=== RUN TestMultiControlPlane/serial/DegradedAfterClusterRestart
ha_test.go:392: (dbg) Run: out/minikube-darwin-amd64 profile list --output json
ha_test.go:415: expected profile "ha-488000" in json of 'profile list' to have "Degraded" status but have "Starting" status. got *"{\"invalid\":[],\"valid\":[{\"Name\":\"ha-488000\",\"Status\":\"Starting\",\"Config\":{\"Name\":\"ha-488000\",\"KeepContext\":false,\"EmbedCerts\":false,\"MinikubeISO\":\"https://storage.googleapis.com/minikube-builds/iso/19774/minikube-v1.34.0-1728382514-19774-amd64.iso\",\"KicBaseImage\":\"gcr.io/k8s-minikube/kicbase-builds:v0.0.45-1728382586-19774@sha256:5d8c4f6d838467365e214e2194dd0153a763e3f78723b5f2a8e06ef7b47409ec\",\"Memory\":2200,\"CPUs\":2,\"DiskSize\":20000,\"Driver\":\"hyperkit\",\"HyperkitVpnKitSock\":\"\",\"HyperkitVSockPorts\":[],\"DockerEnv\":null,\"ContainerVolumeMounts\":null,\"InsecureRegistry\":null,\"RegistryMirror\":[],\"HostOnlyCIDR\":\"192.168.59.1/24\",\"HypervVirtualSwitch\":\"\",\"HypervUseExternalSwitch\":false,\"HypervExternalAdapter\":\"\",\"KVMNetwork\":\"default\",\"KVMQemuURI\":\"qemu:///system\",\"KVMGPU\":false,\"KVMHidden\":false,\"KVMNUMACo
unt\":1,\"APIServerPort\":8443,\"DockerOpt\":null,\"DisableDriverMounts\":false,\"NFSShare\":[],\"NFSSharesRoot\":\"/nfsshares\",\"UUID\":\"\",\"NoVTXCheck\":false,\"DNSProxy\":false,\"HostDNSResolver\":true,\"HostOnlyNicType\":\"virtio\",\"NatNicType\":\"virtio\",\"SSHIPAddress\":\"\",\"SSHUser\":\"root\",\"SSHKey\":\"\",\"SSHPort\":22,\"KubernetesConfig\":{\"KubernetesVersion\":\"v1.31.1\",\"ClusterName\":\"ha-488000\",\"Namespace\":\"default\",\"APIServerHAVIP\":\"192.169.0.254\",\"APIServerName\":\"minikubeCA\",\"APIServerNames\":null,\"APIServerIPs\":null,\"DNSDomain\":\"cluster.local\",\"ContainerRuntime\":\"docker\",\"CRISocket\":\"\",\"NetworkPlugin\":\"cni\",\"FeatureGates\":\"\",\"ServiceCIDR\":\"10.96.0.0/12\",\"ImageRepository\":\"\",\"LoadBalancerStartIP\":\"\",\"LoadBalancerEndIP\":\"\",\"CustomIngressCert\":\"\",\"RegistryAliases\":\"\",\"ExtraOptions\":null,\"ShouldLoadCachedImages\":true,\"EnableDefaultCNI\":false,\"CNI\":\"\"},\"Nodes\":[{\"Name\":\"\",\"IP\":\"192.169.0.5\",\"Port\":8443,\"
KubernetesVersion\":\"v1.31.1\",\"ContainerRuntime\":\"docker\",\"ControlPlane\":true,\"Worker\":true},{\"Name\":\"m02\",\"IP\":\"192.169.0.6\",\"Port\":8443,\"KubernetesVersion\":\"v1.31.1\",\"ContainerRuntime\":\"docker\",\"ControlPlane\":true,\"Worker\":true},{\"Name\":\"m04\",\"IP\":\"192.169.0.8\",\"Port\":0,\"KubernetesVersion\":\"v1.31.1\",\"ContainerRuntime\":\"docker\",\"ControlPlane\":false,\"Worker\":true}],\"Addons\":{\"ambassador\":false,\"auto-pause\":false,\"cloud-spanner\":false,\"csi-hostpath-driver\":false,\"dashboard\":false,\"default-storageclass\":false,\"efk\":false,\"freshpod\":false,\"gcp-auth\":false,\"gvisor\":false,\"headlamp\":false,\"inaccel\":false,\"ingress\":false,\"ingress-dns\":false,\"inspektor-gadget\":false,\"istio\":false,\"istio-provisioner\":false,\"kong\":false,\"kubeflow\":false,\"kubevirt\":false,\"logviewer\":false,\"metallb\":false,\"metrics-server\":false,\"nvidia-device-plugin\":false,\"nvidia-driver-installer\":false,\"nvidia-gpu-device-plugin\":false,\"olm\":fa
lse,\"pod-security-policy\":false,\"portainer\":false,\"registry\":false,\"registry-aliases\":false,\"registry-creds\":false,\"storage-provisioner\":false,\"storage-provisioner-gluster\":false,\"storage-provisioner-rancher\":false,\"volcano\":false,\"volumesnapshots\":false,\"yakd\":false},\"CustomAddonImages\":null,\"CustomAddonRegistries\":null,\"VerifyComponents\":{\"apiserver\":true,\"apps_running\":true,\"default_sa\":true,\"extra\":true,\"kubelet\":true,\"node_ready\":true,\"system_pods\":true},\"StartHostTimeout\":360000000000,\"ScheduledStop\":null,\"ExposedPorts\":[],\"ListenAddress\":\"\",\"Network\":\"\",\"Subnet\":\"\",\"MultiNodeRequested\":true,\"ExtraDisks\":0,\"CertExpiration\":94608000000000000,\"Mount\":false,\"MountString\":\"/Users:/minikube-host\",\"Mount9PVersion\":\"9p2000.L\",\"MountGID\":\"docker\",\"MountIP\":\"\",\"MountMSize\":262144,\"MountOptions\":[],\"MountPort\":0,\"MountType\":\"9p\",\"MountUID\":\"docker\",\"BinaryMirror\":\"\",\"DisableOptimizations\":false,\"DisableMetrics
\":false,\"CustomQemuFirmwarePath\":\"\",\"SocketVMnetClientPath\":\"\",\"SocketVMnetPath\":\"\",\"StaticIP\":\"\",\"SSHAuthSock\":\"\",\"SSHAgentPID\":0,\"GPUs\":\"\",\"AutoPauseInterval\":60000000000},\"Active\":false,\"ActiveKubeContext\":true}]}"*. args: "out/minikube-darwin-amd64 profile list --output json"
helpers_test.go:222: -----------------------post-mortem--------------------------------
helpers_test.go:239: (dbg) Run: out/minikube-darwin-amd64 status --format={{.Host}} -p ha-488000 -n ha-488000
helpers_test.go:239: (dbg) Non-zero exit: out/minikube-darwin-amd64 status --format={{.Host}} -p ha-488000 -n ha-488000: exit status 2 (170.501947ms)
-- stdout --
Running
-- /stdout --
helpers_test.go:239: status error: exit status 2 (may be ok)
helpers_test.go:244: <<< TestMultiControlPlane/serial/DegradedAfterClusterRestart FAILED: start of post-mortem logs <<<
helpers_test.go:245: ======> post-mortem[TestMultiControlPlane/serial/DegradedAfterClusterRestart]: minikube logs <======
helpers_test.go:247: (dbg) Run: out/minikube-darwin-amd64 -p ha-488000 logs -n 25
helpers_test.go:247: (dbg) Done: out/minikube-darwin-amd64 -p ha-488000 logs -n 25: (2.20074743s)
helpers_test.go:252: TestMultiControlPlane/serial/DegradedAfterClusterRestart logs:
-- stdout --
==> Audit <==
|---------|------------------------------------------------------------------------------------------------------------------------------|-----------|---------|---------|---------------------|---------------------|
| Command | Args | Profile | User | Version | Start Time | End Time |
|---------|------------------------------------------------------------------------------------------------------------------------------|-----------|---------|---------|---------------------|---------------------|
| cp | ha-488000 cp ha-488000-m03:/home/docker/cp-test.txt | ha-488000 | jenkins | v1.34.0 | 09 Oct 24 13:04 PDT | 09 Oct 24 13:04 PDT |
| | ha-488000-m04:/home/docker/cp-test_ha-488000-m03_ha-488000-m04.txt | | | | | |
| ssh | ha-488000 ssh -n | ha-488000 | jenkins | v1.34.0 | 09 Oct 24 13:04 PDT | 09 Oct 24 13:04 PDT |
| | ha-488000-m03 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| ssh | ha-488000 ssh -n ha-488000-m04 sudo cat | ha-488000 | jenkins | v1.34.0 | 09 Oct 24 13:04 PDT | 09 Oct 24 13:04 PDT |
| | /home/docker/cp-test_ha-488000-m03_ha-488000-m04.txt | | | | | |
| cp | ha-488000 cp testdata/cp-test.txt | ha-488000 | jenkins | v1.34.0 | 09 Oct 24 13:04 PDT | 09 Oct 24 13:04 PDT |
| | ha-488000-m04:/home/docker/cp-test.txt | | | | | |
| ssh | ha-488000 ssh -n | ha-488000 | jenkins | v1.34.0 | 09 Oct 24 13:04 PDT | 09 Oct 24 13:04 PDT |
| | ha-488000-m04 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| cp | ha-488000 cp ha-488000-m04:/home/docker/cp-test.txt | ha-488000 | jenkins | v1.34.0 | 09 Oct 24 13:04 PDT | 09 Oct 24 13:04 PDT |
| | /var/folders/0y/_8hvl7v13q38_kkh25vpxkz00000gp/T/TestMultiControlPlaneserialCopyFile2325467248/001/cp-test_ha-488000-m04.txt | | | | | |
| ssh | ha-488000 ssh -n | ha-488000 | jenkins | v1.34.0 | 09 Oct 24 13:04 PDT | 09 Oct 24 13:04 PDT |
| | ha-488000-m04 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| cp | ha-488000 cp ha-488000-m04:/home/docker/cp-test.txt | ha-488000 | jenkins | v1.34.0 | 09 Oct 24 13:04 PDT | 09 Oct 24 13:04 PDT |
| | ha-488000:/home/docker/cp-test_ha-488000-m04_ha-488000.txt | | | | | |
| ssh | ha-488000 ssh -n | ha-488000 | jenkins | v1.34.0 | 09 Oct 24 13:04 PDT | 09 Oct 24 13:04 PDT |
| | ha-488000-m04 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| ssh | ha-488000 ssh -n ha-488000 sudo cat | ha-488000 | jenkins | v1.34.0 | 09 Oct 24 13:04 PDT | 09 Oct 24 13:04 PDT |
| | /home/docker/cp-test_ha-488000-m04_ha-488000.txt | | | | | |
| cp | ha-488000 cp ha-488000-m04:/home/docker/cp-test.txt | ha-488000 | jenkins | v1.34.0 | 09 Oct 24 13:04 PDT | 09 Oct 24 13:04 PDT |
| | ha-488000-m02:/home/docker/cp-test_ha-488000-m04_ha-488000-m02.txt | | | | | |
| ssh | ha-488000 ssh -n | ha-488000 | jenkins | v1.34.0 | 09 Oct 24 13:04 PDT | 09 Oct 24 13:04 PDT |
| | ha-488000-m04 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| ssh | ha-488000 ssh -n ha-488000-m02 sudo cat | ha-488000 | jenkins | v1.34.0 | 09 Oct 24 13:04 PDT | 09 Oct 24 13:04 PDT |
| | /home/docker/cp-test_ha-488000-m04_ha-488000-m02.txt | | | | | |
| cp | ha-488000 cp ha-488000-m04:/home/docker/cp-test.txt | ha-488000 | jenkins | v1.34.0 | 09 Oct 24 13:04 PDT | 09 Oct 24 13:04 PDT |
| | ha-488000-m03:/home/docker/cp-test_ha-488000-m04_ha-488000-m03.txt | | | | | |
| ssh | ha-488000 ssh -n | ha-488000 | jenkins | v1.34.0 | 09 Oct 24 13:04 PDT | 09 Oct 24 13:04 PDT |
| | ha-488000-m04 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| ssh | ha-488000 ssh -n ha-488000-m03 sudo cat | ha-488000 | jenkins | v1.34.0 | 09 Oct 24 13:04 PDT | 09 Oct 24 13:04 PDT |
| | /home/docker/cp-test_ha-488000-m04_ha-488000-m03.txt | | | | | |
| node | ha-488000 node stop m02 -v=7 | ha-488000 | jenkins | v1.34.0 | 09 Oct 24 13:04 PDT | 09 Oct 24 13:04 PDT |
| | --alsologtostderr | | | | | |
| node | ha-488000 node start m02 -v=7 | ha-488000 | jenkins | v1.34.0 | 09 Oct 24 13:04 PDT | 09 Oct 24 13:05 PDT |
| | --alsologtostderr | | | | | |
| node | list -p ha-488000 -v=7 | ha-488000 | jenkins | v1.34.0 | 09 Oct 24 13:05 PDT | |
| | --alsologtostderr | | | | | |
| stop | -p ha-488000 -v=7 | ha-488000 | jenkins | v1.34.0 | 09 Oct 24 13:05 PDT | 09 Oct 24 13:05 PDT |
| | --alsologtostderr | | | | | |
| start | -p ha-488000 --wait=true -v=7 | ha-488000 | jenkins | v1.34.0 | 09 Oct 24 13:05 PDT | 09 Oct 24 13:08 PDT |
| | --alsologtostderr | | | | | |
| node | list -p ha-488000 | ha-488000 | jenkins | v1.34.0 | 09 Oct 24 13:08 PDT | |
| node | ha-488000 node delete m03 -v=7 | ha-488000 | jenkins | v1.34.0 | 09 Oct 24 13:08 PDT | 09 Oct 24 13:08 PDT |
| | --alsologtostderr | | | | | |
| stop | ha-488000 stop -v=7 | ha-488000 | jenkins | v1.34.0 | 09 Oct 24 13:08 PDT | 09 Oct 24 13:09 PDT |
| | --alsologtostderr | | | | | |
| start | -p ha-488000 --wait=true | ha-488000 | jenkins | v1.34.0 | 09 Oct 24 13:09 PDT | |
| | -v=7 --alsologtostderr | | | | | |
| | --driver=hyperkit | | | | | |
|---------|------------------------------------------------------------------------------------------------------------------------------|-----------|---------|---------|---------------------|---------------------|
==> Last Start <==
Log file created at: 2024/10/09 13:09:18
Running on machine: MacOS-Agent-4
Binary: Built with gc go1.23.1 for darwin/amd64
Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg
I1009 13:09:18.768079 10047 out.go:345] Setting OutFile to fd 1 ...
I1009 13:09:18.768390 10047 out.go:392] TERM=,COLORTERM=, which probably does not support color
I1009 13:09:18.768395 10047 out.go:358] Setting ErrFile to fd 2...
I1009 13:09:18.768399 10047 out.go:392] TERM=,COLORTERM=, which probably does not support color
I1009 13:09:18.768577 10047 root.go:338] Updating PATH: /Users/jenkins/minikube-integration/19780-6775/.minikube/bin
I1009 13:09:18.770041 10047 out.go:352] Setting JSON to false
I1009 13:09:18.797661 10047 start.go:129] hostinfo: {"hostname":"MacOS-Agent-4.local","uptime":13127,"bootTime":1728491431,"procs":561,"os":"darwin","platform":"darwin","platformFamily":"Standalone Workstation","platformVersion":"15.0.1","kernelVersion":"24.0.0","kernelArch":"x86_64","virtualizationSystem":"","virtualizationRole":"","hostId":"f2f27e25-cfda-5ffd-9706-e98286194e62"}
W1009 13:09:18.797809 10047 start.go:137] gopshost.Virtualization returned error: not implemented yet
I1009 13:09:18.819553 10047 out.go:177] * [ha-488000] minikube v1.34.0 on Darwin 15.0.1
I1009 13:09:18.861286 10047 out.go:177] - MINIKUBE_LOCATION=19780
I1009 13:09:18.861343 10047 notify.go:220] Checking for updates...
I1009 13:09:18.904309 10047 out.go:177] - KUBECONFIG=/Users/jenkins/minikube-integration/19780-6775/kubeconfig
I1009 13:09:18.925208 10047 out.go:177] - MINIKUBE_BIN=out/minikube-darwin-amd64
I1009 13:09:18.946087 10047 out.go:177] - MINIKUBE_SUPPRESS_DOCKER_PERFORMANCE=true
I1009 13:09:18.967204 10047 out.go:177] - MINIKUBE_HOME=/Users/jenkins/minikube-integration/19780-6775/.minikube
I1009 13:09:18.988184 10047 out.go:177] - MINIKUBE_FORCE_SYSTEMD=
I1009 13:09:19.009785 10047 config.go:182] Loaded profile config "ha-488000": Driver=hyperkit, ContainerRuntime=docker, KubernetesVersion=v1.31.1
I1009 13:09:19.010304 10047 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I1009 13:09:19.010355 10047 main.go:141] libmachine: Launching plugin server for driver hyperkit
I1009 13:09:19.021725 10047 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:52269
I1009 13:09:19.022076 10047 main.go:141] libmachine: () Calling .GetVersion
I1009 13:09:19.022542 10047 main.go:141] libmachine: Using API Version 1
I1009 13:09:19.022558 10047 main.go:141] libmachine: () Calling .SetConfigRaw
I1009 13:09:19.022779 10047 main.go:141] libmachine: () Calling .GetMachineName
I1009 13:09:19.022901 10047 main.go:141] libmachine: (ha-488000) Calling .DriverName
I1009 13:09:19.023172 10047 driver.go:394] Setting default libvirt URI to qemu:///system
I1009 13:09:19.023430 10047 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I1009 13:09:19.023455 10047 main.go:141] libmachine: Launching plugin server for driver hyperkit
I1009 13:09:19.034167 10047 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:52271
I1009 13:09:19.034478 10047 main.go:141] libmachine: () Calling .GetVersion
I1009 13:09:19.034820 10047 main.go:141] libmachine: Using API Version 1
I1009 13:09:19.034846 10047 main.go:141] libmachine: () Calling .SetConfigRaw
I1009 13:09:19.035096 10047 main.go:141] libmachine: () Calling .GetMachineName
I1009 13:09:19.035239 10047 main.go:141] libmachine: (ha-488000) Calling .DriverName
I1009 13:09:19.066249 10047 out.go:177] * Using the hyperkit driver based on existing profile
I1009 13:09:19.108243 10047 start.go:297] selected driver: hyperkit
I1009 13:09:19.108271 10047 start.go:901] validating driver "hyperkit" against &{Name:ha-488000 KeepContext:false EmbedCerts:false MinikubeISO:https://storage.googleapis.com/minikube-builds/iso/19774/minikube-v1.34.0-1728382514-19774-amd64.iso KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.45-1728382586-19774@sha256:5d8c4f6d838467365e214e2194dd0153a763e3f78723b5f2a8e06ef7b47409ec Memory:2200 CPUs:2 DiskSize:20000 Driver:hyperkit HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{Kuber
netesVersion:v1.31.1 ClusterName:ha-488000 Namespace:default APIServerHAVIP:192.169.0.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:docker CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.169.0.5 Port:8443 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m02 IP:192.169.0.6 Port:8443 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m04 IP:192.169.0.8 Port:0 KubernetesVersion:v1.31.1 ContainerRuntime: ControlPlane:false Worker:true}] Addons:map[ambassador:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false inaccel:false ingress:false ingress-dns:false
inspektor-gadget:false istio:false istio-provisioner:false kong:false kubeflow:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-gluster:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s Mount:false MountString:/Users:/minikube-host Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOptimiza
tions:false DisableMetrics:false CustomQemuFirmwarePath: SocketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s}
I1009 13:09:19.108538 10047 start.go:912] status for hyperkit: {Installed:true Healthy:true Running:true NeedsImprovement:false Error:<nil> Reason: Fix: Doc: Version:}
I1009 13:09:19.108747 10047 install.go:52] acquiring lock: {Name:mk4023283b30b374c3f04c8805d539e68824c0b8 Clock:{} Delay:500ms Timeout:10m0s Cancel:<nil>}
I1009 13:09:19.108996 10047 install.go:117] Validating docker-machine-driver-hyperkit, PATH=/Users/jenkins/minikube-integration/19780-6775/.minikube/bin:/Users/jenkins/workspace/out/:/usr/bin:/bin:/usr/sbin:/sbin:/Users/jenkins/google-cloud-sdk/bin:/usr/local/bin/:/usr/local/go/bin/:/Users/jenkins/go/bin
I1009 13:09:19.120995 10047 install.go:137] /Users/jenkins/workspace/out/docker-machine-driver-hyperkit version is 1.34.0
I1009 13:09:19.127230 10047 install.go:79] stdout: /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I1009 13:09:19.127259 10047 install.go:81] /Users/jenkins/workspace/out/docker-machine-driver-hyperkit looks good
I1009 13:09:19.132310 10047 start_flags.go:947] Waiting for all components: map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true]
I1009 13:09:19.132350 10047 cni.go:84] Creating CNI manager for ""
I1009 13:09:19.132386 10047 cni.go:136] multinode detected (3 nodes found), recommending kindnet
I1009 13:09:19.132460 10047 start.go:340] cluster config:
{Name:ha-488000 KeepContext:false EmbedCerts:false MinikubeISO:https://storage.googleapis.com/minikube-builds/iso/19774/minikube-v1.34.0-1728382514-19774-amd64.iso KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.45-1728382586-19774@sha256:5d8c4f6d838467365e214e2194dd0153a763e3f78723b5f2a8e06ef7b47409ec Memory:2200 CPUs:2 DiskSize:20000 Driver:hyperkit HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.31.1 ClusterName:ha-488000 Namespace:default APIServerHAVIP:192.16
9.0.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:docker CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.169.0.5 Port:8443 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m02 IP:192.169.0.6 Port:8443 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m04 IP:192.169.0.8 Port:0 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:false Worker:true}] Addons:map[ambassador:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false inaccel:false ingress:false ingress-dns:false inspektor-gadget:false istio:false istio-provisioner:false kong:false kubefl
ow:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-gluster:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s Mount:false MountString:/Users:/minikube-host Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOptimizations:false DisableMetrics:false CustomQemuFirmwarePath: SocketVMnetClientPa
th: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s}
I1009 13:09:19.132572 10047 iso.go:125] acquiring lock: {Name:mke8b37658708256f245ab3e8787456047421569 Clock:{} Delay:500ms Timeout:10m0s Cancel:<nil>}
I1009 13:09:19.174935 10047 out.go:177] * Starting "ha-488000" primary control-plane node in "ha-488000" cluster
I1009 13:09:19.196187 10047 preload.go:131] Checking if preload exists for k8s version v1.31.1 and runtime docker
I1009 13:09:19.196280 10047 preload.go:146] Found local preload: /Users/jenkins/minikube-integration/19780-6775/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.31.1-docker-overlay2-amd64.tar.lz4
I1009 13:09:19.196301 10047 cache.go:56] Caching tarball of preloaded images
I1009 13:09:19.196527 10047 preload.go:172] Found /Users/jenkins/minikube-integration/19780-6775/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.31.1-docker-overlay2-amd64.tar.lz4 in cache, skipping download
I1009 13:09:19.196547 10047 cache.go:59] Finished verifying existence of preloaded tar for v1.31.1 on docker
I1009 13:09:19.196716 10047 profile.go:143] Saving config to /Users/jenkins/minikube-integration/19780-6775/.minikube/profiles/ha-488000/config.json ...
I1009 13:09:19.197618 10047 start.go:360] acquireMachinesLock for ha-488000: {Name:mk693e628b9fda7cceeeedd30d9cc7939119b68f Clock:{} Delay:500ms Timeout:13m0s Cancel:<nil>}
I1009 13:09:19.197765 10047 start.go:364] duration metric: took 124.332µs to acquireMachinesLock for "ha-488000"
I1009 13:09:19.197803 10047 start.go:96] Skipping create...Using existing machine configuration
I1009 13:09:19.197815 10047 fix.go:54] fixHost starting:
I1009 13:09:19.198237 10047 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I1009 13:09:19.198263 10047 main.go:141] libmachine: Launching plugin server for driver hyperkit
I1009 13:09:19.209652 10047 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:52273
I1009 13:09:19.209999 10047 main.go:141] libmachine: () Calling .GetVersion
I1009 13:09:19.210331 10047 main.go:141] libmachine: Using API Version 1
I1009 13:09:19.210346 10047 main.go:141] libmachine: () Calling .SetConfigRaw
I1009 13:09:19.210562 10047 main.go:141] libmachine: () Calling .GetMachineName
I1009 13:09:19.210697 10047 main.go:141] libmachine: (ha-488000) Calling .DriverName
I1009 13:09:19.210793 10047 main.go:141] libmachine: (ha-488000) Calling .GetState
I1009 13:09:19.210878 10047 main.go:141] libmachine: (ha-488000) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I1009 13:09:19.210954 10047 main.go:141] libmachine: (ha-488000) DBG | hyperkit pid from json: 9932
I1009 13:09:19.212121 10047 main.go:141] libmachine: (ha-488000) DBG | hyperkit pid 9932 missing from process table
I1009 13:09:19.212157 10047 fix.go:112] recreateIfNeeded on ha-488000: state=Stopped err=<nil>
I1009 13:09:19.212185 10047 main.go:141] libmachine: (ha-488000) Calling .DriverName
W1009 13:09:19.212275 10047 fix.go:138] unexpected machine state, will restart: <nil>
I1009 13:09:19.232953 10047 out.go:177] * Restarting existing hyperkit VM for "ha-488000" ...
I1009 13:09:19.275044 10047 main.go:141] libmachine: (ha-488000) Calling .Start
I1009 13:09:19.275288 10047 main.go:141] libmachine: (ha-488000) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I1009 13:09:19.275311 10047 main.go:141] libmachine: (ha-488000) minikube might have been shutdown in an unclean way, the hyperkit pid file still exists: /Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000/hyperkit.pid
I1009 13:09:19.277516 10047 main.go:141] libmachine: (ha-488000) DBG | hyperkit pid 9932 missing from process table
I1009 13:09:19.277528 10047 main.go:141] libmachine: (ha-488000) DBG | pid 9932 is in state "Stopped"
I1009 13:09:19.277546 10047 main.go:141] libmachine: (ha-488000) DBG | Removing stale pid file /Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000/hyperkit.pid...
I1009 13:09:19.277882 10047 main.go:141] libmachine: (ha-488000) DBG | Using UUID 0874ca5c-0dc0-4ec2-b18f-ac8b6605962a
I1009 13:09:19.390561 10047 main.go:141] libmachine: (ha-488000) DBG | Generated MAC 22:8b:d6:6a:e4:67
I1009 13:09:19.390595 10047 main.go:141] libmachine: (ha-488000) DBG | Starting with cmdline: loglevel=3 console=ttyS0 console=tty0 noembed nomodeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-488000
I1009 13:09:19.390738 10047 main.go:141] libmachine: (ha-488000) DBG | 2024/10/09 13:09:19 DEBUG: hyperkit: Start &hyperkit.HyperKit{HyperKit:"/usr/local/bin/hyperkit", Argv0:"", StateDir:"/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000", VPNKitSock:"", VPNKitUUID:"", VPNKitPreferredIPv4:"", UUID:"0874ca5c-0dc0-4ec2-b18f-ac8b6605962a", Disks:[]hyperkit.Disk{(*hyperkit.RawDisk)(0xc00042b9e0)}, ISOImages:[]string{"/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000/boot2docker.iso"}, VSock:false, VSockDir:"", VSockPorts:[]int(nil), VSockGuestCID:3, VMNet:true, Sockets9P:[]hyperkit.Socket9P(nil), Kernel:"/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000/bzimage", Initrd:"/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000/initrd", Bootrom:"", CPUs:2, Memory:2200, Console:1, Serials:[]hyperkit.Serial(nil), Pid:0, Arguments:[]string(nil), CmdLine:"", process:(*os.Process)(nil)}
I1009 13:09:19.390770 10047 main.go:141] libmachine: (ha-488000) DBG | 2024/10/09 13:09:19 DEBUG: hyperkit: check &hyperkit.HyperKit{HyperKit:"/usr/local/bin/hyperkit", Argv0:"", StateDir:"/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000", VPNKitSock:"", VPNKitUUID:"", VPNKitPreferredIPv4:"", UUID:"0874ca5c-0dc0-4ec2-b18f-ac8b6605962a", Disks:[]hyperkit.Disk{(*hyperkit.RawDisk)(0xc00042b9e0)}, ISOImages:[]string{"/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000/boot2docker.iso"}, VSock:false, VSockDir:"", VSockPorts:[]int(nil), VSockGuestCID:3, VMNet:true, Sockets9P:[]hyperkit.Socket9P(nil), Kernel:"/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000/bzimage", Initrd:"/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000/initrd", Bootrom:"", CPUs:2, Memory:2200, Console:1, Serials:[]hyperkit.Serial(nil), Pid:0, Arguments:[]string(nil), CmdLine:"", process:(*os.Process)(nil)}
I1009 13:09:19.390833 10047 main.go:141] libmachine: (ha-488000) DBG | 2024/10/09 13:09:19 DEBUG: hyperkit: Arguments: []string{"-A", "-u", "-F", "/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000/hyperkit.pid", "-c", "2", "-m", "2200M", "-s", "0:0,hostbridge", "-s", "31,lpc", "-s", "1:0,virtio-net", "-U", "0874ca5c-0dc0-4ec2-b18f-ac8b6605962a", "-s", "2:0,virtio-blk,/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000/ha-488000.rawdisk", "-s", "3,ahci-cd,/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000/boot2docker.iso", "-s", "4,virtio-rnd", "-l", "com1,autopty=/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000/tty,log=/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000/console-ring", "-f", "kexec,/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000/bzimage,/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000/initrd,earlyprintk=s
erial loglevel=3 console=ttyS0 console=tty0 noembed nomodeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-488000"}
I1009 13:09:19.390884 10047 main.go:141] libmachine: (ha-488000) DBG | 2024/10/09 13:09:19 DEBUG: hyperkit: CmdLine: "/usr/local/bin/hyperkit -A -u -F /Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000/hyperkit.pid -c 2 -m 2200M -s 0:0,hostbridge -s 31,lpc -s 1:0,virtio-net -U 0874ca5c-0dc0-4ec2-b18f-ac8b6605962a -s 2:0,virtio-blk,/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000/ha-488000.rawdisk -s 3,ahci-cd,/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000/boot2docker.iso -s 4,virtio-rnd -l com1,autopty=/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000/tty,log=/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000/console-ring -f kexec,/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000/bzimage,/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000/initrd,earlyprintk=serial loglevel=3 console=ttyS0 console=tty0 noembed nomodeset
norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-488000"
I1009 13:09:19.390956 10047 main.go:141] libmachine: (ha-488000) DBG | 2024/10/09 13:09:19 DEBUG: hyperkit: Redirecting stdout/stderr to logger
I1009 13:09:19.392704 10047 main.go:141] libmachine: (ha-488000) DBG | 2024/10/09 13:09:19 DEBUG: hyperkit: Pid is 10061
I1009 13:09:19.393125 10047 main.go:141] libmachine: (ha-488000) DBG | Attempt 0
I1009 13:09:19.393141 10047 main.go:141] libmachine: (ha-488000) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I1009 13:09:19.393800 10047 main.go:141] libmachine: (ha-488000) DBG | hyperkit pid from json: 10061
I1009 13:09:19.395518 10047 main.go:141] libmachine: (ha-488000) DBG | Searching for 22:8b:d6:6a:e4:67 in /var/db/dhcpd_leases ...
I1009 13:09:19.395655 10047 main.go:141] libmachine: (ha-488000) DBG | Found 7 entries in /var/db/dhcpd_leases!
I1009 13:09:19.395677 10047 main.go:141] libmachine: (ha-488000) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.7 HWAddress:b6:2:d4:66:de:30 ID:1,b6:2:d4:66:de:30 Lease:0x6706e2d3}
I1009 13:09:19.395696 10047 main.go:141] libmachine: (ha-488000) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.8 HWAddress:c6:88:ed:5f:e9:2a ID:1,c6:88:ed:5f:e9:2a Lease:0x6706f0b6}
I1009 13:09:19.395703 10047 main.go:141] libmachine: (ha-488000) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.6 HWAddress:76:27:9b:6:7d:7a ID:1,76:27:9b:6:7d:7a Lease:0x6706f04c}
I1009 13:09:19.395711 10047 main.go:141] libmachine: (ha-488000) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.5 HWAddress:22:8b:d6:6a:e4:67 ID:1,22:8b:d6:6a:e4:67 Lease:0x6706f022}
I1009 13:09:19.395718 10047 main.go:141] libmachine: (ha-488000) DBG | Found match: 22:8b:d6:6a:e4:67
I1009 13:09:19.395721 10047 main.go:141] libmachine: (ha-488000) DBG | IP: 192.169.0.5
I1009 13:09:19.395781 10047 main.go:141] libmachine: (ha-488000) Calling .GetConfigRaw
I1009 13:09:19.396534 10047 main.go:141] libmachine: (ha-488000) Calling .GetIP
I1009 13:09:19.396819 10047 profile.go:143] Saving config to /Users/jenkins/minikube-integration/19780-6775/.minikube/profiles/ha-488000/config.json ...
I1009 13:09:19.397360 10047 machine.go:93] provisionDockerMachine start ...
I1009 13:09:19.397385 10047 main.go:141] libmachine: (ha-488000) Calling .DriverName
I1009 13:09:19.397563 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHHostname
I1009 13:09:19.397720 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHPort
I1009 13:09:19.397862 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHKeyPath
I1009 13:09:19.398002 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHKeyPath
I1009 13:09:19.398131 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHUsername
I1009 13:09:19.398316 10047 main.go:141] libmachine: Using SSH client type: native
I1009 13:09:19.398513 10047 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xf225520] 0xf228200 <nil> [] 0s} 192.169.0.5 22 <nil> <nil>}
I1009 13:09:19.398522 10047 main.go:141] libmachine: About to run SSH command:
hostname
I1009 13:09:19.403285 10047 main.go:141] libmachine: (ha-488000) DBG | 2024/10/09 13:09:19 INFO : hyperkit: stderr: Using fd 6 for I/O notifications
I1009 13:09:19.460253 10047 main.go:141] libmachine: (ha-488000) DBG | 2024/10/09 13:09:19 INFO : hyperkit: stderr: /Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000/boot2docker.iso: fcntl(F_PUNCHHOLE) Operation not permitted: block device will not support TRIM/DISCARD
I1009 13:09:19.461040 10047 main.go:141] libmachine: (ha-488000) DBG | 2024/10/09 13:09:19 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 2 bit: 22 unspecified don't care: bit is 0
I1009 13:09:19.461067 10047 main.go:141] libmachine: (ha-488000) DBG | 2024/10/09 13:09:19 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 12 unspecified don't care: bit is 0
I1009 13:09:19.461089 10047 main.go:141] libmachine: (ha-488000) DBG | 2024/10/09 13:09:19 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 20 unspecified don't care: bit is 0
I1009 13:09:19.461106 10047 main.go:141] libmachine: (ha-488000) DBG | 2024/10/09 13:09:19 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 3 bit: 13 unspecified don't care: bit is 0
I1009 13:09:19.846111 10047 main.go:141] libmachine: (ha-488000) DBG | 2024/10/09 13:09:19 INFO : hyperkit: stderr: rdmsr to register 0x3a on vcpu 0
I1009 13:09:19.846132 10047 main.go:141] libmachine: (ha-488000) DBG | 2024/10/09 13:09:19 INFO : hyperkit: stderr: rdmsr to register 0x140 on vcpu 0
I1009 13:09:19.960930 10047 main.go:141] libmachine: (ha-488000) DBG | 2024/10/09 13:09:19 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 2 bit: 22 unspecified don't care: bit is 0
I1009 13:09:19.960945 10047 main.go:141] libmachine: (ha-488000) DBG | 2024/10/09 13:09:19 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 12 unspecified don't care: bit is 0
I1009 13:09:19.960958 10047 main.go:141] libmachine: (ha-488000) DBG | 2024/10/09 13:09:19 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 20 unspecified don't care: bit is 0
I1009 13:09:19.960999 10047 main.go:141] libmachine: (ha-488000) DBG | 2024/10/09 13:09:19 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 3 bit: 13 unspecified don't care: bit is 0
I1009 13:09:19.961838 10047 main.go:141] libmachine: (ha-488000) DBG | 2024/10/09 13:09:19 INFO : hyperkit: stderr: rdmsr to register 0x3a on vcpu 1
I1009 13:09:19.961850 10047 main.go:141] libmachine: (ha-488000) DBG | 2024/10/09 13:09:19 INFO : hyperkit: stderr: rdmsr to register 0x140 on vcpu 1
I1009 13:09:25.663947 10047 main.go:141] libmachine: (ha-488000) DBG | 2024/10/09 13:09:25 INFO : hyperkit: stderr: rdmsr to register 0x64d on vcpu 1
I1009 13:09:25.663995 10047 main.go:141] libmachine: (ha-488000) DBG | 2024/10/09 13:09:25 INFO : hyperkit: stderr: rdmsr to register 0x64e on vcpu 1
I1009 13:09:25.664006 10047 main.go:141] libmachine: (ha-488000) DBG | 2024/10/09 13:09:25 INFO : hyperkit: stderr: rdmsr to register 0x34 on vcpu 1
I1009 13:09:25.687729 10047 main.go:141] libmachine: (ha-488000) DBG | 2024/10/09 13:09:25 INFO : hyperkit: stderr: rdmsr to register 0xc0011029 on vcpu 1
I1009 13:09:30.469040 10047 main.go:141] libmachine: SSH cmd err, output: <nil>: minikube
I1009 13:09:30.469065 10047 main.go:141] libmachine: (ha-488000) Calling .GetMachineName
I1009 13:09:30.469216 10047 buildroot.go:166] provisioning hostname "ha-488000"
I1009 13:09:30.469238 10047 main.go:141] libmachine: (ha-488000) Calling .GetMachineName
I1009 13:09:30.469352 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHHostname
I1009 13:09:30.469454 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHPort
I1009 13:09:30.469557 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHKeyPath
I1009 13:09:30.469673 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHKeyPath
I1009 13:09:30.469797 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHUsername
I1009 13:09:30.469932 10047 main.go:141] libmachine: Using SSH client type: native
I1009 13:09:30.470074 10047 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xf225520] 0xf228200 <nil> [] 0s} 192.169.0.5 22 <nil> <nil>}
I1009 13:09:30.470082 10047 main.go:141] libmachine: About to run SSH command:
sudo hostname ha-488000 && echo "ha-488000" | sudo tee /etc/hostname
I1009 13:09:30.543330 10047 main.go:141] libmachine: SSH cmd err, output: <nil>: ha-488000
I1009 13:09:30.543348 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHHostname
I1009 13:09:30.543512 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHPort
I1009 13:09:30.543625 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHKeyPath
I1009 13:09:30.543718 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHKeyPath
I1009 13:09:30.543822 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHUsername
I1009 13:09:30.543987 10047 main.go:141] libmachine: Using SSH client type: native
I1009 13:09:30.544116 10047 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xf225520] 0xf228200 <nil> [] 0s} 192.169.0.5 22 <nil> <nil>}
I1009 13:09:30.544127 10047 main.go:141] libmachine: About to run SSH command:
if ! grep -xq '.*\sha-488000' /etc/hosts; then
if grep -xq '127.0.1.1\s.*' /etc/hosts; then
sudo sed -i 's/^127.0.1.1\s.*/127.0.1.1 ha-488000/g' /etc/hosts;
else
echo '127.0.1.1 ha-488000' | sudo tee -a /etc/hosts;
fi
fi
I1009 13:09:30.616984 10047 main.go:141] libmachine: SSH cmd err, output: <nil>:
I1009 13:09:30.617019 10047 buildroot.go:172] set auth options {CertDir:/Users/jenkins/minikube-integration/19780-6775/.minikube CaCertPath:/Users/jenkins/minikube-integration/19780-6775/.minikube/certs/ca.pem CaPrivateKeyPath:/Users/jenkins/minikube-integration/19780-6775/.minikube/certs/ca-key.pem CaCertRemotePath:/etc/docker/ca.pem ServerCertPath:/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/server.pem ServerKeyPath:/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/server-key.pem ClientKeyPath:/Users/jenkins/minikube-integration/19780-6775/.minikube/certs/key.pem ServerCertRemotePath:/etc/docker/server.pem ServerKeyRemotePath:/etc/docker/server-key.pem ClientCertPath:/Users/jenkins/minikube-integration/19780-6775/.minikube/certs/cert.pem ServerCertSANs:[] StorePath:/Users/jenkins/minikube-integration/19780-6775/.minikube}
I1009 13:09:30.617035 10047 buildroot.go:174] setting up certificates
I1009 13:09:30.617042 10047 provision.go:84] configureAuth start
I1009 13:09:30.617048 10047 main.go:141] libmachine: (ha-488000) Calling .GetMachineName
I1009 13:09:30.617188 10047 main.go:141] libmachine: (ha-488000) Calling .GetIP
I1009 13:09:30.617278 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHHostname
I1009 13:09:30.617361 10047 provision.go:143] copyHostCerts
I1009 13:09:30.617391 10047 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19780-6775/.minikube/certs/ca.pem -> /Users/jenkins/minikube-integration/19780-6775/.minikube/ca.pem
I1009 13:09:30.617469 10047 exec_runner.go:144] found /Users/jenkins/minikube-integration/19780-6775/.minikube/ca.pem, removing ...
I1009 13:09:30.617478 10047 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19780-6775/.minikube/ca.pem
I1009 13:09:30.617626 10047 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19780-6775/.minikube/certs/ca.pem --> /Users/jenkins/minikube-integration/19780-6775/.minikube/ca.pem (1082 bytes)
I1009 13:09:30.617840 10047 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19780-6775/.minikube/certs/cert.pem -> /Users/jenkins/minikube-integration/19780-6775/.minikube/cert.pem
I1009 13:09:30.617890 10047 exec_runner.go:144] found /Users/jenkins/minikube-integration/19780-6775/.minikube/cert.pem, removing ...
I1009 13:09:30.617895 10047 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19780-6775/.minikube/cert.pem
I1009 13:09:30.617991 10047 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19780-6775/.minikube/certs/cert.pem --> /Users/jenkins/minikube-integration/19780-6775/.minikube/cert.pem (1123 bytes)
I1009 13:09:30.618178 10047 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19780-6775/.minikube/certs/key.pem -> /Users/jenkins/minikube-integration/19780-6775/.minikube/key.pem
I1009 13:09:30.618229 10047 exec_runner.go:144] found /Users/jenkins/minikube-integration/19780-6775/.minikube/key.pem, removing ...
I1009 13:09:30.618233 10047 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19780-6775/.minikube/key.pem
I1009 13:09:30.618325 10047 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19780-6775/.minikube/certs/key.pem --> /Users/jenkins/minikube-integration/19780-6775/.minikube/key.pem (1675 bytes)
I1009 13:09:30.618478 10047 provision.go:117] generating server cert: /Users/jenkins/minikube-integration/19780-6775/.minikube/machines/server.pem ca-key=/Users/jenkins/minikube-integration/19780-6775/.minikube/certs/ca.pem private-key=/Users/jenkins/minikube-integration/19780-6775/.minikube/certs/ca-key.pem org=jenkins.ha-488000 san=[127.0.0.1 192.169.0.5 ha-488000 localhost minikube]
I1009 13:09:30.980295 10047 provision.go:177] copyRemoteCerts
I1009 13:09:30.980369 10047 ssh_runner.go:195] Run: sudo mkdir -p /etc/docker /etc/docker /etc/docker
I1009 13:09:30.980385 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHHostname
I1009 13:09:30.980528 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHPort
I1009 13:09:30.980631 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHKeyPath
I1009 13:09:30.980722 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHUsername
I1009 13:09:30.980823 10047 sshutil.go:53] new ssh client: &{IP:192.169.0.5 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000/id_rsa Username:docker}
I1009 13:09:31.019403 10047 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19780-6775/.minikube/machines/server-key.pem -> /etc/docker/server-key.pem
I1009 13:09:31.019477 10047 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19780-6775/.minikube/machines/server-key.pem --> /etc/docker/server-key.pem (1675 bytes)
I1009 13:09:31.038237 10047 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19780-6775/.minikube/certs/ca.pem -> /etc/docker/ca.pem
I1009 13:09:31.038297 10047 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19780-6775/.minikube/certs/ca.pem --> /etc/docker/ca.pem (1082 bytes)
I1009 13:09:31.057226 10047 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19780-6775/.minikube/machines/server.pem -> /etc/docker/server.pem
I1009 13:09:31.057302 10047 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19780-6775/.minikube/machines/server.pem --> /etc/docker/server.pem (1200 bytes)
I1009 13:09:31.076516 10047 provision.go:87] duration metric: took 459.469923ms to configureAuth
I1009 13:09:31.076531 10047 buildroot.go:189] setting minikube options for container-runtime
I1009 13:09:31.076695 10047 config.go:182] Loaded profile config "ha-488000": Driver=hyperkit, ContainerRuntime=docker, KubernetesVersion=v1.31.1
I1009 13:09:31.076707 10047 main.go:141] libmachine: (ha-488000) Calling .DriverName
I1009 13:09:31.076842 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHHostname
I1009 13:09:31.076924 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHPort
I1009 13:09:31.077021 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHKeyPath
I1009 13:09:31.077107 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHKeyPath
I1009 13:09:31.077194 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHUsername
I1009 13:09:31.077315 10047 main.go:141] libmachine: Using SSH client type: native
I1009 13:09:31.077451 10047 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xf225520] 0xf228200 <nil> [] 0s} 192.169.0.5 22 <nil> <nil>}
I1009 13:09:31.077459 10047 main.go:141] libmachine: About to run SSH command:
df --output=fstype / | tail -n 1
I1009 13:09:31.141799 10047 main.go:141] libmachine: SSH cmd err, output: <nil>: tmpfs
I1009 13:09:31.141816 10047 buildroot.go:70] root file system type: tmpfs
I1009 13:09:31.141897 10047 provision.go:314] Updating docker unit: /lib/systemd/system/docker.service ...
I1009 13:09:31.141912 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHHostname
I1009 13:09:31.142054 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHPort
I1009 13:09:31.142150 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHKeyPath
I1009 13:09:31.142249 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHKeyPath
I1009 13:09:31.142347 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHUsername
I1009 13:09:31.142517 10047 main.go:141] libmachine: Using SSH client type: native
I1009 13:09:31.142652 10047 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xf225520] 0xf228200 <nil> [] 0s} 192.169.0.5 22 <nil> <nil>}
I1009 13:09:31.142698 10047 main.go:141] libmachine: About to run SSH command:
sudo mkdir -p /lib/systemd/system && printf %s "[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network.target minikube-automount.service docker.socket
Requires= minikube-automount.service docker.socket
StartLimitBurst=3
StartLimitIntervalSec=60
[Service]
Type=notify
Restart=on-failure
# This file is a systemd drop-in unit that inherits from the base dockerd configuration.
# The base configuration already specifies an 'ExecStart=...' command. The first directive
# here is to clear out that command inherited from the base configuration. Without this,
# the command from the base configuration and the command specified here are treated as
# a sequence of commands, which is not the desired behavior, nor is it valid -- systemd
# will catch this invalid input and refuse to start the service with an error like:
# Service has more than one ExecStart= setting, which is only allowed for Type=oneshot services.
# NOTE: default-ulimit=nofile is set to an arbitrary number for consistency with other
# container runtimes. If left unlimited, it may result in OOM issues with MySQL.
ExecStart=
ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --default-ulimit=nofile=1048576:1048576 --tlsverify --tlscacert /etc/docker/ca.pem --tlscert /etc/docker/server.pem --tlskey /etc/docker/server-key.pem --label provider=hyperkit --insecure-registry 10.96.0.0/12
ExecReload=/bin/kill -s HUP \$MAINPID
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
TimeoutStartSec=0
# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes
# kill only the docker process, not all processes in the cgroup
KillMode=process
[Install]
WantedBy=multi-user.target
" | sudo tee /lib/systemd/system/docker.service.new
I1009 13:09:31.218589 10047 main.go:141] libmachine: SSH cmd err, output: <nil>: [Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network.target minikube-automount.service docker.socket
Requires= minikube-automount.service docker.socket
StartLimitBurst=3
StartLimitIntervalSec=60
[Service]
Type=notify
Restart=on-failure
# This file is a systemd drop-in unit that inherits from the base dockerd configuration.
# The base configuration already specifies an 'ExecStart=...' command. The first directive
# here is to clear out that command inherited from the base configuration. Without this,
# the command from the base configuration and the command specified here are treated as
# a sequence of commands, which is not the desired behavior, nor is it valid -- systemd
# will catch this invalid input and refuse to start the service with an error like:
# Service has more than one ExecStart= setting, which is only allowed for Type=oneshot services.
# NOTE: default-ulimit=nofile is set to an arbitrary number for consistency with other
# container runtimes. If left unlimited, it may result in OOM issues with MySQL.
ExecStart=
ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --default-ulimit=nofile=1048576:1048576 --tlsverify --tlscacert /etc/docker/ca.pem --tlscert /etc/docker/server.pem --tlskey /etc/docker/server-key.pem --label provider=hyperkit --insecure-registry 10.96.0.0/12
ExecReload=/bin/kill -s HUP $MAINPID
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
TimeoutStartSec=0
# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes
# kill only the docker process, not all processes in the cgroup
KillMode=process
[Install]
WantedBy=multi-user.target
I1009 13:09:31.218612 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHHostname
I1009 13:09:31.218744 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHPort
I1009 13:09:31.218853 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHKeyPath
I1009 13:09:31.218956 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHKeyPath
I1009 13:09:31.219053 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHUsername
I1009 13:09:31.219230 10047 main.go:141] libmachine: Using SSH client type: native
I1009 13:09:31.219362 10047 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xf225520] 0xf228200 <nil> [] 0s} 192.169.0.5 22 <nil> <nil>}
I1009 13:09:31.219375 10047 main.go:141] libmachine: About to run SSH command:
sudo diff -u /lib/systemd/system/docker.service /lib/systemd/system/docker.service.new || { sudo mv /lib/systemd/system/docker.service.new /lib/systemd/system/docker.service; sudo systemctl -f daemon-reload && sudo systemctl -f enable docker && sudo systemctl -f restart docker; }
I1009 13:09:32.944604 10047 main.go:141] libmachine: SSH cmd err, output: <nil>: diff: can't stat '/lib/systemd/system/docker.service': No such file or directory
Created symlink /etc/systemd/system/multi-user.target.wants/docker.service → /usr/lib/systemd/system/docker.service.
I1009 13:09:32.944617 10047 machine.go:96] duration metric: took 13.547532863s to provisionDockerMachine
I1009 13:09:32.944630 10047 start.go:293] postStartSetup for "ha-488000" (driver="hyperkit")
I1009 13:09:32.944637 10047 start.go:322] creating required directories: [/etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs]
I1009 13:09:32.944652 10047 main.go:141] libmachine: (ha-488000) Calling .DriverName
I1009 13:09:32.944886 10047 ssh_runner.go:195] Run: sudo mkdir -p /etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs
I1009 13:09:32.944906 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHHostname
I1009 13:09:32.945018 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHPort
I1009 13:09:32.945137 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHKeyPath
I1009 13:09:32.945242 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHUsername
I1009 13:09:32.945359 10047 sshutil.go:53] new ssh client: &{IP:192.169.0.5 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000/id_rsa Username:docker}
I1009 13:09:32.986533 10047 ssh_runner.go:195] Run: cat /etc/os-release
I1009 13:09:32.990384 10047 info.go:137] Remote host: Buildroot 2023.02.9
I1009 13:09:32.990398 10047 filesync.go:126] Scanning /Users/jenkins/minikube-integration/19780-6775/.minikube/addons for local assets ...
I1009 13:09:32.990505 10047 filesync.go:126] Scanning /Users/jenkins/minikube-integration/19780-6775/.minikube/files for local assets ...
I1009 13:09:32.990721 10047 filesync.go:149] local asset: /Users/jenkins/minikube-integration/19780-6775/.minikube/files/etc/ssl/certs/73182.pem -> 73182.pem in /etc/ssl/certs
I1009 13:09:32.990727 10047 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19780-6775/.minikube/files/etc/ssl/certs/73182.pem -> /etc/ssl/certs/73182.pem
I1009 13:09:32.991001 10047 ssh_runner.go:195] Run: sudo mkdir -p /etc/ssl/certs
I1009 13:09:33.001446 10047 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19780-6775/.minikube/files/etc/ssl/certs/73182.pem --> /etc/ssl/certs/73182.pem (1708 bytes)
I1009 13:09:33.030289 10047 start.go:296] duration metric: took 85.651886ms for postStartSetup
I1009 13:09:33.030311 10047 main.go:141] libmachine: (ha-488000) Calling .DriverName
I1009 13:09:33.030502 10047 ssh_runner.go:195] Run: sudo ls --almost-all -1 /var/lib/minikube/backup
I1009 13:09:33.030529 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHHostname
I1009 13:09:33.030614 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHPort
I1009 13:09:33.030698 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHKeyPath
I1009 13:09:33.030781 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHUsername
I1009 13:09:33.030858 10047 sshutil.go:53] new ssh client: &{IP:192.169.0.5 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000/id_rsa Username:docker}
I1009 13:09:33.069175 10047 machine.go:197] restoring vm config from /var/lib/minikube/backup: [etc]
I1009 13:09:33.069245 10047 ssh_runner.go:195] Run: sudo rsync --archive --update /var/lib/minikube/backup/etc /
I1009 13:09:33.103875 10047 fix.go:56] duration metric: took 13.906364255s for fixHost
I1009 13:09:33.103897 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHHostname
I1009 13:09:33.104037 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHPort
I1009 13:09:33.104152 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHKeyPath
I1009 13:09:33.104249 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHKeyPath
I1009 13:09:33.104347 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHUsername
I1009 13:09:33.104512 10047 main.go:141] libmachine: Using SSH client type: native
I1009 13:09:33.104664 10047 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xf225520] 0xf228200 <nil> [] 0s} 192.169.0.5 22 <nil> <nil>}
I1009 13:09:33.104671 10047 main.go:141] libmachine: About to run SSH command:
date +%s.%N
I1009 13:09:33.168354 10047 main.go:141] libmachine: SSH cmd err, output: <nil>: 1728504573.354414759
I1009 13:09:33.168365 10047 fix.go:216] guest clock: 1728504573.354414759
I1009 13:09:33.168371 10047 fix.go:229] Guest: 2024-10-09 13:09:33.354414759 -0700 PDT Remote: 2024-10-09 13:09:33.103888 -0700 PDT m=+14.376622939 (delta=250.526759ms)
I1009 13:09:33.168391 10047 fix.go:200] guest clock delta is within tolerance: 250.526759ms
I1009 13:09:33.168401 10047 start.go:83] releasing machines lock for "ha-488000", held for 13.970927059s
I1009 13:09:33.168420 10047 main.go:141] libmachine: (ha-488000) Calling .DriverName
I1009 13:09:33.168548 10047 main.go:141] libmachine: (ha-488000) Calling .GetIP
I1009 13:09:33.168667 10047 main.go:141] libmachine: (ha-488000) Calling .DriverName
I1009 13:09:33.168995 10047 main.go:141] libmachine: (ha-488000) Calling .DriverName
I1009 13:09:33.169103 10047 main.go:141] libmachine: (ha-488000) Calling .DriverName
I1009 13:09:33.169202 10047 ssh_runner.go:195] Run: curl -sS -m 2 https://registry.k8s.io/
I1009 13:09:33.169231 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHHostname
I1009 13:09:33.169287 10047 ssh_runner.go:195] Run: cat /version.json
I1009 13:09:33.169322 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHHostname
I1009 13:09:33.169356 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHPort
I1009 13:09:33.169418 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHPort
I1009 13:09:33.169458 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHKeyPath
I1009 13:09:33.169525 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHKeyPath
I1009 13:09:33.169546 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHUsername
I1009 13:09:33.169620 10047 sshutil.go:53] new ssh client: &{IP:192.169.0.5 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000/id_rsa Username:docker}
I1009 13:09:33.169643 10047 main.go:141] libmachine: (ha-488000) Calling .GetSSHUsername
I1009 13:09:33.169726 10047 sshutil.go:53] new ssh client: &{IP:192.169.0.5 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000/id_rsa Username:docker}
I1009 13:09:33.249679 10047 ssh_runner.go:195] Run: systemctl --version
I1009 13:09:33.254533 10047 ssh_runner.go:195] Run: sh -c "stat /etc/cni/net.d/*loopback.conf*"
W1009 13:09:33.258620 10047 cni.go:209] loopback cni configuration skipped: "/etc/cni/net.d/*loopback.conf*" not found
I1009 13:09:33.258674 10047 ssh_runner.go:195] Run: sudo find /etc/cni/net.d -maxdepth 1 -type f ( ( -name *bridge* -or -name *podman* ) -and -not -name *.mk_disabled ) -printf "%p, " -exec sh -c "sudo mv {} {}.mk_disabled" ;
I1009 13:09:33.272957 10047 cni.go:262] disabled [/etc/cni/net.d/87-podman-bridge.conflist] bridge cni config(s)
I1009 13:09:33.272971 10047 start.go:495] detecting cgroup driver to use...
I1009 13:09:33.273071 10047 ssh_runner.go:195] Run: /bin/bash -c "sudo mkdir -p /etc && printf %s "runtime-endpoint: unix:///run/containerd/containerd.sock
" | sudo tee /etc/crictl.yaml"
I1009 13:09:33.290160 10047 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)sandbox_image = .*$|\1sandbox_image = "registry.k8s.io/pause:3.10"|' /etc/containerd/config.toml"
I1009 13:09:33.298936 10047 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)restrict_oom_score_adj = .*$|\1restrict_oom_score_adj = false|' /etc/containerd/config.toml"
I1009 13:09:33.307535 10047 containerd.go:146] configuring containerd to use "cgroupfs" as cgroup driver...
I1009 13:09:33.307588 10047 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)SystemdCgroup = .*$|\1SystemdCgroup = false|g' /etc/containerd/config.toml"
I1009 13:09:33.316105 10047 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|"io.containerd.runtime.v1.linux"|"io.containerd.runc.v2"|g' /etc/containerd/config.toml"
I1009 13:09:33.324804 10047 ssh_runner.go:195] Run: sh -c "sudo sed -i '/systemd_cgroup/d' /etc/containerd/config.toml"
I1009 13:09:33.333456 10047 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|"io.containerd.runc.v1"|"io.containerd.runc.v2"|g' /etc/containerd/config.toml"
I1009 13:09:33.342046 10047 ssh_runner.go:195] Run: sh -c "sudo rm -rf /etc/cni/net.mk"
I1009 13:09:33.350970 10047 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)conf_dir = .*$|\1conf_dir = "/etc/cni/net.d"|g' /etc/containerd/config.toml"
I1009 13:09:33.359608 10047 ssh_runner.go:195] Run: sh -c "sudo sed -i '/^ *enable_unprivileged_ports = .*/d' /etc/containerd/config.toml"
I1009 13:09:33.368305 10047 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)\[plugins."io.containerd.grpc.v1.cri"\]|&\n\1 enable_unprivileged_ports = true|' /etc/containerd/config.toml"
I1009 13:09:33.377192 10047 ssh_runner.go:195] Run: sudo sysctl net.bridge.bridge-nf-call-iptables
I1009 13:09:33.385072 10047 crio.go:166] couldn't verify netfilter by "sudo sysctl net.bridge.bridge-nf-call-iptables" which might be okay. error: sudo sysctl net.bridge.bridge-nf-call-iptables: Process exited with status 255
stdout:
stderr:
sysctl: cannot stat /proc/sys/net/bridge/bridge-nf-call-iptables: No such file or directory
I1009 13:09:33.385126 10047 ssh_runner.go:195] Run: sudo modprobe br_netfilter
I1009 13:09:33.394195 10047 ssh_runner.go:195] Run: sudo sh -c "echo 1 > /proc/sys/net/ipv4/ip_forward"
I1009 13:09:33.402276 10047 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I1009 13:09:33.508469 10047 ssh_runner.go:195] Run: sudo systemctl restart containerd
I1009 13:09:33.527676 10047 start.go:495] detecting cgroup driver to use...
I1009 13:09:33.527772 10047 ssh_runner.go:195] Run: sudo systemctl cat docker.service
I1009 13:09:33.541265 10047 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service containerd
I1009 13:09:33.552624 10047 ssh_runner.go:195] Run: sudo systemctl stop -f containerd
I1009 13:09:33.571884 10047 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service containerd
I1009 13:09:33.583061 10047 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service crio
I1009 13:09:33.594079 10047 ssh_runner.go:195] Run: sudo systemctl stop -f crio
I1009 13:09:33.618423 10047 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service crio
I1009 13:09:33.629832 10047 ssh_runner.go:195] Run: /bin/bash -c "sudo mkdir -p /etc && printf %s "runtime-endpoint: unix:///var/run/cri-dockerd.sock
" | sudo tee /etc/crictl.yaml"
I1009 13:09:33.644591 10047 ssh_runner.go:195] Run: which cri-dockerd
I1009 13:09:33.647426 10047 ssh_runner.go:195] Run: sudo mkdir -p /etc/systemd/system/cri-docker.service.d
I1009 13:09:33.655690 10047 ssh_runner.go:362] scp memory --> /etc/systemd/system/cri-docker.service.d/10-cni.conf (190 bytes)
I1009 13:09:33.669165 10047 ssh_runner.go:195] Run: sudo systemctl unmask docker.service
I1009 13:09:33.764831 10047 ssh_runner.go:195] Run: sudo systemctl enable docker.socket
I1009 13:09:33.862413 10047 docker.go:574] configuring docker to use "cgroupfs" as cgroup driver...
I1009 13:09:33.862497 10047 ssh_runner.go:362] scp memory --> /etc/docker/daemon.json (130 bytes)
I1009 13:09:33.876453 10047 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I1009 13:09:33.969231 10047 ssh_runner.go:195] Run: sudo systemctl restart docker
I1009 13:09:36.310629 10047 ssh_runner.go:235] Completed: sudo systemctl restart docker: (2.341431098s)
I1009 13:09:36.310720 10047 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service cri-docker.socket
I1009 13:09:36.320976 10047 ssh_runner.go:195] Run: sudo systemctl stop cri-docker.socket
I1009 13:09:36.334364 10047 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service cri-docker.service
I1009 13:09:36.344871 10047 ssh_runner.go:195] Run: sudo systemctl unmask cri-docker.socket
I1009 13:09:36.435639 10047 ssh_runner.go:195] Run: sudo systemctl enable cri-docker.socket
I1009 13:09:36.540034 10047 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I1009 13:09:36.648604 10047 ssh_runner.go:195] Run: sudo systemctl restart cri-docker.socket
I1009 13:09:36.662190 10047 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service cri-docker.service
I1009 13:09:36.673340 10047 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I1009 13:09:36.769920 10047 ssh_runner.go:195] Run: sudo systemctl restart cri-docker.service
I1009 13:09:36.837660 10047 start.go:542] Will wait 60s for socket path /var/run/cri-dockerd.sock
I1009 13:09:36.837762 10047 ssh_runner.go:195] Run: stat /var/run/cri-dockerd.sock
I1009 13:09:36.842030 10047 start.go:563] Will wait 60s for crictl version
I1009 13:09:36.842086 10047 ssh_runner.go:195] Run: which crictl
I1009 13:09:36.845280 10047 ssh_runner.go:195] Run: sudo /usr/bin/crictl version
I1009 13:09:36.877615 10047 start.go:579] Version: 0.1.0
RuntimeName: docker
RuntimeVersion: 27.3.1
RuntimeApiVersion: v1
I1009 13:09:36.877701 10047 ssh_runner.go:195] Run: docker version --format {{.Server.Version}}
I1009 13:09:36.895487 10047 ssh_runner.go:195] Run: docker version --format {{.Server.Version}}
I1009 13:09:36.936720 10047 out.go:235] * Preparing Kubernetes v1.31.1 on Docker 27.3.1 ...
I1009 13:09:36.937152 10047 main.go:141] libmachine: (ha-488000) Calling .GetIP
I1009 13:09:36.937546 10047 ssh_runner.go:195] Run: grep 192.169.0.1 host.minikube.internal$ /etc/hosts
I1009 13:09:36.941666 10047 ssh_runner.go:195] Run: /bin/bash -c "{ grep -v $'\thost.minikube.internal$' "/etc/hosts"; echo "192.169.0.1 host.minikube.internal"; } > /tmp/h.$$; sudo cp /tmp/h.$$ "/etc/hosts""
I1009 13:09:36.951622 10047 kubeadm.go:883] updating cluster {Name:ha-488000 KeepContext:false EmbedCerts:false MinikubeISO:https://storage.googleapis.com/minikube-builds/iso/19774/minikube-v1.34.0-1728382514-19774-amd64.iso KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.45-1728382586-19774@sha256:5d8c4f6d838467365e214e2194dd0153a763e3f78723b5f2a8e06ef7b47409ec Memory:2200 CPUs:2 DiskSize:20000 Driver:hyperkit HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.31.
1 ClusterName:ha-488000 Namespace:default APIServerHAVIP:192.169.0.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:docker CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.169.0.5 Port:8443 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m02 IP:192.169.0.6 Port:8443 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m04 IP:192.169.0.8 Port:0 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:false Worker:true}] Addons:map[ambassador:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false inaccel:false ingress:false ingress-dns:false inspektor-gad
get:false istio:false istio-provisioner:false kong:false kubeflow:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-gluster:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s Mount:false MountString:/Users:/minikube-host Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOptimizations:false D
isableMetrics:false CustomQemuFirmwarePath: SocketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s} ...
I1009 13:09:36.951699 10047 preload.go:131] Checking if preload exists for k8s version v1.31.1 and runtime docker
I1009 13:09:36.951770 10047 ssh_runner.go:195] Run: docker images --format {{.Repository}}:{{.Tag}}
I1009 13:09:36.964999 10047 docker.go:685] Got preloaded images: -- stdout --
kindest/kindnetd:v20241007-36f62932
registry.k8s.io/kube-apiserver:v1.31.1
registry.k8s.io/kube-controller-manager:v1.31.1
registry.k8s.io/kube-scheduler:v1.31.1
registry.k8s.io/kube-proxy:v1.31.1
ghcr.io/kube-vip/kube-vip:v0.8.3
registry.k8s.io/coredns/coredns:v1.11.3
registry.k8s.io/etcd:3.5.15-0
registry.k8s.io/pause:3.10
gcr.io/k8s-minikube/storage-provisioner:v5
gcr.io/k8s-minikube/busybox:1.28
-- /stdout --
I1009 13:09:36.965010 10047 docker.go:615] Images already preloaded, skipping extraction
I1009 13:09:36.965094 10047 ssh_runner.go:195] Run: docker images --format {{.Repository}}:{{.Tag}}
I1009 13:09:36.977363 10047 docker.go:685] Got preloaded images: -- stdout --
kindest/kindnetd:v20241007-36f62932
registry.k8s.io/kube-apiserver:v1.31.1
registry.k8s.io/kube-scheduler:v1.31.1
registry.k8s.io/kube-controller-manager:v1.31.1
registry.k8s.io/kube-proxy:v1.31.1
ghcr.io/kube-vip/kube-vip:v0.8.3
registry.k8s.io/coredns/coredns:v1.11.3
registry.k8s.io/etcd:3.5.15-0
registry.k8s.io/pause:3.10
gcr.io/k8s-minikube/storage-provisioner:v5
gcr.io/k8s-minikube/busybox:1.28
-- /stdout --
I1009 13:09:36.977380 10047 cache_images.go:84] Images are preloaded, skipping loading
I1009 13:09:36.977391 10047 kubeadm.go:934] updating node { 192.169.0.5 8443 v1.31.1 docker true true} ...
I1009 13:09:36.977484 10047 kubeadm.go:946] kubelet [Unit]
Wants=docker.socket
[Service]
ExecStart=
ExecStart=/var/lib/minikube/binaries/v1.31.1/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --config=/var/lib/kubelet/config.yaml --hostname-override=ha-488000 --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=192.169.0.5
[Install]
config:
{KubernetesVersion:v1.31.1 ClusterName:ha-488000 Namespace:default APIServerHAVIP:192.169.0.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:docker CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:}
I1009 13:09:36.977567 10047 ssh_runner.go:195] Run: docker info --format {{.CgroupDriver}}
I1009 13:09:37.011230 10047 cni.go:84] Creating CNI manager for ""
I1009 13:09:37.011243 10047 cni.go:136] multinode detected (3 nodes found), recommending kindnet
I1009 13:09:37.011254 10047 kubeadm.go:84] Using pod CIDR: 10.244.0.0/16
I1009 13:09:37.011268 10047 kubeadm.go:181] kubeadm options: {CertDir:/var/lib/minikube/certs ServiceCIDR:10.96.0.0/12 PodSubnet:10.244.0.0/16 AdvertiseAddress:192.169.0.5 APIServerPort:8443 KubernetesVersion:v1.31.1 EtcdDataDir:/var/lib/minikube/etcd EtcdExtraArgs:map[] ClusterName:ha-488000 NodeName:ha-488000 DNSDomain:cluster.local CRISocket:/var/run/cri-dockerd.sock ImageRepository: ComponentOptions:[{Component:apiServer ExtraArgs:map[enable-admission-plugins:NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota] Pairs:map[certSANs:["127.0.0.1", "localhost", "192.169.0.5"]]} {Component:controllerManager ExtraArgs:map[allocate-node-cidrs:true leader-elect:false] Pairs:map[]} {Component:scheduler ExtraArgs:map[leader-elect:false] Pairs:map[]}] FeatureArgs:map[] NodeIP:192.169.0.5 CgroupDriver:cgroupfs ClientCAFile:/var/lib/minikube/certs/ca.crt StaticPodPath:/etc/kubernetes/manif
ests ControlPlaneAddress:control-plane.minikube.internal KubeProxyOptions:map[] ResolvConfSearchRegression:false KubeletConfigOpts:map[containerRuntimeEndpoint:unix:///var/run/cri-dockerd.sock hairpinMode:hairpin-veth runtimeRequestTimeout:15m] PrependCriSocketUnix:true}
I1009 13:09:37.011350 10047 kubeadm.go:187] kubeadm config:
apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 192.169.0.5
bindPort: 8443
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
ttl: 24h0m0s
usages:
- signing
- authentication
nodeRegistration:
criSocket: unix:///var/run/cri-dockerd.sock
name: "ha-488000"
kubeletExtraArgs:
node-ip: 192.169.0.5
taints: []
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
apiServer:
certSANs: ["127.0.0.1", "localhost", "192.169.0.5"]
extraArgs:
enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
controllerManager:
extraArgs:
allocate-node-cidrs: "true"
leader-elect: "false"
scheduler:
extraArgs:
leader-elect: "false"
certificatesDir: /var/lib/minikube/certs
clusterName: mk
controlPlaneEndpoint: control-plane.minikube.internal:8443
etcd:
local:
dataDir: /var/lib/minikube/etcd
extraArgs:
proxy-refresh-interval: "70000"
kubernetesVersion: v1.31.1
networking:
dnsDomain: cluster.local
podSubnet: "10.244.0.0/16"
serviceSubnet: 10.96.0.0/12
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
authentication:
x509:
clientCAFile: /var/lib/minikube/certs/ca.crt
cgroupDriver: cgroupfs
containerRuntimeEndpoint: unix:///var/run/cri-dockerd.sock
hairpinMode: hairpin-veth
runtimeRequestTimeout: 15m
clusterDomain: "cluster.local"
# disable disk resource management by default
imageGCHighThresholdPercent: 100
evictionHard:
nodefs.available: "0%"
nodefs.inodesFree: "0%"
imagefs.available: "0%"
failSwapOn: false
staticPodPath: /etc/kubernetes/manifests
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
clusterCIDR: "10.244.0.0/16"
metricsBindAddress: 0.0.0.0:10249
conntrack:
maxPerCore: 0
# Skip setting "net.netfilter.nf_conntrack_tcp_timeout_established"
tcpEstablishedTimeout: 0s
# Skip setting "net.netfilter.nf_conntrack_tcp_timeout_close"
tcpCloseWaitTimeout: 0s
I1009 13:09:37.011361 10047 kube-vip.go:115] generating kube-vip config ...
I1009 13:09:37.011427 10047 ssh_runner.go:195] Run: sudo sh -c "modprobe --all ip_vs ip_vs_rr ip_vs_wrr ip_vs_sh nf_conntrack"
I1009 13:09:37.023952 10047 kube-vip.go:167] auto-enabling control-plane load-balancing in kube-vip
I1009 13:09:37.024023 10047 kube-vip.go:137] kube-vip config:
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: null
name: kube-vip
namespace: kube-system
spec:
containers:
- args:
- manager
env:
- name: vip_arp
value: "true"
- name: port
value: "8443"
- name: vip_nodename
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: vip_interface
value: eth0
- name: vip_cidr
value: "32"
- name: dns_mode
value: first
- name: cp_enable
value: "true"
- name: cp_namespace
value: kube-system
- name: vip_leaderelection
value: "true"
- name: vip_leasename
value: plndr-cp-lock
- name: vip_leaseduration
value: "5"
- name: vip_renewdeadline
value: "3"
- name: vip_retryperiod
value: "1"
- name: address
value: 192.169.0.254
- name: prometheus_server
value: :2112
- name : lb_enable
value: "true"
- name: lb_port
value: "8443"
image: ghcr.io/kube-vip/kube-vip:v0.8.3
imagePullPolicy: IfNotPresent
name: kube-vip
resources: {}
securityContext:
capabilities:
add:
- NET_ADMIN
- NET_RAW
volumeMounts:
- mountPath: /etc/kubernetes/admin.conf
name: kubeconfig
hostAliases:
- hostnames:
- kubernetes
ip: 127.0.0.1
hostNetwork: true
volumes:
- hostPath:
path: "/etc/kubernetes/admin.conf"
name: kubeconfig
status: {}
I1009 13:09:37.024094 10047 ssh_runner.go:195] Run: sudo ls /var/lib/minikube/binaries/v1.31.1
I1009 13:09:37.032117 10047 binaries.go:44] Found k8s binaries, skipping transfer
I1009 13:09:37.032176 10047 ssh_runner.go:195] Run: sudo mkdir -p /etc/systemd/system/kubelet.service.d /lib/systemd/system /var/tmp/minikube /etc/kubernetes/manifests
I1009 13:09:37.039581 10047 ssh_runner.go:362] scp memory --> /etc/systemd/system/kubelet.service.d/10-kubeadm.conf (307 bytes)
I1009 13:09:37.053153 10047 ssh_runner.go:362] scp memory --> /lib/systemd/system/kubelet.service (352 bytes)
I1009 13:09:37.066418 10047 ssh_runner.go:362] scp memory --> /var/tmp/minikube/kubeadm.yaml.new (2148 bytes)
I1009 13:09:37.080185 10047 ssh_runner.go:362] scp memory --> /etc/kubernetes/manifests/kube-vip.yaml (1440 bytes)
I1009 13:09:37.093694 10047 ssh_runner.go:195] Run: grep 192.169.0.254 control-plane.minikube.internal$ /etc/hosts
I1009 13:09:37.096707 10047 ssh_runner.go:195] Run: /bin/bash -c "{ grep -v $'\tcontrol-plane.minikube.internal$' "/etc/hosts"; echo "192.169.0.254 control-plane.minikube.internal"; } > /tmp/h.$$; sudo cp /tmp/h.$$ "/etc/hosts""
I1009 13:09:37.106213 10047 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I1009 13:09:37.209835 10047 ssh_runner.go:195] Run: sudo systemctl start kubelet
I1009 13:09:37.224797 10047 certs.go:68] Setting up /Users/jenkins/minikube-integration/19780-6775/.minikube/profiles/ha-488000 for IP: 192.169.0.5
I1009 13:09:37.224813 10047 certs.go:194] generating shared ca certs ...
I1009 13:09:37.224823 10047 certs.go:226] acquiring lock for ca certs: {Name:mkbb267bf2c184481fcce626a487648eb2365342 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1009 13:09:37.225030 10047 certs.go:235] skipping valid "minikubeCA" ca cert: /Users/jenkins/minikube-integration/19780-6775/.minikube/ca.key
I1009 13:09:37.225117 10047 certs.go:235] skipping valid "proxyClientCA" ca cert: /Users/jenkins/minikube-integration/19780-6775/.minikube/proxy-client-ca.key
I1009 13:09:37.225127 10047 certs.go:256] generating profile certs ...
I1009 13:09:37.225217 10047 certs.go:359] skipping valid signed profile cert regeneration for "minikube-user": /Users/jenkins/minikube-integration/19780-6775/.minikube/profiles/ha-488000/client.key
I1009 13:09:37.225241 10047 certs.go:363] generating signed profile cert for "minikube": /Users/jenkins/minikube-integration/19780-6775/.minikube/profiles/ha-488000/apiserver.key.7bd327ec
I1009 13:09:37.225260 10047 crypto.go:68] Generating cert /Users/jenkins/minikube-integration/19780-6775/.minikube/profiles/ha-488000/apiserver.crt.7bd327ec with IP's: [10.96.0.1 127.0.0.1 10.0.0.1 192.169.0.5 192.169.0.6 192.169.0.254]
I1009 13:09:37.369755 10047 crypto.go:156] Writing cert to /Users/jenkins/minikube-integration/19780-6775/.minikube/profiles/ha-488000/apiserver.crt.7bd327ec ...
I1009 13:09:37.369771 10047 lock.go:35] WriteFile acquiring /Users/jenkins/minikube-integration/19780-6775/.minikube/profiles/ha-488000/apiserver.crt.7bd327ec: {Name:mk39ae7f7b77bd754c2aecf642c8318b58d69821 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1009 13:09:37.370439 10047 crypto.go:164] Writing key to /Users/jenkins/minikube-integration/19780-6775/.minikube/profiles/ha-488000/apiserver.key.7bd327ec ...
I1009 13:09:37.370450 10047 lock.go:35] WriteFile acquiring /Users/jenkins/minikube-integration/19780-6775/.minikube/profiles/ha-488000/apiserver.key.7bd327ec: {Name:mkc06752c93863b38901f4ba65c0c7246b44478c Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1009 13:09:37.370733 10047 certs.go:381] copying /Users/jenkins/minikube-integration/19780-6775/.minikube/profiles/ha-488000/apiserver.crt.7bd327ec -> /Users/jenkins/minikube-integration/19780-6775/.minikube/profiles/ha-488000/apiserver.crt
I1009 13:09:37.370951 10047 certs.go:385] copying /Users/jenkins/minikube-integration/19780-6775/.minikube/profiles/ha-488000/apiserver.key.7bd327ec -> /Users/jenkins/minikube-integration/19780-6775/.minikube/profiles/ha-488000/apiserver.key
I1009 13:09:37.371230 10047 certs.go:359] skipping valid signed profile cert regeneration for "aggregator": /Users/jenkins/minikube-integration/19780-6775/.minikube/profiles/ha-488000/proxy-client.key
I1009 13:09:37.371239 10047 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19780-6775/.minikube/ca.crt -> /var/lib/minikube/certs/ca.crt
I1009 13:09:37.371288 10047 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19780-6775/.minikube/ca.key -> /var/lib/minikube/certs/ca.key
I1009 13:09:37.371311 10047 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19780-6775/.minikube/proxy-client-ca.crt -> /var/lib/minikube/certs/proxy-client-ca.crt
I1009 13:09:37.371331 10047 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19780-6775/.minikube/proxy-client-ca.key -> /var/lib/minikube/certs/proxy-client-ca.key
I1009 13:09:37.371356 10047 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19780-6775/.minikube/profiles/ha-488000/apiserver.crt -> /var/lib/minikube/certs/apiserver.crt
I1009 13:09:37.371381 10047 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19780-6775/.minikube/profiles/ha-488000/apiserver.key -> /var/lib/minikube/certs/apiserver.key
I1009 13:09:37.371401 10047 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19780-6775/.minikube/profiles/ha-488000/proxy-client.crt -> /var/lib/minikube/certs/proxy-client.crt
I1009 13:09:37.371422 10047 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19780-6775/.minikube/profiles/ha-488000/proxy-client.key -> /var/lib/minikube/certs/proxy-client.key
I1009 13:09:37.371522 10047 certs.go:484] found cert: /Users/jenkins/minikube-integration/19780-6775/.minikube/certs/7318.pem (1338 bytes)
W1009 13:09:37.371583 10047 certs.go:480] ignoring /Users/jenkins/minikube-integration/19780-6775/.minikube/certs/7318_empty.pem, impossibly tiny 0 bytes
I1009 13:09:37.371592 10047 certs.go:484] found cert: /Users/jenkins/minikube-integration/19780-6775/.minikube/certs/ca-key.pem (1675 bytes)
I1009 13:09:37.371622 10047 certs.go:484] found cert: /Users/jenkins/minikube-integration/19780-6775/.minikube/certs/ca.pem (1082 bytes)
I1009 13:09:37.371652 10047 certs.go:484] found cert: /Users/jenkins/minikube-integration/19780-6775/.minikube/certs/cert.pem (1123 bytes)
I1009 13:09:37.371686 10047 certs.go:484] found cert: /Users/jenkins/minikube-integration/19780-6775/.minikube/certs/key.pem (1675 bytes)
I1009 13:09:37.371752 10047 certs.go:484] found cert: /Users/jenkins/minikube-integration/19780-6775/.minikube/files/etc/ssl/certs/73182.pem (1708 bytes)
I1009 13:09:37.371790 10047 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19780-6775/.minikube/certs/7318.pem -> /usr/share/ca-certificates/7318.pem
I1009 13:09:37.371811 10047 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19780-6775/.minikube/files/etc/ssl/certs/73182.pem -> /usr/share/ca-certificates/73182.pem
I1009 13:09:37.371829 10047 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19780-6775/.minikube/ca.crt -> /usr/share/ca-certificates/minikubeCA.pem
I1009 13:09:37.372273 10047 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19780-6775/.minikube/ca.crt --> /var/lib/minikube/certs/ca.crt (1111 bytes)
I1009 13:09:37.414818 10047 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19780-6775/.minikube/ca.key --> /var/lib/minikube/certs/ca.key (1679 bytes)
I1009 13:09:37.476985 10047 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19780-6775/.minikube/proxy-client-ca.crt --> /var/lib/minikube/certs/proxy-client-ca.crt (1119 bytes)
I1009 13:09:37.528825 10047 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19780-6775/.minikube/proxy-client-ca.key --> /var/lib/minikube/certs/proxy-client-ca.key (1679 bytes)
I1009 13:09:37.567078 10047 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19780-6775/.minikube/profiles/ha-488000/apiserver.crt --> /var/lib/minikube/certs/apiserver.crt (1432 bytes)
I1009 13:09:37.605942 10047 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19780-6775/.minikube/profiles/ha-488000/apiserver.key --> /var/lib/minikube/certs/apiserver.key (1675 bytes)
I1009 13:09:37.641809 10047 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19780-6775/.minikube/profiles/ha-488000/proxy-client.crt --> /var/lib/minikube/certs/proxy-client.crt (1147 bytes)
I1009 13:09:37.671064 10047 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19780-6775/.minikube/profiles/ha-488000/proxy-client.key --> /var/lib/minikube/certs/proxy-client.key (1679 bytes)
I1009 13:09:37.698648 10047 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19780-6775/.minikube/certs/7318.pem --> /usr/share/ca-certificates/7318.pem (1338 bytes)
I1009 13:09:37.722047 10047 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19780-6775/.minikube/files/etc/ssl/certs/73182.pem --> /usr/share/ca-certificates/73182.pem (1708 bytes)
I1009 13:09:37.741855 10047 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19780-6775/.minikube/ca.crt --> /usr/share/ca-certificates/minikubeCA.pem (1111 bytes)
I1009 13:09:37.761569 10047 ssh_runner.go:362] scp memory --> /var/lib/minikube/kubeconfig (738 bytes)
I1009 13:09:37.775142 10047 ssh_runner.go:195] Run: openssl version
I1009 13:09:37.779547 10047 ssh_runner.go:195] Run: sudo /bin/bash -c "test -s /usr/share/ca-certificates/7318.pem && ln -fs /usr/share/ca-certificates/7318.pem /etc/ssl/certs/7318.pem"
I1009 13:09:37.787837 10047 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/7318.pem
I1009 13:09:37.791355 10047 certs.go:528] hashing: -rw-r--r-- 1 root root 1338 Oct 9 19:53 /usr/share/ca-certificates/7318.pem
I1009 13:09:37.791401 10047 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/7318.pem
I1009 13:09:37.795647 10047 ssh_runner.go:195] Run: sudo /bin/bash -c "test -L /etc/ssl/certs/51391683.0 || ln -fs /etc/ssl/certs/7318.pem /etc/ssl/certs/51391683.0"
I1009 13:09:37.804027 10047 ssh_runner.go:195] Run: sudo /bin/bash -c "test -s /usr/share/ca-certificates/73182.pem && ln -fs /usr/share/ca-certificates/73182.pem /etc/ssl/certs/73182.pem"
I1009 13:09:37.812262 10047 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/73182.pem
I1009 13:09:37.815655 10047 certs.go:528] hashing: -rw-r--r-- 1 root root 1708 Oct 9 19:53 /usr/share/ca-certificates/73182.pem
I1009 13:09:37.815700 10047 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/73182.pem
I1009 13:09:37.819981 10047 ssh_runner.go:195] Run: sudo /bin/bash -c "test -L /etc/ssl/certs/3ec20f2e.0 || ln -fs /etc/ssl/certs/73182.pem /etc/ssl/certs/3ec20f2e.0"
I1009 13:09:37.828434 10047 ssh_runner.go:195] Run: sudo /bin/bash -c "test -s /usr/share/ca-certificates/minikubeCA.pem && ln -fs /usr/share/ca-certificates/minikubeCA.pem /etc/ssl/certs/minikubeCA.pem"
I1009 13:09:37.837311 10047 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/minikubeCA.pem
I1009 13:09:37.840959 10047 certs.go:528] hashing: -rw-r--r-- 1 root root 1111 Oct 9 19:35 /usr/share/ca-certificates/minikubeCA.pem
I1009 13:09:37.841023 10047 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/minikubeCA.pem
I1009 13:09:37.845407 10047 ssh_runner.go:195] Run: sudo /bin/bash -c "test -L /etc/ssl/certs/b5213941.0 || ln -fs /etc/ssl/certs/minikubeCA.pem /etc/ssl/certs/b5213941.0"
I1009 13:09:37.854209 10047 ssh_runner.go:195] Run: stat /var/lib/minikube/certs/apiserver-kubelet-client.crt
I1009 13:09:37.857713 10047 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/apiserver-etcd-client.crt -checkend 86400
I1009 13:09:37.862027 10047 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/apiserver-kubelet-client.crt -checkend 86400
I1009 13:09:37.866278 10047 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/etcd/server.crt -checkend 86400
I1009 13:09:37.870716 10047 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/etcd/healthcheck-client.crt -checkend 86400
I1009 13:09:37.874960 10047 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/etcd/peer.crt -checkend 86400
I1009 13:09:37.879257 10047 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/front-proxy-client.crt -checkend 86400
I1009 13:09:37.883490 10047 kubeadm.go:392] StartCluster: {Name:ha-488000 KeepContext:false EmbedCerts:false MinikubeISO:https://storage.googleapis.com/minikube-builds/iso/19774/minikube-v1.34.0-1728382514-19774-amd64.iso KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.45-1728382586-19774@sha256:5d8c4f6d838467365e214e2194dd0153a763e3f78723b5f2a8e06ef7b47409ec Memory:2200 CPUs:2 DiskSize:20000 Driver:hyperkit HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.31.1 C
lusterName:ha-488000 Namespace:default APIServerHAVIP:192.169.0.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:docker CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.169.0.5 Port:8443 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m02 IP:192.169.0.6 Port:8443 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m04 IP:192.169.0.8 Port:0 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:false Worker:true}] Addons:map[ambassador:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false inaccel:false ingress:false ingress-dns:false inspektor-gadget
:false istio:false istio-provisioner:false kong:false kubeflow:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-gluster:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s Mount:false MountString:/Users:/minikube-host Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOptimizations:false Disa
bleMetrics:false CustomQemuFirmwarePath: SocketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s}
I1009 13:09:37.883613 10047 ssh_runner.go:195] Run: docker ps --filter status=paused --filter=name=k8s_.*_(kube-system)_ --format={{.ID}}
I1009 13:09:37.895010 10047 ssh_runner.go:195] Run: sudo ls /var/lib/kubelet/kubeadm-flags.env /var/lib/kubelet/config.yaml /var/lib/minikube/etcd
I1009 13:09:37.902655 10047 kubeadm.go:408] found existing configuration files, will attempt cluster restart
I1009 13:09:37.902666 10047 kubeadm.go:593] restartPrimaryControlPlane start ...
I1009 13:09:37.902720 10047 ssh_runner.go:195] Run: sudo test -d /data/minikube
I1009 13:09:37.910165 10047 kubeadm.go:130] /data/minikube skipping compat symlinks: sudo test -d /data/minikube: Process exited with status 1
stdout:
stderr:
I1009 13:09:37.910499 10047 kubeconfig.go:47] verify endpoint returned: get endpoint: "ha-488000" does not appear in /Users/jenkins/minikube-integration/19780-6775/kubeconfig
I1009 13:09:37.910582 10047 kubeconfig.go:62] /Users/jenkins/minikube-integration/19780-6775/kubeconfig needs updating (will repair): [kubeconfig missing "ha-488000" cluster setting kubeconfig missing "ha-488000" context setting]
I1009 13:09:37.910801 10047 lock.go:35] WriteFile acquiring /Users/jenkins/minikube-integration/19780-6775/kubeconfig: {Name:mkcde35a6d2de52f61718633b30bfba6bb83db45 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1009 13:09:37.911478 10047 loader.go:395] Config loaded from file: /Users/jenkins/minikube-integration/19780-6775/kubeconfig
I1009 13:09:37.911695 10047 kapi.go:59] client config for ha-488000: &rest.Config{Host:"https://192.169.0.5:8443", APIPath:"", ContentConfig:rest.ContentConfig{AcceptContentTypes:"", ContentType:"", GroupVersion:(*schema.GroupVersion)(nil), NegotiatedSerializer:runtime.NegotiatedSerializer(nil)}, Username:"", Password:"", BearerToken:"", BearerTokenFile:"", Impersonate:rest.ImpersonationConfig{UserName:"", UID:"", Groups:[]string(nil), Extra:map[string][]string(nil)}, AuthProvider:<nil>, AuthConfigPersister:rest.AuthProviderConfigPersister(nil), ExecProvider:<nil>, TLSClientConfig:rest.sanitizedTLSClientConfig{Insecure:false, ServerName:"", CertFile:"/Users/jenkins/minikube-integration/19780-6775/.minikube/profiles/ha-488000/client.crt", KeyFile:"/Users/jenkins/minikube-integration/19780-6775/.minikube/profiles/ha-488000/client.key", CAFile:"/Users/jenkins/minikube-integration/19780-6775/.minikube/ca.crt", CertData:[]uint8(nil), KeyData:[]uint8(nil), CAData:[]uint8(nil), NextProtos:[]string(nil)},
UserAgent:"", DisableCompression:false, Transport:http.RoundTripper(nil), WrapTransport:(transport.WrapperFunc)(0x10dc21c0), QPS:0, Burst:0, RateLimiter:flowcontrol.RateLimiter(nil), WarningHandler:rest.WarningHandler(nil), Timeout:0, Dial:(func(context.Context, string, string) (net.Conn, error))(nil), Proxy:(func(*http.Request) (*url.URL, error))(nil)}
I1009 13:09:37.912037 10047 cert_rotation.go:140] Starting client certificate rotation controller
I1009 13:09:37.912181 10047 ssh_runner.go:195] Run: sudo diff -u /var/tmp/minikube/kubeadm.yaml /var/tmp/minikube/kubeadm.yaml.new
I1009 13:09:37.919406 10047 kubeadm.go:630] The running cluster does not require reconfiguration: 192.169.0.5
I1009 13:09:37.919420 10047 kubeadm.go:597] duration metric: took 16.750098ms to restartPrimaryControlPlane
I1009 13:09:37.919425 10047 kubeadm.go:394] duration metric: took 35.941754ms to StartCluster
I1009 13:09:37.919434 10047 settings.go:142] acquiring lock: {Name:mk03dee51c54ceeed0a9c6565977bdceffdeaba5 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1009 13:09:37.919522 10047 settings.go:150] Updating kubeconfig: /Users/jenkins/minikube-integration/19780-6775/kubeconfig
I1009 13:09:37.919841 10047 lock.go:35] WriteFile acquiring /Users/jenkins/minikube-integration/19780-6775/kubeconfig: {Name:mkcde35a6d2de52f61718633b30bfba6bb83db45 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1009 13:09:37.920060 10047 start.go:233] HA (multi-control plane) cluster: will skip waiting for primary control-plane node &{Name: IP:192.169.0.5 Port:8443 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:true Worker:true}
I1009 13:09:37.920074 10047 start.go:241] waiting for startup goroutines ...
I1009 13:09:37.920079 10047 addons.go:507] enable addons start: toEnable=map[ambassador:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false inaccel:false ingress:false ingress-dns:false inspektor-gadget:false istio:false istio-provisioner:false kong:false kubeflow:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-gluster:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false]
I1009 13:09:37.920200 10047 config.go:182] Loaded profile config "ha-488000": Driver=hyperkit, ContainerRuntime=docker, KubernetesVersion=v1.31.1
I1009 13:09:37.942407 10047 out.go:177] * Enabled addons:
I1009 13:09:37.963170 10047 addons.go:510] duration metric: took 43.092882ms for enable addons: enabled=[]
I1009 13:09:37.963210 10047 start.go:246] waiting for cluster config update ...
I1009 13:09:37.963222 10047 start.go:255] writing updated cluster config ...
I1009 13:09:37.985051 10047 out.go:201]
I1009 13:09:38.006662 10047 config.go:182] Loaded profile config "ha-488000": Driver=hyperkit, ContainerRuntime=docker, KubernetesVersion=v1.31.1
I1009 13:09:38.006798 10047 profile.go:143] Saving config to /Users/jenkins/minikube-integration/19780-6775/.minikube/profiles/ha-488000/config.json ...
I1009 13:09:38.029272 10047 out.go:177] * Starting "ha-488000-m02" control-plane node in "ha-488000" cluster
I1009 13:09:38.071348 10047 preload.go:131] Checking if preload exists for k8s version v1.31.1 and runtime docker
I1009 13:09:38.071384 10047 cache.go:56] Caching tarball of preloaded images
I1009 13:09:38.071588 10047 preload.go:172] Found /Users/jenkins/minikube-integration/19780-6775/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.31.1-docker-overlay2-amd64.tar.lz4 in cache, skipping download
I1009 13:09:38.071608 10047 cache.go:59] Finished verifying existence of preloaded tar for v1.31.1 on docker
I1009 13:09:38.071728 10047 profile.go:143] Saving config to /Users/jenkins/minikube-integration/19780-6775/.minikube/profiles/ha-488000/config.json ...
I1009 13:09:38.072664 10047 start.go:360] acquireMachinesLock for ha-488000-m02: {Name:mk693e628b9fda7cceeeedd30d9cc7939119b68f Clock:{} Delay:500ms Timeout:13m0s Cancel:<nil>}
I1009 13:09:38.072766 10047 start.go:364] duration metric: took 78.93µs to acquireMachinesLock for "ha-488000-m02"
I1009 13:09:38.072789 10047 start.go:96] Skipping create...Using existing machine configuration
I1009 13:09:38.072797 10047 fix.go:54] fixHost starting: m02
I1009 13:09:38.073216 10047 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I1009 13:09:38.073251 10047 main.go:141] libmachine: Launching plugin server for driver hyperkit
I1009 13:09:38.085004 10047 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:52295
I1009 13:09:38.085321 10047 main.go:141] libmachine: () Calling .GetVersion
I1009 13:09:38.085709 10047 main.go:141] libmachine: Using API Version 1
I1009 13:09:38.085727 10047 main.go:141] libmachine: () Calling .SetConfigRaw
I1009 13:09:38.085992 10047 main.go:141] libmachine: () Calling .GetMachineName
I1009 13:09:38.086120 10047 main.go:141] libmachine: (ha-488000-m02) Calling .DriverName
I1009 13:09:38.086216 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetState
I1009 13:09:38.086295 10047 main.go:141] libmachine: (ha-488000-m02) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I1009 13:09:38.086370 10047 main.go:141] libmachine: (ha-488000-m02) DBG | hyperkit pid from json: 9945
I1009 13:09:38.087526 10047 main.go:141] libmachine: (ha-488000-m02) DBG | hyperkit pid 9945 missing from process table
I1009 13:09:38.087558 10047 fix.go:112] recreateIfNeeded on ha-488000-m02: state=Stopped err=<nil>
I1009 13:09:38.087574 10047 main.go:141] libmachine: (ha-488000-m02) Calling .DriverName
W1009 13:09:38.087670 10047 fix.go:138] unexpected machine state, will restart: <nil>
I1009 13:09:38.109324 10047 out.go:177] * Restarting existing hyperkit VM for "ha-488000-m02" ...
I1009 13:09:38.151234 10047 main.go:141] libmachine: (ha-488000-m02) Calling .Start
I1009 13:09:38.151504 10047 main.go:141] libmachine: (ha-488000-m02) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I1009 13:09:38.151603 10047 main.go:141] libmachine: (ha-488000-m02) minikube might have been shutdown in an unclean way, the hyperkit pid file still exists: /Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02/hyperkit.pid
I1009 13:09:38.153797 10047 main.go:141] libmachine: (ha-488000-m02) DBG | hyperkit pid 9945 missing from process table
I1009 13:09:38.153812 10047 main.go:141] libmachine: (ha-488000-m02) DBG | pid 9945 is in state "Stopped"
I1009 13:09:38.153831 10047 main.go:141] libmachine: (ha-488000-m02) DBG | Removing stale pid file /Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02/hyperkit.pid...
I1009 13:09:38.154128 10047 main.go:141] libmachine: (ha-488000-m02) DBG | Using UUID 40ee58d1-2daa-4825-9128-943d98a2eb00
I1009 13:09:38.180157 10047 main.go:141] libmachine: (ha-488000-m02) DBG | Generated MAC 76:27:9b:6:7d:7a
I1009 13:09:38.180175 10047 main.go:141] libmachine: (ha-488000-m02) DBG | Starting with cmdline: loglevel=3 console=ttyS0 console=tty0 noembed nomodeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-488000
I1009 13:09:38.180342 10047 main.go:141] libmachine: (ha-488000-m02) DBG | 2024/10/09 13:09:38 DEBUG: hyperkit: Start &hyperkit.HyperKit{HyperKit:"/usr/local/bin/hyperkit", Argv0:"", StateDir:"/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02", VPNKitSock:"", VPNKitUUID:"", VPNKitPreferredIPv4:"", UUID:"40ee58d1-2daa-4825-9128-943d98a2eb00", Disks:[]hyperkit.Disk{(*hyperkit.RawDisk)(0xc00042daa0)}, ISOImages:[]string{"/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02/boot2docker.iso"}, VSock:false, VSockDir:"", VSockPorts:[]int(nil), VSockGuestCID:3, VMNet:true, Sockets9P:[]hyperkit.Socket9P(nil), Kernel:"/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02/bzimage", Initrd:"/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02/initrd", Bootrom:"", CPUs:2, Memory:2200, Console:1, Serials:[]hyperkit.Serial(nil), Pid:0, Arguments:[]string(nil), CmdLine:"", process:(*os.Process)(nil)}
I1009 13:09:38.180368 10047 main.go:141] libmachine: (ha-488000-m02) DBG | 2024/10/09 13:09:38 DEBUG: hyperkit: check &hyperkit.HyperKit{HyperKit:"/usr/local/bin/hyperkit", Argv0:"", StateDir:"/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02", VPNKitSock:"", VPNKitUUID:"", VPNKitPreferredIPv4:"", UUID:"40ee58d1-2daa-4825-9128-943d98a2eb00", Disks:[]hyperkit.Disk{(*hyperkit.RawDisk)(0xc00042daa0)}, ISOImages:[]string{"/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02/boot2docker.iso"}, VSock:false, VSockDir:"", VSockPorts:[]int(nil), VSockGuestCID:3, VMNet:true, Sockets9P:[]hyperkit.Socket9P(nil), Kernel:"/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02/bzimage", Initrd:"/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02/initrd", Bootrom:"", CPUs:2, Memory:2200, Console:1, Serials:[]hyperkit.Serial(nil), Pid:0, Arguments:[]string(nil), CmdLine:"", process:(*os.Process)(nil)}
I1009 13:09:38.180451 10047 main.go:141] libmachine: (ha-488000-m02) DBG | 2024/10/09 13:09:38 DEBUG: hyperkit: Arguments: []string{"-A", "-u", "-F", "/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02/hyperkit.pid", "-c", "2", "-m", "2200M", "-s", "0:0,hostbridge", "-s", "31,lpc", "-s", "1:0,virtio-net", "-U", "40ee58d1-2daa-4825-9128-943d98a2eb00", "-s", "2:0,virtio-blk,/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02/ha-488000-m02.rawdisk", "-s", "3,ahci-cd,/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02/boot2docker.iso", "-s", "4,virtio-rnd", "-l", "com1,autopty=/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02/tty,log=/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02/console-ring", "-f", "kexec,/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02/bzimage,/Users/jenkins/minikube-integration/19780-6775/.minikube/machine
s/ha-488000-m02/initrd,earlyprintk=serial loglevel=3 console=ttyS0 console=tty0 noembed nomodeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-488000"}
I1009 13:09:38.180488 10047 main.go:141] libmachine: (ha-488000-m02) DBG | 2024/10/09 13:09:38 DEBUG: hyperkit: CmdLine: "/usr/local/bin/hyperkit -A -u -F /Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02/hyperkit.pid -c 2 -m 2200M -s 0:0,hostbridge -s 31,lpc -s 1:0,virtio-net -U 40ee58d1-2daa-4825-9128-943d98a2eb00 -s 2:0,virtio-blk,/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02/ha-488000-m02.rawdisk -s 3,ahci-cd,/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02/boot2docker.iso -s 4,virtio-rnd -l com1,autopty=/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02/tty,log=/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02/console-ring -f kexec,/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02/bzimage,/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02/initrd,earlyprintk=serial loglevel=3 console=t
tyS0 console=tty0 noembed nomodeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-488000"
I1009 13:09:38.180509 10047 main.go:141] libmachine: (ha-488000-m02) DBG | 2024/10/09 13:09:38 DEBUG: hyperkit: Redirecting stdout/stderr to logger
I1009 13:09:38.181760 10047 main.go:141] libmachine: (ha-488000-m02) DBG | 2024/10/09 13:09:38 DEBUG: hyperkit: Pid is 10071
I1009 13:09:38.182230 10047 main.go:141] libmachine: (ha-488000-m02) DBG | Attempt 0
I1009 13:09:38.182250 10047 main.go:141] libmachine: (ha-488000-m02) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I1009 13:09:38.182347 10047 main.go:141] libmachine: (ha-488000-m02) DBG | hyperkit pid from json: 10071
I1009 13:09:38.184103 10047 main.go:141] libmachine: (ha-488000-m02) DBG | Searching for 76:27:9b:6:7d:7a in /var/db/dhcpd_leases ...
I1009 13:09:38.184182 10047 main.go:141] libmachine: (ha-488000-m02) DBG | Found 7 entries in /var/db/dhcpd_leases!
I1009 13:09:38.184202 10047 main.go:141] libmachine: (ha-488000-m02) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.5 HWAddress:22:8b:d6:6a:e4:67 ID:1,22:8b:d6:6a:e4:67 Lease:0x6706f108}
I1009 13:09:38.184229 10047 main.go:141] libmachine: (ha-488000-m02) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.7 HWAddress:b6:2:d4:66:de:30 ID:1,b6:2:d4:66:de:30 Lease:0x6706e2d3}
I1009 13:09:38.184244 10047 main.go:141] libmachine: (ha-488000-m02) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.8 HWAddress:c6:88:ed:5f:e9:2a ID:1,c6:88:ed:5f:e9:2a Lease:0x6706f0b6}
I1009 13:09:38.184262 10047 main.go:141] libmachine: (ha-488000-m02) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.6 HWAddress:76:27:9b:6:7d:7a ID:1,76:27:9b:6:7d:7a Lease:0x6706f04c}
I1009 13:09:38.184282 10047 main.go:141] libmachine: (ha-488000-m02) DBG | Found match: 76:27:9b:6:7d:7a
I1009 13:09:38.184302 10047 main.go:141] libmachine: (ha-488000-m02) DBG | IP: 192.169.0.6
I1009 13:09:38.184346 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetConfigRaw
I1009 13:09:38.185061 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetIP
I1009 13:09:38.185258 10047 profile.go:143] Saving config to /Users/jenkins/minikube-integration/19780-6775/.minikube/profiles/ha-488000/config.json ...
I1009 13:09:38.185710 10047 machine.go:93] provisionDockerMachine start ...
I1009 13:09:38.185720 10047 main.go:141] libmachine: (ha-488000-m02) Calling .DriverName
I1009 13:09:38.185833 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHHostname
I1009 13:09:38.185940 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHPort
I1009 13:09:38.186125 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHKeyPath
I1009 13:09:38.186226 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHKeyPath
I1009 13:09:38.186313 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHUsername
I1009 13:09:38.186449 10047 main.go:141] libmachine: Using SSH client type: native
I1009 13:09:38.186613 10047 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xf225520] 0xf228200 <nil> [] 0s} 192.169.0.6 22 <nil> <nil>}
I1009 13:09:38.186620 10047 main.go:141] libmachine: About to run SSH command:
hostname
I1009 13:09:38.192524 10047 main.go:141] libmachine: (ha-488000-m02) DBG | 2024/10/09 13:09:38 INFO : hyperkit: stderr: Using fd 6 for I/O notifications
I1009 13:09:38.202252 10047 main.go:141] libmachine: (ha-488000-m02) DBG | 2024/10/09 13:09:38 INFO : hyperkit: stderr: /Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02/boot2docker.iso: fcntl(F_PUNCHHOLE) Operation not permitted: block device will not support TRIM/DISCARD
I1009 13:09:38.203425 10047 main.go:141] libmachine: (ha-488000-m02) DBG | 2024/10/09 13:09:38 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 2 bit: 22 unspecified don't care: bit is 0
I1009 13:09:38.203448 10047 main.go:141] libmachine: (ha-488000-m02) DBG | 2024/10/09 13:09:38 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 12 unspecified don't care: bit is 0
I1009 13:09:38.203471 10047 main.go:141] libmachine: (ha-488000-m02) DBG | 2024/10/09 13:09:38 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 20 unspecified don't care: bit is 0
I1009 13:09:38.203487 10047 main.go:141] libmachine: (ha-488000-m02) DBG | 2024/10/09 13:09:38 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 3 bit: 13 unspecified don't care: bit is 0
I1009 13:09:38.593433 10047 main.go:141] libmachine: (ha-488000-m02) DBG | 2024/10/09 13:09:38 INFO : hyperkit: stderr: rdmsr to register 0x3a on vcpu 0
I1009 13:09:38.593447 10047 main.go:141] libmachine: (ha-488000-m02) DBG | 2024/10/09 13:09:38 INFO : hyperkit: stderr: rdmsr to register 0x140 on vcpu 0
I1009 13:09:38.708191 10047 main.go:141] libmachine: (ha-488000-m02) DBG | 2024/10/09 13:09:38 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 2 bit: 22 unspecified don't care: bit is 0
I1009 13:09:38.708209 10047 main.go:141] libmachine: (ha-488000-m02) DBG | 2024/10/09 13:09:38 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 12 unspecified don't care: bit is 0
I1009 13:09:38.708216 10047 main.go:141] libmachine: (ha-488000-m02) DBG | 2024/10/09 13:09:38 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 20 unspecified don't care: bit is 0
I1009 13:09:38.708221 10047 main.go:141] libmachine: (ha-488000-m02) DBG | 2024/10/09 13:09:38 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 3 bit: 13 unspecified don't care: bit is 0
I1009 13:09:38.709046 10047 main.go:141] libmachine: (ha-488000-m02) DBG | 2024/10/09 13:09:38 INFO : hyperkit: stderr: rdmsr to register 0x3a on vcpu 1
I1009 13:09:38.709056 10047 main.go:141] libmachine: (ha-488000-m02) DBG | 2024/10/09 13:09:38 INFO : hyperkit: stderr: rdmsr to register 0x140 on vcpu 1
I1009 13:09:44.424189 10047 main.go:141] libmachine: (ha-488000-m02) DBG | 2024/10/09 13:09:44 INFO : hyperkit: stderr: rdmsr to register 0x64d on vcpu 1
I1009 13:09:44.424263 10047 main.go:141] libmachine: (ha-488000-m02) DBG | 2024/10/09 13:09:44 INFO : hyperkit: stderr: rdmsr to register 0x64e on vcpu 1
I1009 13:09:44.424275 10047 main.go:141] libmachine: (ha-488000-m02) DBG | 2024/10/09 13:09:44 INFO : hyperkit: stderr: rdmsr to register 0x34 on vcpu 1
I1009 13:09:44.448309 10047 main.go:141] libmachine: (ha-488000-m02) DBG | 2024/10/09 13:09:44 INFO : hyperkit: stderr: rdmsr to register 0xc0011029 on vcpu 1
I1009 13:10:13.250877 10047 main.go:141] libmachine: SSH cmd err, output: <nil>: minikube
I1009 13:10:13.250894 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetMachineName
I1009 13:10:13.251045 10047 buildroot.go:166] provisioning hostname "ha-488000-m02"
I1009 13:10:13.251055 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetMachineName
I1009 13:10:13.251173 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHHostname
I1009 13:10:13.251353 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHPort
I1009 13:10:13.251455 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHKeyPath
I1009 13:10:13.251562 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHKeyPath
I1009 13:10:13.251656 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHUsername
I1009 13:10:13.251811 10047 main.go:141] libmachine: Using SSH client type: native
I1009 13:10:13.251953 10047 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xf225520] 0xf228200 <nil> [] 0s} 192.169.0.6 22 <nil> <nil>}
I1009 13:10:13.251963 10047 main.go:141] libmachine: About to run SSH command:
sudo hostname ha-488000-m02 && echo "ha-488000-m02" | sudo tee /etc/hostname
I1009 13:10:13.320140 10047 main.go:141] libmachine: SSH cmd err, output: <nil>: ha-488000-m02
I1009 13:10:13.320155 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHHostname
I1009 13:10:13.320282 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHPort
I1009 13:10:13.320385 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHKeyPath
I1009 13:10:13.320476 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHKeyPath
I1009 13:10:13.320579 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHUsername
I1009 13:10:13.320733 10047 main.go:141] libmachine: Using SSH client type: native
I1009 13:10:13.320878 10047 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xf225520] 0xf228200 <nil> [] 0s} 192.169.0.6 22 <nil> <nil>}
I1009 13:10:13.320890 10047 main.go:141] libmachine: About to run SSH command:
if ! grep -xq '.*\sha-488000-m02' /etc/hosts; then
if grep -xq '127.0.1.1\s.*' /etc/hosts; then
sudo sed -i 's/^127.0.1.1\s.*/127.0.1.1 ha-488000-m02/g' /etc/hosts;
else
echo '127.0.1.1 ha-488000-m02' | sudo tee -a /etc/hosts;
fi
fi
I1009 13:10:13.388315 10047 main.go:141] libmachine: SSH cmd err, output: <nil>:
I1009 13:10:13.388330 10047 buildroot.go:172] set auth options {CertDir:/Users/jenkins/minikube-integration/19780-6775/.minikube CaCertPath:/Users/jenkins/minikube-integration/19780-6775/.minikube/certs/ca.pem CaPrivateKeyPath:/Users/jenkins/minikube-integration/19780-6775/.minikube/certs/ca-key.pem CaCertRemotePath:/etc/docker/ca.pem ServerCertPath:/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/server.pem ServerKeyPath:/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/server-key.pem ClientKeyPath:/Users/jenkins/minikube-integration/19780-6775/.minikube/certs/key.pem ServerCertRemotePath:/etc/docker/server.pem ServerKeyRemotePath:/etc/docker/server-key.pem ClientCertPath:/Users/jenkins/minikube-integration/19780-6775/.minikube/certs/cert.pem ServerCertSANs:[] StorePath:/Users/jenkins/minikube-integration/19780-6775/.minikube}
I1009 13:10:13.388339 10047 buildroot.go:174] setting up certificates
I1009 13:10:13.388345 10047 provision.go:84] configureAuth start
I1009 13:10:13.388351 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetMachineName
I1009 13:10:13.388485 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetIP
I1009 13:10:13.388591 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHHostname
I1009 13:10:13.388669 10047 provision.go:143] copyHostCerts
I1009 13:10:13.388695 10047 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19780-6775/.minikube/certs/ca.pem -> /Users/jenkins/minikube-integration/19780-6775/.minikube/ca.pem
I1009 13:10:13.388745 10047 exec_runner.go:144] found /Users/jenkins/minikube-integration/19780-6775/.minikube/ca.pem, removing ...
I1009 13:10:13.388756 10047 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19780-6775/.minikube/ca.pem
I1009 13:10:13.388888 10047 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19780-6775/.minikube/certs/ca.pem --> /Users/jenkins/minikube-integration/19780-6775/.minikube/ca.pem (1082 bytes)
I1009 13:10:13.389105 10047 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19780-6775/.minikube/certs/cert.pem -> /Users/jenkins/minikube-integration/19780-6775/.minikube/cert.pem
I1009 13:10:13.389135 10047 exec_runner.go:144] found /Users/jenkins/minikube-integration/19780-6775/.minikube/cert.pem, removing ...
I1009 13:10:13.389140 10047 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19780-6775/.minikube/cert.pem
I1009 13:10:13.389210 10047 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19780-6775/.minikube/certs/cert.pem --> /Users/jenkins/minikube-integration/19780-6775/.minikube/cert.pem (1123 bytes)
I1009 13:10:13.389380 10047 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19780-6775/.minikube/certs/key.pem -> /Users/jenkins/minikube-integration/19780-6775/.minikube/key.pem
I1009 13:10:13.389408 10047 exec_runner.go:144] found /Users/jenkins/minikube-integration/19780-6775/.minikube/key.pem, removing ...
I1009 13:10:13.389412 10047 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19780-6775/.minikube/key.pem
I1009 13:10:13.389476 10047 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19780-6775/.minikube/certs/key.pem --> /Users/jenkins/minikube-integration/19780-6775/.minikube/key.pem (1675 bytes)
I1009 13:10:13.389632 10047 provision.go:117] generating server cert: /Users/jenkins/minikube-integration/19780-6775/.minikube/machines/server.pem ca-key=/Users/jenkins/minikube-integration/19780-6775/.minikube/certs/ca.pem private-key=/Users/jenkins/minikube-integration/19780-6775/.minikube/certs/ca-key.pem org=jenkins.ha-488000-m02 san=[127.0.0.1 192.169.0.6 ha-488000-m02 localhost minikube]
I1009 13:10:13.439036 10047 provision.go:177] copyRemoteCerts
I1009 13:10:13.439096 10047 ssh_runner.go:195] Run: sudo mkdir -p /etc/docker /etc/docker /etc/docker
I1009 13:10:13.439110 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHHostname
I1009 13:10:13.439261 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHPort
I1009 13:10:13.439362 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHKeyPath
I1009 13:10:13.439460 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHUsername
I1009 13:10:13.439554 10047 sshutil.go:53] new ssh client: &{IP:192.169.0.6 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02/id_rsa Username:docker}
I1009 13:10:13.475839 10047 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19780-6775/.minikube/certs/ca.pem -> /etc/docker/ca.pem
I1009 13:10:13.475906 10047 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19780-6775/.minikube/certs/ca.pem --> /etc/docker/ca.pem (1082 bytes)
I1009 13:10:13.495317 10047 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19780-6775/.minikube/machines/server.pem -> /etc/docker/server.pem
I1009 13:10:13.495386 10047 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19780-6775/.minikube/machines/server.pem --> /etc/docker/server.pem (1208 bytes)
I1009 13:10:13.514482 10047 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19780-6775/.minikube/machines/server-key.pem -> /etc/docker/server-key.pem
I1009 13:10:13.514552 10047 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19780-6775/.minikube/machines/server-key.pem --> /etc/docker/server-key.pem (1679 bytes)
I1009 13:10:13.533687 10047 provision.go:87] duration metric: took 145.338042ms to configureAuth
I1009 13:10:13.533699 10047 buildroot.go:189] setting minikube options for container-runtime
I1009 13:10:13.533861 10047 config.go:182] Loaded profile config "ha-488000": Driver=hyperkit, ContainerRuntime=docker, KubernetesVersion=v1.31.1
I1009 13:10:13.533875 10047 main.go:141] libmachine: (ha-488000-m02) Calling .DriverName
I1009 13:10:13.534005 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHHostname
I1009 13:10:13.534086 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHPort
I1009 13:10:13.534166 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHKeyPath
I1009 13:10:13.534256 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHKeyPath
I1009 13:10:13.534337 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHUsername
I1009 13:10:13.534456 10047 main.go:141] libmachine: Using SSH client type: native
I1009 13:10:13.534588 10047 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xf225520] 0xf228200 <nil> [] 0s} 192.169.0.6 22 <nil> <nil>}
I1009 13:10:13.534595 10047 main.go:141] libmachine: About to run SSH command:
df --output=fstype / | tail -n 1
I1009 13:10:13.594094 10047 main.go:141] libmachine: SSH cmd err, output: <nil>: tmpfs
I1009 13:10:13.594110 10047 buildroot.go:70] root file system type: tmpfs
I1009 13:10:13.594214 10047 provision.go:314] Updating docker unit: /lib/systemd/system/docker.service ...
I1009 13:10:13.594226 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHHostname
I1009 13:10:13.594372 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHPort
I1009 13:10:13.594470 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHKeyPath
I1009 13:10:13.594568 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHKeyPath
I1009 13:10:13.594647 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHUsername
I1009 13:10:13.594802 10047 main.go:141] libmachine: Using SSH client type: native
I1009 13:10:13.594934 10047 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xf225520] 0xf228200 <nil> [] 0s} 192.169.0.6 22 <nil> <nil>}
I1009 13:10:13.594981 10047 main.go:141] libmachine: About to run SSH command:
sudo mkdir -p /lib/systemd/system && printf %s "[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network.target minikube-automount.service docker.socket
Requires= minikube-automount.service docker.socket
StartLimitBurst=3
StartLimitIntervalSec=60
[Service]
Type=notify
Restart=on-failure
Environment="NO_PROXY=192.169.0.5"
# This file is a systemd drop-in unit that inherits from the base dockerd configuration.
# The base configuration already specifies an 'ExecStart=...' command. The first directive
# here is to clear out that command inherited from the base configuration. Without this,
# the command from the base configuration and the command specified here are treated as
# a sequence of commands, which is not the desired behavior, nor is it valid -- systemd
# will catch this invalid input and refuse to start the service with an error like:
# Service has more than one ExecStart= setting, which is only allowed for Type=oneshot services.
# NOTE: default-ulimit=nofile is set to an arbitrary number for consistency with other
# container runtimes. If left unlimited, it may result in OOM issues with MySQL.
ExecStart=
ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --default-ulimit=nofile=1048576:1048576 --tlsverify --tlscacert /etc/docker/ca.pem --tlscert /etc/docker/server.pem --tlskey /etc/docker/server-key.pem --label provider=hyperkit --insecure-registry 10.96.0.0/12
ExecReload=/bin/kill -s HUP \$MAINPID
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
TimeoutStartSec=0
# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes
# kill only the docker process, not all processes in the cgroup
KillMode=process
[Install]
WantedBy=multi-user.target
" | sudo tee /lib/systemd/system/docker.service.new
I1009 13:10:13.665260 10047 main.go:141] libmachine: SSH cmd err, output: <nil>: [Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network.target minikube-automount.service docker.socket
Requires= minikube-automount.service docker.socket
StartLimitBurst=3
StartLimitIntervalSec=60
[Service]
Type=notify
Restart=on-failure
Environment=NO_PROXY=192.169.0.5
# This file is a systemd drop-in unit that inherits from the base dockerd configuration.
# The base configuration already specifies an 'ExecStart=...' command. The first directive
# here is to clear out that command inherited from the base configuration. Without this,
# the command from the base configuration and the command specified here are treated as
# a sequence of commands, which is not the desired behavior, nor is it valid -- systemd
# will catch this invalid input and refuse to start the service with an error like:
# Service has more than one ExecStart= setting, which is only allowed for Type=oneshot services.
# NOTE: default-ulimit=nofile is set to an arbitrary number for consistency with other
# container runtimes. If left unlimited, it may result in OOM issues with MySQL.
ExecStart=
ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --default-ulimit=nofile=1048576:1048576 --tlsverify --tlscacert /etc/docker/ca.pem --tlscert /etc/docker/server.pem --tlskey /etc/docker/server-key.pem --label provider=hyperkit --insecure-registry 10.96.0.0/12
ExecReload=/bin/kill -s HUP $MAINPID
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
TimeoutStartSec=0
# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes
# kill only the docker process, not all processes in the cgroup
KillMode=process
[Install]
WantedBy=multi-user.target
I1009 13:10:13.665277 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHHostname
I1009 13:10:13.665418 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHPort
I1009 13:10:13.665518 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHKeyPath
I1009 13:10:13.665611 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHKeyPath
I1009 13:10:13.665691 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHUsername
I1009 13:10:13.665841 10047 main.go:141] libmachine: Using SSH client type: native
I1009 13:10:13.665979 10047 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xf225520] 0xf228200 <nil> [] 0s} 192.169.0.6 22 <nil> <nil>}
I1009 13:10:13.665992 10047 main.go:141] libmachine: About to run SSH command:
sudo diff -u /lib/systemd/system/docker.service /lib/systemd/system/docker.service.new || { sudo mv /lib/systemd/system/docker.service.new /lib/systemd/system/docker.service; sudo systemctl -f daemon-reload && sudo systemctl -f enable docker && sudo systemctl -f restart docker; }
I1009 13:10:15.335048 10047 main.go:141] libmachine: SSH cmd err, output: <nil>: diff: can't stat '/lib/systemd/system/docker.service': No such file or directory
Created symlink /etc/systemd/system/multi-user.target.wants/docker.service → /usr/lib/systemd/system/docker.service.
I1009 13:10:15.335075 10047 machine.go:96] duration metric: took 37.150170616s to provisionDockerMachine
I1009 13:10:15.335082 10047 start.go:293] postStartSetup for "ha-488000-m02" (driver="hyperkit")
I1009 13:10:15.335096 10047 start.go:322] creating required directories: [/etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs]
I1009 13:10:15.335106 10047 main.go:141] libmachine: (ha-488000-m02) Calling .DriverName
I1009 13:10:15.335324 10047 ssh_runner.go:195] Run: sudo mkdir -p /etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs
I1009 13:10:15.335337 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHHostname
I1009 13:10:15.335446 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHPort
I1009 13:10:15.335554 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHKeyPath
I1009 13:10:15.335650 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHUsername
I1009 13:10:15.335758 10047 sshutil.go:53] new ssh client: &{IP:192.169.0.6 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02/id_rsa Username:docker}
I1009 13:10:15.371443 10047 ssh_runner.go:195] Run: cat /etc/os-release
I1009 13:10:15.374577 10047 info.go:137] Remote host: Buildroot 2023.02.9
I1009 13:10:15.374586 10047 filesync.go:126] Scanning /Users/jenkins/minikube-integration/19780-6775/.minikube/addons for local assets ...
I1009 13:10:15.374664 10047 filesync.go:126] Scanning /Users/jenkins/minikube-integration/19780-6775/.minikube/files for local assets ...
I1009 13:10:15.374800 10047 filesync.go:149] local asset: /Users/jenkins/minikube-integration/19780-6775/.minikube/files/etc/ssl/certs/73182.pem -> 73182.pem in /etc/ssl/certs
I1009 13:10:15.374806 10047 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19780-6775/.minikube/files/etc/ssl/certs/73182.pem -> /etc/ssl/certs/73182.pem
I1009 13:10:15.374969 10047 ssh_runner.go:195] Run: sudo mkdir -p /etc/ssl/certs
I1009 13:10:15.382233 10047 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19780-6775/.minikube/files/etc/ssl/certs/73182.pem --> /etc/ssl/certs/73182.pem (1708 bytes)
I1009 13:10:15.402021 10047 start.go:296] duration metric: took 66.924904ms for postStartSetup
I1009 13:10:15.402042 10047 main.go:141] libmachine: (ha-488000-m02) Calling .DriverName
I1009 13:10:15.402244 10047 ssh_runner.go:195] Run: sudo ls --almost-all -1 /var/lib/minikube/backup
I1009 13:10:15.402258 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHHostname
I1009 13:10:15.402371 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHPort
I1009 13:10:15.402470 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHKeyPath
I1009 13:10:15.402557 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHUsername
I1009 13:10:15.402638 10047 sshutil.go:53] new ssh client: &{IP:192.169.0.6 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02/id_rsa Username:docker}
I1009 13:10:15.438474 10047 machine.go:197] restoring vm config from /var/lib/minikube/backup: [etc]
I1009 13:10:15.438545 10047 ssh_runner.go:195] Run: sudo rsync --archive --update /var/lib/minikube/backup/etc /
I1009 13:10:15.491531 10047 fix.go:56] duration metric: took 37.419550792s for fixHost
I1009 13:10:15.491556 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHHostname
I1009 13:10:15.491714 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHPort
I1009 13:10:15.491818 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHKeyPath
I1009 13:10:15.491900 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHKeyPath
I1009 13:10:15.491994 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHUsername
I1009 13:10:15.492140 10047 main.go:141] libmachine: Using SSH client type: native
I1009 13:10:15.492269 10047 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xf225520] 0xf228200 <nil> [] 0s} 192.169.0.6 22 <nil> <nil>}
I1009 13:10:15.492276 10047 main.go:141] libmachine: About to run SSH command:
date +%s.%N
I1009 13:10:15.551788 10047 main.go:141] libmachine: SSH cmd err, output: <nil>: 1728504615.736945552
I1009 13:10:15.551800 10047 fix.go:216] guest clock: 1728504615.736945552
I1009 13:10:15.551805 10047 fix.go:229] Guest: 2024-10-09 13:10:15.736945552 -0700 PDT Remote: 2024-10-09 13:10:15.491546 -0700 PDT m=+56.765207871 (delta=245.399552ms)
I1009 13:10:15.551815 10047 fix.go:200] guest clock delta is within tolerance: 245.399552ms
I1009 13:10:15.551823 10047 start.go:83] releasing machines lock for "ha-488000-m02", held for 37.479862859s
I1009 13:10:15.551839 10047 main.go:141] libmachine: (ha-488000-m02) Calling .DriverName
I1009 13:10:15.551970 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetIP
I1009 13:10:15.576574 10047 out.go:177] * Found network options:
I1009 13:10:15.597202 10047 out.go:177] - NO_PROXY=192.169.0.5
W1009 13:10:15.618594 10047 proxy.go:119] fail to check proxy env: Error ip not in block
I1009 13:10:15.618633 10047 main.go:141] libmachine: (ha-488000-m02) Calling .DriverName
I1009 13:10:15.619501 10047 main.go:141] libmachine: (ha-488000-m02) Calling .DriverName
I1009 13:10:15.619754 10047 main.go:141] libmachine: (ha-488000-m02) Calling .DriverName
I1009 13:10:15.619912 10047 ssh_runner.go:195] Run: curl -sS -m 2 https://registry.k8s.io/
I1009 13:10:15.619966 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHHostname
W1009 13:10:15.620015 10047 proxy.go:119] fail to check proxy env: Error ip not in block
I1009 13:10:15.620140 10047 ssh_runner.go:195] Run: sh -c "stat /etc/cni/net.d/*loopback.conf*"
I1009 13:10:15.620159 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHHostname
I1009 13:10:15.620181 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHPort
I1009 13:10:15.620355 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHKeyPath
I1009 13:10:15.620424 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHPort
I1009 13:10:15.620529 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHUsername
I1009 13:10:15.620568 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHKeyPath
I1009 13:10:15.620681 10047 main.go:141] libmachine: (ha-488000-m02) Calling .GetSSHUsername
I1009 13:10:15.620713 10047 sshutil.go:53] new ssh client: &{IP:192.169.0.6 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02/id_rsa Username:docker}
I1009 13:10:15.620808 10047 sshutil.go:53] new ssh client: &{IP:192.169.0.6 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19780-6775/.minikube/machines/ha-488000-m02/id_rsa Username:docker}
W1009 13:10:15.656116 10047 cni.go:209] loopback cni configuration skipped: "/etc/cni/net.d/*loopback.conf*" not found
I1009 13:10:15.656191 10047 ssh_runner.go:195] Run: sudo find /etc/cni/net.d -maxdepth 1 -type f ( ( -name *bridge* -or -name *podman* ) -and -not -name *.mk_disabled ) -printf "%p, " -exec sh -c "sudo mv {} {}.mk_disabled" ;
I1009 13:10:15.705550 10047 cni.go:262] disabled [/etc/cni/net.d/87-podman-bridge.conflist] bridge cni config(s)
I1009 13:10:15.705574 10047 start.go:495] detecting cgroup driver to use...
I1009 13:10:15.705674 10047 ssh_runner.go:195] Run: /bin/bash -c "sudo mkdir -p /etc && printf %s "runtime-endpoint: unix:///run/containerd/containerd.sock
" | sudo tee /etc/crictl.yaml"
I1009 13:10:15.721749 10047 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)sandbox_image = .*$|\1sandbox_image = "registry.k8s.io/pause:3.10"|' /etc/containerd/config.toml"
I1009 13:10:15.730768 10047 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)restrict_oom_score_adj = .*$|\1restrict_oom_score_adj = false|' /etc/containerd/config.toml"
I1009 13:10:15.739682 10047 containerd.go:146] configuring containerd to use "cgroupfs" as cgroup driver...
I1009 13:10:15.739741 10047 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)SystemdCgroup = .*$|\1SystemdCgroup = false|g' /etc/containerd/config.toml"
I1009 13:10:15.748701 10047 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|"io.containerd.runtime.v1.linux"|"io.containerd.runc.v2"|g' /etc/containerd/config.toml"
I1009 13:10:15.757525 10047 ssh_runner.go:195] Run: sh -c "sudo sed -i '/systemd_cgroup/d' /etc/containerd/config.toml"
I1009 13:10:15.766268 10047 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|"io.containerd.runc.v1"|"io.containerd.runc.v2"|g' /etc/containerd/config.toml"
I1009 13:10:15.775114 10047 ssh_runner.go:195] Run: sh -c "sudo rm -rf /etc/cni/net.mk"
I1009 13:10:15.784378 10047 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)conf_dir = .*$|\1conf_dir = "/etc/cni/net.d"|g' /etc/containerd/config.toml"
I1009 13:10:15.793555 10047 ssh_runner.go:195] Run: sh -c "sudo sed -i '/^ *enable_unprivileged_ports = .*/d' /etc/containerd/config.toml"
I1009 13:10:15.802407 10047 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)\[plugins."io.containerd.grpc.v1.cri"\]|&\n\1 enable_unprivileged_ports = true|' /etc/containerd/config.toml"
I1009 13:10:15.811290 10047 ssh_runner.go:195] Run: sudo sysctl net.bridge.bridge-nf-call-iptables
I1009 13:10:15.819212 10047 crio.go:166] couldn't verify netfilter by "sudo sysctl net.bridge.bridge-nf-call-iptables" which might be okay. error: sudo sysctl net.bridge.bridge-nf-call-iptables: Process exited with status 255
stdout:
stderr:
sysctl: cannot stat /proc/sys/net/bridge/bridge-nf-call-iptables: No such file or directory
I1009 13:10:15.819270 10047 ssh_runner.go:195] Run: sudo modprobe br_netfilter
I1009 13:10:15.828283 10047 ssh_runner.go:195] Run: sudo sh -c "echo 1 > /proc/sys/net/ipv4/ip_forward"
I1009 13:10:15.836495 10047 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I1009 13:10:15.932846 10047 ssh_runner.go:195] Run: sudo systemctl restart containerd
I1009 13:10:15.952775 10047 start.go:495] detecting cgroup driver to use...
I1009 13:10:15.952853 10047 ssh_runner.go:195] Run: sudo systemctl cat docker.service
I1009 13:10:15.967942 10047 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service containerd
I1009 13:10:15.983217 10047 ssh_runner.go:195] Run: sudo systemctl stop -f containerd
I1009 13:10:16.004294 10047 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service containerd
I1009 13:10:16.015793 10047 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service crio
I1009 13:10:16.025951 10047 ssh_runner.go:195] Run: sudo systemctl stop -f crio
I1009 13:10:16.048721 10047 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service crio
I1009 13:10:16.059134 10047 ssh_runner.go:195] Run: /bin/bash -c "sudo mkdir -p /etc && printf %s "runtime-endpoint: unix:///var/run/cri-dockerd.sock
" | sudo tee /etc/crictl.yaml"
I1009 13:10:16.074105 10047 ssh_runner.go:195] Run: which cri-dockerd
I1009 13:10:16.077076 10047 ssh_runner.go:195] Run: sudo mkdir -p /etc/systemd/system/cri-docker.service.d
I1009 13:10:16.084473 10047 ssh_runner.go:362] scp memory --> /etc/systemd/system/cri-docker.service.d/10-cni.conf (190 bytes)
I1009 13:10:16.098203 10047 ssh_runner.go:195] Run: sudo systemctl unmask docker.service
I1009 13:10:16.191140 10047 ssh_runner.go:195] Run: sudo systemctl enable docker.socket
I1009 13:10:16.303139 10047 docker.go:574] configuring docker to use "cgroupfs" as cgroup driver...
I1009 13:10:16.303167 10047 ssh_runner.go:362] scp memory --> /etc/docker/daemon.json (130 bytes)
I1009 13:10:16.317229 10047 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I1009 13:10:16.419571 10047 ssh_runner.go:195] Run: sudo systemctl restart docker
I1009 13:11:17.566677 10047 ssh_runner.go:235] Completed: sudo systemctl restart docker: (1m1.02320066s)
I1009 13:11:17.566758 10047 ssh_runner.go:195] Run: sudo journalctl --no-pager -u docker
I1009 13:11:17.603432 10047 out.go:201]
W1009 13:11:17.624525 10047 out.go:270] X Exiting due to RUNTIME_ENABLE: Failed to enable container runtime: sudo systemctl restart docker: Process exited with status 1
stdout:
stderr:
Job for docker.service failed because the control process exited with error code.
See "systemctl status docker.service" and "journalctl -xeu docker.service" for details.
sudo journalctl --no-pager -u docker:
-- stdout --
Oct 09 20:10:13 ha-488000-m02 systemd[1]: Starting Docker Application Container Engine...
Oct 09 20:10:13 ha-488000-m02 dockerd[506]: time="2024-10-09T20:10:13.356746736Z" level=info msg="Starting up"
Oct 09 20:10:13 ha-488000-m02 dockerd[506]: time="2024-10-09T20:10:13.357427132Z" level=info msg="containerd not running, starting managed containerd"
Oct 09 20:10:13 ha-488000-m02 dockerd[506]: time="2024-10-09T20:10:13.357979007Z" level=info msg="started new containerd process" address=/var/run/docker/containerd/containerd.sock module=libcontainerd pid=513
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.373447274Z" level=info msg="starting containerd" revision=7f7fdf5fed64eb6a7caf99b3e12efcf9d60e311c version=v1.7.22
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.388646250Z" level=info msg="loading plugin \"io.containerd.event.v1.exchange\"..." type=io.containerd.event.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.388741695Z" level=info msg="loading plugin \"io.containerd.internal.v1.opt\"..." type=io.containerd.internal.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.388813351Z" level=info msg="loading plugin \"io.containerd.warning.v1.deprecations\"..." type=io.containerd.warning.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.388849092Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.blockfile\"..." type=io.containerd.snapshotter.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.389023538Z" level=info msg="skip loading plugin \"io.containerd.snapshotter.v1.blockfile\"..." error="no scratch file generator: skip plugin" type=io.containerd.snapshotter.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.389083673Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.btrfs\"..." type=io.containerd.snapshotter.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.389227258Z" level=info msg="skip loading plugin \"io.containerd.snapshotter.v1.btrfs\"..." error="path /var/lib/docker/containerd/daemon/io.containerd.snapshotter.v1.btrfs (ext4) must be a btrfs filesystem to be used with the btrfs snapshotter: skip plugin" type=io.containerd.snapshotter.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.389270781Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.devmapper\"..." type=io.containerd.snapshotter.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.389302939Z" level=info msg="skip loading plugin \"io.containerd.snapshotter.v1.devmapper\"..." error="devmapper not configured: skip plugin" type=io.containerd.snapshotter.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.389334622Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.native\"..." type=io.containerd.snapshotter.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.389432191Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.overlayfs\"..." type=io.containerd.snapshotter.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.389591824Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.aufs\"..." type=io.containerd.snapshotter.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.391579466Z" level=info msg="skip loading plugin \"io.containerd.snapshotter.v1.aufs\"..." error="aufs is not supported (modprobe aufs failed: exit status 1 \"modprobe: FATAL: Module aufs not found in directory /lib/modules/5.10.207\\n\"): skip plugin" type=io.containerd.snapshotter.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.391629484Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.zfs\"..." type=io.containerd.snapshotter.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.391760548Z" level=info msg="skip loading plugin \"io.containerd.snapshotter.v1.zfs\"..." error="path /var/lib/docker/containerd/daemon/io.containerd.snapshotter.v1.zfs must be a zfs filesystem to be used with the zfs snapshotter: skip plugin" type=io.containerd.snapshotter.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.391803524Z" level=info msg="loading plugin \"io.containerd.content.v1.content\"..." type=io.containerd.content.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.391904291Z" level=info msg="loading plugin \"io.containerd.metadata.v1.bolt\"..." type=io.containerd.metadata.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.391986458Z" level=info msg="metadata content store policy set" policy=shared
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.394061479Z" level=info msg="loading plugin \"io.containerd.gc.v1.scheduler\"..." type=io.containerd.gc.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.394169732Z" level=info msg="loading plugin \"io.containerd.differ.v1.walking\"..." type=io.containerd.differ.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.394218728Z" level=info msg="loading plugin \"io.containerd.lease.v1.manager\"..." type=io.containerd.lease.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.394314013Z" level=info msg="loading plugin \"io.containerd.streaming.v1.manager\"..." type=io.containerd.streaming.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.394362049Z" level=info msg="loading plugin \"io.containerd.runtime.v1.linux\"..." type=io.containerd.runtime.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.394430562Z" level=info msg="loading plugin \"io.containerd.monitor.v1.cgroups\"..." type=io.containerd.monitor.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.394652532Z" level=info msg="loading plugin \"io.containerd.runtime.v2.task\"..." type=io.containerd.runtime.v2
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.394762641Z" level=info msg="loading plugin \"io.containerd.runtime.v2.shim\"..." type=io.containerd.runtime.v2
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.394800064Z" level=info msg="loading plugin \"io.containerd.sandbox.store.v1.local\"..." type=io.containerd.sandbox.store.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.394830971Z" level=info msg="loading plugin \"io.containerd.sandbox.controller.v1.local\"..." type=io.containerd.sandbox.controller.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.394861458Z" level=info msg="loading plugin \"io.containerd.service.v1.containers-service\"..." type=io.containerd.service.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.394892471Z" level=info msg="loading plugin \"io.containerd.service.v1.content-service\"..." type=io.containerd.service.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.394974260Z" level=info msg="loading plugin \"io.containerd.service.v1.diff-service\"..." type=io.containerd.service.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.395016600Z" level=info msg="loading plugin \"io.containerd.service.v1.images-service\"..." type=io.containerd.service.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.395048465Z" level=info msg="loading plugin \"io.containerd.service.v1.introspection-service\"..." type=io.containerd.service.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.395078685Z" level=info msg="loading plugin \"io.containerd.service.v1.namespaces-service\"..." type=io.containerd.service.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.395107707Z" level=info msg="loading plugin \"io.containerd.service.v1.snapshots-service\"..." type=io.containerd.service.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.395139117Z" level=info msg="loading plugin \"io.containerd.service.v1.tasks-service\"..." type=io.containerd.service.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.395176031Z" level=info msg="loading plugin \"io.containerd.grpc.v1.containers\"..." type=io.containerd.grpc.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.395211902Z" level=info msg="loading plugin \"io.containerd.grpc.v1.content\"..." type=io.containerd.grpc.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.395242081Z" level=info msg="loading plugin \"io.containerd.grpc.v1.diff\"..." type=io.containerd.grpc.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.395275351Z" level=info msg="loading plugin \"io.containerd.grpc.v1.events\"..." type=io.containerd.grpc.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.395308790Z" level=info msg="loading plugin \"io.containerd.grpc.v1.images\"..." type=io.containerd.grpc.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.395341976Z" level=info msg="loading plugin \"io.containerd.grpc.v1.introspection\"..." type=io.containerd.grpc.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.395372003Z" level=info msg="loading plugin \"io.containerd.grpc.v1.leases\"..." type=io.containerd.grpc.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.395401615Z" level=info msg="loading plugin \"io.containerd.grpc.v1.namespaces\"..." type=io.containerd.grpc.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.395431176Z" level=info msg="loading plugin \"io.containerd.grpc.v1.sandbox-controllers\"..." type=io.containerd.grpc.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.395461321Z" level=info msg="loading plugin \"io.containerd.grpc.v1.sandboxes\"..." type=io.containerd.grpc.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.395493133Z" level=info msg="loading plugin \"io.containerd.grpc.v1.snapshots\"..." type=io.containerd.grpc.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.395524326Z" level=info msg="loading plugin \"io.containerd.grpc.v1.streaming\"..." type=io.containerd.grpc.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.395554155Z" level=info msg="loading plugin \"io.containerd.grpc.v1.tasks\"..." type=io.containerd.grpc.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.395585737Z" level=info msg="loading plugin \"io.containerd.transfer.v1.local\"..." type=io.containerd.transfer.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.395621809Z" level=info msg="loading plugin \"io.containerd.grpc.v1.transfer\"..." type=io.containerd.grpc.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.395652366Z" level=info msg="loading plugin \"io.containerd.grpc.v1.version\"..." type=io.containerd.grpc.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.395683996Z" level=info msg="loading plugin \"io.containerd.internal.v1.restart\"..." type=io.containerd.internal.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.395750209Z" level=info msg="loading plugin \"io.containerd.tracing.processor.v1.otlp\"..." type=io.containerd.tracing.processor.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.395795296Z" level=info msg="skip loading plugin \"io.containerd.tracing.processor.v1.otlp\"..." error="skip plugin: tracing endpoint not configured" type=io.containerd.tracing.processor.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.395976946Z" level=info msg="loading plugin \"io.containerd.internal.v1.tracing\"..." type=io.containerd.internal.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.396014920Z" level=info msg="skip loading plugin \"io.containerd.internal.v1.tracing\"..." error="skip plugin: tracing endpoint not configured" type=io.containerd.internal.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.396025742Z" level=info msg="loading plugin \"io.containerd.grpc.v1.healthcheck\"..." type=io.containerd.grpc.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.396035016Z" level=info msg="loading plugin \"io.containerd.nri.v1.nri\"..." type=io.containerd.nri.v1
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.396045124Z" level=info msg="NRI interface is disabled by configuration."
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.396159569Z" level=info msg=serving... address=/var/run/docker/containerd/containerd-debug.sock
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.396229447Z" level=info msg=serving... address=/var/run/docker/containerd/containerd.sock.ttrpc
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.396355539Z" level=info msg=serving... address=/var/run/docker/containerd/containerd.sock
Oct 09 20:10:13 ha-488000-m02 dockerd[513]: time="2024-10-09T20:10:13.396391066Z" level=info msg="containerd successfully booted in 0.023771s"
Oct 09 20:10:14 ha-488000-m02 dockerd[506]: time="2024-10-09T20:10:14.382785560Z" level=info msg="[graphdriver] trying configured driver: overlay2"
Oct 09 20:10:14 ha-488000-m02 dockerd[506]: time="2024-10-09T20:10:14.415839817Z" level=info msg="Loading containers: start."
Oct 09 20:10:14 ha-488000-m02 dockerd[506]: time="2024-10-09T20:10:14.586933172Z" level=warning msg="ip6tables is enabled, but cannot set up ip6tables chains" error="failed to create NAT chain DOCKER: iptables failed: ip6tables --wait -t nat -N DOCKER: ip6tables v1.8.9 (legacy): can't initialize ip6tables table `nat': Table does not exist (do you need to insmod?)\nPerhaps ip6tables or your kernel needs to be upgraded.\n (exit status 3)"
Oct 09 20:10:15 ha-488000-m02 dockerd[506]: time="2024-10-09T20:10:15.439119356Z" level=info msg="Default bridge (docker0) is assigned with an IP address 172.17.0.0/16. Daemon option --bip can be used to set a preferred IP address"
Oct 09 20:10:15 ha-488000-m02 dockerd[506]: time="2024-10-09T20:10:15.484657854Z" level=info msg="Loading containers: done."
Oct 09 20:10:15 ha-488000-m02 dockerd[506]: time="2024-10-09T20:10:15.497584051Z" level=warning msg="WARNING: bridge-nf-call-iptables is disabled"
Oct 09 20:10:15 ha-488000-m02 dockerd[506]: time="2024-10-09T20:10:15.497671888Z" level=warning msg="WARNING: bridge-nf-call-ip6tables is disabled"
Oct 09 20:10:15 ha-488000-m02 dockerd[506]: time="2024-10-09T20:10:15.497749571Z" level=info msg="Docker daemon" commit=41ca978 containerd-snapshotter=false storage-driver=overlay2 version=27.3.1
Oct 09 20:10:15 ha-488000-m02 dockerd[506]: time="2024-10-09T20:10:15.498015136Z" level=info msg="Daemon has completed initialization"
Oct 09 20:10:15 ha-488000-m02 dockerd[506]: time="2024-10-09T20:10:15.518976953Z" level=info msg="API listen on /var/run/docker.sock"
Oct 09 20:10:15 ha-488000-m02 dockerd[506]: time="2024-10-09T20:10:15.519137360Z" level=info msg="API listen on [::]:2376"
Oct 09 20:10:15 ha-488000-m02 systemd[1]: Started Docker Application Container Engine.
Oct 09 20:10:16 ha-488000-m02 systemd[1]: Stopping Docker Application Container Engine...
Oct 09 20:10:16 ha-488000-m02 dockerd[506]: time="2024-10-09T20:10:16.623435159Z" level=info msg="Processing signal 'terminated'"
Oct 09 20:10:16 ha-488000-m02 dockerd[506]: time="2024-10-09T20:10:16.624265213Z" level=info msg="stopping event stream following graceful shutdown" error="<nil>" module=libcontainerd namespace=moby
Oct 09 20:10:16 ha-488000-m02 dockerd[506]: time="2024-10-09T20:10:16.624650441Z" level=info msg="Daemon shutdown complete"
Oct 09 20:10:16 ha-488000-m02 dockerd[506]: time="2024-10-09T20:10:16.624738571Z" level=info msg="stopping event stream following graceful shutdown" error="context canceled" module=libcontainerd namespace=plugins.moby
Oct 09 20:10:16 ha-488000-m02 dockerd[506]: time="2024-10-09T20:10:16.624750053Z" level=info msg="stopping healthcheck following graceful shutdown" module=libcontainerd
Oct 09 20:10:17 ha-488000-m02 systemd[1]: docker.service: Deactivated successfully.
Oct 09 20:10:17 ha-488000-m02 systemd[1]: Stopped Docker Application Container Engine.
Oct 09 20:10:17 ha-488000-m02 systemd[1]: Starting Docker Application Container Engine...
Oct 09 20:10:17 ha-488000-m02 dockerd[1109]: time="2024-10-09T20:10:17.660494115Z" level=info msg="Starting up"
Oct 09 20:11:17 ha-488000-m02 dockerd[1109]: failed to start daemon: failed to dial "/run/containerd/containerd.sock": failed to dial "/run/containerd/containerd.sock": context deadline exceeded
Oct 09 20:11:17 ha-488000-m02 systemd[1]: docker.service: Main process exited, code=exited, status=1/FAILURE
Oct 09 20:11:17 ha-488000-m02 systemd[1]: docker.service: Failed with result 'exit-code'.
Oct 09 20:11:17 ha-488000-m02 systemd[1]: Failed to start Docker Application Container Engine.
-- /stdout --
W1009 13:11:17.624580 10047 out.go:270] *
W1009 13:11:17.625308 10047 out.go:293] ╭─────────────────────────────────────────────────────────────────────────────────────────────╮
│ │
│ * If the above advice does not help, please let us know: │
│ https://github.com/kubernetes/minikube/issues/new/choose │
│ │
│ * Please run `minikube logs --file=logs.txt` and attach logs.txt to the GitHub issue. │
│ │
╰─────────────────────────────────────────────────────────────────────────────────────────────╯
I1009 13:11:17.687575 10047 out.go:201]
==> Docker <==
Oct 09 20:10:21 ha-488000 dockerd[1176]: time="2024-10-09T20:10:21.654241352Z" level=info msg="loading plugin \"io.containerd.ttrpc.v1.pause\"..." runtime=io.containerd.runc.v2 type=io.containerd.ttrpc.v1
Oct 09 20:10:30 ha-488000 dockerd[1176]: time="2024-10-09T20:10:30.655206235Z" level=info msg="loading plugin \"io.containerd.event.v1.publisher\"..." runtime=io.containerd.runc.v2 type=io.containerd.event.v1
Oct 09 20:10:30 ha-488000 dockerd[1176]: time="2024-10-09T20:10:30.655395641Z" level=info msg="loading plugin \"io.containerd.internal.v1.shutdown\"..." runtime=io.containerd.runc.v2 type=io.containerd.internal.v1
Oct 09 20:10:30 ha-488000 dockerd[1176]: time="2024-10-09T20:10:30.655424593Z" level=info msg="loading plugin \"io.containerd.ttrpc.v1.task\"..." runtime=io.containerd.runc.v2 type=io.containerd.ttrpc.v1
Oct 09 20:10:30 ha-488000 dockerd[1176]: time="2024-10-09T20:10:30.655553116Z" level=info msg="loading plugin \"io.containerd.ttrpc.v1.pause\"..." runtime=io.containerd.runc.v2 type=io.containerd.ttrpc.v1
Oct 09 20:10:42 ha-488000 dockerd[1169]: time="2024-10-09T20:10:42.364041777Z" level=info msg="ignoring event" container=f8b55a7e3128b1a465dc68d9bfabdeeb6564e4b3492c807b6148ec4010f16d50 module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Oct 09 20:10:42 ha-488000 dockerd[1176]: time="2024-10-09T20:10:42.363750038Z" level=info msg="shim disconnected" id=f8b55a7e3128b1a465dc68d9bfabdeeb6564e4b3492c807b6148ec4010f16d50 namespace=moby
Oct 09 20:10:42 ha-488000 dockerd[1176]: time="2024-10-09T20:10:42.364182586Z" level=warning msg="cleaning up after shim disconnected" id=f8b55a7e3128b1a465dc68d9bfabdeeb6564e4b3492c807b6148ec4010f16d50 namespace=moby
Oct 09 20:10:42 ha-488000 dockerd[1176]: time="2024-10-09T20:10:42.364192318Z" level=info msg="cleaning up dead shim" namespace=moby
Oct 09 20:10:51 ha-488000 dockerd[1169]: time="2024-10-09T20:10:51.714035724Z" level=info msg="ignoring event" container=b42491e1964fdcd16c9570ec02db0cc321669af0c97f2f1c510efc0f7cb702c8 module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Oct 09 20:10:51 ha-488000 dockerd[1176]: time="2024-10-09T20:10:51.714828914Z" level=info msg="shim disconnected" id=b42491e1964fdcd16c9570ec02db0cc321669af0c97f2f1c510efc0f7cb702c8 namespace=moby
Oct 09 20:10:51 ha-488000 dockerd[1176]: time="2024-10-09T20:10:51.715024991Z" level=warning msg="cleaning up after shim disconnected" id=b42491e1964fdcd16c9570ec02db0cc321669af0c97f2f1c510efc0f7cb702c8 namespace=moby
Oct 09 20:10:51 ha-488000 dockerd[1176]: time="2024-10-09T20:10:51.715062821Z" level=info msg="cleaning up dead shim" namespace=moby
Oct 09 20:11:16 ha-488000 dockerd[1176]: time="2024-10-09T20:11:16.672494154Z" level=info msg="loading plugin \"io.containerd.event.v1.publisher\"..." runtime=io.containerd.runc.v2 type=io.containerd.event.v1
Oct 09 20:11:16 ha-488000 dockerd[1176]: time="2024-10-09T20:11:16.672595411Z" level=info msg="loading plugin \"io.containerd.internal.v1.shutdown\"..." runtime=io.containerd.runc.v2 type=io.containerd.internal.v1
Oct 09 20:11:16 ha-488000 dockerd[1176]: time="2024-10-09T20:11:16.672909479Z" level=info msg="loading plugin \"io.containerd.ttrpc.v1.task\"..." runtime=io.containerd.runc.v2 type=io.containerd.ttrpc.v1
Oct 09 20:11:16 ha-488000 dockerd[1176]: time="2024-10-09T20:11:16.673193463Z" level=info msg="loading plugin \"io.containerd.ttrpc.v1.pause\"..." runtime=io.containerd.runc.v2 type=io.containerd.ttrpc.v1
Oct 09 20:11:23 ha-488000 dockerd[1176]: time="2024-10-09T20:11:23.653214607Z" level=info msg="loading plugin \"io.containerd.event.v1.publisher\"..." runtime=io.containerd.runc.v2 type=io.containerd.event.v1
Oct 09 20:11:23 ha-488000 dockerd[1176]: time="2024-10-09T20:11:23.653269931Z" level=info msg="loading plugin \"io.containerd.internal.v1.shutdown\"..." runtime=io.containerd.runc.v2 type=io.containerd.internal.v1
Oct 09 20:11:23 ha-488000 dockerd[1176]: time="2024-10-09T20:11:23.653279264Z" level=info msg="loading plugin \"io.containerd.ttrpc.v1.task\"..." runtime=io.containerd.runc.v2 type=io.containerd.ttrpc.v1
Oct 09 20:11:23 ha-488000 dockerd[1176]: time="2024-10-09T20:11:23.654176815Z" level=info msg="loading plugin \"io.containerd.ttrpc.v1.pause\"..." runtime=io.containerd.runc.v2 type=io.containerd.ttrpc.v1
Oct 09 20:11:37 ha-488000 dockerd[1169]: time="2024-10-09T20:11:37.114354136Z" level=info msg="ignoring event" container=5231dd26c41cd6b23dc0e184aed6d620565385c4dd3cde5ed2ef978d08ddbfcf module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Oct 09 20:11:37 ha-488000 dockerd[1176]: time="2024-10-09T20:11:37.114602264Z" level=info msg="shim disconnected" id=5231dd26c41cd6b23dc0e184aed6d620565385c4dd3cde5ed2ef978d08ddbfcf namespace=moby
Oct 09 20:11:37 ha-488000 dockerd[1176]: time="2024-10-09T20:11:37.114961282Z" level=warning msg="cleaning up after shim disconnected" id=5231dd26c41cd6b23dc0e184aed6d620565385c4dd3cde5ed2ef978d08ddbfcf namespace=moby
Oct 09 20:11:37 ha-488000 dockerd[1176]: time="2024-10-09T20:11:37.114994001Z" level=info msg="cleaning up dead shim" namespace=moby
==> container status <==
CONTAINER IMAGE CREATED STATE NAME ATTEMPT POD ID POD
85817da55476a 175ffd71cce3d 17 seconds ago Running kube-controller-manager 5 5ff606cee990e kube-controller-manager-ha-488000
5231dd26c41cd 6bab7719df100 24 seconds ago Exited kube-apiserver 4 164f30dc9da17 kube-apiserver-ha-488000
b42491e1964fd 175ffd71cce3d About a minute ago Exited kube-controller-manager 4 5ff606cee990e kube-controller-manager-ha-488000
f07e6a056b436 18b729c2288dc About a minute ago Running kube-vip 1 d6a5c88c0f019 kube-vip-ha-488000
3fe682afe2b2d 2e96e5913fc06 About a minute ago Running etcd 2 bc04e127497e7 etcd-ha-488000
f4310b949f12c 9aa1fad941575 About a minute ago Running kube-scheduler 2 7f610f8d73148 kube-scheduler-ha-488000
b7b900ab000cc 6e38f40d628db 2 minutes ago Exited storage-provisioner 2 63171dbe373cc storage-provisioner
d04adcbe9c31e c69fa2e9cbf5f 4 minutes ago Exited coredns 1 2741bc94126d7 coredns-7c65d6cfc9-5bf4n
4ab82047ebed0 8c811b4aec35f 4 minutes ago Exited busybox 1 303a6748110d4 busybox-7dff88458-jbnhb
36fd51046f117 3a5bc24055c9e 4 minutes ago Exited kindnet-cni 1 b9d3ea7d4cb42 kindnet-hsbm6
abc8bfa16b04b 60c005f310ff3 4 minutes ago Exited kube-proxy 1 29e43f6f8dbeb kube-proxy-lqcs9
20d11dc1772d7 c69fa2e9cbf5f 4 minutes ago Exited coredns 1 86a2dcf36000a coredns-7c65d6cfc9-685s7
4449ac45fb78b 18b729c2288dc 5 minutes ago Exited kube-vip 0 0126ce94f6121 kube-vip-ha-488000
36ff1bf35b2d2 9aa1fad941575 5 minutes ago Exited kube-scheduler 1 66cb6cdee9c05 kube-scheduler-ha-488000
fdcb3a204eb93 2e96e5913fc06 5 minutes ago Exited etcd 1 5a5cdc64553f5 etcd-ha-488000
==> coredns [20d11dc1772d] <==
[WARNING] plugin/kubernetes: starting server with unsynced Kubernetes API
.:53
[INFO] plugin/reload: Running configuration SHA512 = 257e111468ef6f1e36f10df061303186c353cd0e51aed8f50f4e4fd21cec02687aef97084fe1f82262f5cee88179d311670a6ae21ae185759728216fc264125f
CoreDNS-1.11.3
linux/amd64, go1.21.11, a6338e9
[INFO] 127.0.0.1:42491 - 57675 "HINFO IN 3556727844209592965.8082697068778491451. udp 57 false 512" NXDOMAIN qr,rd,ra 132 0.00992987s
[INFO] plugin/ready: Still waiting on: "kubernetes"
[INFO] plugin/ready: Still waiting on: "kubernetes"
[INFO] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.29.3/tools/cache/reflector.go:229: failed to list *v1.Namespace: Get "https://10.96.0.1:443/api/v1/namespaces?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.29.3/tools/cache/reflector.go:229: failed to list *v1.EndpointSlice: Get "https://10.96.0.1:443/apis/discovery.k8s.io/v1/endpointslices?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] plugin/kubernetes: Trace[106125298]: "Reflector ListAndWatch" name:pkg/mod/k8s.io/client-go@v0.29.3/tools/cache/reflector.go:229 (09-Oct-2024 20:07:11.766) (total time: 30002ms):
Trace[106125298]: ---"Objects listed" error:Get "https://10.96.0.1:443/apis/discovery.k8s.io/v1/endpointslices?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout 30001ms (20:07:41.768)
Trace[106125298]: [30.002198282s] [30.002198282s] END
[ERROR] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.29.3/tools/cache/reflector.go:229: Failed to watch *v1.EndpointSlice: failed to list *v1.EndpointSlice: Get "https://10.96.0.1:443/apis/discovery.k8s.io/v1/endpointslices?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.29.3/tools/cache/reflector.go:229: failed to list *v1.Service: Get "https://10.96.0.1:443/api/v1/services?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] plugin/kubernetes: Trace[1764859561]: "Reflector ListAndWatch" name:pkg/mod/k8s.io/client-go@v0.29.3/tools/cache/reflector.go:229 (09-Oct-2024 20:07:11.767) (total time: 30001ms):
Trace[1764859561]: ---"Objects listed" error:Get "https://10.96.0.1:443/api/v1/services?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout 30000ms (20:07:41.768)
Trace[1764859561]: [30.001368179s] [30.001368179s] END
[ERROR] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.29.3/tools/cache/reflector.go:229: Failed to watch *v1.Service: failed to list *v1.Service: Get "https://10.96.0.1:443/api/v1/services?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] plugin/kubernetes: Trace[1230817515]: "Reflector ListAndWatch" name:pkg/mod/k8s.io/client-go@v0.29.3/tools/cache/reflector.go:229 (09-Oct-2024 20:07:11.764) (total time: 30004ms):
Trace[1230817515]: ---"Objects listed" error:Get "https://10.96.0.1:443/api/v1/namespaces?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout 30003ms (20:07:41.768)
Trace[1230817515]: [30.00417792s] [30.00417792s] END
[ERROR] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.29.3/tools/cache/reflector.go:229: Failed to watch *v1.Namespace: failed to list *v1.Namespace: Get "https://10.96.0.1:443/api/v1/namespaces?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] SIGTERM: Shutting down servers then terminating
[INFO] plugin/health: Going into lameduck mode for 5s
==> coredns [d04adcbe9c31] <==
.:53
[INFO] plugin/reload: Running configuration SHA512 = 257e111468ef6f1e36f10df061303186c353cd0e51aed8f50f4e4fd21cec02687aef97084fe1f82262f5cee88179d311670a6ae21ae185759728216fc264125f
CoreDNS-1.11.3
linux/amd64, go1.21.11, a6338e9
[INFO] 127.0.0.1:55055 - 35441 "HINFO IN 2601592654826122702.5755887470621473835. udp 57 false 512" NXDOMAIN qr,rd,ra 132 0.010784765s
[INFO] SIGTERM: Shutting down servers then terminating
[INFO] plugin/health: Going into lameduck mode for 5s
==> describe nodes <==
command /bin/bash -c "sudo /var/lib/minikube/binaries/v1.31.1/kubectl describe nodes --kubeconfig=/var/lib/minikube/kubeconfig" failed with error: /bin/bash -c "sudo /var/lib/minikube/binaries/v1.31.1/kubectl describe nodes --kubeconfig=/var/lib/minikube/kubeconfig": Process exited with status 1
stdout:
stderr:
E1009 20:11:41.129863 3093 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://localhost:8443/api?timeout=32s\": dial tcp 127.0.0.1:8443: connect: connection refused"
E1009 20:11:41.131686 3093 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://localhost:8443/api?timeout=32s\": dial tcp 127.0.0.1:8443: connect: connection refused"
E1009 20:11:41.133206 3093 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://localhost:8443/api?timeout=32s\": dial tcp 127.0.0.1:8443: connect: connection refused"
E1009 20:11:41.134840 3093 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://localhost:8443/api?timeout=32s\": dial tcp 127.0.0.1:8443: connect: connection refused"
E1009 20:11:41.136270 3093 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://localhost:8443/api?timeout=32s\": dial tcp 127.0.0.1:8443: connect: connection refused"
The connection to the server localhost:8443 was refused - did you specify the right host or port?
==> dmesg <==
[ +0.000001] Any video related functionality will be severely degraded, and you may not even be able to suspend the system properly
[ +0.000000] Unless you actually understand what nomodeset does, you should reboot without enabling it
[ +0.035892] ACPI BIOS Warning (bug): Incorrect checksum in table [DSDT] - 0xBE, should be 0x1B (20200925/tbprint-173)
[ +0.007892] RETBleed: WARNING: Spectre v2 mitigation leaves CPU vulnerable to RETBleed attacks, data leaks possible!
[ +5.805731] ACPI Error: Could not enable RealTimeClock event (20200925/evxfevnt-182)
[ +0.000002] ACPI Warning: Could not enable fixed event - RealTimeClock (4) (20200925/evxface-618)
[ +0.007135] platform regulatory.0: Direct firmware load for regulatory.db failed with error -2
[ +2.857226] systemd-fstab-generator[127]: Ignoring "noauto" option for root device
[ +2.221471] NFSD: Using /var/lib/nfs/v4recovery as the NFSv4 state recovery directory
[ +0.000004] NFSD: unable to find recovery directory /var/lib/nfs/v4recovery
[ +0.000001] NFSD: Unable to initialize client recovery tracking! (-2)
[ +0.546516] systemd-fstab-generator[462]: Ignoring "noauto" option for root device
[ +0.093765] systemd-fstab-generator[473]: Ignoring "noauto" option for root device
[ +2.022592] systemd-fstab-generator[1097]: Ignoring "noauto" option for root device
[ +0.268471] systemd-fstab-generator[1135]: Ignoring "noauto" option for root device
[ +0.053236] kauditd_printk_skb: 101 callbacks suppressed
[ +0.041055] systemd-fstab-generator[1147]: Ignoring "noauto" option for root device
[ +0.107758] systemd-fstab-generator[1161]: Ignoring "noauto" option for root device
[ +2.477260] systemd-fstab-generator[1382]: Ignoring "noauto" option for root device
[ +0.087260] systemd-fstab-generator[1394]: Ignoring "noauto" option for root device
[ +0.111203] systemd-fstab-generator[1406]: Ignoring "noauto" option for root device
[ +0.134770] systemd-fstab-generator[1421]: Ignoring "noauto" option for root device
[ +0.422544] systemd-fstab-generator[1582]: Ignoring "noauto" option for root device
[ +6.877987] kauditd_printk_skb: 212 callbacks suppressed
[Oct 9 20:10] kauditd_printk_skb: 40 callbacks suppressed
==> etcd [3fe682afe2b2] <==
{"level":"info","ts":"2024-10-09T20:11:36.911339Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 [logterm: 3, index: 2851] sent MsgPreVote request to 7d77dd3e41d535c1 at term 3"}
{"level":"warn","ts":"2024-10-09T20:11:37.195024Z","caller":"etcdserver/v3_server.go:932","msg":"timed out waiting for read index response (local node might have slow network)","timeout":"7s"}
{"level":"warn","ts":"2024-10-09T20:11:37.195150Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"7.001154649s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"","error":"etcdserver: request timed out"}
{"level":"info","ts":"2024-10-09T20:11:37.195197Z","caller":"traceutil/trace.go:171","msg":"trace[1931515355] range","detail":"{range_begin:; range_end:; }","duration":"7.00124768s","start":"2024-10-09T20:11:30.193940Z","end":"2024-10-09T20:11:37.195188Z","steps":["trace[1931515355] 'agreement among raft nodes before linearized reading' (duration: 7.001153032s)"],"step_count":1}
{"level":"error","ts":"2024-10-09T20:11:37.195252Z","caller":"etcdhttp/health.go:367","msg":"Health check error","path":"/readyz","reason":"[+]serializable_read ok\n[-]linearizable_read failed: etcdserver: request timed out\n[+]data_corruption ok\n","status-code":503,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp.(*CheckRegistry).installRootHttpEndpoint.newHealthHandler.func2\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp/health.go:367\nnet/http.HandlerFunc.ServeHTTP\n\tnet/http/server.go:2141\nnet/http.(*ServeMux).ServeHTTP\n\tnet/http/server.go:2519\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:2943\nnet/http.(*conn).serve\n\tnet/http/server.go:2014"}
{"level":"info","ts":"2024-10-09T20:11:38.011144Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 is starting a new election at term 3"}
{"level":"info","ts":"2024-10-09T20:11:38.011180Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 became pre-candidate at term 3"}
{"level":"info","ts":"2024-10-09T20:11:38.011191Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 received MsgPreVoteResp from b8c6c7563d17d844 at term 3"}
{"level":"info","ts":"2024-10-09T20:11:38.011204Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 [logterm: 3, index: 2851] sent MsgPreVote request to 7d77dd3e41d535c1 at term 3"}
{"level":"info","ts":"2024-10-09T20:11:39.111077Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 is starting a new election at term 3"}
{"level":"info","ts":"2024-10-09T20:11:39.111143Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 became pre-candidate at term 3"}
{"level":"info","ts":"2024-10-09T20:11:39.111154Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 received MsgPreVoteResp from b8c6c7563d17d844 at term 3"}
{"level":"info","ts":"2024-10-09T20:11:39.111164Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 [logterm: 3, index: 2851] sent MsgPreVote request to 7d77dd3e41d535c1 at term 3"}
{"level":"warn","ts":"2024-10-09T20:11:39.771503Z","caller":"rafthttp/probing_status.go:68","msg":"prober detected unhealthy status","round-tripper-name":"ROUND_TRIPPER_RAFT_MESSAGE","remote-peer-id":"7d77dd3e41d535c1","rtt":"0s","error":"dial tcp 192.169.0.6:2380: connect: connection refused"}
{"level":"warn","ts":"2024-10-09T20:11:39.771590Z","caller":"rafthttp/probing_status.go:68","msg":"prober detected unhealthy status","round-tripper-name":"ROUND_TRIPPER_SNAPSHOT","remote-peer-id":"7d77dd3e41d535c1","rtt":"0s","error":"dial tcp 192.169.0.6:2380: connect: connection refused"}
{"level":"info","ts":"2024-10-09T20:11:40.210674Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 is starting a new election at term 3"}
{"level":"info","ts":"2024-10-09T20:11:40.210701Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 became pre-candidate at term 3"}
{"level":"info","ts":"2024-10-09T20:11:40.210709Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 received MsgPreVoteResp from b8c6c7563d17d844 at term 3"}
{"level":"info","ts":"2024-10-09T20:11:40.210719Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 [logterm: 3, index: 2851] sent MsgPreVote request to 7d77dd3e41d535c1 at term 3"}
{"level":"warn","ts":"2024-10-09T20:11:40.693576Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":15583741632805733916,"retry-timeout":"500ms"}
{"level":"warn","ts":"2024-10-09T20:11:41.196045Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":15583741632805733916,"retry-timeout":"500ms"}
{"level":"info","ts":"2024-10-09T20:11:41.314209Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 is starting a new election at term 3"}
{"level":"info","ts":"2024-10-09T20:11:41.314260Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 became pre-candidate at term 3"}
{"level":"info","ts":"2024-10-09T20:11:41.314275Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 received MsgPreVoteResp from b8c6c7563d17d844 at term 3"}
{"level":"info","ts":"2024-10-09T20:11:41.314292Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 [logterm: 3, index: 2851] sent MsgPreVote request to 7d77dd3e41d535c1 at term 3"}
==> etcd [fdcb3a204eb9] <==
{"level":"warn","ts":"2024-10-09T20:09:10.987239Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"7.760377096s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/serviceaccounts/\" range_end:\"/registry/serviceaccounts0\" count_only:true ","response":"","error":"context canceled"}
{"level":"info","ts":"2024-10-09T20:09:10.987251Z","caller":"traceutil/trace.go:171","msg":"trace[730832004] range","detail":"{range_begin:/registry/serviceaccounts/; range_end:/registry/serviceaccounts0; }","duration":"7.760390219s","start":"2024-10-09T20:09:03.226857Z","end":"2024-10-09T20:09:10.987248Z","steps":["trace[730832004] 'agreement among raft nodes before linearized reading' (duration: 7.760377043s)"],"step_count":1}
{"level":"warn","ts":"2024-10-09T20:09:10.987263Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2024-10-09T20:09:03.226829Z","time spent":"7.760430675s","remote":"127.0.0.1:48330","response type":"/etcdserverpb.KV/Range","request count":0,"request size":58,"response count":0,"response size":0,"request content":"key:\"/registry/serviceaccounts/\" range_end:\"/registry/serviceaccounts0\" count_only:true "}
2024/10/09 20:09:10 WARNING: [core] [Server #7] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing"
{"level":"warn","ts":"2024-10-09T20:09:10.987332Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"1.770678775s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/apiregistration.k8s.io/apiservices/\" range_end:\"/registry/apiregistration.k8s.io/apiservices0\" count_only:true ","response":"","error":"context canceled"}
{"level":"info","ts":"2024-10-09T20:09:10.987344Z","caller":"traceutil/trace.go:171","msg":"trace[981794759] range","detail":"{range_begin:/registry/apiregistration.k8s.io/apiservices/; range_end:/registry/apiregistration.k8s.io/apiservices0; }","duration":"1.770692903s","start":"2024-10-09T20:09:09.216648Z","end":"2024-10-09T20:09:10.987340Z","steps":["trace[981794759] 'agreement among raft nodes before linearized reading' (duration: 1.770678564s)"],"step_count":1}
{"level":"warn","ts":"2024-10-09T20:09:10.987356Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2024-10-09T20:09:09.216629Z","time spent":"1.770723093s","remote":"127.0.0.1:48692","response type":"/etcdserverpb.KV/Range","request count":0,"request size":96,"response count":0,"response size":0,"request content":"key:\"/registry/apiregistration.k8s.io/apiservices/\" range_end:\"/registry/apiregistration.k8s.io/apiservices0\" count_only:true "}
2024/10/09 20:09:10 WARNING: [core] [Server #7] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing"
{"level":"warn","ts":"2024-10-09T20:09:10.987407Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"803.198639ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/192.169.0.5\" ","response":"","error":"context canceled"}
{"level":"info","ts":"2024-10-09T20:09:10.987417Z","caller":"traceutil/trace.go:171","msg":"trace[1310673979] range","detail":"{range_begin:/registry/masterleases/192.169.0.5; range_end:; }","duration":"803.209271ms","start":"2024-10-09T20:09:10.184205Z","end":"2024-10-09T20:09:10.987414Z","steps":["trace[1310673979] 'agreement among raft nodes before linearized reading' (duration: 803.198569ms)"],"step_count":1}
{"level":"warn","ts":"2024-10-09T20:09:10.987426Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2024-10-09T20:09:10.184187Z","time spent":"803.236607ms","remote":"127.0.0.1:48142","response type":"/etcdserverpb.KV/Range","request count":0,"request size":36,"response count":0,"response size":0,"request content":"key:\"/registry/masterleases/192.169.0.5\" "}
2024/10/09 20:09:10 WARNING: [core] [Server #7] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing"
{"level":"warn","ts":"2024-10-09T20:09:11.017872Z","caller":"embed/serve.go:212","msg":"stopping secure grpc server due to error","error":"accept tcp 192.169.0.5:2379: use of closed network connection"}
{"level":"warn","ts":"2024-10-09T20:09:11.017918Z","caller":"embed/serve.go:214","msg":"stopped secure grpc server due to error","error":"accept tcp 192.169.0.5:2379: use of closed network connection"}
{"level":"info","ts":"2024-10-09T20:09:11.018006Z","caller":"etcdserver/server.go:1512","msg":"skipped leadership transfer; local server is not leader","local-member-id":"b8c6c7563d17d844","current-leader-member-id":"0"}
{"level":"info","ts":"2024-10-09T20:09:11.018207Z","caller":"rafthttp/peer.go:330","msg":"stopping remote peer","remote-peer-id":"7d77dd3e41d535c1"}
{"level":"info","ts":"2024-10-09T20:09:11.018238Z","caller":"rafthttp/stream.go:294","msg":"stopped TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","remote-peer-id":"7d77dd3e41d535c1"}
{"level":"info","ts":"2024-10-09T20:09:11.018304Z","caller":"rafthttp/stream.go:294","msg":"stopped TCP streaming connection with remote peer","stream-writer-type":"stream Message","remote-peer-id":"7d77dd3e41d535c1"}
{"level":"info","ts":"2024-10-09T20:09:11.018356Z","caller":"rafthttp/pipeline.go:85","msg":"stopped HTTP pipelining with remote peer","local-member-id":"b8c6c7563d17d844","remote-peer-id":"7d77dd3e41d535c1"}
{"level":"info","ts":"2024-10-09T20:09:11.018445Z","caller":"rafthttp/stream.go:442","msg":"stopped stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"b8c6c7563d17d844","remote-peer-id":"7d77dd3e41d535c1"}
{"level":"info","ts":"2024-10-09T20:09:11.018495Z","caller":"rafthttp/stream.go:442","msg":"stopped stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"b8c6c7563d17d844","remote-peer-id":"7d77dd3e41d535c1"}
{"level":"info","ts":"2024-10-09T20:09:11.018505Z","caller":"rafthttp/peer.go:335","msg":"stopped remote peer","remote-peer-id":"7d77dd3e41d535c1"}
{"level":"info","ts":"2024-10-09T20:09:11.020858Z","caller":"embed/etcd.go:581","msg":"stopping serving peer traffic","address":"192.169.0.5:2380"}
{"level":"info","ts":"2024-10-09T20:09:11.020922Z","caller":"embed/etcd.go:586","msg":"stopped serving peer traffic","address":"192.169.0.5:2380"}
{"level":"info","ts":"2024-10-09T20:09:11.020946Z","caller":"embed/etcd.go:379","msg":"closed etcd server","name":"ha-488000","data-dir":"/var/lib/minikube/etcd","advertise-peer-urls":["https://192.169.0.5:2380"],"advertise-client-urls":["https://192.169.0.5:2379"]}
==> kernel <==
20:11:41 up 2 min, 0 users, load average: 0.37, 0.18, 0.07
Linux ha-488000 5.10.207 #1 SMP Tue Oct 8 15:16:25 UTC 2024 x86_64 GNU/Linux
PRETTY_NAME="Buildroot 2023.02.9"
==> kindnet [36fd51046f11] <==
I1009 20:08:32.797447 1 main.go:300] handling current node
I1009 20:08:32.797457 1 main.go:296] Handling node with IPs: map[192.169.0.6:{}]
I1009 20:08:32.797461 1 main.go:323] Node ha-488000-m02 has CIDR [10.244.1.0/24]
I1009 20:08:32.797600 1 main.go:296] Handling node with IPs: map[192.169.0.7:{}]
I1009 20:08:32.797686 1 main.go:323] Node ha-488000-m03 has CIDR [10.244.2.0/24]
I1009 20:08:42.788489 1 main.go:296] Handling node with IPs: map[192.169.0.5:{}]
I1009 20:08:42.788520 1 main.go:300] handling current node
I1009 20:08:42.788535 1 main.go:296] Handling node with IPs: map[192.169.0.6:{}]
I1009 20:08:42.788542 1 main.go:323] Node ha-488000-m02 has CIDR [10.244.1.0/24]
I1009 20:08:42.788931 1 main.go:296] Handling node with IPs: map[192.169.0.7:{}]
I1009 20:08:42.788984 1 main.go:323] Node ha-488000-m03 has CIDR [10.244.2.0/24]
I1009 20:08:42.789077 1 main.go:296] Handling node with IPs: map[192.169.0.8:{}]
I1009 20:08:42.789125 1 main.go:323] Node ha-488000-m04 has CIDR [10.244.3.0/24]
I1009 20:08:52.790103 1 main.go:296] Handling node with IPs: map[192.169.0.6:{}]
I1009 20:08:52.790509 1 main.go:323] Node ha-488000-m02 has CIDR [10.244.1.0/24]
I1009 20:08:52.790748 1 main.go:296] Handling node with IPs: map[192.169.0.8:{}]
I1009 20:08:52.790761 1 main.go:323] Node ha-488000-m04 has CIDR [10.244.3.0/24]
I1009 20:08:52.790887 1 main.go:296] Handling node with IPs: map[192.169.0.5:{}]
I1009 20:08:52.790898 1 main.go:300] handling current node
I1009 20:09:02.797272 1 main.go:296] Handling node with IPs: map[192.169.0.5:{}]
I1009 20:09:02.797310 1 main.go:300] handling current node
I1009 20:09:02.797322 1 main.go:296] Handling node with IPs: map[192.169.0.6:{}]
I1009 20:09:02.797327 1 main.go:323] Node ha-488000-m02 has CIDR [10.244.1.0/24]
I1009 20:09:02.797539 1 main.go:296] Handling node with IPs: map[192.169.0.8:{}]
I1009 20:09:02.797569 1 main.go:323] Node ha-488000-m04 has CIDR [10.244.3.0/24]
==> kube-apiserver [5231dd26c41c] <==
I1009 20:11:16.774346 1 options.go:228] external host was not specified, using 192.169.0.5
I1009 20:11:16.775647 1 server.go:142] Version: v1.31.1
I1009 20:11:16.775691 1 server.go:144] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
I1009 20:11:17.087066 1 shared_informer.go:313] Waiting for caches to sync for node_authorizer
I1009 20:11:17.095753 1 shared_informer.go:313] Waiting for caches to sync for *generic.policySource[*k8s.io/api/admissionregistration/v1.ValidatingAdmissionPolicy,*k8s.io/api/admissionregistration/v1.ValidatingAdmissionPolicyBinding,k8s.io/apiserver/pkg/admission/plugin/policy/validating.Validator]
I1009 20:11:17.098173 1 plugins.go:157] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
I1009 20:11:17.098201 1 plugins.go:160] Loaded 13 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,PodSecurity,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,ClusterTrustBundleAttest,CertificateSubjectRestriction,ValidatingAdmissionPolicy,ValidatingAdmissionWebhook,ResourceQuota.
I1009 20:11:17.098380 1 instance.go:232] Using reconciler: lease
W1009 20:11:37.086252 1 logging.go:55] [core] [Channel #1 SubChannel #3]grpc: addrConn.createTransport failed to connect to {Addr: "127.0.0.1:2379", ServerName: "127.0.0.1:2379", }. Err: connection error: desc = "transport: authentication handshake failed: context canceled"
W1009 20:11:37.086775 1 logging.go:55] [core] [Channel #2 SubChannel #4]grpc: addrConn.createTransport failed to connect to {Addr: "127.0.0.1:2379", ServerName: "127.0.0.1:2379", }. Err: connection error: desc = "transport: authentication handshake failed: context canceled"
W1009 20:11:37.100340 1 logging.go:55] [core] [Channel #5 SubChannel #6]grpc: addrConn.createTransport failed to connect to {Addr: "127.0.0.1:2379", ServerName: "127.0.0.1:2379", }. Err: connection error: desc = "transport: authentication handshake failed: context deadline exceeded"
F1009 20:11:37.100351 1 instance.go:225] Error creating leases: error creating storage factory: context deadline exceeded
==> kube-controller-manager [85817da55476] <==
I1009 20:11:24.070889 1 serving.go:386] Generated self-signed cert in-memory
I1009 20:11:24.433457 1 controllermanager.go:197] "Starting" version="v1.31.1"
I1009 20:11:24.433623 1 controllermanager.go:199] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
I1009 20:11:24.434771 1 dynamic_cafile_content.go:160] "Starting controller" name="request-header::/var/lib/minikube/certs/front-proxy-ca.crt"
I1009 20:11:24.434878 1 secure_serving.go:213] Serving securely on 127.0.0.1:10257
I1009 20:11:24.434946 1 dynamic_cafile_content.go:160] "Starting controller" name="client-ca-bundle::/var/lib/minikube/certs/ca.crt"
I1009 20:11:24.435219 1 tlsconfig.go:243] "Starting DynamicServingCertificateController"
==> kube-controller-manager [b42491e1964f] <==
I1009 20:10:31.211886 1 serving.go:386] Generated self-signed cert in-memory
I1009 20:10:31.693567 1 controllermanager.go:197] "Starting" version="v1.31.1"
I1009 20:10:31.693627 1 controllermanager.go:199] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
I1009 20:10:31.694813 1 dynamic_cafile_content.go:160] "Starting controller" name="request-header::/var/lib/minikube/certs/front-proxy-ca.crt"
I1009 20:10:31.694931 1 dynamic_cafile_content.go:160] "Starting controller" name="client-ca-bundle::/var/lib/minikube/certs/ca.crt"
I1009 20:10:31.695258 1 secure_serving.go:213] Serving securely on 127.0.0.1:10257
I1009 20:10:31.695348 1 tlsconfig.go:243] "Starting DynamicServingCertificateController"
E1009 20:10:51.697851 1 controllermanager.go:242] "Error building controller context" err="failed to wait for apiserver being healthy: timed out waiting for the condition: failed to get apiserver /healthz status: Get \"https://192.169.0.5:8443/healthz\": dial tcp 192.169.0.5:8443: connect: connection refused"
==> kube-proxy [abc8bfa16b04] <==
add table ip kube-proxy
^^^^^^^^^^^^^^^^^^^^^^^^
>
E1009 20:07:12.018097 1 proxier.go:734] "Error cleaning up nftables rules" err=<
could not run nftables command: /dev/stdin:1:1-25: Error: Could not process rule: Operation not supported
add table ip6 kube-proxy
^^^^^^^^^^^^^^^^^^^^^^^^^
>
I1009 20:07:12.041669 1 server.go:677] "Successfully retrieved node IP(s)" IPs=["192.169.0.5"]
E1009 20:07:12.041829 1 server.go:234] "Kube-proxy configuration may be incomplete or incorrect" err="nodePortAddresses is unset; NodePort connections will be accepted on all local IPs. Consider using `--nodeport-addresses primary`"
I1009 20:07:12.071268 1 server_linux.go:146] "No iptables support for family" ipFamily="IPv6"
I1009 20:07:12.071315 1 server.go:245] "kube-proxy running in single-stack mode" ipFamily="IPv4"
I1009 20:07:12.071334 1 server_linux.go:169] "Using iptables Proxier"
I1009 20:07:12.073882 1 proxier.go:255] "Setting route_localnet=1 to allow node-ports on localhost; to change this either disable iptables.localhostNodePorts (--iptables-localhost-nodeports) or set nodePortAddresses (--nodeport-addresses) to filter loopback addresses" ipFamily="IPv4"
I1009 20:07:12.075084 1 server.go:483] "Version info" version="v1.31.1"
I1009 20:07:12.075115 1 server.go:485] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
I1009 20:07:12.079603 1 config.go:199] "Starting service config controller"
I1009 20:07:12.079862 1 shared_informer.go:313] Waiting for caches to sync for service config
I1009 20:07:12.079908 1 config.go:328] "Starting node config controller"
I1009 20:07:12.079934 1 shared_informer.go:313] Waiting for caches to sync for node config
I1009 20:07:12.080892 1 config.go:105] "Starting endpoint slice config controller"
I1009 20:07:12.080953 1 shared_informer.go:313] Waiting for caches to sync for endpoint slice config
I1009 20:07:12.180023 1 shared_informer.go:320] Caches are synced for node config
I1009 20:07:12.180484 1 shared_informer.go:320] Caches are synced for service config
I1009 20:07:12.182238 1 shared_informer.go:320] Caches are synced for endpoint slice config
==> kube-scheduler [36ff1bf35b2d] <==
I1009 20:06:19.481269 1 serving.go:386] Generated self-signed cert in-memory
W1009 20:06:29.843992 1 authentication.go:370] Error looking up in-cluster authentication configuration: Get "https://192.169.0.5:8443/api/v1/namespaces/kube-system/configmaps/extension-apiserver-authentication": net/http: TLS handshake timeout
W1009 20:06:29.844036 1 authentication.go:371] Continuing without authentication configuration. This may treat all requests as anonymous.
W1009 20:06:29.844042 1 authentication.go:372] To require authentication configuration lookup to succeed, set --authentication-tolerate-lookup-failure=false
I1009 20:06:38.982918 1 server.go:167] "Starting Kubernetes Scheduler" version="v1.31.1"
I1009 20:06:38.982968 1 server.go:169] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
I1009 20:06:38.989508 1 secure_serving.go:213] Serving securely on 127.0.0.1:10259
I1009 20:06:38.995047 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file"
I1009 20:06:38.995083 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
I1009 20:06:38.995099 1 tlsconfig.go:243] "Starting DynamicServingCertificateController"
I1009 20:06:39.095758 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
E1009 20:08:46.700744 1 framework.go:1305] "Plugin Failed" err="Operation cannot be fulfilled on pods/binding \"busybox-7dff88458-hmkwj\": pod busybox-7dff88458-hmkwj is already assigned to node \"ha-488000-m04\"" plugin="DefaultBinder" pod="default/busybox-7dff88458-hmkwj" node="ha-488000-m04"
E1009 20:08:46.701136 1 schedule_one.go:348] "scheduler cache ForgetPod failed" err="pod b885517b-eeb8-4d24-8fc6-2bed3662909d(default/busybox-7dff88458-hmkwj) wasn't assumed so cannot be forgotten" pod="default/busybox-7dff88458-hmkwj"
E1009 20:08:46.701580 1 schedule_one.go:1057] "Error scheduling pod; retrying" err="running Bind plugin \"DefaultBinder\": Operation cannot be fulfilled on pods/binding \"busybox-7dff88458-hmkwj\": pod busybox-7dff88458-hmkwj is already assigned to node \"ha-488000-m04\"" pod="default/busybox-7dff88458-hmkwj"
I1009 20:08:46.701720 1 schedule_one.go:1070] "Pod has been assigned to node. Abort adding it back to queue." pod="default/busybox-7dff88458-hmkwj" node="ha-488000-m04"
E1009 20:09:10.890992 1 run.go:72] "command failed" err="finished without leader elect"
==> kube-scheduler [f4310b949f12] <==
E1009 20:10:58.812419 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.StatefulSet: failed to list *v1.StatefulSet: Get \"https://192.169.0.5:8443/apis/apps/v1/statefulsets?limit=500&resourceVersion=0\": dial tcp 192.169.0.5:8443: connect: connection refused" logger="UnhandledError"
W1009 20:10:59.062686 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.PodDisruptionBudget: Get "https://192.169.0.5:8443/apis/policy/v1/poddisruptionbudgets?limit=500&resourceVersion=0": dial tcp 192.169.0.5:8443: connect: connection refused
E1009 20:10:59.062788 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.PodDisruptionBudget: failed to list *v1.PodDisruptionBudget: Get \"https://192.169.0.5:8443/apis/policy/v1/poddisruptionbudgets?limit=500&resourceVersion=0\": dial tcp 192.169.0.5:8443: connect: connection refused" logger="UnhandledError"
W1009 20:11:27.482198 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.Pod: Get "https://192.169.0.5:8443/api/v1/pods?fieldSelector=status.phase%21%3DSucceeded%2Cstatus.phase%21%3DFailed&limit=500&resourceVersion=0": net/http: TLS handshake timeout
E1009 20:11:27.482613 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.Pod: failed to list *v1.Pod: Get \"https://192.169.0.5:8443/api/v1/pods?fieldSelector=status.phase%21%3DSucceeded%2Cstatus.phase%21%3DFailed&limit=500&resourceVersion=0\": net/http: TLS handshake timeout" logger="UnhandledError"
W1009 20:11:30.117975 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.CSINode: Get "https://192.169.0.5:8443/apis/storage.k8s.io/v1/csinodes?limit=500&resourceVersion=0": net/http: TLS handshake timeout
E1009 20:11:30.118198 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.CSINode: failed to list *v1.CSINode: Get \"https://192.169.0.5:8443/apis/storage.k8s.io/v1/csinodes?limit=500&resourceVersion=0\": net/http: TLS handshake timeout" logger="UnhandledError"
W1009 20:11:32.216973 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.StorageClass: Get "https://192.169.0.5:8443/apis/storage.k8s.io/v1/storageclasses?limit=500&resourceVersion=0": net/http: TLS handshake timeout
E1009 20:11:32.217404 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.StorageClass: failed to list *v1.StorageClass: Get \"https://192.169.0.5:8443/apis/storage.k8s.io/v1/storageclasses?limit=500&resourceVersion=0\": net/http: TLS handshake timeout" logger="UnhandledError"
W1009 20:11:33.953673 1 reflector.go:561] runtime/asm_amd64.s:1695: failed to list *v1.ConfigMap: Get "https://192.169.0.5:8443/api/v1/namespaces/kube-system/configmaps?fieldSelector=metadata.name%3Dextension-apiserver-authentication&limit=500&resourceVersion=0": net/http: TLS handshake timeout
E1009 20:11:33.953771 1 reflector.go:158] "Unhandled Error" err="runtime/asm_amd64.s:1695: Failed to watch *v1.ConfigMap: failed to list *v1.ConfigMap: Get \"https://192.169.0.5:8443/api/v1/namespaces/kube-system/configmaps?fieldSelector=metadata.name%3Dextension-apiserver-authentication&limit=500&resourceVersion=0\": net/http: TLS handshake timeout" logger="UnhandledError"
W1009 20:11:38.105908 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.PodDisruptionBudget: Get "https://192.169.0.5:8443/apis/policy/v1/poddisruptionbudgets?limit=500&resourceVersion=0": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:43064->192.169.0.5:8443: read: connection reset by peer
E1009 20:11:38.105957 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.PodDisruptionBudget: failed to list *v1.PodDisruptionBudget: Get \"https://192.169.0.5:8443/apis/policy/v1/poddisruptionbudgets?limit=500&resourceVersion=0\": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:43064->192.169.0.5:8443: read: connection reset by peer" logger="UnhandledError"
W1009 20:11:38.106066 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.Service: Get "https://192.169.0.5:8443/api/v1/services?limit=500&resourceVersion=0": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:43090->192.169.0.5:8443: read: connection reset by peer
E1009 20:11:38.106141 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.Service: failed to list *v1.Service: Get \"https://192.169.0.5:8443/api/v1/services?limit=500&resourceVersion=0\": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:43090->192.169.0.5:8443: read: connection reset by peer" logger="UnhandledError"
W1009 20:11:38.106170 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.CSIStorageCapacity: Get "https://192.169.0.5:8443/apis/storage.k8s.io/v1/csistoragecapacities?limit=500&resourceVersion=0": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:43112->192.169.0.5:8443: read: connection reset by peer
E1009 20:11:38.106199 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.CSIStorageCapacity: failed to list *v1.CSIStorageCapacity: Get \"https://192.169.0.5:8443/apis/storage.k8s.io/v1/csistoragecapacities?limit=500&resourceVersion=0\": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:43112->192.169.0.5:8443: read: connection reset by peer" logger="UnhandledError"
W1009 20:11:38.106279 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.CSIDriver: Get "https://192.169.0.5:8443/apis/storage.k8s.io/v1/csidrivers?limit=500&resourceVersion=0": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:53428->192.169.0.5:8443: read: connection reset by peer
E1009 20:11:38.106300 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.CSIDriver: failed to list *v1.CSIDriver: Get \"https://192.169.0.5:8443/apis/storage.k8s.io/v1/csidrivers?limit=500&resourceVersion=0\": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:53428->192.169.0.5:8443: read: connection reset by peer" logger="UnhandledError"
W1009 20:11:38.106853 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.Node: Get "https://192.169.0.5:8443/api/v1/nodes?limit=500&resourceVersion=0": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:43100->192.169.0.5:8443: read: connection reset by peer
E1009 20:11:38.106879 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.Node: failed to list *v1.Node: Get \"https://192.169.0.5:8443/api/v1/nodes?limit=500&resourceVersion=0\": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:43100->192.169.0.5:8443: read: connection reset by peer" logger="UnhandledError"
W1009 20:11:38.107458 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.PersistentVolumeClaim: Get "https://192.169.0.5:8443/api/v1/persistentvolumeclaims?limit=500&resourceVersion=0": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:43118->192.169.0.5:8443: read: connection reset by peer
E1009 20:11:38.107491 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.PersistentVolumeClaim: failed to list *v1.PersistentVolumeClaim: Get \"https://192.169.0.5:8443/api/v1/persistentvolumeclaims?limit=500&resourceVersion=0\": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:43118->192.169.0.5:8443: read: connection reset by peer" logger="UnhandledError"
W1009 20:11:38.898660 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.PersistentVolume: Get "https://192.169.0.5:8443/api/v1/persistentvolumes?limit=500&resourceVersion=0": dial tcp 192.169.0.5:8443: connect: connection refused
E1009 20:11:38.898710 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.PersistentVolume: failed to list *v1.PersistentVolume: Get \"https://192.169.0.5:8443/api/v1/persistentvolumes?limit=500&resourceVersion=0\": dial tcp 192.169.0.5:8443: connect: connection refused" logger="UnhandledError"
==> kubelet <==
Oct 09 20:11:22 ha-488000 kubelet[1589]: W1009 20:11:22.012831 1589 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.Service: Get "https://control-plane.minikube.internal:8443/api/v1/services?fieldSelector=spec.clusterIP%21%3DNone&limit=500&resourceVersion=0": dial tcp 192.169.0.254:8443: connect: no route to host
Oct 09 20:11:22 ha-488000 kubelet[1589]: E1009 20:11:22.013748 1589 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.Service: failed to list *v1.Service: Get \"https://control-plane.minikube.internal:8443/api/v1/services?fieldSelector=spec.clusterIP%21%3DNone&limit=500&resourceVersion=0\": dial tcp 192.169.0.254:8443: connect: no route to host" logger="UnhandledError"
Oct 09 20:11:23 ha-488000 kubelet[1589]: I1009 20:11:23.607934 1589 scope.go:117] "RemoveContainer" containerID="b42491e1964fdcd16c9570ec02db0cc321669af0c97f2f1c510efc0f7cb702c8"
Oct 09 20:11:25 ha-488000 kubelet[1589]: E1009 20:11:25.083907 1589 event.go:368] "Unable to write event (may retry after sleeping)" err="Post \"https://control-plane.minikube.internal:8443/api/v1/namespaces/default/events\": dial tcp 192.169.0.254:8443: connect: no route to host" event="&Event{ObjectMeta:{ha-488000.17fce1c158b0052d default 0 0001-01-01 00:00:00 +0000 UTC <nil> <nil> map[] map[] [] [] []},InvolvedObject:ObjectReference{Kind:Node,Namespace:,Name:ha-488000,UID:ha-488000,APIVersion:,ResourceVersion:,FieldPath:,},Reason:Starting,Message:Starting kubelet.,Source:EventSource{Component:kubelet,Host:ha-488000,},FirstTimestamp:2024-10-09 20:09:37.536296237 +0000 UTC m=+0.109841447,LastTimestamp:2024-10-09 20:09:37.536296237 +0000 UTC m=+0.109841447,Count:1,Type:Normal,EventTime:0001-01-01 00:00:00 +0000 UTC,Series:nil,Action:,Related:nil,ReportingController:kubelet,ReportingInstance:ha-488000,}"
Oct 09 20:11:27 ha-488000 kubelet[1589]: E1009 20:11:27.633249 1589 eviction_manager.go:285] "Eviction manager: failed to get summary stats" err="failed to get node info: node \"ha-488000\" not found"
Oct 09 20:11:29 ha-488000 kubelet[1589]: I1009 20:11:29.016395 1589 kubelet_node_status.go:72] "Attempting to register node" node="ha-488000"
Oct 09 20:11:31 ha-488000 kubelet[1589]: E1009 20:11:31.230784 1589 kubelet_node_status.go:95] "Unable to register node with API server" err="Post \"https://control-plane.minikube.internal:8443/api/v1/nodes\": dial tcp 192.169.0.254:8443: connect: no route to host" node="ha-488000"
Oct 09 20:11:31 ha-488000 kubelet[1589]: E1009 20:11:31.230818 1589 controller.go:145] "Failed to ensure lease exists, will retry" err="Get \"https://control-plane.minikube.internal:8443/apis/coordination.k8s.io/v1/namespaces/kube-node-lease/leases/ha-488000?timeout=10s\": dial tcp 192.169.0.254:8443: connect: no route to host" interval="7s"
Oct 09 20:11:37 ha-488000 kubelet[1589]: E1009 20:11:37.378428 1589 event.go:368] "Unable to write event (may retry after sleeping)" err="Post \"https://control-plane.minikube.internal:8443/api/v1/namespaces/default/events\": dial tcp 192.169.0.254:8443: connect: no route to host" event="&Event{ObjectMeta:{ha-488000.17fce1c158b0052d default 0 0001-01-01 00:00:00 +0000 UTC <nil> <nil> map[] map[] [] [] []},InvolvedObject:ObjectReference{Kind:Node,Namespace:,Name:ha-488000,UID:ha-488000,APIVersion:,ResourceVersion:,FieldPath:,},Reason:Starting,Message:Starting kubelet.,Source:EventSource{Component:kubelet,Host:ha-488000,},FirstTimestamp:2024-10-09 20:09:37.536296237 +0000 UTC m=+0.109841447,LastTimestamp:2024-10-09 20:09:37.536296237 +0000 UTC m=+0.109841447,Count:1,Type:Normal,EventTime:0001-01-01 00:00:00 +0000 UTC,Series:nil,Action:,Related:nil,ReportingController:kubelet,ReportingInstance:ha-488000,}"
Oct 09 20:11:37 ha-488000 kubelet[1589]: E1009 20:11:37.634017 1589 eviction_manager.go:285] "Eviction manager: failed to get summary stats" err="failed to get node info: node \"ha-488000\" not found"
Oct 09 20:11:37 ha-488000 kubelet[1589]: E1009 20:11:37.639252 1589 iptables.go:577] "Could not set up iptables canary" err=<
Oct 09 20:11:37 ha-488000 kubelet[1589]: error creating chain "KUBE-KUBELET-CANARY": exit status 3: Ignoring deprecated --wait-interval option.
Oct 09 20:11:37 ha-488000 kubelet[1589]: ip6tables v1.8.9 (legacy): can't initialize ip6tables table `nat': Table does not exist (do you need to insmod?)
Oct 09 20:11:37 ha-488000 kubelet[1589]: Perhaps ip6tables or your kernel needs to be upgraded.
Oct 09 20:11:37 ha-488000 kubelet[1589]: > table="nat" chain="KUBE-KUBELET-CANARY"
Oct 09 20:11:37 ha-488000 kubelet[1589]: I1009 20:11:37.914885 1589 scope.go:117] "RemoveContainer" containerID="f8b55a7e3128b1a465dc68d9bfabdeeb6564e4b3492c807b6148ec4010f16d50"
Oct 09 20:11:37 ha-488000 kubelet[1589]: I1009 20:11:37.915598 1589 scope.go:117] "RemoveContainer" containerID="5231dd26c41cd6b23dc0e184aed6d620565385c4dd3cde5ed2ef978d08ddbfcf"
Oct 09 20:11:37 ha-488000 kubelet[1589]: E1009 20:11:37.915719 1589 pod_workers.go:1301] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"kube-apiserver\" with CrashLoopBackOff: \"back-off 40s restarting failed container=kube-apiserver pod=kube-apiserver-ha-488000_kube-system(3c80eeb5526c12484f72fb4888b1bcf1)\"" pod="kube-system/kube-apiserver-ha-488000" podUID="3c80eeb5526c12484f72fb4888b1bcf1"
Oct 09 20:11:38 ha-488000 kubelet[1589]: I1009 20:11:38.232428 1589 kubelet_node_status.go:72] "Attempting to register node" node="ha-488000"
Oct 09 20:11:38 ha-488000 kubelet[1589]: I1009 20:11:38.927507 1589 scope.go:117] "RemoveContainer" containerID="5231dd26c41cd6b23dc0e184aed6d620565385c4dd3cde5ed2ef978d08ddbfcf"
Oct 09 20:11:38 ha-488000 kubelet[1589]: E1009 20:11:38.927611 1589 pod_workers.go:1301] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"kube-apiserver\" with CrashLoopBackOff: \"back-off 40s restarting failed container=kube-apiserver pod=kube-apiserver-ha-488000_kube-system(3c80eeb5526c12484f72fb4888b1bcf1)\"" pod="kube-system/kube-apiserver-ha-488000" podUID="3c80eeb5526c12484f72fb4888b1bcf1"
Oct 09 20:11:39 ha-488000 kubelet[1589]: I1009 20:11:39.934352 1589 scope.go:117] "RemoveContainer" containerID="5231dd26c41cd6b23dc0e184aed6d620565385c4dd3cde5ed2ef978d08ddbfcf"
Oct 09 20:11:39 ha-488000 kubelet[1589]: E1009 20:11:39.934564 1589 pod_workers.go:1301] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"kube-apiserver\" with CrashLoopBackOff: \"back-off 40s restarting failed container=kube-apiserver pod=kube-apiserver-ha-488000_kube-system(3c80eeb5526c12484f72fb4888b1bcf1)\"" pod="kube-system/kube-apiserver-ha-488000" podUID="3c80eeb5526c12484f72fb4888b1bcf1"
Oct 09 20:11:40 ha-488000 kubelet[1589]: E1009 20:11:40.443824 1589 kubelet_node_status.go:95] "Unable to register node with API server" err="Post \"https://control-plane.minikube.internal:8443/api/v1/nodes\": dial tcp 192.169.0.254:8443: connect: no route to host" node="ha-488000"
Oct 09 20:11:40 ha-488000 kubelet[1589]: E1009 20:11:40.443840 1589 controller.go:145] "Failed to ensure lease exists, will retry" err="Get \"https://control-plane.minikube.internal:8443/apis/coordination.k8s.io/v1/namespaces/kube-node-lease/leases/ha-488000?timeout=10s\": dial tcp 192.169.0.254:8443: connect: no route to host" interval="7s"
-- /stdout --
helpers_test.go:254: (dbg) Run: out/minikube-darwin-amd64 status --format={{.APIServer}} -p ha-488000 -n ha-488000
helpers_test.go:254: (dbg) Non-zero exit: out/minikube-darwin-amd64 status --format={{.APIServer}} -p ha-488000 -n ha-488000: exit status 2 (174.530635ms)
-- stdout --
Stopped
-- /stdout --
helpers_test.go:254: status error: exit status 2 (may be ok)
helpers_test.go:256: "ha-488000" apiserver is not running, skipping kubectl commands (state="Stopped")
--- FAIL: TestMultiControlPlane/serial/DegradedAfterClusterRestart (2.92s)