=== RUN TestMultiControlPlane/serial/DegradedAfterClusterRestart
ha_test.go:390: (dbg) Run: out/minikube-darwin-amd64 profile list --output json
ha_test.go:413: expected profile "ha-356000" in json of 'profile list' to have "Degraded" status but have "Stopped" status. got *"{\"invalid\":[],\"valid\":[{\"Name\":\"ha-356000\",\"Status\":\"Stopped\",\"Config\":{\"Name\":\"ha-356000\",\"KeepContext\":false,\"EmbedCerts\":false,\"MinikubeISO\":\"https://storage.googleapis.com/minikube-builds/iso/19452/minikube-v1.33.1-1723740674-19452-amd64.iso\",\"KicBaseImage\":\"gcr.io/k8s-minikube/kicbase-builds:v0.0.44-1723740748-19452@sha256:2211a6931895d2d502e957e9667096db10734a96767d670cb4dbffdd37397b0d\",\"Memory\":2200,\"CPUs\":2,\"DiskSize\":20000,\"Driver\":\"hyperkit\",\"HyperkitVpnKitSock\":\"\",\"HyperkitVSockPorts\":[],\"DockerEnv\":null,\"ContainerVolumeMounts\":null,\"InsecureRegistry\":null,\"RegistryMirror\":[],\"HostOnlyCIDR\":\"192.168.59.1/24\",\"HypervVirtualSwitch\":\"\",\"HypervUseExternalSwitch\":false,\"HypervExternalAdapter\":\"\",\"KVMNetwork\":\"default\",\"KVMQemuURI\":\"qemu:///system\",\"KVMGPU\":false,\"KVMHidden\":false,\"KVMNUMACoun
t\":1,\"APIServerPort\":8443,\"DockerOpt\":null,\"DisableDriverMounts\":false,\"NFSShare\":[],\"NFSSharesRoot\":\"/nfsshares\",\"UUID\":\"\",\"NoVTXCheck\":false,\"DNSProxy\":false,\"HostDNSResolver\":true,\"HostOnlyNicType\":\"virtio\",\"NatNicType\":\"virtio\",\"SSHIPAddress\":\"\",\"SSHUser\":\"root\",\"SSHKey\":\"\",\"SSHPort\":22,\"KubernetesConfig\":{\"KubernetesVersion\":\"v1.31.0\",\"ClusterName\":\"ha-356000\",\"Namespace\":\"default\",\"APIServerHAVIP\":\"192.169.0.254\",\"APIServerName\":\"minikubeCA\",\"APIServerNames\":null,\"APIServerIPs\":null,\"DNSDomain\":\"cluster.local\",\"ContainerRuntime\":\"docker\",\"CRISocket\":\"\",\"NetworkPlugin\":\"cni\",\"FeatureGates\":\"\",\"ServiceCIDR\":\"10.96.0.0/12\",\"ImageRepository\":\"\",\"LoadBalancerStartIP\":\"\",\"LoadBalancerEndIP\":\"\",\"CustomIngressCert\":\"\",\"RegistryAliases\":\"\",\"ExtraOptions\":null,\"ShouldLoadCachedImages\":true,\"EnableDefaultCNI\":false,\"CNI\":\"\"},\"Nodes\":[{\"Name\":\"\",\"IP\":\"192.169.0.38\",\"Port\":8443,\"K
ubernetesVersion\":\"v1.31.0\",\"ContainerRuntime\":\"docker\",\"ControlPlane\":true,\"Worker\":true},{\"Name\":\"m02\",\"IP\":\"192.169.0.39\",\"Port\":8443,\"KubernetesVersion\":\"v1.31.0\",\"ContainerRuntime\":\"docker\",\"ControlPlane\":true,\"Worker\":true},{\"Name\":\"m04\",\"IP\":\"192.169.0.41\",\"Port\":0,\"KubernetesVersion\":\"v1.31.0\",\"ContainerRuntime\":\"docker\",\"ControlPlane\":false,\"Worker\":true}],\"Addons\":{\"ambassador\":false,\"auto-pause\":false,\"cloud-spanner\":false,\"csi-hostpath-driver\":false,\"dashboard\":false,\"default-storageclass\":false,\"efk\":false,\"freshpod\":false,\"gcp-auth\":false,\"gvisor\":false,\"headlamp\":false,\"helm-tiller\":false,\"inaccel\":false,\"ingress\":false,\"ingress-dns\":false,\"inspektor-gadget\":false,\"istio\":false,\"istio-provisioner\":false,\"kong\":false,\"kubeflow\":false,\"kubevirt\":false,\"logviewer\":false,\"metallb\":false,\"metrics-server\":false,\"nvidia-device-plugin\":false,\"nvidia-driver-installer\":false,\"nvidia-gpu-device-pl
ugin\":false,\"olm\":false,\"pod-security-policy\":false,\"portainer\":false,\"registry\":false,\"registry-aliases\":false,\"registry-creds\":false,\"storage-provisioner\":false,\"storage-provisioner-gluster\":false,\"storage-provisioner-rancher\":false,\"volcano\":false,\"volumesnapshots\":false,\"yakd\":false},\"CustomAddonImages\":null,\"CustomAddonRegistries\":null,\"VerifyComponents\":{\"apiserver\":true,\"apps_running\":true,\"default_sa\":true,\"extra\":true,\"kubelet\":true,\"node_ready\":true,\"system_pods\":true},\"StartHostTimeout\":360000000000,\"ScheduledStop\":null,\"ExposedPorts\":[],\"ListenAddress\":\"\",\"Network\":\"\",\"Subnet\":\"\",\"MultiNodeRequested\":true,\"ExtraDisks\":0,\"CertExpiration\":94608000000000000,\"Mount\":false,\"MountString\":\"/Users:/minikube-host\",\"Mount9PVersion\":\"9p2000.L\",\"MountGID\":\"docker\",\"MountIP\":\"\",\"MountMSize\":262144,\"MountOptions\":[],\"MountPort\":0,\"MountType\":\"9p\",\"MountUID\":\"docker\",\"BinaryMirror\":\"\",\"DisableOptimizations\"
:false,\"DisableMetrics\":false,\"CustomQemuFirmwarePath\":\"\",\"SocketVMnetClientPath\":\"\",\"SocketVMnetPath\":\"\",\"StaticIP\":\"\",\"SSHAuthSock\":\"\",\"SSHAgentPID\":0,\"GPUs\":\"\",\"AutoPauseInterval\":60000000000},\"Active\":false,\"ActiveKubeContext\":true}]}"*. args: "out/minikube-darwin-amd64 profile list --output json"
helpers_test.go:222: -----------------------post-mortem--------------------------------
helpers_test.go:239: (dbg) Run: out/minikube-darwin-amd64 status --format={{.Host}} -p ha-356000 -n ha-356000
helpers_test.go:239: (dbg) Non-zero exit: out/minikube-darwin-amd64 status --format={{.Host}} -p ha-356000 -n ha-356000: exit status 2 (148.806133ms)
-- stdout --
Running
-- /stdout --
helpers_test.go:239: status error: exit status 2 (may be ok)
helpers_test.go:244: <<< TestMultiControlPlane/serial/DegradedAfterClusterRestart FAILED: start of post-mortem logs <<<
helpers_test.go:245: ======> post-mortem[TestMultiControlPlane/serial/DegradedAfterClusterRestart]: minikube logs <======
helpers_test.go:247: (dbg) Run: out/minikube-darwin-amd64 -p ha-356000 logs -n 25
helpers_test.go:247: (dbg) Done: out/minikube-darwin-amd64 -p ha-356000 logs -n 25: (2.131818617s)
helpers_test.go:252: TestMultiControlPlane/serial/DegradedAfterClusterRestart logs:
-- stdout --
==> Audit <==
|---------|------------------------------------------------------------------------------------------------------------------------------|-----------|---------|---------|---------------------|---------------------|
| Command | Args | Profile | User | Version | Start Time | End Time |
|---------|------------------------------------------------------------------------------------------------------------------------------|-----------|---------|---------|---------------------|---------------------|
| cp | ha-356000 cp ha-356000-m03:/home/docker/cp-test.txt | ha-356000 | jenkins | v1.33.1 | 19 Aug 24 04:29 PDT | 19 Aug 24 04:29 PDT |
| | ha-356000-m04:/home/docker/cp-test_ha-356000-m03_ha-356000-m04.txt | | | | | |
| ssh | ha-356000 ssh -n | ha-356000 | jenkins | v1.33.1 | 19 Aug 24 04:29 PDT | 19 Aug 24 04:29 PDT |
| | ha-356000-m03 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| ssh | ha-356000 ssh -n ha-356000-m04 sudo cat | ha-356000 | jenkins | v1.33.1 | 19 Aug 24 04:29 PDT | 19 Aug 24 04:29 PDT |
| | /home/docker/cp-test_ha-356000-m03_ha-356000-m04.txt | | | | | |
| cp | ha-356000 cp testdata/cp-test.txt | ha-356000 | jenkins | v1.33.1 | 19 Aug 24 04:29 PDT | 19 Aug 24 04:29 PDT |
| | ha-356000-m04:/home/docker/cp-test.txt | | | | | |
| ssh | ha-356000 ssh -n | ha-356000 | jenkins | v1.33.1 | 19 Aug 24 04:29 PDT | 19 Aug 24 04:29 PDT |
| | ha-356000-m04 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| cp | ha-356000 cp ha-356000-m04:/home/docker/cp-test.txt | ha-356000 | jenkins | v1.33.1 | 19 Aug 24 04:29 PDT | 19 Aug 24 04:29 PDT |
| | /var/folders/0y/_8hvl7v13q38_kkh25vpxkz00000gp/T/TestMultiControlPlaneserialCopyFile3080543681/001/cp-test_ha-356000-m04.txt | | | | | |
| ssh | ha-356000 ssh -n | ha-356000 | jenkins | v1.33.1 | 19 Aug 24 04:29 PDT | 19 Aug 24 04:29 PDT |
| | ha-356000-m04 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| cp | ha-356000 cp ha-356000-m04:/home/docker/cp-test.txt | ha-356000 | jenkins | v1.33.1 | 19 Aug 24 04:29 PDT | 19 Aug 24 04:29 PDT |
| | ha-356000:/home/docker/cp-test_ha-356000-m04_ha-356000.txt | | | | | |
| ssh | ha-356000 ssh -n | ha-356000 | jenkins | v1.33.1 | 19 Aug 24 04:29 PDT | 19 Aug 24 04:29 PDT |
| | ha-356000-m04 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| ssh | ha-356000 ssh -n ha-356000 sudo cat | ha-356000 | jenkins | v1.33.1 | 19 Aug 24 04:29 PDT | 19 Aug 24 04:29 PDT |
| | /home/docker/cp-test_ha-356000-m04_ha-356000.txt | | | | | |
| cp | ha-356000 cp ha-356000-m04:/home/docker/cp-test.txt | ha-356000 | jenkins | v1.33.1 | 19 Aug 24 04:29 PDT | 19 Aug 24 04:29 PDT |
| | ha-356000-m02:/home/docker/cp-test_ha-356000-m04_ha-356000-m02.txt | | | | | |
| ssh | ha-356000 ssh -n | ha-356000 | jenkins | v1.33.1 | 19 Aug 24 04:29 PDT | 19 Aug 24 04:29 PDT |
| | ha-356000-m04 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| ssh | ha-356000 ssh -n ha-356000-m02 sudo cat | ha-356000 | jenkins | v1.33.1 | 19 Aug 24 04:29 PDT | 19 Aug 24 04:29 PDT |
| | /home/docker/cp-test_ha-356000-m04_ha-356000-m02.txt | | | | | |
| cp | ha-356000 cp ha-356000-m04:/home/docker/cp-test.txt | ha-356000 | jenkins | v1.33.1 | 19 Aug 24 04:29 PDT | 19 Aug 24 04:29 PDT |
| | ha-356000-m03:/home/docker/cp-test_ha-356000-m04_ha-356000-m03.txt | | | | | |
| ssh | ha-356000 ssh -n | ha-356000 | jenkins | v1.33.1 | 19 Aug 24 04:29 PDT | 19 Aug 24 04:29 PDT |
| | ha-356000-m04 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| ssh | ha-356000 ssh -n ha-356000-m03 sudo cat | ha-356000 | jenkins | v1.33.1 | 19 Aug 24 04:29 PDT | 19 Aug 24 04:29 PDT |
| | /home/docker/cp-test_ha-356000-m04_ha-356000-m03.txt | | | | | |
| node | ha-356000 node stop m02 -v=7 | ha-356000 | jenkins | v1.33.1 | 19 Aug 24 04:29 PDT | 19 Aug 24 04:29 PDT |
| | --alsologtostderr | | | | | |
| node | ha-356000 node start m02 -v=7 | ha-356000 | jenkins | v1.33.1 | 19 Aug 24 04:29 PDT | 19 Aug 24 04:30 PDT |
| | --alsologtostderr | | | | | |
| node | list -p ha-356000 -v=7 | ha-356000 | jenkins | v1.33.1 | 19 Aug 24 04:30 PDT | |
| | --alsologtostderr | | | | | |
| stop | -p ha-356000 -v=7 | ha-356000 | jenkins | v1.33.1 | 19 Aug 24 04:30 PDT | 19 Aug 24 04:31 PDT |
| | --alsologtostderr | | | | | |
| start | -p ha-356000 --wait=true -v=7 | ha-356000 | jenkins | v1.33.1 | 19 Aug 24 04:31 PDT | 19 Aug 24 04:33 PDT |
| | --alsologtostderr | | | | | |
| node | list -p ha-356000 | ha-356000 | jenkins | v1.33.1 | 19 Aug 24 04:33 PDT | |
| node | ha-356000 node delete m03 -v=7 | ha-356000 | jenkins | v1.33.1 | 19 Aug 24 04:33 PDT | 19 Aug 24 04:33 PDT |
| | --alsologtostderr | | | | | |
| stop | ha-356000 stop -v=7 | ha-356000 | jenkins | v1.33.1 | 19 Aug 24 04:33 PDT | 19 Aug 24 04:34 PDT |
| | --alsologtostderr | | | | | |
| start | -p ha-356000 --wait=true | ha-356000 | jenkins | v1.33.1 | 19 Aug 24 04:34 PDT | |
| | -v=7 --alsologtostderr | | | | | |
| | --driver=hyperkit | | | | | |
|---------|------------------------------------------------------------------------------------------------------------------------------|-----------|---------|---------|---------------------|---------------------|
==> Last Start <==
Log file created at: 2024/08/19 04:34:11
Running on machine: MacOS-Agent-4
Binary: Built with gc go1.22.5 for darwin/amd64
Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg
I0819 04:34:11.004233 13066 out.go:345] Setting OutFile to fd 1 ...
I0819 04:34:11.004503 13066 out.go:392] TERM=,COLORTERM=, which probably does not support color
I0819 04:34:11.004508 13066 out.go:358] Setting ErrFile to fd 2...
I0819 04:34:11.004512 13066 out.go:392] TERM=,COLORTERM=, which probably does not support color
I0819 04:34:11.004695 13066 root.go:338] Updating PATH: /Users/jenkins/minikube-integration/19476-10399/.minikube/bin
I0819 04:34:11.006109 13066 out.go:352] Setting JSON to false
I0819 04:34:11.028926 13066 start.go:129] hostinfo: {"hostname":"MacOS-Agent-4.local","uptime":9220,"bootTime":1724058031,"procs":483,"os":"darwin","platform":"darwin","platformFamily":"Standalone Workstation","platformVersion":"14.6.1","kernelVersion":"23.6.0","kernelArch":"x86_64","virtualizationSystem":"","virtualizationRole":"","hostId":"f2f27e25-cfda-5ffd-9706-e98286194e62"}
W0819 04:34:11.029017 13066 start.go:137] gopshost.Virtualization returned error: not implemented yet
I0819 04:34:11.050926 13066 out.go:177] * [ha-356000] minikube v1.33.1 on Darwin 14.6.1
I0819 04:34:11.093405 13066 out.go:177] - MINIKUBE_LOCATION=19476
I0819 04:34:11.093483 13066 notify.go:220] Checking for updates...
I0819 04:34:11.136422 13066 out.go:177] - KUBECONFIG=/Users/jenkins/minikube-integration/19476-10399/kubeconfig
I0819 04:34:11.157315 13066 out.go:177] - MINIKUBE_BIN=out/minikube-darwin-amd64
I0819 04:34:11.178299 13066 out.go:177] - MINIKUBE_SUPPRESS_DOCKER_PERFORMANCE=true
I0819 04:34:11.199478 13066 out.go:177] - MINIKUBE_HOME=/Users/jenkins/minikube-integration/19476-10399/.minikube
I0819 04:34:11.220396 13066 out.go:177] - MINIKUBE_FORCE_SYSTEMD=
I0819 04:34:11.241974 13066 config.go:182] Loaded profile config "ha-356000": Driver=hyperkit, ContainerRuntime=docker, KubernetesVersion=v1.31.0
I0819 04:34:11.242628 13066 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0819 04:34:11.242738 13066 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0819 04:34:11.252467 13066 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:60842
I0819 04:34:11.252840 13066 main.go:141] libmachine: () Calling .GetVersion
I0819 04:34:11.253240 13066 main.go:141] libmachine: Using API Version 1
I0819 04:34:11.253266 13066 main.go:141] libmachine: () Calling .SetConfigRaw
I0819 04:34:11.253534 13066 main.go:141] libmachine: () Calling .GetMachineName
I0819 04:34:11.253664 13066 main.go:141] libmachine: (ha-356000) Calling .DriverName
I0819 04:34:11.253895 13066 driver.go:392] Setting default libvirt URI to qemu:///system
I0819 04:34:11.254156 13066 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0819 04:34:11.254178 13066 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0819 04:34:11.262598 13066 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:60844
I0819 04:34:11.262924 13066 main.go:141] libmachine: () Calling .GetVersion
I0819 04:34:11.263280 13066 main.go:141] libmachine: Using API Version 1
I0819 04:34:11.263296 13066 main.go:141] libmachine: () Calling .SetConfigRaw
I0819 04:34:11.263519 13066 main.go:141] libmachine: () Calling .GetMachineName
I0819 04:34:11.263638 13066 main.go:141] libmachine: (ha-356000) Calling .DriverName
I0819 04:34:11.292302 13066 out.go:177] * Using the hyperkit driver based on existing profile
I0819 04:34:11.334251 13066 start.go:297] selected driver: hyperkit
I0819 04:34:11.334279 13066 start.go:901] validating driver "hyperkit" against &{Name:ha-356000 KeepContext:false EmbedCerts:false MinikubeISO:https://storage.googleapis.com/minikube-builds/iso/19452/minikube-v1.33.1-1723740674-19452-amd64.iso KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.44-1723740748-19452@sha256:2211a6931895d2d502e957e9667096db10734a96767d670cb4dbffdd37397b0d Memory:2200 CPUs:2 DiskSize:20000 Driver:hyperkit HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{Kuber
netesVersion:v1.31.0 ClusterName:ha-356000 Namespace:default APIServerHAVIP:192.169.0.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:docker CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.169.0.38 Port:8443 KubernetesVersion:v1.31.0 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m02 IP:192.169.0.39 Port:8443 KubernetesVersion:v1.31.0 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m04 IP:192.169.0.41 Port:0 KubernetesVersion:v1.31.0 ContainerRuntime: ControlPlane:false Worker:true}] Addons:map[ambassador:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false helm-tiller:false inaccel:false ingress:fal
se ingress-dns:false inspektor-gadget:false istio:false istio-provisioner:false kong:false kubeflow:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-gluster:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s Mount:false MountString:/Users:/minikube-host Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMi
rror: DisableOptimizations:false DisableMetrics:false CustomQemuFirmwarePath: SocketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s}
I0819 04:34:11.334524 13066 start.go:912] status for hyperkit: {Installed:true Healthy:true Running:true NeedsImprovement:false Error:<nil> Reason: Fix: Doc: Version:}
I0819 04:34:11.334698 13066 install.go:52] acquiring lock: {Name:mk4023283b30b374c3f04c8805d539e68824c0b8 Clock:{} Delay:500ms Timeout:10m0s Cancel:<nil>}
I0819 04:34:11.334896 13066 install.go:117] Validating docker-machine-driver-hyperkit, PATH=/Users/jenkins/minikube-integration/19476-10399/.minikube/bin:/Users/jenkins/workspace/out/:/usr/bin:/bin:/usr/sbin:/sbin:/Users/jenkins/google-cloud-sdk/bin:/usr/local/bin/:/usr/local/go/bin/:/Users/jenkins/go/bin
I0819 04:34:11.344610 13066 install.go:137] /Users/jenkins/workspace/out/docker-machine-driver-hyperkit version is 1.33.1
I0819 04:34:11.348437 13066 install.go:79] stdout: /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0819 04:34:11.348459 13066 install.go:81] /Users/jenkins/workspace/out/docker-machine-driver-hyperkit looks good
I0819 04:34:11.351059 13066 start_flags.go:947] Waiting for all components: map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true]
I0819 04:34:11.351101 13066 cni.go:84] Creating CNI manager for ""
I0819 04:34:11.351109 13066 cni.go:136] multinode detected (3 nodes found), recommending kindnet
I0819 04:34:11.351207 13066 start.go:340] cluster config:
{Name:ha-356000 KeepContext:false EmbedCerts:false MinikubeISO:https://storage.googleapis.com/minikube-builds/iso/19452/minikube-v1.33.1-1723740674-19452-amd64.iso KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.44-1723740748-19452@sha256:2211a6931895d2d502e957e9667096db10734a96767d670cb4dbffdd37397b0d Memory:2200 CPUs:2 DiskSize:20000 Driver:hyperkit HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.31.0 ClusterName:ha-356000 Namespace:default APIServerHAVIP:192.16
9.0.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:docker CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.169.0.38 Port:8443 KubernetesVersion:v1.31.0 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m02 IP:192.169.0.39 Port:8443 KubernetesVersion:v1.31.0 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m04 IP:192.169.0.41 Port:0 KubernetesVersion:v1.31.0 ContainerRuntime:docker ControlPlane:false Worker:true}] Addons:map[ambassador:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false helm-tiller:false inaccel:false ingress:false ingress-dns:false inspektor-gadget:false istio:false istio-provisioner:fa
lse kong:false kubeflow:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-gluster:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s Mount:false MountString:/Users:/minikube-host Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOptimizations:false DisableMetrics:false CustomQemuFirmwarePath
: SocketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s}
I0819 04:34:11.351330 13066 iso.go:125] acquiring lock: {Name:mk52ecce0f4fb189477ea6707504fde804652b35 Clock:{} Delay:500ms Timeout:10m0s Cancel:<nil>}
I0819 04:34:11.393199 13066 out.go:177] * Starting "ha-356000" primary control-plane node in "ha-356000" cluster
I0819 04:34:11.414253 13066 preload.go:131] Checking if preload exists for k8s version v1.31.0 and runtime docker
I0819 04:34:11.414325 13066 preload.go:146] Found local preload: /Users/jenkins/minikube-integration/19476-10399/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.31.0-docker-overlay2-amd64.tar.lz4
I0819 04:34:11.414353 13066 cache.go:56] Caching tarball of preloaded images
I0819 04:34:11.414576 13066 preload.go:172] Found /Users/jenkins/minikube-integration/19476-10399/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.31.0-docker-overlay2-amd64.tar.lz4 in cache, skipping download
I0819 04:34:11.414596 13066 cache.go:59] Finished verifying existence of preloaded tar for v1.31.0 on docker
I0819 04:34:11.414760 13066 profile.go:143] Saving config to /Users/jenkins/minikube-integration/19476-10399/.minikube/profiles/ha-356000/config.json ...
I0819 04:34:11.415734 13066 start.go:360] acquireMachinesLock for ha-356000: {Name:mk90c69d5b026abe257122d2e8eeec3d8f4767c3 Clock:{} Delay:500ms Timeout:13m0s Cancel:<nil>}
I0819 04:34:11.415859 13066 start.go:364] duration metric: took 100.094µs to acquireMachinesLock for "ha-356000"
I0819 04:34:11.415898 13066 start.go:96] Skipping create...Using existing machine configuration
I0819 04:34:11.415917 13066 fix.go:54] fixHost starting:
I0819 04:34:11.416261 13066 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0819 04:34:11.416290 13066 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0819 04:34:11.425113 13066 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:60846
I0819 04:34:11.425455 13066 main.go:141] libmachine: () Calling .GetVersion
I0819 04:34:11.425788 13066 main.go:141] libmachine: Using API Version 1
I0819 04:34:11.425805 13066 main.go:141] libmachine: () Calling .SetConfigRaw
I0819 04:34:11.426021 13066 main.go:141] libmachine: () Calling .GetMachineName
I0819 04:34:11.426150 13066 main.go:141] libmachine: (ha-356000) Calling .DriverName
I0819 04:34:11.426245 13066 main.go:141] libmachine: (ha-356000) Calling .GetState
I0819 04:34:11.426393 13066 main.go:141] libmachine: (ha-356000) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I0819 04:34:11.426464 13066 main.go:141] libmachine: (ha-356000) DBG | hyperkit pid from json: 12970
I0819 04:34:11.427497 13066 main.go:141] libmachine: (ha-356000) DBG | hyperkit pid 12970 missing from process table
I0819 04:34:11.427564 13066 fix.go:112] recreateIfNeeded on ha-356000: state=Stopped err=<nil>
I0819 04:34:11.427610 13066 main.go:141] libmachine: (ha-356000) Calling .DriverName
W0819 04:34:11.427703 13066 fix.go:138] unexpected machine state, will restart: <nil>
I0819 04:34:11.470340 13066 out.go:177] * Restarting existing hyperkit VM for "ha-356000" ...
I0819 04:34:11.493521 13066 main.go:141] libmachine: (ha-356000) Calling .Start
I0819 04:34:11.493795 13066 main.go:141] libmachine: (ha-356000) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I0819 04:34:11.493838 13066 main.go:141] libmachine: (ha-356000) minikube might have been shutdown in an unclean way, the hyperkit pid file still exists: /Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000/hyperkit.pid
I0819 04:34:11.495770 13066 main.go:141] libmachine: (ha-356000) DBG | hyperkit pid 12970 missing from process table
I0819 04:34:11.495782 13066 main.go:141] libmachine: (ha-356000) DBG | pid 12970 is in state "Stopped"
I0819 04:34:11.495796 13066 main.go:141] libmachine: (ha-356000) DBG | Removing stale pid file /Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000/hyperkit.pid...
I0819 04:34:11.495989 13066 main.go:141] libmachine: (ha-356000) DBG | Using UUID fd1bcd67-0cbf-4f86-aa8e-4d9e80bb5466
I0819 04:34:11.608854 13066 main.go:141] libmachine: (ha-356000) DBG | Generated MAC aa:ca:df:56:43:33
I0819 04:34:11.608880 13066 main.go:141] libmachine: (ha-356000) DBG | Starting with cmdline: loglevel=3 console=ttyS0 console=tty0 noembed nomodeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-356000
I0819 04:34:11.609031 13066 main.go:141] libmachine: (ha-356000) DBG | 2024/08/19 04:34:11 DEBUG: hyperkit: Start &hyperkit.HyperKit{HyperKit:"/usr/local/bin/hyperkit", Argv0:"", StateDir:"/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000", VPNKitSock:"", VPNKitUUID:"", VPNKitPreferredIPv4:"", UUID:"fd1bcd67-0cbf-4f86-aa8e-4d9e80bb5466", Disks:[]hyperkit.Disk{(*hyperkit.RawDisk)(0xc0003b7bc0)}, ISOImages:[]string{"/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000/boot2docker.iso"}, VSock:false, VSockDir:"", VSockPorts:[]int(nil), VSockGuestCID:3, VMNet:true, Sockets9P:[]hyperkit.Socket9P(nil), Kernel:"/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000/bzimage", Initrd:"/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000/initrd", Bootrom:"", CPUs:2, Memory:2200, Console:1, Serials:[]hyperkit.Serial(nil), Pid:0, Arguments:[]string(nil), CmdLine:"", process:(*os.Process)(nil)}
I0819 04:34:11.609066 13066 main.go:141] libmachine: (ha-356000) DBG | 2024/08/19 04:34:11 DEBUG: hyperkit: check &hyperkit.HyperKit{HyperKit:"/usr/local/bin/hyperkit", Argv0:"", StateDir:"/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000", VPNKitSock:"", VPNKitUUID:"", VPNKitPreferredIPv4:"", UUID:"fd1bcd67-0cbf-4f86-aa8e-4d9e80bb5466", Disks:[]hyperkit.Disk{(*hyperkit.RawDisk)(0xc0003b7bc0)}, ISOImages:[]string{"/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000/boot2docker.iso"}, VSock:false, VSockDir:"", VSockPorts:[]int(nil), VSockGuestCID:3, VMNet:true, Sockets9P:[]hyperkit.Socket9P(nil), Kernel:"/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000/bzimage", Initrd:"/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000/initrd", Bootrom:"", CPUs:2, Memory:2200, Console:1, Serials:[]hyperkit.Serial(nil), Pid:0, Arguments:[]string(nil), CmdLine:"", process:(*os.Process)(nil)}
I0819 04:34:11.609096 13066 main.go:141] libmachine: (ha-356000) DBG | 2024/08/19 04:34:11 DEBUG: hyperkit: Arguments: []string{"-A", "-u", "-F", "/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000/hyperkit.pid", "-c", "2", "-m", "2200M", "-s", "0:0,hostbridge", "-s", "31,lpc", "-s", "1:0,virtio-net", "-U", "fd1bcd67-0cbf-4f86-aa8e-4d9e80bb5466", "-s", "2:0,virtio-blk,/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000/ha-356000.rawdisk", "-s", "3,ahci-cd,/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000/boot2docker.iso", "-s", "4,virtio-rnd", "-l", "com1,autopty=/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000/tty,log=/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000/console-ring", "-f", "kexec,/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000/bzimage,/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000/initrd,earlyp
rintk=serial loglevel=3 console=ttyS0 console=tty0 noembed nomodeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-356000"}
I0819 04:34:11.609122 13066 main.go:141] libmachine: (ha-356000) DBG | 2024/08/19 04:34:11 DEBUG: hyperkit: CmdLine: "/usr/local/bin/hyperkit -A -u -F /Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000/hyperkit.pid -c 2 -m 2200M -s 0:0,hostbridge -s 31,lpc -s 1:0,virtio-net -U fd1bcd67-0cbf-4f86-aa8e-4d9e80bb5466 -s 2:0,virtio-blk,/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000/ha-356000.rawdisk -s 3,ahci-cd,/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000/boot2docker.iso -s 4,virtio-rnd -l com1,autopty=/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000/tty,log=/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000/console-ring -f kexec,/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000/bzimage,/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000/initrd,earlyprintk=serial loglevel=3 console=ttyS0 console=tty0 noembed nom
odeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-356000"
I0819 04:34:11.609156 13066 main.go:141] libmachine: (ha-356000) DBG | 2024/08/19 04:34:11 DEBUG: hyperkit: Redirecting stdout/stderr to logger
I0819 04:34:11.610580 13066 main.go:141] libmachine: (ha-356000) DBG | 2024/08/19 04:34:11 DEBUG: hyperkit: Pid is 13079
I0819 04:34:11.611062 13066 main.go:141] libmachine: (ha-356000) DBG | Attempt 0
I0819 04:34:11.611076 13066 main.go:141] libmachine: (ha-356000) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I0819 04:34:11.611152 13066 main.go:141] libmachine: (ha-356000) DBG | hyperkit pid from json: 13079
I0819 04:34:11.612930 13066 main.go:141] libmachine: (ha-356000) DBG | Searching for aa:ca:df:56:43:33 in /var/db/dhcpd_leases ...
I0819 04:34:11.613030 13066 main.go:141] libmachine: (ha-356000) DBG | Found 40 entries in /var/db/dhcpd_leases!
I0819 04:34:11.613054 13066 main.go:141] libmachine: (ha-356000) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.40 HWAddress:c6:be:b:f7:f5:ce ID:1,c6:be:b:f7:f5:ce Lease:0x66c32d97}
I0819 04:34:11.613088 13066 main.go:141] libmachine: (ha-356000) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.41 HWAddress:6e:45:87:58:5c:30 ID:1,6e:45:87:58:5c:30 Lease:0x66c47f02}
I0819 04:34:11.613113 13066 main.go:141] libmachine: (ha-356000) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.39 HWAddress:d6:df:55:3d:f:ed ID:1,d6:df:55:3d:f:ed Lease:0x66c47e95}
I0819 04:34:11.613128 13066 main.go:141] libmachine: (ha-356000) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.38 HWAddress:aa:ca:df:56:43:33 ID:1,aa:ca:df:56:43:33 Lease:0x66c47e83}
I0819 04:34:11.613136 13066 main.go:141] libmachine: (ha-356000) DBG | Found match: aa:ca:df:56:43:33
I0819 04:34:11.613143 13066 main.go:141] libmachine: (ha-356000) DBG | IP: 192.169.0.38
I0819 04:34:11.613167 13066 main.go:141] libmachine: (ha-356000) Calling .GetConfigRaw
I0819 04:34:11.613806 13066 main.go:141] libmachine: (ha-356000) Calling .GetIP
I0819 04:34:11.613979 13066 profile.go:143] Saving config to /Users/jenkins/minikube-integration/19476-10399/.minikube/profiles/ha-356000/config.json ...
I0819 04:34:11.614401 13066 machine.go:93] provisionDockerMachine start ...
I0819 04:34:11.614412 13066 main.go:141] libmachine: (ha-356000) Calling .DriverName
I0819 04:34:11.614561 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHHostname
I0819 04:34:11.614667 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHPort
I0819 04:34:11.614761 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHKeyPath
I0819 04:34:11.614881 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHKeyPath
I0819 04:34:11.614974 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHUsername
I0819 04:34:11.615096 13066 main.go:141] libmachine: Using SSH client type: native
I0819 04:34:11.615289 13066 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xa900ea0] 0xa903c00 <nil> [] 0s} 192.169.0.38 22 <nil> <nil>}
I0819 04:34:11.615297 13066 main.go:141] libmachine: About to run SSH command:
hostname
I0819 04:34:11.618877 13066 main.go:141] libmachine: (ha-356000) DBG | 2024/08/19 04:34:11 INFO : hyperkit: stderr: Using fd 5 for I/O notifications
I0819 04:34:11.672399 13066 main.go:141] libmachine: (ha-356000) DBG | 2024/08/19 04:34:11 INFO : hyperkit: stderr: /Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000/boot2docker.iso: fcntl(F_PUNCHHOLE) Operation not permitted: block device will not support TRIM/DISCARD
I0819 04:34:11.673084 13066 main.go:141] libmachine: (ha-356000) DBG | 2024/08/19 04:34:11 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 2 bit: 22 unspecified don't care: bit is 0
I0819 04:34:11.673108 13066 main.go:141] libmachine: (ha-356000) DBG | 2024/08/19 04:34:11 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 12 unspecified don't care: bit is 0
I0819 04:34:11.673142 13066 main.go:141] libmachine: (ha-356000) DBG | 2024/08/19 04:34:11 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 20 unspecified don't care: bit is 0
I0819 04:34:11.673161 13066 main.go:141] libmachine: (ha-356000) DBG | 2024/08/19 04:34:11 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 3 bit: 13 unspecified don't care: bit is 0
I0819 04:34:12.059865 13066 main.go:141] libmachine: (ha-356000) DBG | 2024/08/19 04:34:12 INFO : hyperkit: stderr: rdmsr to register 0x3a on vcpu 0
I0819 04:34:12.059882 13066 main.go:141] libmachine: (ha-356000) DBG | 2024/08/19 04:34:12 INFO : hyperkit: stderr: rdmsr to register 0x140 on vcpu 0
I0819 04:34:12.174829 13066 main.go:141] libmachine: (ha-356000) DBG | 2024/08/19 04:34:12 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 2 bit: 22 unspecified don't care: bit is 0
I0819 04:34:12.174850 13066 main.go:141] libmachine: (ha-356000) DBG | 2024/08/19 04:34:12 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 12 unspecified don't care: bit is 0
I0819 04:34:12.174882 13066 main.go:141] libmachine: (ha-356000) DBG | 2024/08/19 04:34:12 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 20 unspecified don't care: bit is 0
I0819 04:34:12.174902 13066 main.go:141] libmachine: (ha-356000) DBG | 2024/08/19 04:34:12 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 3 bit: 13 unspecified don't care: bit is 0
I0819 04:34:12.175709 13066 main.go:141] libmachine: (ha-356000) DBG | 2024/08/19 04:34:12 INFO : hyperkit: stderr: rdmsr to register 0x3a on vcpu 1
I0819 04:34:12.175719 13066 main.go:141] libmachine: (ha-356000) DBG | 2024/08/19 04:34:12 INFO : hyperkit: stderr: rdmsr to register 0x140 on vcpu 1
I0819 04:34:17.763459 13066 main.go:141] libmachine: (ha-356000) DBG | 2024/08/19 04:34:17 INFO : hyperkit: stderr: rdmsr to register 0x64d on vcpu 1
I0819 04:34:17.763492 13066 main.go:141] libmachine: (ha-356000) DBG | 2024/08/19 04:34:17 INFO : hyperkit: stderr: rdmsr to register 0x64e on vcpu 1
I0819 04:34:17.763505 13066 main.go:141] libmachine: (ha-356000) DBG | 2024/08/19 04:34:17 INFO : hyperkit: stderr: rdmsr to register 0x34 on vcpu 1
I0819 04:34:17.787984 13066 main.go:141] libmachine: (ha-356000) DBG | 2024/08/19 04:34:17 INFO : hyperkit: stderr: rdmsr to register 0xc0011029 on vcpu 1
I0819 04:34:46.681872 13066 main.go:141] libmachine: SSH cmd err, output: <nil>: minikube
I0819 04:34:46.681885 13066 main.go:141] libmachine: (ha-356000) Calling .GetMachineName
I0819 04:34:46.682015 13066 buildroot.go:166] provisioning hostname "ha-356000"
I0819 04:34:46.682027 13066 main.go:141] libmachine: (ha-356000) Calling .GetMachineName
I0819 04:34:46.682143 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHHostname
I0819 04:34:46.682233 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHPort
I0819 04:34:46.682316 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHKeyPath
I0819 04:34:46.682402 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHKeyPath
I0819 04:34:46.682496 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHUsername
I0819 04:34:46.682630 13066 main.go:141] libmachine: Using SSH client type: native
I0819 04:34:46.682777 13066 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xa900ea0] 0xa903c00 <nil> [] 0s} 192.169.0.38 22 <nil> <nil>}
I0819 04:34:46.682790 13066 main.go:141] libmachine: About to run SSH command:
sudo hostname ha-356000 && echo "ha-356000" | sudo tee /etc/hostname
I0819 04:34:46.746059 13066 main.go:141] libmachine: SSH cmd err, output: <nil>: ha-356000
I0819 04:34:46.746077 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHHostname
I0819 04:34:46.746217 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHPort
I0819 04:34:46.746324 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHKeyPath
I0819 04:34:46.746423 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHKeyPath
I0819 04:34:46.746528 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHUsername
I0819 04:34:46.746680 13066 main.go:141] libmachine: Using SSH client type: native
I0819 04:34:46.746835 13066 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xa900ea0] 0xa903c00 <nil> [] 0s} 192.169.0.38 22 <nil> <nil>}
I0819 04:34:46.746847 13066 main.go:141] libmachine: About to run SSH command:
if ! grep -xq '.*\sha-356000' /etc/hosts; then
if grep -xq '127.0.1.1\s.*' /etc/hosts; then
sudo sed -i 's/^127.0.1.1\s.*/127.0.1.1 ha-356000/g' /etc/hosts;
else
echo '127.0.1.1 ha-356000' | sudo tee -a /etc/hosts;
fi
fi
I0819 04:34:46.806873 13066 main.go:141] libmachine: SSH cmd err, output: <nil>:
I0819 04:34:46.806893 13066 buildroot.go:172] set auth options {CertDir:/Users/jenkins/minikube-integration/19476-10399/.minikube CaCertPath:/Users/jenkins/minikube-integration/19476-10399/.minikube/certs/ca.pem CaPrivateKeyPath:/Users/jenkins/minikube-integration/19476-10399/.minikube/certs/ca-key.pem CaCertRemotePath:/etc/docker/ca.pem ServerCertPath:/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/server.pem ServerKeyPath:/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/server-key.pem ClientKeyPath:/Users/jenkins/minikube-integration/19476-10399/.minikube/certs/key.pem ServerCertRemotePath:/etc/docker/server.pem ServerKeyRemotePath:/etc/docker/server-key.pem ClientCertPath:/Users/jenkins/minikube-integration/19476-10399/.minikube/certs/cert.pem ServerCertSANs:[] StorePath:/Users/jenkins/minikube-integration/19476-10399/.minikube}
I0819 04:34:46.806915 13066 buildroot.go:174] setting up certificates
I0819 04:34:46.806935 13066 provision.go:84] configureAuth start
I0819 04:34:46.806955 13066 main.go:141] libmachine: (ha-356000) Calling .GetMachineName
I0819 04:34:46.807092 13066 main.go:141] libmachine: (ha-356000) Calling .GetIP
I0819 04:34:46.807181 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHHostname
I0819 04:34:46.807283 13066 provision.go:143] copyHostCerts
I0819 04:34:46.807315 13066 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19476-10399/.minikube/certs/ca.pem -> /Users/jenkins/minikube-integration/19476-10399/.minikube/ca.pem
I0819 04:34:46.807384 13066 exec_runner.go:144] found /Users/jenkins/minikube-integration/19476-10399/.minikube/ca.pem, removing ...
I0819 04:34:46.807392 13066 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19476-10399/.minikube/ca.pem
I0819 04:34:46.807579 13066 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19476-10399/.minikube/certs/ca.pem --> /Users/jenkins/minikube-integration/19476-10399/.minikube/ca.pem (1078 bytes)
I0819 04:34:46.807804 13066 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19476-10399/.minikube/certs/cert.pem -> /Users/jenkins/minikube-integration/19476-10399/.minikube/cert.pem
I0819 04:34:46.807844 13066 exec_runner.go:144] found /Users/jenkins/minikube-integration/19476-10399/.minikube/cert.pem, removing ...
I0819 04:34:46.807849 13066 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19476-10399/.minikube/cert.pem
I0819 04:34:46.807932 13066 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19476-10399/.minikube/certs/cert.pem --> /Users/jenkins/minikube-integration/19476-10399/.minikube/cert.pem (1123 bytes)
I0819 04:34:46.808089 13066 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19476-10399/.minikube/certs/key.pem -> /Users/jenkins/minikube-integration/19476-10399/.minikube/key.pem
I0819 04:34:46.808129 13066 exec_runner.go:144] found /Users/jenkins/minikube-integration/19476-10399/.minikube/key.pem, removing ...
I0819 04:34:46.808133 13066 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19476-10399/.minikube/key.pem
I0819 04:34:46.808209 13066 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19476-10399/.minikube/certs/key.pem --> /Users/jenkins/minikube-integration/19476-10399/.minikube/key.pem (1675 bytes)
I0819 04:34:46.808369 13066 provision.go:117] generating server cert: /Users/jenkins/minikube-integration/19476-10399/.minikube/machines/server.pem ca-key=/Users/jenkins/minikube-integration/19476-10399/.minikube/certs/ca.pem private-key=/Users/jenkins/minikube-integration/19476-10399/.minikube/certs/ca-key.pem org=jenkins.ha-356000 san=[127.0.0.1 192.169.0.38 ha-356000 localhost minikube]
I0819 04:34:47.054969 13066 provision.go:177] copyRemoteCerts
I0819 04:34:47.055059 13066 ssh_runner.go:195] Run: sudo mkdir -p /etc/docker /etc/docker /etc/docker
I0819 04:34:47.055098 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHHostname
I0819 04:34:47.055262 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHPort
I0819 04:34:47.055364 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHKeyPath
I0819 04:34:47.055545 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHUsername
I0819 04:34:47.055721 13066 sshutil.go:53] new ssh client: &{IP:192.169.0.38 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000/id_rsa Username:docker}
I0819 04:34:47.091847 13066 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19476-10399/.minikube/certs/ca.pem -> /etc/docker/ca.pem
I0819 04:34:47.091938 13066 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19476-10399/.minikube/certs/ca.pem --> /etc/docker/ca.pem (1078 bytes)
I0819 04:34:47.111425 13066 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19476-10399/.minikube/machines/server.pem -> /etc/docker/server.pem
I0819 04:34:47.111484 13066 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19476-10399/.minikube/machines/server.pem --> /etc/docker/server.pem (1196 bytes)
I0819 04:34:47.131474 13066 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19476-10399/.minikube/machines/server-key.pem -> /etc/docker/server-key.pem
I0819 04:34:47.131534 13066 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19476-10399/.minikube/machines/server-key.pem --> /etc/docker/server-key.pem (1679 bytes)
I0819 04:34:47.151115 13066 provision.go:87] duration metric: took 344.154728ms to configureAuth
I0819 04:34:47.151128 13066 buildroot.go:189] setting minikube options for container-runtime
I0819 04:34:47.151298 13066 config.go:182] Loaded profile config "ha-356000": Driver=hyperkit, ContainerRuntime=docker, KubernetesVersion=v1.31.0
I0819 04:34:47.151312 13066 main.go:141] libmachine: (ha-356000) Calling .DriverName
I0819 04:34:47.151454 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHHostname
I0819 04:34:47.151540 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHPort
I0819 04:34:47.151620 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHKeyPath
I0819 04:34:47.151702 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHKeyPath
I0819 04:34:47.151791 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHUsername
I0819 04:34:47.151911 13066 main.go:141] libmachine: Using SSH client type: native
I0819 04:34:47.152038 13066 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xa900ea0] 0xa903c00 <nil> [] 0s} 192.169.0.38 22 <nil> <nil>}
I0819 04:34:47.152046 13066 main.go:141] libmachine: About to run SSH command:
df --output=fstype / | tail -n 1
I0819 04:34:47.203430 13066 main.go:141] libmachine: SSH cmd err, output: <nil>: tmpfs
I0819 04:34:47.203449 13066 buildroot.go:70] root file system type: tmpfs
I0819 04:34:47.203528 13066 provision.go:314] Updating docker unit: /lib/systemd/system/docker.service ...
I0819 04:34:47.203542 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHHostname
I0819 04:34:47.203684 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHPort
I0819 04:34:47.203781 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHKeyPath
I0819 04:34:47.203877 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHKeyPath
I0819 04:34:47.203965 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHUsername
I0819 04:34:47.204095 13066 main.go:141] libmachine: Using SSH client type: native
I0819 04:34:47.204235 13066 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xa900ea0] 0xa903c00 <nil> [] 0s} 192.169.0.38 22 <nil> <nil>}
I0819 04:34:47.204316 13066 main.go:141] libmachine: About to run SSH command:
sudo mkdir -p /lib/systemd/system && printf %s "[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network.target minikube-automount.service docker.socket
Requires= minikube-automount.service docker.socket
StartLimitBurst=3
StartLimitIntervalSec=60
[Service]
Type=notify
Restart=on-failure
# This file is a systemd drop-in unit that inherits from the base dockerd configuration.
# The base configuration already specifies an 'ExecStart=...' command. The first directive
# here is to clear out that command inherited from the base configuration. Without this,
# the command from the base configuration and the command specified here are treated as
# a sequence of commands, which is not the desired behavior, nor is it valid -- systemd
# will catch this invalid input and refuse to start the service with an error like:
# Service has more than one ExecStart= setting, which is only allowed for Type=oneshot services.
# NOTE: default-ulimit=nofile is set to an arbitrary number for consistency with other
# container runtimes. If left unlimited, it may result in OOM issues with MySQL.
ExecStart=
ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --default-ulimit=nofile=1048576:1048576 --tlsverify --tlscacert /etc/docker/ca.pem --tlscert /etc/docker/server.pem --tlskey /etc/docker/server-key.pem --label provider=hyperkit --insecure-registry 10.96.0.0/12
ExecReload=/bin/kill -s HUP \$MAINPID
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
TimeoutStartSec=0
# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes
# kill only the docker process, not all processes in the cgroup
KillMode=process
[Install]
WantedBy=multi-user.target
" | sudo tee /lib/systemd/system/docker.service.new
I0819 04:34:47.266306 13066 main.go:141] libmachine: SSH cmd err, output: <nil>: [Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network.target minikube-automount.service docker.socket
Requires= minikube-automount.service docker.socket
StartLimitBurst=3
StartLimitIntervalSec=60
[Service]
Type=notify
Restart=on-failure
# This file is a systemd drop-in unit that inherits from the base dockerd configuration.
# The base configuration already specifies an 'ExecStart=...' command. The first directive
# here is to clear out that command inherited from the base configuration. Without this,
# the command from the base configuration and the command specified here are treated as
# a sequence of commands, which is not the desired behavior, nor is it valid -- systemd
# will catch this invalid input and refuse to start the service with an error like:
# Service has more than one ExecStart= setting, which is only allowed for Type=oneshot services.
# NOTE: default-ulimit=nofile is set to an arbitrary number for consistency with other
# container runtimes. If left unlimited, it may result in OOM issues with MySQL.
ExecStart=
ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --default-ulimit=nofile=1048576:1048576 --tlsverify --tlscacert /etc/docker/ca.pem --tlscert /etc/docker/server.pem --tlskey /etc/docker/server-key.pem --label provider=hyperkit --insecure-registry 10.96.0.0/12
ExecReload=/bin/kill -s HUP $MAINPID
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
TimeoutStartSec=0
# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes
# kill only the docker process, not all processes in the cgroup
KillMode=process
[Install]
WantedBy=multi-user.target
I0819 04:34:47.266328 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHHostname
I0819 04:34:47.266466 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHPort
I0819 04:34:47.266571 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHKeyPath
I0819 04:34:47.266661 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHKeyPath
I0819 04:34:47.266748 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHUsername
I0819 04:34:47.266881 13066 main.go:141] libmachine: Using SSH client type: native
I0819 04:34:47.267018 13066 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xa900ea0] 0xa903c00 <nil> [] 0s} 192.169.0.38 22 <nil> <nil>}
I0819 04:34:47.267030 13066 main.go:141] libmachine: About to run SSH command:
sudo diff -u /lib/systemd/system/docker.service /lib/systemd/system/docker.service.new || { sudo mv /lib/systemd/system/docker.service.new /lib/systemd/system/docker.service; sudo systemctl -f daemon-reload && sudo systemctl -f enable docker && sudo systemctl -f restart docker; }
I0819 04:34:48.954677 13066 main.go:141] libmachine: SSH cmd err, output: <nil>: diff: can't stat '/lib/systemd/system/docker.service': No such file or directory
Created symlink /etc/systemd/system/multi-user.target.wants/docker.service → /usr/lib/systemd/system/docker.service.
I0819 04:34:48.954693 13066 machine.go:96] duration metric: took 37.339379659s to provisionDockerMachine
I0819 04:34:48.954705 13066 start.go:293] postStartSetup for "ha-356000" (driver="hyperkit")
I0819 04:34:48.954714 13066 start.go:322] creating required directories: [/etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs]
I0819 04:34:48.954723 13066 main.go:141] libmachine: (ha-356000) Calling .DriverName
I0819 04:34:48.954900 13066 ssh_runner.go:195] Run: sudo mkdir -p /etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs
I0819 04:34:48.954926 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHHostname
I0819 04:34:48.955027 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHPort
I0819 04:34:48.955115 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHKeyPath
I0819 04:34:48.955209 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHUsername
I0819 04:34:48.955293 13066 sshutil.go:53] new ssh client: &{IP:192.169.0.38 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000/id_rsa Username:docker}
I0819 04:34:48.989262 13066 ssh_runner.go:195] Run: cat /etc/os-release
I0819 04:34:48.992376 13066 info.go:137] Remote host: Buildroot 2023.02.9
I0819 04:34:48.992388 13066 filesync.go:126] Scanning /Users/jenkins/minikube-integration/19476-10399/.minikube/addons for local assets ...
I0819 04:34:48.992488 13066 filesync.go:126] Scanning /Users/jenkins/minikube-integration/19476-10399/.minikube/files for local assets ...
I0819 04:34:48.992673 13066 filesync.go:149] local asset: /Users/jenkins/minikube-integration/19476-10399/.minikube/files/etc/ssl/certs/109402.pem -> 109402.pem in /etc/ssl/certs
I0819 04:34:48.992680 13066 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19476-10399/.minikube/files/etc/ssl/certs/109402.pem -> /etc/ssl/certs/109402.pem
I0819 04:34:48.992879 13066 ssh_runner.go:195] Run: sudo mkdir -p /etc/ssl/certs
I0819 04:34:49.000815 13066 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19476-10399/.minikube/files/etc/ssl/certs/109402.pem --> /etc/ssl/certs/109402.pem (1708 bytes)
I0819 04:34:49.019798 13066 start.go:296] duration metric: took 65.083423ms for postStartSetup
I0819 04:34:49.019820 13066 main.go:141] libmachine: (ha-356000) Calling .DriverName
I0819 04:34:49.020013 13066 ssh_runner.go:195] Run: sudo ls --almost-all -1 /var/lib/minikube/backup
I0819 04:34:49.020026 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHHostname
I0819 04:34:49.020119 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHPort
I0819 04:34:49.020201 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHKeyPath
I0819 04:34:49.020284 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHUsername
I0819 04:34:49.020368 13066 sshutil.go:53] new ssh client: &{IP:192.169.0.38 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000/id_rsa Username:docker}
I0819 04:34:49.052920 13066 machine.go:197] restoring vm config from /var/lib/minikube/backup: [etc]
I0819 04:34:49.052971 13066 ssh_runner.go:195] Run: sudo rsync --archive --update /var/lib/minikube/backup/etc /
I0819 04:34:49.106237 13066 fix.go:56] duration metric: took 37.689411977s for fixHost
I0819 04:34:49.106260 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHHostname
I0819 04:34:49.106404 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHPort
I0819 04:34:49.106498 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHKeyPath
I0819 04:34:49.106578 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHKeyPath
I0819 04:34:49.106672 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHUsername
I0819 04:34:49.106806 13066 main.go:141] libmachine: Using SSH client type: native
I0819 04:34:49.106952 13066 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xa900ea0] 0xa903c00 <nil> [] 0s} 192.169.0.38 22 <nil> <nil>}
I0819 04:34:49.106960 13066 main.go:141] libmachine: About to run SSH command:
date +%s.%N
I0819 04:34:49.158055 13066 main.go:141] libmachine: SSH cmd err, output: <nil>: 1724067289.089321399
I0819 04:34:49.158070 13066 fix.go:216] guest clock: 1724067289.089321399
I0819 04:34:49.158075 13066 fix.go:229] Guest: 2024-08-19 04:34:49.089321399 -0700 PDT Remote: 2024-08-19 04:34:49.10625 -0700 PDT m=+38.136429560 (delta=-16.928601ms)
I0819 04:34:49.158090 13066 fix.go:200] guest clock delta is within tolerance: -16.928601ms
I0819 04:34:49.158096 13066 start.go:83] releasing machines lock for "ha-356000", held for 37.741312167s
I0819 04:34:49.158115 13066 main.go:141] libmachine: (ha-356000) Calling .DriverName
I0819 04:34:49.158254 13066 main.go:141] libmachine: (ha-356000) Calling .GetIP
I0819 04:34:49.158351 13066 main.go:141] libmachine: (ha-356000) Calling .DriverName
I0819 04:34:49.158648 13066 main.go:141] libmachine: (ha-356000) Calling .DriverName
I0819 04:34:49.158759 13066 main.go:141] libmachine: (ha-356000) Calling .DriverName
I0819 04:34:49.158841 13066 ssh_runner.go:195] Run: curl -sS -m 2 https://registry.k8s.io/
I0819 04:34:49.158876 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHHostname
I0819 04:34:49.158911 13066 ssh_runner.go:195] Run: cat /version.json
I0819 04:34:49.158922 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHHostname
I0819 04:34:49.158972 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHPort
I0819 04:34:49.159015 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHPort
I0819 04:34:49.159049 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHKeyPath
I0819 04:34:49.159094 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHKeyPath
I0819 04:34:49.159110 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHUsername
I0819 04:34:49.159176 13066 sshutil.go:53] new ssh client: &{IP:192.169.0.38 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000/id_rsa Username:docker}
I0819 04:34:49.159196 13066 main.go:141] libmachine: (ha-356000) Calling .GetSSHUsername
I0819 04:34:49.159269 13066 sshutil.go:53] new ssh client: &{IP:192.169.0.38 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000/id_rsa Username:docker}
I0819 04:34:49.187266 13066 ssh_runner.go:195] Run: systemctl --version
I0819 04:34:49.233417 13066 ssh_runner.go:195] Run: sh -c "stat /etc/cni/net.d/*loopback.conf*"
W0819 04:34:49.238362 13066 cni.go:209] loopback cni configuration skipped: "/etc/cni/net.d/*loopback.conf*" not found
I0819 04:34:49.238398 13066 ssh_runner.go:195] Run: sudo find /etc/cni/net.d -maxdepth 1 -type f ( ( -name *bridge* -or -name *podman* ) -and -not -name *.mk_disabled ) -printf "%p, " -exec sh -c "sudo mv {} {}.mk_disabled" ;
I0819 04:34:49.252658 13066 cni.go:262] disabled [/etc/cni/net.d/87-podman-bridge.conflist] bridge cni config(s)
I0819 04:34:49.252669 13066 start.go:495] detecting cgroup driver to use...
I0819 04:34:49.252771 13066 ssh_runner.go:195] Run: /bin/bash -c "sudo mkdir -p /etc && printf %s "runtime-endpoint: unix:///run/containerd/containerd.sock
" | sudo tee /etc/crictl.yaml"
I0819 04:34:49.269794 13066 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)sandbox_image = .*$|\1sandbox_image = "registry.k8s.io/pause:3.10"|' /etc/containerd/config.toml"
I0819 04:34:49.278790 13066 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)restrict_oom_score_adj = .*$|\1restrict_oom_score_adj = false|' /etc/containerd/config.toml"
I0819 04:34:49.286910 13066 containerd.go:146] configuring containerd to use "cgroupfs" as cgroup driver...
I0819 04:34:49.286947 13066 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)SystemdCgroup = .*$|\1SystemdCgroup = false|g' /etc/containerd/config.toml"
I0819 04:34:49.295052 13066 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|"io.containerd.runtime.v1.linux"|"io.containerd.runc.v2"|g' /etc/containerd/config.toml"
I0819 04:34:49.303200 13066 ssh_runner.go:195] Run: sh -c "sudo sed -i '/systemd_cgroup/d' /etc/containerd/config.toml"
I0819 04:34:49.311278 13066 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|"io.containerd.runc.v1"|"io.containerd.runc.v2"|g' /etc/containerd/config.toml"
I0819 04:34:49.319565 13066 ssh_runner.go:195] Run: sh -c "sudo rm -rf /etc/cni/net.mk"
I0819 04:34:49.327849 13066 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)conf_dir = .*$|\1conf_dir = "/etc/cni/net.d"|g' /etc/containerd/config.toml"
I0819 04:34:49.336090 13066 ssh_runner.go:195] Run: sh -c "sudo sed -i '/^ *enable_unprivileged_ports = .*/d' /etc/containerd/config.toml"
I0819 04:34:49.344364 13066 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)\[plugins."io.containerd.grpc.v1.cri"\]|&\n\1 enable_unprivileged_ports = true|' /etc/containerd/config.toml"
I0819 04:34:49.352675 13066 ssh_runner.go:195] Run: sudo sysctl net.bridge.bridge-nf-call-iptables
I0819 04:34:49.360032 13066 ssh_runner.go:195] Run: sudo sh -c "echo 1 > /proc/sys/net/ipv4/ip_forward"
I0819 04:34:49.367513 13066 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I0819 04:34:49.460847 13066 ssh_runner.go:195] Run: sudo systemctl restart containerd
I0819 04:34:49.480319 13066 start.go:495] detecting cgroup driver to use...
I0819 04:34:49.480397 13066 ssh_runner.go:195] Run: sudo systemctl cat docker.service
I0819 04:34:49.499139 13066 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service containerd
I0819 04:34:49.511544 13066 ssh_runner.go:195] Run: sudo systemctl stop -f containerd
I0819 04:34:49.528832 13066 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service containerd
I0819 04:34:49.539148 13066 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service crio
I0819 04:34:49.549199 13066 ssh_runner.go:195] Run: sudo systemctl stop -f crio
I0819 04:34:49.570183 13066 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service crio
I0819 04:34:49.580417 13066 ssh_runner.go:195] Run: /bin/bash -c "sudo mkdir -p /etc && printf %s "runtime-endpoint: unix:///var/run/cri-dockerd.sock
" | sudo tee /etc/crictl.yaml"
I0819 04:34:49.597405 13066 ssh_runner.go:195] Run: which cri-dockerd
I0819 04:34:49.600367 13066 ssh_runner.go:195] Run: sudo mkdir -p /etc/systemd/system/cri-docker.service.d
I0819 04:34:49.607460 13066 ssh_runner.go:362] scp memory --> /etc/systemd/system/cri-docker.service.d/10-cni.conf (190 bytes)
I0819 04:34:49.620743 13066 ssh_runner.go:195] Run: sudo systemctl unmask docker.service
I0819 04:34:49.714669 13066 ssh_runner.go:195] Run: sudo systemctl enable docker.socket
I0819 04:34:49.810300 13066 docker.go:574] configuring docker to use "cgroupfs" as cgroup driver...
I0819 04:34:49.810372 13066 ssh_runner.go:362] scp memory --> /etc/docker/daemon.json (130 bytes)
I0819 04:34:49.830236 13066 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I0819 04:34:49.931137 13066 ssh_runner.go:195] Run: sudo systemctl restart docker
I0819 04:34:52.270416 13066 ssh_runner.go:235] Completed: sudo systemctl restart docker: (2.339202169s)
I0819 04:34:52.270482 13066 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service cri-docker.socket
I0819 04:34:52.280954 13066 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service cri-docker.service
I0819 04:34:52.290974 13066 ssh_runner.go:195] Run: sudo systemctl unmask cri-docker.socket
I0819 04:34:52.383158 13066 ssh_runner.go:195] Run: sudo systemctl enable cri-docker.socket
I0819 04:34:52.483475 13066 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I0819 04:34:52.585444 13066 ssh_runner.go:195] Run: sudo systemctl restart cri-docker.socket
I0819 04:34:52.598156 13066 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service cri-docker.service
I0819 04:34:52.608339 13066 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I0819 04:34:52.704734 13066 ssh_runner.go:195] Run: sudo systemctl restart cri-docker.service
I0819 04:34:52.770110 13066 start.go:542] Will wait 60s for socket path /var/run/cri-dockerd.sock
I0819 04:34:52.770184 13066 ssh_runner.go:195] Run: stat /var/run/cri-dockerd.sock
I0819 04:34:52.774538 13066 start.go:563] Will wait 60s for crictl version
I0819 04:34:52.774592 13066 ssh_runner.go:195] Run: which crictl
I0819 04:34:52.777881 13066 ssh_runner.go:195] Run: sudo /usr/bin/crictl version
I0819 04:34:52.802866 13066 start.go:579] Version: 0.1.0
RuntimeName: docker
RuntimeVersion: 27.1.2
RuntimeApiVersion: v1
I0819 04:34:52.802935 13066 ssh_runner.go:195] Run: docker version --format {{.Server.Version}}
I0819 04:34:52.820162 13066 ssh_runner.go:195] Run: docker version --format {{.Server.Version}}
I0819 04:34:52.861230 13066 out.go:235] * Preparing Kubernetes v1.31.0 on Docker 27.1.2 ...
I0819 04:34:52.861279 13066 main.go:141] libmachine: (ha-356000) Calling .GetIP
I0819 04:34:52.861664 13066 ssh_runner.go:195] Run: grep 192.169.0.1 host.minikube.internal$ /etc/hosts
I0819 04:34:52.866368 13066 ssh_runner.go:195] Run: /bin/bash -c "{ grep -v $'\thost.minikube.internal$' "/etc/hosts"; echo "192.169.0.1 host.minikube.internal"; } > /tmp/h.$$; sudo cp /tmp/h.$$ "/etc/hosts""
I0819 04:34:52.876609 13066 kubeadm.go:883] updating cluster {Name:ha-356000 KeepContext:false EmbedCerts:false MinikubeISO:https://storage.googleapis.com/minikube-builds/iso/19452/minikube-v1.33.1-1723740674-19452-amd64.iso KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.44-1723740748-19452@sha256:2211a6931895d2d502e957e9667096db10734a96767d670cb4dbffdd37397b0d Memory:2200 CPUs:2 DiskSize:20000 Driver:hyperkit HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.31.
0 ClusterName:ha-356000 Namespace:default APIServerHAVIP:192.169.0.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:docker CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.169.0.38 Port:8443 KubernetesVersion:v1.31.0 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m02 IP:192.169.0.39 Port:8443 KubernetesVersion:v1.31.0 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m04 IP:192.169.0.41 Port:0 KubernetesVersion:v1.31.0 ContainerRuntime:docker ControlPlane:false Worker:true}] Addons:map[ambassador:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false helm-tiller:false inaccel:false ingress:false ingress-dn
s:false inspektor-gadget:false istio:false istio-provisioner:false kong:false kubeflow:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-gluster:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s Mount:false MountString:/Users:/minikube-host Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: Disable
Optimizations:false DisableMetrics:false CustomQemuFirmwarePath: SocketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s} ...
I0819 04:34:52.876712 13066 preload.go:131] Checking if preload exists for k8s version v1.31.0 and runtime docker
I0819 04:34:52.876771 13066 ssh_runner.go:195] Run: docker images --format {{.Repository}}:{{.Tag}}
I0819 04:34:52.890832 13066 docker.go:685] Got preloaded images: -- stdout --
kindest/kindnetd:v20240813-c6f155d6
registry.k8s.io/kube-apiserver:v1.31.0
registry.k8s.io/kube-controller-manager:v1.31.0
registry.k8s.io/kube-scheduler:v1.31.0
registry.k8s.io/kube-proxy:v1.31.0
registry.k8s.io/etcd:3.5.15-0
registry.k8s.io/pause:3.10
ghcr.io/kube-vip/kube-vip:v0.8.0
registry.k8s.io/coredns/coredns:v1.11.1
gcr.io/k8s-minikube/storage-provisioner:v5
gcr.io/k8s-minikube/busybox:1.28
-- /stdout --
I0819 04:34:52.890845 13066 docker.go:615] Images already preloaded, skipping extraction
I0819 04:34:52.890917 13066 ssh_runner.go:195] Run: docker images --format {{.Repository}}:{{.Tag}}
I0819 04:34:52.904319 13066 docker.go:685] Got preloaded images: -- stdout --
kindest/kindnetd:v20240813-c6f155d6
registry.k8s.io/kube-scheduler:v1.31.0
registry.k8s.io/kube-controller-manager:v1.31.0
registry.k8s.io/kube-apiserver:v1.31.0
registry.k8s.io/kube-proxy:v1.31.0
registry.k8s.io/etcd:3.5.15-0
registry.k8s.io/pause:3.10
ghcr.io/kube-vip/kube-vip:v0.8.0
registry.k8s.io/coredns/coredns:v1.11.1
gcr.io/k8s-minikube/storage-provisioner:v5
gcr.io/k8s-minikube/busybox:1.28
-- /stdout --
I0819 04:34:52.904337 13066 cache_images.go:84] Images are preloaded, skipping loading
I0819 04:34:52.904346 13066 kubeadm.go:934] updating node { 192.169.0.38 8443 v1.31.0 docker true true} ...
I0819 04:34:52.904431 13066 kubeadm.go:946] kubelet [Unit]
Wants=docker.socket
[Service]
ExecStart=
ExecStart=/var/lib/minikube/binaries/v1.31.0/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --config=/var/lib/kubelet/config.yaml --hostname-override=ha-356000 --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=192.169.0.38
[Install]
config:
{KubernetesVersion:v1.31.0 ClusterName:ha-356000 Namespace:default APIServerHAVIP:192.169.0.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:docker CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:}
I0819 04:34:52.904497 13066 ssh_runner.go:195] Run: docker info --format {{.CgroupDriver}}
I0819 04:34:52.942405 13066 cni.go:84] Creating CNI manager for ""
I0819 04:34:52.942418 13066 cni.go:136] multinode detected (3 nodes found), recommending kindnet
I0819 04:34:52.942430 13066 kubeadm.go:84] Using pod CIDR: 10.244.0.0/16
I0819 04:34:52.942450 13066 kubeadm.go:181] kubeadm options: {CertDir:/var/lib/minikube/certs ServiceCIDR:10.96.0.0/12 PodSubnet:10.244.0.0/16 AdvertiseAddress:192.169.0.38 APIServerPort:8443 KubernetesVersion:v1.31.0 EtcdDataDir:/var/lib/minikube/etcd EtcdExtraArgs:map[] ClusterName:ha-356000 NodeName:ha-356000 DNSDomain:cluster.local CRISocket:/var/run/cri-dockerd.sock ImageRepository: ComponentOptions:[{Component:apiServer ExtraArgs:map[enable-admission-plugins:NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota] Pairs:map[certSANs:["127.0.0.1", "localhost", "192.169.0.38"]]} {Component:controllerManager ExtraArgs:map[allocate-node-cidrs:true leader-elect:false] Pairs:map[]} {Component:scheduler ExtraArgs:map[leader-elect:false] Pairs:map[]}] FeatureArgs:map[] NodeIP:192.169.0.38 CgroupDriver:cgroupfs ClientCAFile:/var/lib/minikube/certs/ca.crt StaticPodPath:/etc/kubernetes/ma
nifests ControlPlaneAddress:control-plane.minikube.internal KubeProxyOptions:map[] ResolvConfSearchRegression:false KubeletConfigOpts:map[containerRuntimeEndpoint:unix:///var/run/cri-dockerd.sock hairpinMode:hairpin-veth runtimeRequestTimeout:15m] PrependCriSocketUnix:true}
I0819 04:34:52.942537 13066 kubeadm.go:187] kubeadm config:
apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 192.169.0.38
bindPort: 8443
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
ttl: 24h0m0s
usages:
- signing
- authentication
nodeRegistration:
criSocket: unix:///var/run/cri-dockerd.sock
name: "ha-356000"
kubeletExtraArgs:
node-ip: 192.169.0.38
taints: []
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
apiServer:
certSANs: ["127.0.0.1", "localhost", "192.169.0.38"]
extraArgs:
enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
controllerManager:
extraArgs:
allocate-node-cidrs: "true"
leader-elect: "false"
scheduler:
extraArgs:
leader-elect: "false"
certificatesDir: /var/lib/minikube/certs
clusterName: mk
controlPlaneEndpoint: control-plane.minikube.internal:8443
etcd:
local:
dataDir: /var/lib/minikube/etcd
extraArgs:
proxy-refresh-interval: "70000"
kubernetesVersion: v1.31.0
networking:
dnsDomain: cluster.local
podSubnet: "10.244.0.0/16"
serviceSubnet: 10.96.0.0/12
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
authentication:
x509:
clientCAFile: /var/lib/minikube/certs/ca.crt
cgroupDriver: cgroupfs
containerRuntimeEndpoint: unix:///var/run/cri-dockerd.sock
hairpinMode: hairpin-veth
runtimeRequestTimeout: 15m
clusterDomain: "cluster.local"
# disable disk resource management by default
imageGCHighThresholdPercent: 100
evictionHard:
nodefs.available: "0%"
nodefs.inodesFree: "0%"
imagefs.available: "0%"
failSwapOn: false
staticPodPath: /etc/kubernetes/manifests
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
clusterCIDR: "10.244.0.0/16"
metricsBindAddress: 0.0.0.0:10249
conntrack:
maxPerCore: 0
# Skip setting "net.netfilter.nf_conntrack_tcp_timeout_established"
tcpEstablishedTimeout: 0s
# Skip setting "net.netfilter.nf_conntrack_tcp_timeout_close"
tcpCloseWaitTimeout: 0s
I0819 04:34:52.942550 13066 kube-vip.go:115] generating kube-vip config ...
I0819 04:34:52.942598 13066 ssh_runner.go:195] Run: sudo sh -c "modprobe --all ip_vs ip_vs_rr ip_vs_wrr ip_vs_sh nf_conntrack"
I0819 04:34:52.955299 13066 kube-vip.go:167] auto-enabling control-plane load-balancing in kube-vip
I0819 04:34:52.955367 13066 kube-vip.go:137] kube-vip config:
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: null
name: kube-vip
namespace: kube-system
spec:
containers:
- args:
- manager
env:
- name: vip_arp
value: "true"
- name: port
value: "8443"
- name: vip_nodename
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: vip_interface
value: eth0
- name: vip_cidr
value: "32"
- name: dns_mode
value: first
- name: cp_enable
value: "true"
- name: cp_namespace
value: kube-system
- name: vip_leaderelection
value: "true"
- name: vip_leasename
value: plndr-cp-lock
- name: vip_leaseduration
value: "5"
- name: vip_renewdeadline
value: "3"
- name: vip_retryperiod
value: "1"
- name: address
value: 192.169.0.254
- name: prometheus_server
value: :2112
- name : lb_enable
value: "true"
- name: lb_port
value: "8443"
image: ghcr.io/kube-vip/kube-vip:v0.8.0
imagePullPolicy: IfNotPresent
name: kube-vip
resources: {}
securityContext:
capabilities:
add:
- NET_ADMIN
- NET_RAW
volumeMounts:
- mountPath: /etc/kubernetes/admin.conf
name: kubeconfig
hostAliases:
- hostnames:
- kubernetes
ip: 127.0.0.1
hostNetwork: true
volumes:
- hostPath:
path: "/etc/kubernetes/admin.conf"
name: kubeconfig
status: {}
I0819 04:34:52.955423 13066 ssh_runner.go:195] Run: sudo ls /var/lib/minikube/binaries/v1.31.0
I0819 04:34:52.964132 13066 binaries.go:44] Found k8s binaries, skipping transfer
I0819 04:34:52.964176 13066 ssh_runner.go:195] Run: sudo mkdir -p /etc/systemd/system/kubelet.service.d /lib/systemd/system /var/tmp/minikube /etc/kubernetes/manifests
I0819 04:34:52.973133 13066 ssh_runner.go:362] scp memory --> /etc/systemd/system/kubelet.service.d/10-kubeadm.conf (308 bytes)
I0819 04:34:52.986657 13066 ssh_runner.go:362] scp memory --> /lib/systemd/system/kubelet.service (352 bytes)
I0819 04:34:52.999990 13066 ssh_runner.go:362] scp memory --> /var/tmp/minikube/kubeadm.yaml.new (2151 bytes)
I0819 04:34:53.013618 13066 ssh_runner.go:362] scp memory --> /etc/kubernetes/manifests/kube-vip.yaml (1440 bytes)
I0819 04:34:53.027049 13066 ssh_runner.go:195] Run: grep 192.169.0.254 control-plane.minikube.internal$ /etc/hosts
I0819 04:34:53.029946 13066 ssh_runner.go:195] Run: /bin/bash -c "{ grep -v $'\tcontrol-plane.minikube.internal$' "/etc/hosts"; echo "192.169.0.254 control-plane.minikube.internal"; } > /tmp/h.$$; sudo cp /tmp/h.$$ "/etc/hosts""
I0819 04:34:53.039212 13066 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I0819 04:34:53.133613 13066 ssh_runner.go:195] Run: sudo systemctl start kubelet
I0819 04:34:53.148441 13066 certs.go:68] Setting up /Users/jenkins/minikube-integration/19476-10399/.minikube/profiles/ha-356000 for IP: 192.169.0.38
I0819 04:34:53.148453 13066 certs.go:194] generating shared ca certs ...
I0819 04:34:53.148463 13066 certs.go:226] acquiring lock for ca certs: {Name:mkce1d4daf32f71f0bd34e5caff0df379c3ce90f Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I0819 04:34:53.148663 13066 certs.go:235] skipping valid "minikubeCA" ca cert: /Users/jenkins/minikube-integration/19476-10399/.minikube/ca.key
I0819 04:34:53.148734 13066 certs.go:235] skipping valid "proxyClientCA" ca cert: /Users/jenkins/minikube-integration/19476-10399/.minikube/proxy-client-ca.key
I0819 04:34:53.148745 13066 certs.go:256] generating profile certs ...
I0819 04:34:53.148842 13066 certs.go:359] skipping valid signed profile cert regeneration for "minikube-user": /Users/jenkins/minikube-integration/19476-10399/.minikube/profiles/ha-356000/client.key
I0819 04:34:53.148866 13066 certs.go:363] generating signed profile cert for "minikube": /Users/jenkins/minikube-integration/19476-10399/.minikube/profiles/ha-356000/apiserver.key.553dc5cd
I0819 04:34:53.148882 13066 crypto.go:68] Generating cert /Users/jenkins/minikube-integration/19476-10399/.minikube/profiles/ha-356000/apiserver.crt.553dc5cd with IP's: [10.96.0.1 127.0.0.1 10.0.0.1 192.169.0.38 192.169.0.39 192.169.0.254]
I0819 04:34:53.330842 13066 crypto.go:156] Writing cert to /Users/jenkins/minikube-integration/19476-10399/.minikube/profiles/ha-356000/apiserver.crt.553dc5cd ...
I0819 04:34:53.330864 13066 lock.go:35] WriteFile acquiring /Users/jenkins/minikube-integration/19476-10399/.minikube/profiles/ha-356000/apiserver.crt.553dc5cd: {Name:mk62722ca7e14168e08072154b82bad3a7055113 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I0819 04:34:53.331169 13066 crypto.go:164] Writing key to /Users/jenkins/minikube-integration/19476-10399/.minikube/profiles/ha-356000/apiserver.key.553dc5cd ...
I0819 04:34:53.331178 13066 lock.go:35] WriteFile acquiring /Users/jenkins/minikube-integration/19476-10399/.minikube/profiles/ha-356000/apiserver.key.553dc5cd: {Name:mk7cd2d8b988328325a2f9f4c7e123b766d7476a Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I0819 04:34:53.331379 13066 certs.go:381] copying /Users/jenkins/minikube-integration/19476-10399/.minikube/profiles/ha-356000/apiserver.crt.553dc5cd -> /Users/jenkins/minikube-integration/19476-10399/.minikube/profiles/ha-356000/apiserver.crt
I0819 04:34:53.331574 13066 certs.go:385] copying /Users/jenkins/minikube-integration/19476-10399/.minikube/profiles/ha-356000/apiserver.key.553dc5cd -> /Users/jenkins/minikube-integration/19476-10399/.minikube/profiles/ha-356000/apiserver.key
I0819 04:34:53.331813 13066 certs.go:359] skipping valid signed profile cert regeneration for "aggregator": /Users/jenkins/minikube-integration/19476-10399/.minikube/profiles/ha-356000/proxy-client.key
I0819 04:34:53.331824 13066 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19476-10399/.minikube/ca.crt -> /var/lib/minikube/certs/ca.crt
I0819 04:34:53.331846 13066 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19476-10399/.minikube/ca.key -> /var/lib/minikube/certs/ca.key
I0819 04:34:53.331865 13066 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19476-10399/.minikube/proxy-client-ca.crt -> /var/lib/minikube/certs/proxy-client-ca.crt
I0819 04:34:53.331883 13066 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19476-10399/.minikube/proxy-client-ca.key -> /var/lib/minikube/certs/proxy-client-ca.key
I0819 04:34:53.331902 13066 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19476-10399/.minikube/profiles/ha-356000/apiserver.crt -> /var/lib/minikube/certs/apiserver.crt
I0819 04:34:53.331919 13066 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19476-10399/.minikube/profiles/ha-356000/apiserver.key -> /var/lib/minikube/certs/apiserver.key
I0819 04:34:53.331938 13066 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19476-10399/.minikube/profiles/ha-356000/proxy-client.crt -> /var/lib/minikube/certs/proxy-client.crt
I0819 04:34:53.331956 13066 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19476-10399/.minikube/profiles/ha-356000/proxy-client.key -> /var/lib/minikube/certs/proxy-client.key
I0819 04:34:53.332053 13066 certs.go:484] found cert: /Users/jenkins/minikube-integration/19476-10399/.minikube/certs/10940.pem (1338 bytes)
W0819 04:34:53.332100 13066 certs.go:480] ignoring /Users/jenkins/minikube-integration/19476-10399/.minikube/certs/10940_empty.pem, impossibly tiny 0 bytes
I0819 04:34:53.332108 13066 certs.go:484] found cert: /Users/jenkins/minikube-integration/19476-10399/.minikube/certs/ca-key.pem (1679 bytes)
I0819 04:34:53.332140 13066 certs.go:484] found cert: /Users/jenkins/minikube-integration/19476-10399/.minikube/certs/ca.pem (1078 bytes)
I0819 04:34:53.332169 13066 certs.go:484] found cert: /Users/jenkins/minikube-integration/19476-10399/.minikube/certs/cert.pem (1123 bytes)
I0819 04:34:53.332196 13066 certs.go:484] found cert: /Users/jenkins/minikube-integration/19476-10399/.minikube/certs/key.pem (1675 bytes)
I0819 04:34:53.332261 13066 certs.go:484] found cert: /Users/jenkins/minikube-integration/19476-10399/.minikube/files/etc/ssl/certs/109402.pem (1708 bytes)
I0819 04:34:53.332293 13066 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19476-10399/.minikube/files/etc/ssl/certs/109402.pem -> /usr/share/ca-certificates/109402.pem
I0819 04:34:53.332314 13066 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19476-10399/.minikube/ca.crt -> /usr/share/ca-certificates/minikubeCA.pem
I0819 04:34:53.332332 13066 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19476-10399/.minikube/certs/10940.pem -> /usr/share/ca-certificates/10940.pem
I0819 04:34:53.332734 13066 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19476-10399/.minikube/ca.crt --> /var/lib/minikube/certs/ca.crt (1111 bytes)
I0819 04:34:53.391411 13066 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19476-10399/.minikube/ca.key --> /var/lib/minikube/certs/ca.key (1679 bytes)
I0819 04:34:53.435935 13066 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19476-10399/.minikube/proxy-client-ca.crt --> /var/lib/minikube/certs/proxy-client-ca.crt (1119 bytes)
I0819 04:34:53.475335 13066 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19476-10399/.minikube/proxy-client-ca.key --> /var/lib/minikube/certs/proxy-client-ca.key (1679 bytes)
I0819 04:34:53.505815 13066 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19476-10399/.minikube/profiles/ha-356000/apiserver.crt --> /var/lib/minikube/certs/apiserver.crt (1432 bytes)
I0819 04:34:53.529932 13066 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19476-10399/.minikube/profiles/ha-356000/apiserver.key --> /var/lib/minikube/certs/apiserver.key (1675 bytes)
I0819 04:34:53.561697 13066 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19476-10399/.minikube/profiles/ha-356000/proxy-client.crt --> /var/lib/minikube/certs/proxy-client.crt (1147 bytes)
I0819 04:34:53.586804 13066 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19476-10399/.minikube/profiles/ha-356000/proxy-client.key --> /var/lib/minikube/certs/proxy-client.key (1675 bytes)
I0819 04:34:53.611958 13066 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19476-10399/.minikube/files/etc/ssl/certs/109402.pem --> /usr/share/ca-certificates/109402.pem (1708 bytes)
I0819 04:34:53.640820 13066 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19476-10399/.minikube/ca.crt --> /usr/share/ca-certificates/minikubeCA.pem (1111 bytes)
I0819 04:34:53.660601 13066 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19476-10399/.minikube/certs/10940.pem --> /usr/share/ca-certificates/10940.pem (1338 bytes)
I0819 04:34:53.680324 13066 ssh_runner.go:362] scp memory --> /var/lib/minikube/kubeconfig (738 bytes)
I0819 04:34:53.693672 13066 ssh_runner.go:195] Run: openssl version
I0819 04:34:53.697948 13066 ssh_runner.go:195] Run: sudo /bin/bash -c "test -s /usr/share/ca-certificates/109402.pem && ln -fs /usr/share/ca-certificates/109402.pem /etc/ssl/certs/109402.pem"
I0819 04:34:53.706382 13066 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/109402.pem
I0819 04:34:53.709892 13066 certs.go:528] hashing: -rw-r--r-- 1 root root 1708 Aug 19 11:21 /usr/share/ca-certificates/109402.pem
I0819 04:34:53.709930 13066 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/109402.pem
I0819 04:34:53.714079 13066 ssh_runner.go:195] Run: sudo /bin/bash -c "test -L /etc/ssl/certs/3ec20f2e.0 || ln -fs /etc/ssl/certs/109402.pem /etc/ssl/certs/3ec20f2e.0"
I0819 04:34:53.722446 13066 ssh_runner.go:195] Run: sudo /bin/bash -c "test -s /usr/share/ca-certificates/minikubeCA.pem && ln -fs /usr/share/ca-certificates/minikubeCA.pem /etc/ssl/certs/minikubeCA.pem"
I0819 04:34:53.730761 13066 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/minikubeCA.pem
I0819 04:34:53.734181 13066 certs.go:528] hashing: -rw-r--r-- 1 root root 1111 Aug 19 11:12 /usr/share/ca-certificates/minikubeCA.pem
I0819 04:34:53.734214 13066 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/minikubeCA.pem
I0819 04:34:53.738418 13066 ssh_runner.go:195] Run: sudo /bin/bash -c "test -L /etc/ssl/certs/b5213941.0 || ln -fs /etc/ssl/certs/minikubeCA.pem /etc/ssl/certs/b5213941.0"
I0819 04:34:53.746788 13066 ssh_runner.go:195] Run: sudo /bin/bash -c "test -s /usr/share/ca-certificates/10940.pem && ln -fs /usr/share/ca-certificates/10940.pem /etc/ssl/certs/10940.pem"
I0819 04:34:53.755027 13066 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/10940.pem
I0819 04:34:53.758472 13066 certs.go:528] hashing: -rw-r--r-- 1 root root 1338 Aug 19 11:21 /usr/share/ca-certificates/10940.pem
I0819 04:34:53.758508 13066 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/10940.pem
I0819 04:34:53.762796 13066 ssh_runner.go:195] Run: sudo /bin/bash -c "test -L /etc/ssl/certs/51391683.0 || ln -fs /etc/ssl/certs/10940.pem /etc/ssl/certs/51391683.0"
I0819 04:34:53.771086 13066 ssh_runner.go:195] Run: stat /var/lib/minikube/certs/apiserver-kubelet-client.crt
I0819 04:34:53.774618 13066 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/apiserver-etcd-client.crt -checkend 86400
I0819 04:34:53.778970 13066 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/apiserver-kubelet-client.crt -checkend 86400
I0819 04:34:53.783186 13066 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/etcd/server.crt -checkend 86400
I0819 04:34:53.787519 13066 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/etcd/healthcheck-client.crt -checkend 86400
I0819 04:34:53.791830 13066 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/etcd/peer.crt -checkend 86400
I0819 04:34:53.796176 13066 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/front-proxy-client.crt -checkend 86400
I0819 04:34:53.800420 13066 kubeadm.go:392] StartCluster: {Name:ha-356000 KeepContext:false EmbedCerts:false MinikubeISO:https://storage.googleapis.com/minikube-builds/iso/19452/minikube-v1.33.1-1723740674-19452-amd64.iso KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.44-1723740748-19452@sha256:2211a6931895d2d502e957e9667096db10734a96767d670cb4dbffdd37397b0d Memory:2200 CPUs:2 DiskSize:20000 Driver:hyperkit HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.31.0 C
lusterName:ha-356000 Namespace:default APIServerHAVIP:192.169.0.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:docker CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.169.0.38 Port:8443 KubernetesVersion:v1.31.0 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m02 IP:192.169.0.39 Port:8443 KubernetesVersion:v1.31.0 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m04 IP:192.169.0.41 Port:0 KubernetesVersion:v1.31.0 ContainerRuntime:docker ControlPlane:false Worker:true}] Addons:map[ambassador:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false helm-tiller:false inaccel:false ingress:false ingress-dns:f
alse inspektor-gadget:false istio:false istio-provisioner:false kong:false kubeflow:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-gluster:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s Mount:false MountString:/Users:/minikube-host Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOpt
imizations:false DisableMetrics:false CustomQemuFirmwarePath: SocketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s}
I0819 04:34:53.800532 13066 ssh_runner.go:195] Run: docker ps --filter status=paused --filter=name=k8s_.*_(kube-system)_ --format={{.ID}}
I0819 04:34:53.814197 13066 ssh_runner.go:195] Run: sudo ls /var/lib/kubelet/kubeadm-flags.env /var/lib/kubelet/config.yaml /var/lib/minikube/etcd
I0819 04:34:53.821854 13066 kubeadm.go:408] found existing configuration files, will attempt cluster restart
I0819 04:34:53.821866 13066 kubeadm.go:593] restartPrimaryControlPlane start ...
I0819 04:34:53.821901 13066 ssh_runner.go:195] Run: sudo test -d /data/minikube
I0819 04:34:53.829341 13066 kubeadm.go:130] /data/minikube skipping compat symlinks: sudo test -d /data/minikube: Process exited with status 1
stdout:
stderr:
I0819 04:34:53.829640 13066 kubeconfig.go:47] verify endpoint returned: get endpoint: "ha-356000" does not appear in /Users/jenkins/minikube-integration/19476-10399/kubeconfig
I0819 04:34:53.829729 13066 kubeconfig.go:62] /Users/jenkins/minikube-integration/19476-10399/kubeconfig needs updating (will repair): [kubeconfig missing "ha-356000" cluster setting kubeconfig missing "ha-356000" context setting]
I0819 04:34:53.829938 13066 lock.go:35] WriteFile acquiring /Users/jenkins/minikube-integration/19476-10399/kubeconfig: {Name:mkf1e57534909363d50de4c4460c1c459f61ca82 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I0819 04:34:53.830637 13066 loader.go:395] Config loaded from file: /Users/jenkins/minikube-integration/19476-10399/kubeconfig
I0819 04:34:53.830832 13066 kapi.go:59] client config for ha-356000: &rest.Config{Host:"https://192.169.0.38:8443", APIPath:"", ContentConfig:rest.ContentConfig{AcceptContentTypes:"", ContentType:"", GroupVersion:(*schema.GroupVersion)(nil), NegotiatedSerializer:runtime.NegotiatedSerializer(nil)}, Username:"", Password:"", BearerToken:"", BearerTokenFile:"", Impersonate:rest.ImpersonationConfig{UserName:"", UID:"", Groups:[]string(nil), Extra:map[string][]string(nil)}, AuthProvider:<nil>, AuthConfigPersister:rest.AuthProviderConfigPersister(nil), ExecProvider:<nil>, TLSClientConfig:rest.sanitizedTLSClientConfig{Insecure:false, ServerName:"", CertFile:"/Users/jenkins/minikube-integration/19476-10399/.minikube/profiles/ha-356000/client.crt", KeyFile:"/Users/jenkins/minikube-integration/19476-10399/.minikube/profiles/ha-356000/client.key", CAFile:"/Users/jenkins/minikube-integration/19476-10399/.minikube/ca.crt", CertData:[]uint8(nil), KeyData:[]uint8(nil), CAData:[]uint8(nil), NextProtos:[]string(nil
)}, UserAgent:"", DisableCompression:false, Transport:http.RoundTripper(nil), WrapTransport:(transport.WrapperFunc)(0xbfb9f60), QPS:0, Burst:0, RateLimiter:flowcontrol.RateLimiter(nil), WarningHandler:rest.WarningHandler(nil), Timeout:0, Dial:(func(context.Context, string, string) (net.Conn, error))(nil), Proxy:(func(*http.Request) (*url.URL, error))(nil)}
I0819 04:34:53.831135 13066 cert_rotation.go:140] Starting client certificate rotation controller
I0819 04:34:53.831318 13066 ssh_runner.go:195] Run: sudo diff -u /var/tmp/minikube/kubeadm.yaml /var/tmp/minikube/kubeadm.yaml.new
I0819 04:34:53.838547 13066 kubeadm.go:630] The running cluster does not require reconfiguration: 192.169.0.38
I0819 04:34:53.838561 13066 kubeadm.go:597] duration metric: took 16.691081ms to restartPrimaryControlPlane
I0819 04:34:53.838565 13066 kubeadm.go:394] duration metric: took 38.149017ms to StartCluster
I0819 04:34:53.838573 13066 settings.go:142] acquiring lock: {Name:mk8742854127df9fd7e94ce523139a7c99e29caa Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I0819 04:34:53.838642 13066 settings.go:150] Updating kubeconfig: /Users/jenkins/minikube-integration/19476-10399/kubeconfig
I0819 04:34:53.838947 13066 lock.go:35] WriteFile acquiring /Users/jenkins/minikube-integration/19476-10399/kubeconfig: {Name:mkf1e57534909363d50de4c4460c1c459f61ca82 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I0819 04:34:53.839156 13066 start.go:233] HA (multi-control plane) cluster: will skip waiting for primary control-plane node &{Name: IP:192.169.0.38 Port:8443 KubernetesVersion:v1.31.0 ContainerRuntime:docker ControlPlane:true Worker:true}
I0819 04:34:53.839171 13066 start.go:241] waiting for startup goroutines ...
I0819 04:34:53.839183 13066 addons.go:507] enable addons start: toEnable=map[ambassador:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false helm-tiller:false inaccel:false ingress:false ingress-dns:false inspektor-gadget:false istio:false istio-provisioner:false kong:false kubeflow:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-gluster:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false]
I0819 04:34:53.839347 13066 config.go:182] Loaded profile config "ha-356000": Driver=hyperkit, ContainerRuntime=docker, KubernetesVersion=v1.31.0
I0819 04:34:53.882730 13066 out.go:177] * Enabled addons:
I0819 04:34:53.904861 13066 addons.go:510] duration metric: took 65.655733ms for enable addons: enabled=[]
I0819 04:34:53.904898 13066 start.go:246] waiting for cluster config update ...
I0819 04:34:53.904910 13066 start.go:255] writing updated cluster config ...
I0819 04:34:53.932717 13066 out.go:201]
I0819 04:34:53.955070 13066 config.go:182] Loaded profile config "ha-356000": Driver=hyperkit, ContainerRuntime=docker, KubernetesVersion=v1.31.0
I0819 04:34:53.955196 13066 profile.go:143] Saving config to /Users/jenkins/minikube-integration/19476-10399/.minikube/profiles/ha-356000/config.json ...
I0819 04:34:53.977497 13066 out.go:177] * Starting "ha-356000-m02" control-plane node in "ha-356000" cluster
I0819 04:34:54.019367 13066 preload.go:131] Checking if preload exists for k8s version v1.31.0 and runtime docker
I0819 04:34:54.019422 13066 cache.go:56] Caching tarball of preloaded images
I0819 04:34:54.019587 13066 preload.go:172] Found /Users/jenkins/minikube-integration/19476-10399/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.31.0-docker-overlay2-amd64.tar.lz4 in cache, skipping download
I0819 04:34:54.019605 13066 cache.go:59] Finished verifying existence of preloaded tar for v1.31.0 on docker
I0819 04:34:54.019714 13066 profile.go:143] Saving config to /Users/jenkins/minikube-integration/19476-10399/.minikube/profiles/ha-356000/config.json ...
I0819 04:34:54.020660 13066 start.go:360] acquireMachinesLock for ha-356000-m02: {Name:mk90c69d5b026abe257122d2e8eeec3d8f4767c3 Clock:{} Delay:500ms Timeout:13m0s Cancel:<nil>}
I0819 04:34:54.020767 13066 start.go:364] duration metric: took 83.596µs to acquireMachinesLock for "ha-356000-m02"
I0819 04:34:54.020794 13066 start.go:96] Skipping create...Using existing machine configuration
I0819 04:34:54.020803 13066 fix.go:54] fixHost starting: m02
I0819 04:34:54.021199 13066 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0819 04:34:54.021225 13066 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0819 04:34:54.030822 13066 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:60868
I0819 04:34:54.031234 13066 main.go:141] libmachine: () Calling .GetVersion
I0819 04:34:54.031723 13066 main.go:141] libmachine: Using API Version 1
I0819 04:34:54.031762 13066 main.go:141] libmachine: () Calling .SetConfigRaw
I0819 04:34:54.032083 13066 main.go:141] libmachine: () Calling .GetMachineName
I0819 04:34:54.032207 13066 main.go:141] libmachine: (ha-356000-m02) Calling .DriverName
I0819 04:34:54.032294 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetState
I0819 04:34:54.032384 13066 main.go:141] libmachine: (ha-356000-m02) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I0819 04:34:54.032506 13066 main.go:141] libmachine: (ha-356000-m02) DBG | hyperkit pid from json: 12976
I0819 04:34:54.033600 13066 main.go:141] libmachine: (ha-356000-m02) DBG | hyperkit pid 12976 missing from process table
I0819 04:34:54.033624 13066 fix.go:112] recreateIfNeeded on ha-356000-m02: state=Stopped err=<nil>
I0819 04:34:54.033634 13066 main.go:141] libmachine: (ha-356000-m02) Calling .DriverName
W0819 04:34:54.033710 13066 fix.go:138] unexpected machine state, will restart: <nil>
I0819 04:34:54.076276 13066 out.go:177] * Restarting existing hyperkit VM for "ha-356000-m02" ...
I0819 04:34:54.097458 13066 main.go:141] libmachine: (ha-356000-m02) Calling .Start
I0819 04:34:54.097739 13066 main.go:141] libmachine: (ha-356000-m02) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I0819 04:34:54.097816 13066 main.go:141] libmachine: (ha-356000-m02) minikube might have been shutdown in an unclean way, the hyperkit pid file still exists: /Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02/hyperkit.pid
I0819 04:34:54.099764 13066 main.go:141] libmachine: (ha-356000-m02) DBG | hyperkit pid 12976 missing from process table
I0819 04:34:54.099780 13066 main.go:141] libmachine: (ha-356000-m02) DBG | pid 12976 is in state "Stopped"
I0819 04:34:54.099797 13066 main.go:141] libmachine: (ha-356000-m02) DBG | Removing stale pid file /Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02/hyperkit.pid...
I0819 04:34:54.100123 13066 main.go:141] libmachine: (ha-356000-m02) DBG | Using UUID 849ad382-b47f-4a64-8340-012df5a36793
I0819 04:34:54.126330 13066 main.go:141] libmachine: (ha-356000-m02) DBG | Generated MAC d6:df:55:3d:f:ed
I0819 04:34:54.126357 13066 main.go:141] libmachine: (ha-356000-m02) DBG | Starting with cmdline: loglevel=3 console=ttyS0 console=tty0 noembed nomodeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-356000
I0819 04:34:54.126519 13066 main.go:141] libmachine: (ha-356000-m02) DBG | 2024/08/19 04:34:54 DEBUG: hyperkit: Start &hyperkit.HyperKit{HyperKit:"/usr/local/bin/hyperkit", Argv0:"", StateDir:"/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02", VPNKitSock:"", VPNKitUUID:"", VPNKitPreferredIPv4:"", UUID:"849ad382-b47f-4a64-8340-012df5a36793", Disks:[]hyperkit.Disk{(*hyperkit.RawDisk)(0xc0003b7bc0)}, ISOImages:[]string{"/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02/boot2docker.iso"}, VSock:false, VSockDir:"", VSockPorts:[]int(nil), VSockGuestCID:3, VMNet:true, Sockets9P:[]hyperkit.Socket9P(nil), Kernel:"/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02/bzimage", Initrd:"/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02/initrd", Bootrom:"", CPUs:2, Memory:2200, Console:1, Serials:[]hyperkit.Serial(nil), Pid:0, Arguments:[]string(nil), CmdLine:"", process:(*os.Process)(nil)}
I0819 04:34:54.126564 13066 main.go:141] libmachine: (ha-356000-m02) DBG | 2024/08/19 04:34:54 DEBUG: hyperkit: check &hyperkit.HyperKit{HyperKit:"/usr/local/bin/hyperkit", Argv0:"", StateDir:"/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02", VPNKitSock:"", VPNKitUUID:"", VPNKitPreferredIPv4:"", UUID:"849ad382-b47f-4a64-8340-012df5a36793", Disks:[]hyperkit.Disk{(*hyperkit.RawDisk)(0xc0003b7bc0)}, ISOImages:[]string{"/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02/boot2docker.iso"}, VSock:false, VSockDir:"", VSockPorts:[]int(nil), VSockGuestCID:3, VMNet:true, Sockets9P:[]hyperkit.Socket9P(nil), Kernel:"/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02/bzimage", Initrd:"/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02/initrd", Bootrom:"", CPUs:2, Memory:2200, Console:1, Serials:[]hyperkit.Serial(nil), Pid:0, Arguments:[]string(nil), CmdLine:"", process:(*os.Process)(nil)}
I0819 04:34:54.126605 13066 main.go:141] libmachine: (ha-356000-m02) DBG | 2024/08/19 04:34:54 DEBUG: hyperkit: Arguments: []string{"-A", "-u", "-F", "/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02/hyperkit.pid", "-c", "2", "-m", "2200M", "-s", "0:0,hostbridge", "-s", "31,lpc", "-s", "1:0,virtio-net", "-U", "849ad382-b47f-4a64-8340-012df5a36793", "-s", "2:0,virtio-blk,/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02/ha-356000-m02.rawdisk", "-s", "3,ahci-cd,/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02/boot2docker.iso", "-s", "4,virtio-rnd", "-l", "com1,autopty=/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02/tty,log=/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02/console-ring", "-f", "kexec,/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02/bzimage,/Users/jenkins/minikube-integration/19476-10399/.minikube/
machines/ha-356000-m02/initrd,earlyprintk=serial loglevel=3 console=ttyS0 console=tty0 noembed nomodeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-356000"}
I0819 04:34:54.126653 13066 main.go:141] libmachine: (ha-356000-m02) DBG | 2024/08/19 04:34:54 DEBUG: hyperkit: CmdLine: "/usr/local/bin/hyperkit -A -u -F /Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02/hyperkit.pid -c 2 -m 2200M -s 0:0,hostbridge -s 31,lpc -s 1:0,virtio-net -U 849ad382-b47f-4a64-8340-012df5a36793 -s 2:0,virtio-blk,/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02/ha-356000-m02.rawdisk -s 3,ahci-cd,/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02/boot2docker.iso -s 4,virtio-rnd -l com1,autopty=/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02/tty,log=/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02/console-ring -f kexec,/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02/bzimage,/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02/initrd,earlyprintk=serial loglevel=3 co
nsole=ttyS0 console=tty0 noembed nomodeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-356000"
I0819 04:34:54.126669 13066 main.go:141] libmachine: (ha-356000-m02) DBG | 2024/08/19 04:34:54 DEBUG: hyperkit: Redirecting stdout/stderr to logger
I0819 04:34:54.128080 13066 main.go:141] libmachine: (ha-356000-m02) DBG | 2024/08/19 04:34:54 DEBUG: hyperkit: Pid is 13087
I0819 04:34:54.128563 13066 main.go:141] libmachine: (ha-356000-m02) DBG | Attempt 0
I0819 04:34:54.128580 13066 main.go:141] libmachine: (ha-356000-m02) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I0819 04:34:54.128657 13066 main.go:141] libmachine: (ha-356000-m02) DBG | hyperkit pid from json: 13087
I0819 04:34:54.130362 13066 main.go:141] libmachine: (ha-356000-m02) DBG | Searching for d6:df:55:3d:f:ed in /var/db/dhcpd_leases ...
I0819 04:34:54.130449 13066 main.go:141] libmachine: (ha-356000-m02) DBG | Found 40 entries in /var/db/dhcpd_leases!
I0819 04:34:54.130463 13066 main.go:141] libmachine: (ha-356000-m02) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.38 HWAddress:aa:ca:df:56:43:33 ID:1,aa:ca:df:56:43:33 Lease:0x66c47f3c}
I0819 04:34:54.130479 13066 main.go:141] libmachine: (ha-356000-m02) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.40 HWAddress:c6:be:b:f7:f5:ce ID:1,c6:be:b:f7:f5:ce Lease:0x66c32d97}
I0819 04:34:54.130493 13066 main.go:141] libmachine: (ha-356000-m02) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.41 HWAddress:6e:45:87:58:5c:30 ID:1,6e:45:87:58:5c:30 Lease:0x66c47f02}
I0819 04:34:54.130510 13066 main.go:141] libmachine: (ha-356000-m02) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.39 HWAddress:d6:df:55:3d:f:ed ID:1,d6:df:55:3d:f:ed Lease:0x66c47e95}
I0819 04:34:54.130523 13066 main.go:141] libmachine: (ha-356000-m02) DBG | Found match: d6:df:55:3d:f:ed
I0819 04:34:54.130534 13066 main.go:141] libmachine: (ha-356000-m02) DBG | IP: 192.169.0.39
I0819 04:34:54.130551 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetConfigRaw
I0819 04:34:54.131219 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetIP
I0819 04:34:54.131420 13066 profile.go:143] Saving config to /Users/jenkins/minikube-integration/19476-10399/.minikube/profiles/ha-356000/config.json ...
I0819 04:34:54.132004 13066 machine.go:93] provisionDockerMachine start ...
I0819 04:34:54.132017 13066 main.go:141] libmachine: (ha-356000-m02) Calling .DriverName
I0819 04:34:54.132150 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHHostname
I0819 04:34:54.132240 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHPort
I0819 04:34:54.132331 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHKeyPath
I0819 04:34:54.132436 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHKeyPath
I0819 04:34:54.132538 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHUsername
I0819 04:34:54.132667 13066 main.go:141] libmachine: Using SSH client type: native
I0819 04:34:54.132824 13066 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xa900ea0] 0xa903c00 <nil> [] 0s} 192.169.0.39 22 <nil> <nil>}
I0819 04:34:54.132832 13066 main.go:141] libmachine: About to run SSH command:
hostname
I0819 04:34:54.136111 13066 main.go:141] libmachine: (ha-356000-m02) DBG | 2024/08/19 04:34:54 INFO : hyperkit: stderr: Using fd 5 for I/O notifications
I0819 04:34:54.144946 13066 main.go:141] libmachine: (ha-356000-m02) DBG | 2024/08/19 04:34:54 INFO : hyperkit: stderr: /Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02/boot2docker.iso: fcntl(F_PUNCHHOLE) Operation not permitted: block device will not support TRIM/DISCARD
I0819 04:34:54.145937 13066 main.go:141] libmachine: (ha-356000-m02) DBG | 2024/08/19 04:34:54 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 2 bit: 22 unspecified don't care: bit is 0
I0819 04:34:54.145960 13066 main.go:141] libmachine: (ha-356000-m02) DBG | 2024/08/19 04:34:54 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 12 unspecified don't care: bit is 0
I0819 04:34:54.145970 13066 main.go:141] libmachine: (ha-356000-m02) DBG | 2024/08/19 04:34:54 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 20 unspecified don't care: bit is 0
I0819 04:34:54.145982 13066 main.go:141] libmachine: (ha-356000-m02) DBG | 2024/08/19 04:34:54 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 3 bit: 13 unspecified don't care: bit is 0
I0819 04:34:54.536704 13066 main.go:141] libmachine: (ha-356000-m02) DBG | 2024/08/19 04:34:54 INFO : hyperkit: stderr: rdmsr to register 0x3a on vcpu 0
I0819 04:34:54.536720 13066 main.go:141] libmachine: (ha-356000-m02) DBG | 2024/08/19 04:34:54 INFO : hyperkit: stderr: rdmsr to register 0x140 on vcpu 0
I0819 04:34:54.651595 13066 main.go:141] libmachine: (ha-356000-m02) DBG | 2024/08/19 04:34:54 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 2 bit: 22 unspecified don't care: bit is 0
I0819 04:34:54.651615 13066 main.go:141] libmachine: (ha-356000-m02) DBG | 2024/08/19 04:34:54 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 12 unspecified don't care: bit is 0
I0819 04:34:54.651623 13066 main.go:141] libmachine: (ha-356000-m02) DBG | 2024/08/19 04:34:54 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 20 unspecified don't care: bit is 0
I0819 04:34:54.651629 13066 main.go:141] libmachine: (ha-356000-m02) DBG | 2024/08/19 04:34:54 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 3 bit: 13 unspecified don't care: bit is 0
I0819 04:34:54.652541 13066 main.go:141] libmachine: (ha-356000-m02) DBG | 2024/08/19 04:34:54 INFO : hyperkit: stderr: rdmsr to register 0x3a on vcpu 1
I0819 04:34:54.652560 13066 main.go:141] libmachine: (ha-356000-m02) DBG | 2024/08/19 04:34:54 INFO : hyperkit: stderr: rdmsr to register 0x140 on vcpu 1
I0819 04:35:00.254714 13066 main.go:141] libmachine: (ha-356000-m02) DBG | 2024/08/19 04:35:00 INFO : hyperkit: stderr: rdmsr to register 0x64d on vcpu 1
I0819 04:35:00.254754 13066 main.go:141] libmachine: (ha-356000-m02) DBG | 2024/08/19 04:35:00 INFO : hyperkit: stderr: rdmsr to register 0x64e on vcpu 1
I0819 04:35:00.254762 13066 main.go:141] libmachine: (ha-356000-m02) DBG | 2024/08/19 04:35:00 INFO : hyperkit: stderr: rdmsr to register 0x34 on vcpu 1
I0819 04:35:00.278500 13066 main.go:141] libmachine: (ha-356000-m02) DBG | 2024/08/19 04:35:00 INFO : hyperkit: stderr: rdmsr to register 0xc0011029 on vcpu 1
I0819 04:35:29.199274 13066 main.go:141] libmachine: SSH cmd err, output: <nil>: minikube
I0819 04:35:29.199288 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetMachineName
I0819 04:35:29.199406 13066 buildroot.go:166] provisioning hostname "ha-356000-m02"
I0819 04:35:29.199416 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetMachineName
I0819 04:35:29.199519 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHHostname
I0819 04:35:29.199618 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHPort
I0819 04:35:29.199702 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHKeyPath
I0819 04:35:29.199782 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHKeyPath
I0819 04:35:29.199874 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHUsername
I0819 04:35:29.200000 13066 main.go:141] libmachine: Using SSH client type: native
I0819 04:35:29.200158 13066 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xa900ea0] 0xa903c00 <nil> [] 0s} 192.169.0.39 22 <nil> <nil>}
I0819 04:35:29.200169 13066 main.go:141] libmachine: About to run SSH command:
sudo hostname ha-356000-m02 && echo "ha-356000-m02" | sudo tee /etc/hostname
I0819 04:35:29.261581 13066 main.go:141] libmachine: SSH cmd err, output: <nil>: ha-356000-m02
I0819 04:35:29.261599 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHHostname
I0819 04:35:29.261738 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHPort
I0819 04:35:29.261842 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHKeyPath
I0819 04:35:29.261937 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHKeyPath
I0819 04:35:29.262028 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHUsername
I0819 04:35:29.262146 13066 main.go:141] libmachine: Using SSH client type: native
I0819 04:35:29.262292 13066 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xa900ea0] 0xa903c00 <nil> [] 0s} 192.169.0.39 22 <nil> <nil>}
I0819 04:35:29.262303 13066 main.go:141] libmachine: About to run SSH command:
if ! grep -xq '.*\sha-356000-m02' /etc/hosts; then
if grep -xq '127.0.1.1\s.*' /etc/hosts; then
sudo sed -i 's/^127.0.1.1\s.*/127.0.1.1 ha-356000-m02/g' /etc/hosts;
else
echo '127.0.1.1 ha-356000-m02' | sudo tee -a /etc/hosts;
fi
fi
I0819 04:35:29.323492 13066 main.go:141] libmachine: SSH cmd err, output: <nil>:
I0819 04:35:29.323509 13066 buildroot.go:172] set auth options {CertDir:/Users/jenkins/minikube-integration/19476-10399/.minikube CaCertPath:/Users/jenkins/minikube-integration/19476-10399/.minikube/certs/ca.pem CaPrivateKeyPath:/Users/jenkins/minikube-integration/19476-10399/.minikube/certs/ca-key.pem CaCertRemotePath:/etc/docker/ca.pem ServerCertPath:/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/server.pem ServerKeyPath:/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/server-key.pem ClientKeyPath:/Users/jenkins/minikube-integration/19476-10399/.minikube/certs/key.pem ServerCertRemotePath:/etc/docker/server.pem ServerKeyRemotePath:/etc/docker/server-key.pem ClientCertPath:/Users/jenkins/minikube-integration/19476-10399/.minikube/certs/cert.pem ServerCertSANs:[] StorePath:/Users/jenkins/minikube-integration/19476-10399/.minikube}
I0819 04:35:29.323518 13066 buildroot.go:174] setting up certificates
I0819 04:35:29.323523 13066 provision.go:84] configureAuth start
I0819 04:35:29.323539 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetMachineName
I0819 04:35:29.323675 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetIP
I0819 04:35:29.323787 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHHostname
I0819 04:35:29.323892 13066 provision.go:143] copyHostCerts
I0819 04:35:29.323923 13066 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19476-10399/.minikube/certs/key.pem -> /Users/jenkins/minikube-integration/19476-10399/.minikube/key.pem
I0819 04:35:29.323986 13066 exec_runner.go:144] found /Users/jenkins/minikube-integration/19476-10399/.minikube/key.pem, removing ...
I0819 04:35:29.323992 13066 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19476-10399/.minikube/key.pem
I0819 04:35:29.324209 13066 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19476-10399/.minikube/certs/key.pem --> /Users/jenkins/minikube-integration/19476-10399/.minikube/key.pem (1675 bytes)
I0819 04:35:29.324421 13066 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19476-10399/.minikube/certs/ca.pem -> /Users/jenkins/minikube-integration/19476-10399/.minikube/ca.pem
I0819 04:35:29.324466 13066 exec_runner.go:144] found /Users/jenkins/minikube-integration/19476-10399/.minikube/ca.pem, removing ...
I0819 04:35:29.324471 13066 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19476-10399/.minikube/ca.pem
I0819 04:35:29.324573 13066 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19476-10399/.minikube/certs/ca.pem --> /Users/jenkins/minikube-integration/19476-10399/.minikube/ca.pem (1078 bytes)
I0819 04:35:29.324723 13066 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19476-10399/.minikube/certs/cert.pem -> /Users/jenkins/minikube-integration/19476-10399/.minikube/cert.pem
I0819 04:35:29.324763 13066 exec_runner.go:144] found /Users/jenkins/minikube-integration/19476-10399/.minikube/cert.pem, removing ...
I0819 04:35:29.324768 13066 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19476-10399/.minikube/cert.pem
I0819 04:35:29.324854 13066 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19476-10399/.minikube/certs/cert.pem --> /Users/jenkins/minikube-integration/19476-10399/.minikube/cert.pem (1123 bytes)
I0819 04:35:29.325000 13066 provision.go:117] generating server cert: /Users/jenkins/minikube-integration/19476-10399/.minikube/machines/server.pem ca-key=/Users/jenkins/minikube-integration/19476-10399/.minikube/certs/ca.pem private-key=/Users/jenkins/minikube-integration/19476-10399/.minikube/certs/ca-key.pem org=jenkins.ha-356000-m02 san=[127.0.0.1 192.169.0.39 ha-356000-m02 localhost minikube]
I0819 04:35:29.385114 13066 provision.go:177] copyRemoteCerts
I0819 04:35:29.385153 13066 ssh_runner.go:195] Run: sudo mkdir -p /etc/docker /etc/docker /etc/docker
I0819 04:35:29.385165 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHHostname
I0819 04:35:29.385286 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHPort
I0819 04:35:29.385386 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHKeyPath
I0819 04:35:29.385478 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHUsername
I0819 04:35:29.385571 13066 sshutil.go:53] new ssh client: &{IP:192.169.0.39 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02/id_rsa Username:docker}
I0819 04:35:29.417589 13066 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19476-10399/.minikube/certs/ca.pem -> /etc/docker/ca.pem
I0819 04:35:29.417662 13066 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19476-10399/.minikube/certs/ca.pem --> /etc/docker/ca.pem (1078 bytes)
I0819 04:35:29.437435 13066 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19476-10399/.minikube/machines/server.pem -> /etc/docker/server.pem
I0819 04:35:29.437499 13066 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19476-10399/.minikube/machines/server.pem --> /etc/docker/server.pem (1208 bytes)
I0819 04:35:29.456875 13066 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19476-10399/.minikube/machines/server-key.pem -> /etc/docker/server-key.pem
I0819 04:35:29.456944 13066 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19476-10399/.minikube/machines/server-key.pem --> /etc/docker/server-key.pem (1675 bytes)
I0819 04:35:29.476511 13066 provision.go:87] duration metric: took 152.975637ms to configureAuth
I0819 04:35:29.476524 13066 buildroot.go:189] setting minikube options for container-runtime
I0819 04:35:29.476693 13066 config.go:182] Loaded profile config "ha-356000": Driver=hyperkit, ContainerRuntime=docker, KubernetesVersion=v1.31.0
I0819 04:35:29.476707 13066 main.go:141] libmachine: (ha-356000-m02) Calling .DriverName
I0819 04:35:29.476843 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHHostname
I0819 04:35:29.476942 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHPort
I0819 04:35:29.477022 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHKeyPath
I0819 04:35:29.477117 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHKeyPath
I0819 04:35:29.477199 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHUsername
I0819 04:35:29.477321 13066 main.go:141] libmachine: Using SSH client type: native
I0819 04:35:29.477450 13066 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xa900ea0] 0xa903c00 <nil> [] 0s} 192.169.0.39 22 <nil> <nil>}
I0819 04:35:29.477459 13066 main.go:141] libmachine: About to run SSH command:
df --output=fstype / | tail -n 1
I0819 04:35:29.527324 13066 main.go:141] libmachine: SSH cmd err, output: <nil>: tmpfs
I0819 04:35:29.527338 13066 buildroot.go:70] root file system type: tmpfs
I0819 04:35:29.527426 13066 provision.go:314] Updating docker unit: /lib/systemd/system/docker.service ...
I0819 04:35:29.527437 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHHostname
I0819 04:35:29.527564 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHPort
I0819 04:35:29.527660 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHKeyPath
I0819 04:35:29.527765 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHKeyPath
I0819 04:35:29.527855 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHUsername
I0819 04:35:29.527982 13066 main.go:141] libmachine: Using SSH client type: native
I0819 04:35:29.528112 13066 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xa900ea0] 0xa903c00 <nil> [] 0s} 192.169.0.39 22 <nil> <nil>}
I0819 04:35:29.528164 13066 main.go:141] libmachine: About to run SSH command:
sudo mkdir -p /lib/systemd/system && printf %s "[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network.target minikube-automount.service docker.socket
Requires= minikube-automount.service docker.socket
StartLimitBurst=3
StartLimitIntervalSec=60
[Service]
Type=notify
Restart=on-failure
Environment="NO_PROXY=192.169.0.38"
# This file is a systemd drop-in unit that inherits from the base dockerd configuration.
# The base configuration already specifies an 'ExecStart=...' command. The first directive
# here is to clear out that command inherited from the base configuration. Without this,
# the command from the base configuration and the command specified here are treated as
# a sequence of commands, which is not the desired behavior, nor is it valid -- systemd
# will catch this invalid input and refuse to start the service with an error like:
# Service has more than one ExecStart= setting, which is only allowed for Type=oneshot services.
# NOTE: default-ulimit=nofile is set to an arbitrary number for consistency with other
# container runtimes. If left unlimited, it may result in OOM issues with MySQL.
ExecStart=
ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --default-ulimit=nofile=1048576:1048576 --tlsverify --tlscacert /etc/docker/ca.pem --tlscert /etc/docker/server.pem --tlskey /etc/docker/server-key.pem --label provider=hyperkit --insecure-registry 10.96.0.0/12
ExecReload=/bin/kill -s HUP \$MAINPID
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
TimeoutStartSec=0
# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes
# kill only the docker process, not all processes in the cgroup
KillMode=process
[Install]
WantedBy=multi-user.target
" | sudo tee /lib/systemd/system/docker.service.new
I0819 04:35:29.591482 13066 main.go:141] libmachine: SSH cmd err, output: <nil>: [Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network.target minikube-automount.service docker.socket
Requires= minikube-automount.service docker.socket
StartLimitBurst=3
StartLimitIntervalSec=60
[Service]
Type=notify
Restart=on-failure
Environment=NO_PROXY=192.169.0.38
# This file is a systemd drop-in unit that inherits from the base dockerd configuration.
# The base configuration already specifies an 'ExecStart=...' command. The first directive
# here is to clear out that command inherited from the base configuration. Without this,
# the command from the base configuration and the command specified here are treated as
# a sequence of commands, which is not the desired behavior, nor is it valid -- systemd
# will catch this invalid input and refuse to start the service with an error like:
# Service has more than one ExecStart= setting, which is only allowed for Type=oneshot services.
# NOTE: default-ulimit=nofile is set to an arbitrary number for consistency with other
# container runtimes. If left unlimited, it may result in OOM issues with MySQL.
ExecStart=
ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --default-ulimit=nofile=1048576:1048576 --tlsverify --tlscacert /etc/docker/ca.pem --tlscert /etc/docker/server.pem --tlskey /etc/docker/server-key.pem --label provider=hyperkit --insecure-registry 10.96.0.0/12
ExecReload=/bin/kill -s HUP $MAINPID
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
TimeoutStartSec=0
# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes
# kill only the docker process, not all processes in the cgroup
KillMode=process
[Install]
WantedBy=multi-user.target
I0819 04:35:29.591503 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHHostname
I0819 04:35:29.591652 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHPort
I0819 04:35:29.591761 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHKeyPath
I0819 04:35:29.591881 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHKeyPath
I0819 04:35:29.591986 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHUsername
I0819 04:35:29.592118 13066 main.go:141] libmachine: Using SSH client type: native
I0819 04:35:29.592261 13066 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xa900ea0] 0xa903c00 <nil> [] 0s} 192.169.0.39 22 <nil> <nil>}
I0819 04:35:29.592276 13066 main.go:141] libmachine: About to run SSH command:
sudo diff -u /lib/systemd/system/docker.service /lib/systemd/system/docker.service.new || { sudo mv /lib/systemd/system/docker.service.new /lib/systemd/system/docker.service; sudo systemctl -f daemon-reload && sudo systemctl -f enable docker && sudo systemctl -f restart docker; }
I0819 04:35:31.295148 13066 main.go:141] libmachine: SSH cmd err, output: <nil>: diff: can't stat '/lib/systemd/system/docker.service': No such file or directory
Created symlink /etc/systemd/system/multi-user.target.wants/docker.service → /usr/lib/systemd/system/docker.service.
I0819 04:35:31.295165 13066 machine.go:96] duration metric: took 37.162252193s to provisionDockerMachine
I0819 04:35:31.295174 13066 start.go:293] postStartSetup for "ha-356000-m02" (driver="hyperkit")
I0819 04:35:31.295181 13066 start.go:322] creating required directories: [/etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs]
I0819 04:35:31.295191 13066 main.go:141] libmachine: (ha-356000-m02) Calling .DriverName
I0819 04:35:31.295355 13066 ssh_runner.go:195] Run: sudo mkdir -p /etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs
I0819 04:35:31.295368 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHHostname
I0819 04:35:31.295467 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHPort
I0819 04:35:31.295556 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHKeyPath
I0819 04:35:31.295643 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHUsername
I0819 04:35:31.295727 13066 sshutil.go:53] new ssh client: &{IP:192.169.0.39 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02/id_rsa Username:docker}
I0819 04:35:31.329891 13066 ssh_runner.go:195] Run: cat /etc/os-release
I0819 04:35:31.332967 13066 info.go:137] Remote host: Buildroot 2023.02.9
I0819 04:35:31.332977 13066 filesync.go:126] Scanning /Users/jenkins/minikube-integration/19476-10399/.minikube/addons for local assets ...
I0819 04:35:31.333083 13066 filesync.go:126] Scanning /Users/jenkins/minikube-integration/19476-10399/.minikube/files for local assets ...
I0819 04:35:31.333270 13066 filesync.go:149] local asset: /Users/jenkins/minikube-integration/19476-10399/.minikube/files/etc/ssl/certs/109402.pem -> 109402.pem in /etc/ssl/certs
I0819 04:35:31.333277 13066 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19476-10399/.minikube/files/etc/ssl/certs/109402.pem -> /etc/ssl/certs/109402.pem
I0819 04:35:31.333474 13066 ssh_runner.go:195] Run: sudo mkdir -p /etc/ssl/certs
I0819 04:35:31.341279 13066 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19476-10399/.minikube/files/etc/ssl/certs/109402.pem --> /etc/ssl/certs/109402.pem (1708 bytes)
I0819 04:35:31.360755 13066 start.go:296] duration metric: took 65.571633ms for postStartSetup
I0819 04:35:31.360772 13066 main.go:141] libmachine: (ha-356000-m02) Calling .DriverName
I0819 04:35:31.360943 13066 ssh_runner.go:195] Run: sudo ls --almost-all -1 /var/lib/minikube/backup
I0819 04:35:31.360956 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHHostname
I0819 04:35:31.361050 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHPort
I0819 04:35:31.361151 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHKeyPath
I0819 04:35:31.361241 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHUsername
I0819 04:35:31.361317 13066 sshutil.go:53] new ssh client: &{IP:192.169.0.39 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02/id_rsa Username:docker}
I0819 04:35:31.394197 13066 machine.go:197] restoring vm config from /var/lib/minikube/backup: [etc]
I0819 04:35:31.394252 13066 ssh_runner.go:195] Run: sudo rsync --archive --update /var/lib/minikube/backup/etc /
I0819 04:35:31.428528 13066 fix.go:56] duration metric: took 37.406817092s for fixHost
I0819 04:35:31.428553 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHHostname
I0819 04:35:31.428684 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHPort
I0819 04:35:31.428789 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHKeyPath
I0819 04:35:31.428878 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHKeyPath
I0819 04:35:31.428962 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHUsername
I0819 04:35:31.429080 13066 main.go:141] libmachine: Using SSH client type: native
I0819 04:35:31.429226 13066 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xa900ea0] 0xa903c00 <nil> [] 0s} 192.169.0.39 22 <nil> <nil>}
I0819 04:35:31.429234 13066 main.go:141] libmachine: About to run SSH command:
date +%s.%N
I0819 04:35:31.481094 13066 main.go:141] libmachine: SSH cmd err, output: <nil>: 1724067331.543934671
I0819 04:35:31.481106 13066 fix.go:216] guest clock: 1724067331.543934671
I0819 04:35:31.481111 13066 fix.go:229] Guest: 2024-08-19 04:35:31.543934671 -0700 PDT Remote: 2024-08-19 04:35:31.428543 -0700 PDT m=+80.457698496 (delta=115.391671ms)
I0819 04:35:31.481121 13066 fix.go:200] guest clock delta is within tolerance: 115.391671ms
I0819 04:35:31.481125 13066 start.go:83] releasing machines lock for "ha-356000-m02", held for 37.459439437s
I0819 04:35:31.481142 13066 main.go:141] libmachine: (ha-356000-m02) Calling .DriverName
I0819 04:35:31.481281 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetIP
I0819 04:35:31.504689 13066 out.go:177] * Found network options:
I0819 04:35:31.524970 13066 out.go:177] - NO_PROXY=192.169.0.38
W0819 04:35:31.546783 13066 proxy.go:119] fail to check proxy env: Error ip not in block
I0819 04:35:31.546823 13066 main.go:141] libmachine: (ha-356000-m02) Calling .DriverName
I0819 04:35:31.547635 13066 main.go:141] libmachine: (ha-356000-m02) Calling .DriverName
I0819 04:35:31.547906 13066 main.go:141] libmachine: (ha-356000-m02) Calling .DriverName
I0819 04:35:31.548013 13066 ssh_runner.go:195] Run: curl -sS -m 2 https://registry.k8s.io/
I0819 04:35:31.548056 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHHostname
W0819 04:35:31.548178 13066 proxy.go:119] fail to check proxy env: Error ip not in block
I0819 04:35:31.548263 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHPort
I0819 04:35:31.548280 13066 ssh_runner.go:195] Run: sh -c "stat /etc/cni/net.d/*loopback.conf*"
I0819 04:35:31.548302 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHHostname
I0819 04:35:31.548473 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHKeyPath
I0819 04:35:31.548494 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHPort
I0819 04:35:31.548724 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHUsername
I0819 04:35:31.548766 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHKeyPath
I0819 04:35:31.548954 13066 sshutil.go:53] new ssh client: &{IP:192.169.0.39 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02/id_rsa Username:docker}
I0819 04:35:31.548985 13066 main.go:141] libmachine: (ha-356000-m02) Calling .GetSSHUsername
I0819 04:35:31.549118 13066 sshutil.go:53] new ssh client: &{IP:192.169.0.39 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19476-10399/.minikube/machines/ha-356000-m02/id_rsa Username:docker}
W0819 04:35:31.578159 13066 cni.go:209] loopback cni configuration skipped: "/etc/cni/net.d/*loopback.conf*" not found
I0819 04:35:31.578218 13066 ssh_runner.go:195] Run: sudo find /etc/cni/net.d -maxdepth 1 -type f ( ( -name *bridge* -or -name *podman* ) -and -not -name *.mk_disabled ) -printf "%p, " -exec sh -c "sudo mv {} {}.mk_disabled" ;
I0819 04:35:31.625637 13066 cni.go:262] disabled [/etc/cni/net.d/87-podman-bridge.conflist] bridge cni config(s)
I0819 04:35:31.625659 13066 start.go:495] detecting cgroup driver to use...
I0819 04:35:31.625759 13066 ssh_runner.go:195] Run: /bin/bash -c "sudo mkdir -p /etc && printf %s "runtime-endpoint: unix:///run/containerd/containerd.sock
" | sudo tee /etc/crictl.yaml"
I0819 04:35:31.641841 13066 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)sandbox_image = .*$|\1sandbox_image = "registry.k8s.io/pause:3.10"|' /etc/containerd/config.toml"
I0819 04:35:31.650587 13066 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)restrict_oom_score_adj = .*$|\1restrict_oom_score_adj = false|' /etc/containerd/config.toml"
I0819 04:35:31.659322 13066 containerd.go:146] configuring containerd to use "cgroupfs" as cgroup driver...
I0819 04:35:31.659364 13066 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)SystemdCgroup = .*$|\1SystemdCgroup = false|g' /etc/containerd/config.toml"
I0819 04:35:31.668282 13066 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|"io.containerd.runtime.v1.linux"|"io.containerd.runc.v2"|g' /etc/containerd/config.toml"
I0819 04:35:31.677045 13066 ssh_runner.go:195] Run: sh -c "sudo sed -i '/systemd_cgroup/d' /etc/containerd/config.toml"
I0819 04:35:31.685786 13066 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|"io.containerd.runc.v1"|"io.containerd.runc.v2"|g' /etc/containerd/config.toml"
I0819 04:35:31.694543 13066 ssh_runner.go:195] Run: sh -c "sudo rm -rf /etc/cni/net.mk"
I0819 04:35:31.703580 13066 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)conf_dir = .*$|\1conf_dir = "/etc/cni/net.d"|g' /etc/containerd/config.toml"
I0819 04:35:31.712257 13066 ssh_runner.go:195] Run: sh -c "sudo sed -i '/^ *enable_unprivileged_ports = .*/d' /etc/containerd/config.toml"
I0819 04:35:31.721030 13066 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)\[plugins."io.containerd.grpc.v1.cri"\]|&\n\1 enable_unprivileged_ports = true|' /etc/containerd/config.toml"
I0819 04:35:31.729900 13066 ssh_runner.go:195] Run: sudo sysctl net.bridge.bridge-nf-call-iptables
I0819 04:35:31.737768 13066 ssh_runner.go:195] Run: sudo sh -c "echo 1 > /proc/sys/net/ipv4/ip_forward"
I0819 04:35:31.745684 13066 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I0819 04:35:31.842314 13066 ssh_runner.go:195] Run: sudo systemctl restart containerd
I0819 04:35:31.861480 13066 start.go:495] detecting cgroup driver to use...
I0819 04:35:31.861545 13066 ssh_runner.go:195] Run: sudo systemctl cat docker.service
I0819 04:35:31.882012 13066 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service containerd
I0819 04:35:31.895265 13066 ssh_runner.go:195] Run: sudo systemctl stop -f containerd
I0819 04:35:31.917304 13066 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service containerd
I0819 04:35:31.928851 13066 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service crio
I0819 04:35:31.940069 13066 ssh_runner.go:195] Run: sudo systemctl stop -f crio
I0819 04:35:31.968123 13066 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service crio
I0819 04:35:31.979770 13066 ssh_runner.go:195] Run: /bin/bash -c "sudo mkdir -p /etc && printf %s "runtime-endpoint: unix:///var/run/cri-dockerd.sock
" | sudo tee /etc/crictl.yaml"
I0819 04:35:31.994906 13066 ssh_runner.go:195] Run: which cri-dockerd
I0819 04:35:31.997851 13066 ssh_runner.go:195] Run: sudo mkdir -p /etc/systemd/system/cri-docker.service.d
I0819 04:35:32.005104 13066 ssh_runner.go:362] scp memory --> /etc/systemd/system/cri-docker.service.d/10-cni.conf (190 bytes)
I0819 04:35:32.018515 13066 ssh_runner.go:195] Run: sudo systemctl unmask docker.service
I0819 04:35:32.123956 13066 ssh_runner.go:195] Run: sudo systemctl enable docker.socket
I0819 04:35:32.216989 13066 docker.go:574] configuring docker to use "cgroupfs" as cgroup driver...
I0819 04:35:32.217011 13066 ssh_runner.go:362] scp memory --> /etc/docker/daemon.json (130 bytes)
I0819 04:35:32.236543 13066 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I0819 04:35:32.335023 13066 ssh_runner.go:195] Run: sudo systemctl restart docker
I0819 04:36:33.359717 13066 ssh_runner.go:235] Completed: sudo systemctl restart docker: (1m1.023197901s)
I0819 04:36:33.359784 13066 ssh_runner.go:195] Run: sudo journalctl --no-pager -u docker
I0819 04:36:33.396053 13066 out.go:201]
W0819 04:36:33.418861 13066 out.go:270] X Exiting due to RUNTIME_ENABLE: Failed to enable container runtime: sudo systemctl restart docker: Process exited with status 1
stdout:
stderr:
Job for docker.service failed because the control process exited with error code.
See "systemctl status docker.service" and "journalctl -xeu docker.service" for details.
sudo journalctl --no-pager -u docker:
-- stdout --
Aug 19 11:35:29 ha-356000-m02 systemd[1]: Starting Docker Application Container Engine...
Aug 19 11:35:29 ha-356000-m02 dockerd[511]: time="2024-08-19T11:35:29.324179830Z" level=info msg="Starting up"
Aug 19 11:35:29 ha-356000-m02 dockerd[511]: time="2024-08-19T11:35:29.324631187Z" level=info msg="containerd not running, starting managed containerd"
Aug 19 11:35:29 ha-356000-m02 dockerd[511]: time="2024-08-19T11:35:29.325219709Z" level=info msg="started new containerd process" address=/var/run/docker/containerd/containerd.sock module=libcontainerd pid=517
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.343048826Z" level=info msg="starting containerd" revision=8fc6bcff51318944179630522a095cc9dbf9f353 version=v1.7.20
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.357776217Z" level=info msg="loading plugin \"io.containerd.event.v1.exchange\"..." type=io.containerd.event.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.357806147Z" level=info msg="loading plugin \"io.containerd.internal.v1.opt\"..." type=io.containerd.internal.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.357853202Z" level=info msg="loading plugin \"io.containerd.warning.v1.deprecations\"..." type=io.containerd.warning.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.357864654Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.blockfile\"..." type=io.containerd.snapshotter.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.358012531Z" level=info msg="skip loading plugin \"io.containerd.snapshotter.v1.blockfile\"..." error="no scratch file generator: skip plugin" type=io.containerd.snapshotter.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.358050455Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.btrfs\"..." type=io.containerd.snapshotter.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.358178841Z" level=info msg="skip loading plugin \"io.containerd.snapshotter.v1.btrfs\"..." error="path /var/lib/docker/containerd/daemon/io.containerd.snapshotter.v1.btrfs (ext4) must be a btrfs filesystem to be used with the btrfs snapshotter: skip plugin" type=io.containerd.snapshotter.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.358192958Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.devmapper\"..." type=io.containerd.snapshotter.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.358202617Z" level=info msg="skip loading plugin \"io.containerd.snapshotter.v1.devmapper\"..." error="devmapper not configured: skip plugin" type=io.containerd.snapshotter.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.358209203Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.native\"..." type=io.containerd.snapshotter.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.358316306Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.overlayfs\"..." type=io.containerd.snapshotter.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.358505651Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.aufs\"..." type=io.containerd.snapshotter.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.360109204Z" level=info msg="skip loading plugin \"io.containerd.snapshotter.v1.aufs\"..." error="aufs is not supported (modprobe aufs failed: exit status 1 \"modprobe: FATAL: Module aufs not found in directory /lib/modules/5.10.207\\n\"): skip plugin" type=io.containerd.snapshotter.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.360145586Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.zfs\"..." type=io.containerd.snapshotter.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.360232283Z" level=info msg="skip loading plugin \"io.containerd.snapshotter.v1.zfs\"..." error="path /var/lib/docker/containerd/daemon/io.containerd.snapshotter.v1.zfs must be a zfs filesystem to be used with the zfs snapshotter: skip plugin" type=io.containerd.snapshotter.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.360265567Z" level=info msg="loading plugin \"io.containerd.content.v1.content\"..." type=io.containerd.content.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.360393818Z" level=info msg="loading plugin \"io.containerd.metadata.v1.bolt\"..." type=io.containerd.metadata.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.360438850Z" level=info msg="metadata content store policy set" policy=shared
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.363978148Z" level=info msg="loading plugin \"io.containerd.gc.v1.scheduler\"..." type=io.containerd.gc.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364024803Z" level=info msg="loading plugin \"io.containerd.differ.v1.walking\"..." type=io.containerd.differ.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364037902Z" level=info msg="loading plugin \"io.containerd.lease.v1.manager\"..." type=io.containerd.lease.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364048191Z" level=info msg="loading plugin \"io.containerd.streaming.v1.manager\"..." type=io.containerd.streaming.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364057433Z" level=info msg="loading plugin \"io.containerd.runtime.v1.linux\"..." type=io.containerd.runtime.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364144924Z" level=info msg="loading plugin \"io.containerd.monitor.v1.cgroups\"..." type=io.containerd.monitor.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364307296Z" level=info msg="loading plugin \"io.containerd.runtime.v2.task\"..." type=io.containerd.runtime.v2
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364380642Z" level=info msg="loading plugin \"io.containerd.runtime.v2.shim\"..." type=io.containerd.runtime.v2
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364402387Z" level=info msg="loading plugin \"io.containerd.sandbox.store.v1.local\"..." type=io.containerd.sandbox.store.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364415018Z" level=info msg="loading plugin \"io.containerd.sandbox.controller.v1.local\"..." type=io.containerd.sandbox.controller.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364424925Z" level=info msg="loading plugin \"io.containerd.service.v1.containers-service\"..." type=io.containerd.service.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364433622Z" level=info msg="loading plugin \"io.containerd.service.v1.content-service\"..." type=io.containerd.service.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364441125Z" level=info msg="loading plugin \"io.containerd.service.v1.diff-service\"..." type=io.containerd.service.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364452476Z" level=info msg="loading plugin \"io.containerd.service.v1.images-service\"..." type=io.containerd.service.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364461452Z" level=info msg="loading plugin \"io.containerd.service.v1.introspection-service\"..." type=io.containerd.service.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364470187Z" level=info msg="loading plugin \"io.containerd.service.v1.namespaces-service\"..." type=io.containerd.service.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364477875Z" level=info msg="loading plugin \"io.containerd.service.v1.snapshots-service\"..." type=io.containerd.service.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364485462Z" level=info msg="loading plugin \"io.containerd.service.v1.tasks-service\"..." type=io.containerd.service.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364506700Z" level=info msg="loading plugin \"io.containerd.grpc.v1.containers\"..." type=io.containerd.grpc.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364518097Z" level=info msg="loading plugin \"io.containerd.grpc.v1.content\"..." type=io.containerd.grpc.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364526770Z" level=info msg="loading plugin \"io.containerd.grpc.v1.diff\"..." type=io.containerd.grpc.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364535397Z" level=info msg="loading plugin \"io.containerd.grpc.v1.events\"..." type=io.containerd.grpc.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364544614Z" level=info msg="loading plugin \"io.containerd.grpc.v1.images\"..." type=io.containerd.grpc.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364553228Z" level=info msg="loading plugin \"io.containerd.grpc.v1.introspection\"..." type=io.containerd.grpc.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364560478Z" level=info msg="loading plugin \"io.containerd.grpc.v1.leases\"..." type=io.containerd.grpc.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364568384Z" level=info msg="loading plugin \"io.containerd.grpc.v1.namespaces\"..." type=io.containerd.grpc.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364576411Z" level=info msg="loading plugin \"io.containerd.grpc.v1.sandbox-controllers\"..." type=io.containerd.grpc.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364586166Z" level=info msg="loading plugin \"io.containerd.grpc.v1.sandboxes\"..." type=io.containerd.grpc.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364593790Z" level=info msg="loading plugin \"io.containerd.grpc.v1.snapshots\"..." type=io.containerd.grpc.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364601258Z" level=info msg="loading plugin \"io.containerd.grpc.v1.streaming\"..." type=io.containerd.grpc.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364608762Z" level=info msg="loading plugin \"io.containerd.grpc.v1.tasks\"..." type=io.containerd.grpc.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364618470Z" level=info msg="loading plugin \"io.containerd.transfer.v1.local\"..." type=io.containerd.transfer.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364631396Z" level=info msg="loading plugin \"io.containerd.grpc.v1.transfer\"..." type=io.containerd.grpc.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364639366Z" level=info msg="loading plugin \"io.containerd.grpc.v1.version\"..." type=io.containerd.grpc.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364646821Z" level=info msg="loading plugin \"io.containerd.internal.v1.restart\"..." type=io.containerd.internal.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364671565Z" level=info msg="loading plugin \"io.containerd.tracing.processor.v1.otlp\"..." type=io.containerd.tracing.processor.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364681986Z" level=info msg="skip loading plugin \"io.containerd.tracing.processor.v1.otlp\"..." error="skip plugin: tracing endpoint not configured" type=io.containerd.tracing.processor.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364689333Z" level=info msg="loading plugin \"io.containerd.internal.v1.tracing\"..." type=io.containerd.internal.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364696823Z" level=info msg="skip loading plugin \"io.containerd.internal.v1.tracing\"..." error="skip plugin: tracing endpoint not configured" type=io.containerd.internal.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364710942Z" level=info msg="loading plugin \"io.containerd.grpc.v1.healthcheck\"..." type=io.containerd.grpc.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364724751Z" level=info msg="loading plugin \"io.containerd.nri.v1.nri\"..." type=io.containerd.nri.v1
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364734276Z" level=info msg="NRI interface is disabled by configuration."
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364881393Z" level=info msg=serving... address=/var/run/docker/containerd/containerd-debug.sock
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364936581Z" level=info msg=serving... address=/var/run/docker/containerd/containerd.sock.ttrpc
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364961767Z" level=info msg=serving... address=/var/run/docker/containerd/containerd.sock
Aug 19 11:35:29 ha-356000-m02 dockerd[517]: time="2024-08-19T11:35:29.364994781Z" level=info msg="containerd successfully booted in 0.022862s"
Aug 19 11:35:30 ha-356000-m02 dockerd[511]: time="2024-08-19T11:35:30.343247358Z" level=info msg="[graphdriver] trying configured driver: overlay2"
Aug 19 11:35:30 ha-356000-m02 dockerd[511]: time="2024-08-19T11:35:30.386657968Z" level=info msg="Loading containers: start."
Aug 19 11:35:30 ha-356000-m02 dockerd[511]: time="2024-08-19T11:35:30.603738249Z" level=warning msg="ip6tables is enabled, but cannot set up ip6tables chains" error="failed to create NAT chain DOCKER: iptables failed: ip6tables --wait -t nat -N DOCKER: ip6tables v1.8.9 (legacy): can't initialize ip6tables table `nat': Table does not exist (do you need to insmod?)\nPerhaps ip6tables or your kernel needs to be upgraded.\n (exit status 3)"
Aug 19 11:35:31 ha-356000-m02 dockerd[511]: time="2024-08-19T11:35:31.282434367Z" level=info msg="Default bridge (docker0) is assigned with an IP address 172.17.0.0/16. Daemon option --bip can be used to set a preferred IP address"
Aug 19 11:35:31 ha-356000-m02 dockerd[511]: time="2024-08-19T11:35:31.327872236Z" level=info msg="Loading containers: done."
Aug 19 11:35:31 ha-356000-m02 dockerd[511]: time="2024-08-19T11:35:31.334567046Z" level=info msg="Docker daemon" commit=f9522e5 containerd-snapshotter=false storage-driver=overlay2 version=27.1.2
Aug 19 11:35:31 ha-356000-m02 dockerd[511]: time="2024-08-19T11:35:31.334747201Z" level=info msg="Daemon has completed initialization"
Aug 19 11:35:31 ha-356000-m02 dockerd[511]: time="2024-08-19T11:35:31.356977166Z" level=info msg="API listen on /var/run/docker.sock"
Aug 19 11:35:31 ha-356000-m02 systemd[1]: Started Docker Application Container Engine.
Aug 19 11:35:31 ha-356000-m02 dockerd[511]: time="2024-08-19T11:35:31.357038858Z" level=info msg="API listen on [::]:2376"
Aug 19 11:35:32 ha-356000-m02 dockerd[511]: time="2024-08-19T11:35:32.410721109Z" level=info msg="Processing signal 'terminated'"
Aug 19 11:35:32 ha-356000-m02 dockerd[511]: time="2024-08-19T11:35:32.411576571Z" level=info msg="stopping event stream following graceful shutdown" error="<nil>" module=libcontainerd namespace=moby
Aug 19 11:35:32 ha-356000-m02 dockerd[511]: time="2024-08-19T11:35:32.411707824Z" level=info msg="Daemon shutdown complete"
Aug 19 11:35:32 ha-356000-m02 dockerd[511]: time="2024-08-19T11:35:32.411744277Z" level=info msg="stopping event stream following graceful shutdown" error="context canceled" module=libcontainerd namespace=plugins.moby
Aug 19 11:35:32 ha-356000-m02 dockerd[511]: time="2024-08-19T11:35:32.411756165Z" level=info msg="stopping healthcheck following graceful shutdown" module=libcontainerd
Aug 19 11:35:32 ha-356000-m02 systemd[1]: Stopping Docker Application Container Engine...
Aug 19 11:35:33 ha-356000-m02 systemd[1]: docker.service: Deactivated successfully.
Aug 19 11:35:33 ha-356000-m02 systemd[1]: Stopped Docker Application Container Engine.
Aug 19 11:35:33 ha-356000-m02 systemd[1]: Starting Docker Application Container Engine...
Aug 19 11:35:33 ha-356000-m02 dockerd[1100]: time="2024-08-19T11:35:33.449480254Z" level=info msg="Starting up"
Aug 19 11:36:33 ha-356000-m02 dockerd[1100]: failed to start daemon: failed to dial "/run/containerd/containerd.sock": failed to dial "/run/containerd/containerd.sock": context deadline exceeded
Aug 19 11:36:33 ha-356000-m02 systemd[1]: docker.service: Main process exited, code=exited, status=1/FAILURE
Aug 19 11:36:33 ha-356000-m02 systemd[1]: docker.service: Failed with result 'exit-code'.
Aug 19 11:36:33 ha-356000-m02 systemd[1]: Failed to start Docker Application Container Engine.
-- /stdout --
W0819 04:36:33.418942 13066 out.go:270] *
W0819 04:36:33.420223 13066 out.go:293] ╭─────────────────────────────────────────────────────────────────────────────────────────────╮
│ │
│ * If the above advice does not help, please let us know: │
│ https://github.com/kubernetes/minikube/issues/new/choose │
│ │
│ * Please run `minikube logs --file=logs.txt` and attach logs.txt to the GitHub issue. │
│ │
╰─────────────────────────────────────────────────────────────────────────────────────────────╯
I0819 04:36:33.482760 13066 out.go:201]
==> Docker <==
Aug 19 11:35:51 ha-356000 dockerd[1109]: time="2024-08-19T11:35:51.024819719Z" level=info msg="ignoring event" container=bc9a2e06b275830d33b503f1b026fb1010d6095ea619910504bbfa7ba0115af5 module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Aug 19 11:35:51 ha-356000 dockerd[1115]: time="2024-08-19T11:35:51.025299906Z" level=info msg="shim disconnected" id=bc9a2e06b275830d33b503f1b026fb1010d6095ea619910504bbfa7ba0115af5 namespace=moby
Aug 19 11:35:51 ha-356000 dockerd[1115]: time="2024-08-19T11:35:51.025856770Z" level=warning msg="cleaning up after shim disconnected" id=bc9a2e06b275830d33b503f1b026fb1010d6095ea619910504bbfa7ba0115af5 namespace=moby
Aug 19 11:35:51 ha-356000 dockerd[1115]: time="2024-08-19T11:35:51.025899443Z" level=info msg="cleaning up dead shim" namespace=moby
Aug 19 11:36:00 ha-356000 dockerd[1115]: time="2024-08-19T11:36:00.406427399Z" level=info msg="shim disconnected" id=259361a45e235c109ea54b1f11a2ffc395a0946fabb67691546f8127ff992e47 namespace=moby
Aug 19 11:36:00 ha-356000 dockerd[1115]: time="2024-08-19T11:36:00.406846486Z" level=warning msg="cleaning up after shim disconnected" id=259361a45e235c109ea54b1f11a2ffc395a0946fabb67691546f8127ff992e47 namespace=moby
Aug 19 11:36:00 ha-356000 dockerd[1115]: time="2024-08-19T11:36:00.406944033Z" level=info msg="cleaning up dead shim" namespace=moby
Aug 19 11:36:00 ha-356000 dockerd[1109]: time="2024-08-19T11:36:00.407634658Z" level=info msg="ignoring event" container=259361a45e235c109ea54b1f11a2ffc395a0946fabb67691546f8127ff992e47 module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Aug 19 11:36:00 ha-356000 dockerd[1115]: time="2024-08-19T11:36:00.417678922Z" level=warning msg="cleanup warnings time=\"2024-08-19T11:36:00Z\" level=warning msg=\"failed to remove runc container\" error=\"runc did not terminate successfully: exit status 255: \" runtime=io.containerd.runc.v2\n" namespace=moby
Aug 19 11:36:14 ha-356000 dockerd[1115]: time="2024-08-19T11:36:14.381775205Z" level=info msg="loading plugin \"io.containerd.event.v1.publisher\"..." runtime=io.containerd.runc.v2 type=io.containerd.event.v1
Aug 19 11:36:14 ha-356000 dockerd[1115]: time="2024-08-19T11:36:14.382997854Z" level=info msg="loading plugin \"io.containerd.internal.v1.shutdown\"..." runtime=io.containerd.runc.v2 type=io.containerd.internal.v1
Aug 19 11:36:14 ha-356000 dockerd[1115]: time="2024-08-19T11:36:14.383025345Z" level=info msg="loading plugin \"io.containerd.ttrpc.v1.task\"..." runtime=io.containerd.runc.v2 type=io.containerd.ttrpc.v1
Aug 19 11:36:14 ha-356000 dockerd[1115]: time="2024-08-19T11:36:14.383167410Z" level=info msg="loading plugin \"io.containerd.ttrpc.v1.pause\"..." runtime=io.containerd.runc.v2 type=io.containerd.ttrpc.v1
Aug 19 11:36:30 ha-356000 dockerd[1115]: time="2024-08-19T11:36:30.386028582Z" level=info msg="loading plugin \"io.containerd.event.v1.publisher\"..." runtime=io.containerd.runc.v2 type=io.containerd.event.v1
Aug 19 11:36:30 ha-356000 dockerd[1115]: time="2024-08-19T11:36:30.386891064Z" level=info msg="loading plugin \"io.containerd.internal.v1.shutdown\"..." runtime=io.containerd.runc.v2 type=io.containerd.internal.v1
Aug 19 11:36:30 ha-356000 dockerd[1115]: time="2024-08-19T11:36:30.386924922Z" level=info msg="loading plugin \"io.containerd.ttrpc.v1.task\"..." runtime=io.containerd.runc.v2 type=io.containerd.ttrpc.v1
Aug 19 11:36:30 ha-356000 dockerd[1115]: time="2024-08-19T11:36:30.387129769Z" level=info msg="loading plugin \"io.containerd.ttrpc.v1.pause\"..." runtime=io.containerd.runc.v2 type=io.containerd.ttrpc.v1
Aug 19 11:36:35 ha-356000 dockerd[1109]: time="2024-08-19T11:36:35.189341935Z" level=info msg="ignoring event" container=860a05c56d106dc2e8fbe54d2d4a3c870b95690d8845f6ae5eabf52b76ab6422 module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Aug 19 11:36:35 ha-356000 dockerd[1115]: time="2024-08-19T11:36:35.189026996Z" level=info msg="shim disconnected" id=860a05c56d106dc2e8fbe54d2d4a3c870b95690d8845f6ae5eabf52b76ab6422 namespace=moby
Aug 19 11:36:35 ha-356000 dockerd[1115]: time="2024-08-19T11:36:35.190435239Z" level=warning msg="cleaning up after shim disconnected" id=860a05c56d106dc2e8fbe54d2d4a3c870b95690d8845f6ae5eabf52b76ab6422 namespace=moby
Aug 19 11:36:35 ha-356000 dockerd[1115]: time="2024-08-19T11:36:35.190477443Z" level=info msg="cleaning up dead shim" namespace=moby
Aug 19 11:36:46 ha-356000 dockerd[1115]: time="2024-08-19T11:36:46.203147977Z" level=info msg="shim disconnected" id=f19e14473bdcae42a398f77919e5f2f5fd297f2cc629eb3c7ca44798a2883025 namespace=moby
Aug 19 11:36:46 ha-356000 dockerd[1109]: time="2024-08-19T11:36:46.203936236Z" level=info msg="ignoring event" container=f19e14473bdcae42a398f77919e5f2f5fd297f2cc629eb3c7ca44798a2883025 module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Aug 19 11:36:46 ha-356000 dockerd[1115]: time="2024-08-19T11:36:46.204231695Z" level=warning msg="cleaning up after shim disconnected" id=f19e14473bdcae42a398f77919e5f2f5fd297f2cc629eb3c7ca44798a2883025 namespace=moby
Aug 19 11:36:46 ha-356000 dockerd[1115]: time="2024-08-19T11:36:46.204342328Z" level=info msg="cleaning up dead shim" namespace=moby
==> container status <==
CONTAINER IMAGE CREATED STATE NAME ATTEMPT POD ID POD
f19e14473bdca 045733566833c 20 seconds ago Exited kube-controller-manager 5 60f516d5fd1cb kube-controller-manager-ha-356000
860a05c56d106 604f5db92eaa8 36 seconds ago Exited kube-apiserver 5 ffda5efe9741d kube-apiserver-ha-356000
b5f04621abe7d 1766f54c897f0 About a minute ago Running kube-scheduler 2 ea2cb107132a1 kube-scheduler-ha-356000
29602b8fea29a 38af8ddebf499 About a minute ago Running kube-vip 1 01b3d54ae5387 kube-vip-ha-356000
09103e728061c 2e96e5913fc06 About a minute ago Running etcd 2 e9f38eabc28da etcd-ha-356000
23d42a24df9b6 cbb01a7bd410d 3 minutes ago Exited coredns 1 dacb252e80d75 coredns-6f6b679f8f-nxpq9
9015de206ec30 8c811b4aec35f 3 minutes ago Exited busybox 1 b26940bff6af3 busybox-7dff88458-hzcbh
5a836e2c09daa 6e38f40d628db 3 minutes ago Exited storage-provisioner 1 98195331b1ef5 storage-provisioner
1407bee0174af 12968670680f4 3 minutes ago Exited kindnet-cni 1 34835bd84b90a kindnet-f774d
e451bf82e08ea cbb01a7bd410d 3 minutes ago Exited coredns 1 1c2dfc549fb7a coredns-6f6b679f8f-pg7rx
e122dbe98fb29 ad83b2ca7b09e 3 minutes ago Exited kube-proxy 1 8b16e5b40fd00 kube-proxy-t4txm
251210cd4644c 38af8ddebf499 5 minutes ago Exited kube-vip 0 6584356547d91 kube-vip-ha-356000
9850a80a5c5f4 2e96e5913fc06 5 minutes ago Exited etcd 1 9d5dc5799203f etcd-ha-356000
21cc5ba42c46e 1766f54c897f0 5 minutes ago Exited kube-scheduler 1 56350ea8ea2cf kube-scheduler-ha-356000
==> coredns [23d42a24df9b] <==
.:53
[INFO] plugin/reload: Running configuration SHA512 = 257e111468ef6f1e36f10df061303186c353cd0e51aed8f50f4e4fd21cec02687aef97084fe1f82262f5cee88179d311670a6ae21ae185759728216fc264125f
CoreDNS-1.11.1
linux/amd64, go1.20.7, ae2bbc2
[INFO] 127.0.0.1:47495 - 15683 "HINFO IN 2368164196805048718.3506173865380983937. udp 57 false 512" NXDOMAIN qr,rd,ra 132 0.010784546s
[INFO] SIGTERM: Shutting down servers then terminating
[INFO] plugin/health: Going into lameduck mode for 5s
==> coredns [e451bf82e08e] <==
.:53
[INFO] plugin/reload: Running configuration SHA512 = 257e111468ef6f1e36f10df061303186c353cd0e51aed8f50f4e4fd21cec02687aef97084fe1f82262f5cee88179d311670a6ae21ae185759728216fc264125f
CoreDNS-1.11.1
linux/amd64, go1.20.7, ae2bbc2
[INFO] 127.0.0.1:41736 - 33655 "HINFO IN 7447381375798715563.6242938424915766931. udp 57 false 512" NXDOMAIN qr,rd,ra 132 0.000706353s
[INFO] 127.0.0.1:32933 - 19797 "HINFO IN 7447381375798715563.6242938424915766931. udp 57 false 512" NXDOMAIN qr,rd,ra 132 4.00282492s
[INFO] SIGTERM: Shutting down servers then terminating
[INFO] plugin/health: Going into lameduck mode for 5s
==> describe nodes <==
command /bin/bash -c "sudo /var/lib/minikube/binaries/v1.31.0/kubectl describe nodes --kubeconfig=/var/lib/minikube/kubeconfig" failed with error: /bin/bash -c "sudo /var/lib/minikube/binaries/v1.31.0/kubectl describe nodes --kubeconfig=/var/lib/minikube/kubeconfig": Process exited with status 1
stdout:
stderr:
E0819 11:36:50.566588 2982 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://localhost:8443/api?timeout=32s\": dial tcp 127.0.0.1:8443: connect: connection refused"
E0819 11:36:50.568307 2982 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://localhost:8443/api?timeout=32s\": dial tcp 127.0.0.1:8443: connect: connection refused"
E0819 11:36:50.569783 2982 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://localhost:8443/api?timeout=32s\": dial tcp 127.0.0.1:8443: connect: connection refused"
E0819 11:36:50.571337 2982 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://localhost:8443/api?timeout=32s\": dial tcp 127.0.0.1:8443: connect: connection refused"
E0819 11:36:50.572667 2982 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://localhost:8443/api?timeout=32s\": dial tcp 127.0.0.1:8443: connect: connection refused"
The connection to the server localhost:8443 was refused - did you specify the right host or port?
==> dmesg <==
[ +0.000000] Any video related functionality will be severely degraded, and you may not even be able to suspend the system properly
[ +0.000000] Unless you actually understand what nomodeset does, you should reboot without enabling it
[ +0.035301] ACPI BIOS Warning (bug): Incorrect checksum in table [DSDT] - 0xBE, should be 0x1B (20200925/tbprint-173)
[ +0.008078] RETBleed: WARNING: Spectre v2 mitigation leaves CPU vulnerable to RETBleed attacks, data leaks possible!
[ +5.692362] ACPI Error: Could not enable RealTimeClock event (20200925/evxfevnt-182)
[ +0.000002] ACPI Warning: Could not enable fixed event - RealTimeClock (4) (20200925/evxface-618)
[ +0.006772] platform regulatory.0: Direct firmware load for regulatory.db failed with error -2
[ +2.649041] systemd-fstab-generator[127]: Ignoring "noauto" option for root device
[ +2.231602] NFSD: Using /var/lib/nfs/v4recovery as the NFSv4 state recovery directory
[ +0.000007] NFSD: unable to find recovery directory /var/lib/nfs/v4recovery
[ +0.000002] NFSD: Unable to initialize client recovery tracking! (-2)
[ +24.708310] systemd-fstab-generator[495]: Ignoring "noauto" option for root device
[ +0.101320] systemd-fstab-generator[507]: Ignoring "noauto" option for root device
[ +1.932536] systemd-fstab-generator[1035]: Ignoring "noauto" option for root device
[ +0.256173] systemd-fstab-generator[1074]: Ignoring "noauto" option for root device
[ +0.097051] systemd-fstab-generator[1086]: Ignoring "noauto" option for root device
[ +0.116369] systemd-fstab-generator[1101]: Ignoring "noauto" option for root device
[ +2.460541] systemd-fstab-generator[1314]: Ignoring "noauto" option for root device
[ +0.050192] kauditd_printk_skb: 195 callbacks suppressed
[ +0.045227] systemd-fstab-generator[1326]: Ignoring "noauto" option for root device
[ +0.101932] systemd-fstab-generator[1338]: Ignoring "noauto" option for root device
[ +0.121540] systemd-fstab-generator[1353]: Ignoring "noauto" option for root device
[ +0.429125] systemd-fstab-generator[1519]: Ignoring "noauto" option for root device
[ +6.504409] kauditd_printk_skb: 110 callbacks suppressed
[Aug19 11:35] kauditd_printk_skb: 40 callbacks suppressed
==> etcd [09103e728061] <==
{"level":"warn","ts":"2024-08-19T11:36:45.472565Z","caller":"rafthttp/probing_status.go:68","msg":"prober detected unhealthy status","round-tripper-name":"ROUND_TRIPPER_SNAPSHOT","remote-peer-id":"83e7f3b71b594713","rtt":"0s","error":"dial tcp 192.169.0.39:2380: connect: connection refused"}
{"level":"warn","ts":"2024-08-19T11:36:45.472706Z","caller":"rafthttp/probing_status.go:68","msg":"prober detected unhealthy status","round-tripper-name":"ROUND_TRIPPER_RAFT_MESSAGE","remote-peer-id":"83e7f3b71b594713","rtt":"0s","error":"dial tcp 192.169.0.39:2380: connect: connection refused"}
{"level":"warn","ts":"2024-08-19T11:36:45.498962Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":12906069043412488472,"retry-timeout":"500ms"}
{"level":"warn","ts":"2024-08-19T11:36:46.003478Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":12906069043412488472,"retry-timeout":"500ms"}
{"level":"warn","ts":"2024-08-19T11:36:46.506306Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":12906069043412488472,"retry-timeout":"500ms"}
{"level":"info","ts":"2024-08-19T11:36:46.934307Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"56078c35c519b31b is starting a new election at term 3"}
{"level":"info","ts":"2024-08-19T11:36:46.934524Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"56078c35c519b31b became pre-candidate at term 3"}
{"level":"info","ts":"2024-08-19T11:36:46.934575Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"56078c35c519b31b received MsgPreVoteResp from 56078c35c519b31b at term 3"}
{"level":"info","ts":"2024-08-19T11:36:46.934660Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"56078c35c519b31b [logterm: 3, index: 2722] sent MsgPreVote request to 83e7f3b71b594713 at term 3"}
{"level":"warn","ts":"2024-08-19T11:36:47.012375Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":12906069043412488472,"retry-timeout":"500ms"}
{"level":"warn","ts":"2024-08-19T11:36:47.512794Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":12906069043412488472,"retry-timeout":"500ms"}
{"level":"warn","ts":"2024-08-19T11:36:47.977808Z","caller":"etcdserver/v3_server.go:932","msg":"timed out waiting for read index response (local node might have slow network)","timeout":"7s"}
{"level":"warn","ts":"2024-08-19T11:36:47.977940Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"7.000614685s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"","error":"etcdserver: request timed out"}
{"level":"info","ts":"2024-08-19T11:36:47.977986Z","caller":"traceutil/trace.go:171","msg":"trace[1775314289] range","detail":"{range_begin:; range_end:; }","duration":"7.000674372s","start":"2024-08-19T11:36:40.977301Z","end":"2024-08-19T11:36:47.977975Z","steps":["trace[1775314289] 'agreement among raft nodes before linearized reading' (duration: 7.000612053s)"],"step_count":1}
{"level":"error","ts":"2024-08-19T11:36:47.978021Z","caller":"etcdhttp/health.go:367","msg":"Health check error","path":"/readyz","reason":"[+]serializable_read ok\n[-]linearizable_read failed: etcdserver: request timed out\n[+]data_corruption ok\n","status-code":503,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp.(*CheckRegistry).installRootHttpEndpoint.newHealthHandler.func2\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp/health.go:367\nnet/http.HandlerFunc.ServeHTTP\n\tnet/http/server.go:2141\nnet/http.(*ServeMux).ServeHTTP\n\tnet/http/server.go:2519\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:2943\nnet/http.(*conn).serve\n\tnet/http/server.go:2014"}
{"level":"info","ts":"2024-08-19T11:36:48.436796Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"56078c35c519b31b is starting a new election at term 3"}
{"level":"info","ts":"2024-08-19T11:36:48.436876Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"56078c35c519b31b became pre-candidate at term 3"}
{"level":"info","ts":"2024-08-19T11:36:48.436895Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"56078c35c519b31b received MsgPreVoteResp from 56078c35c519b31b at term 3"}
{"level":"info","ts":"2024-08-19T11:36:48.436912Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"56078c35c519b31b [logterm: 3, index: 2722] sent MsgPreVote request to 83e7f3b71b594713 at term 3"}
{"level":"info","ts":"2024-08-19T11:36:49.934103Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"56078c35c519b31b is starting a new election at term 3"}
{"level":"info","ts":"2024-08-19T11:36:49.934164Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"56078c35c519b31b became pre-candidate at term 3"}
{"level":"info","ts":"2024-08-19T11:36:49.934176Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"56078c35c519b31b received MsgPreVoteResp from 56078c35c519b31b at term 3"}
{"level":"info","ts":"2024-08-19T11:36:49.934190Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"56078c35c519b31b [logterm: 3, index: 2722] sent MsgPreVote request to 83e7f3b71b594713 at term 3"}
{"level":"warn","ts":"2024-08-19T11:36:50.473627Z","caller":"rafthttp/probing_status.go:68","msg":"prober detected unhealthy status","round-tripper-name":"ROUND_TRIPPER_SNAPSHOT","remote-peer-id":"83e7f3b71b594713","rtt":"0s","error":"dial tcp 192.169.0.39:2380: connect: connection refused"}
{"level":"warn","ts":"2024-08-19T11:36:50.473770Z","caller":"rafthttp/probing_status.go:68","msg":"prober detected unhealthy status","round-tripper-name":"ROUND_TRIPPER_RAFT_MESSAGE","remote-peer-id":"83e7f3b71b594713","rtt":"0s","error":"dial tcp 192.169.0.39:2380: connect: connection refused"}
==> etcd [9850a80a5c5f] <==
{"level":"warn","ts":"2024-08-19T11:34:03.014505Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"3.559016378s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/horizontalpodautoscalers/\" range_end:\"/registry/horizontalpodautoscalers0\" count_only:true ","response":"","error":"context canceled"}
{"level":"info","ts":"2024-08-19T11:34:03.014515Z","caller":"traceutil/trace.go:171","msg":"trace[1990453727] range","detail":"{range_begin:/registry/horizontalpodautoscalers/; range_end:/registry/horizontalpodautoscalers0; }","duration":"3.559027803s","start":"2024-08-19T11:33:59.455485Z","end":"2024-08-19T11:34:03.014512Z","steps":["trace[1990453727] 'agreement among raft nodes before linearized reading' (duration: 3.559016106s)"],"step_count":1}
{"level":"warn","ts":"2024-08-19T11:34:03.014526Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2024-08-19T11:33:59.455468Z","time spent":"3.559055154s","remote":"127.0.0.1:60858","response type":"/etcdserverpb.KV/Range","request count":0,"request size":76,"response count":0,"response size":0,"request content":"key:\"/registry/horizontalpodautoscalers/\" range_end:\"/registry/horizontalpodautoscalers0\" count_only:true "}
2024/08/19 11:34:03 WARNING: [core] [Server #8] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing"
{"level":"warn","ts":"2024-08-19T11:34:03.014571Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"4.134292776s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/csistoragecapacities/\" range_end:\"/registry/csistoragecapacities0\" count_only:true ","response":"","error":"context canceled"}
{"level":"info","ts":"2024-08-19T11:34:03.014581Z","caller":"traceutil/trace.go:171","msg":"trace[231858823] range","detail":"{range_begin:/registry/csistoragecapacities/; range_end:/registry/csistoragecapacities0; }","duration":"4.134304954s","start":"2024-08-19T11:33:58.880273Z","end":"2024-08-19T11:34:03.014578Z","steps":["trace[231858823] 'agreement among raft nodes before linearized reading' (duration: 4.134292485s)"],"step_count":1}
{"level":"warn","ts":"2024-08-19T11:34:03.014597Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2024-08-19T11:33:58.880257Z","time spent":"4.134336256s","remote":"127.0.0.1:32810","response type":"/etcdserverpb.KV/Range","request count":0,"request size":68,"response count":0,"response size":0,"request content":"key:\"/registry/csistoragecapacities/\" range_end:\"/registry/csistoragecapacities0\" count_only:true "}
2024/08/19 11:34:03 WARNING: [core] [Server #8] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing"
{"level":"warn","ts":"2024-08-19T11:34:03.014642Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"6.391471892s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/endpoints/kube-system/k8s.io-minikube-hostpath\" ","response":"","error":"context canceled"}
{"level":"info","ts":"2024-08-19T11:34:03.014652Z","caller":"traceutil/trace.go:171","msg":"trace[2044934831] range","detail":"{range_begin:/registry/services/endpoints/kube-system/k8s.io-minikube-hostpath; range_end:; }","duration":"6.391483063s","start":"2024-08-19T11:33:56.623166Z","end":"2024-08-19T11:34:03.014649Z","steps":["trace[2044934831] 'agreement among raft nodes before linearized reading' (duration: 6.391471814s)"],"step_count":1}
{"level":"warn","ts":"2024-08-19T11:34:03.014661Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2024-08-19T11:33:56.623143Z","time spent":"6.391515438s","remote":"127.0.0.1:60806","response type":"/etcdserverpb.KV/Range","request count":0,"request size":67,"response count":0,"response size":0,"request content":"key:\"/registry/services/endpoints/kube-system/k8s.io-minikube-hostpath\" "}
2024/08/19 11:34:03 WARNING: [core] [Server #8] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing"
{"level":"warn","ts":"2024-08-19T11:34:03.041071Z","caller":"embed/serve.go:212","msg":"stopping secure grpc server due to error","error":"accept tcp 192.169.0.38:2379: use of closed network connection"}
{"level":"warn","ts":"2024-08-19T11:34:03.041099Z","caller":"embed/serve.go:214","msg":"stopped secure grpc server due to error","error":"accept tcp 192.169.0.38:2379: use of closed network connection"}
{"level":"info","ts":"2024-08-19T11:34:03.041162Z","caller":"etcdserver/server.go:1512","msg":"skipped leadership transfer; local server is not leader","local-member-id":"56078c35c519b31b","current-leader-member-id":"0"}
{"level":"info","ts":"2024-08-19T11:34:03.041249Z","caller":"rafthttp/peer.go:330","msg":"stopping remote peer","remote-peer-id":"83e7f3b71b594713"}
{"level":"info","ts":"2024-08-19T11:34:03.041258Z","caller":"rafthttp/stream.go:294","msg":"stopped TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","remote-peer-id":"83e7f3b71b594713"}
{"level":"info","ts":"2024-08-19T11:34:03.041270Z","caller":"rafthttp/stream.go:294","msg":"stopped TCP streaming connection with remote peer","stream-writer-type":"stream Message","remote-peer-id":"83e7f3b71b594713"}
{"level":"info","ts":"2024-08-19T11:34:03.041318Z","caller":"rafthttp/pipeline.go:85","msg":"stopped HTTP pipelining with remote peer","local-member-id":"56078c35c519b31b","remote-peer-id":"83e7f3b71b594713"}
{"level":"info","ts":"2024-08-19T11:34:03.041341Z","caller":"rafthttp/stream.go:442","msg":"stopped stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"56078c35c519b31b","remote-peer-id":"83e7f3b71b594713"}
{"level":"info","ts":"2024-08-19T11:34:03.041365Z","caller":"rafthttp/stream.go:442","msg":"stopped stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"56078c35c519b31b","remote-peer-id":"83e7f3b71b594713"}
{"level":"info","ts":"2024-08-19T11:34:03.041373Z","caller":"rafthttp/peer.go:335","msg":"stopped remote peer","remote-peer-id":"83e7f3b71b594713"}
{"level":"info","ts":"2024-08-19T11:34:03.045625Z","caller":"embed/etcd.go:581","msg":"stopping serving peer traffic","address":"192.169.0.38:2380"}
{"level":"info","ts":"2024-08-19T11:34:03.045696Z","caller":"embed/etcd.go:586","msg":"stopped serving peer traffic","address":"192.169.0.38:2380"}
{"level":"info","ts":"2024-08-19T11:34:03.045705Z","caller":"embed/etcd.go:379","msg":"closed etcd server","name":"ha-356000","data-dir":"/var/lib/minikube/etcd","advertise-peer-urls":["https://192.169.0.38:2380"],"advertise-client-urls":["https://192.169.0.38:2379"]}
==> kernel <==
11:36:50 up 2 min, 0 users, load average: 0.09, 0.07, 0.02
Linux ha-356000 5.10.207 #1 SMP Thu Aug 15 21:30:57 UTC 2024 x86_64 GNU/Linux
PRETTY_NAME="Buildroot 2023.02.9"
==> kindnet [1407bee0174a] <==
I0819 11:33:25.345422 1 main.go:322] Node ha-356000-m03 has CIDR [10.244.2.0/24]
I0819 11:33:25.345458 1 routes.go:62] Adding route {Ifindex: 0 Dst: 10.244.2.0/24 Src: <nil> Gw: 192.169.0.40 Flags: [] Table: 0}
I0819 11:33:25.346198 1 main.go:295] Handling node with IPs: map[192.169.0.41:{}]
I0819 11:33:25.346226 1 main.go:322] Node ha-356000-m04 has CIDR [10.244.3.0/24]
I0819 11:33:25.346371 1 routes.go:62] Adding route {Ifindex: 0 Dst: 10.244.3.0/24 Src: <nil> Gw: 192.169.0.41 Flags: [] Table: 0}
I0819 11:33:35.344515 1 main.go:295] Handling node with IPs: map[192.169.0.38:{}]
I0819 11:33:35.344939 1 main.go:299] handling current node
I0819 11:33:35.345156 1 main.go:295] Handling node with IPs: map[192.169.0.39:{}]
I0819 11:33:35.345839 1 main.go:322] Node ha-356000-m02 has CIDR [10.244.1.0/24]
I0819 11:33:35.347804 1 main.go:295] Handling node with IPs: map[192.169.0.40:{}]
I0819 11:33:35.347879 1 main.go:322] Node ha-356000-m03 has CIDR [10.244.2.0/24]
I0819 11:33:35.348003 1 main.go:295] Handling node with IPs: map[192.169.0.41:{}]
I0819 11:33:35.348073 1 main.go:322] Node ha-356000-m04 has CIDR [10.244.3.0/24]
I0819 11:33:45.345811 1 main.go:295] Handling node with IPs: map[192.169.0.41:{}]
I0819 11:33:45.346089 1 main.go:322] Node ha-356000-m04 has CIDR [10.244.3.0/24]
I0819 11:33:45.346629 1 main.go:295] Handling node with IPs: map[192.169.0.38:{}]
I0819 11:33:45.347248 1 main.go:299] handling current node
I0819 11:33:45.347372 1 main.go:295] Handling node with IPs: map[192.169.0.39:{}]
I0819 11:33:45.347524 1 main.go:322] Node ha-356000-m02 has CIDR [10.244.1.0/24]
I0819 11:33:55.343909 1 main.go:295] Handling node with IPs: map[192.169.0.38:{}]
I0819 11:33:55.343979 1 main.go:299] handling current node
I0819 11:33:55.343997 1 main.go:295] Handling node with IPs: map[192.169.0.39:{}]
I0819 11:33:55.344006 1 main.go:322] Node ha-356000-m02 has CIDR [10.244.1.0/24]
I0819 11:33:55.344203 1 main.go:295] Handling node with IPs: map[192.169.0.41:{}]
I0819 11:33:55.344386 1 main.go:322] Node ha-356000-m04 has CIDR [10.244.3.0/24]
==> kube-apiserver [860a05c56d10] <==
I0819 11:36:14.482465 1 options.go:228] external host was not specified, using 192.169.0.38
I0819 11:36:14.484845 1 server.go:142] Version: v1.31.0
I0819 11:36:14.484928 1 server.go:144] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
I0819 11:36:15.166151 1 shared_informer.go:313] Waiting for caches to sync for node_authorizer
I0819 11:36:15.169857 1 shared_informer.go:313] Waiting for caches to sync for *generic.policySource[*k8s.io/api/admissionregistration/v1.ValidatingAdmissionPolicy,*k8s.io/api/admissionregistration/v1.ValidatingAdmissionPolicyBinding,k8s.io/apiserver/pkg/admission/plugin/policy/validating.Validator]
I0819 11:36:15.172490 1 plugins.go:157] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
I0819 11:36:15.172582 1 plugins.go:160] Loaded 13 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,PodSecurity,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,ClusterTrustBundleAttest,CertificateSubjectRestriction,ValidatingAdmissionPolicy,ValidatingAdmissionWebhook,ResourceQuota.
I0819 11:36:15.172783 1 instance.go:232] Using reconciler: lease
W0819 11:36:35.166612 1 logging.go:55] [core] [Channel #1 SubChannel #3]grpc: addrConn.createTransport failed to connect to {Addr: "127.0.0.1:2379", ServerName: "127.0.0.1:2379", }. Err: connection error: desc = "transport: authentication handshake failed: context deadline exceeded"
W0819 11:36:35.166790 1 logging.go:55] [core] [Channel #2 SubChannel #4]grpc: addrConn.createTransport failed to connect to {Addr: "127.0.0.1:2379", ServerName: "127.0.0.1:2379", }. Err: connection error: desc = "transport: authentication handshake failed: context deadline exceeded"
W0819 11:36:35.173883 1 logging.go:55] [core] [Channel #5 SubChannel #6]grpc: addrConn.createTransport failed to connect to {Addr: "127.0.0.1:2379", ServerName: "127.0.0.1:2379", }. Err: connection error: desc = "transport: authentication handshake failed: context deadline exceeded"
F0819 11:36:35.173915 1 instance.go:225] Error creating leases: error creating storage factory: context deadline exceeded
==> kube-controller-manager [f19e14473bdc] <==
I0819 11:36:30.755338 1 serving.go:386] Generated self-signed cert in-memory
I0819 11:36:31.269815 1 controllermanager.go:197] "Starting" version="v1.31.0"
I0819 11:36:31.269888 1 controllermanager.go:199] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
I0819 11:36:31.270901 1 dynamic_cafile_content.go:160] "Starting controller" name="request-header::/var/lib/minikube/certs/front-proxy-ca.crt"
I0819 11:36:31.270918 1 secure_serving.go:213] Serving securely on 127.0.0.1:10257
I0819 11:36:31.271179 1 dynamic_cafile_content.go:160] "Starting controller" name="client-ca-bundle::/var/lib/minikube/certs/ca.crt"
I0819 11:36:31.271422 1 tlsconfig.go:243] "Starting DynamicServingCertificateController"
E0819 11:36:46.183957 1 controllermanager.go:242] "Error building controller context" err="failed to wait for apiserver being healthy: timed out waiting for the condition: failed to get apiserver /healthz status: Get \"https://192.169.0.38:8443/healthz\": dial tcp 192.169.0.38:8443: connect: connection refused"
==> kube-proxy [e122dbe98fb2] <==
add table ip kube-proxy
^^^^^^^^^^^^^^^^^^^^^^^^
>
E0819 11:33:10.263610 1 proxier.go:734] "Error cleaning up nftables rules" err=<
could not run nftables command: /dev/stdin:1:1-25: Error: Could not process rule: Operation not supported
add table ip6 kube-proxy
^^^^^^^^^^^^^^^^^^^^^^^^^
>
I0819 11:33:10.283145 1 server.go:677] "Successfully retrieved node IP(s)" IPs=["192.169.0.38"]
E0819 11:33:10.283216 1 server.go:234] "Kube-proxy configuration may be incomplete or incorrect" err="nodePortAddresses is unset; NodePort connections will be accepted on all local IPs. Consider using `--nodeport-addresses primary`"
I0819 11:33:10.317451 1 server_linux.go:146] "No iptables support for family" ipFamily="IPv6"
I0819 11:33:10.317537 1 server.go:245] "kube-proxy running in single-stack mode" ipFamily="IPv4"
I0819 11:33:10.317683 1 server_linux.go:169] "Using iptables Proxier"
I0819 11:33:10.319910 1 proxier.go:255] "Setting route_localnet=1 to allow node-ports on localhost; to change this either disable iptables.localhostNodePorts (--iptables-localhost-nodeports) or set nodePortAddresses (--nodeport-addresses) to filter loopback addresses" ipFamily="IPv4"
I0819 11:33:10.320687 1 server.go:483] "Version info" version="v1.31.0"
I0819 11:33:10.320739 1 server.go:485] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
I0819 11:33:10.323332 1 config.go:197] "Starting service config controller"
I0819 11:33:10.324220 1 shared_informer.go:313] Waiting for caches to sync for service config
I0819 11:33:10.324628 1 config.go:104] "Starting endpoint slice config controller"
I0819 11:33:10.324706 1 shared_informer.go:313] Waiting for caches to sync for endpoint slice config
I0819 11:33:10.326294 1 config.go:326] "Starting node config controller"
I0819 11:33:10.327153 1 shared_informer.go:313] Waiting for caches to sync for node config
I0819 11:33:10.424846 1 shared_informer.go:320] Caches are synced for service config
I0819 11:33:10.424854 1 shared_informer.go:320] Caches are synced for endpoint slice config
I0819 11:33:10.427282 1 shared_informer.go:320] Caches are synced for node config
==> kube-scheduler [21cc5ba42c46] <==
W0819 11:32:15.071469 1 reflector.go:561] runtime/asm_amd64.s:1695: failed to list *v1.ConfigMap: configmaps "extension-apiserver-authentication" is forbidden: User "system:kube-scheduler" cannot list resource "configmaps" in API group "" in the namespace "kube-system"
E0819 11:32:15.071529 1 reflector.go:158] "Unhandled Error" err="runtime/asm_amd64.s:1695: Failed to watch *v1.ConfigMap: failed to list *v1.ConfigMap: configmaps \"extension-apiserver-authentication\" is forbidden: User \"system:kube-scheduler\" cannot list resource \"configmaps\" in API group \"\" in the namespace \"kube-system\"" logger="UnhandledError"
W0819 11:32:15.104873 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.CSIStorageCapacity: csistoragecapacities.storage.k8s.io is forbidden: User "system:kube-scheduler" cannot list resource "csistoragecapacities" in API group "storage.k8s.io" at the cluster scope
E0819 11:32:15.104921 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.CSIStorageCapacity: failed to list *v1.CSIStorageCapacity: csistoragecapacities.storage.k8s.io is forbidden: User \"system:kube-scheduler\" cannot list resource \"csistoragecapacities\" in API group \"storage.k8s.io\" at the cluster scope" logger="UnhandledError"
W0819 11:32:15.104982 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.StatefulSet: statefulsets.apps is forbidden: User "system:kube-scheduler" cannot list resource "statefulsets" in API group "apps" at the cluster scope
E0819 11:32:15.105010 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.StatefulSet: failed to list *v1.StatefulSet: statefulsets.apps is forbidden: User \"system:kube-scheduler\" cannot list resource \"statefulsets\" in API group \"apps\" at the cluster scope" logger="UnhandledError"
W0819 11:32:15.105096 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.CSINode: csinodes.storage.k8s.io is forbidden: User "system:kube-scheduler" cannot list resource "csinodes" in API group "storage.k8s.io" at the cluster scope
E0819 11:32:15.105126 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.CSINode: failed to list *v1.CSINode: csinodes.storage.k8s.io is forbidden: User \"system:kube-scheduler\" cannot list resource \"csinodes\" in API group \"storage.k8s.io\" at the cluster scope" logger="UnhandledError"
W0819 11:32:15.105174 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.PersistentVolume: persistentvolumes is forbidden: User "system:kube-scheduler" cannot list resource "persistentvolumes" in API group "" at the cluster scope
E0819 11:32:15.105223 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.PersistentVolume: failed to list *v1.PersistentVolume: persistentvolumes is forbidden: User \"system:kube-scheduler\" cannot list resource \"persistentvolumes\" in API group \"\" at the cluster scope" logger="UnhandledError"
W0819 11:32:15.105274 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.Pod: pods is forbidden: User "system:kube-scheduler" cannot list resource "pods" in API group "" at the cluster scope
E0819 11:32:15.105323 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.Pod: failed to list *v1.Pod: pods is forbidden: User \"system:kube-scheduler\" cannot list resource \"pods\" in API group \"\" at the cluster scope" logger="UnhandledError"
W0819 11:32:15.105372 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.ReplicationController: replicationcontrollers is forbidden: User "system:kube-scheduler" cannot list resource "replicationcontrollers" in API group "" at the cluster scope
E0819 11:32:15.105419 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.ReplicationController: failed to list *v1.ReplicationController: replicationcontrollers is forbidden: User \"system:kube-scheduler\" cannot list resource \"replicationcontrollers\" in API group \"\" at the cluster scope" logger="UnhandledError"
W0819 11:32:15.105514 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.Node: nodes is forbidden: User "system:kube-scheduler" cannot list resource "nodes" in API group "" at the cluster scope
E0819 11:32:15.105791 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.Node: failed to list *v1.Node: nodes is forbidden: User \"system:kube-scheduler\" cannot list resource \"nodes\" in API group \"\" at the cluster scope" logger="UnhandledError"
W0819 11:32:15.105520 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.PodDisruptionBudget: poddisruptionbudgets.policy is forbidden: User "system:kube-scheduler" cannot list resource "poddisruptionbudgets" in API group "policy" at the cluster scope
E0819 11:32:15.105878 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.PodDisruptionBudget: failed to list *v1.PodDisruptionBudget: poddisruptionbudgets.policy is forbidden: User \"system:kube-scheduler\" cannot list resource \"poddisruptionbudgets\" in API group \"policy\" at the cluster scope" logger="UnhandledError"
I0819 11:32:33.917473 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
E0819 11:33:38.836990 1 framework.go:1305] "Plugin Failed" err="Operation cannot be fulfilled on pods/binding \"busybox-7dff88458-7hgk4\": pod busybox-7dff88458-7hgk4 is already assigned to node \"ha-356000-m04\"" plugin="DefaultBinder" pod="default/busybox-7dff88458-7hgk4" node="ha-356000-m04"
E0819 11:33:38.837329 1 schedule_one.go:1057] "Error scheduling pod; retrying" err="running Bind plugin \"DefaultBinder\": Operation cannot be fulfilled on pods/binding \"busybox-7dff88458-7hgk4\": pod busybox-7dff88458-7hgk4 is already assigned to node \"ha-356000-m04\"" pod="default/busybox-7dff88458-7hgk4"
I0819 11:34:03.098001 1 secure_serving.go:258] Stopped listening on 127.0.0.1:10259
I0819 11:34:03.098130 1 tlsconfig.go:258] "Shutting down DynamicServingCertificateController"
I0819 11:34:03.098362 1 configmap_cafile_content.go:226] "Shutting down controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file"
E0819 11:34:03.098476 1 run.go:72] "command failed" err="finished without leader elect"
==> kube-scheduler [b5f04621abe7] <==
E0819 11:36:11.814681 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.StatefulSet: failed to list *v1.StatefulSet: Get \"https://192.169.0.38:8443/apis/apps/v1/statefulsets?limit=500&resourceVersion=0\": dial tcp 192.169.0.38:8443: connect: connection refused" logger="UnhandledError"
W0819 11:36:25.038026 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.Namespace: Get "https://192.169.0.38:8443/api/v1/namespaces?limit=500&resourceVersion=0": net/http: TLS handshake timeout
E0819 11:36:25.038233 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.Namespace: failed to list *v1.Namespace: Get \"https://192.169.0.38:8443/api/v1/namespaces?limit=500&resourceVersion=0\": net/http: TLS handshake timeout" logger="UnhandledError"
W0819 11:36:25.217382 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.Node: Get "https://192.169.0.38:8443/api/v1/nodes?limit=500&resourceVersion=0": net/http: TLS handshake timeout
E0819 11:36:25.217474 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.Node: failed to list *v1.Node: Get \"https://192.169.0.38:8443/api/v1/nodes?limit=500&resourceVersion=0\": net/http: TLS handshake timeout" logger="UnhandledError"
W0819 11:36:25.244110 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.CSIDriver: Get "https://192.169.0.38:8443/apis/storage.k8s.io/v1/csidrivers?limit=500&resourceVersion=0": net/http: TLS handshake timeout
E0819 11:36:25.244395 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.CSIDriver: failed to list *v1.CSIDriver: Get \"https://192.169.0.38:8443/apis/storage.k8s.io/v1/csidrivers?limit=500&resourceVersion=0\": net/http: TLS handshake timeout" logger="UnhandledError"
W0819 11:36:27.092612 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.PodDisruptionBudget: Get "https://192.169.0.38:8443/apis/policy/v1/poddisruptionbudgets?limit=500&resourceVersion=0": net/http: TLS handshake timeout
E0819 11:36:27.092712 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.PodDisruptionBudget: failed to list *v1.PodDisruptionBudget: Get \"https://192.169.0.38:8443/apis/policy/v1/poddisruptionbudgets?limit=500&resourceVersion=0\": net/http: TLS handshake timeout" logger="UnhandledError"
W0819 11:36:36.179492 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.Pod: Get "https://192.169.0.38:8443/api/v1/pods?fieldSelector=status.phase%21%3DSucceeded%2Cstatus.phase%21%3DFailed&limit=500&resourceVersion=0": dial tcp 192.169.0.38:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.38:48806->192.169.0.38:8443: read: connection reset by peer
E0819 11:36:36.179591 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.Pod: failed to list *v1.Pod: Get \"https://192.169.0.38:8443/api/v1/pods?fieldSelector=status.phase%21%3DSucceeded%2Cstatus.phase%21%3DFailed&limit=500&resourceVersion=0\": dial tcp 192.169.0.38:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.38:48806->192.169.0.38:8443: read: connection reset by peer" logger="UnhandledError"
W0819 11:36:36.712136 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.StorageClass: Get "https://192.169.0.38:8443/apis/storage.k8s.io/v1/storageclasses?limit=500&resourceVersion=0": dial tcp 192.169.0.38:8443: connect: connection refused
E0819 11:36:36.712331 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.StorageClass: failed to list *v1.StorageClass: Get \"https://192.169.0.38:8443/apis/storage.k8s.io/v1/storageclasses?limit=500&resourceVersion=0\": dial tcp 192.169.0.38:8443: connect: connection refused" logger="UnhandledError"
W0819 11:36:37.151477 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.PersistentVolumeClaim: Get "https://192.169.0.38:8443/api/v1/persistentvolumeclaims?limit=500&resourceVersion=0": dial tcp 192.169.0.38:8443: connect: connection refused
E0819 11:36:37.151577 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.PersistentVolumeClaim: failed to list *v1.PersistentVolumeClaim: Get \"https://192.169.0.38:8443/api/v1/persistentvolumeclaims?limit=500&resourceVersion=0\": dial tcp 192.169.0.38:8443: connect: connection refused" logger="UnhandledError"
W0819 11:36:39.003559 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.StatefulSet: Get "https://192.169.0.38:8443/apis/apps/v1/statefulsets?limit=500&resourceVersion=0": dial tcp 192.169.0.38:8443: connect: connection refused
E0819 11:36:39.003937 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.StatefulSet: failed to list *v1.StatefulSet: Get \"https://192.169.0.38:8443/apis/apps/v1/statefulsets?limit=500&resourceVersion=0\": dial tcp 192.169.0.38:8443: connect: connection refused" logger="UnhandledError"
W0819 11:36:43.603583 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.Service: Get "https://192.169.0.38:8443/api/v1/services?limit=500&resourceVersion=0": dial tcp 192.169.0.38:8443: connect: connection refused
E0819 11:36:43.603753 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.Service: failed to list *v1.Service: Get \"https://192.169.0.38:8443/api/v1/services?limit=500&resourceVersion=0\": dial tcp 192.169.0.38:8443: connect: connection refused" logger="UnhandledError"
W0819 11:36:45.464756 1 reflector.go:561] runtime/asm_amd64.s:1695: failed to list *v1.ConfigMap: Get "https://192.169.0.38:8443/api/v1/namespaces/kube-system/configmaps?fieldSelector=metadata.name%3Dextension-apiserver-authentication&limit=500&resourceVersion=0": dial tcp 192.169.0.38:8443: connect: connection refused
E0819 11:36:45.464822 1 reflector.go:158] "Unhandled Error" err="runtime/asm_amd64.s:1695: Failed to watch *v1.ConfigMap: failed to list *v1.ConfigMap: Get \"https://192.169.0.38:8443/api/v1/namespaces/kube-system/configmaps?fieldSelector=metadata.name%3Dextension-apiserver-authentication&limit=500&resourceVersion=0\": dial tcp 192.169.0.38:8443: connect: connection refused" logger="UnhandledError"
W0819 11:36:46.286002 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.CSINode: Get "https://192.169.0.38:8443/apis/storage.k8s.io/v1/csinodes?limit=500&resourceVersion=0": dial tcp 192.169.0.38:8443: connect: connection refused
E0819 11:36:46.286213 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.CSINode: failed to list *v1.CSINode: Get \"https://192.169.0.38:8443/apis/storage.k8s.io/v1/csinodes?limit=500&resourceVersion=0\": dial tcp 192.169.0.38:8443: connect: connection refused" logger="UnhandledError"
W0819 11:36:46.832941 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.CSIStorageCapacity: Get "https://192.169.0.38:8443/apis/storage.k8s.io/v1/csistoragecapacities?limit=500&resourceVersion=0": dial tcp 192.169.0.38:8443: connect: connection refused
E0819 11:36:46.833039 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.CSIStorageCapacity: failed to list *v1.CSIStorageCapacity: Get \"https://192.169.0.38:8443/apis/storage.k8s.io/v1/csistoragecapacities?limit=500&resourceVersion=0\": dial tcp 192.169.0.38:8443: connect: connection refused" logger="UnhandledError"
==> kubelet <==
Aug 19 11:36:33 ha-356000 kubelet[1526]: E0819 11:36:33.335010 1526 eviction_manager.go:285] "Eviction manager: failed to get summary stats" err="failed to get node info: node \"ha-356000\" not found"
Aug 19 11:36:35 ha-356000 kubelet[1526]: I0819 11:36:35.483337 1526 kubelet_node_status.go:72] "Attempting to register node" node="ha-356000"
Aug 19 11:36:35 ha-356000 kubelet[1526]: I0819 11:36:35.628932 1526 scope.go:117] "RemoveContainer" containerID="bc9a2e06b275830d33b503f1b026fb1010d6095ea619910504bbfa7ba0115af5"
Aug 19 11:36:35 ha-356000 kubelet[1526]: I0819 11:36:35.629568 1526 scope.go:117] "RemoveContainer" containerID="860a05c56d106dc2e8fbe54d2d4a3c870b95690d8845f6ae5eabf52b76ab6422"
Aug 19 11:36:35 ha-356000 kubelet[1526]: E0819 11:36:35.629666 1526 pod_workers.go:1301] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"kube-apiserver\" with CrashLoopBackOff: \"back-off 40s restarting failed container=kube-apiserver pod=kube-apiserver-ha-356000_kube-system(718e22639bba4cf009fc5275fb2200cd)\"" pod="kube-system/kube-apiserver-ha-356000" podUID="718e22639bba4cf009fc5275fb2200cd"
Aug 19 11:36:37 ha-356000 kubelet[1526]: E0819 11:36:37.705269 1526 kubelet_node_status.go:95] "Unable to register node with API server" err="Post \"https://control-plane.minikube.internal:8443/api/v1/nodes\": dial tcp 192.169.0.254:8443: connect: no route to host" node="ha-356000"
Aug 19 11:36:37 ha-356000 kubelet[1526]: E0819 11:36:37.705307 1526 controller.go:145] "Failed to ensure lease exists, will retry" err="Get \"https://control-plane.minikube.internal:8443/apis/coordination.k8s.io/v1/namespaces/kube-node-lease/leases/ha-356000?timeout=10s\": dial tcp 192.169.0.254:8443: connect: no route to host" interval="7s"
Aug 19 11:36:37 ha-356000 kubelet[1526]: W0819 11:36:37.705407 1526 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.CSIDriver: Get "https://control-plane.minikube.internal:8443/apis/storage.k8s.io/v1/csidrivers?limit=500&resourceVersion=0": dial tcp 192.169.0.254:8443: connect: no route to host
Aug 19 11:36:37 ha-356000 kubelet[1526]: E0819 11:36:37.706045 1526 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.CSIDriver: failed to list *v1.CSIDriver: Get \"https://control-plane.minikube.internal:8443/apis/storage.k8s.io/v1/csidrivers?limit=500&resourceVersion=0\": dial tcp 192.169.0.254:8443: connect: no route to host" logger="UnhandledError"
Aug 19 11:36:40 ha-356000 kubelet[1526]: I0819 11:36:40.680524 1526 scope.go:117] "RemoveContainer" containerID="860a05c56d106dc2e8fbe54d2d4a3c870b95690d8845f6ae5eabf52b76ab6422"
Aug 19 11:36:40 ha-356000 kubelet[1526]: E0819 11:36:40.680715 1526 pod_workers.go:1301] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"kube-apiserver\" with CrashLoopBackOff: \"back-off 40s restarting failed container=kube-apiserver pod=kube-apiserver-ha-356000_kube-system(718e22639bba4cf009fc5275fb2200cd)\"" pod="kube-system/kube-apiserver-ha-356000" podUID="718e22639bba4cf009fc5275fb2200cd"
Aug 19 11:36:43 ha-356000 kubelet[1526]: E0819 11:36:43.335222 1526 eviction_manager.go:285] "Eviction manager: failed to get summary stats" err="failed to get node info: node \"ha-356000\" not found"
Aug 19 11:36:43 ha-356000 kubelet[1526]: I0819 11:36:43.570340 1526 scope.go:117] "RemoveContainer" containerID="860a05c56d106dc2e8fbe54d2d4a3c870b95690d8845f6ae5eabf52b76ab6422"
Aug 19 11:36:43 ha-356000 kubelet[1526]: E0819 11:36:43.570557 1526 pod_workers.go:1301] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"kube-apiserver\" with CrashLoopBackOff: \"back-off 40s restarting failed container=kube-apiserver pod=kube-apiserver-ha-356000_kube-system(718e22639bba4cf009fc5275fb2200cd)\"" pod="kube-system/kube-apiserver-ha-356000" podUID="718e22639bba4cf009fc5275fb2200cd"
Aug 19 11:36:43 ha-356000 kubelet[1526]: E0819 11:36:43.842461 1526 event.go:368] "Unable to write event (may retry after sleeping)" err="Post \"https://control-plane.minikube.internal:8443/api/v1/namespaces/default/events\": dial tcp 192.169.0.254:8443: connect: no route to host" event="&Event{ObjectMeta:{ha-356000.17ed1e119800e8aa default 0 0001-01-01 00:00:00 +0000 UTC <nil> <nil> map[] map[] [] [] []},InvolvedObject:ObjectReference{Kind:Node,Namespace:,Name:ha-356000,UID:ha-356000,APIVersion:,ResourceVersion:,FieldPath:,},Reason:Starting,Message:Starting kubelet.,Source:EventSource{Component:kubelet,Host:ha-356000,},FirstTimestamp:2024-08-19 11:34:53.266241706 +0000 UTC m=+0.114912176,LastTimestamp:2024-08-19 11:34:53.266241706 +0000 UTC m=+0.114912176,Count:1,Type:Normal,EventTime:0001-01-01 00:00:00 +0000 UTC,Series:nil,Action:,Related:nil,ReportingController:kubelet,ReportingInstance:ha-356000,}"
Aug 19 11:36:44 ha-356000 kubelet[1526]: I0819 11:36:44.709298 1526 kubelet_node_status.go:72] "Attempting to register node" node="ha-356000"
Aug 19 11:36:46 ha-356000 kubelet[1526]: I0819 11:36:46.736845 1526 scope.go:117] "RemoveContainer" containerID="259361a45e235c109ea54b1f11a2ffc395a0946fabb67691546f8127ff992e47"
Aug 19 11:36:46 ha-356000 kubelet[1526]: I0819 11:36:46.737635 1526 scope.go:117] "RemoveContainer" containerID="f19e14473bdcae42a398f77919e5f2f5fd297f2cc629eb3c7ca44798a2883025"
Aug 19 11:36:46 ha-356000 kubelet[1526]: E0819 11:36:46.737713 1526 pod_workers.go:1301] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"kube-controller-manager\" with CrashLoopBackOff: \"back-off 40s restarting failed container=kube-controller-manager pod=kube-controller-manager-ha-356000_kube-system(869c3bcad30564e753dc51559e76cb65)\"" pod="kube-system/kube-controller-manager-ha-356000" podUID="869c3bcad30564e753dc51559e76cb65"
Aug 19 11:36:46 ha-356000 kubelet[1526]: E0819 11:36:46.917152 1526 kubelet_node_status.go:95] "Unable to register node with API server" err="Post \"https://control-plane.minikube.internal:8443/api/v1/nodes\": dial tcp 192.169.0.254:8443: connect: no route to host" node="ha-356000"
Aug 19 11:36:46 ha-356000 kubelet[1526]: E0819 11:36:46.917176 1526 controller.go:145] "Failed to ensure lease exists, will retry" err="Get \"https://control-plane.minikube.internal:8443/apis/coordination.k8s.io/v1/namespaces/kube-node-lease/leases/ha-356000?timeout=10s\": dial tcp 192.169.0.254:8443: connect: no route to host" interval="7s"
Aug 19 11:36:47 ha-356000 kubelet[1526]: I0819 11:36:47.748625 1526 scope.go:117] "RemoveContainer" containerID="f19e14473bdcae42a398f77919e5f2f5fd297f2cc629eb3c7ca44798a2883025"
Aug 19 11:36:47 ha-356000 kubelet[1526]: E0819 11:36:47.748746 1526 pod_workers.go:1301] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"kube-controller-manager\" with CrashLoopBackOff: \"back-off 40s restarting failed container=kube-controller-manager pod=kube-controller-manager-ha-356000_kube-system(869c3bcad30564e753dc51559e76cb65)\"" pod="kube-system/kube-controller-manager-ha-356000" podUID="869c3bcad30564e753dc51559e76cb65"
Aug 19 11:36:48 ha-356000 kubelet[1526]: I0819 11:36:48.755665 1526 scope.go:117] "RemoveContainer" containerID="f19e14473bdcae42a398f77919e5f2f5fd297f2cc629eb3c7ca44798a2883025"
Aug 19 11:36:48 ha-356000 kubelet[1526]: E0819 11:36:48.755778 1526 pod_workers.go:1301] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"kube-controller-manager\" with CrashLoopBackOff: \"back-off 40s restarting failed container=kube-controller-manager pod=kube-controller-manager-ha-356000_kube-system(869c3bcad30564e753dc51559e76cb65)\"" pod="kube-system/kube-controller-manager-ha-356000" podUID="869c3bcad30564e753dc51559e76cb65"
-- /stdout --
helpers_test.go:254: (dbg) Run: out/minikube-darwin-amd64 status --format={{.APIServer}} -p ha-356000 -n ha-356000
helpers_test.go:254: (dbg) Non-zero exit: out/minikube-darwin-amd64 status --format={{.APIServer}} -p ha-356000 -n ha-356000: exit status 2 (150.579614ms)
-- stdout --
Stopped
-- /stdout --
helpers_test.go:254: status error: exit status 2 (may be ok)
helpers_test.go:256: "ha-356000" apiserver is not running, skipping kubectl commands (state="Stopped")
--- FAIL: TestMultiControlPlane/serial/DegradedAfterClusterRestart (2.70s)