=== RUN TestMultiControlPlane/serial/DegradedAfterClusterRestart
ha_test.go:390: (dbg) Run: out/minikube-darwin-amd64 profile list --output json
ha_test.go:413: expected profile "ha-373000" in json of 'profile list' to have "Degraded" status but have "Stopped" status. got *"{\"invalid\":[],\"valid\":[{\"Name\":\"ha-373000\",\"Status\":\"Stopped\",\"Config\":{\"Name\":\"ha-373000\",\"KeepContext\":false,\"EmbedCerts\":false,\"MinikubeISO\":\"https://storage.googleapis.com/minikube-builds/iso/19452/minikube-v1.33.1-1723740674-19452-amd64.iso\",\"KicBaseImage\":\"gcr.io/k8s-minikube/kicbase-builds:v0.0.44-1723740748-19452@sha256:2211a6931895d2d502e957e9667096db10734a96767d670cb4dbffdd37397b0d\",\"Memory\":2200,\"CPUs\":2,\"DiskSize\":20000,\"Driver\":\"hyperkit\",\"HyperkitVpnKitSock\":\"\",\"HyperkitVSockPorts\":[],\"DockerEnv\":null,\"ContainerVolumeMounts\":null,\"InsecureRegistry\":null,\"RegistryMirror\":[],\"HostOnlyCIDR\":\"192.168.59.1/24\",\"HypervVirtualSwitch\":\"\",\"HypervUseExternalSwitch\":false,\"HypervExternalAdapter\":\"\",\"KVMNetwork\":\"default\",\"KVMQemuURI\":\"qemu:///system\",\"KVMGPU\":false,\"KVMHidden\":false,\"KVMNUMACoun
t\":1,\"APIServerPort\":8443,\"DockerOpt\":null,\"DisableDriverMounts\":false,\"NFSShare\":[],\"NFSSharesRoot\":\"/nfsshares\",\"UUID\":\"\",\"NoVTXCheck\":false,\"DNSProxy\":false,\"HostDNSResolver\":true,\"HostOnlyNicType\":\"virtio\",\"NatNicType\":\"virtio\",\"SSHIPAddress\":\"\",\"SSHUser\":\"root\",\"SSHKey\":\"\",\"SSHPort\":22,\"KubernetesConfig\":{\"KubernetesVersion\":\"v1.31.0\",\"ClusterName\":\"ha-373000\",\"Namespace\":\"default\",\"APIServerHAVIP\":\"192.169.0.254\",\"APIServerName\":\"minikubeCA\",\"APIServerNames\":null,\"APIServerIPs\":null,\"DNSDomain\":\"cluster.local\",\"ContainerRuntime\":\"docker\",\"CRISocket\":\"\",\"NetworkPlugin\":\"cni\",\"FeatureGates\":\"\",\"ServiceCIDR\":\"10.96.0.0/12\",\"ImageRepository\":\"\",\"LoadBalancerStartIP\":\"\",\"LoadBalancerEndIP\":\"\",\"CustomIngressCert\":\"\",\"RegistryAliases\":\"\",\"ExtraOptions\":null,\"ShouldLoadCachedImages\":true,\"EnableDefaultCNI\":false,\"CNI\":\"\"},\"Nodes\":[{\"Name\":\"\",\"IP\":\"192.169.0.5\",\"Port\":8443,\"Ku
bernetesVersion\":\"v1.31.0\",\"ContainerRuntime\":\"docker\",\"ControlPlane\":true,\"Worker\":true},{\"Name\":\"m02\",\"IP\":\"192.169.0.6\",\"Port\":8443,\"KubernetesVersion\":\"v1.31.0\",\"ContainerRuntime\":\"docker\",\"ControlPlane\":true,\"Worker\":true},{\"Name\":\"m04\",\"IP\":\"192.169.0.8\",\"Port\":0,\"KubernetesVersion\":\"v1.31.0\",\"ContainerRuntime\":\"docker\",\"ControlPlane\":false,\"Worker\":true}],\"Addons\":{\"ambassador\":false,\"auto-pause\":false,\"cloud-spanner\":false,\"csi-hostpath-driver\":false,\"dashboard\":false,\"default-storageclass\":false,\"efk\":false,\"freshpod\":false,\"gcp-auth\":false,\"gvisor\":false,\"headlamp\":false,\"helm-tiller\":false,\"inaccel\":false,\"ingress\":false,\"ingress-dns\":false,\"inspektor-gadget\":false,\"istio\":false,\"istio-provisioner\":false,\"kong\":false,\"kubeflow\":false,\"kubevirt\":false,\"logviewer\":false,\"metallb\":false,\"metrics-server\":false,\"nvidia-device-plugin\":false,\"nvidia-driver-installer\":false,\"nvidia-gpu-device-plugi
n\":false,\"olm\":false,\"pod-security-policy\":false,\"portainer\":false,\"registry\":false,\"registry-aliases\":false,\"registry-creds\":false,\"storage-provisioner\":false,\"storage-provisioner-gluster\":false,\"storage-provisioner-rancher\":false,\"volcano\":false,\"volumesnapshots\":false,\"yakd\":false},\"CustomAddonImages\":null,\"CustomAddonRegistries\":null,\"VerifyComponents\":{\"apiserver\":true,\"apps_running\":true,\"default_sa\":true,\"extra\":true,\"kubelet\":true,\"node_ready\":true,\"system_pods\":true},\"StartHostTimeout\":360000000000,\"ScheduledStop\":null,\"ExposedPorts\":[],\"ListenAddress\":\"\",\"Network\":\"\",\"Subnet\":\"\",\"MultiNodeRequested\":true,\"ExtraDisks\":0,\"CertExpiration\":94608000000000000,\"Mount\":false,\"MountString\":\"/Users:/minikube-host\",\"Mount9PVersion\":\"9p2000.L\",\"MountGID\":\"docker\",\"MountIP\":\"\",\"MountMSize\":262144,\"MountOptions\":[],\"MountPort\":0,\"MountType\":\"9p\",\"MountUID\":\"docker\",\"BinaryMirror\":\"\",\"DisableOptimizations\":fa
lse,\"DisableMetrics\":false,\"CustomQemuFirmwarePath\":\"\",\"SocketVMnetClientPath\":\"\",\"SocketVMnetPath\":\"\",\"StaticIP\":\"\",\"SSHAuthSock\":\"\",\"SSHAgentPID\":0,\"GPUs\":\"\",\"AutoPauseInterval\":60000000000},\"Active\":false,\"ActiveKubeContext\":true}]}"*. args: "out/minikube-darwin-amd64 profile list --output json"
helpers_test.go:222: -----------------------post-mortem--------------------------------
helpers_test.go:239: (dbg) Run: out/minikube-darwin-amd64 status --format={{.Host}} -p ha-373000 -n ha-373000
helpers_test.go:239: (dbg) Non-zero exit: out/minikube-darwin-amd64 status --format={{.Host}} -p ha-373000 -n ha-373000: exit status 2 (149.589788ms)
-- stdout --
Running
-- /stdout --
helpers_test.go:239: status error: exit status 2 (may be ok)
helpers_test.go:244: <<< TestMultiControlPlane/serial/DegradedAfterClusterRestart FAILED: start of post-mortem logs <<<
helpers_test.go:245: ======> post-mortem[TestMultiControlPlane/serial/DegradedAfterClusterRestart]: minikube logs <======
helpers_test.go:247: (dbg) Run: out/minikube-darwin-amd64 -p ha-373000 logs -n 25
helpers_test.go:247: (dbg) Done: out/minikube-darwin-amd64 -p ha-373000 logs -n 25: (2.093832712s)
helpers_test.go:252: TestMultiControlPlane/serial/DegradedAfterClusterRestart logs:
-- stdout --
==> Audit <==
|---------|------------------------------------------------------------------------------------------------------------------------------|-----------|---------|---------|---------------------|---------------------|
| Command | Args | Profile | User | Version | Start Time | End Time |
|---------|------------------------------------------------------------------------------------------------------------------------------|-----------|---------|---------|---------------------|---------------------|
| cp | ha-373000 cp ha-373000-m03:/home/docker/cp-test.txt | ha-373000 | jenkins | v1.33.1 | 18 Aug 24 12:03 PDT | 18 Aug 24 12:03 PDT |
| | ha-373000-m04:/home/docker/cp-test_ha-373000-m03_ha-373000-m04.txt | | | | | |
| ssh | ha-373000 ssh -n | ha-373000 | jenkins | v1.33.1 | 18 Aug 24 12:03 PDT | 18 Aug 24 12:03 PDT |
| | ha-373000-m03 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| ssh | ha-373000 ssh -n ha-373000-m04 sudo cat | ha-373000 | jenkins | v1.33.1 | 18 Aug 24 12:03 PDT | 18 Aug 24 12:03 PDT |
| | /home/docker/cp-test_ha-373000-m03_ha-373000-m04.txt | | | | | |
| cp | ha-373000 cp testdata/cp-test.txt | ha-373000 | jenkins | v1.33.1 | 18 Aug 24 12:03 PDT | 18 Aug 24 12:03 PDT |
| | ha-373000-m04:/home/docker/cp-test.txt | | | | | |
| ssh | ha-373000 ssh -n | ha-373000 | jenkins | v1.33.1 | 18 Aug 24 12:03 PDT | 18 Aug 24 12:03 PDT |
| | ha-373000-m04 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| cp | ha-373000 cp ha-373000-m04:/home/docker/cp-test.txt | ha-373000 | jenkins | v1.33.1 | 18 Aug 24 12:03 PDT | 18 Aug 24 12:03 PDT |
| | /var/folders/vq/yhv778t970xgml0dzm5fdwlr0000gp/T/TestMultiControlPlaneserialCopyFile1633039751/001/cp-test_ha-373000-m04.txt | | | | | |
| ssh | ha-373000 ssh -n | ha-373000 | jenkins | v1.33.1 | 18 Aug 24 12:03 PDT | 18 Aug 24 12:03 PDT |
| | ha-373000-m04 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| cp | ha-373000 cp ha-373000-m04:/home/docker/cp-test.txt | ha-373000 | jenkins | v1.33.1 | 18 Aug 24 12:03 PDT | 18 Aug 24 12:03 PDT |
| | ha-373000:/home/docker/cp-test_ha-373000-m04_ha-373000.txt | | | | | |
| ssh | ha-373000 ssh -n | ha-373000 | jenkins | v1.33.1 | 18 Aug 24 12:03 PDT | 18 Aug 24 12:03 PDT |
| | ha-373000-m04 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| ssh | ha-373000 ssh -n ha-373000 sudo cat | ha-373000 | jenkins | v1.33.1 | 18 Aug 24 12:03 PDT | 18 Aug 24 12:03 PDT |
| | /home/docker/cp-test_ha-373000-m04_ha-373000.txt | | | | | |
| cp | ha-373000 cp ha-373000-m04:/home/docker/cp-test.txt | ha-373000 | jenkins | v1.33.1 | 18 Aug 24 12:03 PDT | 18 Aug 24 12:03 PDT |
| | ha-373000-m02:/home/docker/cp-test_ha-373000-m04_ha-373000-m02.txt | | | | | |
| ssh | ha-373000 ssh -n | ha-373000 | jenkins | v1.33.1 | 18 Aug 24 12:03 PDT | 18 Aug 24 12:03 PDT |
| | ha-373000-m04 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| ssh | ha-373000 ssh -n ha-373000-m02 sudo cat | ha-373000 | jenkins | v1.33.1 | 18 Aug 24 12:03 PDT | 18 Aug 24 12:03 PDT |
| | /home/docker/cp-test_ha-373000-m04_ha-373000-m02.txt | | | | | |
| cp | ha-373000 cp ha-373000-m04:/home/docker/cp-test.txt | ha-373000 | jenkins | v1.33.1 | 18 Aug 24 12:03 PDT | 18 Aug 24 12:03 PDT |
| | ha-373000-m03:/home/docker/cp-test_ha-373000-m04_ha-373000-m03.txt | | | | | |
| ssh | ha-373000 ssh -n | ha-373000 | jenkins | v1.33.1 | 18 Aug 24 12:03 PDT | 18 Aug 24 12:03 PDT |
| | ha-373000-m04 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| ssh | ha-373000 ssh -n ha-373000-m03 sudo cat | ha-373000 | jenkins | v1.33.1 | 18 Aug 24 12:03 PDT | 18 Aug 24 12:03 PDT |
| | /home/docker/cp-test_ha-373000-m04_ha-373000-m03.txt | | | | | |
| node | ha-373000 node stop m02 -v=7 | ha-373000 | jenkins | v1.33.1 | 18 Aug 24 12:03 PDT | 18 Aug 24 12:03 PDT |
| | --alsologtostderr | | | | | |
| node | ha-373000 node start m02 -v=7 | ha-373000 | jenkins | v1.33.1 | 18 Aug 24 12:03 PDT | 18 Aug 24 12:04 PDT |
| | --alsologtostderr | | | | | |
| node | list -p ha-373000 -v=7 | ha-373000 | jenkins | v1.33.1 | 18 Aug 24 12:04 PDT | |
| | --alsologtostderr | | | | | |
| stop | -p ha-373000 -v=7 | ha-373000 | jenkins | v1.33.1 | 18 Aug 24 12:04 PDT | 18 Aug 24 12:04 PDT |
| | --alsologtostderr | | | | | |
| start | -p ha-373000 --wait=true -v=7 | ha-373000 | jenkins | v1.33.1 | 18 Aug 24 12:04 PDT | |
| | --alsologtostderr | | | | | |
| node | list -p ha-373000 | ha-373000 | jenkins | v1.33.1 | 18 Aug 24 12:08 PDT | |
| node | ha-373000 node delete m03 -v=7 | ha-373000 | jenkins | v1.33.1 | 18 Aug 24 12:08 PDT | 18 Aug 24 12:08 PDT |
| | --alsologtostderr | | | | | |
| stop | ha-373000 stop -v=7 | ha-373000 | jenkins | v1.33.1 | 18 Aug 24 12:08 PDT | 18 Aug 24 12:09 PDT |
| | --alsologtostderr | | | | | |
| start | -p ha-373000 --wait=true | ha-373000 | jenkins | v1.33.1 | 18 Aug 24 12:09 PDT | |
| | -v=7 --alsologtostderr | | | | | |
| | --driver=hyperkit | | | | | |
|---------|------------------------------------------------------------------------------------------------------------------------------|-----------|---------|---------|---------------------|---------------------|
==> Last Start <==
Log file created at: 2024/08/18 12:09:00
Running on machine: MacOS-Agent-2
Binary: Built with gc go1.22.5 for darwin/amd64
Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg
I0818 12:09:00.388954 3976 out.go:345] Setting OutFile to fd 1 ...
I0818 12:09:00.389224 3976 out.go:392] TERM=,COLORTERM=, which probably does not support color
I0818 12:09:00.389230 3976 out.go:358] Setting ErrFile to fd 2...
I0818 12:09:00.389234 3976 out.go:392] TERM=,COLORTERM=, which probably does not support color
I0818 12:09:00.389403 3976 root.go:338] Updating PATH: /Users/jenkins/minikube-integration/19423-1007/.minikube/bin
I0818 12:09:00.390788 3976 out.go:352] Setting JSON to false
I0818 12:09:00.412980 3976 start.go:129] hostinfo: {"hostname":"MacOS-Agent-2.local","uptime":2311,"bootTime":1724005829,"procs":422,"os":"darwin","platform":"darwin","platformFamily":"Standalone Workstation","platformVersion":"14.6.1","kernelVersion":"23.6.0","kernelArch":"x86_64","virtualizationSystem":"","virtualizationRole":"","hostId":"2965c349-98a5-5970-aaa9-9eedd3ae5959"}
W0818 12:09:00.413073 3976 start.go:137] gopshost.Virtualization returned error: not implemented yet
I0818 12:09:00.435491 3976 out.go:177] * [ha-373000] minikube v1.33.1 on Darwin 14.6.1
I0818 12:09:00.478012 3976 out.go:177] - MINIKUBE_LOCATION=19423
I0818 12:09:00.478041 3976 notify.go:220] Checking for updates...
I0818 12:09:00.520842 3976 out.go:177] - KUBECONFIG=/Users/jenkins/minikube-integration/19423-1007/kubeconfig
I0818 12:09:00.541902 3976 out.go:177] - MINIKUBE_BIN=out/minikube-darwin-amd64
I0818 12:09:00.562974 3976 out.go:177] - MINIKUBE_SUPPRESS_DOCKER_PERFORMANCE=true
I0818 12:09:00.583978 3976 out.go:177] - MINIKUBE_HOME=/Users/jenkins/minikube-integration/19423-1007/.minikube
I0818 12:09:00.604937 3976 out.go:177] - MINIKUBE_FORCE_SYSTEMD=
I0818 12:09:00.626633 3976 config.go:182] Loaded profile config "ha-373000": Driver=hyperkit, ContainerRuntime=docker, KubernetesVersion=v1.31.0
I0818 12:09:00.627309 3976 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0818 12:09:00.627392 3976 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0818 12:09:00.636929 3976 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:52014
I0818 12:09:00.637287 3976 main.go:141] libmachine: () Calling .GetVersion
I0818 12:09:00.637735 3976 main.go:141] libmachine: Using API Version 1
I0818 12:09:00.637744 3976 main.go:141] libmachine: () Calling .SetConfigRaw
I0818 12:09:00.637948 3976 main.go:141] libmachine: () Calling .GetMachineName
I0818 12:09:00.638063 3976 main.go:141] libmachine: (ha-373000) Calling .DriverName
I0818 12:09:00.638277 3976 driver.go:394] Setting default libvirt URI to qemu:///system
I0818 12:09:00.638525 3976 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0818 12:09:00.638545 3976 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0818 12:09:00.646880 3976 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:52016
I0818 12:09:00.647224 3976 main.go:141] libmachine: () Calling .GetVersion
I0818 12:09:00.647595 3976 main.go:141] libmachine: Using API Version 1
I0818 12:09:00.647613 3976 main.go:141] libmachine: () Calling .SetConfigRaw
I0818 12:09:00.647826 3976 main.go:141] libmachine: () Calling .GetMachineName
I0818 12:09:00.647950 3976 main.go:141] libmachine: (ha-373000) Calling .DriverName
I0818 12:09:00.676977 3976 out.go:177] * Using the hyperkit driver based on existing profile
I0818 12:09:00.718931 3976 start.go:297] selected driver: hyperkit
I0818 12:09:00.718961 3976 start.go:901] validating driver "hyperkit" against &{Name:ha-373000 KeepContext:false EmbedCerts:false MinikubeISO:https://storage.googleapis.com/minikube-builds/iso/19452/minikube-v1.33.1-1723740674-19452-amd64.iso KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.44-1723740748-19452@sha256:2211a6931895d2d502e957e9667096db10734a96767d670cb4dbffdd37397b0d Memory:2200 CPUs:2 DiskSize:20000 Driver:hyperkit HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{Kuber
netesVersion:v1.31.0 ClusterName:ha-373000 Namespace:default APIServerHAVIP:192.169.0.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:docker CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.169.0.5 Port:8443 KubernetesVersion:v1.31.0 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m02 IP:192.169.0.6 Port:8443 KubernetesVersion:v1.31.0 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m04 IP:192.169.0.8 Port:0 KubernetesVersion:v1.31.0 ContainerRuntime: ControlPlane:false Worker:true}] Addons:map[ambassador:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false helm-tiller:false inaccel:false ingress:false
ingress-dns:false inspektor-gadget:false istio:false istio-provisioner:false kong:false kubeflow:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-gluster:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s Mount:false MountString:/Users:/minikube-host Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirro
r: DisableOptimizations:false DisableMetrics:false CustomQemuFirmwarePath: SocketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s}
I0818 12:09:00.719183 3976 start.go:912] status for hyperkit: {Installed:true Healthy:true Running:true NeedsImprovement:false Error:<nil> Reason: Fix: Doc: Version:}
I0818 12:09:00.719386 3976 install.go:52] acquiring lock: {Name:mk4023283b30b374c3f04c8805d539e68824c0b8 Clock:{} Delay:500ms Timeout:10m0s Cancel:<nil>}
I0818 12:09:00.719595 3976 install.go:117] Validating docker-machine-driver-hyperkit, PATH=/Users/jenkins/minikube-integration/19423-1007/.minikube/bin:/Users/jenkins/workspace/out/:/usr/bin:/bin:/usr/sbin:/sbin:/Users/jenkins/google-cloud-sdk/bin:/usr/local/bin/:/usr/local/go/bin/:/Users/jenkins/go/bin
I0818 12:09:00.729307 3976 install.go:137] /Users/jenkins/workspace/out/docker-machine-driver-hyperkit version is 1.33.1
I0818 12:09:00.733175 3976 install.go:79] stdout: /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0818 12:09:00.733199 3976 install.go:81] /Users/jenkins/workspace/out/docker-machine-driver-hyperkit looks good
I0818 12:09:00.735834 3976 start_flags.go:947] Waiting for all components: map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true]
I0818 12:09:00.735880 3976 cni.go:84] Creating CNI manager for ""
I0818 12:09:00.735888 3976 cni.go:136] multinode detected (3 nodes found), recommending kindnet
I0818 12:09:00.735960 3976 start.go:340] cluster config:
{Name:ha-373000 KeepContext:false EmbedCerts:false MinikubeISO:https://storage.googleapis.com/minikube-builds/iso/19452/minikube-v1.33.1-1723740674-19452-amd64.iso KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.44-1723740748-19452@sha256:2211a6931895d2d502e957e9667096db10734a96767d670cb4dbffdd37397b0d Memory:2200 CPUs:2 DiskSize:20000 Driver:hyperkit HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.31.0 ClusterName:ha-373000 Namespace:default APIServerHAVIP:192.16
9.0.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:docker CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.169.0.5 Port:8443 KubernetesVersion:v1.31.0 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m02 IP:192.169.0.6 Port:8443 KubernetesVersion:v1.31.0 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m04 IP:192.169.0.8 Port:0 KubernetesVersion:v1.31.0 ContainerRuntime:docker ControlPlane:false Worker:true}] Addons:map[ambassador:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false helm-tiller:false inaccel:false ingress:false ingress-dns:false inspektor-gadget:false istio:false istio-provisioner:false
kong:false kubeflow:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-gluster:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s Mount:false MountString:/Users:/minikube-host Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOptimizations:false DisableMetrics:false CustomQemuFirmwarePath: S
ocketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s}
I0818 12:09:00.736064 3976 iso.go:125] acquiring lock: {Name:mkfcc70059a4397f76252ba67d02448d3569c468 Clock:{} Delay:500ms Timeout:10m0s Cancel:<nil>}
I0818 12:09:00.757023 3976 out.go:177] * Starting "ha-373000" primary control-plane node in "ha-373000" cluster
I0818 12:09:00.777783 3976 preload.go:131] Checking if preload exists for k8s version v1.31.0 and runtime docker
I0818 12:09:00.777901 3976 preload.go:146] Found local preload: /Users/jenkins/minikube-integration/19423-1007/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.31.0-docker-overlay2-amd64.tar.lz4
I0818 12:09:00.777924 3976 cache.go:56] Caching tarball of preloaded images
I0818 12:09:00.778128 3976 preload.go:172] Found /Users/jenkins/minikube-integration/19423-1007/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.31.0-docker-overlay2-amd64.tar.lz4 in cache, skipping download
I0818 12:09:00.778148 3976 cache.go:59] Finished verifying existence of preloaded tar for v1.31.0 on docker
I0818 12:09:00.778333 3976 profile.go:143] Saving config to /Users/jenkins/minikube-integration/19423-1007/.minikube/profiles/ha-373000/config.json ...
I0818 12:09:00.779289 3976 start.go:360] acquireMachinesLock for ha-373000: {Name:mk31060d3397796eb838e7923e770e40ab40e545 Clock:{} Delay:500ms Timeout:13m0s Cancel:<nil>}
I0818 12:09:00.779483 3976 start.go:364] duration metric: took 143.76µs to acquireMachinesLock for "ha-373000"
I0818 12:09:00.779521 3976 start.go:96] Skipping create...Using existing machine configuration
I0818 12:09:00.779537 3976 fix.go:54] fixHost starting:
I0818 12:09:00.779956 3976 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0818 12:09:00.779984 3976 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0818 12:09:00.789309 3976 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:52018
I0818 12:09:00.789666 3976 main.go:141] libmachine: () Calling .GetVersion
I0818 12:09:00.790031 3976 main.go:141] libmachine: Using API Version 1
I0818 12:09:00.790040 3976 main.go:141] libmachine: () Calling .SetConfigRaw
I0818 12:09:00.790251 3976 main.go:141] libmachine: () Calling .GetMachineName
I0818 12:09:00.790366 3976 main.go:141] libmachine: (ha-373000) Calling .DriverName
I0818 12:09:00.790468 3976 main.go:141] libmachine: (ha-373000) Calling .GetState
I0818 12:09:00.790556 3976 main.go:141] libmachine: (ha-373000) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I0818 12:09:00.790639 3976 main.go:141] libmachine: (ha-373000) DBG | hyperkit pid from json: 3836
I0818 12:09:00.791548 3976 main.go:141] libmachine: (ha-373000) DBG | hyperkit pid 3836 missing from process table
I0818 12:09:00.791593 3976 fix.go:112] recreateIfNeeded on ha-373000: state=Stopped err=<nil>
I0818 12:09:00.791619 3976 main.go:141] libmachine: (ha-373000) Calling .DriverName
W0818 12:09:00.791703 3976 fix.go:138] unexpected machine state, will restart: <nil>
I0818 12:09:00.833742 3976 out.go:177] * Restarting existing hyperkit VM for "ha-373000" ...
I0818 12:09:00.854617 3976 main.go:141] libmachine: (ha-373000) Calling .Start
I0818 12:09:00.854890 3976 main.go:141] libmachine: (ha-373000) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I0818 12:09:00.854917 3976 main.go:141] libmachine: (ha-373000) minikube might have been shutdown in an unclean way, the hyperkit pid file still exists: /Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000/hyperkit.pid
I0818 12:09:00.856657 3976 main.go:141] libmachine: (ha-373000) DBG | hyperkit pid 3836 missing from process table
I0818 12:09:00.856693 3976 main.go:141] libmachine: (ha-373000) DBG | pid 3836 is in state "Stopped"
I0818 12:09:00.856718 3976 main.go:141] libmachine: (ha-373000) DBG | Removing stale pid file /Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000/hyperkit.pid...
I0818 12:09:00.856984 3976 main.go:141] libmachine: (ha-373000) DBG | Using UUID 2f6e5f86-d003-4f9b-8f55-d5f48a14c3df
I0818 12:09:00.989123 3976 main.go:141] libmachine: (ha-373000) DBG | Generated MAC be:21:66:25:9a:b1
I0818 12:09:00.989174 3976 main.go:141] libmachine: (ha-373000) DBG | Starting with cmdline: loglevel=3 console=ttyS0 console=tty0 noembed nomodeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-373000
I0818 12:09:00.989237 3976 main.go:141] libmachine: (ha-373000) DBG | 2024/08/18 12:09:00 DEBUG: hyperkit: Start &hyperkit.HyperKit{HyperKit:"/usr/local/bin/hyperkit", Argv0:"", StateDir:"/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000", VPNKitSock:"", VPNKitUUID:"", VPNKitPreferredIPv4:"", UUID:"2f6e5f86-d003-4f9b-8f55-d5f48a14c3df", Disks:[]hyperkit.Disk{(*hyperkit.RawDisk)(0xc0003c2540)}, ISOImages:[]string{"/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000/boot2docker.iso"}, VSock:false, VSockDir:"", VSockPorts:[]int(nil), VSockGuestCID:3, VMNet:true, Sockets9P:[]hyperkit.Socket9P(nil), Kernel:"/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000/bzimage", Initrd:"/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000/initrd", Bootrom:"", CPUs:2, Memory:2200, Console:1, Serials:[]hyperkit.Serial(nil), Pid:0, Arguments:[]string(nil), CmdLine:"", process:(*os.Process)(nil)}
I0818 12:09:00.989280 3976 main.go:141] libmachine: (ha-373000) DBG | 2024/08/18 12:09:00 DEBUG: hyperkit: check &hyperkit.HyperKit{HyperKit:"/usr/local/bin/hyperkit", Argv0:"", StateDir:"/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000", VPNKitSock:"", VPNKitUUID:"", VPNKitPreferredIPv4:"", UUID:"2f6e5f86-d003-4f9b-8f55-d5f48a14c3df", Disks:[]hyperkit.Disk{(*hyperkit.RawDisk)(0xc0003c2540)}, ISOImages:[]string{"/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000/boot2docker.iso"}, VSock:false, VSockDir:"", VSockPorts:[]int(nil), VSockGuestCID:3, VMNet:true, Sockets9P:[]hyperkit.Socket9P(nil), Kernel:"/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000/bzimage", Initrd:"/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000/initrd", Bootrom:"", CPUs:2, Memory:2200, Console:1, Serials:[]hyperkit.Serial(nil), Pid:0, Arguments:[]string(nil), CmdLine:"", process:(*os.Process)(nil)}
I0818 12:09:00.989323 3976 main.go:141] libmachine: (ha-373000) DBG | 2024/08/18 12:09:00 DEBUG: hyperkit: Arguments: []string{"-A", "-u", "-F", "/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000/hyperkit.pid", "-c", "2", "-m", "2200M", "-s", "0:0,hostbridge", "-s", "31,lpc", "-s", "1:0,virtio-net", "-U", "2f6e5f86-d003-4f9b-8f55-d5f48a14c3df", "-s", "2:0,virtio-blk,/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000/ha-373000.rawdisk", "-s", "3,ahci-cd,/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000/boot2docker.iso", "-s", "4,virtio-rnd", "-l", "com1,autopty=/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000/tty,log=/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000/console-ring", "-f", "kexec,/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000/bzimage,/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000/initrd,earlyprintk=s
erial loglevel=3 console=ttyS0 console=tty0 noembed nomodeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-373000"}
I0818 12:09:00.989366 3976 main.go:141] libmachine: (ha-373000) DBG | 2024/08/18 12:09:00 DEBUG: hyperkit: CmdLine: "/usr/local/bin/hyperkit -A -u -F /Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000/hyperkit.pid -c 2 -m 2200M -s 0:0,hostbridge -s 31,lpc -s 1:0,virtio-net -U 2f6e5f86-d003-4f9b-8f55-d5f48a14c3df -s 2:0,virtio-blk,/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000/ha-373000.rawdisk -s 3,ahci-cd,/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000/boot2docker.iso -s 4,virtio-rnd -l com1,autopty=/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000/tty,log=/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000/console-ring -f kexec,/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000/bzimage,/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000/initrd,earlyprintk=serial loglevel=3 console=ttyS0 console=tty0 noembed nomodeset
norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-373000"
I0818 12:09:00.989381 3976 main.go:141] libmachine: (ha-373000) DBG | 2024/08/18 12:09:00 DEBUG: hyperkit: Redirecting stdout/stderr to logger
I0818 12:09:00.990799 3976 main.go:141] libmachine: (ha-373000) DBG | 2024/08/18 12:09:00 DEBUG: hyperkit: Pid is 3990
I0818 12:09:00.991176 3976 main.go:141] libmachine: (ha-373000) DBG | Attempt 0
I0818 12:09:00.991196 3976 main.go:141] libmachine: (ha-373000) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I0818 12:09:00.991218 3976 main.go:141] libmachine: (ha-373000) DBG | hyperkit pid from json: 3990
I0818 12:09:00.993000 3976 main.go:141] libmachine: (ha-373000) DBG | Searching for be:21:66:25:9a:b1 in /var/db/dhcpd_leases ...
I0818 12:09:00.993068 3976 main.go:141] libmachine: (ha-373000) DBG | Found 7 entries in /var/db/dhcpd_leases!
I0818 12:09:00.993082 3976 main.go:141] libmachine: (ha-373000) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.7 HWAddress:72:9e:9b:7f:e6:a8 ID:1,72:9e:9b:7f:e6:a8 Lease:0x66c246ad}
I0818 12:09:00.993090 3976 main.go:141] libmachine: (ha-373000) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.8 HWAddress:f2:8c:91:ee:dd:c0 ID:1,f2:8c:91:ee:dd:c0 Lease:0x66c397e0}
I0818 12:09:00.993097 3976 main.go:141] libmachine: (ha-373000) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.6 HWAddress:ca:b5:c4:e6:47:79 ID:1,ca:b5:c4:e6:47:79 Lease:0x66c3975b}
I0818 12:09:00.993119 3976 main.go:141] libmachine: (ha-373000) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.5 HWAddress:be:21:66:25:9a:b1 ID:1,be:21:66:25:9a:b1 Lease:0x66c39749}
I0818 12:09:00.993129 3976 main.go:141] libmachine: (ha-373000) DBG | Found match: be:21:66:25:9a:b1
I0818 12:09:00.993139 3976 main.go:141] libmachine: (ha-373000) DBG | IP: 192.169.0.5
I0818 12:09:00.993184 3976 main.go:141] libmachine: (ha-373000) Calling .GetConfigRaw
I0818 12:09:00.994094 3976 main.go:141] libmachine: (ha-373000) Calling .GetIP
I0818 12:09:00.994333 3976 profile.go:143] Saving config to /Users/jenkins/minikube-integration/19423-1007/.minikube/profiles/ha-373000/config.json ...
I0818 12:09:00.994945 3976 machine.go:93] provisionDockerMachine start ...
I0818 12:09:00.994967 3976 main.go:141] libmachine: (ha-373000) Calling .DriverName
I0818 12:09:00.995142 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHHostname
I0818 12:09:00.995271 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHPort
I0818 12:09:00.995391 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHKeyPath
I0818 12:09:00.995521 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHKeyPath
I0818 12:09:00.995632 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHUsername
I0818 12:09:00.995768 3976 main.go:141] libmachine: Using SSH client type: native
I0818 12:09:00.996051 3976 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0x3bf3ea0] 0x3bf6c00 <nil> [] 0s} 192.169.0.5 22 <nil> <nil>}
I0818 12:09:00.996062 3976 main.go:141] libmachine: About to run SSH command:
hostname
I0818 12:09:00.999904 3976 main.go:141] libmachine: (ha-373000) DBG | 2024/08/18 12:09:00 INFO : hyperkit: stderr: Using fd 5 for I/O notifications
I0818 12:09:01.080830 3976 main.go:141] libmachine: (ha-373000) DBG | 2024/08/18 12:09:01 INFO : hyperkit: stderr: /Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000/boot2docker.iso: fcntl(F_PUNCHHOLE) Operation not permitted: block device will not support TRIM/DISCARD
I0818 12:09:01.081571 3976 main.go:141] libmachine: (ha-373000) DBG | 2024/08/18 12:09:01 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 2 bit: 22 unspecified don't care: bit is 0
I0818 12:09:01.081587 3976 main.go:141] libmachine: (ha-373000) DBG | 2024/08/18 12:09:01 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 12 unspecified don't care: bit is 0
I0818 12:09:01.081595 3976 main.go:141] libmachine: (ha-373000) DBG | 2024/08/18 12:09:01 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 20 unspecified don't care: bit is 0
I0818 12:09:01.081604 3976 main.go:141] libmachine: (ha-373000) DBG | 2024/08/18 12:09:01 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 3 bit: 13 unspecified don't care: bit is 0
I0818 12:09:01.460230 3976 main.go:141] libmachine: (ha-373000) DBG | 2024/08/18 12:09:01 INFO : hyperkit: stderr: rdmsr to register 0x3a on vcpu 0
I0818 12:09:01.460268 3976 main.go:141] libmachine: (ha-373000) DBG | 2024/08/18 12:09:01 INFO : hyperkit: stderr: rdmsr to register 0x140 on vcpu 0
I0818 12:09:01.574713 3976 main.go:141] libmachine: (ha-373000) DBG | 2024/08/18 12:09:01 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 2 bit: 22 unspecified don't care: bit is 0
I0818 12:09:01.574755 3976 main.go:141] libmachine: (ha-373000) DBG | 2024/08/18 12:09:01 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 12 unspecified don't care: bit is 0
I0818 12:09:01.574768 3976 main.go:141] libmachine: (ha-373000) DBG | 2024/08/18 12:09:01 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 20 unspecified don't care: bit is 0
I0818 12:09:01.574787 3976 main.go:141] libmachine: (ha-373000) DBG | 2024/08/18 12:09:01 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 3 bit: 13 unspecified don't care: bit is 0
I0818 12:09:01.575699 3976 main.go:141] libmachine: (ha-373000) DBG | 2024/08/18 12:09:01 INFO : hyperkit: stderr: rdmsr to register 0x3a on vcpu 1
I0818 12:09:01.575710 3976 main.go:141] libmachine: (ha-373000) DBG | 2024/08/18 12:09:01 INFO : hyperkit: stderr: rdmsr to register 0x140 on vcpu 1
I0818 12:09:07.163001 3976 main.go:141] libmachine: (ha-373000) DBG | 2024/08/18 12:09:07 INFO : hyperkit: stderr: rdmsr to register 0x64d on vcpu 1
I0818 12:09:07.163029 3976 main.go:141] libmachine: (ha-373000) DBG | 2024/08/18 12:09:07 INFO : hyperkit: stderr: rdmsr to register 0x64e on vcpu 1
I0818 12:09:07.163053 3976 main.go:141] libmachine: (ha-373000) DBG | 2024/08/18 12:09:07 INFO : hyperkit: stderr: rdmsr to register 0x34 on vcpu 1
I0818 12:09:07.186829 3976 main.go:141] libmachine: (ha-373000) DBG | 2024/08/18 12:09:07 INFO : hyperkit: stderr: rdmsr to register 0xc0011029 on vcpu 1
I0818 12:09:12.062770 3976 main.go:141] libmachine: SSH cmd err, output: <nil>: minikube
I0818 12:09:12.062784 3976 main.go:141] libmachine: (ha-373000) Calling .GetMachineName
I0818 12:09:12.062975 3976 buildroot.go:166] provisioning hostname "ha-373000"
I0818 12:09:12.062986 3976 main.go:141] libmachine: (ha-373000) Calling .GetMachineName
I0818 12:09:12.063087 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHHostname
I0818 12:09:12.063175 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHPort
I0818 12:09:12.063280 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHKeyPath
I0818 12:09:12.063371 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHKeyPath
I0818 12:09:12.063480 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHUsername
I0818 12:09:12.063605 3976 main.go:141] libmachine: Using SSH client type: native
I0818 12:09:12.063750 3976 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0x3bf3ea0] 0x3bf6c00 <nil> [] 0s} 192.169.0.5 22 <nil> <nil>}
I0818 12:09:12.063759 3976 main.go:141] libmachine: About to run SSH command:
sudo hostname ha-373000 && echo "ha-373000" | sudo tee /etc/hostname
I0818 12:09:12.131801 3976 main.go:141] libmachine: SSH cmd err, output: <nil>: ha-373000
I0818 12:09:12.131819 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHHostname
I0818 12:09:12.131954 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHPort
I0818 12:09:12.132061 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHKeyPath
I0818 12:09:12.132144 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHKeyPath
I0818 12:09:12.132224 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHUsername
I0818 12:09:12.132376 3976 main.go:141] libmachine: Using SSH client type: native
I0818 12:09:12.132528 3976 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0x3bf3ea0] 0x3bf6c00 <nil> [] 0s} 192.169.0.5 22 <nil> <nil>}
I0818 12:09:12.132546 3976 main.go:141] libmachine: About to run SSH command:
if ! grep -xq '.*\sha-373000' /etc/hosts; then
if grep -xq '127.0.1.1\s.*' /etc/hosts; then
sudo sed -i 's/^127.0.1.1\s.*/127.0.1.1 ha-373000/g' /etc/hosts;
else
echo '127.0.1.1 ha-373000' | sudo tee -a /etc/hosts;
fi
fi
I0818 12:09:12.199331 3976 main.go:141] libmachine: SSH cmd err, output: <nil>:
I0818 12:09:12.199349 3976 buildroot.go:172] set auth options {CertDir:/Users/jenkins/minikube-integration/19423-1007/.minikube CaCertPath:/Users/jenkins/minikube-integration/19423-1007/.minikube/certs/ca.pem CaPrivateKeyPath:/Users/jenkins/minikube-integration/19423-1007/.minikube/certs/ca-key.pem CaCertRemotePath:/etc/docker/ca.pem ServerCertPath:/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/server.pem ServerKeyPath:/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/server-key.pem ClientKeyPath:/Users/jenkins/minikube-integration/19423-1007/.minikube/certs/key.pem ServerCertRemotePath:/etc/docker/server.pem ServerKeyRemotePath:/etc/docker/server-key.pem ClientCertPath:/Users/jenkins/minikube-integration/19423-1007/.minikube/certs/cert.pem ServerCertSANs:[] StorePath:/Users/jenkins/minikube-integration/19423-1007/.minikube}
I0818 12:09:12.199369 3976 buildroot.go:174] setting up certificates
I0818 12:09:12.199383 3976 provision.go:84] configureAuth start
I0818 12:09:12.199391 3976 main.go:141] libmachine: (ha-373000) Calling .GetMachineName
I0818 12:09:12.199540 3976 main.go:141] libmachine: (ha-373000) Calling .GetIP
I0818 12:09:12.199634 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHHostname
I0818 12:09:12.199719 3976 provision.go:143] copyHostCerts
I0818 12:09:12.199749 3976 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19423-1007/.minikube/certs/ca.pem -> /Users/jenkins/minikube-integration/19423-1007/.minikube/ca.pem
I0818 12:09:12.199819 3976 exec_runner.go:144] found /Users/jenkins/minikube-integration/19423-1007/.minikube/ca.pem, removing ...
I0818 12:09:12.199828 3976 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19423-1007/.minikube/ca.pem
I0818 12:09:12.199960 3976 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19423-1007/.minikube/certs/ca.pem --> /Users/jenkins/minikube-integration/19423-1007/.minikube/ca.pem (1082 bytes)
I0818 12:09:12.200176 3976 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19423-1007/.minikube/certs/cert.pem -> /Users/jenkins/minikube-integration/19423-1007/.minikube/cert.pem
I0818 12:09:12.200222 3976 exec_runner.go:144] found /Users/jenkins/minikube-integration/19423-1007/.minikube/cert.pem, removing ...
I0818 12:09:12.200227 3976 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19423-1007/.minikube/cert.pem
I0818 12:09:12.200306 3976 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19423-1007/.minikube/certs/cert.pem --> /Users/jenkins/minikube-integration/19423-1007/.minikube/cert.pem (1123 bytes)
I0818 12:09:12.200461 3976 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19423-1007/.minikube/certs/key.pem -> /Users/jenkins/minikube-integration/19423-1007/.minikube/key.pem
I0818 12:09:12.200505 3976 exec_runner.go:144] found /Users/jenkins/minikube-integration/19423-1007/.minikube/key.pem, removing ...
I0818 12:09:12.200509 3976 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19423-1007/.minikube/key.pem
I0818 12:09:12.200584 3976 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19423-1007/.minikube/certs/key.pem --> /Users/jenkins/minikube-integration/19423-1007/.minikube/key.pem (1675 bytes)
I0818 12:09:12.200731 3976 provision.go:117] generating server cert: /Users/jenkins/minikube-integration/19423-1007/.minikube/machines/server.pem ca-key=/Users/jenkins/minikube-integration/19423-1007/.minikube/certs/ca.pem private-key=/Users/jenkins/minikube-integration/19423-1007/.minikube/certs/ca-key.pem org=jenkins.ha-373000 san=[127.0.0.1 192.169.0.5 ha-373000 localhost minikube]
I0818 12:09:12.289022 3976 provision.go:177] copyRemoteCerts
I0818 12:09:12.289076 3976 ssh_runner.go:195] Run: sudo mkdir -p /etc/docker /etc/docker /etc/docker
I0818 12:09:12.289091 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHHostname
I0818 12:09:12.289227 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHPort
I0818 12:09:12.289322 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHKeyPath
I0818 12:09:12.289416 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHUsername
I0818 12:09:12.289508 3976 sshutil.go:53] new ssh client: &{IP:192.169.0.5 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000/id_rsa Username:docker}
I0818 12:09:12.325856 3976 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19423-1007/.minikube/certs/ca.pem -> /etc/docker/ca.pem
I0818 12:09:12.325929 3976 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19423-1007/.minikube/certs/ca.pem --> /etc/docker/ca.pem (1082 bytes)
I0818 12:09:12.345953 3976 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19423-1007/.minikube/machines/server.pem -> /etc/docker/server.pem
I0818 12:09:12.346012 3976 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19423-1007/.minikube/machines/server.pem --> /etc/docker/server.pem (1200 bytes)
I0818 12:09:12.366027 3976 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19423-1007/.minikube/machines/server-key.pem -> /etc/docker/server-key.pem
I0818 12:09:12.366092 3976 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19423-1007/.minikube/machines/server-key.pem --> /etc/docker/server-key.pem (1679 bytes)
I0818 12:09:12.386212 3976 provision.go:87] duration metric: took 186.823558ms to configureAuth
I0818 12:09:12.386225 3976 buildroot.go:189] setting minikube options for container-runtime
I0818 12:09:12.386405 3976 config.go:182] Loaded profile config "ha-373000": Driver=hyperkit, ContainerRuntime=docker, KubernetesVersion=v1.31.0
I0818 12:09:12.386418 3976 main.go:141] libmachine: (ha-373000) Calling .DriverName
I0818 12:09:12.386551 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHHostname
I0818 12:09:12.386643 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHPort
I0818 12:09:12.386731 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHKeyPath
I0818 12:09:12.386817 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHKeyPath
I0818 12:09:12.386909 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHUsername
I0818 12:09:12.387025 3976 main.go:141] libmachine: Using SSH client type: native
I0818 12:09:12.387159 3976 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0x3bf3ea0] 0x3bf6c00 <nil> [] 0s} 192.169.0.5 22 <nil> <nil>}
I0818 12:09:12.387167 3976 main.go:141] libmachine: About to run SSH command:
df --output=fstype / | tail -n 1
I0818 12:09:12.445833 3976 main.go:141] libmachine: SSH cmd err, output: <nil>: tmpfs
I0818 12:09:12.445851 3976 buildroot.go:70] root file system type: tmpfs
I0818 12:09:12.445930 3976 provision.go:314] Updating docker unit: /lib/systemd/system/docker.service ...
I0818 12:09:12.445943 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHHostname
I0818 12:09:12.446067 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHPort
I0818 12:09:12.446173 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHKeyPath
I0818 12:09:12.446279 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHKeyPath
I0818 12:09:12.446389 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHUsername
I0818 12:09:12.446543 3976 main.go:141] libmachine: Using SSH client type: native
I0818 12:09:12.446679 3976 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0x3bf3ea0] 0x3bf6c00 <nil> [] 0s} 192.169.0.5 22 <nil> <nil>}
I0818 12:09:12.446725 3976 main.go:141] libmachine: About to run SSH command:
sudo mkdir -p /lib/systemd/system && printf %s "[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network.target minikube-automount.service docker.socket
Requires= minikube-automount.service docker.socket
StartLimitBurst=3
StartLimitIntervalSec=60
[Service]
Type=notify
Restart=on-failure
# This file is a systemd drop-in unit that inherits from the base dockerd configuration.
# The base configuration already specifies an 'ExecStart=...' command. The first directive
# here is to clear out that command inherited from the base configuration. Without this,
# the command from the base configuration and the command specified here are treated as
# a sequence of commands, which is not the desired behavior, nor is it valid -- systemd
# will catch this invalid input and refuse to start the service with an error like:
# Service has more than one ExecStart= setting, which is only allowed for Type=oneshot services.
# NOTE: default-ulimit=nofile is set to an arbitrary number for consistency with other
# container runtimes. If left unlimited, it may result in OOM issues with MySQL.
ExecStart=
ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --default-ulimit=nofile=1048576:1048576 --tlsverify --tlscacert /etc/docker/ca.pem --tlscert /etc/docker/server.pem --tlskey /etc/docker/server-key.pem --label provider=hyperkit --insecure-registry 10.96.0.0/12
ExecReload=/bin/kill -s HUP \$MAINPID
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
TimeoutStartSec=0
# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes
# kill only the docker process, not all processes in the cgroup
KillMode=process
[Install]
WantedBy=multi-user.target
" | sudo tee /lib/systemd/system/docker.service.new
I0818 12:09:12.516077 3976 main.go:141] libmachine: SSH cmd err, output: <nil>: [Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network.target minikube-automount.service docker.socket
Requires= minikube-automount.service docker.socket
StartLimitBurst=3
StartLimitIntervalSec=60
[Service]
Type=notify
Restart=on-failure
# This file is a systemd drop-in unit that inherits from the base dockerd configuration.
# The base configuration already specifies an 'ExecStart=...' command. The first directive
# here is to clear out that command inherited from the base configuration. Without this,
# the command from the base configuration and the command specified here are treated as
# a sequence of commands, which is not the desired behavior, nor is it valid -- systemd
# will catch this invalid input and refuse to start the service with an error like:
# Service has more than one ExecStart= setting, which is only allowed for Type=oneshot services.
# NOTE: default-ulimit=nofile is set to an arbitrary number for consistency with other
# container runtimes. If left unlimited, it may result in OOM issues with MySQL.
ExecStart=
ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --default-ulimit=nofile=1048576:1048576 --tlsverify --tlscacert /etc/docker/ca.pem --tlscert /etc/docker/server.pem --tlskey /etc/docker/server-key.pem --label provider=hyperkit --insecure-registry 10.96.0.0/12
ExecReload=/bin/kill -s HUP $MAINPID
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
TimeoutStartSec=0
# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes
# kill only the docker process, not all processes in the cgroup
KillMode=process
[Install]
WantedBy=multi-user.target
I0818 12:09:12.516100 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHHostname
I0818 12:09:12.516233 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHPort
I0818 12:09:12.516348 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHKeyPath
I0818 12:09:12.516437 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHKeyPath
I0818 12:09:12.516526 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHUsername
I0818 12:09:12.516667 3976 main.go:141] libmachine: Using SSH client type: native
I0818 12:09:12.516813 3976 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0x3bf3ea0] 0x3bf6c00 <nil> [] 0s} 192.169.0.5 22 <nil> <nil>}
I0818 12:09:12.516825 3976 main.go:141] libmachine: About to run SSH command:
sudo diff -u /lib/systemd/system/docker.service /lib/systemd/system/docker.service.new || { sudo mv /lib/systemd/system/docker.service.new /lib/systemd/system/docker.service; sudo systemctl -f daemon-reload && sudo systemctl -f enable docker && sudo systemctl -f restart docker; }
I0818 12:09:14.219167 3976 main.go:141] libmachine: SSH cmd err, output: <nil>: diff: can't stat '/lib/systemd/system/docker.service': No such file or directory
Created symlink /etc/systemd/system/multi-user.target.wants/docker.service → /usr/lib/systemd/system/docker.service.
I0818 12:09:14.219181 3976 machine.go:96] duration metric: took 13.22463913s to provisionDockerMachine
I0818 12:09:14.219193 3976 start.go:293] postStartSetup for "ha-373000" (driver="hyperkit")
I0818 12:09:14.219201 3976 start.go:322] creating required directories: [/etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs]
I0818 12:09:14.219211 3976 main.go:141] libmachine: (ha-373000) Calling .DriverName
I0818 12:09:14.219390 3976 ssh_runner.go:195] Run: sudo mkdir -p /etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs
I0818 12:09:14.219417 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHHostname
I0818 12:09:14.219519 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHPort
I0818 12:09:14.219630 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHKeyPath
I0818 12:09:14.219724 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHUsername
I0818 12:09:14.219808 3976 sshutil.go:53] new ssh client: &{IP:192.169.0.5 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000/id_rsa Username:docker}
I0818 12:09:14.259561 3976 ssh_runner.go:195] Run: cat /etc/os-release
I0818 12:09:14.263959 3976 info.go:137] Remote host: Buildroot 2023.02.9
I0818 12:09:14.263976 3976 filesync.go:126] Scanning /Users/jenkins/minikube-integration/19423-1007/.minikube/addons for local assets ...
I0818 12:09:14.264080 3976 filesync.go:126] Scanning /Users/jenkins/minikube-integration/19423-1007/.minikube/files for local assets ...
I0818 12:09:14.264273 3976 filesync.go:149] local asset: /Users/jenkins/minikube-integration/19423-1007/.minikube/files/etc/ssl/certs/15262.pem -> 15262.pem in /etc/ssl/certs
I0818 12:09:14.264280 3976 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19423-1007/.minikube/files/etc/ssl/certs/15262.pem -> /etc/ssl/certs/15262.pem
I0818 12:09:14.264487 3976 ssh_runner.go:195] Run: sudo mkdir -p /etc/ssl/certs
I0818 12:09:14.272283 3976 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19423-1007/.minikube/files/etc/ssl/certs/15262.pem --> /etc/ssl/certs/15262.pem (1708 bytes)
I0818 12:09:14.302942 3976 start.go:296] duration metric: took 83.742133ms for postStartSetup
I0818 12:09:14.302965 3976 main.go:141] libmachine: (ha-373000) Calling .DriverName
I0818 12:09:14.303146 3976 ssh_runner.go:195] Run: sudo ls --almost-all -1 /var/lib/minikube/backup
I0818 12:09:14.303160 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHHostname
I0818 12:09:14.303248 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHPort
I0818 12:09:14.303361 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHKeyPath
I0818 12:09:14.303436 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHUsername
I0818 12:09:14.303526 3976 sshutil.go:53] new ssh client: &{IP:192.169.0.5 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000/id_rsa Username:docker}
I0818 12:09:14.338080 3976 machine.go:197] restoring vm config from /var/lib/minikube/backup: [etc]
I0818 12:09:14.338142 3976 ssh_runner.go:195] Run: sudo rsync --archive --update /var/lib/minikube/backup/etc /
I0818 12:09:14.391638 3976 fix.go:56] duration metric: took 13.612527396s for fixHost
I0818 12:09:14.391662 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHHostname
I0818 12:09:14.391810 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHPort
I0818 12:09:14.391899 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHKeyPath
I0818 12:09:14.391991 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHKeyPath
I0818 12:09:14.392074 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHUsername
I0818 12:09:14.392222 3976 main.go:141] libmachine: Using SSH client type: native
I0818 12:09:14.392364 3976 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0x3bf3ea0] 0x3bf6c00 <nil> [] 0s} 192.169.0.5 22 <nil> <nil>}
I0818 12:09:14.392372 3976 main.go:141] libmachine: About to run SSH command:
date +%s.%N
I0818 12:09:14.449746 3976 main.go:141] libmachine: SSH cmd err, output: <nil>: 1724008154.620949792
I0818 12:09:14.449760 3976 fix.go:216] guest clock: 1724008154.620949792
I0818 12:09:14.449772 3976 fix.go:229] Guest: 2024-08-18 12:09:14.620949792 -0700 PDT Remote: 2024-08-18 12:09:14.391652 -0700 PDT m=+14.038170292 (delta=229.297792ms)
I0818 12:09:14.449789 3976 fix.go:200] guest clock delta is within tolerance: 229.297792ms
I0818 12:09:14.449793 3976 start.go:83] releasing machines lock for "ha-373000", held for 13.670724274s
I0818 12:09:14.449812 3976 main.go:141] libmachine: (ha-373000) Calling .DriverName
I0818 12:09:14.449942 3976 main.go:141] libmachine: (ha-373000) Calling .GetIP
I0818 12:09:14.450037 3976 main.go:141] libmachine: (ha-373000) Calling .DriverName
I0818 12:09:14.450349 3976 main.go:141] libmachine: (ha-373000) Calling .DriverName
I0818 12:09:14.450474 3976 main.go:141] libmachine: (ha-373000) Calling .DriverName
I0818 12:09:14.450548 3976 ssh_runner.go:195] Run: curl -sS -m 2 https://registry.k8s.io/
I0818 12:09:14.450580 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHHostname
I0818 12:09:14.450637 3976 ssh_runner.go:195] Run: cat /version.json
I0818 12:09:14.450648 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHHostname
I0818 12:09:14.450688 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHPort
I0818 12:09:14.450746 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHPort
I0818 12:09:14.450782 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHKeyPath
I0818 12:09:14.450836 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHKeyPath
I0818 12:09:14.450854 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHUsername
I0818 12:09:14.450935 3976 main.go:141] libmachine: (ha-373000) Calling .GetSSHUsername
I0818 12:09:14.450952 3976 sshutil.go:53] new ssh client: &{IP:192.169.0.5 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000/id_rsa Username:docker}
I0818 12:09:14.451045 3976 sshutil.go:53] new ssh client: &{IP:192.169.0.5 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000/id_rsa Username:docker}
I0818 12:09:14.541757 3976 ssh_runner.go:195] Run: systemctl --version
I0818 12:09:14.546793 3976 ssh_runner.go:195] Run: sh -c "stat /etc/cni/net.d/*loopback.conf*"
W0818 12:09:14.550801 3976 cni.go:209] loopback cni configuration skipped: "/etc/cni/net.d/*loopback.conf*" not found
I0818 12:09:14.550839 3976 ssh_runner.go:195] Run: sudo find /etc/cni/net.d -maxdepth 1 -type f ( ( -name *bridge* -or -name *podman* ) -and -not -name *.mk_disabled ) -printf "%p, " -exec sh -c "sudo mv {} {}.mk_disabled" ;
I0818 12:09:14.564129 3976 cni.go:262] disabled [/etc/cni/net.d/87-podman-bridge.conflist] bridge cni config(s)
I0818 12:09:14.564141 3976 start.go:495] detecting cgroup driver to use...
I0818 12:09:14.564243 3976 ssh_runner.go:195] Run: /bin/bash -c "sudo mkdir -p /etc && printf %s "runtime-endpoint: unix:///run/containerd/containerd.sock
" | sudo tee /etc/crictl.yaml"
I0818 12:09:14.581664 3976 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)sandbox_image = .*$|\1sandbox_image = "registry.k8s.io/pause:3.10"|' /etc/containerd/config.toml"
I0818 12:09:14.590425 3976 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)restrict_oom_score_adj = .*$|\1restrict_oom_score_adj = false|' /etc/containerd/config.toml"
I0818 12:09:14.599077 3976 containerd.go:146] configuring containerd to use "cgroupfs" as cgroup driver...
I0818 12:09:14.599120 3976 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)SystemdCgroup = .*$|\1SystemdCgroup = false|g' /etc/containerd/config.toml"
I0818 12:09:14.607868 3976 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|"io.containerd.runtime.v1.linux"|"io.containerd.runc.v2"|g' /etc/containerd/config.toml"
I0818 12:09:14.616526 3976 ssh_runner.go:195] Run: sh -c "sudo sed -i '/systemd_cgroup/d' /etc/containerd/config.toml"
I0818 12:09:14.625074 3976 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|"io.containerd.runc.v1"|"io.containerd.runc.v2"|g' /etc/containerd/config.toml"
I0818 12:09:14.633725 3976 ssh_runner.go:195] Run: sh -c "sudo rm -rf /etc/cni/net.mk"
I0818 12:09:14.642461 3976 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)conf_dir = .*$|\1conf_dir = "/etc/cni/net.d"|g' /etc/containerd/config.toml"
I0818 12:09:14.651030 3976 ssh_runner.go:195] Run: sh -c "sudo sed -i '/^ *enable_unprivileged_ports = .*/d' /etc/containerd/config.toml"
I0818 12:09:14.659717 3976 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)\[plugins."io.containerd.grpc.v1.cri"\]|&\n\1 enable_unprivileged_ports = true|' /etc/containerd/config.toml"
I0818 12:09:14.668509 3976 ssh_runner.go:195] Run: sudo sysctl net.bridge.bridge-nf-call-iptables
I0818 12:09:14.676419 3976 ssh_runner.go:195] Run: sudo sh -c "echo 1 > /proc/sys/net/ipv4/ip_forward"
I0818 12:09:14.684357 3976 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I0818 12:09:14.777696 3976 ssh_runner.go:195] Run: sudo systemctl restart containerd
I0818 12:09:14.795379 3976 start.go:495] detecting cgroup driver to use...
I0818 12:09:14.795465 3976 ssh_runner.go:195] Run: sudo systemctl cat docker.service
I0818 12:09:14.808091 3976 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service containerd
I0818 12:09:14.819351 3976 ssh_runner.go:195] Run: sudo systemctl stop -f containerd
I0818 12:09:14.834858 3976 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service containerd
I0818 12:09:14.845068 3976 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service crio
I0818 12:09:14.855088 3976 ssh_runner.go:195] Run: sudo systemctl stop -f crio
I0818 12:09:14.879151 3976 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service crio
I0818 12:09:14.889782 3976 ssh_runner.go:195] Run: /bin/bash -c "sudo mkdir -p /etc && printf %s "runtime-endpoint: unix:///var/run/cri-dockerd.sock
" | sudo tee /etc/crictl.yaml"
I0818 12:09:14.904555 3976 ssh_runner.go:195] Run: which cri-dockerd
I0818 12:09:14.907616 3976 ssh_runner.go:195] Run: sudo mkdir -p /etc/systemd/system/cri-docker.service.d
I0818 12:09:14.914893 3976 ssh_runner.go:362] scp memory --> /etc/systemd/system/cri-docker.service.d/10-cni.conf (190 bytes)
I0818 12:09:14.928498 3976 ssh_runner.go:195] Run: sudo systemctl unmask docker.service
I0818 12:09:15.021302 3976 ssh_runner.go:195] Run: sudo systemctl enable docker.socket
I0818 12:09:15.126534 3976 docker.go:574] configuring docker to use "cgroupfs" as cgroup driver...
I0818 12:09:15.126611 3976 ssh_runner.go:362] scp memory --> /etc/docker/daemon.json (130 bytes)
I0818 12:09:15.141437 3976 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I0818 12:09:15.238491 3976 ssh_runner.go:195] Run: sudo systemctl restart docker
I0818 12:09:17.633635 3976 ssh_runner.go:235] Completed: sudo systemctl restart docker: (2.395193434s)
I0818 12:09:17.633701 3976 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service cri-docker.socket
I0818 12:09:17.644119 3976 ssh_runner.go:195] Run: sudo systemctl stop cri-docker.socket
I0818 12:09:17.657413 3976 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service cri-docker.service
I0818 12:09:17.668074 3976 ssh_runner.go:195] Run: sudo systemctl unmask cri-docker.socket
I0818 12:09:17.762478 3976 ssh_runner.go:195] Run: sudo systemctl enable cri-docker.socket
I0818 12:09:17.858367 3976 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I0818 12:09:17.948600 3976 ssh_runner.go:195] Run: sudo systemctl restart cri-docker.socket
I0818 12:09:17.962148 3976 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service cri-docker.service
I0818 12:09:17.972120 3976 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I0818 12:09:18.070649 3976 ssh_runner.go:195] Run: sudo systemctl restart cri-docker.service
I0818 12:09:18.132791 3976 start.go:542] Will wait 60s for socket path /var/run/cri-dockerd.sock
I0818 12:09:18.132869 3976 ssh_runner.go:195] Run: stat /var/run/cri-dockerd.sock
I0818 12:09:18.137140 3976 start.go:563] Will wait 60s for crictl version
I0818 12:09:18.137200 3976 ssh_runner.go:195] Run: which crictl
I0818 12:09:18.140608 3976 ssh_runner.go:195] Run: sudo /usr/bin/crictl version
I0818 12:09:18.167352 3976 start.go:579] Version: 0.1.0
RuntimeName: docker
RuntimeVersion: 27.1.2
RuntimeApiVersion: v1
I0818 12:09:18.167422 3976 ssh_runner.go:195] Run: docker version --format {{.Server.Version}}
I0818 12:09:18.186476 3976 ssh_runner.go:195] Run: docker version --format {{.Server.Version}}
I0818 12:09:18.224169 3976 out.go:235] * Preparing Kubernetes v1.31.0 on Docker 27.1.2 ...
I0818 12:09:18.224214 3976 main.go:141] libmachine: (ha-373000) Calling .GetIP
I0818 12:09:18.224595 3976 ssh_runner.go:195] Run: grep 192.169.0.1 host.minikube.internal$ /etc/hosts
I0818 12:09:18.229086 3976 ssh_runner.go:195] Run: /bin/bash -c "{ grep -v $'\thost.minikube.internal$' "/etc/hosts"; echo "192.169.0.1 host.minikube.internal"; } > /tmp/h.$$; sudo cp /tmp/h.$$ "/etc/hosts""
I0818 12:09:18.238631 3976 kubeadm.go:883] updating cluster {Name:ha-373000 KeepContext:false EmbedCerts:false MinikubeISO:https://storage.googleapis.com/minikube-builds/iso/19452/minikube-v1.33.1-1723740674-19452-amd64.iso KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.44-1723740748-19452@sha256:2211a6931895d2d502e957e9667096db10734a96767d670cb4dbffdd37397b0d Memory:2200 CPUs:2 DiskSize:20000 Driver:hyperkit HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.31.
0 ClusterName:ha-373000 Namespace:default APIServerHAVIP:192.169.0.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:docker CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.169.0.5 Port:8443 KubernetesVersion:v1.31.0 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m02 IP:192.169.0.6 Port:8443 KubernetesVersion:v1.31.0 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m04 IP:192.169.0.8 Port:0 KubernetesVersion:v1.31.0 ContainerRuntime:docker ControlPlane:false Worker:true}] Addons:map[ambassador:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false helm-tiller:false inaccel:false ingress:false ingress-dns:f
alse inspektor-gadget:false istio:false istio-provisioner:false kong:false kubeflow:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-gluster:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s Mount:false MountString:/Users:/minikube-host Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOpt
imizations:false DisableMetrics:false CustomQemuFirmwarePath: SocketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s} ...
I0818 12:09:18.238717 3976 preload.go:131] Checking if preload exists for k8s version v1.31.0 and runtime docker
I0818 12:09:18.238780 3976 ssh_runner.go:195] Run: docker images --format {{.Repository}}:{{.Tag}}
I0818 12:09:18.252546 3976 docker.go:685] Got preloaded images: -- stdout --
kindest/kindnetd:v20240813-c6f155d6
registry.k8s.io/kube-scheduler:v1.31.0
registry.k8s.io/kube-apiserver:v1.31.0
registry.k8s.io/kube-controller-manager:v1.31.0
registry.k8s.io/kube-proxy:v1.31.0
registry.k8s.io/etcd:3.5.15-0
registry.k8s.io/pause:3.10
ghcr.io/kube-vip/kube-vip:v0.8.0
registry.k8s.io/coredns/coredns:v1.11.1
gcr.io/k8s-minikube/storage-provisioner:v5
gcr.io/k8s-minikube/busybox:1.28
-- /stdout --
I0818 12:09:18.252557 3976 docker.go:615] Images already preloaded, skipping extraction
I0818 12:09:18.252627 3976 ssh_runner.go:195] Run: docker images --format {{.Repository}}:{{.Tag}}
I0818 12:09:18.266684 3976 docker.go:685] Got preloaded images: -- stdout --
kindest/kindnetd:v20240813-c6f155d6
registry.k8s.io/kube-scheduler:v1.31.0
registry.k8s.io/kube-apiserver:v1.31.0
registry.k8s.io/kube-controller-manager:v1.31.0
registry.k8s.io/kube-proxy:v1.31.0
registry.k8s.io/etcd:3.5.15-0
registry.k8s.io/pause:3.10
ghcr.io/kube-vip/kube-vip:v0.8.0
registry.k8s.io/coredns/coredns:v1.11.1
gcr.io/k8s-minikube/storage-provisioner:v5
gcr.io/k8s-minikube/busybox:1.28
-- /stdout --
I0818 12:09:18.266703 3976 cache_images.go:84] Images are preloaded, skipping loading
I0818 12:09:18.266713 3976 kubeadm.go:934] updating node { 192.169.0.5 8443 v1.31.0 docker true true} ...
I0818 12:09:18.266790 3976 kubeadm.go:946] kubelet [Unit]
Wants=docker.socket
[Service]
ExecStart=
ExecStart=/var/lib/minikube/binaries/v1.31.0/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --config=/var/lib/kubelet/config.yaml --hostname-override=ha-373000 --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=192.169.0.5
[Install]
config:
{KubernetesVersion:v1.31.0 ClusterName:ha-373000 Namespace:default APIServerHAVIP:192.169.0.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:docker CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:}
I0818 12:09:18.266861 3976 ssh_runner.go:195] Run: docker info --format {{.CgroupDriver}}
I0818 12:09:18.304192 3976 cni.go:84] Creating CNI manager for ""
I0818 12:09:18.304204 3976 cni.go:136] multinode detected (3 nodes found), recommending kindnet
I0818 12:09:18.304213 3976 kubeadm.go:84] Using pod CIDR: 10.244.0.0/16
I0818 12:09:18.304229 3976 kubeadm.go:181] kubeadm options: {CertDir:/var/lib/minikube/certs ServiceCIDR:10.96.0.0/12 PodSubnet:10.244.0.0/16 AdvertiseAddress:192.169.0.5 APIServerPort:8443 KubernetesVersion:v1.31.0 EtcdDataDir:/var/lib/minikube/etcd EtcdExtraArgs:map[] ClusterName:ha-373000 NodeName:ha-373000 DNSDomain:cluster.local CRISocket:/var/run/cri-dockerd.sock ImageRepository: ComponentOptions:[{Component:apiServer ExtraArgs:map[enable-admission-plugins:NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota] Pairs:map[certSANs:["127.0.0.1", "localhost", "192.169.0.5"]]} {Component:controllerManager ExtraArgs:map[allocate-node-cidrs:true leader-elect:false] Pairs:map[]} {Component:scheduler ExtraArgs:map[leader-elect:false] Pairs:map[]}] FeatureArgs:map[] NodeIP:192.169.0.5 CgroupDriver:cgroupfs ClientCAFile:/var/lib/minikube/certs/ca.crt StaticPodPath:/etc/kubernetes/manif
ests ControlPlaneAddress:control-plane.minikube.internal KubeProxyOptions:map[] ResolvConfSearchRegression:false KubeletConfigOpts:map[containerRuntimeEndpoint:unix:///var/run/cri-dockerd.sock hairpinMode:hairpin-veth runtimeRequestTimeout:15m] PrependCriSocketUnix:true}
I0818 12:09:18.304320 3976 kubeadm.go:187] kubeadm config:
apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 192.169.0.5
bindPort: 8443
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
ttl: 24h0m0s
usages:
- signing
- authentication
nodeRegistration:
criSocket: unix:///var/run/cri-dockerd.sock
name: "ha-373000"
kubeletExtraArgs:
node-ip: 192.169.0.5
taints: []
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
apiServer:
certSANs: ["127.0.0.1", "localhost", "192.169.0.5"]
extraArgs:
enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
controllerManager:
extraArgs:
allocate-node-cidrs: "true"
leader-elect: "false"
scheduler:
extraArgs:
leader-elect: "false"
certificatesDir: /var/lib/minikube/certs
clusterName: mk
controlPlaneEndpoint: control-plane.minikube.internal:8443
etcd:
local:
dataDir: /var/lib/minikube/etcd
extraArgs:
proxy-refresh-interval: "70000"
kubernetesVersion: v1.31.0
networking:
dnsDomain: cluster.local
podSubnet: "10.244.0.0/16"
serviceSubnet: 10.96.0.0/12
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
authentication:
x509:
clientCAFile: /var/lib/minikube/certs/ca.crt
cgroupDriver: cgroupfs
containerRuntimeEndpoint: unix:///var/run/cri-dockerd.sock
hairpinMode: hairpin-veth
runtimeRequestTimeout: 15m
clusterDomain: "cluster.local"
# disable disk resource management by default
imageGCHighThresholdPercent: 100
evictionHard:
nodefs.available: "0%"
nodefs.inodesFree: "0%"
imagefs.available: "0%"
failSwapOn: false
staticPodPath: /etc/kubernetes/manifests
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
clusterCIDR: "10.244.0.0/16"
metricsBindAddress: 0.0.0.0:10249
conntrack:
maxPerCore: 0
# Skip setting "net.netfilter.nf_conntrack_tcp_timeout_established"
tcpEstablishedTimeout: 0s
# Skip setting "net.netfilter.nf_conntrack_tcp_timeout_close"
tcpCloseWaitTimeout: 0s
I0818 12:09:18.304334 3976 kube-vip.go:115] generating kube-vip config ...
I0818 12:09:18.304382 3976 ssh_runner.go:195] Run: sudo sh -c "modprobe --all ip_vs ip_vs_rr ip_vs_wrr ip_vs_sh nf_conntrack"
I0818 12:09:18.316732 3976 kube-vip.go:167] auto-enabling control-plane load-balancing in kube-vip
I0818 12:09:18.316793 3976 kube-vip.go:137] kube-vip config:
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: null
name: kube-vip
namespace: kube-system
spec:
containers:
- args:
- manager
env:
- name: vip_arp
value: "true"
- name: port
value: "8443"
- name: vip_nodename
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: vip_interface
value: eth0
- name: vip_cidr
value: "32"
- name: dns_mode
value: first
- name: cp_enable
value: "true"
- name: cp_namespace
value: kube-system
- name: vip_leaderelection
value: "true"
- name: vip_leasename
value: plndr-cp-lock
- name: vip_leaseduration
value: "5"
- name: vip_renewdeadline
value: "3"
- name: vip_retryperiod
value: "1"
- name: address
value: 192.169.0.254
- name: prometheus_server
value: :2112
- name : lb_enable
value: "true"
- name: lb_port
value: "8443"
image: ghcr.io/kube-vip/kube-vip:v0.8.0
imagePullPolicy: IfNotPresent
name: kube-vip
resources: {}
securityContext:
capabilities:
add:
- NET_ADMIN
- NET_RAW
volumeMounts:
- mountPath: /etc/kubernetes/admin.conf
name: kubeconfig
hostAliases:
- hostnames:
- kubernetes
ip: 127.0.0.1
hostNetwork: true
volumes:
- hostPath:
path: "/etc/kubernetes/admin.conf"
name: kubeconfig
status: {}
I0818 12:09:18.316840 3976 ssh_runner.go:195] Run: sudo ls /var/lib/minikube/binaries/v1.31.0
I0818 12:09:18.324597 3976 binaries.go:44] Found k8s binaries, skipping transfer
I0818 12:09:18.324641 3976 ssh_runner.go:195] Run: sudo mkdir -p /etc/systemd/system/kubelet.service.d /lib/systemd/system /var/tmp/minikube /etc/kubernetes/manifests
I0818 12:09:18.331779 3976 ssh_runner.go:362] scp memory --> /etc/systemd/system/kubelet.service.d/10-kubeadm.conf (307 bytes)
I0818 12:09:18.345158 3976 ssh_runner.go:362] scp memory --> /lib/systemd/system/kubelet.service (352 bytes)
I0818 12:09:18.358298 3976 ssh_runner.go:362] scp memory --> /var/tmp/minikube/kubeadm.yaml.new (2148 bytes)
I0818 12:09:18.372286 3976 ssh_runner.go:362] scp memory --> /etc/kubernetes/manifests/kube-vip.yaml (1440 bytes)
I0818 12:09:18.385485 3976 ssh_runner.go:195] Run: grep 192.169.0.254 control-plane.minikube.internal$ /etc/hosts
I0818 12:09:18.388341 3976 ssh_runner.go:195] Run: /bin/bash -c "{ grep -v $'\tcontrol-plane.minikube.internal$' "/etc/hosts"; echo "192.169.0.254 control-plane.minikube.internal"; } > /tmp/h.$$; sudo cp /tmp/h.$$ "/etc/hosts""
I0818 12:09:18.397526 3976 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I0818 12:09:18.496612 3976 ssh_runner.go:195] Run: sudo systemctl start kubelet
I0818 12:09:18.511160 3976 certs.go:68] Setting up /Users/jenkins/minikube-integration/19423-1007/.minikube/profiles/ha-373000 for IP: 192.169.0.5
I0818 12:09:18.511172 3976 certs.go:194] generating shared ca certs ...
I0818 12:09:18.511184 3976 certs.go:226] acquiring lock for ca certs: {Name:mkf9c4ce6e92bb713042213e4605fc93500c1ab9 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I0818 12:09:18.511356 3976 certs.go:235] skipping valid "minikubeCA" ca cert: /Users/jenkins/minikube-integration/19423-1007/.minikube/ca.key
I0818 12:09:18.511436 3976 certs.go:235] skipping valid "proxyClientCA" ca cert: /Users/jenkins/minikube-integration/19423-1007/.minikube/proxy-client-ca.key
I0818 12:09:18.511446 3976 certs.go:256] generating profile certs ...
I0818 12:09:18.511538 3976 certs.go:359] skipping valid signed profile cert regeneration for "minikube-user": /Users/jenkins/minikube-integration/19423-1007/.minikube/profiles/ha-373000/client.key
I0818 12:09:18.511564 3976 certs.go:363] generating signed profile cert for "minikube": /Users/jenkins/minikube-integration/19423-1007/.minikube/profiles/ha-373000/apiserver.key.55d44b69
I0818 12:09:18.511579 3976 crypto.go:68] Generating cert /Users/jenkins/minikube-integration/19423-1007/.minikube/profiles/ha-373000/apiserver.crt.55d44b69 with IP's: [10.96.0.1 127.0.0.1 10.0.0.1 192.169.0.5 192.169.0.6 192.169.0.254]
I0818 12:09:18.678090 3976 crypto.go:156] Writing cert to /Users/jenkins/minikube-integration/19423-1007/.minikube/profiles/ha-373000/apiserver.crt.55d44b69 ...
I0818 12:09:18.678108 3976 lock.go:35] WriteFile acquiring /Users/jenkins/minikube-integration/19423-1007/.minikube/profiles/ha-373000/apiserver.crt.55d44b69: {Name:mk412ce60d50ec37c24febde03f7225e8a48a24c Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I0818 12:09:18.678466 3976 crypto.go:164] Writing key to /Users/jenkins/minikube-integration/19423-1007/.minikube/profiles/ha-373000/apiserver.key.55d44b69 ...
I0818 12:09:18.678480 3976 lock.go:35] WriteFile acquiring /Users/jenkins/minikube-integration/19423-1007/.minikube/profiles/ha-373000/apiserver.key.55d44b69: {Name:mke31239238122280f7cbf00316b2acd43533e9b Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I0818 12:09:18.678743 3976 certs.go:381] copying /Users/jenkins/minikube-integration/19423-1007/.minikube/profiles/ha-373000/apiserver.crt.55d44b69 -> /Users/jenkins/minikube-integration/19423-1007/.minikube/profiles/ha-373000/apiserver.crt
I0818 12:09:18.678987 3976 certs.go:385] copying /Users/jenkins/minikube-integration/19423-1007/.minikube/profiles/ha-373000/apiserver.key.55d44b69 -> /Users/jenkins/minikube-integration/19423-1007/.minikube/profiles/ha-373000/apiserver.key
I0818 12:09:18.679293 3976 certs.go:359] skipping valid signed profile cert regeneration for "aggregator": /Users/jenkins/minikube-integration/19423-1007/.minikube/profiles/ha-373000/proxy-client.key
I0818 12:09:18.679306 3976 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19423-1007/.minikube/ca.crt -> /var/lib/minikube/certs/ca.crt
I0818 12:09:18.679332 3976 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19423-1007/.minikube/ca.key -> /var/lib/minikube/certs/ca.key
I0818 12:09:18.679353 3976 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19423-1007/.minikube/proxy-client-ca.crt -> /var/lib/minikube/certs/proxy-client-ca.crt
I0818 12:09:18.679374 3976 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19423-1007/.minikube/proxy-client-ca.key -> /var/lib/minikube/certs/proxy-client-ca.key
I0818 12:09:18.679394 3976 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19423-1007/.minikube/profiles/ha-373000/apiserver.crt -> /var/lib/minikube/certs/apiserver.crt
I0818 12:09:18.679414 3976 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19423-1007/.minikube/profiles/ha-373000/apiserver.key -> /var/lib/minikube/certs/apiserver.key
I0818 12:09:18.679441 3976 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19423-1007/.minikube/profiles/ha-373000/proxy-client.crt -> /var/lib/minikube/certs/proxy-client.crt
I0818 12:09:18.679462 3976 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19423-1007/.minikube/profiles/ha-373000/proxy-client.key -> /var/lib/minikube/certs/proxy-client.key
I0818 12:09:18.679567 3976 certs.go:484] found cert: /Users/jenkins/minikube-integration/19423-1007/.minikube/certs/1526.pem (1338 bytes)
W0818 12:09:18.679618 3976 certs.go:480] ignoring /Users/jenkins/minikube-integration/19423-1007/.minikube/certs/1526_empty.pem, impossibly tiny 0 bytes
I0818 12:09:18.679629 3976 certs.go:484] found cert: /Users/jenkins/minikube-integration/19423-1007/.minikube/certs/ca-key.pem (1675 bytes)
I0818 12:09:18.679662 3976 certs.go:484] found cert: /Users/jenkins/minikube-integration/19423-1007/.minikube/certs/ca.pem (1082 bytes)
I0818 12:09:18.679695 3976 certs.go:484] found cert: /Users/jenkins/minikube-integration/19423-1007/.minikube/certs/cert.pem (1123 bytes)
I0818 12:09:18.679735 3976 certs.go:484] found cert: /Users/jenkins/minikube-integration/19423-1007/.minikube/certs/key.pem (1675 bytes)
I0818 12:09:18.679815 3976 certs.go:484] found cert: /Users/jenkins/minikube-integration/19423-1007/.minikube/files/etc/ssl/certs/15262.pem (1708 bytes)
I0818 12:09:18.679851 3976 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19423-1007/.minikube/files/etc/ssl/certs/15262.pem -> /usr/share/ca-certificates/15262.pem
I0818 12:09:18.679895 3976 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19423-1007/.minikube/ca.crt -> /usr/share/ca-certificates/minikubeCA.pem
I0818 12:09:18.679917 3976 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19423-1007/.minikube/certs/1526.pem -> /usr/share/ca-certificates/1526.pem
I0818 12:09:18.680416 3976 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19423-1007/.minikube/ca.crt --> /var/lib/minikube/certs/ca.crt (1111 bytes)
I0818 12:09:18.731491 3976 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19423-1007/.minikube/ca.key --> /var/lib/minikube/certs/ca.key (1675 bytes)
I0818 12:09:18.777149 3976 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19423-1007/.minikube/proxy-client-ca.crt --> /var/lib/minikube/certs/proxy-client-ca.crt (1119 bytes)
I0818 12:09:18.836957 3976 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19423-1007/.minikube/proxy-client-ca.key --> /var/lib/minikube/certs/proxy-client-ca.key (1679 bytes)
I0818 12:09:18.879727 3976 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19423-1007/.minikube/profiles/ha-373000/apiserver.crt --> /var/lib/minikube/certs/apiserver.crt (1432 bytes)
I0818 12:09:18.904838 3976 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19423-1007/.minikube/profiles/ha-373000/apiserver.key --> /var/lib/minikube/certs/apiserver.key (1679 bytes)
I0818 12:09:18.933787 3976 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19423-1007/.minikube/profiles/ha-373000/proxy-client.crt --> /var/lib/minikube/certs/proxy-client.crt (1147 bytes)
I0818 12:09:18.969389 3976 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19423-1007/.minikube/profiles/ha-373000/proxy-client.key --> /var/lib/minikube/certs/proxy-client.key (1679 bytes)
I0818 12:09:18.994753 3976 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19423-1007/.minikube/files/etc/ssl/certs/15262.pem --> /usr/share/ca-certificates/15262.pem (1708 bytes)
I0818 12:09:19.013849 3976 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19423-1007/.minikube/ca.crt --> /usr/share/ca-certificates/minikubeCA.pem (1111 bytes)
I0818 12:09:19.033471 3976 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19423-1007/.minikube/certs/1526.pem --> /usr/share/ca-certificates/1526.pem (1338 bytes)
I0818 12:09:19.052595 3976 ssh_runner.go:362] scp memory --> /var/lib/minikube/kubeconfig (738 bytes)
I0818 12:09:19.066128 3976 ssh_runner.go:195] Run: openssl version
I0818 12:09:19.070271 3976 ssh_runner.go:195] Run: sudo /bin/bash -c "test -s /usr/share/ca-certificates/15262.pem && ln -fs /usr/share/ca-certificates/15262.pem /etc/ssl/certs/15262.pem"
I0818 12:09:19.079228 3976 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/15262.pem
I0818 12:09:19.082728 3976 certs.go:528] hashing: -rw-r--r-- 1 root root 1708 Aug 18 18:54 /usr/share/ca-certificates/15262.pem
I0818 12:09:19.082763 3976 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/15262.pem
I0818 12:09:19.086877 3976 ssh_runner.go:195] Run: sudo /bin/bash -c "test -L /etc/ssl/certs/3ec20f2e.0 || ln -fs /etc/ssl/certs/15262.pem /etc/ssl/certs/3ec20f2e.0"
I0818 12:09:19.095804 3976 ssh_runner.go:195] Run: sudo /bin/bash -c "test -s /usr/share/ca-certificates/minikubeCA.pem && ln -fs /usr/share/ca-certificates/minikubeCA.pem /etc/ssl/certs/minikubeCA.pem"
I0818 12:09:19.104889 3976 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/minikubeCA.pem
I0818 12:09:19.108208 3976 certs.go:528] hashing: -rw-r--r-- 1 root root 1111 Aug 18 18:38 /usr/share/ca-certificates/minikubeCA.pem
I0818 12:09:19.108241 3976 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/minikubeCA.pem
I0818 12:09:19.112406 3976 ssh_runner.go:195] Run: sudo /bin/bash -c "test -L /etc/ssl/certs/b5213941.0 || ln -fs /etc/ssl/certs/minikubeCA.pem /etc/ssl/certs/b5213941.0"
I0818 12:09:19.121720 3976 ssh_runner.go:195] Run: sudo /bin/bash -c "test -s /usr/share/ca-certificates/1526.pem && ln -fs /usr/share/ca-certificates/1526.pem /etc/ssl/certs/1526.pem"
I0818 12:09:19.130845 3976 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/1526.pem
I0818 12:09:19.134345 3976 certs.go:528] hashing: -rw-r--r-- 1 root root 1338 Aug 18 18:54 /usr/share/ca-certificates/1526.pem
I0818 12:09:19.134389 3976 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/1526.pem
I0818 12:09:19.138941 3976 ssh_runner.go:195] Run: sudo /bin/bash -c "test -L /etc/ssl/certs/51391683.0 || ln -fs /etc/ssl/certs/1526.pem /etc/ssl/certs/51391683.0"
I0818 12:09:19.148376 3976 ssh_runner.go:195] Run: stat /var/lib/minikube/certs/apiserver-kubelet-client.crt
I0818 12:09:19.151715 3976 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/apiserver-etcd-client.crt -checkend 86400
I0818 12:09:19.155985 3976 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/apiserver-kubelet-client.crt -checkend 86400
I0818 12:09:19.160273 3976 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/etcd/server.crt -checkend 86400
I0818 12:09:19.165064 3976 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/etcd/healthcheck-client.crt -checkend 86400
I0818 12:09:19.169962 3976 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/etcd/peer.crt -checkend 86400
I0818 12:09:19.174244 3976 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/front-proxy-client.crt -checkend 86400
I0818 12:09:19.178473 3976 kubeadm.go:392] StartCluster: {Name:ha-373000 KeepContext:false EmbedCerts:false MinikubeISO:https://storage.googleapis.com/minikube-builds/iso/19452/minikube-v1.33.1-1723740674-19452-amd64.iso KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.44-1723740748-19452@sha256:2211a6931895d2d502e957e9667096db10734a96767d670cb4dbffdd37397b0d Memory:2200 CPUs:2 DiskSize:20000 Driver:hyperkit HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.31.0 C
lusterName:ha-373000 Namespace:default APIServerHAVIP:192.169.0.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:docker CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.169.0.5 Port:8443 KubernetesVersion:v1.31.0 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m02 IP:192.169.0.6 Port:8443 KubernetesVersion:v1.31.0 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m04 IP:192.169.0.8 Port:0 KubernetesVersion:v1.31.0 ContainerRuntime:docker ControlPlane:false Worker:true}] Addons:map[ambassador:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false helm-tiller:false inaccel:false ingress:false ingress-dns:fals
e inspektor-gadget:false istio:false istio-provisioner:false kong:false kubeflow:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-gluster:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s Mount:false MountString:/Users:/minikube-host Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOptimi
zations:false DisableMetrics:false CustomQemuFirmwarePath: SocketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s}
I0818 12:09:19.178593 3976 ssh_runner.go:195] Run: docker ps --filter status=paused --filter=name=k8s_.*_(kube-system)_ --format={{.ID}}
I0818 12:09:19.190838 3976 ssh_runner.go:195] Run: sudo ls /var/lib/kubelet/kubeadm-flags.env /var/lib/kubelet/config.yaml /var/lib/minikube/etcd
I0818 12:09:19.199172 3976 kubeadm.go:408] found existing configuration files, will attempt cluster restart
I0818 12:09:19.199186 3976 kubeadm.go:593] restartPrimaryControlPlane start ...
I0818 12:09:19.199227 3976 ssh_runner.go:195] Run: sudo test -d /data/minikube
I0818 12:09:19.207402 3976 kubeadm.go:130] /data/minikube skipping compat symlinks: sudo test -d /data/minikube: Process exited with status 1
stdout:
stderr:
I0818 12:09:19.207710 3976 kubeconfig.go:47] verify endpoint returned: get endpoint: "ha-373000" does not appear in /Users/jenkins/minikube-integration/19423-1007/kubeconfig
I0818 12:09:19.207791 3976 kubeconfig.go:62] /Users/jenkins/minikube-integration/19423-1007/kubeconfig needs updating (will repair): [kubeconfig missing "ha-373000" cluster setting kubeconfig missing "ha-373000" context setting]
I0818 12:09:19.207967 3976 lock.go:35] WriteFile acquiring /Users/jenkins/minikube-integration/19423-1007/kubeconfig: {Name:mk884388b2510ace5b23e8fdd3343cda9524a1f2 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I0818 12:09:19.208584 3976 loader.go:395] Config loaded from file: /Users/jenkins/minikube-integration/19423-1007/kubeconfig
I0818 12:09:19.208770 3976 kapi.go:59] client config for ha-373000: &rest.Config{Host:"https://192.169.0.5:8443", APIPath:"", ContentConfig:rest.ContentConfig{AcceptContentTypes:"", ContentType:"", GroupVersion:(*schema.GroupVersion)(nil), NegotiatedSerializer:runtime.NegotiatedSerializer(nil)}, Username:"", Password:"", BearerToken:"", BearerTokenFile:"", Impersonate:rest.ImpersonationConfig{UserName:"", UID:"", Groups:[]string(nil), Extra:map[string][]string(nil)}, AuthProvider:<nil>, AuthConfigPersister:rest.AuthProviderConfigPersister(nil), ExecProvider:<nil>, TLSClientConfig:rest.sanitizedTLSClientConfig{Insecure:false, ServerName:"", CertFile:"/Users/jenkins/minikube-integration/19423-1007/.minikube/profiles/ha-373000/client.crt", KeyFile:"/Users/jenkins/minikube-integration/19423-1007/.minikube/profiles/ha-373000/client.key", CAFile:"/Users/jenkins/minikube-integration/19423-1007/.minikube/ca.crt", CertData:[]uint8(nil), KeyData:[]uint8(nil), CAData:[]uint8(nil), NextProtos:[]string(nil)},
UserAgent:"", DisableCompression:false, Transport:http.RoundTripper(nil), WrapTransport:(transport.WrapperFunc)(0x52acf60), QPS:0, Burst:0, RateLimiter:flowcontrol.RateLimiter(nil), WarningHandler:rest.WarningHandler(nil), Timeout:0, Dial:(func(context.Context, string, string) (net.Conn, error))(nil), Proxy:(func(*http.Request) (*url.URL, error))(nil)}
I0818 12:09:19.209064 3976 cert_rotation.go:140] Starting client certificate rotation controller
I0818 12:09:19.209255 3976 ssh_runner.go:195] Run: sudo diff -u /var/tmp/minikube/kubeadm.yaml /var/tmp/minikube/kubeadm.yaml.new
I0818 12:09:19.217108 3976 kubeadm.go:630] The running cluster does not require reconfiguration: 192.169.0.5
I0818 12:09:19.217125 3976 kubeadm.go:597] duration metric: took 17.934031ms to restartPrimaryControlPlane
I0818 12:09:19.217132 3976 kubeadm.go:394] duration metric: took 38.665023ms to StartCluster
I0818 12:09:19.217145 3976 settings.go:142] acquiring lock: {Name:mk54874f9a6ab5b428c8697c9ef71cbcdde2d89e Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I0818 12:09:19.217216 3976 settings.go:150] Updating kubeconfig: /Users/jenkins/minikube-integration/19423-1007/kubeconfig
I0818 12:09:19.217617 3976 lock.go:35] WriteFile acquiring /Users/jenkins/minikube-integration/19423-1007/kubeconfig: {Name:mk884388b2510ace5b23e8fdd3343cda9524a1f2 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I0818 12:09:19.217869 3976 start.go:233] HA (multi-control plane) cluster: will skip waiting for primary control-plane node &{Name: IP:192.169.0.5 Port:8443 KubernetesVersion:v1.31.0 ContainerRuntime:docker ControlPlane:true Worker:true}
I0818 12:09:19.217886 3976 start.go:241] waiting for startup goroutines ...
I0818 12:09:19.217906 3976 addons.go:507] enable addons start: toEnable=map[ambassador:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false helm-tiller:false inaccel:false ingress:false ingress-dns:false inspektor-gadget:false istio:false istio-provisioner:false kong:false kubeflow:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-gluster:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false]
I0818 12:09:19.217983 3976 config.go:182] Loaded profile config "ha-373000": Driver=hyperkit, ContainerRuntime=docker, KubernetesVersion=v1.31.0
I0818 12:09:19.263234 3976 out.go:177] * Enabled addons:
I0818 12:09:19.284302 3976 addons.go:510] duration metric: took 66.388858ms for enable addons: enabled=[]
I0818 12:09:19.284387 3976 start.go:246] waiting for cluster config update ...
I0818 12:09:19.284400 3976 start.go:255] writing updated cluster config ...
I0818 12:09:19.306484 3976 out.go:201]
I0818 12:09:19.327608 3976 config.go:182] Loaded profile config "ha-373000": Driver=hyperkit, ContainerRuntime=docker, KubernetesVersion=v1.31.0
I0818 12:09:19.327742 3976 profile.go:143] Saving config to /Users/jenkins/minikube-integration/19423-1007/.minikube/profiles/ha-373000/config.json ...
I0818 12:09:19.350369 3976 out.go:177] * Starting "ha-373000-m02" control-plane node in "ha-373000" cluster
I0818 12:09:19.392104 3976 preload.go:131] Checking if preload exists for k8s version v1.31.0 and runtime docker
I0818 12:09:19.392164 3976 cache.go:56] Caching tarball of preloaded images
I0818 12:09:19.392336 3976 preload.go:172] Found /Users/jenkins/minikube-integration/19423-1007/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.31.0-docker-overlay2-amd64.tar.lz4 in cache, skipping download
I0818 12:09:19.392355 3976 cache.go:59] Finished verifying existence of preloaded tar for v1.31.0 on docker
I0818 12:09:19.392486 3976 profile.go:143] Saving config to /Users/jenkins/minikube-integration/19423-1007/.minikube/profiles/ha-373000/config.json ...
I0818 12:09:19.393415 3976 start.go:360] acquireMachinesLock for ha-373000-m02: {Name:mk31060d3397796eb838e7923e770e40ab40e545 Clock:{} Delay:500ms Timeout:13m0s Cancel:<nil>}
I0818 12:09:19.393522 3976 start.go:364] duration metric: took 80.918µs to acquireMachinesLock for "ha-373000-m02"
I0818 12:09:19.393546 3976 start.go:96] Skipping create...Using existing machine configuration
I0818 12:09:19.393556 3976 fix.go:54] fixHost starting: m02
I0818 12:09:19.393965 3976 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0818 12:09:19.393990 3976 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0818 12:09:19.403655 3976 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:52040
I0818 12:09:19.404217 3976 main.go:141] libmachine: () Calling .GetVersion
I0818 12:09:19.404634 3976 main.go:141] libmachine: Using API Version 1
I0818 12:09:19.404650 3976 main.go:141] libmachine: () Calling .SetConfigRaw
I0818 12:09:19.405004 3976 main.go:141] libmachine: () Calling .GetMachineName
I0818 12:09:19.405118 3976 main.go:141] libmachine: (ha-373000-m02) Calling .DriverName
I0818 12:09:19.405222 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetState
I0818 12:09:19.405303 3976 main.go:141] libmachine: (ha-373000-m02) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I0818 12:09:19.405380 3976 main.go:141] libmachine: (ha-373000-m02) DBG | hyperkit pid from json: 3847
I0818 12:09:19.406287 3976 main.go:141] libmachine: (ha-373000-m02) DBG | hyperkit pid 3847 missing from process table
I0818 12:09:19.406302 3976 fix.go:112] recreateIfNeeded on ha-373000-m02: state=Stopped err=<nil>
I0818 12:09:19.406312 3976 main.go:141] libmachine: (ha-373000-m02) Calling .DriverName
W0818 12:09:19.406463 3976 fix.go:138] unexpected machine state, will restart: <nil>
I0818 12:09:19.448356 3976 out.go:177] * Restarting existing hyperkit VM for "ha-373000-m02" ...
I0818 12:09:19.469229 3976 main.go:141] libmachine: (ha-373000-m02) Calling .Start
I0818 12:09:19.469501 3976 main.go:141] libmachine: (ha-373000-m02) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I0818 12:09:19.469542 3976 main.go:141] libmachine: (ha-373000-m02) minikube might have been shutdown in an unclean way, the hyperkit pid file still exists: /Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02/hyperkit.pid
I0818 12:09:19.471314 3976 main.go:141] libmachine: (ha-373000-m02) DBG | hyperkit pid 3847 missing from process table
I0818 12:09:19.471327 3976 main.go:141] libmachine: (ha-373000-m02) DBG | pid 3847 is in state "Stopped"
I0818 12:09:19.471351 3976 main.go:141] libmachine: (ha-373000-m02) DBG | Removing stale pid file /Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02/hyperkit.pid...
I0818 12:09:19.471584 3976 main.go:141] libmachine: (ha-373000-m02) DBG | Using UUID 7a237572-4e62-4b98-a476-83254bfde967
I0818 12:09:19.500704 3976 main.go:141] libmachine: (ha-373000-m02) DBG | Generated MAC ca:b5:c4:e6:47:79
I0818 12:09:19.500730 3976 main.go:141] libmachine: (ha-373000-m02) DBG | Starting with cmdline: loglevel=3 console=ttyS0 console=tty0 noembed nomodeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-373000
I0818 12:09:19.500855 3976 main.go:141] libmachine: (ha-373000-m02) DBG | 2024/08/18 12:09:19 DEBUG: hyperkit: Start &hyperkit.HyperKit{HyperKit:"/usr/local/bin/hyperkit", Argv0:"", StateDir:"/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02", VPNKitSock:"", VPNKitUUID:"", VPNKitPreferredIPv4:"", UUID:"7a237572-4e62-4b98-a476-83254bfde967", Disks:[]hyperkit.Disk{(*hyperkit.RawDisk)(0xc0003c4600)}, ISOImages:[]string{"/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02/boot2docker.iso"}, VSock:false, VSockDir:"", VSockPorts:[]int(nil), VSockGuestCID:3, VMNet:true, Sockets9P:[]hyperkit.Socket9P(nil), Kernel:"/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02/bzimage", Initrd:"/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02/initrd", Bootrom:"", CPUs:2, Memory:2200, Console:1, Serials:[]hyperkit.Serial(nil), Pid:0, Arguments:[]string(nil), CmdLine:"", process:(*os.Process)(nil)}
I0818 12:09:19.500885 3976 main.go:141] libmachine: (ha-373000-m02) DBG | 2024/08/18 12:09:19 DEBUG: hyperkit: check &hyperkit.HyperKit{HyperKit:"/usr/local/bin/hyperkit", Argv0:"", StateDir:"/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02", VPNKitSock:"", VPNKitUUID:"", VPNKitPreferredIPv4:"", UUID:"7a237572-4e62-4b98-a476-83254bfde967", Disks:[]hyperkit.Disk{(*hyperkit.RawDisk)(0xc0003c4600)}, ISOImages:[]string{"/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02/boot2docker.iso"}, VSock:false, VSockDir:"", VSockPorts:[]int(nil), VSockGuestCID:3, VMNet:true, Sockets9P:[]hyperkit.Socket9P(nil), Kernel:"/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02/bzimage", Initrd:"/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02/initrd", Bootrom:"", CPUs:2, Memory:2200, Console:1, Serials:[]hyperkit.Serial(nil), Pid:0, Arguments:[]string(nil), CmdLine:"", process:(*os.Process)(nil)}
I0818 12:09:19.500929 3976 main.go:141] libmachine: (ha-373000-m02) DBG | 2024/08/18 12:09:19 DEBUG: hyperkit: Arguments: []string{"-A", "-u", "-F", "/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02/hyperkit.pid", "-c", "2", "-m", "2200M", "-s", "0:0,hostbridge", "-s", "31,lpc", "-s", "1:0,virtio-net", "-U", "7a237572-4e62-4b98-a476-83254bfde967", "-s", "2:0,virtio-blk,/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02/ha-373000-m02.rawdisk", "-s", "3,ahci-cd,/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02/boot2docker.iso", "-s", "4,virtio-rnd", "-l", "com1,autopty=/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02/tty,log=/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02/console-ring", "-f", "kexec,/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02/bzimage,/Users/jenkins/minikube-integration/19423-1007/.minikube/machine
s/ha-373000-m02/initrd,earlyprintk=serial loglevel=3 console=ttyS0 console=tty0 noembed nomodeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-373000"}
I0818 12:09:19.500977 3976 main.go:141] libmachine: (ha-373000-m02) DBG | 2024/08/18 12:09:19 DEBUG: hyperkit: CmdLine: "/usr/local/bin/hyperkit -A -u -F /Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02/hyperkit.pid -c 2 -m 2200M -s 0:0,hostbridge -s 31,lpc -s 1:0,virtio-net -U 7a237572-4e62-4b98-a476-83254bfde967 -s 2:0,virtio-blk,/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02/ha-373000-m02.rawdisk -s 3,ahci-cd,/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02/boot2docker.iso -s 4,virtio-rnd -l com1,autopty=/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02/tty,log=/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02/console-ring -f kexec,/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02/bzimage,/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02/initrd,earlyprintk=serial loglevel=3 console=t
tyS0 console=tty0 noembed nomodeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-373000"
I0818 12:09:19.500998 3976 main.go:141] libmachine: (ha-373000-m02) DBG | 2024/08/18 12:09:19 DEBUG: hyperkit: Redirecting stdout/stderr to logger
I0818 12:09:19.502361 3976 main.go:141] libmachine: (ha-373000-m02) DBG | 2024/08/18 12:09:19 DEBUG: hyperkit: Pid is 3997
I0818 12:09:19.502828 3976 main.go:141] libmachine: (ha-373000-m02) DBG | Attempt 0
I0818 12:09:19.502885 3976 main.go:141] libmachine: (ha-373000-m02) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I0818 12:09:19.502920 3976 main.go:141] libmachine: (ha-373000-m02) DBG | hyperkit pid from json: 3997
I0818 12:09:19.504725 3976 main.go:141] libmachine: (ha-373000-m02) DBG | Searching for ca:b5:c4:e6:47:79 in /var/db/dhcpd_leases ...
I0818 12:09:19.504780 3976 main.go:141] libmachine: (ha-373000-m02) DBG | Found 7 entries in /var/db/dhcpd_leases!
I0818 12:09:19.504828 3976 main.go:141] libmachine: (ha-373000-m02) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.5 HWAddress:be:21:66:25:9a:b1 ID:1,be:21:66:25:9a:b1 Lease:0x66c39856}
I0818 12:09:19.504848 3976 main.go:141] libmachine: (ha-373000-m02) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.7 HWAddress:72:9e:9b:7f:e6:a8 ID:1,72:9e:9b:7f:e6:a8 Lease:0x66c246ad}
I0818 12:09:19.504870 3976 main.go:141] libmachine: (ha-373000-m02) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.8 HWAddress:f2:8c:91:ee:dd:c0 ID:1,f2:8c:91:ee:dd:c0 Lease:0x66c397e0}
I0818 12:09:19.504882 3976 main.go:141] libmachine: (ha-373000-m02) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.6 HWAddress:ca:b5:c4:e6:47:79 ID:1,ca:b5:c4:e6:47:79 Lease:0x66c3975b}
I0818 12:09:19.504895 3976 main.go:141] libmachine: (ha-373000-m02) DBG | Found match: ca:b5:c4:e6:47:79
I0818 12:09:19.504900 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetConfigRaw
I0818 12:09:19.504907 3976 main.go:141] libmachine: (ha-373000-m02) DBG | IP: 192.169.0.6
I0818 12:09:19.505665 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetIP
I0818 12:09:19.505858 3976 profile.go:143] Saving config to /Users/jenkins/minikube-integration/19423-1007/.minikube/profiles/ha-373000/config.json ...
I0818 12:09:19.506316 3976 machine.go:93] provisionDockerMachine start ...
I0818 12:09:19.506328 3976 main.go:141] libmachine: (ha-373000-m02) Calling .DriverName
I0818 12:09:19.506474 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHHostname
I0818 12:09:19.506602 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHPort
I0818 12:09:19.506707 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHKeyPath
I0818 12:09:19.506790 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHKeyPath
I0818 12:09:19.506894 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHUsername
I0818 12:09:19.507039 3976 main.go:141] libmachine: Using SSH client type: native
I0818 12:09:19.507197 3976 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0x3bf3ea0] 0x3bf6c00 <nil> [] 0s} 192.169.0.6 22 <nil> <nil>}
I0818 12:09:19.507205 3976 main.go:141] libmachine: About to run SSH command:
hostname
I0818 12:09:19.510551 3976 main.go:141] libmachine: (ha-373000-m02) DBG | 2024/08/18 12:09:19 INFO : hyperkit: stderr: Using fd 5 for I/O notifications
I0818 12:09:19.519215 3976 main.go:141] libmachine: (ha-373000-m02) DBG | 2024/08/18 12:09:19 INFO : hyperkit: stderr: /Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02/boot2docker.iso: fcntl(F_PUNCHHOLE) Operation not permitted: block device will not support TRIM/DISCARD
I0818 12:09:19.520168 3976 main.go:141] libmachine: (ha-373000-m02) DBG | 2024/08/18 12:09:19 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 2 bit: 22 unspecified don't care: bit is 0
I0818 12:09:19.520203 3976 main.go:141] libmachine: (ha-373000-m02) DBG | 2024/08/18 12:09:19 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 12 unspecified don't care: bit is 0
I0818 12:09:19.520228 3976 main.go:141] libmachine: (ha-373000-m02) DBG | 2024/08/18 12:09:19 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 20 unspecified don't care: bit is 0
I0818 12:09:19.520254 3976 main.go:141] libmachine: (ha-373000-m02) DBG | 2024/08/18 12:09:19 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 3 bit: 13 unspecified don't care: bit is 0
I0818 12:09:19.902342 3976 main.go:141] libmachine: (ha-373000-m02) DBG | 2024/08/18 12:09:19 INFO : hyperkit: stderr: rdmsr to register 0x3a on vcpu 0
I0818 12:09:19.902357 3976 main.go:141] libmachine: (ha-373000-m02) DBG | 2024/08/18 12:09:19 INFO : hyperkit: stderr: rdmsr to register 0x140 on vcpu 0
I0818 12:09:20.017440 3976 main.go:141] libmachine: (ha-373000-m02) DBG | 2024/08/18 12:09:20 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 2 bit: 22 unspecified don't care: bit is 0
I0818 12:09:20.017463 3976 main.go:141] libmachine: (ha-373000-m02) DBG | 2024/08/18 12:09:20 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 12 unspecified don't care: bit is 0
I0818 12:09:20.017471 3976 main.go:141] libmachine: (ha-373000-m02) DBG | 2024/08/18 12:09:20 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 20 unspecified don't care: bit is 0
I0818 12:09:20.017477 3976 main.go:141] libmachine: (ha-373000-m02) DBG | 2024/08/18 12:09:20 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 3 bit: 13 unspecified don't care: bit is 0
I0818 12:09:20.018303 3976 main.go:141] libmachine: (ha-373000-m02) DBG | 2024/08/18 12:09:20 INFO : hyperkit: stderr: rdmsr to register 0x3a on vcpu 1
I0818 12:09:20.018315 3976 main.go:141] libmachine: (ha-373000-m02) DBG | 2024/08/18 12:09:20 INFO : hyperkit: stderr: rdmsr to register 0x140 on vcpu 1
I0818 12:09:25.632462 3976 main.go:141] libmachine: (ha-373000-m02) DBG | 2024/08/18 12:09:25 INFO : hyperkit: stderr: rdmsr to register 0x64d on vcpu 1
I0818 12:09:25.632549 3976 main.go:141] libmachine: (ha-373000-m02) DBG | 2024/08/18 12:09:25 INFO : hyperkit: stderr: rdmsr to register 0x64e on vcpu 1
I0818 12:09:25.632559 3976 main.go:141] libmachine: (ha-373000-m02) DBG | 2024/08/18 12:09:25 INFO : hyperkit: stderr: rdmsr to register 0x34 on vcpu 1
I0818 12:09:25.657887 3976 main.go:141] libmachine: (ha-373000-m02) DBG | 2024/08/18 12:09:25 INFO : hyperkit: stderr: rdmsr to register 0xc0011029 on vcpu 1
I0818 12:09:28.954523 3976 main.go:141] libmachine: Error dialing TCP: dial tcp 192.169.0.6:22: connect: connection refused
I0818 12:09:32.012675 3976 main.go:141] libmachine: SSH cmd err, output: <nil>: minikube
I0818 12:09:32.012690 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetMachineName
I0818 12:09:32.012844 3976 buildroot.go:166] provisioning hostname "ha-373000-m02"
I0818 12:09:32.012857 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetMachineName
I0818 12:09:32.012969 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHHostname
I0818 12:09:32.013100 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHPort
I0818 12:09:32.013206 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHKeyPath
I0818 12:09:32.013295 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHKeyPath
I0818 12:09:32.013399 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHUsername
I0818 12:09:32.013577 3976 main.go:141] libmachine: Using SSH client type: native
I0818 12:09:32.013797 3976 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0x3bf3ea0] 0x3bf6c00 <nil> [] 0s} 192.169.0.6 22 <nil> <nil>}
I0818 12:09:32.013807 3976 main.go:141] libmachine: About to run SSH command:
sudo hostname ha-373000-m02 && echo "ha-373000-m02" | sudo tee /etc/hostname
I0818 12:09:32.083655 3976 main.go:141] libmachine: SSH cmd err, output: <nil>: ha-373000-m02
I0818 12:09:32.083671 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHHostname
I0818 12:09:32.083802 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHPort
I0818 12:09:32.083888 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHKeyPath
I0818 12:09:32.083968 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHKeyPath
I0818 12:09:32.084051 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHUsername
I0818 12:09:32.084177 3976 main.go:141] libmachine: Using SSH client type: native
I0818 12:09:32.084328 3976 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0x3bf3ea0] 0x3bf6c00 <nil> [] 0s} 192.169.0.6 22 <nil> <nil>}
I0818 12:09:32.084343 3976 main.go:141] libmachine: About to run SSH command:
if ! grep -xq '.*\sha-373000-m02' /etc/hosts; then
if grep -xq '127.0.1.1\s.*' /etc/hosts; then
sudo sed -i 's/^127.0.1.1\s.*/127.0.1.1 ha-373000-m02/g' /etc/hosts;
else
echo '127.0.1.1 ha-373000-m02' | sudo tee -a /etc/hosts;
fi
fi
I0818 12:09:32.145743 3976 main.go:141] libmachine: SSH cmd err, output: <nil>:
I0818 12:09:32.145757 3976 buildroot.go:172] set auth options {CertDir:/Users/jenkins/minikube-integration/19423-1007/.minikube CaCertPath:/Users/jenkins/minikube-integration/19423-1007/.minikube/certs/ca.pem CaPrivateKeyPath:/Users/jenkins/minikube-integration/19423-1007/.minikube/certs/ca-key.pem CaCertRemotePath:/etc/docker/ca.pem ServerCertPath:/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/server.pem ServerKeyPath:/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/server-key.pem ClientKeyPath:/Users/jenkins/minikube-integration/19423-1007/.minikube/certs/key.pem ServerCertRemotePath:/etc/docker/server.pem ServerKeyRemotePath:/etc/docker/server-key.pem ClientCertPath:/Users/jenkins/minikube-integration/19423-1007/.minikube/certs/cert.pem ServerCertSANs:[] StorePath:/Users/jenkins/minikube-integration/19423-1007/.minikube}
I0818 12:09:32.145771 3976 buildroot.go:174] setting up certificates
I0818 12:09:32.145778 3976 provision.go:84] configureAuth start
I0818 12:09:32.145785 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetMachineName
I0818 12:09:32.145913 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetIP
I0818 12:09:32.146013 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHHostname
I0818 12:09:32.146119 3976 provision.go:143] copyHostCerts
I0818 12:09:32.146155 3976 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19423-1007/.minikube/certs/ca.pem -> /Users/jenkins/minikube-integration/19423-1007/.minikube/ca.pem
I0818 12:09:32.146207 3976 exec_runner.go:144] found /Users/jenkins/minikube-integration/19423-1007/.minikube/ca.pem, removing ...
I0818 12:09:32.146213 3976 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19423-1007/.minikube/ca.pem
I0818 12:09:32.146346 3976 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19423-1007/.minikube/certs/ca.pem --> /Users/jenkins/minikube-integration/19423-1007/.minikube/ca.pem (1082 bytes)
I0818 12:09:32.146563 3976 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19423-1007/.minikube/certs/cert.pem -> /Users/jenkins/minikube-integration/19423-1007/.minikube/cert.pem
I0818 12:09:32.146599 3976 exec_runner.go:144] found /Users/jenkins/minikube-integration/19423-1007/.minikube/cert.pem, removing ...
I0818 12:09:32.146604 3976 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19423-1007/.minikube/cert.pem
I0818 12:09:32.146673 3976 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19423-1007/.minikube/certs/cert.pem --> /Users/jenkins/minikube-integration/19423-1007/.minikube/cert.pem (1123 bytes)
I0818 12:09:32.146816 3976 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19423-1007/.minikube/certs/key.pem -> /Users/jenkins/minikube-integration/19423-1007/.minikube/key.pem
I0818 12:09:32.146847 3976 exec_runner.go:144] found /Users/jenkins/minikube-integration/19423-1007/.minikube/key.pem, removing ...
I0818 12:09:32.146852 3976 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19423-1007/.minikube/key.pem
I0818 12:09:32.146916 3976 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19423-1007/.minikube/certs/key.pem --> /Users/jenkins/minikube-integration/19423-1007/.minikube/key.pem (1675 bytes)
I0818 12:09:32.147063 3976 provision.go:117] generating server cert: /Users/jenkins/minikube-integration/19423-1007/.minikube/machines/server.pem ca-key=/Users/jenkins/minikube-integration/19423-1007/.minikube/certs/ca.pem private-key=/Users/jenkins/minikube-integration/19423-1007/.minikube/certs/ca-key.pem org=jenkins.ha-373000-m02 san=[127.0.0.1 192.169.0.6 ha-373000-m02 localhost minikube]
I0818 12:09:32.439235 3976 provision.go:177] copyRemoteCerts
I0818 12:09:32.439288 3976 ssh_runner.go:195] Run: sudo mkdir -p /etc/docker /etc/docker /etc/docker
I0818 12:09:32.439303 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHHostname
I0818 12:09:32.439451 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHPort
I0818 12:09:32.439555 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHKeyPath
I0818 12:09:32.439662 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHUsername
I0818 12:09:32.439767 3976 sshutil.go:53] new ssh client: &{IP:192.169.0.6 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02/id_rsa Username:docker}
I0818 12:09:32.473899 3976 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19423-1007/.minikube/certs/ca.pem -> /etc/docker/ca.pem
I0818 12:09:32.473971 3976 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19423-1007/.minikube/certs/ca.pem --> /etc/docker/ca.pem (1082 bytes)
I0818 12:09:32.492902 3976 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19423-1007/.minikube/machines/server.pem -> /etc/docker/server.pem
I0818 12:09:32.492977 3976 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19423-1007/.minikube/machines/server.pem --> /etc/docker/server.pem (1208 bytes)
I0818 12:09:32.512205 3976 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19423-1007/.minikube/machines/server-key.pem -> /etc/docker/server-key.pem
I0818 12:09:32.512269 3976 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19423-1007/.minikube/machines/server-key.pem --> /etc/docker/server-key.pem (1675 bytes)
I0818 12:09:32.531269 3976 provision.go:87] duration metric: took 385.496037ms to configureAuth
I0818 12:09:32.531282 3976 buildroot.go:189] setting minikube options for container-runtime
I0818 12:09:32.531440 3976 config.go:182] Loaded profile config "ha-373000": Driver=hyperkit, ContainerRuntime=docker, KubernetesVersion=v1.31.0
I0818 12:09:32.531454 3976 main.go:141] libmachine: (ha-373000-m02) Calling .DriverName
I0818 12:09:32.531586 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHHostname
I0818 12:09:32.531687 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHPort
I0818 12:09:32.531797 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHKeyPath
I0818 12:09:32.531905 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHKeyPath
I0818 12:09:32.531985 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHUsername
I0818 12:09:32.532087 3976 main.go:141] libmachine: Using SSH client type: native
I0818 12:09:32.532212 3976 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0x3bf3ea0] 0x3bf6c00 <nil> [] 0s} 192.169.0.6 22 <nil> <nil>}
I0818 12:09:32.532220 3976 main.go:141] libmachine: About to run SSH command:
df --output=fstype / | tail -n 1
I0818 12:09:32.586134 3976 main.go:141] libmachine: SSH cmd err, output: <nil>: tmpfs
I0818 12:09:32.586145 3976 buildroot.go:70] root file system type: tmpfs
I0818 12:09:32.586228 3976 provision.go:314] Updating docker unit: /lib/systemd/system/docker.service ...
I0818 12:09:32.586239 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHHostname
I0818 12:09:32.586366 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHPort
I0818 12:09:32.586454 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHKeyPath
I0818 12:09:32.586566 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHKeyPath
I0818 12:09:32.586649 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHUsername
I0818 12:09:32.586801 3976 main.go:141] libmachine: Using SSH client type: native
I0818 12:09:32.586940 3976 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0x3bf3ea0] 0x3bf6c00 <nil> [] 0s} 192.169.0.6 22 <nil> <nil>}
I0818 12:09:32.586986 3976 main.go:141] libmachine: About to run SSH command:
sudo mkdir -p /lib/systemd/system && printf %s "[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network.target minikube-automount.service docker.socket
Requires= minikube-automount.service docker.socket
StartLimitBurst=3
StartLimitIntervalSec=60
[Service]
Type=notify
Restart=on-failure
Environment="NO_PROXY=192.169.0.5"
# This file is a systemd drop-in unit that inherits from the base dockerd configuration.
# The base configuration already specifies an 'ExecStart=...' command. The first directive
# here is to clear out that command inherited from the base configuration. Without this,
# the command from the base configuration and the command specified here are treated as
# a sequence of commands, which is not the desired behavior, nor is it valid -- systemd
# will catch this invalid input and refuse to start the service with an error like:
# Service has more than one ExecStart= setting, which is only allowed for Type=oneshot services.
# NOTE: default-ulimit=nofile is set to an arbitrary number for consistency with other
# container runtimes. If left unlimited, it may result in OOM issues with MySQL.
ExecStart=
ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --default-ulimit=nofile=1048576:1048576 --tlsverify --tlscacert /etc/docker/ca.pem --tlscert /etc/docker/server.pem --tlskey /etc/docker/server-key.pem --label provider=hyperkit --insecure-registry 10.96.0.0/12
ExecReload=/bin/kill -s HUP \$MAINPID
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
TimeoutStartSec=0
# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes
# kill only the docker process, not all processes in the cgroup
KillMode=process
[Install]
WantedBy=multi-user.target
" | sudo tee /lib/systemd/system/docker.service.new
I0818 12:09:32.654663 3976 main.go:141] libmachine: SSH cmd err, output: <nil>: [Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network.target minikube-automount.service docker.socket
Requires= minikube-automount.service docker.socket
StartLimitBurst=3
StartLimitIntervalSec=60
[Service]
Type=notify
Restart=on-failure
Environment=NO_PROXY=192.169.0.5
# This file is a systemd drop-in unit that inherits from the base dockerd configuration.
# The base configuration already specifies an 'ExecStart=...' command. The first directive
# here is to clear out that command inherited from the base configuration. Without this,
# the command from the base configuration and the command specified here are treated as
# a sequence of commands, which is not the desired behavior, nor is it valid -- systemd
# will catch this invalid input and refuse to start the service with an error like:
# Service has more than one ExecStart= setting, which is only allowed for Type=oneshot services.
# NOTE: default-ulimit=nofile is set to an arbitrary number for consistency with other
# container runtimes. If left unlimited, it may result in OOM issues with MySQL.
ExecStart=
ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --default-ulimit=nofile=1048576:1048576 --tlsverify --tlscacert /etc/docker/ca.pem --tlscert /etc/docker/server.pem --tlskey /etc/docker/server-key.pem --label provider=hyperkit --insecure-registry 10.96.0.0/12
ExecReload=/bin/kill -s HUP $MAINPID
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
TimeoutStartSec=0
# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes
# kill only the docker process, not all processes in the cgroup
KillMode=process
[Install]
WantedBy=multi-user.target
I0818 12:09:32.654688 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHHostname
I0818 12:09:32.654820 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHPort
I0818 12:09:32.654904 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHKeyPath
I0818 12:09:32.654974 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHKeyPath
I0818 12:09:32.655053 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHUsername
I0818 12:09:32.655180 3976 main.go:141] libmachine: Using SSH client type: native
I0818 12:09:32.655330 3976 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0x3bf3ea0] 0x3bf6c00 <nil> [] 0s} 192.169.0.6 22 <nil> <nil>}
I0818 12:09:32.655343 3976 main.go:141] libmachine: About to run SSH command:
sudo diff -u /lib/systemd/system/docker.service /lib/systemd/system/docker.service.new || { sudo mv /lib/systemd/system/docker.service.new /lib/systemd/system/docker.service; sudo systemctl -f daemon-reload && sudo systemctl -f enable docker && sudo systemctl -f restart docker; }
I0818 12:09:34.321102 3976 main.go:141] libmachine: SSH cmd err, output: <nil>: diff: can't stat '/lib/systemd/system/docker.service': No such file or directory
Created symlink /etc/systemd/system/multi-user.target.wants/docker.service → /usr/lib/systemd/system/docker.service.
I0818 12:09:34.321115 3976 machine.go:96] duration metric: took 14.8152512s to provisionDockerMachine
I0818 12:09:34.321123 3976 start.go:293] postStartSetup for "ha-373000-m02" (driver="hyperkit")
I0818 12:09:34.321131 3976 start.go:322] creating required directories: [/etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs]
I0818 12:09:34.321140 3976 main.go:141] libmachine: (ha-373000-m02) Calling .DriverName
I0818 12:09:34.321324 3976 ssh_runner.go:195] Run: sudo mkdir -p /etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs
I0818 12:09:34.321348 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHHostname
I0818 12:09:34.321440 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHPort
I0818 12:09:34.321528 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHKeyPath
I0818 12:09:34.321619 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHUsername
I0818 12:09:34.321715 3976 sshutil.go:53] new ssh client: &{IP:192.169.0.6 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02/id_rsa Username:docker}
I0818 12:09:34.356724 3976 ssh_runner.go:195] Run: cat /etc/os-release
I0818 12:09:34.363921 3976 info.go:137] Remote host: Buildroot 2023.02.9
I0818 12:09:34.363935 3976 filesync.go:126] Scanning /Users/jenkins/minikube-integration/19423-1007/.minikube/addons for local assets ...
I0818 12:09:34.364038 3976 filesync.go:126] Scanning /Users/jenkins/minikube-integration/19423-1007/.minikube/files for local assets ...
I0818 12:09:34.364185 3976 filesync.go:149] local asset: /Users/jenkins/minikube-integration/19423-1007/.minikube/files/etc/ssl/certs/15262.pem -> 15262.pem in /etc/ssl/certs
I0818 12:09:34.364192 3976 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19423-1007/.minikube/files/etc/ssl/certs/15262.pem -> /etc/ssl/certs/15262.pem
I0818 12:09:34.364347 3976 ssh_runner.go:195] Run: sudo mkdir -p /etc/ssl/certs
I0818 12:09:34.379409 3976 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19423-1007/.minikube/files/etc/ssl/certs/15262.pem --> /etc/ssl/certs/15262.pem (1708 bytes)
I0818 12:09:34.407459 3976 start.go:296] duration metric: took 86.328927ms for postStartSetup
I0818 12:09:34.407481 3976 main.go:141] libmachine: (ha-373000-m02) Calling .DriverName
I0818 12:09:34.407638 3976 ssh_runner.go:195] Run: sudo ls --almost-all -1 /var/lib/minikube/backup
I0818 12:09:34.407658 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHHostname
I0818 12:09:34.407738 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHPort
I0818 12:09:34.407823 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHKeyPath
I0818 12:09:34.407908 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHUsername
I0818 12:09:34.407985 3976 sshutil.go:53] new ssh client: &{IP:192.169.0.6 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02/id_rsa Username:docker}
I0818 12:09:34.441305 3976 machine.go:197] restoring vm config from /var/lib/minikube/backup: [etc]
I0818 12:09:34.441365 3976 ssh_runner.go:195] Run: sudo rsync --archive --update /var/lib/minikube/backup/etc /
I0818 12:09:34.475756 3976 fix.go:56] duration metric: took 15.082665832s for fixHost
I0818 12:09:34.475780 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHHostname
I0818 12:09:34.475917 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHPort
I0818 12:09:34.476014 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHKeyPath
I0818 12:09:34.476109 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHKeyPath
I0818 12:09:34.476204 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHUsername
I0818 12:09:34.476334 3976 main.go:141] libmachine: Using SSH client type: native
I0818 12:09:34.476475 3976 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0x3bf3ea0] 0x3bf6c00 <nil> [] 0s} 192.169.0.6 22 <nil> <nil>}
I0818 12:09:34.476483 3976 main.go:141] libmachine: About to run SSH command:
date +%s.%N
I0818 12:09:34.531245 3976 main.go:141] libmachine: SSH cmd err, output: <nil>: 1724008174.705830135
I0818 12:09:34.531256 3976 fix.go:216] guest clock: 1724008174.705830135
I0818 12:09:34.531265 3976 fix.go:229] Guest: 2024-08-18 12:09:34.705830135 -0700 PDT Remote: 2024-08-18 12:09:34.475769 -0700 PDT m=+34.122913514 (delta=230.061135ms)
I0818 12:09:34.531276 3976 fix.go:200] guest clock delta is within tolerance: 230.061135ms
I0818 12:09:34.531281 3976 start.go:83] releasing machines lock for "ha-373000-m02", held for 15.138221498s
I0818 12:09:34.531298 3976 main.go:141] libmachine: (ha-373000-m02) Calling .DriverName
I0818 12:09:34.531428 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetIP
I0818 12:09:34.555141 3976 out.go:177] * Found network options:
I0818 12:09:34.576875 3976 out.go:177] - NO_PROXY=192.169.0.5
W0818 12:09:34.597784 3976 proxy.go:119] fail to check proxy env: Error ip not in block
I0818 12:09:34.597830 3976 main.go:141] libmachine: (ha-373000-m02) Calling .DriverName
I0818 12:09:34.598688 3976 main.go:141] libmachine: (ha-373000-m02) Calling .DriverName
I0818 12:09:34.598932 3976 main.go:141] libmachine: (ha-373000-m02) Calling .DriverName
I0818 12:09:34.599031 3976 ssh_runner.go:195] Run: curl -sS -m 2 https://registry.k8s.io/
I0818 12:09:34.599086 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHHostname
W0818 12:09:34.599150 3976 proxy.go:119] fail to check proxy env: Error ip not in block
I0818 12:09:34.599257 3976 ssh_runner.go:195] Run: sh -c "stat /etc/cni/net.d/*loopback.conf*"
I0818 12:09:34.599278 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHHostname
I0818 12:09:34.599308 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHPort
I0818 12:09:34.599482 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHPort
I0818 12:09:34.599521 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHKeyPath
I0818 12:09:34.599684 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHKeyPath
I0818 12:09:34.599720 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHUsername
I0818 12:09:34.599871 3976 main.go:141] libmachine: (ha-373000-m02) Calling .GetSSHUsername
I0818 12:09:34.599921 3976 sshutil.go:53] new ssh client: &{IP:192.169.0.6 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02/id_rsa Username:docker}
I0818 12:09:34.600032 3976 sshutil.go:53] new ssh client: &{IP:192.169.0.6 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19423-1007/.minikube/machines/ha-373000-m02/id_rsa Username:docker}
W0818 12:09:34.631739 3976 cni.go:209] loopback cni configuration skipped: "/etc/cni/net.d/*loopback.conf*" not found
I0818 12:09:34.631799 3976 ssh_runner.go:195] Run: sudo find /etc/cni/net.d -maxdepth 1 -type f ( ( -name *bridge* -or -name *podman* ) -and -not -name *.mk_disabled ) -printf "%p, " -exec sh -c "sudo mv {} {}.mk_disabled" ;
I0818 12:09:34.677593 3976 cni.go:262] disabled [/etc/cni/net.d/87-podman-bridge.conflist] bridge cni config(s)
I0818 12:09:34.677615 3976 start.go:495] detecting cgroup driver to use...
I0818 12:09:34.677737 3976 ssh_runner.go:195] Run: /bin/bash -c "sudo mkdir -p /etc && printf %s "runtime-endpoint: unix:///run/containerd/containerd.sock
" | sudo tee /etc/crictl.yaml"
I0818 12:09:34.693773 3976 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)sandbox_image = .*$|\1sandbox_image = "registry.k8s.io/pause:3.10"|' /etc/containerd/config.toml"
I0818 12:09:34.702951 3976 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)restrict_oom_score_adj = .*$|\1restrict_oom_score_adj = false|' /etc/containerd/config.toml"
I0818 12:09:34.711799 3976 containerd.go:146] configuring containerd to use "cgroupfs" as cgroup driver...
I0818 12:09:34.711840 3976 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)SystemdCgroup = .*$|\1SystemdCgroup = false|g' /etc/containerd/config.toml"
I0818 12:09:34.720906 3976 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|"io.containerd.runtime.v1.linux"|"io.containerd.runc.v2"|g' /etc/containerd/config.toml"
I0818 12:09:34.729957 3976 ssh_runner.go:195] Run: sh -c "sudo sed -i '/systemd_cgroup/d' /etc/containerd/config.toml"
I0818 12:09:34.738902 3976 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|"io.containerd.runc.v1"|"io.containerd.runc.v2"|g' /etc/containerd/config.toml"
I0818 12:09:34.747932 3976 ssh_runner.go:195] Run: sh -c "sudo rm -rf /etc/cni/net.mk"
I0818 12:09:34.757312 3976 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)conf_dir = .*$|\1conf_dir = "/etc/cni/net.d"|g' /etc/containerd/config.toml"
I0818 12:09:34.766375 3976 ssh_runner.go:195] Run: sh -c "sudo sed -i '/^ *enable_unprivileged_ports = .*/d' /etc/containerd/config.toml"
I0818 12:09:34.775307 3976 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)\[plugins."io.containerd.grpc.v1.cri"\]|&\n\1 enable_unprivileged_ports = true|' /etc/containerd/config.toml"
I0818 12:09:34.784400 3976 ssh_runner.go:195] Run: sudo sysctl net.bridge.bridge-nf-call-iptables
I0818 12:09:34.792630 3976 ssh_runner.go:195] Run: sudo sh -c "echo 1 > /proc/sys/net/ipv4/ip_forward"
I0818 12:09:34.801021 3976 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I0818 12:09:34.911872 3976 ssh_runner.go:195] Run: sudo systemctl restart containerd
I0818 12:09:34.930682 3976 start.go:495] detecting cgroup driver to use...
I0818 12:09:34.930753 3976 ssh_runner.go:195] Run: sudo systemctl cat docker.service
I0818 12:09:34.944697 3976 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service containerd
I0818 12:09:34.956782 3976 ssh_runner.go:195] Run: sudo systemctl stop -f containerd
I0818 12:09:34.974233 3976 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service containerd
I0818 12:09:34.986114 3976 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service crio
I0818 12:09:34.998297 3976 ssh_runner.go:195] Run: sudo systemctl stop -f crio
I0818 12:09:35.018378 3976 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service crio
I0818 12:09:35.029759 3976 ssh_runner.go:195] Run: /bin/bash -c "sudo mkdir -p /etc && printf %s "runtime-endpoint: unix:///var/run/cri-dockerd.sock
" | sudo tee /etc/crictl.yaml"
I0818 12:09:35.044553 3976 ssh_runner.go:195] Run: which cri-dockerd
I0818 12:09:35.047654 3976 ssh_runner.go:195] Run: sudo mkdir -p /etc/systemd/system/cri-docker.service.d
I0818 12:09:35.055897 3976 ssh_runner.go:362] scp memory --> /etc/systemd/system/cri-docker.service.d/10-cni.conf (190 bytes)
I0818 12:09:35.069339 3976 ssh_runner.go:195] Run: sudo systemctl unmask docker.service
I0818 12:09:35.163048 3976 ssh_runner.go:195] Run: sudo systemctl enable docker.socket
I0818 12:09:35.263866 3976 docker.go:574] configuring docker to use "cgroupfs" as cgroup driver...
I0818 12:09:35.263888 3976 ssh_runner.go:362] scp memory --> /etc/docker/daemon.json (130 bytes)
I0818 12:09:35.281642 3976 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I0818 12:09:35.375004 3976 ssh_runner.go:195] Run: sudo systemctl restart docker
I0818 12:10:36.400829 3976 ssh_runner.go:235] Completed: sudo systemctl restart docker: (1m1.027707476s)
I0818 12:10:36.400907 3976 ssh_runner.go:195] Run: sudo journalctl --no-pager -u docker
I0818 12:10:36.437434 3976 out.go:201]
W0818 12:10:36.459246 3976 out.go:270] X Exiting due to RUNTIME_ENABLE: Failed to enable container runtime: sudo systemctl restart docker: Process exited with status 1
stdout:
stderr:
Job for docker.service failed because the control process exited with error code.
See "systemctl status docker.service" and "journalctl -xeu docker.service" for details.
sudo journalctl --no-pager -u docker:
-- stdout --
Aug 18 19:09:33 ha-373000-m02 systemd[1]: Starting Docker Application Container Engine...
Aug 18 19:09:33 ha-373000-m02 dockerd[496]: time="2024-08-18T19:09:33.132734313Z" level=info msg="Starting up"
Aug 18 19:09:33 ha-373000-m02 dockerd[496]: time="2024-08-18T19:09:33.133217341Z" level=info msg="containerd not running, starting managed containerd"
Aug 18 19:09:33 ha-373000-m02 dockerd[496]: time="2024-08-18T19:09:33.133706453Z" level=info msg="started new containerd process" address=/var/run/docker/containerd/containerd.sock module=libcontainerd pid=503
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.150884592Z" level=info msg="starting containerd" revision=8fc6bcff51318944179630522a095cc9dbf9f353 version=v1.7.20
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.165526624Z" level=info msg="loading plugin \"io.containerd.event.v1.exchange\"..." type=io.containerd.event.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.165600672Z" level=info msg="loading plugin \"io.containerd.internal.v1.opt\"..." type=io.containerd.internal.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.165665661Z" level=info msg="loading plugin \"io.containerd.warning.v1.deprecations\"..." type=io.containerd.warning.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.165701505Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.blockfile\"..." type=io.containerd.snapshotter.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.165883163Z" level=info msg="skip loading plugin \"io.containerd.snapshotter.v1.blockfile\"..." error="no scratch file generator: skip plugin" type=io.containerd.snapshotter.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.165980711Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.btrfs\"..." type=io.containerd.snapshotter.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.166114419Z" level=info msg="skip loading plugin \"io.containerd.snapshotter.v1.btrfs\"..." error="path /var/lib/docker/containerd/daemon/io.containerd.snapshotter.v1.btrfs (ext4) must be a btrfs filesystem to be used with the btrfs snapshotter: skip plugin" type=io.containerd.snapshotter.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.166158739Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.devmapper\"..." type=io.containerd.snapshotter.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.166192923Z" level=info msg="skip loading plugin \"io.containerd.snapshotter.v1.devmapper\"..." error="devmapper not configured: skip plugin" type=io.containerd.snapshotter.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.166222480Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.native\"..." type=io.containerd.snapshotter.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.166373263Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.overlayfs\"..." type=io.containerd.snapshotter.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.166624364Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.aufs\"..." type=io.containerd.snapshotter.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.168284638Z" level=info msg="skip loading plugin \"io.containerd.snapshotter.v1.aufs\"..." error="aufs is not supported (modprobe aufs failed: exit status 1 \"modprobe: FATAL: Module aufs not found in directory /lib/modules/5.10.207\\n\"): skip plugin" type=io.containerd.snapshotter.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.168338968Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.zfs\"..." type=io.containerd.snapshotter.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.168477528Z" level=info msg="skip loading plugin \"io.containerd.snapshotter.v1.zfs\"..." error="path /var/lib/docker/containerd/daemon/io.containerd.snapshotter.v1.zfs must be a zfs filesystem to be used with the zfs snapshotter: skip plugin" type=io.containerd.snapshotter.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.168522410Z" level=info msg="loading plugin \"io.containerd.content.v1.content\"..." type=io.containerd.content.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.168684236Z" level=info msg="loading plugin \"io.containerd.metadata.v1.bolt\"..." type=io.containerd.metadata.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.168742254Z" level=info msg="metadata content store policy set" policy=shared
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.172229271Z" level=info msg="loading plugin \"io.containerd.gc.v1.scheduler\"..." type=io.containerd.gc.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.172291175Z" level=info msg="loading plugin \"io.containerd.differ.v1.walking\"..." type=io.containerd.differ.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.172328725Z" level=info msg="loading plugin \"io.containerd.lease.v1.manager\"..." type=io.containerd.lease.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.172361584Z" level=info msg="loading plugin \"io.containerd.streaming.v1.manager\"..." type=io.containerd.streaming.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.172397084Z" level=info msg="loading plugin \"io.containerd.runtime.v1.linux\"..." type=io.containerd.runtime.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.172469115Z" level=info msg="loading plugin \"io.containerd.monitor.v1.cgroups\"..." type=io.containerd.monitor.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.172636000Z" level=info msg="loading plugin \"io.containerd.runtime.v2.task\"..." type=io.containerd.runtime.v2
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.172713269Z" level=info msg="loading plugin \"io.containerd.runtime.v2.shim\"..." type=io.containerd.runtime.v2
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.172756026Z" level=info msg="loading plugin \"io.containerd.sandbox.store.v1.local\"..." type=io.containerd.sandbox.store.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.172790721Z" level=info msg="loading plugin \"io.containerd.sandbox.controller.v1.local\"..." type=io.containerd.sandbox.controller.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.172822478Z" level=info msg="loading plugin \"io.containerd.service.v1.containers-service\"..." type=io.containerd.service.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.172857013Z" level=info msg="loading plugin \"io.containerd.service.v1.content-service\"..." type=io.containerd.service.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.172889097Z" level=info msg="loading plugin \"io.containerd.service.v1.diff-service\"..." type=io.containerd.service.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.172923123Z" level=info msg="loading plugin \"io.containerd.service.v1.images-service\"..." type=io.containerd.service.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.172955052Z" level=info msg="loading plugin \"io.containerd.service.v1.introspection-service\"..." type=io.containerd.service.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.172985350Z" level=info msg="loading plugin \"io.containerd.service.v1.namespaces-service\"..." type=io.containerd.service.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.173017995Z" level=info msg="loading plugin \"io.containerd.service.v1.snapshots-service\"..." type=io.containerd.service.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.173047134Z" level=info msg="loading plugin \"io.containerd.service.v1.tasks-service\"..." type=io.containerd.service.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.173082956Z" level=info msg="loading plugin \"io.containerd.grpc.v1.containers\"..." type=io.containerd.grpc.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.173138952Z" level=info msg="loading plugin \"io.containerd.grpc.v1.content\"..." type=io.containerd.grpc.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.173171857Z" level=info msg="loading plugin \"io.containerd.grpc.v1.diff\"..." type=io.containerd.grpc.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.173266115Z" level=info msg="loading plugin \"io.containerd.grpc.v1.events\"..." type=io.containerd.grpc.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.173303729Z" level=info msg="loading plugin \"io.containerd.grpc.v1.images\"..." type=io.containerd.grpc.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.173337305Z" level=info msg="loading plugin \"io.containerd.grpc.v1.introspection\"..." type=io.containerd.grpc.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.173367548Z" level=info msg="loading plugin \"io.containerd.grpc.v1.leases\"..." type=io.containerd.grpc.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.173397195Z" level=info msg="loading plugin \"io.containerd.grpc.v1.namespaces\"..." type=io.containerd.grpc.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.173426651Z" level=info msg="loading plugin \"io.containerd.grpc.v1.sandbox-controllers\"..." type=io.containerd.grpc.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.173461907Z" level=info msg="loading plugin \"io.containerd.grpc.v1.sandboxes\"..." type=io.containerd.grpc.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.173491945Z" level=info msg="loading plugin \"io.containerd.grpc.v1.snapshots\"..." type=io.containerd.grpc.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.173521151Z" level=info msg="loading plugin \"io.containerd.grpc.v1.streaming\"..." type=io.containerd.grpc.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.173551817Z" level=info msg="loading plugin \"io.containerd.grpc.v1.tasks\"..." type=io.containerd.grpc.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.173584158Z" level=info msg="loading plugin \"io.containerd.transfer.v1.local\"..." type=io.containerd.transfer.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.173620017Z" level=info msg="loading plugin \"io.containerd.grpc.v1.transfer\"..." type=io.containerd.grpc.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.173651734Z" level=info msg="loading plugin \"io.containerd.grpc.v1.version\"..." type=io.containerd.grpc.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.173681138Z" level=info msg="loading plugin \"io.containerd.internal.v1.restart\"..." type=io.containerd.internal.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.173753818Z" level=info msg="loading plugin \"io.containerd.tracing.processor.v1.otlp\"..." type=io.containerd.tracing.processor.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.173797160Z" level=info msg="skip loading plugin \"io.containerd.tracing.processor.v1.otlp\"..." error="skip plugin: tracing endpoint not configured" type=io.containerd.tracing.processor.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.173851051Z" level=info msg="loading plugin \"io.containerd.internal.v1.tracing\"..." type=io.containerd.internal.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.173888629Z" level=info msg="skip loading plugin \"io.containerd.internal.v1.tracing\"..." error="skip plugin: tracing endpoint not configured" type=io.containerd.internal.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.173919044Z" level=info msg="loading plugin \"io.containerd.grpc.v1.healthcheck\"..." type=io.containerd.grpc.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.173948712Z" level=info msg="loading plugin \"io.containerd.nri.v1.nri\"..." type=io.containerd.nri.v1
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.173979628Z" level=info msg="NRI interface is disabled by configuration."
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.174202763Z" level=info msg=serving... address=/var/run/docker/containerd/containerd-debug.sock
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.174288578Z" level=info msg=serving... address=/var/run/docker/containerd/containerd.sock.ttrpc
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.174373231Z" level=info msg=serving... address=/var/run/docker/containerd/containerd.sock
Aug 18 19:09:33 ha-373000-m02 dockerd[503]: time="2024-08-18T19:09:33.174419718Z" level=info msg="containerd successfully booted in 0.024281s"
Aug 18 19:09:34 ha-373000-m02 dockerd[496]: time="2024-08-18T19:09:34.163281667Z" level=info msg="[graphdriver] trying configured driver: overlay2"
Aug 18 19:09:34 ha-373000-m02 dockerd[496]: time="2024-08-18T19:09:34.193454663Z" level=info msg="Loading containers: start."
Aug 18 19:09:34 ha-373000-m02 dockerd[496]: time="2024-08-18T19:09:34.358483324Z" level=warning msg="ip6tables is enabled, but cannot set up ip6tables chains" error="failed to create NAT chain DOCKER: iptables failed: ip6tables --wait -t nat -N DOCKER: ip6tables v1.8.9 (legacy): can't initialize ip6tables table `nat': Table does not exist (do you need to insmod?)\nPerhaps ip6tables or your kernel needs to be upgraded.\n (exit status 3)"
Aug 18 19:09:34 ha-373000-m02 dockerd[496]: time="2024-08-18T19:09:34.419779026Z" level=info msg="Default bridge (docker0) is assigned with an IP address 172.17.0.0/16. Daemon option --bip can be used to set a preferred IP address"
Aug 18 19:09:34 ha-373000-m02 dockerd[496]: time="2024-08-18T19:09:34.464759087Z" level=info msg="Loading containers: done."
Aug 18 19:09:34 ha-373000-m02 dockerd[496]: time="2024-08-18T19:09:34.475407585Z" level=info msg="Docker daemon" commit=f9522e5 containerd-snapshotter=false storage-driver=overlay2 version=27.1.2
Aug 18 19:09:34 ha-373000-m02 dockerd[496]: time="2024-08-18T19:09:34.475556691Z" level=info msg="Daemon has completed initialization"
Aug 18 19:09:34 ha-373000-m02 dockerd[496]: time="2024-08-18T19:09:34.493178383Z" level=info msg="API listen on /var/run/docker.sock"
Aug 18 19:09:34 ha-373000-m02 dockerd[496]: time="2024-08-18T19:09:34.493236047Z" level=info msg="API listen on [::]:2376"
Aug 18 19:09:34 ha-373000-m02 systemd[1]: Started Docker Application Container Engine.
Aug 18 19:09:35 ha-373000-m02 dockerd[496]: time="2024-08-18T19:09:35.562066100Z" level=info msg="Processing signal 'terminated'"
Aug 18 19:09:35 ha-373000-m02 systemd[1]: Stopping Docker Application Container Engine...
Aug 18 19:09:35 ha-373000-m02 dockerd[496]: time="2024-08-18T19:09:35.563196599Z" level=info msg="stopping event stream following graceful shutdown" error="<nil>" module=libcontainerd namespace=moby
Aug 18 19:09:35 ha-373000-m02 dockerd[496]: time="2024-08-18T19:09:35.563381674Z" level=info msg="Daemon shutdown complete"
Aug 18 19:09:35 ha-373000-m02 dockerd[496]: time="2024-08-18T19:09:35.563404669Z" level=info msg="stopping healthcheck following graceful shutdown" module=libcontainerd
Aug 18 19:09:35 ha-373000-m02 dockerd[496]: time="2024-08-18T19:09:35.563423915Z" level=info msg="stopping event stream following graceful shutdown" error="context canceled" module=libcontainerd namespace=plugins.moby
Aug 18 19:09:36 ha-373000-m02 systemd[1]: docker.service: Deactivated successfully.
Aug 18 19:09:36 ha-373000-m02 systemd[1]: Stopped Docker Application Container Engine.
Aug 18 19:09:36 ha-373000-m02 systemd[1]: Starting Docker Application Container Engine...
Aug 18 19:09:36 ha-373000-m02 dockerd[1172]: time="2024-08-18T19:09:36.603637435Z" level=info msg="Starting up"
Aug 18 19:10:36 ha-373000-m02 dockerd[1172]: failed to start daemon: failed to dial "/run/containerd/containerd.sock": failed to dial "/run/containerd/containerd.sock": context deadline exceeded
Aug 18 19:10:36 ha-373000-m02 systemd[1]: docker.service: Main process exited, code=exited, status=1/FAILURE
Aug 18 19:10:36 ha-373000-m02 systemd[1]: docker.service: Failed with result 'exit-code'.
Aug 18 19:10:36 ha-373000-m02 systemd[1]: Failed to start Docker Application Container Engine.
-- /stdout --
W0818 12:10:36.459348 3976 out.go:270] *
W0818 12:10:36.460605 3976 out.go:293] ╭─────────────────────────────────────────────────────────────────────────────────────────────╮
│ │
│ * If the above advice does not help, please let us know: │
│ https://github.com/kubernetes/minikube/issues/new/choose │
│ │
│ * Please run `minikube logs --file=logs.txt` and attach logs.txt to the GitHub issue. │
│ │
╰─────────────────────────────────────────────────────────────────────────────────────────────╯
I0818 12:10:36.503171 3976 out.go:201]
==> Docker <==
Aug 18 19:09:46 ha-373000 dockerd[1177]: time="2024-08-18T19:09:46.453438095Z" level=info msg="shim disconnected" id=6f6bc1d9592b465e3b7c4d9db0e74b67040cbbed37626775b4aad68af80ddeb2 namespace=moby
Aug 18 19:09:46 ha-373000 dockerd[1177]: time="2024-08-18T19:09:46.453510509Z" level=warning msg="cleaning up after shim disconnected" id=6f6bc1d9592b465e3b7c4d9db0e74b67040cbbed37626775b4aad68af80ddeb2 namespace=moby
Aug 18 19:09:46 ha-373000 dockerd[1177]: time="2024-08-18T19:09:46.453519178Z" level=info msg="cleaning up dead shim" namespace=moby
Aug 18 19:09:46 ha-373000 dockerd[1169]: time="2024-08-18T19:09:46.453809871Z" level=info msg="ignoring event" container=6f6bc1d9592b465e3b7c4d9db0e74b67040cbbed37626775b4aad68af80ddeb2 module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Aug 18 19:09:47 ha-373000 dockerd[1177]: time="2024-08-18T19:09:47.461847284Z" level=info msg="shim disconnected" id=0a3644cb0db7c69e66c1b1d0f3234fba20e233a15e27e5772148d80d14f33f4e namespace=moby
Aug 18 19:09:47 ha-373000 dockerd[1177]: time="2024-08-18T19:09:47.461900797Z" level=warning msg="cleaning up after shim disconnected" id=0a3644cb0db7c69e66c1b1d0f3234fba20e233a15e27e5772148d80d14f33f4e namespace=moby
Aug 18 19:09:47 ha-373000 dockerd[1177]: time="2024-08-18T19:09:47.461909634Z" level=info msg="cleaning up dead shim" namespace=moby
Aug 18 19:09:47 ha-373000 dockerd[1169]: time="2024-08-18T19:09:47.462210879Z" level=info msg="ignoring event" container=0a3644cb0db7c69e66c1b1d0f3234fba20e233a15e27e5772148d80d14f33f4e module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Aug 18 19:10:04 ha-373000 dockerd[1177]: time="2024-08-18T19:10:04.870147305Z" level=info msg="loading plugin \"io.containerd.event.v1.publisher\"..." runtime=io.containerd.runc.v2 type=io.containerd.event.v1
Aug 18 19:10:04 ha-373000 dockerd[1177]: time="2024-08-18T19:10:04.870333575Z" level=info msg="loading plugin \"io.containerd.internal.v1.shutdown\"..." runtime=io.containerd.runc.v2 type=io.containerd.internal.v1
Aug 18 19:10:04 ha-373000 dockerd[1177]: time="2024-08-18T19:10:04.870347019Z" level=info msg="loading plugin \"io.containerd.ttrpc.v1.task\"..." runtime=io.containerd.runc.v2 type=io.containerd.ttrpc.v1
Aug 18 19:10:04 ha-373000 dockerd[1177]: time="2024-08-18T19:10:04.870447403Z" level=info msg="loading plugin \"io.containerd.ttrpc.v1.pause\"..." runtime=io.containerd.runc.v2 type=io.containerd.ttrpc.v1
Aug 18 19:10:06 ha-373000 dockerd[1177]: time="2024-08-18T19:10:06.866261869Z" level=info msg="loading plugin \"io.containerd.event.v1.publisher\"..." runtime=io.containerd.runc.v2 type=io.containerd.event.v1
Aug 18 19:10:06 ha-373000 dockerd[1177]: time="2024-08-18T19:10:06.866358878Z" level=info msg="loading plugin \"io.containerd.internal.v1.shutdown\"..." runtime=io.containerd.runc.v2 type=io.containerd.internal.v1
Aug 18 19:10:06 ha-373000 dockerd[1177]: time="2024-08-18T19:10:06.866371963Z" level=info msg="loading plugin \"io.containerd.ttrpc.v1.task\"..." runtime=io.containerd.runc.v2 type=io.containerd.ttrpc.v1
Aug 18 19:10:06 ha-373000 dockerd[1177]: time="2024-08-18T19:10:06.866695913Z" level=info msg="loading plugin \"io.containerd.ttrpc.v1.pause\"..." runtime=io.containerd.runc.v2 type=io.containerd.ttrpc.v1
Aug 18 19:10:27 ha-373000 dockerd[1177]: time="2024-08-18T19:10:27.508003021Z" level=info msg="shim disconnected" id=24788de6a779b6b3b57b2e2becb6248a4b461c40237836c9a085858f19bf537e namespace=moby
Aug 18 19:10:27 ha-373000 dockerd[1177]: time="2024-08-18T19:10:27.508123156Z" level=warning msg="cleaning up after shim disconnected" id=24788de6a779b6b3b57b2e2becb6248a4b461c40237836c9a085858f19bf537e namespace=moby
Aug 18 19:10:27 ha-373000 dockerd[1177]: time="2024-08-18T19:10:27.508131683Z" level=info msg="cleaning up dead shim" namespace=moby
Aug 18 19:10:27 ha-373000 dockerd[1169]: time="2024-08-18T19:10:27.508457282Z" level=info msg="ignoring event" container=24788de6a779b6b3b57b2e2becb6248a4b461c40237836c9a085858f19bf537e module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Aug 18 19:10:27 ha-373000 dockerd[1177]: time="2024-08-18T19:10:27.520349911Z" level=warning msg="cleanup warnings time=\"2024-08-18T19:10:27Z\" level=warning msg=\"failed to remove runc container\" error=\"runc did not terminate successfully: exit status 255: \" runtime=io.containerd.runc.v2\n" namespace=moby
Aug 18 19:10:27 ha-373000 dockerd[1177]: time="2024-08-18T19:10:27.569676280Z" level=info msg="shim disconnected" id=7e27aa53db964848f64d9282a63325b77e8066d4f7afd7e078ca49d74f625756 namespace=moby
Aug 18 19:10:27 ha-373000 dockerd[1177]: time="2024-08-18T19:10:27.569734174Z" level=warning msg="cleaning up after shim disconnected" id=7e27aa53db964848f64d9282a63325b77e8066d4f7afd7e078ca49d74f625756 namespace=moby
Aug 18 19:10:27 ha-373000 dockerd[1177]: time="2024-08-18T19:10:27.569742722Z" level=info msg="cleaning up dead shim" namespace=moby
Aug 18 19:10:27 ha-373000 dockerd[1169]: time="2024-08-18T19:10:27.569876898Z" level=info msg="ignoring event" container=7e27aa53db964848f64d9282a63325b77e8066d4f7afd7e078ca49d74f625756 module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
==> container status <==
CONTAINER IMAGE CREATED STATE NAME ATTEMPT POD ID POD
7e27aa53db964 604f5db92eaa8 34 seconds ago Exited kube-apiserver 3 5f2bcb86e47be kube-apiserver-ha-373000
24788de6a779b 045733566833c 36 seconds ago Exited kube-controller-manager 4 45b85b05f9eab kube-controller-manager-ha-373000
e7bf93d680505 38af8ddebf499 About a minute ago Running kube-vip 1 37cbb7af9134a kube-vip-ha-373000
5bb7217cec87f 1766f54c897f0 About a minute ago Running kube-scheduler 2 11d6e68c74890 kube-scheduler-ha-373000
4ad014ace2b0a 2e96e5913fc06 About a minute ago Running etcd 2 4905344ca55ee etcd-ha-373000
eb459a6cac5c5 6e38f40d628db 4 minutes ago Exited storage-provisioner 2 3772c138aa65e storage-provisioner
fc1b30cd2c8f2 8c811b4aec35f 4 minutes ago Exited busybox 1 eb4ed9664dda9 busybox-7dff88458-hdg8r
f3dbf3c176d9d cbb01a7bd410d 4 minutes ago Exited coredns 1 fc1f2fb60f7c5 coredns-6f6b679f8f-rcfmc
09b8ded75e80f cbb01a7bd410d 4 minutes ago Exited coredns 1 bfce6a3dd1783 coredns-6f6b679f8f-hv98f
530d580001894 ad83b2ca7b09e 4 minutes ago Exited kube-proxy 1 c8f48c6f44e55 kube-proxy-2xkhp
fbeef7aab770f 12968670680f4 4 minutes ago Exited kindnet-cni 1 32a6ca59d02e7 kindnet-k4c4p
ebe78e53d91d8 38af8ddebf499 5 minutes ago Exited kube-vip 0 32cc18cf0bf63 kube-vip-ha-373000
a9e532272f1be 2e96e5913fc06 5 minutes ago Exited etcd 1 4c11500a40693 etcd-ha-373000
de016fdbd6fe9 1766f54c897f0 5 minutes ago Exited kube-scheduler 1 a3cc486386c46 kube-scheduler-ha-373000
==> coredns [09b8ded75e80] <==
.:53
[INFO] plugin/reload: Running configuration SHA512 = 257e111468ef6f1e36f10df061303186c353cd0e51aed8f50f4e4fd21cec02687aef97084fe1f82262f5cee88179d311670a6ae21ae185759728216fc264125f
CoreDNS-1.11.1
linux/amd64, go1.20.7, ae2bbc2
[INFO] 127.0.0.1:54168 - 48100 "HINFO IN 5449853140043981156.1960656544577820065. udp 57 false 512" NXDOMAIN qr,rd,ra 132 0.012696853s
[INFO] plugin/ready: Still waiting on: "kubernetes"
[INFO] plugin/ready: Still waiting on: "kubernetes"
[INFO] plugin/ready: Still waiting on: "kubernetes"
[INFO] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.27.4/tools/cache/reflector.go:231: failed to list *v1.Namespace: Get "https://10.96.0.1:443/api/v1/namespaces?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.27.4/tools/cache/reflector.go:231: failed to list *v1.Service: Get "https://10.96.0.1:443/api/v1/services?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] plugin/kubernetes: Trace[1317389180]: "Reflector ListAndWatch" name:pkg/mod/k8s.io/client-go@v0.27.4/tools/cache/reflector.go:231 (18-Aug-2024 19:05:43.060) (total time: 30002ms):
Trace[1317389180]: ---"Objects listed" error:Get "https://10.96.0.1:443/api/v1/services?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout 30002ms (19:06:13.063)
Trace[1317389180]: [30.002782846s] [30.002782846s] END
[ERROR] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.27.4/tools/cache/reflector.go:231: Failed to watch *v1.Service: failed to list *v1.Service: Get "https://10.96.0.1:443/api/v1/services?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] plugin/kubernetes: Trace[804407349]: "Reflector ListAndWatch" name:pkg/mod/k8s.io/client-go@v0.27.4/tools/cache/reflector.go:231 (18-Aug-2024 19:05:43.060) (total time: 30003ms):
Trace[804407349]: ---"Objects listed" error:Get "https://10.96.0.1:443/api/v1/namespaces?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout 30003ms (19:06:13.063)
Trace[804407349]: [30.003234686s] [30.003234686s] END
[ERROR] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.27.4/tools/cache/reflector.go:231: Failed to watch *v1.Namespace: failed to list *v1.Namespace: Get "https://10.96.0.1:443/api/v1/namespaces?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.27.4/tools/cache/reflector.go:231: failed to list *v1.EndpointSlice: Get "https://10.96.0.1:443/apis/discovery.k8s.io/v1/endpointslices?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] plugin/kubernetes: Trace[1407395902]: "Reflector ListAndWatch" name:pkg/mod/k8s.io/client-go@v0.27.4/tools/cache/reflector.go:231 (18-Aug-2024 19:05:43.063) (total time: 30001ms):
Trace[1407395902]: ---"Objects listed" error:Get "https://10.96.0.1:443/apis/discovery.k8s.io/v1/endpointslices?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout 30001ms (19:06:13.064)
Trace[1407395902]: [30.001205512s] [30.001205512s] END
[ERROR] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.27.4/tools/cache/reflector.go:231: Failed to watch *v1.EndpointSlice: failed to list *v1.EndpointSlice: Get "https://10.96.0.1:443/apis/discovery.k8s.io/v1/endpointslices?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] SIGTERM: Shutting down servers then terminating
[INFO] plugin/health: Going into lameduck mode for 5s
==> coredns [f3dbf3c176d9] <==
.:53
[INFO] plugin/reload: Running configuration SHA512 = 257e111468ef6f1e36f10df061303186c353cd0e51aed8f50f4e4fd21cec02687aef97084fe1f82262f5cee88179d311670a6ae21ae185759728216fc264125f
CoreDNS-1.11.1
linux/amd64, go1.20.7, ae2bbc2
[INFO] 127.0.0.1:48257 - 13179 "HINFO IN 3102078210809204073.2916918949998232158. udp 57 false 512" NXDOMAIN qr,rd,ra 132 0.013387746s
[INFO] plugin/ready: Still waiting on: "kubernetes"
[INFO] plugin/ready: Still waiting on: "kubernetes"
[INFO] plugin/ready: Still waiting on: "kubernetes"
[INFO] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.27.4/tools/cache/reflector.go:231: failed to list *v1.Namespace: Get "https://10.96.0.1:443/api/v1/namespaces?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] plugin/kubernetes: Trace[1929152146]: "Reflector ListAndWatch" name:pkg/mod/k8s.io/client-go@v0.27.4/tools/cache/reflector.go:231 (18-Aug-2024 19:05:43.060) (total time: 30003ms):
Trace[1929152146]: ---"Objects listed" error:Get "https://10.96.0.1:443/api/v1/namespaces?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout 30003ms (19:06:13.063)
Trace[1929152146]: [30.003742558s] [30.003742558s] END
[ERROR] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.27.4/tools/cache/reflector.go:231: Failed to watch *v1.Namespace: failed to list *v1.Namespace: Get "https://10.96.0.1:443/api/v1/namespaces?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.27.4/tools/cache/reflector.go:231: failed to list *v1.Service: Get "https://10.96.0.1:443/api/v1/services?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] plugin/kubernetes: Trace[763765503]: "Reflector ListAndWatch" name:pkg/mod/k8s.io/client-go@v0.27.4/tools/cache/reflector.go:231 (18-Aug-2024 19:05:43.060) (total time: 30003ms):
Trace[763765503]: ---"Objects listed" error:Get "https://10.96.0.1:443/api/v1/services?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout 30003ms (19:06:13.064)
Trace[763765503]: [30.003508272s] [30.003508272s] END
[ERROR] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.27.4/tools/cache/reflector.go:231: Failed to watch *v1.Service: failed to list *v1.Service: Get "https://10.96.0.1:443/api/v1/services?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.27.4/tools/cache/reflector.go:231: failed to list *v1.EndpointSlice: Get "https://10.96.0.1:443/apis/discovery.k8s.io/v1/endpointslices?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] plugin/kubernetes: Trace[1437534784]: "Reflector ListAndWatch" name:pkg/mod/k8s.io/client-go@v0.27.4/tools/cache/reflector.go:231 (18-Aug-2024 19:05:43.063) (total time: 30000ms):
Trace[1437534784]: ---"Objects listed" error:Get "https://10.96.0.1:443/apis/discovery.k8s.io/v1/endpointslices?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout 30000ms (19:06:13.064)
Trace[1437534784]: [30.000417221s] [30.000417221s] END
[ERROR] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.27.4/tools/cache/reflector.go:231: Failed to watch *v1.EndpointSlice: failed to list *v1.EndpointSlice: Get "https://10.96.0.1:443/apis/discovery.k8s.io/v1/endpointslices?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] SIGTERM: Shutting down servers then terminating
[INFO] plugin/health: Going into lameduck mode for 5s
==> describe nodes <==
command /bin/bash -c "sudo /var/lib/minikube/binaries/v1.31.0/kubectl describe nodes --kubeconfig=/var/lib/minikube/kubeconfig" failed with error: /bin/bash -c "sudo /var/lib/minikube/binaries/v1.31.0/kubectl describe nodes --kubeconfig=/var/lib/minikube/kubeconfig": Process exited with status 1
stdout:
stderr:
E0818 19:10:40.618965 2863 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://localhost:8443/api?timeout=32s\": dial tcp 127.0.0.1:8443: connect: connection refused"
E0818 19:10:40.620649 2863 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://localhost:8443/api?timeout=32s\": dial tcp 127.0.0.1:8443: connect: connection refused"
E0818 19:10:40.621991 2863 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://localhost:8443/api?timeout=32s\": dial tcp 127.0.0.1:8443: connect: connection refused"
E0818 19:10:40.623432 2863 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://localhost:8443/api?timeout=32s\": dial tcp 127.0.0.1:8443: connect: connection refused"
E0818 19:10:40.624811 2863 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://localhost:8443/api?timeout=32s\": dial tcp 127.0.0.1:8443: connect: connection refused"
The connection to the server localhost:8443 was refused - did you specify the right host or port?
==> dmesg <==
[ +0.000001] Any video related functionality will be severely degraded, and you may not even be able to suspend the system properly
[ +0.000000] Unless you actually understand what nomodeset does, you should reboot without enabling it
[ +0.035419] ACPI BIOS Warning (bug): Incorrect checksum in table [DSDT] - 0xBE, should be 0x1B (20200925/tbprint-173)
[ +0.007963] RETBleed: WARNING: Spectre v2 mitigation leaves CPU vulnerable to RETBleed attacks, data leaks possible!
[ +5.691053] ACPI Error: Could not enable RealTimeClock event (20200925/evxfevnt-182)
[ +0.000000] ACPI Warning: Could not enable fixed event - RealTimeClock (4) (20200925/evxface-618)
[ +0.006881] platform regulatory.0: Direct firmware load for regulatory.db failed with error -2
[ +2.891457] systemd-fstab-generator[127]: Ignoring "noauto" option for root device
[ +2.229875] NFSD: Using /var/lib/nfs/v4recovery as the NFSv4 state recovery directory
[ +0.000003] NFSD: unable to find recovery directory /var/lib/nfs/v4recovery
[ +0.000001] NFSD: Unable to initialize client recovery tracking! (-2)
[ +0.300202] systemd-fstab-generator[467]: Ignoring "noauto" option for root device
[ +0.101114] systemd-fstab-generator[479]: Ignoring "noauto" option for root device
[ +2.001813] systemd-fstab-generator[1098]: Ignoring "noauto" option for root device
[ +0.247527] systemd-fstab-generator[1135]: Ignoring "noauto" option for root device
[ +0.100995] systemd-fstab-generator[1147]: Ignoring "noauto" option for root device
[ +0.114396] systemd-fstab-generator[1161]: Ignoring "noauto" option for root device
[ +0.050935] kauditd_printk_skb: 145 callbacks suppressed
[ +2.471749] systemd-fstab-generator[1379]: Ignoring "noauto" option for root device
[ +0.100344] systemd-fstab-generator[1391]: Ignoring "noauto" option for root device
[ +0.088670] systemd-fstab-generator[1403]: Ignoring "noauto" option for root device
[ +0.117158] systemd-fstab-generator[1418]: Ignoring "noauto" option for root device
[ +0.433473] systemd-fstab-generator[1580]: Ignoring "noauto" option for root device
[ +6.511307] kauditd_printk_skb: 168 callbacks suppressed
[ +21.355887] kauditd_printk_skb: 40 callbacks suppressed
==> etcd [4ad014ace2b0] <==
{"level":"warn","ts":"2024-08-18T19:10:35.522345Z","caller":"rafthttp/probing_status.go:68","msg":"prober detected unhealthy status","round-tripper-name":"ROUND_TRIPPER_RAFT_MESSAGE","remote-peer-id":"64535d9f3a4791ce","rtt":"0s","error":"dial tcp 192.169.0.6:2380: connect: connection refused"}
{"level":"info","ts":"2024-08-18T19:10:35.645755Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 is starting a new election at term 3"}
{"level":"info","ts":"2024-08-18T19:10:35.645965Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 became pre-candidate at term 3"}
{"level":"info","ts":"2024-08-18T19:10:35.646145Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 received MsgPreVoteResp from b8c6c7563d17d844 at term 3"}
{"level":"info","ts":"2024-08-18T19:10:35.646837Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 [logterm: 3, index: 2939] sent MsgPreVote request to 64535d9f3a4791ce at term 3"}
{"level":"warn","ts":"2024-08-18T19:10:35.902167Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":15583740481722400528,"retry-timeout":"500ms"}
{"level":"warn","ts":"2024-08-18T19:10:36.409295Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":15583740481722400528,"retry-timeout":"500ms"}
{"level":"warn","ts":"2024-08-18T19:10:36.909620Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":15583740481722400528,"retry-timeout":"500ms"}
{"level":"warn","ts":"2024-08-18T19:10:37.410029Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":15583740481722400528,"retry-timeout":"500ms"}
{"level":"info","ts":"2024-08-18T19:10:37.439596Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 is starting a new election at term 3"}
{"level":"info","ts":"2024-08-18T19:10:37.439912Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 became pre-candidate at term 3"}
{"level":"info","ts":"2024-08-18T19:10:37.440199Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 received MsgPreVoteResp from b8c6c7563d17d844 at term 3"}
{"level":"info","ts":"2024-08-18T19:10:37.440412Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 [logterm: 3, index: 2939] sent MsgPreVote request to 64535d9f3a4791ce at term 3"}
{"level":"warn","ts":"2024-08-18T19:10:37.910246Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":15583740481722400528,"retry-timeout":"500ms"}
{"level":"warn","ts":"2024-08-18T19:10:38.411386Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":15583740481722400528,"retry-timeout":"500ms"}
{"level":"warn","ts":"2024-08-18T19:10:38.911588Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":15583740481722400528,"retry-timeout":"500ms"}
{"level":"info","ts":"2024-08-18T19:10:39.239512Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 is starting a new election at term 3"}
{"level":"info","ts":"2024-08-18T19:10:39.239538Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 became pre-candidate at term 3"}
{"level":"info","ts":"2024-08-18T19:10:39.239546Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 received MsgPreVoteResp from b8c6c7563d17d844 at term 3"}
{"level":"info","ts":"2024-08-18T19:10:39.239557Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"b8c6c7563d17d844 [logterm: 3, index: 2939] sent MsgPreVote request to 64535d9f3a4791ce at term 3"}
{"level":"warn","ts":"2024-08-18T19:10:39.412430Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":15583740481722400528,"retry-timeout":"500ms"}
{"level":"warn","ts":"2024-08-18T19:10:39.917438Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":15583740481722400528,"retry-timeout":"500ms"}
{"level":"warn","ts":"2024-08-18T19:10:40.418292Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":15583740481722400528,"retry-timeout":"500ms"}
{"level":"warn","ts":"2024-08-18T19:10:40.523142Z","caller":"rafthttp/probing_status.go:68","msg":"prober detected unhealthy status","round-tripper-name":"ROUND_TRIPPER_RAFT_MESSAGE","remote-peer-id":"64535d9f3a4791ce","rtt":"0s","error":"dial tcp 192.169.0.6:2380: connect: connection refused"}
{"level":"warn","ts":"2024-08-18T19:10:40.523187Z","caller":"rafthttp/probing_status.go:68","msg":"prober detected unhealthy status","round-tripper-name":"ROUND_TRIPPER_SNAPSHOT","remote-peer-id":"64535d9f3a4791ce","rtt":"0s","error":"dial tcp 192.169.0.6:2380: connect: connection refused"}
==> etcd [a9e532272f1b] <==
{"level":"warn","ts":"2024-08-18T19:08:52.581006Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2024-08-18T19:08:49.021104Z","time spent":"3.559898593s","remote":"127.0.0.1:56420","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":0,"response size":0,"request content":"key:\"/registry/services/specs/\" range_end:\"/registry/services/specs0\" count_only:true "}
2024/08/18 19:08:52 WARNING: [core] [Server #7] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing"
{"level":"warn","ts":"2024-08-18T19:08:52.581089Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"6.986891044s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/services/endpoints/kube-system/k8s.io-minikube-hostpath\" ","response":"","error":"context canceled"}
{"level":"info","ts":"2024-08-18T19:08:52.581101Z","caller":"traceutil/trace.go:171","msg":"trace[1676890744] range","detail":"{range_begin:/registry/services/endpoints/kube-system/k8s.io-minikube-hostpath; range_end:; }","duration":"6.986905757s","start":"2024-08-18T19:08:45.594192Z","end":"2024-08-18T19:08:52.581098Z","steps":["trace[1676890744] 'agreement among raft nodes before linearized reading' (duration: 6.986891942s)"],"step_count":1}
{"level":"warn","ts":"2024-08-18T19:08:52.581111Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2024-08-18T19:08:45.594168Z","time spent":"6.986940437s","remote":"127.0.0.1:56392","response type":"/etcdserverpb.KV/Range","request count":0,"request size":67,"response count":0,"response size":0,"request content":"key:\"/registry/services/endpoints/kube-system/k8s.io-minikube-hostpath\" "}
2024/08/18 19:08:52 WARNING: [core] [Server #7] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing"
{"level":"warn","ts":"2024-08-18T19:08:52.581159Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"5.633749967s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/endpointslices/\" range_end:\"/registry/endpointslices0\" count_only:true ","response":"","error":"context canceled"}
{"level":"info","ts":"2024-08-18T19:08:52.581170Z","caller":"traceutil/trace.go:171","msg":"trace[1130682409] range","detail":"{range_begin:/registry/endpointslices/; range_end:/registry/endpointslices0; }","duration":"5.633762365s","start":"2024-08-18T19:08:46.947405Z","end":"2024-08-18T19:08:52.581167Z","steps":["trace[1130682409] 'agreement among raft nodes before linearized reading' (duration: 5.633750027s)"],"step_count":1}
{"level":"warn","ts":"2024-08-18T19:08:52.581180Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2024-08-18T19:08:46.947373Z","time spent":"5.633803888s","remote":"127.0.0.1:56504","response type":"/etcdserverpb.KV/Range","request count":0,"request size":56,"response count":0,"response size":0,"request content":"key:\"/registry/endpointslices/\" range_end:\"/registry/endpointslices0\" count_only:true "}
2024/08/18 19:08:52 WARNING: [core] [Server #7] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing"
{"level":"warn","ts":"2024-08-18T19:08:52.581225Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2024-08-18T19:08:52.217567Z","time spent":"363.656855ms","remote":"127.0.0.1:56498","response type":"/etcdserverpb.KV/Txn","request count":0,"request size":0,"response count":0,"response size":0,"request content":""}
2024/08/18 19:08:52 WARNING: [core] [Server #7] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing"
{"level":"warn","ts":"2024-08-18T19:08:52.608176Z","caller":"embed/serve.go:212","msg":"stopping secure grpc server due to error","error":"accept tcp 192.169.0.5:2379: use of closed network connection"}
{"level":"warn","ts":"2024-08-18T19:08:52.608248Z","caller":"embed/serve.go:214","msg":"stopped secure grpc server due to error","error":"accept tcp 192.169.0.5:2379: use of closed network connection"}
{"level":"info","ts":"2024-08-18T19:08:52.608286Z","caller":"etcdserver/server.go:1512","msg":"skipped leadership transfer; local server is not leader","local-member-id":"b8c6c7563d17d844","current-leader-member-id":"0"}
{"level":"info","ts":"2024-08-18T19:08:52.608395Z","caller":"rafthttp/peer.go:330","msg":"stopping remote peer","remote-peer-id":"64535d9f3a4791ce"}
{"level":"info","ts":"2024-08-18T19:08:52.608428Z","caller":"rafthttp/stream.go:294","msg":"stopped TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","remote-peer-id":"64535d9f3a4791ce"}
{"level":"info","ts":"2024-08-18T19:08:52.608446Z","caller":"rafthttp/stream.go:294","msg":"stopped TCP streaming connection with remote peer","stream-writer-type":"stream Message","remote-peer-id":"64535d9f3a4791ce"}
{"level":"info","ts":"2024-08-18T19:08:52.608520Z","caller":"rafthttp/pipeline.go:85","msg":"stopped HTTP pipelining with remote peer","local-member-id":"b8c6c7563d17d844","remote-peer-id":"64535d9f3a4791ce"}
{"level":"info","ts":"2024-08-18T19:08:52.608546Z","caller":"rafthttp/stream.go:442","msg":"stopped stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"b8c6c7563d17d844","remote-peer-id":"64535d9f3a4791ce"}
{"level":"info","ts":"2024-08-18T19:08:52.608595Z","caller":"rafthttp/stream.go:442","msg":"stopped stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"b8c6c7563d17d844","remote-peer-id":"64535d9f3a4791ce"}
{"level":"info","ts":"2024-08-18T19:08:52.608606Z","caller":"rafthttp/peer.go:335","msg":"stopped remote peer","remote-peer-id":"64535d9f3a4791ce"}
{"level":"info","ts":"2024-08-18T19:08:52.610214Z","caller":"embed/etcd.go:581","msg":"stopping serving peer traffic","address":"192.169.0.5:2380"}
{"level":"info","ts":"2024-08-18T19:08:52.610316Z","caller":"embed/etcd.go:586","msg":"stopped serving peer traffic","address":"192.169.0.5:2380"}
{"level":"info","ts":"2024-08-18T19:08:52.610348Z","caller":"embed/etcd.go:379","msg":"closed etcd server","name":"ha-373000","data-dir":"/var/lib/minikube/etcd","advertise-peer-urls":["https://192.169.0.5:2380"],"advertise-client-urls":["https://192.169.0.5:2379"]}
==> kernel <==
19:10:41 up 1 min, 0 users, load average: 0.17, 0.10, 0.04
Linux ha-373000 5.10.207 #1 SMP Thu Aug 15 21:30:57 UTC 2024 x86_64 GNU/Linux
PRETTY_NAME="Buildroot 2023.02.9"
==> kindnet [fbeef7aab770] <==
I0818 19:08:13.321287 1 main.go:322] Node ha-373000-m04 has CIDR [10.244.3.0/24]
I0818 19:08:13.321527 1 main.go:295] Handling node with IPs: map[192.169.0.5:{}]
I0818 19:08:13.321536 1 main.go:299] handling current node
I0818 19:08:13.321545 1 main.go:295] Handling node with IPs: map[192.169.0.6:{}]
I0818 19:08:13.321548 1 main.go:322] Node ha-373000-m02 has CIDR [10.244.1.0/24]
I0818 19:08:23.318236 1 main.go:295] Handling node with IPs: map[192.169.0.6:{}]
I0818 19:08:23.318272 1 main.go:322] Node ha-373000-m02 has CIDR [10.244.1.0/24]
I0818 19:08:23.318358 1 main.go:295] Handling node with IPs: map[192.169.0.7:{}]
I0818 19:08:23.318384 1 main.go:322] Node ha-373000-m03 has CIDR [10.244.2.0/24]
I0818 19:08:23.318431 1 main.go:295] Handling node with IPs: map[192.169.0.8:{}]
I0818 19:08:23.318455 1 main.go:322] Node ha-373000-m04 has CIDR [10.244.3.0/24]
I0818 19:08:23.318492 1 main.go:295] Handling node with IPs: map[192.169.0.5:{}]
I0818 19:08:23.318516 1 main.go:299] handling current node
I0818 19:08:33.318121 1 main.go:295] Handling node with IPs: map[192.169.0.5:{}]
I0818 19:08:33.318160 1 main.go:299] handling current node
I0818 19:08:33.318171 1 main.go:295] Handling node with IPs: map[192.169.0.6:{}]
I0818 19:08:33.318175 1 main.go:322] Node ha-373000-m02 has CIDR [10.244.1.0/24]
I0818 19:08:33.318256 1 main.go:295] Handling node with IPs: map[192.169.0.8:{}]
I0818 19:08:33.318261 1 main.go:322] Node ha-373000-m04 has CIDR [10.244.3.0/24]
I0818 19:08:43.314133 1 main.go:295] Handling node with IPs: map[192.169.0.8:{}]
I0818 19:08:43.314185 1 main.go:322] Node ha-373000-m04 has CIDR [10.244.3.0/24]
I0818 19:08:43.314278 1 main.go:295] Handling node with IPs: map[192.169.0.5:{}]
I0818 19:08:43.314360 1 main.go:299] handling current node
I0818 19:08:43.314444 1 main.go:295] Handling node with IPs: map[192.169.0.6:{}]
I0818 19:08:43.314482 1 main.go:322] Node ha-373000-m02 has CIDR [10.244.1.0/24]
==> kube-apiserver [7e27aa53db96] <==
I0818 19:10:06.959907 1 options.go:228] external host was not specified, using 192.169.0.5
I0818 19:10:06.961347 1 server.go:142] Version: v1.31.0
I0818 19:10:06.961387 1 server.go:144] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
I0818 19:10:07.546161 1 shared_informer.go:313] Waiting for caches to sync for node_authorizer
I0818 19:10:07.549946 1 shared_informer.go:313] Waiting for caches to sync for *generic.policySource[*k8s.io/api/admissionregistration/v1.ValidatingAdmissionPolicy,*k8s.io/api/admissionregistration/v1.ValidatingAdmissionPolicyBinding,k8s.io/apiserver/pkg/admission/plugin/policy/validating.Validator]
I0818 19:10:07.552371 1 plugins.go:157] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
I0818 19:10:07.552381 1 plugins.go:160] Loaded 13 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,PodSecurity,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,ClusterTrustBundleAttest,CertificateSubjectRestriction,ValidatingAdmissionPolicy,ValidatingAdmissionWebhook,ResourceQuota.
I0818 19:10:07.552555 1 instance.go:232] Using reconciler: lease
W0818 19:10:27.545475 1 logging.go:55] [core] [Channel #2 SubChannel #4]grpc: addrConn.createTransport failed to connect to {Addr: "127.0.0.1:2379", ServerName: "127.0.0.1:2379", }. Err: connection error: desc = "transport: authentication handshake failed: context canceled"
W0818 19:10:27.545529 1 logging.go:55] [core] [Channel #1 SubChannel #3]grpc: addrConn.createTransport failed to connect to {Addr: "127.0.0.1:2379", ServerName: "127.0.0.1:2379", }. Err: connection error: desc = "transport: authentication handshake failed: context canceled"
F0818 19:10:27.554420 1 instance.go:225] Error creating leases: error creating storage factory: context deadline exceeded
W0818 19:10:27.554432 1 logging.go:55] [core] [Channel #5 SubChannel #6]grpc: addrConn.createTransport failed to connect to {Addr: "127.0.0.1:2379", ServerName: "127.0.0.1:2379", }. Err: connection error: desc = "transport: authentication handshake failed: context deadline exceeded"
==> kube-controller-manager [24788de6a779] <==
I0818 19:10:05.103965 1 serving.go:386] Generated self-signed cert in-memory
I0818 19:10:05.483625 1 controllermanager.go:197] "Starting" version="v1.31.0"
I0818 19:10:05.483663 1 controllermanager.go:199] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
I0818 19:10:05.484840 1 dynamic_cafile_content.go:160] "Starting controller" name="request-header::/var/lib/minikube/certs/front-proxy-ca.crt"
I0818 19:10:05.484954 1 tlsconfig.go:243] "Starting DynamicServingCertificateController"
I0818 19:10:05.484863 1 secure_serving.go:213] Serving securely on 127.0.0.1:10257
I0818 19:10:05.485038 1 dynamic_cafile_content.go:160] "Starting controller" name="client-ca-bundle::/var/lib/minikube/certs/ca.crt"
E0818 19:10:27.488487 1 controllermanager.go:242] "Error building controller context" err="failed to wait for apiserver being healthy: timed out waiting for the condition: failed to get apiserver /healthz status: Get \"https://192.169.0.5:8443/healthz\": net/http: TLS handshake timeout"
==> kube-proxy [530d58000189] <==
add table ip kube-proxy
^^^^^^^^^^^^^^^^^^^^^^^^
>
E0818 19:05:43.260298 1 proxier.go:734] "Error cleaning up nftables rules" err=<
could not run nftables command: /dev/stdin:1:1-25: Error: Could not process rule: Operation not supported
add table ip6 kube-proxy
^^^^^^^^^^^^^^^^^^^^^^^^^
>
I0818 19:05:43.283054 1 server.go:677] "Successfully retrieved node IP(s)" IPs=["192.169.0.5"]
E0818 19:05:43.283201 1 server.go:234] "Kube-proxy configuration may be incomplete or incorrect" err="nodePortAddresses is unset; NodePort connections will be accepted on all local IPs. Consider using `--nodeport-addresses primary`"
I0818 19:05:43.332462 1 server_linux.go:146] "No iptables support for family" ipFamily="IPv6"
I0818 19:05:43.332509 1 server.go:245] "kube-proxy running in single-stack mode" ipFamily="IPv4"
I0818 19:05:43.332527 1 server_linux.go:169] "Using iptables Proxier"
I0818 19:05:43.335382 1 proxier.go:255] "Setting route_localnet=1 to allow node-ports on localhost; to change this either disable iptables.localhostNodePorts (--iptables-localhost-nodeports) or set nodePortAddresses (--nodeport-addresses) to filter loopback addresses" ipFamily="IPv4"
I0818 19:05:43.336178 1 server.go:483] "Version info" version="v1.31.0"
I0818 19:05:43.336209 1 server.go:485] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
I0818 19:05:43.339664 1 config.go:197] "Starting service config controller"
I0818 19:05:43.340475 1 shared_informer.go:313] Waiting for caches to sync for service config
I0818 19:05:43.340854 1 config.go:104] "Starting endpoint slice config controller"
I0818 19:05:43.340884 1 shared_informer.go:313] Waiting for caches to sync for endpoint slice config
I0818 19:05:43.342595 1 config.go:326] "Starting node config controller"
I0818 19:05:43.342621 1 shared_informer.go:313] Waiting for caches to sync for node config
I0818 19:05:43.440978 1 shared_informer.go:320] Caches are synced for endpoint slice config
I0818 19:05:43.441099 1 shared_informer.go:320] Caches are synced for service config
I0818 19:05:43.442676 1 shared_informer.go:320] Caches are synced for node config
==> kube-scheduler [5bb7217cec87] <==
E0818 19:10:28.561278 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.ReplicationController: failed to list *v1.ReplicationController: Get \"https://192.169.0.5:8443/api/v1/replicationcontrollers?limit=500&resourceVersion=0\": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:48758->192.169.0.5:8443: read: connection reset by peer" logger="UnhandledError"
W0818 19:10:28.561149 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.PersistentVolume: Get "https://192.169.0.5:8443/api/v1/persistentvolumes?limit=500&resourceVersion=0": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:48800->192.169.0.5:8443: read: connection reset by peer
E0818 19:10:28.561322 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.PersistentVolume: failed to list *v1.PersistentVolume: Get \"https://192.169.0.5:8443/api/v1/persistentvolumes?limit=500&resourceVersion=0\": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:48800->192.169.0.5:8443: read: connection reset by peer" logger="UnhandledError"
W0818 19:10:28.561446 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.Pod: Get "https://192.169.0.5:8443/api/v1/pods?fieldSelector=status.phase%21%3DSucceeded%2Cstatus.phase%21%3DFailed&limit=500&resourceVersion=0": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:48768->192.169.0.5:8443: read: connection reset by peer
E0818 19:10:28.561856 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.Pod: failed to list *v1.Pod: Get \"https://192.169.0.5:8443/api/v1/pods?fieldSelector=status.phase%21%3DSucceeded%2Cstatus.phase%21%3DFailed&limit=500&resourceVersion=0\": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:48768->192.169.0.5:8443: read: connection reset by peer" logger="UnhandledError"
W0818 19:10:28.562037 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.StatefulSet: Get "https://192.169.0.5:8443/apis/apps/v1/statefulsets?limit=500&resourceVersion=0": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:48784->192.169.0.5:8443: read: connection reset by peer
E0818 19:10:28.562108 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.StatefulSet: failed to list *v1.StatefulSet: Get \"https://192.169.0.5:8443/apis/apps/v1/statefulsets?limit=500&resourceVersion=0\": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:48784->192.169.0.5:8443: read: connection reset by peer" logger="UnhandledError"
W0818 19:10:28.562348 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.PersistentVolumeClaim: Get "https://192.169.0.5:8443/api/v1/persistentvolumeclaims?limit=500&resourceVersion=0": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:55098->192.169.0.5:8443: read: connection reset by peer
W0818 19:10:28.562424 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.ReplicaSet: Get "https://192.169.0.5:8443/apis/apps/v1/replicasets?limit=500&resourceVersion=0": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:48820->192.169.0.5:8443: read: connection reset by peer
E0818 19:10:28.562538 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.ReplicaSet: failed to list *v1.ReplicaSet: Get \"https://192.169.0.5:8443/apis/apps/v1/replicasets?limit=500&resourceVersion=0\": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:48820->192.169.0.5:8443: read: connection reset by peer" logger="UnhandledError"
E0818 19:10:28.562490 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.PersistentVolumeClaim: failed to list *v1.PersistentVolumeClaim: Get \"https://192.169.0.5:8443/api/v1/persistentvolumeclaims?limit=500&resourceVersion=0\": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:55098->192.169.0.5:8443: read: connection reset by peer" logger="UnhandledError"
W0818 19:10:28.562730 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.CSINode: Get "https://192.169.0.5:8443/apis/storage.k8s.io/v1/csinodes?limit=500&resourceVersion=0": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:48804->192.169.0.5:8443: read: connection reset by peer
E0818 19:10:28.563108 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.CSINode: failed to list *v1.CSINode: Get \"https://192.169.0.5:8443/apis/storage.k8s.io/v1/csinodes?limit=500&resourceVersion=0\": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:48804->192.169.0.5:8443: read: connection reset by peer" logger="UnhandledError"
W0818 19:10:28.562973 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.CSIDriver: Get "https://192.169.0.5:8443/apis/storage.k8s.io/v1/csidrivers?limit=500&resourceVersion=0": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:48786->192.169.0.5:8443: read: connection reset by peer
E0818 19:10:28.563171 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.CSIDriver: failed to list *v1.CSIDriver: Get \"https://192.169.0.5:8443/apis/storage.k8s.io/v1/csidrivers?limit=500&resourceVersion=0\": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:48786->192.169.0.5:8443: read: connection reset by peer" logger="UnhandledError"
W0818 19:10:28.563330 1 reflector.go:561] runtime/asm_amd64.s:1695: failed to list *v1.ConfigMap: Get "https://192.169.0.5:8443/api/v1/namespaces/kube-system/configmaps?fieldSelector=metadata.name%3Dextension-apiserver-authentication&limit=500&resourceVersion=0": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:48824->192.169.0.5:8443: read: connection reset by peer
E0818 19:10:28.563535 1 reflector.go:158] "Unhandled Error" err="runtime/asm_amd64.s:1695: Failed to watch *v1.ConfigMap: failed to list *v1.ConfigMap: Get \"https://192.169.0.5:8443/api/v1/namespaces/kube-system/configmaps?fieldSelector=metadata.name%3Dextension-apiserver-authentication&limit=500&resourceVersion=0\": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:48824->192.169.0.5:8443: read: connection reset by peer" logger="UnhandledError"
W0818 19:10:28.563376 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.Node: Get "https://192.169.0.5:8443/api/v1/nodes?limit=500&resourceVersion=0": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:48782->192.169.0.5:8443: read: connection reset by peer
E0818 19:10:28.563736 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.Node: failed to list *v1.Node: Get \"https://192.169.0.5:8443/api/v1/nodes?limit=500&resourceVersion=0\": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:48782->192.169.0.5:8443: read: connection reset by peer" logger="UnhandledError"
W0818 19:10:28.563801 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.Service: Get "https://192.169.0.5:8443/api/v1/services?limit=500&resourceVersion=0": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:48790->192.169.0.5:8443: read: connection reset by peer
E0818 19:10:28.563955 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.Service: failed to list *v1.Service: Get \"https://192.169.0.5:8443/api/v1/services?limit=500&resourceVersion=0\": dial tcp 192.169.0.5:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.5:48790->192.169.0.5:8443: read: connection reset by peer" logger="UnhandledError"
W0818 19:10:28.680351 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.PodDisruptionBudget: Get "https://192.169.0.5:8443/apis/policy/v1/poddisruptionbudgets?limit=500&resourceVersion=0": dial tcp 192.169.0.5:8443: connect: connection refused
E0818 19:10:28.680700 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.PodDisruptionBudget: failed to list *v1.PodDisruptionBudget: Get \"https://192.169.0.5:8443/apis/policy/v1/poddisruptionbudgets?limit=500&resourceVersion=0\": dial tcp 192.169.0.5:8443: connect: connection refused" logger="UnhandledError"
W0818 19:10:29.353980 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.Namespace: Get "https://192.169.0.5:8443/api/v1/namespaces?limit=500&resourceVersion=0": dial tcp 192.169.0.5:8443: connect: connection refused
E0818 19:10:29.354184 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.Namespace: failed to list *v1.Namespace: Get \"https://192.169.0.5:8443/api/v1/namespaces?limit=500&resourceVersion=0\": dial tcp 192.169.0.5:8443: connect: connection refused" logger="UnhandledError"
==> kube-scheduler [de016fdbd6fe] <==
I0818 19:04:58.645297 1 serving.go:386] Generated self-signed cert in-memory
W0818 19:05:08.939365 1 authentication.go:370] Error looking up in-cluster authentication configuration: Get "https://192.169.0.5:8443/api/v1/namespaces/kube-system/configmaps/extension-apiserver-authentication": net/http: TLS handshake timeout
W0818 19:05:08.939390 1 authentication.go:371] Continuing without authentication configuration. This may treat all requests as anonymous.
W0818 19:05:08.939395 1 authentication.go:372] To require authentication configuration lookup to succeed, set --authentication-tolerate-lookup-failure=false
I0818 19:05:17.672661 1 server.go:167] "Starting Kubernetes Scheduler" version="v1.31.0"
I0818 19:05:17.674961 1 server.go:169] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
I0818 19:05:17.680297 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file"
I0818 19:05:17.680709 1 secure_serving.go:213] Serving securely on 127.0.0.1:10259
I0818 19:05:17.683175 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
I0818 19:05:17.689784 1 tlsconfig.go:243] "Starting DynamicServingCertificateController"
I0818 19:05:17.786103 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
I0818 19:08:52.663744 1 secure_serving.go:258] Stopped listening on 127.0.0.1:10259
I0818 19:08:52.664520 1 tlsconfig.go:258] "Shutting down DynamicServingCertificateController"
I0818 19:08:52.664805 1 configmap_cafile_content.go:226] "Shutting down controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file"
E0818 19:08:52.665618 1 run.go:72] "command failed" err="finished without leader elect"
==> kubelet <==
Aug 18 19:10:24 ha-373000 kubelet[1587]: I0818 19:10:24.149998 1587 kubelet_node_status.go:72] "Attempting to register node" node="ha-373000"
Aug 18 19:10:26 ha-373000 kubelet[1587]: E0818 19:10:26.363782 1587 kubelet_node_status.go:95] "Unable to register node with API server" err="Post \"https://control-plane.minikube.internal:8443/api/v1/nodes\": dial tcp 192.169.0.254:8443: connect: no route to host" node="ha-373000"
Aug 18 19:10:26 ha-373000 kubelet[1587]: E0818 19:10:26.364415 1587 controller.go:145] "Failed to ensure lease exists, will retry" err="Get \"https://control-plane.minikube.internal:8443/apis/coordination.k8s.io/v1/namespaces/kube-node-lease/leases/ha-373000?timeout=10s\": dial tcp 192.169.0.254:8443: connect: no route to host" interval="7s"
Aug 18 19:10:27 ha-373000 kubelet[1587]: I0818 19:10:27.784218 1587 scope.go:117] "RemoveContainer" containerID="6f6bc1d9592b465e3b7c4d9db0e74b67040cbbed37626775b4aad68af80ddeb2"
Aug 18 19:10:27 ha-373000 kubelet[1587]: I0818 19:10:27.785184 1587 scope.go:117] "RemoveContainer" containerID="7e27aa53db964848f64d9282a63325b77e8066d4f7afd7e078ca49d74f625756"
Aug 18 19:10:27 ha-373000 kubelet[1587]: E0818 19:10:27.785346 1587 pod_workers.go:1301] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"kube-apiserver\" with CrashLoopBackOff: \"back-off 20s restarting failed container=kube-apiserver pod=kube-apiserver-ha-373000_kube-system(be3d0c0dc33917295e5fed284f29b0d0)\"" pod="kube-system/kube-apiserver-ha-373000" podUID="be3d0c0dc33917295e5fed284f29b0d0"
Aug 18 19:10:27 ha-373000 kubelet[1587]: I0818 19:10:27.792994 1587 scope.go:117] "RemoveContainer" containerID="24788de6a779b6b3b57b2e2becb6248a4b461c40237836c9a085858f19bf537e"
Aug 18 19:10:27 ha-373000 kubelet[1587]: E0818 19:10:27.793092 1587 pod_workers.go:1301] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"kube-controller-manager\" with CrashLoopBackOff: \"back-off 20s restarting failed container=kube-controller-manager pod=kube-controller-manager-ha-373000_kube-system(ed589efba4f3e45b48b475449dc23ab8)\"" pod="kube-system/kube-controller-manager-ha-373000" podUID="ed589efba4f3e45b48b475449dc23ab8"
Aug 18 19:10:27 ha-373000 kubelet[1587]: I0818 19:10:27.797222 1587 scope.go:117] "RemoveContainer" containerID="0a3644cb0db7c69e66c1b1d0f3234fba20e233a15e27e5772148d80d14f33f4e"
Aug 18 19:10:28 ha-373000 kubelet[1587]: E0818 19:10:28.895968 1587 eviction_manager.go:285] "Eviction manager: failed to get summary stats" err="failed to get node info: node \"ha-373000\" not found"
Aug 18 19:10:29 ha-373000 kubelet[1587]: I0818 19:10:29.422762 1587 scope.go:117] "RemoveContainer" containerID="24788de6a779b6b3b57b2e2becb6248a4b461c40237836c9a085858f19bf537e"
Aug 18 19:10:29 ha-373000 kubelet[1587]: E0818 19:10:29.423034 1587 pod_workers.go:1301] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"kube-controller-manager\" with CrashLoopBackOff: \"back-off 20s restarting failed container=kube-controller-manager pod=kube-controller-manager-ha-373000_kube-system(ed589efba4f3e45b48b475449dc23ab8)\"" pod="kube-system/kube-controller-manager-ha-373000" podUID="ed589efba4f3e45b48b475449dc23ab8"
Aug 18 19:10:32 ha-373000 kubelet[1587]: E0818 19:10:32.507755 1587 event.go:368] "Unable to write event (may retry after sleeping)" err="Post \"https://control-plane.minikube.internal:8443/api/v1/namespaces/default/events\": dial tcp 192.169.0.254:8443: connect: no route to host" event="&Event{ObjectMeta:{ha-373000.17ece84946cc9aa1 default 0 0001-01-01 00:00:00 +0000 UTC <nil> <nil> map[] map[] [] [] []},InvolvedObject:ObjectReference{Kind:Node,Namespace:,Name:ha-373000,UID:ha-373000,APIVersion:,ResourceVersion:,FieldPath:,},Reason:Starting,Message:Starting kubelet.,Source:EventSource{Component:kubelet,Host:ha-373000,},FirstTimestamp:2024-08-18 19:09:18.794128033 +0000 UTC m=+0.098935686,LastTimestamp:2024-08-18 19:09:18.794128033 +0000 UTC m=+0.098935686,Count:1,Type:Normal,EventTime:0001-01-01 00:00:00 +0000 UTC,Series:nil,Action:,Related:nil,ReportingController:kubelet,ReportingInstance:ha-373000,}"
Aug 18 19:10:33 ha-373000 kubelet[1587]: I0818 19:10:33.051084 1587 scope.go:117] "RemoveContainer" containerID="24788de6a779b6b3b57b2e2becb6248a4b461c40237836c9a085858f19bf537e"
Aug 18 19:10:33 ha-373000 kubelet[1587]: E0818 19:10:33.051343 1587 pod_workers.go:1301] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"kube-controller-manager\" with CrashLoopBackOff: \"back-off 20s restarting failed container=kube-controller-manager pod=kube-controller-manager-ha-373000_kube-system(ed589efba4f3e45b48b475449dc23ab8)\"" pod="kube-system/kube-controller-manager-ha-373000" podUID="ed589efba4f3e45b48b475449dc23ab8"
Aug 18 19:10:33 ha-373000 kubelet[1587]: I0818 19:10:33.366165 1587 kubelet_node_status.go:72] "Attempting to register node" node="ha-373000"
Aug 18 19:10:34 ha-373000 kubelet[1587]: I0818 19:10:34.986485 1587 scope.go:117] "RemoveContainer" containerID="7e27aa53db964848f64d9282a63325b77e8066d4f7afd7e078ca49d74f625756"
Aug 18 19:10:34 ha-373000 kubelet[1587]: E0818 19:10:34.987184 1587 pod_workers.go:1301] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"kube-apiserver\" with CrashLoopBackOff: \"back-off 20s restarting failed container=kube-apiserver pod=kube-apiserver-ha-373000_kube-system(be3d0c0dc33917295e5fed284f29b0d0)\"" pod="kube-system/kube-apiserver-ha-373000" podUID="be3d0c0dc33917295e5fed284f29b0d0"
Aug 18 19:10:35 ha-373000 kubelet[1587]: E0818 19:10:35.579478 1587 kubelet_node_status.go:95] "Unable to register node with API server" err="Post \"https://control-plane.minikube.internal:8443/api/v1/nodes\": dial tcp 192.169.0.254:8443: connect: no route to host" node="ha-373000"
Aug 18 19:10:35 ha-373000 kubelet[1587]: E0818 19:10:35.579545 1587 controller.go:145] "Failed to ensure lease exists, will retry" err="Get \"https://control-plane.minikube.internal:8443/apis/coordination.k8s.io/v1/namespaces/kube-node-lease/leases/ha-373000?timeout=10s\": dial tcp 192.169.0.254:8443: connect: no route to host" interval="7s"
Aug 18 19:10:35 ha-373000 kubelet[1587]: I0818 19:10:35.861801 1587 scope.go:117] "RemoveContainer" containerID="7e27aa53db964848f64d9282a63325b77e8066d4f7afd7e078ca49d74f625756"
Aug 18 19:10:35 ha-373000 kubelet[1587]: E0818 19:10:35.861956 1587 pod_workers.go:1301] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"kube-apiserver\" with CrashLoopBackOff: \"back-off 20s restarting failed container=kube-apiserver pod=kube-apiserver-ha-373000_kube-system(be3d0c0dc33917295e5fed284f29b0d0)\"" pod="kube-system/kube-apiserver-ha-373000" podUID="be3d0c0dc33917295e5fed284f29b0d0"
Aug 18 19:10:38 ha-373000 kubelet[1587]: W0818 19:10:38.650520 1587 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.RuntimeClass: Get "https://control-plane.minikube.internal:8443/apis/node.k8s.io/v1/runtimeclasses?limit=500&resourceVersion=0": dial tcp 192.169.0.254:8443: connect: no route to host
Aug 18 19:10:38 ha-373000 kubelet[1587]: E0818 19:10:38.650612 1587 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.RuntimeClass: failed to list *v1.RuntimeClass: Get \"https://control-plane.minikube.internal:8443/apis/node.k8s.io/v1/runtimeclasses?limit=500&resourceVersion=0\": dial tcp 192.169.0.254:8443: connect: no route to host" logger="UnhandledError"
Aug 18 19:10:38 ha-373000 kubelet[1587]: E0818 19:10:38.896386 1587 eviction_manager.go:285] "Eviction manager: failed to get summary stats" err="failed to get node info: node \"ha-373000\" not found"
-- /stdout --
helpers_test.go:254: (dbg) Run: out/minikube-darwin-amd64 status --format={{.APIServer}} -p ha-373000 -n ha-373000
helpers_test.go:254: (dbg) Non-zero exit: out/minikube-darwin-amd64 status --format={{.APIServer}} -p ha-373000 -n ha-373000: exit status 2 (147.447933ms)
-- stdout --
Stopped
-- /stdout --
helpers_test.go:254: status error: exit status 2 (may be ok)
helpers_test.go:256: "ha-373000" apiserver is not running, skipping kubectl commands (state="Stopped")
--- FAIL: TestMultiControlPlane/serial/DegradedAfterClusterRestart (2.66s)