=== RUN TestMultiControlPlane/serial/CopyFile
ha_test.go:326: (dbg) Run: out/minikube-darwin-amd64 -p ha-184000 status --output json -v=7 --alsologtostderr
E0703 16:34:37.201414 7038 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18859-6498/.minikube/profiles/addons-198000/client.crt: no such file or directory
E0703 16:36:00.256935 7038 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18859-6498/.minikube/profiles/addons-198000/client.crt: no such file or directory
E0703 16:37:52.109366 7038 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18859-6498/.minikube/profiles/functional-957000/client.crt: no such file or directory
ha_test.go:326: (dbg) Non-zero exit: out/minikube-darwin-amd64 -p ha-184000 status --output json -v=7 --alsologtostderr: exit status 3 (5m0.191163028s)
-- stdout --
[{"Name":"ha-184000","Host":"Error","Kubelet":"Nonexistent","APIServer":"Nonexistent","Kubeconfig":"Configured","Worker":false},{"Name":"ha-184000-m02","Host":"Error","Kubelet":"Nonexistent","APIServer":"Nonexistent","Kubeconfig":"Configured","Worker":false},{"Name":"ha-184000-m03","Host":"Error","Kubelet":"Nonexistent","APIServer":"Nonexistent","Kubeconfig":"Configured","Worker":false},{"Name":"ha-184000-m04","Host":"Error","Kubelet":"Nonexistent","APIServer":"Irrelevant","Kubeconfig":"Irrelevant","Worker":true}]
-- /stdout --
** stderr **
I0703 16:33:54.270110 8950 out.go:291] Setting OutFile to fd 1 ...
I0703 16:33:54.270327 8950 out.go:338] TERM=,COLORTERM=, which probably does not support color
I0703 16:33:54.270333 8950 out.go:304] Setting ErrFile to fd 2...
I0703 16:33:54.270336 8950 out.go:338] TERM=,COLORTERM=, which probably does not support color
I0703 16:33:54.270523 8950 root.go:338] Updating PATH: /Users/jenkins/minikube-integration/18859-6498/.minikube/bin
I0703 16:33:54.270714 8950 out.go:298] Setting JSON to true
I0703 16:33:54.270742 8950 mustload.go:65] Loading cluster: ha-184000
I0703 16:33:54.270774 8950 notify.go:220] Checking for updates...
I0703 16:33:54.271068 8950 config.go:182] Loaded profile config "ha-184000": Driver=hyperkit, ContainerRuntime=docker, KubernetesVersion=v1.30.2
I0703 16:33:54.271089 8950 status.go:255] checking status of ha-184000 ...
I0703 16:33:54.271466 8950 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0703 16:33:54.271530 8950 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0703 16:33:54.280349 8950 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:53973
I0703 16:33:54.280756 8950 main.go:141] libmachine: () Calling .GetVersion
I0703 16:33:54.281179 8950 main.go:141] libmachine: Using API Version 1
I0703 16:33:54.281220 8950 main.go:141] libmachine: () Calling .SetConfigRaw
I0703 16:33:54.281413 8950 main.go:141] libmachine: () Calling .GetMachineName
I0703 16:33:54.281517 8950 main.go:141] libmachine: (ha-184000) Calling .GetState
I0703 16:33:54.281596 8950 main.go:141] libmachine: (ha-184000) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I0703 16:33:54.281676 8950 main.go:141] libmachine: (ha-184000) DBG | hyperkit pid from json: 8581
I0703 16:33:54.282683 8950 status.go:330] ha-184000 host status = "Running" (err=<nil>)
I0703 16:33:54.282706 8950 host.go:66] Checking if "ha-184000" exists ...
I0703 16:33:54.282947 8950 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0703 16:33:54.282966 8950 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0703 16:33:54.291486 8950 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:53975
I0703 16:33:54.291814 8950 main.go:141] libmachine: () Calling .GetVersion
I0703 16:33:54.292135 8950 main.go:141] libmachine: Using API Version 1
I0703 16:33:54.292154 8950 main.go:141] libmachine: () Calling .SetConfigRaw
I0703 16:33:54.292357 8950 main.go:141] libmachine: () Calling .GetMachineName
I0703 16:33:54.292464 8950 main.go:141] libmachine: (ha-184000) Calling .GetIP
I0703 16:33:54.292551 8950 host.go:66] Checking if "ha-184000" exists ...
I0703 16:33:54.292806 8950 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0703 16:33:54.292827 8950 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0703 16:33:54.301665 8950 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:53977
I0703 16:33:54.302013 8950 main.go:141] libmachine: () Calling .GetVersion
I0703 16:33:54.302352 8950 main.go:141] libmachine: Using API Version 1
I0703 16:33:54.302364 8950 main.go:141] libmachine: () Calling .SetConfigRaw
I0703 16:33:54.302553 8950 main.go:141] libmachine: () Calling .GetMachineName
I0703 16:33:54.302652 8950 main.go:141] libmachine: (ha-184000) Calling .DriverName
I0703 16:33:54.302791 8950 ssh_runner.go:195] Run: sh -c "df -h /var | awk 'NR==2{print $5}'"
I0703 16:33:54.302813 8950 main.go:141] libmachine: (ha-184000) Calling .GetSSHHostname
I0703 16:33:54.302900 8950 main.go:141] libmachine: (ha-184000) Calling .GetSSHPort
I0703 16:33:54.302978 8950 main.go:141] libmachine: (ha-184000) Calling .GetSSHKeyPath
I0703 16:33:54.303068 8950 main.go:141] libmachine: (ha-184000) Calling .GetSSHUsername
I0703 16:33:54.303145 8950 sshutil.go:53] new ssh client: &{IP:192.169.0.7 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/18859-6498/.minikube/machines/ha-184000/id_rsa Username:docker}
W0703 16:35:09.304495 8950 sshutil.go:64] dial failure (will retry): dial tcp 192.169.0.7:22: connect: operation timed out
W0703 16:35:09.304608 8950 start.go:268] error running df -h /var: NewSession: new client: new client: dial tcp 192.169.0.7:22: connect: operation timed out
E0703 16:35:09.304646 8950 status.go:376] failed to get storage capacity of /var: NewSession: new client: new client: dial tcp 192.169.0.7:22: connect: operation timed out
I0703 16:35:09.304667 8950 status.go:257] ha-184000 status: &{Name:ha-184000 Host:Error Kubelet:Nonexistent APIServer:Nonexistent Kubeconfig:Configured Worker:false TimeToStop: DockerEnv: PodManEnv:}
E0703 16:35:09.304689 8950 status.go:260] status error: NewSession: new client: new client: dial tcp 192.169.0.7:22: connect: operation timed out
I0703 16:35:09.304701 8950 status.go:255] checking status of ha-184000-m02 ...
I0703 16:35:09.305198 8950 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0703 16:35:09.305237 8950 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0703 16:35:09.315312 8950 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:53980
I0703 16:35:09.315666 8950 main.go:141] libmachine: () Calling .GetVersion
I0703 16:35:09.315993 8950 main.go:141] libmachine: Using API Version 1
I0703 16:35:09.316005 8950 main.go:141] libmachine: () Calling .SetConfigRaw
I0703 16:35:09.316207 8950 main.go:141] libmachine: () Calling .GetMachineName
I0703 16:35:09.316320 8950 main.go:141] libmachine: (ha-184000-m02) Calling .GetState
I0703 16:35:09.316403 8950 main.go:141] libmachine: (ha-184000-m02) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I0703 16:35:09.316488 8950 main.go:141] libmachine: (ha-184000-m02) DBG | hyperkit pid from json: 8608
I0703 16:35:09.317494 8950 status.go:330] ha-184000-m02 host status = "Running" (err=<nil>)
I0703 16:35:09.317505 8950 host.go:66] Checking if "ha-184000-m02" exists ...
I0703 16:35:09.317780 8950 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0703 16:35:09.317818 8950 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0703 16:35:09.326795 8950 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:53982
I0703 16:35:09.327147 8950 main.go:141] libmachine: () Calling .GetVersion
I0703 16:35:09.327494 8950 main.go:141] libmachine: Using API Version 1
I0703 16:35:09.327513 8950 main.go:141] libmachine: () Calling .SetConfigRaw
I0703 16:35:09.327724 8950 main.go:141] libmachine: () Calling .GetMachineName
I0703 16:35:09.327847 8950 main.go:141] libmachine: (ha-184000-m02) Calling .GetIP
I0703 16:35:09.327935 8950 host.go:66] Checking if "ha-184000-m02" exists ...
I0703 16:35:09.328191 8950 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0703 16:35:09.328219 8950 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0703 16:35:09.336948 8950 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:53984
I0703 16:35:09.337264 8950 main.go:141] libmachine: () Calling .GetVersion
I0703 16:35:09.337587 8950 main.go:141] libmachine: Using API Version 1
I0703 16:35:09.337598 8950 main.go:141] libmachine: () Calling .SetConfigRaw
I0703 16:35:09.337814 8950 main.go:141] libmachine: () Calling .GetMachineName
I0703 16:35:09.337927 8950 main.go:141] libmachine: (ha-184000-m02) Calling .DriverName
I0703 16:35:09.338055 8950 ssh_runner.go:195] Run: sh -c "df -h /var | awk 'NR==2{print $5}'"
I0703 16:35:09.338066 8950 main.go:141] libmachine: (ha-184000-m02) Calling .GetSSHHostname
I0703 16:35:09.338140 8950 main.go:141] libmachine: (ha-184000-m02) Calling .GetSSHPort
I0703 16:35:09.338214 8950 main.go:141] libmachine: (ha-184000-m02) Calling .GetSSHKeyPath
I0703 16:35:09.338300 8950 main.go:141] libmachine: (ha-184000-m02) Calling .GetSSHUsername
I0703 16:35:09.338400 8950 sshutil.go:53] new ssh client: &{IP:192.169.0.8 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/18859-6498/.minikube/machines/ha-184000-m02/id_rsa Username:docker}
W0703 16:36:24.337142 8950 sshutil.go:64] dial failure (will retry): dial tcp 192.169.0.8:22: connect: operation timed out
W0703 16:36:24.337216 8950 start.go:268] error running df -h /var: NewSession: new client: new client: dial tcp 192.169.0.8:22: connect: operation timed out
E0703 16:36:24.337233 8950 status.go:376] failed to get storage capacity of /var: NewSession: new client: new client: dial tcp 192.169.0.8:22: connect: operation timed out
I0703 16:36:24.337252 8950 status.go:257] ha-184000-m02 status: &{Name:ha-184000-m02 Host:Error Kubelet:Nonexistent APIServer:Nonexistent Kubeconfig:Configured Worker:false TimeToStop: DockerEnv: PodManEnv:}
E0703 16:36:24.337265 8950 status.go:260] status error: NewSession: new client: new client: dial tcp 192.169.0.8:22: connect: operation timed out
I0703 16:36:24.337272 8950 status.go:255] checking status of ha-184000-m03 ...
I0703 16:36:24.337656 8950 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0703 16:36:24.337688 8950 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0703 16:36:24.346920 8950 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:53987
I0703 16:36:24.347287 8950 main.go:141] libmachine: () Calling .GetVersion
I0703 16:36:24.347714 8950 main.go:141] libmachine: Using API Version 1
I0703 16:36:24.347737 8950 main.go:141] libmachine: () Calling .SetConfigRaw
I0703 16:36:24.347947 8950 main.go:141] libmachine: () Calling .GetMachineName
I0703 16:36:24.348049 8950 main.go:141] libmachine: (ha-184000-m03) Calling .GetState
I0703 16:36:24.348123 8950 main.go:141] libmachine: (ha-184000-m03) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I0703 16:36:24.348225 8950 main.go:141] libmachine: (ha-184000-m03) DBG | hyperkit pid from json: 8676
I0703 16:36:24.349246 8950 status.go:330] ha-184000-m03 host status = "Running" (err=<nil>)
I0703 16:36:24.349255 8950 host.go:66] Checking if "ha-184000-m03" exists ...
I0703 16:36:24.349497 8950 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0703 16:36:24.349524 8950 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0703 16:36:24.357987 8950 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:53989
I0703 16:36:24.358297 8950 main.go:141] libmachine: () Calling .GetVersion
I0703 16:36:24.358626 8950 main.go:141] libmachine: Using API Version 1
I0703 16:36:24.358643 8950 main.go:141] libmachine: () Calling .SetConfigRaw
I0703 16:36:24.358879 8950 main.go:141] libmachine: () Calling .GetMachineName
I0703 16:36:24.358993 8950 main.go:141] libmachine: (ha-184000-m03) Calling .GetIP
I0703 16:36:24.359075 8950 host.go:66] Checking if "ha-184000-m03" exists ...
I0703 16:36:24.359351 8950 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0703 16:36:24.359376 8950 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0703 16:36:24.368192 8950 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:53991
I0703 16:36:24.368616 8950 main.go:141] libmachine: () Calling .GetVersion
I0703 16:36:24.368970 8950 main.go:141] libmachine: Using API Version 1
I0703 16:36:24.368987 8950 main.go:141] libmachine: () Calling .SetConfigRaw
I0703 16:36:24.369215 8950 main.go:141] libmachine: () Calling .GetMachineName
I0703 16:36:24.369328 8950 main.go:141] libmachine: (ha-184000-m03) Calling .DriverName
I0703 16:36:24.369474 8950 ssh_runner.go:195] Run: sh -c "df -h /var | awk 'NR==2{print $5}'"
I0703 16:36:24.369487 8950 main.go:141] libmachine: (ha-184000-m03) Calling .GetSSHHostname
I0703 16:36:24.369571 8950 main.go:141] libmachine: (ha-184000-m03) Calling .GetSSHPort
I0703 16:36:24.369669 8950 main.go:141] libmachine: (ha-184000-m03) Calling .GetSSHKeyPath
I0703 16:36:24.369758 8950 main.go:141] libmachine: (ha-184000-m03) Calling .GetSSHUsername
I0703 16:36:24.369864 8950 sshutil.go:53] new ssh client: &{IP:192.169.0.9 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/18859-6498/.minikube/machines/ha-184000-m03/id_rsa Username:docker}
W0703 16:37:39.369683 8950 sshutil.go:64] dial failure (will retry): dial tcp 192.169.0.9:22: connect: operation timed out
W0703 16:37:39.369778 8950 start.go:268] error running df -h /var: NewSession: new client: new client: dial tcp 192.169.0.9:22: connect: operation timed out
E0703 16:37:39.369806 8950 status.go:376] failed to get storage capacity of /var: NewSession: new client: new client: dial tcp 192.169.0.9:22: connect: operation timed out
I0703 16:37:39.369821 8950 status.go:257] ha-184000-m03 status: &{Name:ha-184000-m03 Host:Error Kubelet:Nonexistent APIServer:Nonexistent Kubeconfig:Configured Worker:false TimeToStop: DockerEnv: PodManEnv:}
E0703 16:37:39.369846 8950 status.go:260] status error: NewSession: new client: new client: dial tcp 192.169.0.9:22: connect: operation timed out
I0703 16:37:39.369857 8950 status.go:255] checking status of ha-184000-m04 ...
I0703 16:37:39.370417 8950 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0703 16:37:39.370603 8950 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0703 16:37:39.380182 8950 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:53994
I0703 16:37:39.380504 8950 main.go:141] libmachine: () Calling .GetVersion
I0703 16:37:39.380893 8950 main.go:141] libmachine: Using API Version 1
I0703 16:37:39.380914 8950 main.go:141] libmachine: () Calling .SetConfigRaw
I0703 16:37:39.381150 8950 main.go:141] libmachine: () Calling .GetMachineName
I0703 16:37:39.381274 8950 main.go:141] libmachine: (ha-184000-m04) Calling .GetState
I0703 16:37:39.381358 8950 main.go:141] libmachine: (ha-184000-m04) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I0703 16:37:39.381459 8950 main.go:141] libmachine: (ha-184000-m04) DBG | hyperkit pid from json: 8797
I0703 16:37:39.382451 8950 status.go:330] ha-184000-m04 host status = "Running" (err=<nil>)
I0703 16:37:39.382461 8950 host.go:66] Checking if "ha-184000-m04" exists ...
I0703 16:37:39.382713 8950 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0703 16:37:39.382738 8950 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0703 16:37:39.391334 8950 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:53996
I0703 16:37:39.391696 8950 main.go:141] libmachine: () Calling .GetVersion
I0703 16:37:39.392022 8950 main.go:141] libmachine: Using API Version 1
I0703 16:37:39.392036 8950 main.go:141] libmachine: () Calling .SetConfigRaw
I0703 16:37:39.392226 8950 main.go:141] libmachine: () Calling .GetMachineName
I0703 16:37:39.392332 8950 main.go:141] libmachine: (ha-184000-m04) Calling .GetIP
I0703 16:37:39.392417 8950 host.go:66] Checking if "ha-184000-m04" exists ...
I0703 16:37:39.392681 8950 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0703 16:37:39.392707 8950 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0703 16:37:39.400943 8950 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:53998
I0703 16:37:39.401269 8950 main.go:141] libmachine: () Calling .GetVersion
I0703 16:37:39.401591 8950 main.go:141] libmachine: Using API Version 1
I0703 16:37:39.401607 8950 main.go:141] libmachine: () Calling .SetConfigRaw
I0703 16:37:39.401803 8950 main.go:141] libmachine: () Calling .GetMachineName
I0703 16:37:39.401907 8950 main.go:141] libmachine: (ha-184000-m04) Calling .DriverName
I0703 16:37:39.402057 8950 ssh_runner.go:195] Run: sh -c "df -h /var | awk 'NR==2{print $5}'"
I0703 16:37:39.402074 8950 main.go:141] libmachine: (ha-184000-m04) Calling .GetSSHHostname
I0703 16:37:39.402156 8950 main.go:141] libmachine: (ha-184000-m04) Calling .GetSSHPort
I0703 16:37:39.402231 8950 main.go:141] libmachine: (ha-184000-m04) Calling .GetSSHKeyPath
I0703 16:37:39.402305 8950 main.go:141] libmachine: (ha-184000-m04) Calling .GetSSHUsername
I0703 16:37:39.402382 8950 sshutil.go:53] new ssh client: &{IP:192.169.0.10 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/18859-6498/.minikube/machines/ha-184000-m04/id_rsa Username:docker}
W0703 16:38:54.401975 8950 sshutil.go:64] dial failure (will retry): dial tcp 192.169.0.10:22: connect: operation timed out
W0703 16:38:54.402032 8950 start.go:268] error running df -h /var: NewSession: new client: new client: dial tcp 192.169.0.10:22: connect: operation timed out
E0703 16:38:54.402043 8950 status.go:376] failed to get storage capacity of /var: NewSession: new client: new client: dial tcp 192.169.0.10:22: connect: operation timed out
I0703 16:38:54.402052 8950 status.go:257] ha-184000-m04 status: &{Name:ha-184000-m04 Host:Error Kubelet:Nonexistent APIServer:Irrelevant Kubeconfig:Irrelevant Worker:true TimeToStop: DockerEnv: PodManEnv:}
E0703 16:38:54.402063 8950 status.go:260] status error: NewSession: new client: new client: dial tcp 192.169.0.10:22: connect: operation timed out
** /stderr **
ha_test.go:328: failed to run minikube status. args "out/minikube-darwin-amd64 -p ha-184000 status --output json -v=7 --alsologtostderr" : exit status 3
helpers_test.go:222: -----------------------post-mortem--------------------------------
helpers_test.go:239: (dbg) Run: out/minikube-darwin-amd64 status --format={{.Host}} -p ha-184000 -n ha-184000
E0703 16:39:37.196532 7038 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18859-6498/.minikube/profiles/addons-198000/client.crt: no such file or directory
helpers_test.go:239: (dbg) Non-zero exit: out/minikube-darwin-amd64 status --format={{.Host}} -p ha-184000 -n ha-184000: exit status 3 (1m15.08950622s)
-- stdout --
Error
-- /stdout --
** stderr **
E0703 16:40:09.491381 9028 status.go:376] failed to get storage capacity of /var: NewSession: new client: new client: dial tcp 192.169.0.7:22: connect: operation timed out
E0703 16:40:09.491400 9028 status.go:249] status error: NewSession: new client: new client: dial tcp 192.169.0.7:22: connect: operation timed out
** /stderr **
helpers_test.go:239: status error: exit status 3 (may be ok)
helpers_test.go:241: "ha-184000" host is not running, skipping log retrieval (state="Error")
--- FAIL: TestMultiControlPlane/serial/CopyFile (375.28s)