=== RUN TestMultiControlPlane/serial/CopyFile
ha_test.go:326: (dbg) Run: out/minikube-darwin-amd64 -p ha-069000 status --output json -v=7 --alsologtostderr
E0422 04:03:44.367728 1484 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18711-1033/.minikube/profiles/functional-984000/client.crt: no such file or directory
E0422 04:04:53.314463 1484 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18711-1033/.minikube/profiles/addons-483000/client.crt: no such file or directory
ha_test.go:326: (dbg) Non-zero exit: out/minikube-darwin-amd64 -p ha-069000 status --output json -v=7 --alsologtostderr: exit status 3 (5m0.218751905s)
-- stdout --
[{"Name":"ha-069000","Host":"Error","Kubelet":"Nonexistent","APIServer":"Nonexistent","Kubeconfig":"Configured","Worker":false},{"Name":"ha-069000-m02","Host":"Error","Kubelet":"Nonexistent","APIServer":"Nonexistent","Kubeconfig":"Configured","Worker":false},{"Name":"ha-069000-m03","Host":"Error","Kubelet":"Nonexistent","APIServer":"Nonexistent","Kubeconfig":"Configured","Worker":false},{"Name":"ha-069000-m04","Host":"Error","Kubelet":"Nonexistent","APIServer":"Irrelevant","Kubeconfig":"Irrelevant","Worker":true}]
-- /stdout --
** stderr **
I0422 04:01:56.117651 3781 out.go:291] Setting OutFile to fd 1 ...
I0422 04:01:56.118228 3781 out.go:338] TERM=,COLORTERM=, which probably does not support color
I0422 04:01:56.118236 3781 out.go:304] Setting ErrFile to fd 2...
I0422 04:01:56.118240 3781 out.go:338] TERM=,COLORTERM=, which probably does not support color
I0422 04:01:56.118440 3781 root.go:338] Updating PATH: /Users/jenkins/minikube-integration/18711-1033/.minikube/bin
I0422 04:01:56.118683 3781 out.go:298] Setting JSON to true
I0422 04:01:56.118706 3781 mustload.go:65] Loading cluster: ha-069000
I0422 04:01:56.118750 3781 notify.go:220] Checking for updates...
I0422 04:01:56.119214 3781 config.go:182] Loaded profile config "ha-069000": Driver=hyperkit, ContainerRuntime=docker, KubernetesVersion=v1.30.0
I0422 04:01:56.119261 3781 status.go:255] checking status of ha-069000 ...
I0422 04:01:56.119712 3781 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0422 04:01:56.119761 3781 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0422 04:01:56.129448 3781 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:50971
I0422 04:01:56.129818 3781 main.go:141] libmachine: () Calling .GetVersion
I0422 04:01:56.130219 3781 main.go:141] libmachine: Using API Version 1
I0422 04:01:56.130258 3781 main.go:141] libmachine: () Calling .SetConfigRaw
I0422 04:01:56.130507 3781 main.go:141] libmachine: () Calling .GetMachineName
I0422 04:01:56.130645 3781 main.go:141] libmachine: (ha-069000) Calling .GetState
I0422 04:01:56.130729 3781 main.go:141] libmachine: (ha-069000) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I0422 04:01:56.130809 3781 main.go:141] libmachine: (ha-069000) DBG | hyperkit pid from json: 3181
I0422 04:01:56.131815 3781 status.go:330] ha-069000 host status = "Running" (err=<nil>)
I0422 04:01:56.131833 3781 host.go:66] Checking if "ha-069000" exists ...
I0422 04:01:56.132077 3781 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0422 04:01:56.132102 3781 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0422 04:01:56.141552 3781 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:50974
I0422 04:01:56.141921 3781 main.go:141] libmachine: () Calling .GetVersion
I0422 04:01:56.142302 3781 main.go:141] libmachine: Using API Version 1
I0422 04:01:56.142312 3781 main.go:141] libmachine: () Calling .SetConfigRaw
I0422 04:01:56.142583 3781 main.go:141] libmachine: () Calling .GetMachineName
I0422 04:01:56.142713 3781 main.go:141] libmachine: (ha-069000) Calling .GetIP
I0422 04:01:56.142816 3781 host.go:66] Checking if "ha-069000" exists ...
I0422 04:01:56.143115 3781 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0422 04:01:56.143152 3781 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0422 04:01:56.153340 3781 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:50976
I0422 04:01:56.153716 3781 main.go:141] libmachine: () Calling .GetVersion
I0422 04:01:56.162172 3781 main.go:141] libmachine: Using API Version 1
I0422 04:01:56.162195 3781 main.go:141] libmachine: () Calling .SetConfigRaw
I0422 04:01:56.162489 3781 main.go:141] libmachine: () Calling .GetMachineName
I0422 04:01:56.162626 3781 main.go:141] libmachine: (ha-069000) Calling .DriverName
I0422 04:01:56.162791 3781 ssh_runner.go:195] Run: sh -c "df -h /var | awk 'NR==2{print $5}'"
I0422 04:01:56.162814 3781 main.go:141] libmachine: (ha-069000) Calling .GetSSHHostname
I0422 04:01:56.162906 3781 main.go:141] libmachine: (ha-069000) Calling .GetSSHPort
I0422 04:01:56.163010 3781 main.go:141] libmachine: (ha-069000) Calling .GetSSHKeyPath
I0422 04:01:56.163108 3781 main.go:141] libmachine: (ha-069000) Calling .GetSSHUsername
I0422 04:01:56.163221 3781 sshutil.go:53] new ssh client: &{IP:192.169.0.6 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/18711-1033/.minikube/machines/ha-069000/id_rsa Username:docker}
W0422 04:03:11.167062 3781 sshutil.go:64] dial failure (will retry): dial tcp 192.169.0.6:22: connect: operation timed out
W0422 04:03:11.167131 3781 start.go:268] error running df -h /var: NewSession: new client: new client: dial tcp 192.169.0.6:22: connect: operation timed out
E0422 04:03:11.167141 3781 status.go:376] failed to get storage capacity of /var: NewSession: new client: new client: dial tcp 192.169.0.6:22: connect: operation timed out
I0422 04:03:11.167151 3781 status.go:257] ha-069000 status: &{Name:ha-069000 Host:Error Kubelet:Nonexistent APIServer:Nonexistent Kubeconfig:Configured Worker:false TimeToStop: DockerEnv: PodManEnv:}
E0422 04:03:11.167162 3781 status.go:260] status error: NewSession: new client: new client: dial tcp 192.169.0.6:22: connect: operation timed out
I0422 04:03:11.167169 3781 status.go:255] checking status of ha-069000-m02 ...
I0422 04:03:11.167450 3781 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0422 04:03:11.167474 3781 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0422 04:03:11.176662 3781 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:50979
I0422 04:03:11.176989 3781 main.go:141] libmachine: () Calling .GetVersion
I0422 04:03:11.177347 3781 main.go:141] libmachine: Using API Version 1
I0422 04:03:11.177362 3781 main.go:141] libmachine: () Calling .SetConfigRaw
I0422 04:03:11.177553 3781 main.go:141] libmachine: () Calling .GetMachineName
I0422 04:03:11.177659 3781 main.go:141] libmachine: (ha-069000-m02) Calling .GetState
I0422 04:03:11.177731 3781 main.go:141] libmachine: (ha-069000-m02) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I0422 04:03:11.177815 3781 main.go:141] libmachine: (ha-069000-m02) DBG | hyperkit pid from json: 3228
I0422 04:03:11.178801 3781 status.go:330] ha-069000-m02 host status = "Running" (err=<nil>)
I0422 04:03:11.178809 3781 host.go:66] Checking if "ha-069000-m02" exists ...
I0422 04:03:11.179055 3781 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0422 04:03:11.179074 3781 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0422 04:03:11.187658 3781 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:50981
I0422 04:03:11.188016 3781 main.go:141] libmachine: () Calling .GetVersion
I0422 04:03:11.188375 3781 main.go:141] libmachine: Using API Version 1
I0422 04:03:11.188407 3781 main.go:141] libmachine: () Calling .SetConfigRaw
I0422 04:03:11.188602 3781 main.go:141] libmachine: () Calling .GetMachineName
I0422 04:03:11.188715 3781 main.go:141] libmachine: (ha-069000-m02) Calling .GetIP
I0422 04:03:11.188808 3781 host.go:66] Checking if "ha-069000-m02" exists ...
I0422 04:03:11.189067 3781 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0422 04:03:11.189090 3781 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0422 04:03:11.197666 3781 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:50983
I0422 04:03:11.198002 3781 main.go:141] libmachine: () Calling .GetVersion
I0422 04:03:11.198365 3781 main.go:141] libmachine: Using API Version 1
I0422 04:03:11.198386 3781 main.go:141] libmachine: () Calling .SetConfigRaw
I0422 04:03:11.198604 3781 main.go:141] libmachine: () Calling .GetMachineName
I0422 04:03:11.198733 3781 main.go:141] libmachine: (ha-069000-m02) Calling .DriverName
I0422 04:03:11.198870 3781 ssh_runner.go:195] Run: sh -c "df -h /var | awk 'NR==2{print $5}'"
I0422 04:03:11.198884 3781 main.go:141] libmachine: (ha-069000-m02) Calling .GetSSHHostname
I0422 04:03:11.198972 3781 main.go:141] libmachine: (ha-069000-m02) Calling .GetSSHPort
I0422 04:03:11.199053 3781 main.go:141] libmachine: (ha-069000-m02) Calling .GetSSHKeyPath
I0422 04:03:11.199166 3781 main.go:141] libmachine: (ha-069000-m02) Calling .GetSSHUsername
I0422 04:03:11.199260 3781 sshutil.go:53] new ssh client: &{IP:192.169.0.7 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/18711-1033/.minikube/machines/ha-069000-m02/id_rsa Username:docker}
W0422 04:04:26.200772 3781 sshutil.go:64] dial failure (will retry): dial tcp 192.169.0.7:22: connect: operation timed out
W0422 04:04:26.200832 3781 start.go:268] error running df -h /var: NewSession: new client: new client: dial tcp 192.169.0.7:22: connect: operation timed out
E0422 04:04:26.200844 3781 status.go:376] failed to get storage capacity of /var: NewSession: new client: new client: dial tcp 192.169.0.7:22: connect: operation timed out
I0422 04:04:26.200852 3781 status.go:257] ha-069000-m02 status: &{Name:ha-069000-m02 Host:Error Kubelet:Nonexistent APIServer:Nonexistent Kubeconfig:Configured Worker:false TimeToStop: DockerEnv: PodManEnv:}
E0422 04:04:26.200865 3781 status.go:260] status error: NewSession: new client: new client: dial tcp 192.169.0.7:22: connect: operation timed out
I0422 04:04:26.200870 3781 status.go:255] checking status of ha-069000-m03 ...
I0422 04:04:26.201148 3781 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0422 04:04:26.201175 3781 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0422 04:04:26.210809 3781 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:50986
I0422 04:04:26.211170 3781 main.go:141] libmachine: () Calling .GetVersion
I0422 04:04:26.211501 3781 main.go:141] libmachine: Using API Version 1
I0422 04:04:26.211515 3781 main.go:141] libmachine: () Calling .SetConfigRaw
I0422 04:04:26.211750 3781 main.go:141] libmachine: () Calling .GetMachineName
I0422 04:04:26.211860 3781 main.go:141] libmachine: (ha-069000-m03) Calling .GetState
I0422 04:04:26.211950 3781 main.go:141] libmachine: (ha-069000-m03) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I0422 04:04:26.212041 3781 main.go:141] libmachine: (ha-069000-m03) DBG | hyperkit pid from json: 3336
I0422 04:04:26.213084 3781 status.go:330] ha-069000-m03 host status = "Running" (err=<nil>)
I0422 04:04:26.213096 3781 host.go:66] Checking if "ha-069000-m03" exists ...
I0422 04:04:26.213380 3781 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0422 04:04:26.213429 3781 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0422 04:04:26.222489 3781 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:50988
I0422 04:04:26.222873 3781 main.go:141] libmachine: () Calling .GetVersion
I0422 04:04:26.223249 3781 main.go:141] libmachine: Using API Version 1
I0422 04:04:26.223266 3781 main.go:141] libmachine: () Calling .SetConfigRaw
I0422 04:04:26.223497 3781 main.go:141] libmachine: () Calling .GetMachineName
I0422 04:04:26.223635 3781 main.go:141] libmachine: (ha-069000-m03) Calling .GetIP
I0422 04:04:26.223727 3781 host.go:66] Checking if "ha-069000-m03" exists ...
I0422 04:04:26.224002 3781 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0422 04:04:26.224026 3781 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0422 04:04:26.232931 3781 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:50990
I0422 04:04:26.233307 3781 main.go:141] libmachine: () Calling .GetVersion
I0422 04:04:26.233644 3781 main.go:141] libmachine: Using API Version 1
I0422 04:04:26.233669 3781 main.go:141] libmachine: () Calling .SetConfigRaw
I0422 04:04:26.233878 3781 main.go:141] libmachine: () Calling .GetMachineName
I0422 04:04:26.233991 3781 main.go:141] libmachine: (ha-069000-m03) Calling .DriverName
I0422 04:04:26.234117 3781 ssh_runner.go:195] Run: sh -c "df -h /var | awk 'NR==2{print $5}'"
I0422 04:04:26.234129 3781 main.go:141] libmachine: (ha-069000-m03) Calling .GetSSHHostname
I0422 04:04:26.234238 3781 main.go:141] libmachine: (ha-069000-m03) Calling .GetSSHPort
I0422 04:04:26.234314 3781 main.go:141] libmachine: (ha-069000-m03) Calling .GetSSHKeyPath
I0422 04:04:26.234412 3781 main.go:141] libmachine: (ha-069000-m03) Calling .GetSSHUsername
I0422 04:04:26.234487 3781 sshutil.go:53] new ssh client: &{IP:192.169.0.8 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/18711-1033/.minikube/machines/ha-069000-m03/id_rsa Username:docker}
W0422 04:05:41.239086 3781 sshutil.go:64] dial failure (will retry): dial tcp 192.169.0.8:22: connect: operation timed out
W0422 04:05:41.239166 3781 start.go:268] error running df -h /var: NewSession: new client: new client: dial tcp 192.169.0.8:22: connect: operation timed out
E0422 04:05:41.239186 3781 status.go:376] failed to get storage capacity of /var: NewSession: new client: new client: dial tcp 192.169.0.8:22: connect: operation timed out
I0422 04:05:41.239197 3781 status.go:257] ha-069000-m03 status: &{Name:ha-069000-m03 Host:Error Kubelet:Nonexistent APIServer:Nonexistent Kubeconfig:Configured Worker:false TimeToStop: DockerEnv: PodManEnv:}
E0422 04:05:41.239213 3781 status.go:260] status error: NewSession: new client: new client: dial tcp 192.169.0.8:22: connect: operation timed out
I0422 04:05:41.239220 3781 status.go:255] checking status of ha-069000-m04 ...
I0422 04:05:41.239625 3781 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0422 04:05:41.239692 3781 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0422 04:05:41.249938 3781 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:50993
I0422 04:05:41.250267 3781 main.go:141] libmachine: () Calling .GetVersion
I0422 04:05:41.250614 3781 main.go:141] libmachine: Using API Version 1
I0422 04:05:41.250627 3781 main.go:141] libmachine: () Calling .SetConfigRaw
I0422 04:05:41.250838 3781 main.go:141] libmachine: () Calling .GetMachineName
I0422 04:05:41.250951 3781 main.go:141] libmachine: (ha-069000-m04) Calling .GetState
I0422 04:05:41.251034 3781 main.go:141] libmachine: (ha-069000-m04) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I0422 04:05:41.251134 3781 main.go:141] libmachine: (ha-069000-m04) DBG | hyperkit pid from json: 3553
I0422 04:05:41.252140 3781 status.go:330] ha-069000-m04 host status = "Running" (err=<nil>)
I0422 04:05:41.252152 3781 host.go:66] Checking if "ha-069000-m04" exists ...
I0422 04:05:41.252399 3781 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0422 04:05:41.252421 3781 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0422 04:05:41.261715 3781 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:50995
I0422 04:05:41.262066 3781 main.go:141] libmachine: () Calling .GetVersion
I0422 04:05:41.262414 3781 main.go:141] libmachine: Using API Version 1
I0422 04:05:41.262431 3781 main.go:141] libmachine: () Calling .SetConfigRaw
I0422 04:05:41.262638 3781 main.go:141] libmachine: () Calling .GetMachineName
I0422 04:05:41.262748 3781 main.go:141] libmachine: (ha-069000-m04) Calling .GetIP
I0422 04:05:41.262831 3781 host.go:66] Checking if "ha-069000-m04" exists ...
I0422 04:05:41.263093 3781 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0422 04:05:41.263115 3781 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0422 04:05:41.271728 3781 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:50997
I0422 04:05:41.272043 3781 main.go:141] libmachine: () Calling .GetVersion
I0422 04:05:41.272442 3781 main.go:141] libmachine: Using API Version 1
I0422 04:05:41.272467 3781 main.go:141] libmachine: () Calling .SetConfigRaw
I0422 04:05:41.272699 3781 main.go:141] libmachine: () Calling .GetMachineName
I0422 04:05:41.272829 3781 main.go:141] libmachine: (ha-069000-m04) Calling .DriverName
I0422 04:05:41.273874 3781 ssh_runner.go:195] Run: sh -c "df -h /var | awk 'NR==2{print $5}'"
I0422 04:05:41.273885 3781 main.go:141] libmachine: (ha-069000-m04) Calling .GetSSHHostname
I0422 04:05:41.273972 3781 main.go:141] libmachine: (ha-069000-m04) Calling .GetSSHPort
I0422 04:05:41.274064 3781 main.go:141] libmachine: (ha-069000-m04) Calling .GetSSHKeyPath
I0422 04:05:41.274175 3781 main.go:141] libmachine: (ha-069000-m04) Calling .GetSSHUsername
I0422 04:05:41.274265 3781 sshutil.go:53] new ssh client: &{IP:192.169.0.9 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/18711-1033/.minikube/machines/ha-069000-m04/id_rsa Username:docker}
W0422 04:06:56.276169 3781 sshutil.go:64] dial failure (will retry): dial tcp 192.169.0.9:22: connect: operation timed out
W0422 04:06:56.276236 3781 start.go:268] error running df -h /var: NewSession: new client: new client: dial tcp 192.169.0.9:22: connect: operation timed out
E0422 04:06:56.276254 3781 status.go:376] failed to get storage capacity of /var: NewSession: new client: new client: dial tcp 192.169.0.9:22: connect: operation timed out
I0422 04:06:56.276266 3781 status.go:257] ha-069000-m04 status: &{Name:ha-069000-m04 Host:Error Kubelet:Nonexistent APIServer:Irrelevant Kubeconfig:Irrelevant Worker:true TimeToStop: DockerEnv: PodManEnv:}
E0422 04:06:56.276279 3781 status.go:260] status error: NewSession: new client: new client: dial tcp 192.169.0.9:22: connect: operation timed out
** /stderr **
ha_test.go:328: failed to run minikube status. args "out/minikube-darwin-amd64 -p ha-069000 status --output json -v=7 --alsologtostderr" : exit status 3
helpers_test.go:222: -----------------------post-mortem--------------------------------
helpers_test.go:239: (dbg) Run: out/minikube-darwin-amd64 status --format={{.Host}} -p ha-069000 -n ha-069000
helpers_test.go:239: (dbg) Non-zero exit: out/minikube-darwin-amd64 status --format={{.Host}} -p ha-069000 -n ha-069000: exit status 3 (1m15.106370457s)
-- stdout --
Error
-- /stdout --
** stderr **
E0422 04:08:11.386087 4140 status.go:376] failed to get storage capacity of /var: NewSession: new client: new client: dial tcp 192.169.0.6:22: connect: operation timed out
E0422 04:08:11.386101 4140 status.go:249] status error: NewSession: new client: new client: dial tcp 192.169.0.6:22: connect: operation timed out
** /stderr **
helpers_test.go:239: status error: exit status 3 (may be ok)
helpers_test.go:241: "ha-069000" host is not running, skipping log retrieval (state="Error")
--- FAIL: TestMultiControlPlane/serial/CopyFile (375.33s)