=== RUN TestStartStop/group/old-k8s-version/serial/FirstStart
start_stop_delete_test.go:186: (dbg) Run: out/minikube-darwin-amd64 start -p old-k8s-version-326000 --memory=2200 --alsologtostderr --wait=true --kvm-network=default --kvm-qemu-uri=qemu:///system --disable-driver-mounts --keep-context=false --driver=docker --kubernetes-version=v1.16.0
E0226 03:25:41.112235 10026 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/auto-722000/client.crt: no such file or directory
E0226 03:25:46.882202 10026 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/flannel-722000/client.crt: no such file or directory
E0226 03:25:46.887316 10026 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/flannel-722000/client.crt: no such file or directory
E0226 03:25:46.898424 10026 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/flannel-722000/client.crt: no such file or directory
E0226 03:25:46.919449 10026 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/flannel-722000/client.crt: no such file or directory
E0226 03:25:46.959875 10026 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/flannel-722000/client.crt: no such file or directory
E0226 03:25:47.040240 10026 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/flannel-722000/client.crt: no such file or directory
E0226 03:25:47.201595 10026 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/flannel-722000/client.crt: no such file or directory
E0226 03:25:47.521993 10026 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/flannel-722000/client.crt: no such file or directory
E0226 03:25:48.162304 10026 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/flannel-722000/client.crt: no such file or directory
E0226 03:25:49.442816 10026 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/flannel-722000/client.crt: no such file or directory
E0226 03:25:52.003009 10026 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/flannel-722000/client.crt: no such file or directory
E0226 03:25:57.124407 10026 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/flannel-722000/client.crt: no such file or directory
E0226 03:26:05.658126 10026 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/kindnet-722000/client.crt: no such file or directory
E0226 03:26:07.364707 10026 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/flannel-722000/client.crt: no such file or directory
E0226 03:26:07.539204 10026 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/enable-default-cni-722000/client.crt: no such file or directory
E0226 03:26:07.544676 10026 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/enable-default-cni-722000/client.crt: no such file or directory
E0226 03:26:07.555176 10026 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/enable-default-cni-722000/client.crt: no such file or directory
E0226 03:26:07.576543 10026 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/enable-default-cni-722000/client.crt: no such file or directory
E0226 03:26:07.616966 10026 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/enable-default-cni-722000/client.crt: no such file or directory
E0226 03:26:07.697183 10026 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/enable-default-cni-722000/client.crt: no such file or directory
E0226 03:26:07.858506 10026 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/enable-default-cni-722000/client.crt: no such file or directory
E0226 03:26:08.179007 10026 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/enable-default-cni-722000/client.crt: no such file or directory
E0226 03:26:08.819324 10026 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/enable-default-cni-722000/client.crt: no such file or directory
E0226 03:26:10.099506 10026 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/enable-default-cni-722000/client.crt: no such file or directory
E0226 03:26:12.660416 10026 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/enable-default-cni-722000/client.crt: no such file or directory
E0226 03:26:17.781159 10026 cert_rotation.go:168] key failed with : open /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/enable-default-cni-722000/client.crt: no such file or directory
start_stop_delete_test.go:186: (dbg) Non-zero exit: out/minikube-darwin-amd64 start -p old-k8s-version-326000 --memory=2200 --alsologtostderr --wait=true --kvm-network=default --kvm-qemu-uri=qemu:///system --disable-driver-mounts --keep-context=false --driver=docker --kubernetes-version=v1.16.0: exit status 109 (4m18.640931809s)
-- stdout --
* [old-k8s-version-326000] minikube v1.32.0 on Darwin 14.3.1
- MINIKUBE_LOCATION=18222
- KUBECONFIG=/Users/jenkins/minikube-integration/18222-9538/kubeconfig
- MINIKUBE_BIN=out/minikube-darwin-amd64
- MINIKUBE_SUPPRESS_DOCKER_PERFORMANCE=true
- MINIKUBE_HOME=/Users/jenkins/minikube-integration/18222-9538/.minikube
- MINIKUBE_FORCE_SYSTEMD=
* Using the docker driver based on user configuration
* Using Docker Desktop driver with root privileges
* Starting control plane node old-k8s-version-326000 in cluster old-k8s-version-326000
* Pulling base image v0.0.42-1708008208-17936 ...
* Creating docker container (CPUs=2, Memory=2200MB) ...
* Preparing Kubernetes v1.16.0 on Docker 25.0.3 ...
- Generating certificates and keys ...
- Booting up control plane ...
- Generating certificates and keys ...
- Booting up control plane ...
-- /stdout --
** stderr **
I0226 03:25:37.769475 25643 out.go:291] Setting OutFile to fd 1 ...
I0226 03:25:37.769749 25643 out.go:338] TERM=,COLORTERM=, which probably does not support color
I0226 03:25:37.769755 25643 out.go:304] Setting ErrFile to fd 2...
I0226 03:25:37.769758 25643 out.go:338] TERM=,COLORTERM=, which probably does not support color
I0226 03:25:37.770010 25643 root.go:338] Updating PATH: /Users/jenkins/minikube-integration/18222-9538/.minikube/bin
I0226 03:25:37.772248 25643 out.go:298] Setting JSON to false
I0226 03:25:37.798753 25643 start.go:129] hostinfo: {"hostname":"MacOS-Agent-4.local","uptime":12308,"bootTime":1708934429,"procs":448,"os":"darwin","platform":"darwin","platformFamily":"Standalone Workstation","platformVersion":"14.3.1","kernelVersion":"23.3.0","kernelArch":"x86_64","virtualizationSystem":"","virtualizationRole":"","hostId":"f2f27e25-cfda-5ffd-9706-e98286194e62"}
W0226 03:25:37.798846 25643 start.go:137] gopshost.Virtualization returned error: not implemented yet
I0226 03:25:37.822175 25643 out.go:177] * [old-k8s-version-326000] minikube v1.32.0 on Darwin 14.3.1
I0226 03:25:37.881228 25643 out.go:177] - MINIKUBE_LOCATION=18222
I0226 03:25:37.860242 25643 notify.go:220] Checking for updates...
I0226 03:25:37.940122 25643 out.go:177] - KUBECONFIG=/Users/jenkins/minikube-integration/18222-9538/kubeconfig
I0226 03:25:37.999276 25643 out.go:177] - MINIKUBE_BIN=out/minikube-darwin-amd64
I0226 03:25:38.058207 25643 out.go:177] - MINIKUBE_SUPPRESS_DOCKER_PERFORMANCE=true
I0226 03:25:38.132002 25643 out.go:177] - MINIKUBE_HOME=/Users/jenkins/minikube-integration/18222-9538/.minikube
I0226 03:25:38.207049 25643 out.go:177] - MINIKUBE_FORCE_SYSTEMD=
I0226 03:25:38.244711 25643 config.go:182] Loaded profile config "calico-722000": Driver=docker, ContainerRuntime=docker, KubernetesVersion=v1.28.4
I0226 03:25:38.244802 25643 driver.go:392] Setting default libvirt URI to qemu:///system
I0226 03:25:38.300356 25643 docker.go:122] docker version: linux-25.0.3:Docker Desktop 4.27.2 (137060)
I0226 03:25:38.300673 25643 cli_runner.go:164] Run: docker system info --format "{{json .}}"
I0226 03:25:38.404560 25643 info.go:266] docker info: {ID:bd95ca90-0161-4940-8de1-bb75c87f79bd Containers:1 ContainersRunning:1 ContainersPaused:0 ContainersStopped:0 Images:3 Driver:overlay2 DriverStatus:[[Backing Filesystem extfs] [Supports d_type true] [Using metacopy false] [Native Overlay Diff true] [userxattr false]] SystemStatus:<nil> Plugins:{Volume:[local] Network:[bridge host ipvlan macvlan null overlay] Authorization:<nil> Log:[awslogs fluentd gcplogs gelf journald json-file local splunk syslog]} MemoryLimit:true SwapLimit:true KernelMemory:false KernelMemoryTCP:false CPUCfsPeriod:true CPUCfsQuota:true CPUShares:true CPUSet:true PidsLimit:true IPv4Forwarding:true BridgeNfIptables:true BridgeNfIP6Tables:true Debug:false NFd:73 OomKillDisable:false NGoroutines:115 SystemTime:2024-02-26 11:25:38.393629526 +0000 UTC LoggingDriver:json-file CgroupDriver:cgroupfs NEventsListener:26 KernelVersion:6.6.12-linuxkit OperatingSystem:Docker Desktop OSType:linux Architecture:x86_64 IndexServerAddress:
https://index.docker.io/v1/ RegistryConfig:{AllowNondistributableArtifactsCIDRs:[] AllowNondistributableArtifactsHostnames:[] InsecureRegistryCIDRs:[127.0.0.0/8] IndexConfigs:{DockerIo:{Name:docker.io Mirrors:[] Secure:true Official:true}} Mirrors:[]} NCPU:12 MemTotal:6213300224 GenericResources:<nil> DockerRootDir:/var/lib/docker HTTPProxy:http.docker.internal:3128 HTTPSProxy:http.docker.internal:3128 NoProxy:hubproxy.docker.internal Name:docker-desktop Labels:[] ExperimentalBuild:false ServerVersion:25.0.3 ClusterStore: ClusterAdvertise: Runtimes:{Runc:{Path:runc}} DefaultRuntime:runc Swarm:{NodeID: NodeAddr: LocalNodeState:inactive ControlAvailable:false Error: RemoteManagers:<nil>} LiveRestoreEnabled:false Isolation: InitBinary:docker-init ContainerdCommit:{ID:ae07eda36dd25f8a1b98dfbf587313b99c0190bb Expected:ae07eda36dd25f8a1b98dfbf587313b99c0190bb} RuncCommit:{ID:v1.1.12-0-g51d5e94 Expected:v1.1.12-0-g51d5e94} InitCommit:{ID:de40ad0 Expected:de40ad0} SecurityOptions:[name=seccomp,profile=unconfined name
=cgroupns] ProductLicense: Warnings:[WARNING: daemon is not using the default seccomp profile] ServerErrors:[] ClientInfo:{Debug:false Plugins:[map[Name:buildx Path:/Users/jenkins/.docker/cli-plugins/docker-buildx SchemaVersion:0.1.0 ShadowedPaths:[/usr/local/lib/docker/cli-plugins/docker-buildx] ShortDescription:Docker Buildx Vendor:Docker Inc. Version:v0.12.1-desktop.4] map[Name:compose Path:/Users/jenkins/.docker/cli-plugins/docker-compose SchemaVersion:0.1.0 ShadowedPaths:[/usr/local/lib/docker/cli-plugins/docker-compose] ShortDescription:Docker Compose Vendor:Docker Inc. Version:v2.24.5-desktop.1] map[Name:debug Path:/Users/jenkins/.docker/cli-plugins/docker-debug SchemaVersion:0.1.0 ShadowedPaths:[/usr/local/lib/docker/cli-plugins/docker-debug] ShortDescription:Get a shell into any image or container. Vendor:Docker Inc. Version:0.0.24] map[Name:dev Path:/Users/jenkins/.docker/cli-plugins/docker-dev SchemaVersion:0.1.0 ShadowedPaths:[/usr/local/lib/docker/cli-plugins/docker-dev] ShortDescription:Docker D
ev Environments Vendor:Docker Inc. Version:v0.1.0] map[Name:extension Path:/Users/jenkins/.docker/cli-plugins/docker-extension SchemaVersion:0.1.0 ShadowedPaths:[/usr/local/lib/docker/cli-plugins/docker-extension] ShortDescription:Manages Docker extensions Vendor:Docker Inc. Version:v0.2.21] map[Name:feedback Path:/Users/jenkins/.docker/cli-plugins/docker-feedback SchemaVersion:0.1.0 ShadowedPaths:[/usr/local/lib/docker/cli-plugins/docker-feedback] ShortDescription:Provide feedback, right in your terminal! Vendor:Docker Inc. Version:v1.0.4] map[Name:init Path:/Users/jenkins/.docker/cli-plugins/docker-init SchemaVersion:0.1.0 ShadowedPaths:[/usr/local/lib/docker/cli-plugins/docker-init] ShortDescription:Creates Docker-related starter files for your project Vendor:Docker Inc. Version:v1.0.0] map[Name:sbom Path:/Users/jenkins/.docker/cli-plugins/docker-sbom SchemaVersion:0.1.0 ShadowedPaths:[/usr/local/lib/docker/cli-plugins/docker-sbom] ShortDescription:View the packaged-based Software Bill Of Materials (SBOM)
for an image URL:https://github.com/docker/sbom-cli-plugin Vendor:Anchore Inc. Version:0.6.0] map[Err:failed to fetch metadata: fork/exec /Users/jenkins/.docker/cli-plugins/docker-scan: no such file or directory Name:scan Path:/Users/jenkins/.docker/cli-plugins/docker-scan] map[Name:scout Path:/Users/jenkins/.docker/cli-plugins/docker-scout SchemaVersion:0.1.0 ShadowedPaths:[/usr/local/lib/docker/cli-plugins/docker-scout] ShortDescription:Docker Scout Vendor:Docker Inc. Version:v1.4.1]] Warnings:<nil>}}
I0226 03:25:38.449376 25643 out.go:177] * Using the docker driver based on user configuration
I0226 03:25:38.470274 25643 start.go:299] selected driver: docker
I0226 03:25:38.470291 25643 start.go:903] validating driver "docker" against <nil>
I0226 03:25:38.470300 25643 start.go:914] status for docker: {Installed:true Healthy:true Running:false NeedsImprovement:false Error:<nil> Reason: Fix: Doc: Version:}
I0226 03:25:38.475160 25643 cli_runner.go:164] Run: docker system info --format "{{json .}}"
I0226 03:25:38.592677 25643 info.go:266] docker info: {ID:bd95ca90-0161-4940-8de1-bb75c87f79bd Containers:1 ContainersRunning:1 ContainersPaused:0 ContainersStopped:0 Images:3 Driver:overlay2 DriverStatus:[[Backing Filesystem extfs] [Supports d_type true] [Using metacopy false] [Native Overlay Diff true] [userxattr false]] SystemStatus:<nil> Plugins:{Volume:[local] Network:[bridge host ipvlan macvlan null overlay] Authorization:<nil> Log:[awslogs fluentd gcplogs gelf journald json-file local splunk syslog]} MemoryLimit:true SwapLimit:true KernelMemory:false KernelMemoryTCP:false CPUCfsPeriod:true CPUCfsQuota:true CPUShares:true CPUSet:true PidsLimit:true IPv4Forwarding:true BridgeNfIptables:true BridgeNfIP6Tables:true Debug:false NFd:73 OomKillDisable:false NGoroutines:115 SystemTime:2024-02-26 11:25:38.580813388 +0000 UTC LoggingDriver:json-file CgroupDriver:cgroupfs NEventsListener:26 KernelVersion:6.6.12-linuxkit OperatingSystem:Docker Desktop OSType:linux Architecture:x86_64 IndexServerAddress:
https://index.docker.io/v1/ RegistryConfig:{AllowNondistributableArtifactsCIDRs:[] AllowNondistributableArtifactsHostnames:[] InsecureRegistryCIDRs:[127.0.0.0/8] IndexConfigs:{DockerIo:{Name:docker.io Mirrors:[] Secure:true Official:true}} Mirrors:[]} NCPU:12 MemTotal:6213300224 GenericResources:<nil> DockerRootDir:/var/lib/docker HTTPProxy:http.docker.internal:3128 HTTPSProxy:http.docker.internal:3128 NoProxy:hubproxy.docker.internal Name:docker-desktop Labels:[] ExperimentalBuild:false ServerVersion:25.0.3 ClusterStore: ClusterAdvertise: Runtimes:{Runc:{Path:runc}} DefaultRuntime:runc Swarm:{NodeID: NodeAddr: LocalNodeState:inactive ControlAvailable:false Error: RemoteManagers:<nil>} LiveRestoreEnabled:false Isolation: InitBinary:docker-init ContainerdCommit:{ID:ae07eda36dd25f8a1b98dfbf587313b99c0190bb Expected:ae07eda36dd25f8a1b98dfbf587313b99c0190bb} RuncCommit:{ID:v1.1.12-0-g51d5e94 Expected:v1.1.12-0-g51d5e94} InitCommit:{ID:de40ad0 Expected:de40ad0} SecurityOptions:[name=seccomp,profile=unconfined name
=cgroupns] ProductLicense: Warnings:[WARNING: daemon is not using the default seccomp profile] ServerErrors:[] ClientInfo:{Debug:false Plugins:[map[Name:buildx Path:/Users/jenkins/.docker/cli-plugins/docker-buildx SchemaVersion:0.1.0 ShadowedPaths:[/usr/local/lib/docker/cli-plugins/docker-buildx] ShortDescription:Docker Buildx Vendor:Docker Inc. Version:v0.12.1-desktop.4] map[Name:compose Path:/Users/jenkins/.docker/cli-plugins/docker-compose SchemaVersion:0.1.0 ShadowedPaths:[/usr/local/lib/docker/cli-plugins/docker-compose] ShortDescription:Docker Compose Vendor:Docker Inc. Version:v2.24.5-desktop.1] map[Name:debug Path:/Users/jenkins/.docker/cli-plugins/docker-debug SchemaVersion:0.1.0 ShadowedPaths:[/usr/local/lib/docker/cli-plugins/docker-debug] ShortDescription:Get a shell into any image or container. Vendor:Docker Inc. Version:0.0.24] map[Name:dev Path:/Users/jenkins/.docker/cli-plugins/docker-dev SchemaVersion:0.1.0 ShadowedPaths:[/usr/local/lib/docker/cli-plugins/docker-dev] ShortDescription:Docker D
ev Environments Vendor:Docker Inc. Version:v0.1.0] map[Name:extension Path:/Users/jenkins/.docker/cli-plugins/docker-extension SchemaVersion:0.1.0 ShadowedPaths:[/usr/local/lib/docker/cli-plugins/docker-extension] ShortDescription:Manages Docker extensions Vendor:Docker Inc. Version:v0.2.21] map[Name:feedback Path:/Users/jenkins/.docker/cli-plugins/docker-feedback SchemaVersion:0.1.0 ShadowedPaths:[/usr/local/lib/docker/cli-plugins/docker-feedback] ShortDescription:Provide feedback, right in your terminal! Vendor:Docker Inc. Version:v1.0.4] map[Name:init Path:/Users/jenkins/.docker/cli-plugins/docker-init SchemaVersion:0.1.0 ShadowedPaths:[/usr/local/lib/docker/cli-plugins/docker-init] ShortDescription:Creates Docker-related starter files for your project Vendor:Docker Inc. Version:v1.0.0] map[Name:sbom Path:/Users/jenkins/.docker/cli-plugins/docker-sbom SchemaVersion:0.1.0 ShadowedPaths:[/usr/local/lib/docker/cli-plugins/docker-sbom] ShortDescription:View the packaged-based Software Bill Of Materials (SBOM)
for an image URL:https://github.com/docker/sbom-cli-plugin Vendor:Anchore Inc. Version:0.6.0] map[Err:failed to fetch metadata: fork/exec /Users/jenkins/.docker/cli-plugins/docker-scan: no such file or directory Name:scan Path:/Users/jenkins/.docker/cli-plugins/docker-scan] map[Name:scout Path:/Users/jenkins/.docker/cli-plugins/docker-scout SchemaVersion:0.1.0 ShadowedPaths:[/usr/local/lib/docker/cli-plugins/docker-scout] ShortDescription:Docker Scout Vendor:Docker Inc. Version:v1.4.1]] Warnings:<nil>}}
I0226 03:25:38.592939 25643 start_flags.go:309] no existing cluster config was found, will generate one from the flags
I0226 03:25:38.593134 25643 start_flags.go:931] Waiting for all components: map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true]
I0226 03:25:38.632831 25643 out.go:177] * Using Docker Desktop driver with root privileges
I0226 03:25:38.668556 25643 cni.go:84] Creating CNI manager for ""
I0226 03:25:38.668585 25643 cni.go:162] CNI unnecessary in this configuration, recommending no CNI
I0226 03:25:38.668595 25643 start_flags.go:323] config:
{Name:old-k8s-version-326000 KeepContext:false EmbedCerts:false MinikubeISO: KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.42-1708008208-17936@sha256:4ea1136332ba1476cda33a97bf12e2f96995cc120674fbafd3ade22d1118ecdf Memory:2200 CPUs:2 DiskSize:20000 VMDriver: Driver:docker HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:0 DockerOpt:[] DisableDriverMounts:true NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.16.0 ClusterName:old-k8s-version-326000 Namespace:default APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local Containe
rRuntime:docker CRISocket: NetworkPlugin: FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI: NodeIP: NodePort:8443 NodeName:} Nodes:[] Addons:map[] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:false ExtraDisks:0 CertExpiration:26280h0m0s Mount:false MountString:/Users:/minikube-host Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOptimizations:false DisableMetrics:false CustomQemuFirmwarePath: SocketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s}
I0226 03:25:38.689664 25643 out.go:177] * Starting control plane node old-k8s-version-326000 in cluster old-k8s-version-326000
I0226 03:25:38.763850 25643 cache.go:121] Beginning downloading kic base image for docker with docker
I0226 03:25:38.784699 25643 out.go:177] * Pulling base image v0.0.42-1708008208-17936 ...
I0226 03:25:38.826628 25643 preload.go:132] Checking if preload exists for k8s version v1.16.0 and runtime docker
I0226 03:25:38.826664 25643 image.go:79] Checking for gcr.io/k8s-minikube/kicbase-builds:v0.0.42-1708008208-17936@sha256:4ea1136332ba1476cda33a97bf12e2f96995cc120674fbafd3ade22d1118ecdf in local docker daemon
I0226 03:25:38.826686 25643 preload.go:148] Found local preload: /Users/jenkins/minikube-integration/18222-9538/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.16.0-docker-overlay2-amd64.tar.lz4
I0226 03:25:38.826698 25643 cache.go:56] Caching tarball of preloaded images
I0226 03:25:38.826880 25643 preload.go:174] Found /Users/jenkins/minikube-integration/18222-9538/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.16.0-docker-overlay2-amd64.tar.lz4 in cache, skipping download
I0226 03:25:38.826896 25643 cache.go:59] Finished verifying existence of preloaded tar for v1.16.0 on docker
I0226 03:25:38.827530 25643 profile.go:148] Saving config to /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/config.json ...
I0226 03:25:38.827698 25643 lock.go:35] WriteFile acquiring /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/config.json: {Name:mk2fc74782874a86ba5a3dce94f94de712abb39f Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I0226 03:25:38.883362 25643 image.go:83] Found gcr.io/k8s-minikube/kicbase-builds:v0.0.42-1708008208-17936@sha256:4ea1136332ba1476cda33a97bf12e2f96995cc120674fbafd3ade22d1118ecdf in local docker daemon, skipping pull
I0226 03:25:38.883417 25643 cache.go:144] gcr.io/k8s-minikube/kicbase-builds:v0.0.42-1708008208-17936@sha256:4ea1136332ba1476cda33a97bf12e2f96995cc120674fbafd3ade22d1118ecdf exists in daemon, skipping load
I0226 03:25:38.883438 25643 cache.go:194] Successfully downloaded all kic artifacts
I0226 03:25:38.883496 25643 start.go:365] acquiring machines lock for old-k8s-version-326000: {Name:mk2beedabea14e6b62e464a057af5ed4bd127b3e Clock:{} Delay:500ms Timeout:10m0s Cancel:<nil>}
I0226 03:25:38.883657 25643 start.go:369] acquired machines lock for "old-k8s-version-326000" in 147.201µs
I0226 03:25:38.883685 25643 start.go:93] Provisioning new machine with config: &{Name:old-k8s-version-326000 KeepContext:false EmbedCerts:false MinikubeISO: KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.42-1708008208-17936@sha256:4ea1136332ba1476cda33a97bf12e2f96995cc120674fbafd3ade22d1118ecdf Memory:2200 CPUs:2 DiskSize:20000 VMDriver: Driver:docker HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:0 DockerOpt:[] DisableDriverMounts:true NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.16.0 ClusterName:old-k8s-version-326000 Namespace:default APIServer
Name:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:docker CRISocket: NetworkPlugin: FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI: NodeIP: NodePort:8443 NodeName:} Nodes:[{Name: IP: Port:8443 KubernetesVersion:v1.16.0 ContainerRuntime:docker ControlPlane:true Worker:true}] Addons:map[] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:false ExtraDisks:0 CertExpiration:26280h0m0s Mount:false MountString:/Users:/minikube-host Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOptimizations:f
alse DisableMetrics:false CustomQemuFirmwarePath: SocketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s} &{Name: IP: Port:8443 KubernetesVersion:v1.16.0 ContainerRuntime:docker ControlPlane:true Worker:true}
I0226 03:25:38.883772 25643 start.go:125] createHost starting for "" (driver="docker")
I0226 03:25:38.905737 25643 out.go:204] * Creating docker container (CPUs=2, Memory=2200MB) ...
I0226 03:25:38.906048 25643 start.go:159] libmachine.API.Create for "old-k8s-version-326000" (driver="docker")
I0226 03:25:38.906097 25643 client.go:168] LocalClient.Create starting
I0226 03:25:38.906217 25643 main.go:141] libmachine: Reading certificate data from /Users/jenkins/minikube-integration/18222-9538/.minikube/certs/ca.pem
I0226 03:25:38.906266 25643 main.go:141] libmachine: Decoding PEM data...
I0226 03:25:38.906286 25643 main.go:141] libmachine: Parsing certificate...
I0226 03:25:38.906350 25643 main.go:141] libmachine: Reading certificate data from /Users/jenkins/minikube-integration/18222-9538/.minikube/certs/cert.pem
I0226 03:25:38.906385 25643 main.go:141] libmachine: Decoding PEM data...
I0226 03:25:38.906392 25643 main.go:141] libmachine: Parsing certificate...
I0226 03:25:38.906851 25643 cli_runner.go:164] Run: docker network inspect old-k8s-version-326000 --format "{"Name": "{{.Name}}","Driver": "{{.Driver}}","Subnet": "{{range .IPAM.Config}}{{.Subnet}}{{end}}","Gateway": "{{range .IPAM.Config}}{{.Gateway}}{{end}}","MTU": {{if (index .Options "com.docker.network.driver.mtu")}}{{(index .Options "com.docker.network.driver.mtu")}}{{else}}0{{end}}, "ContainerIPs": [{{range $k,$v := .Containers }}"{{$v.IPv4Address}}",{{end}}]}"
W0226 03:25:38.963168 25643 cli_runner.go:211] docker network inspect old-k8s-version-326000 --format "{"Name": "{{.Name}}","Driver": "{{.Driver}}","Subnet": "{{range .IPAM.Config}}{{.Subnet}}{{end}}","Gateway": "{{range .IPAM.Config}}{{.Gateway}}{{end}}","MTU": {{if (index .Options "com.docker.network.driver.mtu")}}{{(index .Options "com.docker.network.driver.mtu")}}{{else}}0{{end}}, "ContainerIPs": [{{range $k,$v := .Containers }}"{{$v.IPv4Address}}",{{end}}]}" returned with exit code 1
I0226 03:25:38.963277 25643 network_create.go:281] running [docker network inspect old-k8s-version-326000] to gather additional debugging logs...
I0226 03:25:38.963298 25643 cli_runner.go:164] Run: docker network inspect old-k8s-version-326000
W0226 03:25:39.015518 25643 cli_runner.go:211] docker network inspect old-k8s-version-326000 returned with exit code 1
I0226 03:25:39.015555 25643 network_create.go:284] error running [docker network inspect old-k8s-version-326000]: docker network inspect old-k8s-version-326000: exit status 1
stdout:
[]
stderr:
Error response from daemon: network old-k8s-version-326000 not found
I0226 03:25:39.015566 25643 network_create.go:286] output of [docker network inspect old-k8s-version-326000]: -- stdout --
[]
-- /stdout --
** stderr **
Error response from daemon: network old-k8s-version-326000 not found
** /stderr **
I0226 03:25:39.015701 25643 cli_runner.go:164] Run: docker network inspect bridge --format "{"Name": "{{.Name}}","Driver": "{{.Driver}}","Subnet": "{{range .IPAM.Config}}{{.Subnet}}{{end}}","Gateway": "{{range .IPAM.Config}}{{.Gateway}}{{end}}","MTU": {{if (index .Options "com.docker.network.driver.mtu")}}{{(index .Options "com.docker.network.driver.mtu")}}{{else}}0{{end}}, "ContainerIPs": [{{range $k,$v := .Containers }}"{{$v.IPv4Address}}",{{end}}]}"
I0226 03:25:39.070889 25643 network.go:210] skipping subnet 192.168.49.0/24 that is reserved: &{IP:192.168.49.0 Netmask:255.255.255.0 Prefix:24 CIDR:192.168.49.0/24 Gateway:192.168.49.1 ClientMin:192.168.49.2 ClientMax:192.168.49.254 Broadcast:192.168.49.255 IsPrivate:true Interface:{IfaceName: IfaceIPv4: IfaceMTU:0 IfaceMAC:} reservation:<nil>}
I0226 03:25:39.072512 25643 network.go:210] skipping subnet 192.168.58.0/24 that is reserved: &{IP:192.168.58.0 Netmask:255.255.255.0 Prefix:24 CIDR:192.168.58.0/24 Gateway:192.168.58.1 ClientMin:192.168.58.2 ClientMax:192.168.58.254 Broadcast:192.168.58.255 IsPrivate:true Interface:{IfaceName: IfaceIPv4: IfaceMTU:0 IfaceMAC:} reservation:<nil>}
I0226 03:25:39.073861 25643 network.go:210] skipping subnet 192.168.67.0/24 that is reserved: &{IP:192.168.67.0 Netmask:255.255.255.0 Prefix:24 CIDR:192.168.67.0/24 Gateway:192.168.67.1 ClientMin:192.168.67.2 ClientMax:192.168.67.254 Broadcast:192.168.67.255 IsPrivate:true Interface:{IfaceName: IfaceIPv4: IfaceMTU:0 IfaceMAC:} reservation:<nil>}
I0226 03:25:39.075186 25643 network.go:210] skipping subnet 192.168.76.0/24 that is reserved: &{IP:192.168.76.0 Netmask:255.255.255.0 Prefix:24 CIDR:192.168.76.0/24 Gateway:192.168.76.1 ClientMin:192.168.76.2 ClientMax:192.168.76.254 Broadcast:192.168.76.255 IsPrivate:true Interface:{IfaceName: IfaceIPv4: IfaceMTU:0 IfaceMAC:} reservation:<nil>}
I0226 03:25:39.075558 25643 network.go:207] using free private subnet 192.168.85.0/24: &{IP:192.168.85.0 Netmask:255.255.255.0 Prefix:24 CIDR:192.168.85.0/24 Gateway:192.168.85.1 ClientMin:192.168.85.2 ClientMax:192.168.85.254 Broadcast:192.168.85.255 IsPrivate:true Interface:{IfaceName: IfaceIPv4: IfaceMTU:0 IfaceMAC:} reservation:0xc0024083f0}
I0226 03:25:39.075576 25643 network_create.go:124] attempt to create docker network old-k8s-version-326000 192.168.85.0/24 with gateway 192.168.85.1 and MTU of 65535 ...
I0226 03:25:39.075647 25643 cli_runner.go:164] Run: docker network create --driver=bridge --subnet=192.168.85.0/24 --gateway=192.168.85.1 -o --ip-masq -o --icc -o com.docker.network.driver.mtu=65535 --label=created_by.minikube.sigs.k8s.io=true --label=name.minikube.sigs.k8s.io=old-k8s-version-326000 old-k8s-version-326000
I0226 03:25:39.162849 25643 network_create.go:108] docker network old-k8s-version-326000 192.168.85.0/24 created
I0226 03:25:39.162895 25643 kic.go:121] calculated static IP "192.168.85.2" for the "old-k8s-version-326000" container
I0226 03:25:39.162998 25643 cli_runner.go:164] Run: docker ps -a --format {{.Names}}
I0226 03:25:39.213222 25643 cli_runner.go:164] Run: docker volume create old-k8s-version-326000 --label name.minikube.sigs.k8s.io=old-k8s-version-326000 --label created_by.minikube.sigs.k8s.io=true
I0226 03:25:39.263456 25643 oci.go:103] Successfully created a docker volume old-k8s-version-326000
I0226 03:25:39.263568 25643 cli_runner.go:164] Run: docker run --rm --name old-k8s-version-326000-preload-sidecar --label created_by.minikube.sigs.k8s.io=true --label name.minikube.sigs.k8s.io=old-k8s-version-326000 --entrypoint /usr/bin/test -v old-k8s-version-326000:/var gcr.io/k8s-minikube/kicbase-builds:v0.0.42-1708008208-17936@sha256:4ea1136332ba1476cda33a97bf12e2f96995cc120674fbafd3ade22d1118ecdf -d /var/lib
I0226 03:25:39.718840 25643 oci.go:107] Successfully prepared a docker volume old-k8s-version-326000
I0226 03:25:39.718929 25643 preload.go:132] Checking if preload exists for k8s version v1.16.0 and runtime docker
I0226 03:25:39.718960 25643 kic.go:194] Starting extracting preloaded images to volume ...
I0226 03:25:39.719110 25643 cli_runner.go:164] Run: docker run --rm --entrypoint /usr/bin/tar -v /Users/jenkins/minikube-integration/18222-9538/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.16.0-docker-overlay2-amd64.tar.lz4:/preloaded.tar:ro -v old-k8s-version-326000:/extractDir gcr.io/k8s-minikube/kicbase-builds:v0.0.42-1708008208-17936@sha256:4ea1136332ba1476cda33a97bf12e2f96995cc120674fbafd3ade22d1118ecdf -I lz4 -xf /preloaded.tar -C /extractDir
I0226 03:25:41.783454 25643 cli_runner.go:217] Completed: docker run --rm --entrypoint /usr/bin/tar -v /Users/jenkins/minikube-integration/18222-9538/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.16.0-docker-overlay2-amd64.tar.lz4:/preloaded.tar:ro -v old-k8s-version-326000:/extractDir gcr.io/k8s-minikube/kicbase-builds:v0.0.42-1708008208-17936@sha256:4ea1136332ba1476cda33a97bf12e2f96995cc120674fbafd3ade22d1118ecdf -I lz4 -xf /preloaded.tar -C /extractDir: (2.064261991s)
I0226 03:25:41.783491 25643 kic.go:203] duration metric: took 2.064524 seconds to extract preloaded images to volume
I0226 03:25:41.783615 25643 cli_runner.go:164] Run: docker info --format "'{{json .SecurityOptions}}'"
I0226 03:25:41.901528 25643 cli_runner.go:164] Run: docker run -d -t --privileged --security-opt seccomp=unconfined --tmpfs /tmp --tmpfs /run -v /lib/modules:/lib/modules:ro --hostname old-k8s-version-326000 --name old-k8s-version-326000 --label created_by.minikube.sigs.k8s.io=true --label name.minikube.sigs.k8s.io=old-k8s-version-326000 --label role.minikube.sigs.k8s.io= --label mode.minikube.sigs.k8s.io=old-k8s-version-326000 --network old-k8s-version-326000 --ip 192.168.85.2 --volume old-k8s-version-326000:/var --security-opt apparmor=unconfined --memory=2200mb --memory-swap=2200mb --cpus=2 -e container=docker --expose 8443 --publish=127.0.0.1::8443 --publish=127.0.0.1::22 --publish=127.0.0.1::2376 --publish=127.0.0.1::5000 --publish=127.0.0.1::32443 gcr.io/k8s-minikube/kicbase-builds:v0.0.42-1708008208-17936@sha256:4ea1136332ba1476cda33a97bf12e2f96995cc120674fbafd3ade22d1118ecdf
I0226 03:25:42.207089 25643 cli_runner.go:164] Run: docker container inspect old-k8s-version-326000 --format={{.State.Running}}
I0226 03:25:42.270013 25643 cli_runner.go:164] Run: docker container inspect old-k8s-version-326000 --format={{.State.Status}}
I0226 03:25:42.327912 25643 cli_runner.go:164] Run: docker exec old-k8s-version-326000 stat /var/lib/dpkg/alternatives/iptables
I0226 03:25:42.441698 25643 oci.go:144] the created container "old-k8s-version-326000" has a running status.
I0226 03:25:42.441741 25643 kic.go:225] Creating ssh key for kic: /Users/jenkins/minikube-integration/18222-9538/.minikube/machines/old-k8s-version-326000/id_rsa...
I0226 03:25:42.523687 25643 kic_runner.go:191] docker (temp): /Users/jenkins/minikube-integration/18222-9538/.minikube/machines/old-k8s-version-326000/id_rsa.pub --> /home/docker/.ssh/authorized_keys (381 bytes)
I0226 03:25:42.588459 25643 cli_runner.go:164] Run: docker container inspect old-k8s-version-326000 --format={{.State.Status}}
I0226 03:25:42.642547 25643 kic_runner.go:93] Run: chown docker:docker /home/docker/.ssh/authorized_keys
I0226 03:25:42.642569 25643 kic_runner.go:114] Args: [docker exec --privileged old-k8s-version-326000 chown docker:docker /home/docker/.ssh/authorized_keys]
I0226 03:25:42.744930 25643 cli_runner.go:164] Run: docker container inspect old-k8s-version-326000 --format={{.State.Status}}
I0226 03:25:42.810539 25643 machine.go:88] provisioning docker machine ...
I0226 03:25:42.810598 25643 ubuntu.go:169] provisioning hostname "old-k8s-version-326000"
I0226 03:25:42.810703 25643 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" old-k8s-version-326000
I0226 03:25:42.862134 25643 main.go:141] libmachine: Using SSH client type: native
I0226 03:25:42.862350 25643 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xd19b920] 0xd19e680 <nil> [] 0s} 127.0.0.1 61670 <nil> <nil>}
I0226 03:25:42.862362 25643 main.go:141] libmachine: About to run SSH command:
sudo hostname old-k8s-version-326000 && echo "old-k8s-version-326000" | sudo tee /etc/hostname
I0226 03:25:43.030542 25643 main.go:141] libmachine: SSH cmd err, output: <nil>: old-k8s-version-326000
I0226 03:25:43.030635 25643 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" old-k8s-version-326000
I0226 03:25:43.088336 25643 main.go:141] libmachine: Using SSH client type: native
I0226 03:25:43.088525 25643 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xd19b920] 0xd19e680 <nil> [] 0s} 127.0.0.1 61670 <nil> <nil>}
I0226 03:25:43.088541 25643 main.go:141] libmachine: About to run SSH command:
if ! grep -xq '.*\sold-k8s-version-326000' /etc/hosts; then
if grep -xq '127.0.1.1\s.*' /etc/hosts; then
sudo sed -i 's/^127.0.1.1\s.*/127.0.1.1 old-k8s-version-326000/g' /etc/hosts;
else
echo '127.0.1.1 old-k8s-version-326000' | sudo tee -a /etc/hosts;
fi
fi
I0226 03:25:43.227422 25643 main.go:141] libmachine: SSH cmd err, output: <nil>:
I0226 03:25:43.227449 25643 ubuntu.go:175] set auth options {CertDir:/Users/jenkins/minikube-integration/18222-9538/.minikube CaCertPath:/Users/jenkins/minikube-integration/18222-9538/.minikube/certs/ca.pem CaPrivateKeyPath:/Users/jenkins/minikube-integration/18222-9538/.minikube/certs/ca-key.pem CaCertRemotePath:/etc/docker/ca.pem ServerCertPath:/Users/jenkins/minikube-integration/18222-9538/.minikube/machines/server.pem ServerKeyPath:/Users/jenkins/minikube-integration/18222-9538/.minikube/machines/server-key.pem ClientKeyPath:/Users/jenkins/minikube-integration/18222-9538/.minikube/certs/key.pem ServerCertRemotePath:/etc/docker/server.pem ServerKeyRemotePath:/etc/docker/server-key.pem ClientCertPath:/Users/jenkins/minikube-integration/18222-9538/.minikube/certs/cert.pem ServerCertSANs:[] StorePath:/Users/jenkins/minikube-integration/18222-9538/.minikube}
I0226 03:25:43.227478 25643 ubuntu.go:177] setting up certificates
I0226 03:25:43.227484 25643 provision.go:83] configureAuth start
I0226 03:25:43.227562 25643 cli_runner.go:164] Run: docker container inspect -f "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}" old-k8s-version-326000
I0226 03:25:43.285063 25643 provision.go:138] copyHostCerts
I0226 03:25:43.285182 25643 exec_runner.go:144] found /Users/jenkins/minikube-integration/18222-9538/.minikube/ca.pem, removing ...
I0226 03:25:43.285218 25643 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/18222-9538/.minikube/ca.pem
I0226 03:25:43.285429 25643 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/18222-9538/.minikube/certs/ca.pem --> /Users/jenkins/minikube-integration/18222-9538/.minikube/ca.pem (1082 bytes)
I0226 03:25:43.285663 25643 exec_runner.go:144] found /Users/jenkins/minikube-integration/18222-9538/.minikube/cert.pem, removing ...
I0226 03:25:43.285671 25643 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/18222-9538/.minikube/cert.pem
I0226 03:25:43.285906 25643 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/18222-9538/.minikube/certs/cert.pem --> /Users/jenkins/minikube-integration/18222-9538/.minikube/cert.pem (1123 bytes)
I0226 03:25:43.286116 25643 exec_runner.go:144] found /Users/jenkins/minikube-integration/18222-9538/.minikube/key.pem, removing ...
I0226 03:25:43.286123 25643 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/18222-9538/.minikube/key.pem
I0226 03:25:43.286208 25643 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/18222-9538/.minikube/certs/key.pem --> /Users/jenkins/minikube-integration/18222-9538/.minikube/key.pem (1675 bytes)
I0226 03:25:43.286359 25643 provision.go:112] generating server cert: /Users/jenkins/minikube-integration/18222-9538/.minikube/machines/server.pem ca-key=/Users/jenkins/minikube-integration/18222-9538/.minikube/certs/ca.pem private-key=/Users/jenkins/minikube-integration/18222-9538/.minikube/certs/ca-key.pem org=jenkins.old-k8s-version-326000 san=[192.168.85.2 127.0.0.1 localhost 127.0.0.1 minikube old-k8s-version-326000]
I0226 03:25:43.381273 25643 provision.go:172] copyRemoteCerts
I0226 03:25:43.381361 25643 ssh_runner.go:195] Run: sudo mkdir -p /etc/docker /etc/docker /etc/docker
I0226 03:25:43.381450 25643 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" old-k8s-version-326000
I0226 03:25:43.434391 25643 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:61670 SSHKeyPath:/Users/jenkins/minikube-integration/18222-9538/.minikube/machines/old-k8s-version-326000/id_rsa Username:docker}
I0226 03:25:43.539450 25643 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/18222-9538/.minikube/certs/ca.pem --> /etc/docker/ca.pem (1082 bytes)
I0226 03:25:43.580147 25643 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/18222-9538/.minikube/machines/server.pem --> /etc/docker/server.pem (1241 bytes)
I0226 03:25:43.620399 25643 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/18222-9538/.minikube/machines/server-key.pem --> /etc/docker/server-key.pem (1679 bytes)
I0226 03:25:43.661190 25643 provision.go:86] duration metric: configureAuth took 433.686375ms
I0226 03:25:43.661209 25643 ubuntu.go:193] setting minikube options for container-runtime
I0226 03:25:43.661382 25643 config.go:182] Loaded profile config "old-k8s-version-326000": Driver=docker, ContainerRuntime=docker, KubernetesVersion=v1.16.0
I0226 03:25:43.661456 25643 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" old-k8s-version-326000
I0226 03:25:43.712346 25643 main.go:141] libmachine: Using SSH client type: native
I0226 03:25:43.712509 25643 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xd19b920] 0xd19e680 <nil> [] 0s} 127.0.0.1 61670 <nil> <nil>}
I0226 03:25:43.712526 25643 main.go:141] libmachine: About to run SSH command:
df --output=fstype / | tail -n 1
I0226 03:25:43.849947 25643 main.go:141] libmachine: SSH cmd err, output: <nil>: overlay
I0226 03:25:43.849959 25643 ubuntu.go:71] root file system type: overlay
I0226 03:25:43.850031 25643 provision.go:309] Updating docker unit: /lib/systemd/system/docker.service ...
I0226 03:25:43.850115 25643 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" old-k8s-version-326000
I0226 03:25:43.901021 25643 main.go:141] libmachine: Using SSH client type: native
I0226 03:25:43.901207 25643 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xd19b920] 0xd19e680 <nil> [] 0s} 127.0.0.1 61670 <nil> <nil>}
I0226 03:25:43.901257 25643 main.go:141] libmachine: About to run SSH command:
sudo mkdir -p /lib/systemd/system && printf %s "[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
BindsTo=containerd.service
After=network-online.target firewalld.service containerd.service
Wants=network-online.target
Requires=docker.socket
StartLimitBurst=3
StartLimitIntervalSec=60
[Service]
Type=notify
Restart=on-failure
# This file is a systemd drop-in unit that inherits from the base dockerd configuration.
# The base configuration already specifies an 'ExecStart=...' command. The first directive
# here is to clear out that command inherited from the base configuration. Without this,
# the command from the base configuration and the command specified here are treated as
# a sequence of commands, which is not the desired behavior, nor is it valid -- systemd
# will catch this invalid input and refuse to start the service with an error like:
# Service has more than one ExecStart= setting, which is only allowed for Type=oneshot services.
# NOTE: default-ulimit=nofile is set to an arbitrary number for consistency with other
# container runtimes. If left unlimited, it may result in OOM issues with MySQL.
ExecStart=
ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --default-ulimit=nofile=1048576:1048576 --tlsverify --tlscacert /etc/docker/ca.pem --tlscert /etc/docker/server.pem --tlskey /etc/docker/server-key.pem --label provider=docker --insecure-registry 10.96.0.0/12
ExecReload=/bin/kill -s HUP \$MAINPID
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
TimeoutStartSec=0
# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes
# kill only the docker process, not all processes in the cgroup
KillMode=process
[Install]
WantedBy=multi-user.target
" | sudo tee /lib/systemd/system/docker.service.new
I0226 03:25:44.063904 25643 main.go:141] libmachine: SSH cmd err, output: <nil>: [Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
BindsTo=containerd.service
After=network-online.target firewalld.service containerd.service
Wants=network-online.target
Requires=docker.socket
StartLimitBurst=3
StartLimitIntervalSec=60
[Service]
Type=notify
Restart=on-failure
# This file is a systemd drop-in unit that inherits from the base dockerd configuration.
# The base configuration already specifies an 'ExecStart=...' command. The first directive
# here is to clear out that command inherited from the base configuration. Without this,
# the command from the base configuration and the command specified here are treated as
# a sequence of commands, which is not the desired behavior, nor is it valid -- systemd
# will catch this invalid input and refuse to start the service with an error like:
# Service has more than one ExecStart= setting, which is only allowed for Type=oneshot services.
# NOTE: default-ulimit=nofile is set to an arbitrary number for consistency with other
# container runtimes. If left unlimited, it may result in OOM issues with MySQL.
ExecStart=
ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --default-ulimit=nofile=1048576:1048576 --tlsverify --tlscacert /etc/docker/ca.pem --tlscert /etc/docker/server.pem --tlskey /etc/docker/server-key.pem --label provider=docker --insecure-registry 10.96.0.0/12
ExecReload=/bin/kill -s HUP $MAINPID
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
TimeoutStartSec=0
# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes
# kill only the docker process, not all processes in the cgroup
KillMode=process
[Install]
WantedBy=multi-user.target
I0226 03:25:44.064037 25643 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" old-k8s-version-326000
I0226 03:25:44.116096 25643 main.go:141] libmachine: Using SSH client type: native
I0226 03:25:44.116289 25643 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xd19b920] 0xd19e680 <nil> [] 0s} 127.0.0.1 61670 <nil> <nil>}
I0226 03:25:44.116302 25643 main.go:141] libmachine: About to run SSH command:
sudo diff -u /lib/systemd/system/docker.service /lib/systemd/system/docker.service.new || { sudo mv /lib/systemd/system/docker.service.new /lib/systemd/system/docker.service; sudo systemctl -f daemon-reload && sudo systemctl -f enable docker && sudo systemctl -f restart docker; }
I0226 03:25:44.787608 25643 main.go:141] libmachine: SSH cmd err, output: <nil>: --- /lib/systemd/system/docker.service 2024-02-06 21:12:51.000000000 +0000
+++ /lib/systemd/system/docker.service.new 2024-02-26 11:25:44.057771063 +0000
@@ -1,46 +1,49 @@
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
-After=network-online.target docker.socket firewalld.service containerd.service time-set.target
-Wants=network-online.target containerd.service
+BindsTo=containerd.service
+After=network-online.target firewalld.service containerd.service
+Wants=network-online.target
Requires=docker.socket
+StartLimitBurst=3
+StartLimitIntervalSec=60
[Service]
Type=notify
-# the default is not to use systemd for cgroups because the delegate issues still
-# exists and systemd currently does not support the cgroup feature set required
-# for containers run by docker
-ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock
-ExecReload=/bin/kill -s HUP $MAINPID
-TimeoutStartSec=0
-RestartSec=2
-Restart=always
+Restart=on-failure
-# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229.
-# Both the old, and new location are accepted by systemd 229 and up, so using the old location
-# to make them work for either version of systemd.
-StartLimitBurst=3
-# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230.
-# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make
-# this option work for either version of systemd.
-StartLimitInterval=60s
+
+# This file is a systemd drop-in unit that inherits from the base dockerd configuration.
+# The base configuration already specifies an 'ExecStart=...' command. The first directive
+# here is to clear out that command inherited from the base configuration. Without this,
+# the command from the base configuration and the command specified here are treated as
+# a sequence of commands, which is not the desired behavior, nor is it valid -- systemd
+# will catch this invalid input and refuse to start the service with an error like:
+# Service has more than one ExecStart= setting, which is only allowed for Type=oneshot services.
+
+# NOTE: default-ulimit=nofile is set to an arbitrary number for consistency with other
+# container runtimes. If left unlimited, it may result in OOM issues with MySQL.
+ExecStart=
+ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --default-ulimit=nofile=1048576:1048576 --tlsverify --tlscacert /etc/docker/ca.pem --tlscert /etc/docker/server.pem --tlskey /etc/docker/server-key.pem --label provider=docker --insecure-registry 10.96.0.0/12
+ExecReload=/bin/kill -s HUP $MAINPID
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
-# Comment TasksMax if your systemd version does not support it.
-# Only systemd 226 and above support this option.
+# Uncomment TasksMax if your systemd version supports it.
+# Only systemd 226 and above support this version.
TasksMax=infinity
+TimeoutStartSec=0
# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes
# kill only the docker process, not all processes in the cgroup
KillMode=process
-OOMScoreAdjust=-500
[Install]
WantedBy=multi-user.target
Synchronizing state of docker.service with SysV service script with /lib/systemd/systemd-sysv-install.
Executing: /lib/systemd/systemd-sysv-install enable docker
I0226 03:25:44.787632 25643 machine.go:91] provisioned docker machine in 1.977047677s
I0226 03:25:44.787644 25643 client.go:171] LocalClient.Create took 5.881499942s
I0226 03:25:44.787662 25643 start.go:167] duration metric: libmachine.API.Create for "old-k8s-version-326000" took 5.881580726s
I0226 03:25:44.787670 25643 start.go:300] post-start starting for "old-k8s-version-326000" (driver="docker")
I0226 03:25:44.787678 25643 start.go:329] creating required directories: [/etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs]
I0226 03:25:44.787740 25643 ssh_runner.go:195] Run: sudo mkdir -p /etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs
I0226 03:25:44.787808 25643 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" old-k8s-version-326000
I0226 03:25:44.838905 25643 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:61670 SSHKeyPath:/Users/jenkins/minikube-integration/18222-9538/.minikube/machines/old-k8s-version-326000/id_rsa Username:docker}
I0226 03:25:44.940436 25643 ssh_runner.go:195] Run: cat /etc/os-release
I0226 03:25:44.944438 25643 main.go:141] libmachine: Couldn't set key VERSION_CODENAME, no corresponding struct field found
I0226 03:25:44.944460 25643 main.go:141] libmachine: Couldn't set key PRIVACY_POLICY_URL, no corresponding struct field found
I0226 03:25:44.944467 25643 main.go:141] libmachine: Couldn't set key UBUNTU_CODENAME, no corresponding struct field found
I0226 03:25:44.944474 25643 info.go:137] Remote host: Ubuntu 22.04.3 LTS
I0226 03:25:44.944484 25643 filesync.go:126] Scanning /Users/jenkins/minikube-integration/18222-9538/.minikube/addons for local assets ...
I0226 03:25:44.944589 25643 filesync.go:126] Scanning /Users/jenkins/minikube-integration/18222-9538/.minikube/files for local assets ...
I0226 03:25:44.944790 25643 filesync.go:149] local asset: /Users/jenkins/minikube-integration/18222-9538/.minikube/files/etc/ssl/certs/100262.pem -> 100262.pem in /etc/ssl/certs
I0226 03:25:44.944997 25643 ssh_runner.go:195] Run: sudo mkdir -p /etc/ssl/certs
I0226 03:25:44.959940 25643 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/18222-9538/.minikube/files/etc/ssl/certs/100262.pem --> /etc/ssl/certs/100262.pem (1708 bytes)
I0226 03:25:45.000637 25643 start.go:303] post-start completed in 212.956114ms
I0226 03:25:45.001237 25643 cli_runner.go:164] Run: docker container inspect -f "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}" old-k8s-version-326000
I0226 03:25:45.051926 25643 profile.go:148] Saving config to /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/config.json ...
I0226 03:25:45.052424 25643 ssh_runner.go:195] Run: sh -c "df -h /var | awk 'NR==2{print $5}'"
I0226 03:25:45.052488 25643 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" old-k8s-version-326000
I0226 03:25:45.103023 25643 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:61670 SSHKeyPath:/Users/jenkins/minikube-integration/18222-9538/.minikube/machines/old-k8s-version-326000/id_rsa Username:docker}
I0226 03:25:45.194939 25643 ssh_runner.go:195] Run: sh -c "df -BG /var | awk 'NR==2{print $4}'"
I0226 03:25:45.201680 25643 start.go:128] duration metric: createHost completed in 6.317849216s
I0226 03:25:45.201704 25643 start.go:83] releasing machines lock for "old-k8s-version-326000", held for 6.318002801s
I0226 03:25:45.201835 25643 cli_runner.go:164] Run: docker container inspect -f "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}" old-k8s-version-326000
I0226 03:25:45.259305 25643 ssh_runner.go:195] Run: curl -sS -m 2 https://registry.k8s.io/
I0226 03:25:45.259314 25643 ssh_runner.go:195] Run: cat /version.json
I0226 03:25:45.259396 25643 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" old-k8s-version-326000
I0226 03:25:45.259409 25643 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" old-k8s-version-326000
I0226 03:25:45.319315 25643 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:61670 SSHKeyPath:/Users/jenkins/minikube-integration/18222-9538/.minikube/machines/old-k8s-version-326000/id_rsa Username:docker}
I0226 03:25:45.319305 25643 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:61670 SSHKeyPath:/Users/jenkins/minikube-integration/18222-9538/.minikube/machines/old-k8s-version-326000/id_rsa Username:docker}
I0226 03:25:45.411951 25643 ssh_runner.go:195] Run: systemctl --version
I0226 03:25:45.524624 25643 ssh_runner.go:195] Run: sh -c "stat /etc/cni/net.d/*loopback.conf*"
I0226 03:25:45.531017 25643 ssh_runner.go:195] Run: sudo find /etc/cni/net.d -maxdepth 1 -type f -name *loopback.conf* -not -name *.mk_disabled -exec sh -c "grep -q loopback {} && ( grep -q name {} || sudo sed -i '/"type": "loopback"/i \ \ \ \ "name": "loopback",' {} ) && sudo sed -i 's|"cniVersion": ".*"|"cniVersion": "1.0.0"|g' {}" ;
I0226 03:25:45.580024 25643 cni.go:230] loopback cni configuration patched: "/etc/cni/net.d/*loopback.conf*" found
I0226 03:25:45.580119 25643 ssh_runner.go:195] Run: sudo find /etc/cni/net.d -maxdepth 1 -type f -name *bridge* -not -name *podman* -not -name *.mk_disabled -printf "%p, " -exec sh -c "sudo sed -i -r -e '/"dst": ".*:.*"/d' -e 's|^(.*)"dst": (.*)[,*]$|\1"dst": \2|g' -e '/"subnet": ".*:.*"/d' -e 's|^(.*)"subnet": ".*"(.*)[,*]$|\1"subnet": "10.244.0.0/16"\2|g' {}" ;
I0226 03:25:45.608786 25643 ssh_runner.go:195] Run: sudo find /etc/cni/net.d -maxdepth 1 -type f -name *podman* -not -name *.mk_disabled -printf "%p, " -exec sh -c "sudo sed -i -r -e 's|^(.*)"subnet": ".*"(.*)$|\1"subnet": "10.244.0.0/16"\2|g' -e 's|^(.*)"gateway": ".*"(.*)$|\1"gateway": "10.244.0.1"\2|g' {}" ;
I0226 03:25:45.641294 25643 cni.go:308] configured [/etc/cni/net.d/100-crio-bridge.conf, /etc/cni/net.d/87-podman-bridge.conflist] bridge cni config(s)
I0226 03:25:45.641315 25643 start.go:475] detecting cgroup driver to use...
I0226 03:25:45.641350 25643 detect.go:196] detected "cgroupfs" cgroup driver on host os
I0226 03:25:45.641542 25643 ssh_runner.go:195] Run: /bin/bash -c "sudo mkdir -p /etc && printf %s "runtime-endpoint: unix:///run/containerd/containerd.sock
" | sudo tee /etc/crictl.yaml"
I0226 03:25:45.672646 25643 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)sandbox_image = .*$|\1sandbox_image = "registry.k8s.io/pause:3.1"|' /etc/containerd/config.toml"
I0226 03:25:45.690593 25643 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)restrict_oom_score_adj = .*$|\1restrict_oom_score_adj = false|' /etc/containerd/config.toml"
I0226 03:25:45.707046 25643 containerd.go:146] configuring containerd to use "cgroupfs" as cgroup driver...
I0226 03:25:45.707109 25643 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)SystemdCgroup = .*$|\1SystemdCgroup = false|g' /etc/containerd/config.toml"
I0226 03:25:45.723177 25643 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|"io.containerd.runtime.v1.linux"|"io.containerd.runc.v2"|g' /etc/containerd/config.toml"
I0226 03:25:45.739267 25643 ssh_runner.go:195] Run: sh -c "sudo sed -i '/systemd_cgroup/d' /etc/containerd/config.toml"
I0226 03:25:45.755337 25643 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|"io.containerd.runc.v1"|"io.containerd.runc.v2"|g' /etc/containerd/config.toml"
I0226 03:25:45.771298 25643 ssh_runner.go:195] Run: sh -c "sudo rm -rf /etc/cni/net.mk"
I0226 03:25:45.787697 25643 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)conf_dir = .*$|\1conf_dir = "/etc/cni/net.d"|g' /etc/containerd/config.toml"
I0226 03:25:45.804513 25643 ssh_runner.go:195] Run: sudo sysctl net.bridge.bridge-nf-call-iptables
I0226 03:25:45.819540 25643 ssh_runner.go:195] Run: sudo sh -c "echo 1 > /proc/sys/net/ipv4/ip_forward"
I0226 03:25:45.834597 25643 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I0226 03:25:45.897490 25643 ssh_runner.go:195] Run: sudo systemctl restart containerd
I0226 03:25:45.988156 25643 start.go:475] detecting cgroup driver to use...
I0226 03:25:45.988184 25643 detect.go:196] detected "cgroupfs" cgroup driver on host os
I0226 03:25:45.988255 25643 ssh_runner.go:195] Run: sudo systemctl cat docker.service
I0226 03:25:46.011182 25643 cruntime.go:279] skipping containerd shutdown because we are bound to it
I0226 03:25:46.011248 25643 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service crio
I0226 03:25:46.030202 25643 ssh_runner.go:195] Run: /bin/bash -c "sudo mkdir -p /etc && printf %s "runtime-endpoint: unix:///var/run/dockershim.sock
" | sudo tee /etc/crictl.yaml"
I0226 03:25:46.060565 25643 ssh_runner.go:195] Run: which cri-dockerd
I0226 03:25:46.065179 25643 ssh_runner.go:195] Run: sudo mkdir -p /etc/systemd/system/cri-docker.service.d
I0226 03:25:46.081616 25643 ssh_runner.go:362] scp memory --> /etc/systemd/system/cri-docker.service.d/10-cni.conf (189 bytes)
I0226 03:25:46.112565 25643 ssh_runner.go:195] Run: sudo systemctl unmask docker.service
I0226 03:25:46.176244 25643 ssh_runner.go:195] Run: sudo systemctl enable docker.socket
I0226 03:25:46.269773 25643 docker.go:574] configuring docker to use "cgroupfs" as cgroup driver...
I0226 03:25:46.269874 25643 ssh_runner.go:362] scp memory --> /etc/docker/daemon.json (130 bytes)
I0226 03:25:46.302031 25643 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I0226 03:25:46.372906 25643 ssh_runner.go:195] Run: sudo systemctl restart docker
I0226 03:25:46.667012 25643 ssh_runner.go:195] Run: docker version --format {{.Server.Version}}
I0226 03:25:46.693486 25643 ssh_runner.go:195] Run: docker version --format {{.Server.Version}}
I0226 03:25:46.762639 25643 out.go:204] * Preparing Kubernetes v1.16.0 on Docker 25.0.3 ...
I0226 03:25:46.762747 25643 cli_runner.go:164] Run: docker exec -t old-k8s-version-326000 dig +short host.docker.internal
I0226 03:25:46.870855 25643 network.go:96] got host ip for mount in container by digging dns: 192.168.65.254
I0226 03:25:46.870966 25643 ssh_runner.go:195] Run: grep 192.168.65.254 host.minikube.internal$ /etc/hosts
I0226 03:25:46.875760 25643 ssh_runner.go:195] Run: /bin/bash -c "{ grep -v $'\thost.minikube.internal$' "/etc/hosts"; echo "192.168.65.254 host.minikube.internal"; } > /tmp/h.$$; sudo cp /tmp/h.$$ "/etc/hosts""
I0226 03:25:46.893250 25643 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "8443/tcp") 0).HostPort}}'" old-k8s-version-326000
I0226 03:25:46.944077 25643 preload.go:132] Checking if preload exists for k8s version v1.16.0 and runtime docker
I0226 03:25:46.944155 25643 ssh_runner.go:195] Run: docker images --format {{.Repository}}:{{.Tag}}
I0226 03:25:46.963743 25643 docker.go:685] Got preloaded images: -- stdout --
gcr.io/k8s-minikube/storage-provisioner:v5
k8s.gcr.io/kube-apiserver:v1.16.0
k8s.gcr.io/kube-proxy:v1.16.0
k8s.gcr.io/kube-controller-manager:v1.16.0
k8s.gcr.io/kube-scheduler:v1.16.0
k8s.gcr.io/etcd:3.3.15-0
k8s.gcr.io/coredns:1.6.2
k8s.gcr.io/pause:3.1
-- /stdout --
I0226 03:25:46.963756 25643 docker.go:691] registry.k8s.io/kube-apiserver:v1.16.0 wasn't preloaded
I0226 03:25:46.963816 25643 ssh_runner.go:195] Run: sudo cat /var/lib/docker/image/overlay2/repositories.json
I0226 03:25:46.979453 25643 ssh_runner.go:195] Run: which lz4
I0226 03:25:46.983845 25643 ssh_runner.go:195] Run: stat -c "%s %y" /preloaded.tar.lz4
I0226 03:25:46.988157 25643 ssh_runner.go:352] existence check for /preloaded.tar.lz4: stat -c "%s %y" /preloaded.tar.lz4: Process exited with status 1
stdout:
stderr:
stat: cannot statx '/preloaded.tar.lz4': No such file or directory
I0226 03:25:46.988179 25643 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/18222-9538/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.16.0-docker-overlay2-amd64.tar.lz4 --> /preloaded.tar.lz4 (369789069 bytes)
I0226 03:25:54.040766 25643 docker.go:649] Took 7.056919 seconds to copy over tarball
I0226 03:25:54.040879 25643 ssh_runner.go:195] Run: sudo tar --xattrs --xattrs-include security.capability -I lz4 -C /var -xf /preloaded.tar.lz4
I0226 03:25:55.908537 25643 ssh_runner.go:235] Completed: sudo tar --xattrs --xattrs-include security.capability -I lz4 -C /var -xf /preloaded.tar.lz4: (1.867613134s)
I0226 03:25:55.908557 25643 ssh_runner.go:146] rm: /preloaded.tar.lz4
I0226 03:25:55.968541 25643 ssh_runner.go:195] Run: sudo cat /var/lib/docker/image/overlay2/repositories.json
I0226 03:25:55.985397 25643 ssh_runner.go:362] scp memory --> /var/lib/docker/image/overlay2/repositories.json (2499 bytes)
I0226 03:25:56.013942 25643 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I0226 03:25:56.099456 25643 ssh_runner.go:195] Run: sudo systemctl restart docker
I0226 03:25:56.623300 25643 ssh_runner.go:195] Run: docker images --format {{.Repository}}:{{.Tag}}
I0226 03:25:56.650170 25643 docker.go:685] Got preloaded images: -- stdout --
gcr.io/k8s-minikube/storage-provisioner:v5
k8s.gcr.io/kube-apiserver:v1.16.0
k8s.gcr.io/kube-proxy:v1.16.0
k8s.gcr.io/kube-controller-manager:v1.16.0
k8s.gcr.io/kube-scheduler:v1.16.0
k8s.gcr.io/etcd:3.3.15-0
k8s.gcr.io/coredns:1.6.2
k8s.gcr.io/pause:3.1
-- /stdout --
I0226 03:25:56.650194 25643 docker.go:691] registry.k8s.io/kube-apiserver:v1.16.0 wasn't preloaded
I0226 03:25:56.650203 25643 cache_images.go:88] LoadImages start: [registry.k8s.io/kube-apiserver:v1.16.0 registry.k8s.io/kube-controller-manager:v1.16.0 registry.k8s.io/kube-scheduler:v1.16.0 registry.k8s.io/kube-proxy:v1.16.0 registry.k8s.io/pause:3.1 registry.k8s.io/etcd:3.3.15-0 registry.k8s.io/coredns:1.6.2 gcr.io/k8s-minikube/storage-provisioner:v5]
I0226 03:25:56.657023 25643 image.go:134] retrieving image: gcr.io/k8s-minikube/storage-provisioner:v5
I0226 03:25:56.657816 25643 image.go:134] retrieving image: registry.k8s.io/coredns:1.6.2
I0226 03:25:56.657874 25643 image.go:134] retrieving image: registry.k8s.io/kube-proxy:v1.16.0
I0226 03:25:56.658232 25643 image.go:134] retrieving image: registry.k8s.io/kube-controller-manager:v1.16.0
I0226 03:25:56.659763 25643 image.go:134] retrieving image: registry.k8s.io/etcd:3.3.15-0
I0226 03:25:56.660771 25643 image.go:134] retrieving image: registry.k8s.io/pause:3.1
I0226 03:25:56.661353 25643 image.go:134] retrieving image: registry.k8s.io/kube-apiserver:v1.16.0
I0226 03:25:56.662055 25643 image.go:134] retrieving image: registry.k8s.io/kube-scheduler:v1.16.0
I0226 03:25:56.663547 25643 image.go:177] daemon lookup for registry.k8s.io/coredns:1.6.2: Error response from daemon: No such image: registry.k8s.io/coredns:1.6.2
I0226 03:25:56.665562 25643 image.go:177] daemon lookup for registry.k8s.io/kube-controller-manager:v1.16.0: Error response from daemon: No such image: registry.k8s.io/kube-controller-manager:v1.16.0
I0226 03:25:56.665807 25643 image.go:177] daemon lookup for registry.k8s.io/kube-proxy:v1.16.0: Error response from daemon: No such image: registry.k8s.io/kube-proxy:v1.16.0
I0226 03:25:56.666134 25643 image.go:177] daemon lookup for gcr.io/k8s-minikube/storage-provisioner:v5: Error response from daemon: No such image: gcr.io/k8s-minikube/storage-provisioner:v5
I0226 03:25:56.667030 25643 image.go:177] daemon lookup for registry.k8s.io/etcd:3.3.15-0: Error response from daemon: No such image: registry.k8s.io/etcd:3.3.15-0
I0226 03:25:56.668582 25643 image.go:177] daemon lookup for registry.k8s.io/kube-scheduler:v1.16.0: Error response from daemon: No such image: registry.k8s.io/kube-scheduler:v1.16.0
I0226 03:25:56.668732 25643 image.go:177] daemon lookup for registry.k8s.io/kube-apiserver:v1.16.0: Error response from daemon: No such image: registry.k8s.io/kube-apiserver:v1.16.0
I0226 03:25:56.668924 25643 image.go:177] daemon lookup for registry.k8s.io/pause:3.1: Error response from daemon: No such image: registry.k8s.io/pause:3.1
I0226 03:25:58.642598 25643 ssh_runner.go:195] Run: docker image inspect --format {{.Id}} registry.k8s.io/kube-controller-manager:v1.16.0
I0226 03:25:58.668329 25643 cache_images.go:116] "registry.k8s.io/kube-controller-manager:v1.16.0" needs transfer: "registry.k8s.io/kube-controller-manager:v1.16.0" does not exist at hash "06a629a7e51cdcc81a5ed6a3e6650348312f20c954ac52ee489a023628ec9c7d" in container runtime
I0226 03:25:58.668405 25643 docker.go:337] Removing image: registry.k8s.io/kube-controller-manager:v1.16.0
I0226 03:25:58.668480 25643 ssh_runner.go:195] Run: docker rmi registry.k8s.io/kube-controller-manager:v1.16.0
I0226 03:25:58.686776 25643 cache_images.go:286] Loading image from: /Users/jenkins/minikube-integration/18222-9538/.minikube/cache/images/amd64/registry.k8s.io/kube-controller-manager_v1.16.0
I0226 03:25:58.723429 25643 ssh_runner.go:195] Run: docker image inspect --format {{.Id}} registry.k8s.io/etcd:3.3.15-0
I0226 03:25:58.746924 25643 cache_images.go:116] "registry.k8s.io/etcd:3.3.15-0" needs transfer: "registry.k8s.io/etcd:3.3.15-0" does not exist at hash "b2756210eeabf84f3221da9959e9483f3919dc2aaab4cd45e7cd072fcbde27ed" in container runtime
I0226 03:25:58.746960 25643 docker.go:337] Removing image: registry.k8s.io/etcd:3.3.15-0
I0226 03:25:58.747045 25643 ssh_runner.go:195] Run: docker rmi registry.k8s.io/etcd:3.3.15-0
I0226 03:25:58.768185 25643 cache_images.go:286] Loading image from: /Users/jenkins/minikube-integration/18222-9538/.minikube/cache/images/amd64/registry.k8s.io/etcd_3.3.15-0
I0226 03:25:58.772139 25643 ssh_runner.go:195] Run: docker image inspect --format {{.Id}} registry.k8s.io/kube-proxy:v1.16.0
I0226 03:25:58.773379 25643 ssh_runner.go:195] Run: docker image inspect --format {{.Id}} registry.k8s.io/pause:3.1
I0226 03:25:58.785061 25643 ssh_runner.go:195] Run: docker image inspect --format {{.Id}} registry.k8s.io/kube-scheduler:v1.16.0
I0226 03:25:58.790109 25643 ssh_runner.go:195] Run: docker image inspect --format {{.Id}} registry.k8s.io/coredns:1.6.2
I0226 03:25:58.790671 25643 cache_images.go:116] "registry.k8s.io/kube-proxy:v1.16.0" needs transfer: "registry.k8s.io/kube-proxy:v1.16.0" does not exist at hash "c21b0c7400f988db4777858edd13b6d3930d62d7ccf026d2415485a52037f384" in container runtime
I0226 03:25:58.790700 25643 docker.go:337] Removing image: registry.k8s.io/kube-proxy:v1.16.0
I0226 03:25:58.790773 25643 ssh_runner.go:195] Run: docker rmi registry.k8s.io/kube-proxy:v1.16.0
I0226 03:25:58.793168 25643 cache_images.go:116] "registry.k8s.io/pause:3.1" needs transfer: "registry.k8s.io/pause:3.1" does not exist at hash "da86e6ba6ca197bf6bc5e9d900febd906b133eaa4750e6bed647b0fbe50ed43e" in container runtime
I0226 03:25:58.793201 25643 docker.go:337] Removing image: registry.k8s.io/pause:3.1
I0226 03:25:58.793288 25643 ssh_runner.go:195] Run: docker rmi registry.k8s.io/pause:3.1
I0226 03:25:58.802113 25643 ssh_runner.go:195] Run: docker image inspect --format {{.Id}} registry.k8s.io/kube-apiserver:v1.16.0
I0226 03:25:58.805133 25643 cache_images.go:116] "registry.k8s.io/kube-scheduler:v1.16.0" needs transfer: "registry.k8s.io/kube-scheduler:v1.16.0" does not exist at hash "301ddc62b80b16315d3c2653cf3888370394277afb3187614cfa20edc352ca0a" in container runtime
I0226 03:25:58.805171 25643 docker.go:337] Removing image: registry.k8s.io/kube-scheduler:v1.16.0
I0226 03:25:58.805278 25643 ssh_runner.go:195] Run: docker rmi registry.k8s.io/kube-scheduler:v1.16.0
I0226 03:25:58.810167 25643 cache_images.go:286] Loading image from: /Users/jenkins/minikube-integration/18222-9538/.minikube/cache/images/amd64/registry.k8s.io/kube-proxy_v1.16.0
I0226 03:25:58.812144 25643 cache_images.go:116] "registry.k8s.io/coredns:1.6.2" needs transfer: "registry.k8s.io/coredns:1.6.2" does not exist at hash "bf261d157914477ee1a5969d28ec687f3fbfc9fb5a664b22df78e57023b0e03b" in container runtime
I0226 03:25:58.812172 25643 docker.go:337] Removing image: registry.k8s.io/coredns:1.6.2
I0226 03:25:58.812237 25643 ssh_runner.go:195] Run: docker rmi registry.k8s.io/coredns:1.6.2
I0226 03:25:58.813286 25643 cache_images.go:286] Loading image from: /Users/jenkins/minikube-integration/18222-9538/.minikube/cache/images/amd64/registry.k8s.io/pause_3.1
I0226 03:25:58.827158 25643 cache_images.go:116] "registry.k8s.io/kube-apiserver:v1.16.0" needs transfer: "registry.k8s.io/kube-apiserver:v1.16.0" does not exist at hash "b305571ca60a5a7818bda47da122683d75e8a1907475681ee8b1efbd06bff12e" in container runtime
I0226 03:25:58.827194 25643 docker.go:337] Removing image: registry.k8s.io/kube-apiserver:v1.16.0
I0226 03:25:58.827313 25643 ssh_runner.go:195] Run: docker rmi registry.k8s.io/kube-apiserver:v1.16.0
I0226 03:25:58.831834 25643 cache_images.go:286] Loading image from: /Users/jenkins/minikube-integration/18222-9538/.minikube/cache/images/amd64/registry.k8s.io/kube-scheduler_v1.16.0
I0226 03:25:58.840306 25643 cache_images.go:286] Loading image from: /Users/jenkins/minikube-integration/18222-9538/.minikube/cache/images/amd64/registry.k8s.io/coredns_1.6.2
I0226 03:25:58.849042 25643 cache_images.go:286] Loading image from: /Users/jenkins/minikube-integration/18222-9538/.minikube/cache/images/amd64/registry.k8s.io/kube-apiserver_v1.16.0
I0226 03:25:59.294659 25643 ssh_runner.go:195] Run: docker image inspect --format {{.Id}} gcr.io/k8s-minikube/storage-provisioner:v5
I0226 03:25:59.314598 25643 cache_images.go:92] LoadImages completed in 2.664364845s
W0226 03:25:59.314660 25643 out.go:239] X Unable to load cached images: loading cached images: stat /Users/jenkins/minikube-integration/18222-9538/.minikube/cache/images/amd64/registry.k8s.io/kube-controller-manager_v1.16.0: no such file or directory
X Unable to load cached images: loading cached images: stat /Users/jenkins/minikube-integration/18222-9538/.minikube/cache/images/amd64/registry.k8s.io/kube-controller-manager_v1.16.0: no such file or directory
I0226 03:25:59.314742 25643 ssh_runner.go:195] Run: docker info --format {{.CgroupDriver}}
I0226 03:25:59.397512 25643 cni.go:84] Creating CNI manager for ""
I0226 03:25:59.397531 25643 cni.go:162] CNI unnecessary in this configuration, recommending no CNI
I0226 03:25:59.397554 25643 kubeadm.go:87] Using pod CIDR: 10.244.0.0/16
I0226 03:25:59.397568 25643 kubeadm.go:176] kubeadm options: {CertDir:/var/lib/minikube/certs ServiceCIDR:10.96.0.0/12 PodSubnet:10.244.0.0/16 AdvertiseAddress:192.168.85.2 APIServerPort:8443 KubernetesVersion:v1.16.0 EtcdDataDir:/var/lib/minikube/etcd EtcdExtraArgs:map[] ClusterName:old-k8s-version-326000 NodeName:old-k8s-version-326000 DNSDomain:cluster.local CRISocket:/var/run/dockershim.sock ImageRepository: ComponentOptions:[{Component:apiServer ExtraArgs:map[enable-admission-plugins:NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota] Pairs:map[certSANs:["127.0.0.1", "localhost", "192.168.85.2"]]} {Component:controllerManager ExtraArgs:map[allocate-node-cidrs:true leader-elect:false] Pairs:map[]} {Component:scheduler ExtraArgs:map[leader-elect:false] Pairs:map[]}] FeatureArgs:map[] NodeIP:192.168.85.2 CgroupDriver:cgroupfs ClientCAFile:/var/lib/minikube/certs/ca.crt StaticP
odPath:/etc/kubernetes/manifests ControlPlaneAddress:control-plane.minikube.internal KubeProxyOptions:map[] ResolvConfSearchRegression:false KubeletConfigOpts:map[hairpinMode:hairpin-veth runtimeRequestTimeout:15m] PrependCriSocketUnix:false}
I0226 03:25:59.397663 25643 kubeadm.go:181] kubeadm config:
apiVersion: kubeadm.k8s.io/v1beta1
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 192.168.85.2
bindPort: 8443
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
ttl: 24h0m0s
usages:
- signing
- authentication
nodeRegistration:
criSocket: /var/run/dockershim.sock
name: "old-k8s-version-326000"
kubeletExtraArgs:
node-ip: 192.168.85.2
taints: []
---
apiVersion: kubeadm.k8s.io/v1beta1
kind: ClusterConfiguration
apiServer:
certSANs: ["127.0.0.1", "localhost", "192.168.85.2"]
extraArgs:
enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
controllerManager:
extraArgs:
allocate-node-cidrs: "true"
leader-elect: "false"
scheduler:
extraArgs:
leader-elect: "false"
certificatesDir: /var/lib/minikube/certs
clusterName: old-k8s-version-326000
controlPlaneEndpoint: control-plane.minikube.internal:8443
dns:
type: CoreDNS
etcd:
local:
dataDir: /var/lib/minikube/etcd
extraArgs:
listen-metrics-urls: http://127.0.0.1:2381,http://192.168.85.2:2381
kubernetesVersion: v1.16.0
networking:
dnsDomain: cluster.local
podSubnet: "10.244.0.0/16"
serviceSubnet: 10.96.0.0/12
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
authentication:
x509:
clientCAFile: /var/lib/minikube/certs/ca.crt
cgroupDriver: cgroupfs
hairpinMode: hairpin-veth
runtimeRequestTimeout: 15m
clusterDomain: "cluster.local"
# disable disk resource management by default
imageGCHighThresholdPercent: 100
evictionHard:
nodefs.available: "0%"
nodefs.inodesFree: "0%"
imagefs.available: "0%"
failSwapOn: false
staticPodPath: /etc/kubernetes/manifests
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
clusterCIDR: "10.244.0.0/16"
metricsBindAddress: 0.0.0.0:10249
conntrack:
maxPerCore: 0
# Skip setting "net.netfilter.nf_conntrack_tcp_timeout_established"
tcpEstablishedTimeout: 0s
# Skip setting "net.netfilter.nf_conntrack_tcp_timeout_close"
tcpCloseWaitTimeout: 0s
I0226 03:25:59.397717 25643 kubeadm.go:976] kubelet [Unit]
Wants=docker.socket
[Service]
ExecStart=
ExecStart=/var/lib/minikube/binaries/v1.16.0/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --config=/var/lib/kubelet/config.yaml --container-runtime=docker --hostname-override=old-k8s-version-326000 --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=192.168.85.2
[Install]
config:
{KubernetesVersion:v1.16.0 ClusterName:old-k8s-version-326000 Namespace:default APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:docker CRISocket: NetworkPlugin: FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI: NodeIP: NodePort:8443 NodeName:}
I0226 03:25:59.397782 25643 ssh_runner.go:195] Run: sudo ls /var/lib/minikube/binaries/v1.16.0
I0226 03:25:59.415350 25643 binaries.go:44] Found k8s binaries, skipping transfer
I0226 03:25:59.415420 25643 ssh_runner.go:195] Run: sudo mkdir -p /etc/systemd/system/kubelet.service.d /lib/systemd/system /var/tmp/minikube
I0226 03:25:59.432436 25643 ssh_runner.go:362] scp memory --> /etc/systemd/system/kubelet.service.d/10-kubeadm.conf (348 bytes)
I0226 03:25:59.465506 25643 ssh_runner.go:362] scp memory --> /lib/systemd/system/kubelet.service (352 bytes)
I0226 03:25:59.498051 25643 ssh_runner.go:362] scp memory --> /var/tmp/minikube/kubeadm.yaml.new (2174 bytes)
I0226 03:25:59.529173 25643 ssh_runner.go:195] Run: grep 192.168.85.2 control-plane.minikube.internal$ /etc/hosts
I0226 03:25:59.534179 25643 ssh_runner.go:195] Run: /bin/bash -c "{ grep -v $'\tcontrol-plane.minikube.internal$' "/etc/hosts"; echo "192.168.85.2 control-plane.minikube.internal"; } > /tmp/h.$$; sudo cp /tmp/h.$$ "/etc/hosts""
I0226 03:25:59.554049 25643 certs.go:56] Setting up /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000 for IP: 192.168.85.2
I0226 03:25:59.554072 25643 certs.go:190] acquiring lock for shared ca certs: {Name:mkac1efdcc7c5f1039385f86b148562f7ea05475 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I0226 03:25:59.554255 25643 certs.go:199] skipping minikubeCA CA generation: /Users/jenkins/minikube-integration/18222-9538/.minikube/ca.key
I0226 03:25:59.554324 25643 certs.go:199] skipping proxyClientCA CA generation: /Users/jenkins/minikube-integration/18222-9538/.minikube/proxy-client-ca.key
I0226 03:25:59.554378 25643 certs.go:319] generating minikube-user signed cert: /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/client.key
I0226 03:25:59.554392 25643 crypto.go:68] Generating cert /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/client.crt with IP's: []
I0226 03:25:59.641018 25643 crypto.go:156] Writing cert to /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/client.crt ...
I0226 03:25:59.641046 25643 lock.go:35] WriteFile acquiring /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/client.crt: {Name:mkd8d58c715fb6140ead109c517b1f913ec1ff04 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I0226 03:25:59.641414 25643 crypto.go:164] Writing key to /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/client.key ...
I0226 03:25:59.641425 25643 lock.go:35] WriteFile acquiring /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/client.key: {Name:mke6c74d38a22b87b281684e14038b0affc2602a Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I0226 03:25:59.641666 25643 certs.go:319] generating minikube signed cert: /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/apiserver.key.43b9df8c
I0226 03:25:59.641681 25643 crypto.go:68] Generating cert /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/apiserver.crt.43b9df8c with IP's: [192.168.85.2 10.96.0.1 127.0.0.1 10.0.0.1]
I0226 03:25:59.796843 25643 crypto.go:156] Writing cert to /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/apiserver.crt.43b9df8c ...
I0226 03:25:59.796859 25643 lock.go:35] WriteFile acquiring /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/apiserver.crt.43b9df8c: {Name:mk53d3613018eabf65c71bfb7b6378f91dd043d0 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I0226 03:25:59.797162 25643 crypto.go:164] Writing key to /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/apiserver.key.43b9df8c ...
I0226 03:25:59.797171 25643 lock.go:35] WriteFile acquiring /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/apiserver.key.43b9df8c: {Name:mke32147866c154fd6b1e35286cf1e6a215c216e Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I0226 03:25:59.797386 25643 certs.go:337] copying /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/apiserver.crt.43b9df8c -> /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/apiserver.crt
I0226 03:25:59.797564 25643 certs.go:341] copying /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/apiserver.key.43b9df8c -> /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/apiserver.key
I0226 03:25:59.797726 25643 certs.go:319] generating aggregator signed cert: /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/proxy-client.key
I0226 03:25:59.797746 25643 crypto.go:68] Generating cert /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/proxy-client.crt with IP's: []
I0226 03:25:59.926004 25643 crypto.go:156] Writing cert to /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/proxy-client.crt ...
I0226 03:25:59.926028 25643 lock.go:35] WriteFile acquiring /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/proxy-client.crt: {Name:mkf19141b6c0713c614eb8b9c7324199e2395558 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I0226 03:25:59.926439 25643 crypto.go:164] Writing key to /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/proxy-client.key ...
I0226 03:25:59.926453 25643 lock.go:35] WriteFile acquiring /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/proxy-client.key: {Name:mkd3f5a85139444f4ce4c694bf88425c69080eef Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I0226 03:25:59.927009 25643 certs.go:437] found cert: /Users/jenkins/minikube-integration/18222-9538/.minikube/certs/Users/jenkins/minikube-integration/18222-9538/.minikube/certs/10026.pem (1338 bytes)
W0226 03:25:59.927106 25643 certs.go:433] ignoring /Users/jenkins/minikube-integration/18222-9538/.minikube/certs/Users/jenkins/minikube-integration/18222-9538/.minikube/certs/10026_empty.pem, impossibly tiny 0 bytes
I0226 03:25:59.927139 25643 certs.go:437] found cert: /Users/jenkins/minikube-integration/18222-9538/.minikube/certs/Users/jenkins/minikube-integration/18222-9538/.minikube/certs/ca-key.pem (1675 bytes)
I0226 03:25:59.927197 25643 certs.go:437] found cert: /Users/jenkins/minikube-integration/18222-9538/.minikube/certs/Users/jenkins/minikube-integration/18222-9538/.minikube/certs/ca.pem (1082 bytes)
I0226 03:25:59.927250 25643 certs.go:437] found cert: /Users/jenkins/minikube-integration/18222-9538/.minikube/certs/Users/jenkins/minikube-integration/18222-9538/.minikube/certs/cert.pem (1123 bytes)
I0226 03:25:59.927315 25643 certs.go:437] found cert: /Users/jenkins/minikube-integration/18222-9538/.minikube/certs/Users/jenkins/minikube-integration/18222-9538/.minikube/certs/key.pem (1675 bytes)
I0226 03:25:59.927445 25643 certs.go:437] found cert: /Users/jenkins/minikube-integration/18222-9538/.minikube/files/etc/ssl/certs/Users/jenkins/minikube-integration/18222-9538/.minikube/files/etc/ssl/certs/100262.pem (1708 bytes)
I0226 03:25:59.928102 25643 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/apiserver.crt --> /var/lib/minikube/certs/apiserver.crt (1399 bytes)
I0226 03:25:59.975796 25643 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/apiserver.key --> /var/lib/minikube/certs/apiserver.key (1679 bytes)
I0226 03:26:00.016136 25643 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/proxy-client.crt --> /var/lib/minikube/certs/proxy-client.crt (1147 bytes)
I0226 03:26:00.064820 25643 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/18222-9538/.minikube/profiles/old-k8s-version-326000/proxy-client.key --> /var/lib/minikube/certs/proxy-client.key (1675 bytes)
I0226 03:26:00.111480 25643 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/18222-9538/.minikube/ca.crt --> /var/lib/minikube/certs/ca.crt (1111 bytes)
I0226 03:26:00.161048 25643 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/18222-9538/.minikube/ca.key --> /var/lib/minikube/certs/ca.key (1675 bytes)
I0226 03:26:00.207224 25643 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/18222-9538/.minikube/proxy-client-ca.crt --> /var/lib/minikube/certs/proxy-client-ca.crt (1119 bytes)
I0226 03:26:00.256025 25643 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/18222-9538/.minikube/proxy-client-ca.key --> /var/lib/minikube/certs/proxy-client-ca.key (1675 bytes)
I0226 03:26:00.301251 25643 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/18222-9538/.minikube/certs/10026.pem --> /usr/share/ca-certificates/10026.pem (1338 bytes)
I0226 03:26:00.349163 25643 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/18222-9538/.minikube/files/etc/ssl/certs/100262.pem --> /usr/share/ca-certificates/100262.pem (1708 bytes)
I0226 03:26:00.394164 25643 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/18222-9538/.minikube/ca.crt --> /usr/share/ca-certificates/minikubeCA.pem (1111 bytes)
I0226 03:26:00.446244 25643 ssh_runner.go:362] scp memory --> /var/lib/minikube/kubeconfig (738 bytes)
I0226 03:26:00.483499 25643 ssh_runner.go:195] Run: openssl version
I0226 03:26:00.489684 25643 ssh_runner.go:195] Run: sudo /bin/bash -c "test -s /usr/share/ca-certificates/100262.pem && ln -fs /usr/share/ca-certificates/100262.pem /etc/ssl/certs/100262.pem"
I0226 03:26:00.505612 25643 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/100262.pem
I0226 03:26:00.509941 25643 certs.go:480] hashing: -rw-r--r-- 1 root root 1708 Feb 26 10:36 /usr/share/ca-certificates/100262.pem
I0226 03:26:00.509989 25643 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/100262.pem
I0226 03:26:00.517577 25643 ssh_runner.go:195] Run: sudo /bin/bash -c "test -L /etc/ssl/certs/3ec20f2e.0 || ln -fs /etc/ssl/certs/100262.pem /etc/ssl/certs/3ec20f2e.0"
I0226 03:26:00.538524 25643 ssh_runner.go:195] Run: sudo /bin/bash -c "test -s /usr/share/ca-certificates/minikubeCA.pem && ln -fs /usr/share/ca-certificates/minikubeCA.pem /etc/ssl/certs/minikubeCA.pem"
I0226 03:26:00.560946 25643 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/minikubeCA.pem
I0226 03:26:00.565430 25643 certs.go:480] hashing: -rw-r--r-- 1 root root 1111 Feb 26 10:29 /usr/share/ca-certificates/minikubeCA.pem
I0226 03:26:00.565484 25643 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/minikubeCA.pem
I0226 03:26:00.572499 25643 ssh_runner.go:195] Run: sudo /bin/bash -c "test -L /etc/ssl/certs/b5213941.0 || ln -fs /etc/ssl/certs/minikubeCA.pem /etc/ssl/certs/b5213941.0"
I0226 03:26:00.588415 25643 ssh_runner.go:195] Run: sudo /bin/bash -c "test -s /usr/share/ca-certificates/10026.pem && ln -fs /usr/share/ca-certificates/10026.pem /etc/ssl/certs/10026.pem"
I0226 03:26:00.604240 25643 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/10026.pem
I0226 03:26:00.608504 25643 certs.go:480] hashing: -rw-r--r-- 1 root root 1338 Feb 26 10:36 /usr/share/ca-certificates/10026.pem
I0226 03:26:00.608553 25643 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/10026.pem
I0226 03:26:00.615227 25643 ssh_runner.go:195] Run: sudo /bin/bash -c "test -L /etc/ssl/certs/51391683.0 || ln -fs /etc/ssl/certs/10026.pem /etc/ssl/certs/51391683.0"
I0226 03:26:00.633976 25643 ssh_runner.go:195] Run: ls /var/lib/minikube/certs/etcd
I0226 03:26:00.640363 25643 certs.go:353] certs directory doesn't exist, likely first start: ls /var/lib/minikube/certs/etcd: Process exited with status 2
stdout:
stderr:
ls: cannot access '/var/lib/minikube/certs/etcd': No such file or directory
I0226 03:26:00.640429 25643 kubeadm.go:404] StartCluster: {Name:old-k8s-version-326000 KeepContext:false EmbedCerts:false MinikubeISO: KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.42-1708008208-17936@sha256:4ea1136332ba1476cda33a97bf12e2f96995cc120674fbafd3ade22d1118ecdf Memory:2200 CPUs:2 DiskSize:20000 VMDriver: Driver:docker HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:0 DockerOpt:[] DisableDriverMounts:true NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.16.0 ClusterName:old-k8s-version-326000 Namespace:default APIServerName:minikubeCA APISer
verNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:docker CRISocket: NetworkPlugin: FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI: NodeIP: NodePort:8443 NodeName:} Nodes:[{Name: IP:192.168.85.2 Port:8443 KubernetesVersion:v1.16.0 ContainerRuntime:docker ControlPlane:true Worker:true}] Addons:map[] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:false ExtraDisks:0 CertExpiration:26280h0m0s Mount:false MountString:/Users:/minikube-host Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOptimizations:false Disab
leMetrics:false CustomQemuFirmwarePath: SocketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s}
I0226 03:26:00.640563 25643 ssh_runner.go:195] Run: docker ps --filter status=paused --filter=name=k8s_.*_(kube-system)_ --format={{.ID}}
I0226 03:26:00.660644 25643 ssh_runner.go:195] Run: sudo ls /var/lib/kubelet/kubeadm-flags.env /var/lib/kubelet/config.yaml /var/lib/minikube/etcd
I0226 03:26:00.677509 25643 ssh_runner.go:195] Run: sudo cp /var/tmp/minikube/kubeadm.yaml.new /var/tmp/minikube/kubeadm.yaml
I0226 03:26:00.694327 25643 kubeadm.go:226] ignoring SystemVerification for kubeadm because of docker driver
I0226 03:26:00.694388 25643 ssh_runner.go:195] Run: sudo ls -la /etc/kubernetes/admin.conf /etc/kubernetes/kubelet.conf /etc/kubernetes/controller-manager.conf /etc/kubernetes/scheduler.conf
I0226 03:26:00.710810 25643 kubeadm.go:152] config check failed, skipping stale config cleanup: sudo ls -la /etc/kubernetes/admin.conf /etc/kubernetes/kubelet.conf /etc/kubernetes/controller-manager.conf /etc/kubernetes/scheduler.conf: Process exited with status 2
stdout:
stderr:
ls: cannot access '/etc/kubernetes/admin.conf': No such file or directory
ls: cannot access '/etc/kubernetes/kubelet.conf': No such file or directory
ls: cannot access '/etc/kubernetes/controller-manager.conf': No such file or directory
ls: cannot access '/etc/kubernetes/scheduler.conf': No such file or directory
I0226 03:26:00.710841 25643 ssh_runner.go:286] Start: /bin/bash -c "sudo env PATH="/var/lib/minikube/binaries/v1.16.0:$PATH" kubeadm init --config /var/tmp/minikube/kubeadm.yaml --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests,DirAvailable--var-lib-minikube,DirAvailable--var-lib-minikube-etcd,FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml,FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml,FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml,FileAvailable--etc-kubernetes-manifests-etcd.yaml,Port-10250,Swap,NumCPU,SystemVerification,FileContent--proc-sys-net-bridge-bridge-nf-call-iptables"
I0226 03:26:00.773444 25643 kubeadm.go:322] [init] Using Kubernetes version: v1.16.0
I0226 03:26:00.773544 25643 kubeadm.go:322] [preflight] Running pre-flight checks
I0226 03:26:01.068490 25643 kubeadm.go:322] [preflight] Pulling images required for setting up a Kubernetes cluster
I0226 03:26:01.068649 25643 kubeadm.go:322] [preflight] This might take a minute or two, depending on the speed of your internet connection
I0226 03:26:01.068822 25643 kubeadm.go:322] [preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
I0226 03:26:01.256642 25643 kubeadm.go:322] [kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
I0226 03:26:01.257819 25643 kubeadm.go:322] [kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
I0226 03:26:01.265816 25643 kubeadm.go:322] [kubelet-start] Activating the kubelet service
I0226 03:26:01.335998 25643 kubeadm.go:322] [certs] Using certificateDir folder "/var/lib/minikube/certs"
I0226 03:26:01.402735 25643 out.go:204] - Generating certificates and keys ...
I0226 03:26:01.402825 25643 kubeadm.go:322] [certs] Using existing ca certificate authority
I0226 03:26:01.402905 25643 kubeadm.go:322] [certs] Using existing apiserver certificate and key on disk
I0226 03:26:01.553767 25643 kubeadm.go:322] [certs] Generating "apiserver-kubelet-client" certificate and key
I0226 03:26:01.678075 25643 kubeadm.go:322] [certs] Generating "front-proxy-ca" certificate and key
I0226 03:26:01.824335 25643 kubeadm.go:322] [certs] Generating "front-proxy-client" certificate and key
I0226 03:26:01.918572 25643 kubeadm.go:322] [certs] Generating "etcd/ca" certificate and key
I0226 03:26:02.298263 25643 kubeadm.go:322] [certs] Generating "etcd/server" certificate and key
I0226 03:26:02.298427 25643 kubeadm.go:322] [certs] etcd/server serving cert is signed for DNS names [old-k8s-version-326000 localhost] and IPs [192.168.85.2 127.0.0.1 ::1]
I0226 03:26:02.449621 25643 kubeadm.go:322] [certs] Generating "etcd/peer" certificate and key
I0226 03:26:02.449746 25643 kubeadm.go:322] [certs] etcd/peer serving cert is signed for DNS names [old-k8s-version-326000 localhost] and IPs [192.168.85.2 127.0.0.1 ::1]
I0226 03:26:02.669496 25643 kubeadm.go:322] [certs] Generating "etcd/healthcheck-client" certificate and key
I0226 03:26:02.870626 25643 kubeadm.go:322] [certs] Generating "apiserver-etcd-client" certificate and key
I0226 03:26:03.141995 25643 kubeadm.go:322] [certs] Generating "sa" key and public key
I0226 03:26:03.142099 25643 kubeadm.go:322] [kubeconfig] Using kubeconfig folder "/etc/kubernetes"
I0226 03:26:03.381603 25643 kubeadm.go:322] [kubeconfig] Writing "admin.conf" kubeconfig file
I0226 03:26:03.603759 25643 kubeadm.go:322] [kubeconfig] Writing "kubelet.conf" kubeconfig file
I0226 03:26:03.780059 25643 kubeadm.go:322] [kubeconfig] Writing "controller-manager.conf" kubeconfig file
I0226 03:26:03.830451 25643 kubeadm.go:322] [kubeconfig] Writing "scheduler.conf" kubeconfig file
I0226 03:26:03.830857 25643 kubeadm.go:322] [control-plane] Using manifest folder "/etc/kubernetes/manifests"
I0226 03:26:03.875185 25643 out.go:204] - Booting up control plane ...
I0226 03:26:03.875340 25643 kubeadm.go:322] [control-plane] Creating static Pod manifest for "kube-apiserver"
I0226 03:26:03.875489 25643 kubeadm.go:322] [control-plane] Creating static Pod manifest for "kube-controller-manager"
I0226 03:26:03.875621 25643 kubeadm.go:322] [control-plane] Creating static Pod manifest for "kube-scheduler"
I0226 03:26:03.875767 25643 kubeadm.go:322] [etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
I0226 03:26:03.876024 25643 kubeadm.go:322] [wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
I0226 03:26:43.846851 25643 kubeadm.go:322] [kubelet-check] Initial timeout of 40s passed.
I0226 03:26:43.847008 25643 kubeadm.go:322] [kubelet-check] It seems like the kubelet isn't running or healthy.
I0226 03:26:43.847224 25643 kubeadm.go:322] [kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
I0226 03:26:48.848685 25643 kubeadm.go:322] [kubelet-check] It seems like the kubelet isn't running or healthy.
I0226 03:26:48.848855 25643 kubeadm.go:322] [kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
I0226 03:26:58.850876 25643 kubeadm.go:322] [kubelet-check] It seems like the kubelet isn't running or healthy.
I0226 03:26:58.851038 25643 kubeadm.go:322] [kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
I0226 03:27:18.852557 25643 kubeadm.go:322] [kubelet-check] It seems like the kubelet isn't running or healthy.
I0226 03:27:18.852728 25643 kubeadm.go:322] [kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
I0226 03:27:58.854656 25643 kubeadm.go:322] [kubelet-check] It seems like the kubelet isn't running or healthy.
I0226 03:27:58.854846 25643 kubeadm.go:322] [kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
I0226 03:27:58.854869 25643 kubeadm.go:322]
I0226 03:27:58.854903 25643 kubeadm.go:322] Unfortunately, an error has occurred:
I0226 03:27:58.854933 25643 kubeadm.go:322] timed out waiting for the condition
I0226 03:27:58.854939 25643 kubeadm.go:322]
I0226 03:27:58.854978 25643 kubeadm.go:322] This error is likely caused by:
I0226 03:27:58.855015 25643 kubeadm.go:322] - The kubelet is not running
I0226 03:27:58.855096 25643 kubeadm.go:322] - The kubelet is unhealthy due to a misconfiguration of the node in some way (required cgroups disabled)
I0226 03:27:58.855104 25643 kubeadm.go:322]
I0226 03:27:58.855204 25643 kubeadm.go:322] If you are on a systemd-powered system, you can try to troubleshoot the error with the following commands:
I0226 03:27:58.855240 25643 kubeadm.go:322] - 'systemctl status kubelet'
I0226 03:27:58.855269 25643 kubeadm.go:322] - 'journalctl -xeu kubelet'
I0226 03:27:58.855281 25643 kubeadm.go:322]
I0226 03:27:58.855375 25643 kubeadm.go:322] Additionally, a control plane component may have crashed or exited when started by the container runtime.
I0226 03:27:58.855453 25643 kubeadm.go:322] To troubleshoot, list all containers using your preferred container runtimes CLI, e.g. docker.
I0226 03:27:58.855527 25643 kubeadm.go:322] Here is one example how you may list all Kubernetes containers running in docker:
I0226 03:27:58.855569 25643 kubeadm.go:322] - 'docker ps -a | grep kube | grep -v pause'
I0226 03:27:58.855633 25643 kubeadm.go:322] Once you have found the failing container, you can inspect its logs with:
I0226 03:27:58.855662 25643 kubeadm.go:322] - 'docker logs CONTAINERID'
I0226 03:27:58.859589 25643 kubeadm.go:322] [WARNING IsDockerSystemdCheck]: detected "cgroupfs" as the Docker cgroup driver. The recommended driver is "systemd". Please follow the guide at https://kubernetes.io/docs/setup/cri/
I0226 03:27:58.859672 25643 kubeadm.go:322] [WARNING Swap]: running with swap on is not supported. Please disable swap
I0226 03:27:58.859790 25643 kubeadm.go:322] [WARNING SystemVerification]: this Docker version is not on the list of validated versions: 25.0.3. Latest validated version: 18.09
I0226 03:27:58.859884 25643 kubeadm.go:322] [WARNING Service-Kubelet]: kubelet service is not enabled, please run 'systemctl enable kubelet.service'
I0226 03:27:58.859967 25643 kubeadm.go:322] error execution phase wait-control-plane: couldn't initialize a Kubernetes cluster
I0226 03:27:58.860053 25643 kubeadm.go:322] To see the stack trace of this error execute with --v=5 or higher
W0226 03:27:58.860139 25643 out.go:239] ! initialization failed, will try again: wait: /bin/bash -c "sudo env PATH="/var/lib/minikube/binaries/v1.16.0:$PATH" kubeadm init --config /var/tmp/minikube/kubeadm.yaml --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests,DirAvailable--var-lib-minikube,DirAvailable--var-lib-minikube-etcd,FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml,FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml,FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml,FileAvailable--etc-kubernetes-manifests-etcd.yaml,Port-10250,Swap,NumCPU,SystemVerification,FileContent--proc-sys-net-bridge-bridge-nf-call-iptables": Process exited with status 1
stdout:
[init] Using Kubernetes version: v1.16.0
[preflight] Running pre-flight checks
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Activating the kubelet service
[certs] Using certificateDir folder "/var/lib/minikube/certs"
[certs] Using existing ca certificate authority
[certs] Using existing apiserver certificate and key on disk
[certs] Generating "apiserver-kubelet-client" certificate and key
[certs] Generating "front-proxy-ca" certificate and key
[certs] Generating "front-proxy-client" certificate and key
[certs] Generating "etcd/ca" certificate and key
[certs] Generating "etcd/server" certificate and key
[certs] etcd/server serving cert is signed for DNS names [old-k8s-version-326000 localhost] and IPs [192.168.85.2 127.0.0.1 ::1]
[certs] Generating "etcd/peer" certificate and key
[certs] etcd/peer serving cert is signed for DNS names [old-k8s-version-326000 localhost] and IPs [192.168.85.2 127.0.0.1 ::1]
[certs] Generating "etcd/healthcheck-client" certificate and key
[certs] Generating "apiserver-etcd-client" certificate and key
[certs] Generating "sa" key and public key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[kubeconfig] Writing "admin.conf" kubeconfig file
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[kubelet-check] Initial timeout of 40s passed.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
Unfortunately, an error has occurred:
timed out waiting for the condition
This error is likely caused by:
- The kubelet is not running
- The kubelet is unhealthy due to a misconfiguration of the node in some way (required cgroups disabled)
If you are on a systemd-powered system, you can try to troubleshoot the error with the following commands:
- 'systemctl status kubelet'
- 'journalctl -xeu kubelet'
Additionally, a control plane component may have crashed or exited when started by the container runtime.
To troubleshoot, list all containers using your preferred container runtimes CLI, e.g. docker.
Here is one example how you may list all Kubernetes containers running in docker:
- 'docker ps -a | grep kube | grep -v pause'
Once you have found the failing container, you can inspect its logs with:
- 'docker logs CONTAINERID'
stderr:
[WARNING IsDockerSystemdCheck]: detected "cgroupfs" as the Docker cgroup driver. The recommended driver is "systemd". Please follow the guide at https://kubernetes.io/docs/setup/cri/
[WARNING Swap]: running with swap on is not supported. Please disable swap
[WARNING SystemVerification]: this Docker version is not on the list of validated versions: 25.0.3. Latest validated version: 18.09
[WARNING Service-Kubelet]: kubelet service is not enabled, please run 'systemctl enable kubelet.service'
error execution phase wait-control-plane: couldn't initialize a Kubernetes cluster
To see the stack trace of this error execute with --v=5 or higher
! initialization failed, will try again: wait: /bin/bash -c "sudo env PATH="/var/lib/minikube/binaries/v1.16.0:$PATH" kubeadm init --config /var/tmp/minikube/kubeadm.yaml --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests,DirAvailable--var-lib-minikube,DirAvailable--var-lib-minikube-etcd,FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml,FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml,FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml,FileAvailable--etc-kubernetes-manifests-etcd.yaml,Port-10250,Swap,NumCPU,SystemVerification,FileContent--proc-sys-net-bridge-bridge-nf-call-iptables": Process exited with status 1
stdout:
[init] Using Kubernetes version: v1.16.0
[preflight] Running pre-flight checks
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Activating the kubelet service
[certs] Using certificateDir folder "/var/lib/minikube/certs"
[certs] Using existing ca certificate authority
[certs] Using existing apiserver certificate and key on disk
[certs] Generating "apiserver-kubelet-client" certificate and key
[certs] Generating "front-proxy-ca" certificate and key
[certs] Generating "front-proxy-client" certificate and key
[certs] Generating "etcd/ca" certificate and key
[certs] Generating "etcd/server" certificate and key
[certs] etcd/server serving cert is signed for DNS names [old-k8s-version-326000 localhost] and IPs [192.168.85.2 127.0.0.1 ::1]
[certs] Generating "etcd/peer" certificate and key
[certs] etcd/peer serving cert is signed for DNS names [old-k8s-version-326000 localhost] and IPs [192.168.85.2 127.0.0.1 ::1]
[certs] Generating "etcd/healthcheck-client" certificate and key
[certs] Generating "apiserver-etcd-client" certificate and key
[certs] Generating "sa" key and public key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[kubeconfig] Writing "admin.conf" kubeconfig file
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[kubelet-check] Initial timeout of 40s passed.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
Unfortunately, an error has occurred:
timed out waiting for the condition
This error is likely caused by:
- The kubelet is not running
- The kubelet is unhealthy due to a misconfiguration of the node in some way (required cgroups disabled)
If you are on a systemd-powered system, you can try to troubleshoot the error with the following commands:
- 'systemctl status kubelet'
- 'journalctl -xeu kubelet'
Additionally, a control plane component may have crashed or exited when started by the container runtime.
To troubleshoot, list all containers using your preferred container runtimes CLI, e.g. docker.
Here is one example how you may list all Kubernetes containers running in docker:
- 'docker ps -a | grep kube | grep -v pause'
Once you have found the failing container, you can inspect its logs with:
- 'docker logs CONTAINERID'
stderr:
[WARNING IsDockerSystemdCheck]: detected "cgroupfs" as the Docker cgroup driver. The recommended driver is "systemd". Please follow the guide at https://kubernetes.io/docs/setup/cri/
[WARNING Swap]: running with swap on is not supported. Please disable swap
[WARNING SystemVerification]: this Docker version is not on the list of validated versions: 25.0.3. Latest validated version: 18.09
[WARNING Service-Kubelet]: kubelet service is not enabled, please run 'systemctl enable kubelet.service'
error execution phase wait-control-plane: couldn't initialize a Kubernetes cluster
To see the stack trace of this error execute with --v=5 or higher
I0226 03:27:58.860186 25643 ssh_runner.go:195] Run: /bin/bash -c "sudo env PATH="/var/lib/minikube/binaries/v1.16.0:$PATH" kubeadm reset --cri-socket /var/run/dockershim.sock --force"
I0226 03:27:59.286743 25643 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service kubelet
I0226 03:27:59.304109 25643 kubeadm.go:226] ignoring SystemVerification for kubeadm because of docker driver
I0226 03:27:59.304162 25643 ssh_runner.go:195] Run: sudo ls -la /etc/kubernetes/admin.conf /etc/kubernetes/kubelet.conf /etc/kubernetes/controller-manager.conf /etc/kubernetes/scheduler.conf
I0226 03:27:59.319319 25643 kubeadm.go:152] config check failed, skipping stale config cleanup: sudo ls -la /etc/kubernetes/admin.conf /etc/kubernetes/kubelet.conf /etc/kubernetes/controller-manager.conf /etc/kubernetes/scheduler.conf: Process exited with status 2
stdout:
stderr:
ls: cannot access '/etc/kubernetes/admin.conf': No such file or directory
ls: cannot access '/etc/kubernetes/kubelet.conf': No such file or directory
ls: cannot access '/etc/kubernetes/controller-manager.conf': No such file or directory
ls: cannot access '/etc/kubernetes/scheduler.conf': No such file or directory
I0226 03:27:59.319343 25643 ssh_runner.go:286] Start: /bin/bash -c "sudo env PATH="/var/lib/minikube/binaries/v1.16.0:$PATH" kubeadm init --config /var/tmp/minikube/kubeadm.yaml --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests,DirAvailable--var-lib-minikube,DirAvailable--var-lib-minikube-etcd,FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml,FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml,FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml,FileAvailable--etc-kubernetes-manifests-etcd.yaml,Port-10250,Swap,NumCPU,SystemVerification,FileContent--proc-sys-net-bridge-bridge-nf-call-iptables"
I0226 03:27:59.379367 25643 kubeadm.go:322] [init] Using Kubernetes version: v1.16.0
I0226 03:27:59.379408 25643 kubeadm.go:322] [preflight] Running pre-flight checks
I0226 03:27:59.660953 25643 kubeadm.go:322] [preflight] Pulling images required for setting up a Kubernetes cluster
I0226 03:27:59.661039 25643 kubeadm.go:322] [preflight] This might take a minute or two, depending on the speed of your internet connection
I0226 03:27:59.661116 25643 kubeadm.go:322] [preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
I0226 03:27:59.833162 25643 kubeadm.go:322] [kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
I0226 03:27:59.833797 25643 kubeadm.go:322] [kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
I0226 03:27:59.840267 25643 kubeadm.go:322] [kubelet-start] Activating the kubelet service
I0226 03:27:59.897619 25643 kubeadm.go:322] [certs] Using certificateDir folder "/var/lib/minikube/certs"
I0226 03:27:59.919208 25643 out.go:204] - Generating certificates and keys ...
I0226 03:27:59.919275 25643 kubeadm.go:322] [certs] Using existing ca certificate authority
I0226 03:27:59.919352 25643 kubeadm.go:322] [certs] Using existing apiserver certificate and key on disk
I0226 03:27:59.919416 25643 kubeadm.go:322] [certs] Using existing apiserver-kubelet-client certificate and key on disk
I0226 03:27:59.919461 25643 kubeadm.go:322] [certs] Using existing front-proxy-ca certificate authority
I0226 03:27:59.919549 25643 kubeadm.go:322] [certs] Using existing front-proxy-client certificate and key on disk
I0226 03:27:59.919619 25643 kubeadm.go:322] [certs] Using existing etcd/ca certificate authority
I0226 03:27:59.919687 25643 kubeadm.go:322] [certs] Using existing etcd/server certificate and key on disk
I0226 03:27:59.919740 25643 kubeadm.go:322] [certs] Using existing etcd/peer certificate and key on disk
I0226 03:27:59.919800 25643 kubeadm.go:322] [certs] Using existing etcd/healthcheck-client certificate and key on disk
I0226 03:27:59.919848 25643 kubeadm.go:322] [certs] Using existing apiserver-etcd-client certificate and key on disk
I0226 03:27:59.919884 25643 kubeadm.go:322] [certs] Using the existing "sa" key
I0226 03:27:59.919932 25643 kubeadm.go:322] [kubeconfig] Using kubeconfig folder "/etc/kubernetes"
I0226 03:28:00.005823 25643 kubeadm.go:322] [kubeconfig] Writing "admin.conf" kubeconfig file
I0226 03:28:00.148730 25643 kubeadm.go:322] [kubeconfig] Writing "kubelet.conf" kubeconfig file
I0226 03:28:00.346066 25643 kubeadm.go:322] [kubeconfig] Writing "controller-manager.conf" kubeconfig file
I0226 03:28:00.763315 25643 kubeadm.go:322] [kubeconfig] Writing "scheduler.conf" kubeconfig file
I0226 03:28:00.763817 25643 kubeadm.go:322] [control-plane] Using manifest folder "/etc/kubernetes/manifests"
I0226 03:28:00.785382 25643 out.go:204] - Booting up control plane ...
I0226 03:28:00.785526 25643 kubeadm.go:322] [control-plane] Creating static Pod manifest for "kube-apiserver"
I0226 03:28:00.785745 25643 kubeadm.go:322] [control-plane] Creating static Pod manifest for "kube-controller-manager"
I0226 03:28:00.785857 25643 kubeadm.go:322] [control-plane] Creating static Pod manifest for "kube-scheduler"
I0226 03:28:00.785977 25643 kubeadm.go:322] [etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
I0226 03:28:00.786167 25643 kubeadm.go:322] [wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
I0226 03:28:40.773217 25643 kubeadm.go:322] [kubelet-check] Initial timeout of 40s passed.
I0226 03:28:40.773753 25643 kubeadm.go:322] [kubelet-check] It seems like the kubelet isn't running or healthy.
I0226 03:28:40.773905 25643 kubeadm.go:322] [kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
I0226 03:28:45.774975 25643 kubeadm.go:322] [kubelet-check] It seems like the kubelet isn't running or healthy.
I0226 03:28:45.775198 25643 kubeadm.go:322] [kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
I0226 03:28:55.776836 25643 kubeadm.go:322] [kubelet-check] It seems like the kubelet isn't running or healthy.
I0226 03:28:55.777145 25643 kubeadm.go:322] [kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
I0226 03:29:15.778721 25643 kubeadm.go:322] [kubelet-check] It seems like the kubelet isn't running or healthy.
I0226 03:29:15.778944 25643 kubeadm.go:322] [kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
I0226 03:29:55.779624 25643 kubeadm.go:322] [kubelet-check] It seems like the kubelet isn't running or healthy.
I0226 03:29:55.779794 25643 kubeadm.go:322] [kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
I0226 03:29:55.779805 25643 kubeadm.go:322]
I0226 03:29:55.779835 25643 kubeadm.go:322] Unfortunately, an error has occurred:
I0226 03:29:55.779867 25643 kubeadm.go:322] timed out waiting for the condition
I0226 03:29:55.779873 25643 kubeadm.go:322]
I0226 03:29:55.779909 25643 kubeadm.go:322] This error is likely caused by:
I0226 03:29:55.779943 25643 kubeadm.go:322] - The kubelet is not running
I0226 03:29:55.780022 25643 kubeadm.go:322] - The kubelet is unhealthy due to a misconfiguration of the node in some way (required cgroups disabled)
I0226 03:29:55.780029 25643 kubeadm.go:322]
I0226 03:29:55.780107 25643 kubeadm.go:322] If you are on a systemd-powered system, you can try to troubleshoot the error with the following commands:
I0226 03:29:55.780135 25643 kubeadm.go:322] - 'systemctl status kubelet'
I0226 03:29:55.780178 25643 kubeadm.go:322] - 'journalctl -xeu kubelet'
I0226 03:29:55.780188 25643 kubeadm.go:322]
I0226 03:29:55.780309 25643 kubeadm.go:322] Additionally, a control plane component may have crashed or exited when started by the container runtime.
I0226 03:29:55.780395 25643 kubeadm.go:322] To troubleshoot, list all containers using your preferred container runtimes CLI, e.g. docker.
I0226 03:29:55.780475 25643 kubeadm.go:322] Here is one example how you may list all Kubernetes containers running in docker:
I0226 03:29:55.780514 25643 kubeadm.go:322] - 'docker ps -a | grep kube | grep -v pause'
I0226 03:29:55.780571 25643 kubeadm.go:322] Once you have found the failing container, you can inspect its logs with:
I0226 03:29:55.780607 25643 kubeadm.go:322] - 'docker logs CONTAINERID'
I0226 03:29:55.784514 25643 kubeadm.go:322] [WARNING IsDockerSystemdCheck]: detected "cgroupfs" as the Docker cgroup driver. The recommended driver is "systemd". Please follow the guide at https://kubernetes.io/docs/setup/cri/
I0226 03:29:55.784581 25643 kubeadm.go:322] [WARNING Swap]: running with swap on is not supported. Please disable swap
I0226 03:29:55.784688 25643 kubeadm.go:322] [WARNING SystemVerification]: this Docker version is not on the list of validated versions: 25.0.3. Latest validated version: 18.09
I0226 03:29:55.784781 25643 kubeadm.go:322] [WARNING Service-Kubelet]: kubelet service is not enabled, please run 'systemctl enable kubelet.service'
I0226 03:29:55.784858 25643 kubeadm.go:322] error execution phase wait-control-plane: couldn't initialize a Kubernetes cluster
I0226 03:29:55.784919 25643 kubeadm.go:322] To see the stack trace of this error execute with --v=5 or higher
I0226 03:29:55.784953 25643 kubeadm.go:406] StartCluster complete in 3m55.143193105s
I0226 03:29:55.785036 25643 ssh_runner.go:195] Run: docker ps -a --filter=name=k8s_kube-apiserver --format={{.ID}}
I0226 03:29:55.800889 25643 logs.go:276] 0 containers: []
W0226 03:29:55.800902 25643 logs.go:278] No container was found matching "kube-apiserver"
I0226 03:29:55.800971 25643 ssh_runner.go:195] Run: docker ps -a --filter=name=k8s_etcd --format={{.ID}}
I0226 03:29:55.817435 25643 logs.go:276] 0 containers: []
W0226 03:29:55.817449 25643 logs.go:278] No container was found matching "etcd"
I0226 03:29:55.817518 25643 ssh_runner.go:195] Run: docker ps -a --filter=name=k8s_coredns --format={{.ID}}
I0226 03:29:55.834181 25643 logs.go:276] 0 containers: []
W0226 03:29:55.834196 25643 logs.go:278] No container was found matching "coredns"
I0226 03:29:55.834274 25643 ssh_runner.go:195] Run: docker ps -a --filter=name=k8s_kube-scheduler --format={{.ID}}
I0226 03:29:55.853280 25643 logs.go:276] 0 containers: []
W0226 03:29:55.853295 25643 logs.go:278] No container was found matching "kube-scheduler"
I0226 03:29:55.853389 25643 ssh_runner.go:195] Run: docker ps -a --filter=name=k8s_kube-proxy --format={{.ID}}
I0226 03:29:55.870671 25643 logs.go:276] 0 containers: []
W0226 03:29:55.870685 25643 logs.go:278] No container was found matching "kube-proxy"
I0226 03:29:55.870750 25643 ssh_runner.go:195] Run: docker ps -a --filter=name=k8s_kube-controller-manager --format={{.ID}}
I0226 03:29:55.887818 25643 logs.go:276] 0 containers: []
W0226 03:29:55.887834 25643 logs.go:278] No container was found matching "kube-controller-manager"
I0226 03:29:55.887921 25643 ssh_runner.go:195] Run: docker ps -a --filter=name=k8s_kindnet --format={{.ID}}
I0226 03:29:55.906026 25643 logs.go:276] 0 containers: []
W0226 03:29:55.906041 25643 logs.go:278] No container was found matching "kindnet"
I0226 03:29:55.906049 25643 logs.go:123] Gathering logs for Docker ...
I0226 03:29:55.906057 25643 ssh_runner.go:195] Run: /bin/bash -c "sudo journalctl -u docker -u cri-docker -n 400"
I0226 03:29:55.931805 25643 logs.go:123] Gathering logs for container status ...
I0226 03:29:55.931819 25643 ssh_runner.go:195] Run: /bin/bash -c "sudo `which crictl || echo crictl` ps -a || sudo docker ps -a"
I0226 03:29:55.990788 25643 logs.go:123] Gathering logs for kubelet ...
I0226 03:29:55.990801 25643 ssh_runner.go:195] Run: /bin/bash -c "sudo journalctl -u kubelet -n 400"
I0226 03:29:56.031854 25643 logs.go:123] Gathering logs for dmesg ...
I0226 03:29:56.031870 25643 ssh_runner.go:195] Run: /bin/bash -c "sudo dmesg -PH -L=never --level warn,err,crit,alert,emerg | tail -n 400"
I0226 03:29:56.052013 25643 logs.go:123] Gathering logs for describe nodes ...
I0226 03:29:56.052028 25643 ssh_runner.go:195] Run: /bin/bash -c "sudo /var/lib/minikube/binaries/v1.16.0/kubectl describe nodes --kubeconfig=/var/lib/minikube/kubeconfig"
W0226 03:29:56.129959 25643 logs.go:130] failed describe nodes: command: /bin/bash -c "sudo /var/lib/minikube/binaries/v1.16.0/kubectl describe nodes --kubeconfig=/var/lib/minikube/kubeconfig" /bin/bash -c "sudo /var/lib/minikube/binaries/v1.16.0/kubectl describe nodes --kubeconfig=/var/lib/minikube/kubeconfig": Process exited with status 1
stdout:
stderr:
The connection to the server localhost:8443 was refused - did you specify the right host or port?
output:
** stderr **
The connection to the server localhost:8443 was refused - did you specify the right host or port?
** /stderr **
W0226 03:29:56.129987 25643 out.go:364] Error starting cluster: wait: /bin/bash -c "sudo env PATH="/var/lib/minikube/binaries/v1.16.0:$PATH" kubeadm init --config /var/tmp/minikube/kubeadm.yaml --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests,DirAvailable--var-lib-minikube,DirAvailable--var-lib-minikube-etcd,FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml,FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml,FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml,FileAvailable--etc-kubernetes-manifests-etcd.yaml,Port-10250,Swap,NumCPU,SystemVerification,FileContent--proc-sys-net-bridge-bridge-nf-call-iptables": Process exited with status 1
stdout:
[init] Using Kubernetes version: v1.16.0
[preflight] Running pre-flight checks
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Activating the kubelet service
[certs] Using certificateDir folder "/var/lib/minikube/certs"
[certs] Using existing ca certificate authority
[certs] Using existing apiserver certificate and key on disk
[certs] Using existing apiserver-kubelet-client certificate and key on disk
[certs] Using existing front-proxy-ca certificate authority
[certs] Using existing front-proxy-client certificate and key on disk
[certs] Using existing etcd/ca certificate authority
[certs] Using existing etcd/server certificate and key on disk
[certs] Using existing etcd/peer certificate and key on disk
[certs] Using existing etcd/healthcheck-client certificate and key on disk
[certs] Using existing apiserver-etcd-client certificate and key on disk
[certs] Using the existing "sa" key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[kubeconfig] Writing "admin.conf" kubeconfig file
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[kubelet-check] Initial timeout of 40s passed.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
Unfortunately, an error has occurred:
timed out waiting for the condition
This error is likely caused by:
- The kubelet is not running
- The kubelet is unhealthy due to a misconfiguration of the node in some way (required cgroups disabled)
If you are on a systemd-powered system, you can try to troubleshoot the error with the following commands:
- 'systemctl status kubelet'
- 'journalctl -xeu kubelet'
Additionally, a control plane component may have crashed or exited when started by the container runtime.
To troubleshoot, list all containers using your preferred container runtimes CLI, e.g. docker.
Here is one example how you may list all Kubernetes containers running in docker:
- 'docker ps -a | grep kube | grep -v pause'
Once you have found the failing container, you can inspect its logs with:
- 'docker logs CONTAINERID'
stderr:
[WARNING IsDockerSystemdCheck]: detected "cgroupfs" as the Docker cgroup driver. The recommended driver is "systemd". Please follow the guide at https://kubernetes.io/docs/setup/cri/
[WARNING Swap]: running with swap on is not supported. Please disable swap
[WARNING SystemVerification]: this Docker version is not on the list of validated versions: 25.0.3. Latest validated version: 18.09
[WARNING Service-Kubelet]: kubelet service is not enabled, please run 'systemctl enable kubelet.service'
error execution phase wait-control-plane: couldn't initialize a Kubernetes cluster
To see the stack trace of this error execute with --v=5 or higher
W0226 03:29:56.130001 25643 out.go:239] *
*
W0226 03:29:56.130042 25643 out.go:239] X Error starting cluster: wait: /bin/bash -c "sudo env PATH="/var/lib/minikube/binaries/v1.16.0:$PATH" kubeadm init --config /var/tmp/minikube/kubeadm.yaml --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests,DirAvailable--var-lib-minikube,DirAvailable--var-lib-minikube-etcd,FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml,FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml,FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml,FileAvailable--etc-kubernetes-manifests-etcd.yaml,Port-10250,Swap,NumCPU,SystemVerification,FileContent--proc-sys-net-bridge-bridge-nf-call-iptables": Process exited with status 1
stdout:
[init] Using Kubernetes version: v1.16.0
[preflight] Running pre-flight checks
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Activating the kubelet service
[certs] Using certificateDir folder "/var/lib/minikube/certs"
[certs] Using existing ca certificate authority
[certs] Using existing apiserver certificate and key on disk
[certs] Using existing apiserver-kubelet-client certificate and key on disk
[certs] Using existing front-proxy-ca certificate authority
[certs] Using existing front-proxy-client certificate and key on disk
[certs] Using existing etcd/ca certificate authority
[certs] Using existing etcd/server certificate and key on disk
[certs] Using existing etcd/peer certificate and key on disk
[certs] Using existing etcd/healthcheck-client certificate and key on disk
[certs] Using existing apiserver-etcd-client certificate and key on disk
[certs] Using the existing "sa" key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[kubeconfig] Writing "admin.conf" kubeconfig file
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[kubelet-check] Initial timeout of 40s passed.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
Unfortunately, an error has occurred:
timed out waiting for the condition
This error is likely caused by:
- The kubelet is not running
- The kubelet is unhealthy due to a misconfiguration of the node in some way (required cgroups disabled)
If you are on a systemd-powered system, you can try to troubleshoot the error with the following commands:
- 'systemctl status kubelet'
- 'journalctl -xeu kubelet'
Additionally, a control plane component may have crashed or exited when started by the container runtime.
To troubleshoot, list all containers using your preferred container runtimes CLI, e.g. docker.
Here is one example how you may list all Kubernetes containers running in docker:
- 'docker ps -a | grep kube | grep -v pause'
Once you have found the failing container, you can inspect its logs with:
- 'docker logs CONTAINERID'
stderr:
[WARNING IsDockerSystemdCheck]: detected "cgroupfs" as the Docker cgroup driver. The recommended driver is "systemd". Please follow the guide at https://kubernetes.io/docs/setup/cri/
[WARNING Swap]: running with swap on is not supported. Please disable swap
[WARNING SystemVerification]: this Docker version is not on the list of validated versions: 25.0.3. Latest validated version: 18.09
[WARNING Service-Kubelet]: kubelet service is not enabled, please run 'systemctl enable kubelet.service'
error execution phase wait-control-plane: couldn't initialize a Kubernetes cluster
To see the stack trace of this error execute with --v=5 or higher
X Error starting cluster: wait: /bin/bash -c "sudo env PATH="/var/lib/minikube/binaries/v1.16.0:$PATH" kubeadm init --config /var/tmp/minikube/kubeadm.yaml --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests,DirAvailable--var-lib-minikube,DirAvailable--var-lib-minikube-etcd,FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml,FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml,FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml,FileAvailable--etc-kubernetes-manifests-etcd.yaml,Port-10250,Swap,NumCPU,SystemVerification,FileContent--proc-sys-net-bridge-bridge-nf-call-iptables": Process exited with status 1
stdout:
[init] Using Kubernetes version: v1.16.0
[preflight] Running pre-flight checks
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Activating the kubelet service
[certs] Using certificateDir folder "/var/lib/minikube/certs"
[certs] Using existing ca certificate authority
[certs] Using existing apiserver certificate and key on disk
[certs] Using existing apiserver-kubelet-client certificate and key on disk
[certs] Using existing front-proxy-ca certificate authority
[certs] Using existing front-proxy-client certificate and key on disk
[certs] Using existing etcd/ca certificate authority
[certs] Using existing etcd/server certificate and key on disk
[certs] Using existing etcd/peer certificate and key on disk
[certs] Using existing etcd/healthcheck-client certificate and key on disk
[certs] Using existing apiserver-etcd-client certificate and key on disk
[certs] Using the existing "sa" key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[kubeconfig] Writing "admin.conf" kubeconfig file
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[kubelet-check] Initial timeout of 40s passed.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
Unfortunately, an error has occurred:
timed out waiting for the condition
This error is likely caused by:
- The kubelet is not running
- The kubelet is unhealthy due to a misconfiguration of the node in some way (required cgroups disabled)
If you are on a systemd-powered system, you can try to troubleshoot the error with the following commands:
- 'systemctl status kubelet'
- 'journalctl -xeu kubelet'
Additionally, a control plane component may have crashed or exited when started by the container runtime.
To troubleshoot, list all containers using your preferred container runtimes CLI, e.g. docker.
Here is one example how you may list all Kubernetes containers running in docker:
- 'docker ps -a | grep kube | grep -v pause'
Once you have found the failing container, you can inspect its logs with:
- 'docker logs CONTAINERID'
stderr:
[WARNING IsDockerSystemdCheck]: detected "cgroupfs" as the Docker cgroup driver. The recommended driver is "systemd". Please follow the guide at https://kubernetes.io/docs/setup/cri/
[WARNING Swap]: running with swap on is not supported. Please disable swap
[WARNING SystemVerification]: this Docker version is not on the list of validated versions: 25.0.3. Latest validated version: 18.09
[WARNING Service-Kubelet]: kubelet service is not enabled, please run 'systemctl enable kubelet.service'
error execution phase wait-control-plane: couldn't initialize a Kubernetes cluster
To see the stack trace of this error execute with --v=5 or higher
W0226 03:29:56.130057 25643 out.go:239] *
*
W0226 03:29:56.130721 25643 out.go:239] ╭─────────────────────────────────────────────────────────────────────────────────────────────╮
│ │
│ * If the above advice does not help, please let us know: │
│ https://github.com/kubernetes/minikube/issues/new/choose │
│ │
│ * Please run `minikube logs --file=logs.txt` and attach logs.txt to the GitHub issue. │
│ │
╰─────────────────────────────────────────────────────────────────────────────────────────────╯
╭─────────────────────────────────────────────────────────────────────────────────────────────╮
│ │
│ * If the above advice does not help, please let us know: │
│ https://github.com/kubernetes/minikube/issues/new/choose │
│ │
│ * Please run `minikube logs --file=logs.txt` and attach logs.txt to the GitHub issue. │
│ │
╰─────────────────────────────────────────────────────────────────────────────────────────────╯
I0226 03:29:56.195252 25643 out.go:177]
W0226 03:29:56.239400 25643 out.go:239] X Exiting due to K8S_KUBELET_NOT_RUNNING: wait: /bin/bash -c "sudo env PATH="/var/lib/minikube/binaries/v1.16.0:$PATH" kubeadm init --config /var/tmp/minikube/kubeadm.yaml --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests,DirAvailable--var-lib-minikube,DirAvailable--var-lib-minikube-etcd,FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml,FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml,FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml,FileAvailable--etc-kubernetes-manifests-etcd.yaml,Port-10250,Swap,NumCPU,SystemVerification,FileContent--proc-sys-net-bridge-bridge-nf-call-iptables": Process exited with status 1
stdout:
[init] Using Kubernetes version: v1.16.0
[preflight] Running pre-flight checks
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Activating the kubelet service
[certs] Using certificateDir folder "/var/lib/minikube/certs"
[certs] Using existing ca certificate authority
[certs] Using existing apiserver certificate and key on disk
[certs] Using existing apiserver-kubelet-client certificate and key on disk
[certs] Using existing front-proxy-ca certificate authority
[certs] Using existing front-proxy-client certificate and key on disk
[certs] Using existing etcd/ca certificate authority
[certs] Using existing etcd/server certificate and key on disk
[certs] Using existing etcd/peer certificate and key on disk
[certs] Using existing etcd/healthcheck-client certificate and key on disk
[certs] Using existing apiserver-etcd-client certificate and key on disk
[certs] Using the existing "sa" key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[kubeconfig] Writing "admin.conf" kubeconfig file
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[kubelet-check] Initial timeout of 40s passed.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
Unfortunately, an error has occurred:
timed out waiting for the condition
This error is likely caused by:
- The kubelet is not running
- The kubelet is unhealthy due to a misconfiguration of the node in some way (required cgroups disabled)
If you are on a systemd-powered system, you can try to troubleshoot the error with the following commands:
- 'systemctl status kubelet'
- 'journalctl -xeu kubelet'
Additionally, a control plane component may have crashed or exited when started by the container runtime.
To troubleshoot, list all containers using your preferred container runtimes CLI, e.g. docker.
Here is one example how you may list all Kubernetes containers running in docker:
- 'docker ps -a | grep kube | grep -v pause'
Once you have found the failing container, you can inspect its logs with:
- 'docker logs CONTAINERID'
stderr:
[WARNING IsDockerSystemdCheck]: detected "cgroupfs" as the Docker cgroup driver. The recommended driver is "systemd". Please follow the guide at https://kubernetes.io/docs/setup/cri/
[WARNING Swap]: running with swap on is not supported. Please disable swap
[WARNING SystemVerification]: this Docker version is not on the list of validated versions: 25.0.3. Latest validated version: 18.09
[WARNING Service-Kubelet]: kubelet service is not enabled, please run 'systemctl enable kubelet.service'
error execution phase wait-control-plane: couldn't initialize a Kubernetes cluster
To see the stack trace of this error execute with --v=5 or higher
X Exiting due to K8S_KUBELET_NOT_RUNNING: wait: /bin/bash -c "sudo env PATH="/var/lib/minikube/binaries/v1.16.0:$PATH" kubeadm init --config /var/tmp/minikube/kubeadm.yaml --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests,DirAvailable--var-lib-minikube,DirAvailable--var-lib-minikube-etcd,FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml,FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml,FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml,FileAvailable--etc-kubernetes-manifests-etcd.yaml,Port-10250,Swap,NumCPU,SystemVerification,FileContent--proc-sys-net-bridge-bridge-nf-call-iptables": Process exited with status 1
stdout:
[init] Using Kubernetes version: v1.16.0
[preflight] Running pre-flight checks
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Activating the kubelet service
[certs] Using certificateDir folder "/var/lib/minikube/certs"
[certs] Using existing ca certificate authority
[certs] Using existing apiserver certificate and key on disk
[certs] Using existing apiserver-kubelet-client certificate and key on disk
[certs] Using existing front-proxy-ca certificate authority
[certs] Using existing front-proxy-client certificate and key on disk
[certs] Using existing etcd/ca certificate authority
[certs] Using existing etcd/server certificate and key on disk
[certs] Using existing etcd/peer certificate and key on disk
[certs] Using existing etcd/healthcheck-client certificate and key on disk
[certs] Using existing apiserver-etcd-client certificate and key on disk
[certs] Using the existing "sa" key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[kubeconfig] Writing "admin.conf" kubeconfig file
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[kubelet-check] Initial timeout of 40s passed.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
[kubelet-check] It seems like the kubelet isn't running or healthy.
[kubelet-check] The HTTP call equal to 'curl -sSL http://localhost:10248/healthz' failed with error: Get http://localhost:10248/healthz: dial tcp 127.0.0.1:10248: connect: connection refused.
Unfortunately, an error has occurred:
timed out waiting for the condition
This error is likely caused by:
- The kubelet is not running
- The kubelet is unhealthy due to a misconfiguration of the node in some way (required cgroups disabled)
If you are on a systemd-powered system, you can try to troubleshoot the error with the following commands:
- 'systemctl status kubelet'
- 'journalctl -xeu kubelet'
Additionally, a control plane component may have crashed or exited when started by the container runtime.
To troubleshoot, list all containers using your preferred container runtimes CLI, e.g. docker.
Here is one example how you may list all Kubernetes containers running in docker:
- 'docker ps -a | grep kube | grep -v pause'
Once you have found the failing container, you can inspect its logs with:
- 'docker logs CONTAINERID'
stderr:
[WARNING IsDockerSystemdCheck]: detected "cgroupfs" as the Docker cgroup driver. The recommended driver is "systemd". Please follow the guide at https://kubernetes.io/docs/setup/cri/
[WARNING Swap]: running with swap on is not supported. Please disable swap
[WARNING SystemVerification]: this Docker version is not on the list of validated versions: 25.0.3. Latest validated version: 18.09
[WARNING Service-Kubelet]: kubelet service is not enabled, please run 'systemctl enable kubelet.service'
error execution phase wait-control-plane: couldn't initialize a Kubernetes cluster
To see the stack trace of this error execute with --v=5 or higher
W0226 03:29:56.239448 25643 out.go:239] * Suggestion: Check output of 'journalctl -xeu kubelet', try passing --extra-config=kubelet.cgroup-driver=systemd to minikube start
* Suggestion: Check output of 'journalctl -xeu kubelet', try passing --extra-config=kubelet.cgroup-driver=systemd to minikube start
W0226 03:29:56.239468 25643 out.go:239] * Related issue: https://github.com/kubernetes/minikube/issues/4172
* Related issue: https://github.com/kubernetes/minikube/issues/4172
I0226 03:29:56.283439 25643 out.go:177]
** /stderr **
start_stop_delete_test.go:188: failed starting minikube -first start-. args "out/minikube-darwin-amd64 start -p old-k8s-version-326000 --memory=2200 --alsologtostderr --wait=true --kvm-network=default --kvm-qemu-uri=qemu:///system --disable-driver-mounts --keep-context=false --driver=docker --kubernetes-version=v1.16.0": exit status 109
helpers_test.go:222: -----------------------post-mortem--------------------------------
helpers_test.go:230: ======> post-mortem[TestStartStop/group/old-k8s-version/serial/FirstStart]: docker inspect <======
helpers_test.go:231: (dbg) Run: docker inspect old-k8s-version-326000
helpers_test.go:235: (dbg) docker inspect old-k8s-version-326000:
-- stdout --
[
{
"Id": "76ad634e3f3ff5bf0b2a50c602c87984102ce0977c187500835978cf6a8c221b",
"Created": "2024-02-26T11:25:41.957182514Z",
"Path": "/usr/local/bin/entrypoint",
"Args": [
"/sbin/init"
],
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 362474,
"ExitCode": 0,
"Error": "",
"StartedAt": "2024-02-26T11:25:42.199375659Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Image": "sha256:78bbddd92c7656e5a7bf9651f59e6b47aa97241efd2e93247c457ec76b2185c5",
"ResolvConfPath": "/var/lib/docker/containers/76ad634e3f3ff5bf0b2a50c602c87984102ce0977c187500835978cf6a8c221b/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/76ad634e3f3ff5bf0b2a50c602c87984102ce0977c187500835978cf6a8c221b/hostname",
"HostsPath": "/var/lib/docker/containers/76ad634e3f3ff5bf0b2a50c602c87984102ce0977c187500835978cf6a8c221b/hosts",
"LogPath": "/var/lib/docker/containers/76ad634e3f3ff5bf0b2a50c602c87984102ce0977c187500835978cf6a8c221b/76ad634e3f3ff5bf0b2a50c602c87984102ce0977c187500835978cf6a8c221b-json.log",
"Name": "/old-k8s-version-326000",
"RestartCount": 0,
"Driver": "overlay2",
"Platform": "linux",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "",
"ExecIDs": null,
"HostConfig": {
"Binds": [
"old-k8s-version-326000:/var",
"/lib/modules:/lib/modules:ro"
],
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {}
},
"NetworkMode": "old-k8s-version-326000",
"PortBindings": {
"22/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "0"
}
],
"2376/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "0"
}
],
"32443/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "0"
}
],
"5000/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "0"
}
],
"8443/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "0"
}
]
},
"RestartPolicy": {
"Name": "no",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"ConsoleSize": [
0,
0
],
"CapAdd": null,
"CapDrop": null,
"CgroupnsMode": "private",
"Dns": [],
"DnsOptions": [],
"DnsSearch": [],
"ExtraHosts": null,
"GroupAdd": null,
"IpcMode": "private",
"Cgroup": "",
"Links": null,
"OomScoreAdj": 0,
"PidMode": "",
"Privileged": true,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": [
"seccomp=unconfined",
"apparmor=unconfined",
"label=disable"
],
"Tmpfs": {
"/run": "",
"/tmp": ""
},
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"Isolation": "",
"CpuShares": 0,
"Memory": 2306867200,
"NanoCpus": 2000000000,
"CgroupParent": "",
"BlkioWeight": 0,
"BlkioWeightDevice": [],
"BlkioDeviceReadBps": [],
"BlkioDeviceWriteBps": [],
"BlkioDeviceReadIOps": [],
"BlkioDeviceWriteIOps": [],
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": [],
"DeviceCgroupRules": null,
"DeviceRequests": null,
"MemoryReservation": 0,
"MemorySwap": 2306867200,
"MemorySwappiness": null,
"OomKillDisable": null,
"PidsLimit": null,
"Ulimits": [],
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0,
"MaskedPaths": null,
"ReadonlyPaths": null
},
"GraphDriver": {
"Data": {
"LowerDir": "/var/lib/docker/overlay2/60f3f0a1425652d22ad9176e8be35f0bcc96402895ea787bafbfdf32dd7af953-init/diff:/var/lib/docker/overlay2/8bb839173c154892efba77c6399a35a6f861ea09086927d7a3ace9b08c2c0425/diff",
"MergedDir": "/var/lib/docker/overlay2/60f3f0a1425652d22ad9176e8be35f0bcc96402895ea787bafbfdf32dd7af953/merged",
"UpperDir": "/var/lib/docker/overlay2/60f3f0a1425652d22ad9176e8be35f0bcc96402895ea787bafbfdf32dd7af953/diff",
"WorkDir": "/var/lib/docker/overlay2/60f3f0a1425652d22ad9176e8be35f0bcc96402895ea787bafbfdf32dd7af953/work"
},
"Name": "overlay2"
},
"Mounts": [
{
"Type": "volume",
"Name": "old-k8s-version-326000",
"Source": "/var/lib/docker/volumes/old-k8s-version-326000/_data",
"Destination": "/var",
"Driver": "local",
"Mode": "z",
"RW": true,
"Propagation": ""
},
{
"Type": "bind",
"Source": "/lib/modules",
"Destination": "/lib/modules",
"Mode": "ro",
"RW": false,
"Propagation": "rprivate"
}
],
"Config": {
"Hostname": "old-k8s-version-326000",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"ExposedPorts": {
"22/tcp": {},
"2376/tcp": {},
"32443/tcp": {},
"5000/tcp": {},
"8443/tcp": {}
},
"Tty": true,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"container=docker",
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
],
"Cmd": null,
"Image": "gcr.io/k8s-minikube/kicbase-builds:v0.0.42-1708008208-17936@sha256:4ea1136332ba1476cda33a97bf12e2f96995cc120674fbafd3ade22d1118ecdf",
"Volumes": null,
"WorkingDir": "/",
"Entrypoint": [
"/usr/local/bin/entrypoint",
"/sbin/init"
],
"OnBuild": null,
"Labels": {
"created_by.minikube.sigs.k8s.io": "true",
"mode.minikube.sigs.k8s.io": "old-k8s-version-326000",
"name.minikube.sigs.k8s.io": "old-k8s-version-326000",
"role.minikube.sigs.k8s.io": ""
},
"StopSignal": "SIGRTMIN+3"
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "fe61ab8db2f03058a0ce9593ef2c3320e14dee5741d4c57f433e0109d3f55670",
"SandboxKey": "/var/run/docker/netns/fe61ab8db2f0",
"Ports": {
"22/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "61670"
}
],
"2376/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "61666"
}
],
"32443/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "61667"
}
],
"5000/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "61668"
}
],
"8443/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "61669"
}
]
},
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "",
"Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"MacAddress": "",
"Networks": {
"old-k8s-version-326000": {
"IPAMConfig": {
"IPv4Address": "192.168.85.2"
},
"Links": null,
"Aliases": [
"76ad634e3f3f",
"old-k8s-version-326000"
],
"MacAddress": "02:42:c0:a8:55:02",
"NetworkID": "d3b7a706276d3d554def8a4b60d3b2d38626f3b90daf1316a28cf50ae9bb155f",
"EndpointID": "b031a27c1724c8987e54177c03d8b226685ddc850d77afdccebb384c1cf0ebf2",
"Gateway": "192.168.85.1",
"IPAddress": "192.168.85.2",
"IPPrefixLen": 24,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"DriverOpts": null,
"DNSNames": [
"old-k8s-version-326000",
"76ad634e3f3f"
]
}
}
}
}
]
-- /stdout --
helpers_test.go:239: (dbg) Run: out/minikube-darwin-amd64 status --format={{.Host}} -p old-k8s-version-326000 -n old-k8s-version-326000
helpers_test.go:239: (dbg) Non-zero exit: out/minikube-darwin-amd64 status --format={{.Host}} -p old-k8s-version-326000 -n old-k8s-version-326000: exit status 6 (448.287149ms)
-- stdout --
Running
WARNING: Your kubectl is pointing to stale minikube-vm.
To fix the kubectl context, run `minikube update-context`
-- /stdout --
** stderr **
E0226 03:29:56.876835 26565 status.go:415] kubeconfig endpoint: extract IP: "old-k8s-version-326000" does not appear in /Users/jenkins/minikube-integration/18222-9538/kubeconfig
** /stderr **
helpers_test.go:239: status error: exit status 6 (may be ok)
helpers_test.go:241: "old-k8s-version-326000" host is not running, skipping log retrieval (state="Running\nWARNING: Your kubectl is pointing to stale minikube-vm.\nTo fix the kubectl context, run `minikube update-context`")
--- FAIL: TestStartStop/group/old-k8s-version/serial/FirstStart (259.23s)