Activated service account credentials for: [prow-build@k8s-infra-prow-build.iam.gserviceaccount.com] fatal: not a git repository (or any of the parent directories): .git + WRAPPED_COMMAND_PID=31 + wait 31 + /workspace/scenarios/kubernetes_e2e.py --cluster=gce-scale-cluster --env=CONCURRENT_SERVICE_SYNCS=5 --env=HEAPSTER_MACHINE_TYPE=e2-standard-32 --extract=ci/latest-fast --extract-ci-bucket=k8s-release-dev '--env=CONTROLLER_MANAGER_TEST_ARGS=--profiling --kube-api-qps=100 --kube-api-burst=100 --endpointslice-updates-batch-period=500ms --endpoint-updates-batch-period=500ms' --gcp-master-image=gci --gcp-node-image=gci --gcp-node-size=e2-small --gcp-nodes=5000 --gcp-project-type=scalability-scale-project --gcp-ssh-proxy-instance-name=gce-scale-cluster-master --gcp-zone=us-east1-b --ginkgo-parallel=40 --provider=gce '--test_args=--ginkgo.skip=\[Driver:.gcepd\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[DisabledForLargeClusters\] --minStartupPods=8 --node-schedulable-timeout=90m' --timeout=240m --use-logexporter --logexporter-gcs-path=gs://k8s-infra-scalability-tests-logs/ci-kubernetes-e2e-gce-scale-correctness/1521822220547002368 starts with local mode Environment: API_SERVER_TEST_LOG_LEVEL=--v=3 ARTIFACTS=/logs/artifacts BAZEL_REMOTE_CACHE_ENABLED=false BAZEL_VERSION=3.4.1 BOSKOS_METRICS_PORT=tcp://10.35.242.179:9090 BOSKOS_METRICS_PORT_9090_TCP=tcp://10.35.242.179:9090 BOSKOS_METRICS_PORT_9090_TCP_ADDR=10.35.242.179 BOSKOS_METRICS_PORT_9090_TCP_PORT=9090 BOSKOS_METRICS_PORT_9090_TCP_PROTO=tcp BOSKOS_METRICS_SERVICE_HOST=10.35.242.179 BOSKOS_METRICS_SERVICE_PORT=9090 BOSKOS_METRICS_SERVICE_PORT_METRICS=9090 BOSKOS_PORT=tcp://10.35.241.148:80 BOSKOS_PORT_80_TCP=tcp://10.35.241.148:80 BOSKOS_PORT_80_TCP_ADDR=10.35.241.148 BOSKOS_PORT_80_TCP_PORT=80 BOSKOS_PORT_80_TCP_PROTO=tcp BOSKOS_SERVICE_HOST=10.35.241.148 BOSKOS_SERVICE_PORT=80 BOSKOS_SERVICE_PORT_DEFAULT=80 BUILD_ID=1521822220547002368 BUILD_NUMBER=1521822220547002368 CI=true CL2_ALLOWED_SLOW_API_CALLS=1 CL2_SCHEDULER_THROUGHPUT_THRESHOLD=90 CLOUDSDK_COMPONENT_MANAGER_DISABLE_UPDATE_CHECK=true CLOUDSDK_CORE_DISABLE_PROMPTS=1 CLOUDSDK_EXPERIMENTAL_FAST_COMPONENT_UPDATE=false CONCURRENT_SERVICE_SYNCS=5 CONTROLLER_MANAGER_TEST_ARGS=--profiling --kube-api-qps=100 --kube-api-burst=100 --endpointslice-updates-batch-period=500ms --endpoint-updates-batch-period=500ms CREATE_CUSTOM_NETWORK=true DEPLOY_GCI_DRIVER=true DOCKER_IN_DOCKER_ENABLED=false DOCKER_IN_DOCKER_IPV6_ENABLED=false DUMP_TO_GCS_ONLY=true E2E_GOOGLE_APPLICATION_CREDENTIALS=/etc/service-account/service-account.json ENABLE_PROMETHEUS_SERVER=true ENTRYPOINT_OPTIONS={"timeout":16200000000000,"grace_period":900000000000,"artifact_dir":"/logs/artifacts","args":["runner.sh","/workspace/scenarios/kubernetes_e2e.py","--cluster=gce-scale-cluster","--env=CONCURRENT_SERVICE_SYNCS=5","--env=HEAPSTER_MACHINE_TYPE=e2-standard-32","--extract=ci/latest-fast","--extract-ci-bucket=k8s-release-dev","--env=CONTROLLER_MANAGER_TEST_ARGS=--profiling --kube-api-qps=100 --kube-api-burst=100 --endpointslice-updates-batch-period=500ms --endpoint-updates-batch-period=500ms","--gcp-master-image=gci","--gcp-node-image=gci","--gcp-node-size=e2-small","--gcp-nodes=5000","--gcp-project-type=scalability-scale-project","--gcp-ssh-proxy-instance-name=gce-scale-cluster-master","--gcp-zone=us-east1-b","--ginkgo-parallel=40","--provider=gce","--test_args=--ginkgo.skip=\\[Driver:.gcepd\\]|\\[Serial\\]|\\[Disruptive\\]|\\[Flaky\\]|\\[Feature:.+\\]|\\[DisabledForLargeClusters\\] --minStartupPods=8 --node-schedulable-timeout=90m","--timeout=240m","--use-logexporter","--logexporter-gcs-path=gs://k8s-infra-scalability-tests-logs/ci-kubernetes-e2e-gce-scale-correctness/1521822220547002368"],"container_name":"test","process_log":"/logs/process-log.txt","marker_file":"/logs/marker-file.txt","metadata_file":"/logs/artifacts/metadata.json"} ETCD_COMPACTION_INTERVAL_SEC=150 ETCD_EXTRA_ARGS=--enable-pprof GCE_SSH_PRIVATE_KEY_FILE=/etc/ssh-key-secret/ssh-private GCE_SSH_PUBLIC_KEY_FILE=/etc/ssh-key-secret/ssh-public GOOGLE_APPLICATION_CREDENTIALS=/etc/service-account/service-account.json GOOGLE_APPLICATION_CREDENTIALS_DEPRECATED=Migrate to workload identity, contact sig-testing GOPATH=/home/prow/go GOPROXY=https://proxy.golang.org GO_TARBALL=go1.18.1.linux-amd64.tar.gz HEAPSTER_MACHINE_TYPE=e2-standard-32 HOME=/workspace HOSTNAME=e0ab2a37-cba1-11ec-87b6-ee8a32404114 IMAGE=gcr.io/k8s-staging-test-infra/kubekins-e2e:v20220428-de61deb68b-master INSTANCE_PREFIX=gce-scale-cluster JENKINS_GCE_SSH_PRIVATE_KEY_FILE=/workspace/.ssh/google_compute_engine JENKINS_GCE_SSH_PUBLIC_KEY_FILE=/workspace/.ssh/google_compute_engine.pub JOB_NAME=ci-kubernetes-e2e-gce-scale-correctness JOB_SPEC={"type":"periodic","job":"ci-kubernetes-e2e-gce-scale-correctness","buildid":"1521822220547002368","prowjobid":"e0ab2a37-cba1-11ec-87b6-ee8a32404114","decoration_config":{"timeout":"4h30m0s","grace_period":"15m0s","utility_images":{"clonerefs":"gcr.io/k8s-prow/clonerefs:v20220503-01ebd50188","initupload":"gcr.io/k8s-prow/initupload:v20220503-01ebd50188","entrypoint":"gcr.io/k8s-prow/entrypoint:v20220503-01ebd50188","sidecar":"gcr.io/k8s-prow/sidecar:v20220503-01ebd50188"},"resources":{"clonerefs":{"requests":{"cpu":"100m"}},"initupload":{"requests":{"cpu":"100m"}},"place_entrypoint":{"requests":{"cpu":"100m"}},"sidecar":{"requests":{"cpu":"100m"}}},"gcs_configuration":{"bucket":"kubernetes-jenkins","path_strategy":"legacy","default_org":"kubernetes","default_repo":"kubernetes"},"gcs_credentials_secret":"service-account"}} JOB_TYPE=periodic KUBECTL_PRUNE_WHITELIST_OVERRIDE=core/v1/ConfigMap core/v1/Endpoints core/v1/Namespace core/v1/PersistentVolumeClaim core/v1/PersistentVolume core/v1/ReplicationController core/v1/Secret core/v1/Service batch/v1/Job batch/v1beta1/CronJob apps/v1/DaemonSet apps/v1/Deployment apps/v1/ReplicaSet apps/v1/StatefulSet networking.k8s.io/v1/Ingress KUBELET_TEST_ARGS=--enable-debugging-handlers --kube-api-qps=100 --kube-api-burst=100 KUBEPROXY_TEST_ARGS=--profiling --metrics-bind-address=0.0.0.0 KUBERNETES_PORT=tcp://10.35.240.1:443 KUBERNETES_PORT_443_TCP=tcp://10.35.240.1:443 KUBERNETES_PORT_443_TCP_ADDR=10.35.240.1 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT_443_TCP_PROTO=tcp KUBERNETES_SERVICE_HOST=10.35.240.1 KUBERNETES_SERVICE_PORT=443 KUBERNETES_SERVICE_PORT_HTTPS=443 KUBETEST_IN_DOCKER=true KUBETEST_MANUAL_DUMP=y KUBE_AWS_INSTANCE_PREFIX=gce-scale-cluster KUBE_ENABLE_CLUSTER_UI=false KUBE_GCE_ENABLE_IP_ALIASES=true KUBE_GCE_INSTANCE_PREFIX=gce-scale-cluster KUBE_GCE_PRIVATE_CLUSTER=true KUBE_MASTER_NODE_LABELS=node.kubernetes.io/node-exporter-ready=true LOGROTATE_FILES_MAX_COUNT=1000 LOGROTATE_MAX_SIZE=5G LOG_DUMP_EXPECTED_SUCCESS_PERCENTAGE=50 LOG_DUMP_EXTRA_FILES=cl2-* LOG_DUMP_SAVE_SERVICES=containerd LOG_DUMP_SCRIPT_PATH=/workspace/log-dump.sh LOG_DUMP_SSH_TIMEOUT_SECONDS=3600 LOG_DUMP_SYSTEMD_JOURNAL=true MASTER_MIN_CPU_ARCHITECTURE=Intel Skylake NODE_DISK_SIZE=50GB NODE_KUBELET_TEST_ARGS=--kube-reserved=cpu=1050m NODE_SIZE=e2-medium PATH=/home/prow/go/bin:/go/bin:/usr/local/go/bin:/google-cloud-sdk/bin:/workspace:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin PERF_TESTS_PRINT_COMMIT_HISTORY=true PROMETHEUS_SCRAPE_ETCD=true PROMETHEUS_SCRAPE_MASTER_KUBELETS=true PROMETHEUS_STORAGE_CLASS_PROVISIONER=pd.csi.storage.gke.io PROW_JOB_ID=e0ab2a37-cba1-11ec-87b6-ee8a32404114 PWD=/workspace REGISTER_MASTER=true SCHEDULER_TEST_ARGS=--profiling --kube-api-qps=100 --kube-api-burst=100 SHLVL=0 SOURCE_DATE_EPOCH= TERM=xterm TEST_CLUSTER_DELETE_COLLECTION_WORKERS=--delete-collection-workers=16 TEST_CLUSTER_LOG_LEVEL=--v=2 TEST_CLUSTER_RESYNC_PERIOD=--min-resync-period=12h USER=prow USE_TEST_INFRA_LOG_DUMPING=true WORKSPACE=/workspace _=/workspace/scenarios/kubernetes_e2e.py Run: ('kubetest', '--dump=/logs/artifacts', '--gcp-service-account=/etc/service-account/service-account.json', '--up', '--down', '--test', '--provider=gce', '--cluster=gce-scale-cluster', '--gcp-network=gce-scale-cluster', '--extract=ci/latest-fast', '--extract-ci-bucket=k8s-release-dev', '--gcp-master-image=gci', '--gcp-node-image=gci', '--gcp-node-size=e2-small', '--gcp-nodes=5000', '--gcp-project-type=scalability-scale-project', '--gcp-ssh-proxy-instance-name=gce-scale-cluster-master', '--gcp-zone=us-east1-b', '--ginkgo-parallel=40', '--test_args=--ginkgo.skip=\\[Driver:.gcepd\\]|\\[Serial\\]|\\[Disruptive\\]|\\[Flaky\\]|\\[Feature:.+\\]|\\[DisabledForLargeClusters\\] --minStartupPods=8 --node-schedulable-timeout=90m', '--timeout=240m', '--logexporter-gcs-path=gs://k8s-infra-scalability-tests-logs/ci-kubernetes-e2e-gce-scale-correctness/1521822220547002368') 2022/05/04 12:02:12 Warning: Couldn't find directory src/k8s.io/kubernetes under any of GOPATH /home/prow/go, defaulting to /home/prow/go/src/k8s.io/kubernetes 2022/05/04 12:02:12 Warning: Couldn't find directory src/sigs.k8s.io/cloud-provider-azure under any of GOPATH /home/prow/go, defaulting to /home/prow/go/src/k8s.io/cloud-provider-azure 2022/05/04 12:02:12 Warning: Couldn't find directory src/k8s.io/kubernetes under any of GOPATH /home/prow/go, defaulting to /home/prow/go/src/k8s.io/kubernetes 2022/05/04 12:02:12 main.go:284: Running kubetest version: 2022/05/04 12:02:12 extract_k8s.go:118: Matched extraction strategy: ^ci/(.+)-fast$ 2022/05/04 12:02:12 main.go:344: Limiting testing to 4h0m0s 2022/05/04 12:02:12 process.go:153: Running: gcloud auth activate-service-account --key-file=/etc/service-account/service-account.json Activated service account credentials for: [prow-build@k8s-infra-prow-build.iam.gserviceaccount.com] 2022/05/04 12:02:12 process.go:155: Step 'gcloud auth activate-service-account --key-file=/etc/service-account/service-account.json' finished in 729.449143ms 2022/05/04 12:02:12 main.go:749: --gcp-project is missing, trying to fetch a project from boskos. (for local runs please set --gcp-project to your dev project) 2022/05/04 12:02:12 main.go:761: provider gce, will acquire project type scalability-scale-project from boskos 2022/05/04 12:02:12 process.go:153: Running: gcloud config set project k8s-infra-e2e-scale-5k-project WARNING: You do not appear to have access to project [k8s-infra-e2e-scale-5k-project] or it does not exist. Updated property [core/project]. 2022/05/04 12:02:13 process.go:155: Step 'gcloud config set project k8s-infra-e2e-scale-5k-project' finished in 611.992676ms 2022/05/04 12:02:13 main.go:800: Checking existing of GCP ssh keys... 2022/05/04 12:02:13 main.go:810: Checking presence of public key in k8s-infra-e2e-scale-5k-project 2022/05/04 12:02:13 process.go:153: Running: gcloud compute --project=k8s-infra-e2e-scale-5k-project project-info describe 2022/05/04 12:02:14 process.go:155: Step 'gcloud compute --project=k8s-infra-e2e-scale-5k-project project-info describe' finished in 977.666453ms 2022/05/04 12:02:14 extract_k8s.go:149: rm kubernetes 2022/05/04 12:02:14 extract_k8s.go:301: U=https://storage.googleapis.com/k8s-release-dev/ci/fast R=v1.25.0-alpha.0.195+094a33ad801065 get-kube.sh 2022/05/04 12:02:14 process.go:153: Running: ./get-kube.sh Downloading kubernetes release v1.25.0-alpha.0.195+094a33ad801065 from https://storage.googleapis.com/k8s-release-dev/ci/fast/v1.25.0-alpha.0.195+094a33ad801065/kubernetes.tar.gz to /workspace/kubernetes.tar.gz Copying gs://k8s-release-dev/ci/fast/v1.25.0-alpha.0.195+094a33ad801065/kubernetes.tar.gz... / [0 files][ 0.0 B/510.3 KiB] / [1 files][510.3 KiB/510.3 KiB] Operation completed over 1 objects/510.3 KiB. Unpacking kubernetes release v1.25.0-alpha.0.195+094a33ad801065 Kubernetes release: v1.25.0-alpha.0.195+094a33ad801065 Server: linux/amd64 (to override, set KUBERNETES_SERVER_ARCH) Client: linux/amd64 (autodetected) (to override, set KUBERNETES_CLIENT_OS and/or KUBERNETES_CLIENT_ARCH) Will download kubernetes-server-linux-amd64.tar.gz from https://storage.googleapis.com/k8s-release-dev/ci/fast/v1.25.0-alpha.0.195+094a33ad801065 Will download and extract kubernetes-client-linux-amd64.tar.gz from https://storage.googleapis.com/k8s-release-dev/ci/fast/v1.25.0-alpha.0.195+094a33ad801065 Will download and extract kubernetes-test tarball(s) from https://storage.googleapis.com/k8s-release-dev/ci/fast/v1.25.0-alpha.0.195+094a33ad801065 Copying gs://k8s-release-dev/ci/fast/v1.25.0-alpha.0.195+094a33ad801065/kubernetes-server-linux-amd64.tar.gz... / [0 files][ 0.0 B/332.7 MiB] ==> NOTE: You are downloading one or more large file(s), which would run significantly faster if you enabled sliced object downloads. This feature is enabled by default but requires that compiled crcmod be installed (see "gsutil help crcmod"). - - [0 files][161.6 MiB/332.7 MiB] \ | | [1 files][332.7 MiB/332.7 MiB] Operation completed over 1 objects/332.7 MiB. md5sum(kubernetes-server-linux-amd64.tar.gz)=5aa2136df7390730e025bd0105729042 sha512sum(kubernetes-server-linux-amd64.tar.gz)=f229de79dd9d2f03c726b57f0d5bd63368311225d396ee40b2cc394848598ccaba107541d0acebfa2cc8bfc3e21cf5d586ee127eec63350d4173f1fa6df81c04 Copying gs://k8s-release-dev/ci/fast/v1.25.0-alpha.0.195+094a33ad801065/kubernetes-client-linux-amd64.tar.gz... / [0 files][ 0.0 B/ 29.0 MiB] / [1 files][ 29.0 MiB/ 29.0 MiB] Operation completed over 1 objects/29.0 MiB. md5sum(kubernetes-client-linux-amd64.tar.gz)=d6233366ee1ae63a0cb28d4b49f01cc3 sha512sum(kubernetes-client-linux-amd64.tar.gz)=cf72d36fc4bdf92707b1af5634cdff3d27cac57590cf96af8eb2a2be89b6e0a8b15a952f2c485df39e50ef9238aee797a770316d6101395ba6a2fd9fcdc65665 Extracting /workspace/kubernetes/client/kubernetes-client-linux-amd64.tar.gz into /workspace/kubernetes/platforms/linux/amd64 Add '/workspace/kubernetes/client/bin' to your PATH to use newly-installed binaries. Copying gs://k8s-release-dev/ci/fast/v1.25.0-alpha.0.195+094a33ad801065/kubernetes-test-portable.tar.gz... / [0 files][ 0.0 B/248.2 KiB] / [1 files][248.2 KiB/248.2 KiB] Operation completed over 1 objects/248.2 KiB. md5sum(kubernetes-test-portable.tar.gz)=74e481e7cbedd6da169eacf68f9e8cc2 sha512sum(kubernetes-test-portable.tar.gz)=ecb04369ce27bc62836175c57763de5603817bac16e986814e3b418fa416eabaf488474ee288ea4ed68110d855c3bac4e59166ce1b237ad6216e2f09144c1b1e Extracting kubernetes-test-portable.tar.gz into /workspace/kubernetes Copying gs://k8s-release-dev/ci/fast/v1.25.0-alpha.0.195+094a33ad801065/kubernetes-test-linux-amd64.tar.gz... / [0 files][ 0.0 B/261.5 MiB] ==> NOTE: You are downloading one or more large file(s), which would run significantly faster if you enabled sliced object downloads. This feature is enabled by default but requires that compiled crcmod be installed (see "gsutil help crcmod"). - - [0 files][157.3 MiB/261.5 MiB] \ \ [1 files][261.5 MiB/261.5 MiB] Operation completed over 1 objects/261.5 MiB. md5sum(kubernetes-test-linux-amd64.tar.gz)=c38d44923c4c04576ce70f4c38029e85 sha512sum(kubernetes-test-linux-amd64.tar.gz)=b9506df2a952b3d3a6a9a5488f6589104c801b85e93afac5ca79244fb73a66d6194d74a3fc1f3e9fb4cdb62e6fff2faff510fdd2cabe8c12a960ef7e7d4a7b83 Extracting /workspace/kubernetes/test/kubernetes-test-linux-amd64.tar.gz into /workspace/kubernetes/platforms/linux/amd64 2022/05/04 12:02:35 process.go:155: Step './get-kube.sh' finished in 20.918562345s 2022/05/04 12:02:35 process.go:153: Running: ./hack/e2e-internal/e2e-down.sh Project: k8s-infra-e2e-scale-5k-project Network Project: k8s-infra-e2e-scale-5k-project Zone: us-east1-b Shutting down test cluster in background. Bringing down cluster using provider: gce ... calling verify-prereqs -c:5: DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead. -c:12: DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead. ... calling verify-kube-binaries ... calling kube-down Project: k8s-infra-e2e-scale-5k-project Network Project: k8s-infra-e2e-scale-5k-project Zone: us-east1-b WARNING: The following filter keys were not present in any resource : name, zone WARNING: The following filter keys were not present in any resource : name, zone INSTANCE_GROUPS= NODE_NAMES=gce-scale-cluster-minion-heapster Bringing down cluster WARNING: The following filter keys were not present in any resource : name WARNING: The following filter keys were not present in any resource : name, zone WARNING: The following filter keys were not present in any resource : name WARNING: The following filter keys were not present in any resource : name WARNING: The following filter keys were not present in any resource : name, zone Deleting firewall rules remaining in network gce-scale-cluster: Removing auto-created subnet gce-scale-cluster:gce-scale-cluster-custom-subnet W0504 12:03:03.457101 1668 loader.go:221] Config not found: /workspace/.kube/config W0504 12:03:03.557882 1722 loader.go:221] Config not found: /workspace/.kube/config W0504 12:03:03.558116 1722 loader.go:221] Config not found: /workspace/.kube/config Property "clusters.k8s-infra-e2e-scale-5k-project_gce-scale-cluster" unset. W0504 12:03:03.661898 1775 loader.go:221] Config not found: /workspace/.kube/config W0504 12:03:03.662059 1775 loader.go:221] Config not found: /workspace/.kube/config Property "users.k8s-infra-e2e-scale-5k-project_gce-scale-cluster" unset. W0504 12:03:03.761950 1828 loader.go:221] Config not found: /workspace/.kube/config W0504 12:03:03.762112 1828 loader.go:221] Config not found: /workspace/.kube/config Property "users.k8s-infra-e2e-scale-5k-project_gce-scale-cluster-basic-auth" unset. W0504 12:03:03.862645 1882 loader.go:221] Config not found: /workspace/.kube/config W0504 12:03:03.862775 1882 loader.go:221] Config not found: /workspace/.kube/config Property "contexts.k8s-infra-e2e-scale-5k-project_gce-scale-cluster" unset. Cleared config for k8s-infra-e2e-scale-5k-project_gce-scale-cluster from /workspace/.kube/config Done 2022/05/04 12:03:03 process.go:155: Step './hack/e2e-internal/e2e-down.sh' finished in 28.367092302s 2022/05/04 12:03:03 process.go:153: Running: ./hack/e2e-internal/e2e-up.sh Project: k8s-infra-e2e-scale-5k-project Network Project: k8s-infra-e2e-scale-5k-project Zone: us-east1-b ... Starting cluster in us-east1-b using provider gce ... calling verify-prereqs -c:5: DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead. -c:12: DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead. ... calling verify-kube-binaries ... calling verify-release-tars ... calling kube-up Project: k8s-infra-e2e-scale-5k-project Network Project: k8s-infra-e2e-scale-5k-project Zone: us-east1-b +++ Staging tars to Google Storage: gs://kubernetes-staging-ae2a94cd61/gce-scale-cluster-devel +++ kubernetes-server-linux-amd64.tar.gz uploaded (sha512 = f229de79dd9d2f03c726b57f0d5bd63368311225d396ee40b2cc394848598ccaba107541d0acebfa2cc8bfc3e21cf5d586ee127eec63350d4173f1fa6df81c04) +++ kubernetes-manifests.tar.gz uploaded (sha512 = 5ac5195f6e212c5a747623c916b0fca0a9e5b7abcd8afc6d7f4490b7f1bc1f4a112d3814187b63bb202dc34aa9dd23a042f5388cfb12cdb1642de4f67ea33457) Creating new custom network: gce-scale-cluster Created [https://www.googleapis.com/compute/v1/projects/k8s-infra-e2e-scale-5k-project/global/networks/gce-scale-cluster]. NAME SUBNET_MODE BGP_ROUTING_MODE IPV4_RANGE GATEWAY_IPV4 gce-scale-cluster CUSTOM REGIONAL Instances on this network will not be reachable until firewall rules are created. As an example, you can allow all internal traffic between instances as well as SSH, RDP, and ICMP by running: $ gcloud compute firewall-rules create --network gce-scale-cluster --allow tcp,udp,icmp --source-ranges $ gcloud compute firewall-rules create --network gce-scale-cluster --allow tcp:22,tcp:3389,icmp IP aliases are enabled. Creating subnetworks. Creating subnet gce-scale-cluster:gce-scale-cluster-custom-subnet Creating firewall... ..Created [https://www.googleapis.com/compute/v1/projects/k8s-infra-e2e-scale-5k-project/global/firewalls/gce-scale-cluster-default-internal-master]. NAME NETWORK DIRECTION PRIORITY ALLOW DENY DISABLED gce-scale-cluster-default-internal-master gce-scale-cluster INGRESS 1000 tcp:1-2379,tcp:2382-65535,udp:1-65535,icmp False done. Creating firewall... ..Created [https://www.googleapis.com/compute/v1/projects/k8s-infra-e2e-scale-5k-project/global/firewalls/gce-scale-cluster-default-ssh]. Creating firewall... ..Created [https://www.googleapis.com/compute/v1/projects/k8s-infra-e2e-scale-5k-project/global/firewalls/gce-scale-cluster-default-internal-node]. NAME NETWORK DIRECTION PRIORITY ALLOW DENY DISABLED gce-scale-cluster-default-ssh gce-scale-cluster INGRESS 1000 tcp:22 False done. NAME NETWORK DIRECTION PRIORITY ALLOW DENY DISABLED gce-scale-cluster-default-internal-node gce-scale-cluster INGRESS 1000 tcp:1-65535,udp:1-65535,icmp False done. Created [https://www.googleapis.com/compute/v1/projects/k8s-infra-e2e-scale-5k-project/regions/us-east1/subnetworks/gce-scale-cluster-custom-subnet]. NAME REGION NETWORK RANGE STACK_TYPE IPV6_ACCESS_TYPE INTERNAL_IPV6_PREFIX EXTERNAL_IPV6_PREFIX gce-scale-cluster-custom-subnet us-east1 gce-scale-cluster 10.40.0.0/19 IPV4_ONLY Created subnetwork gce-scale-cluster-custom-subnet Using subnet gce-scale-cluster-custom-subnet NAME REGION NETWORK gce-scale-cluster-nat-router us-east1 gce-scale-cluster Creating router [gce-scale-cluster-nat-router]... .....done. Creating NAT [gce-scale-cluster-nat-config] in router [gce-scale-cluster-nat-router]... .................................................done. Starting master and configuring firewalls Configuring firewall for apiserver konnectivity server Created [https://www.googleapis.com/compute/v1/projects/k8s-infra-e2e-scale-5k-project/zones/us-east1-b/disks/gce-scale-cluster-master-pd]. NAME ZONE SIZE_GB TYPE STATUS gce-scale-cluster-master-pd us-east1-b 200 pd-ssd READY New disks are unformatted. You must format and mount a disk before it can be used. You can find instructions on how to do this at: https://cloud.google.com/compute/docs/disks/add-persistent-disk#formatting Created [https://www.googleapis.com/compute/v1/projects/k8s-infra-e2e-scale-5k-project/regions/us-east1/addresses/gce-scale-cluster-master-ip]. Created [https://www.googleapis.com/compute/v1/projects/k8s-infra-e2e-scale-5k-project/regions/us-east1/addresses/gce-scale-cluster-master-internal-ip]. Creating firewall... ..Created [https://www.googleapis.com/compute/v1/projects/k8s-infra-e2e-scale-5k-project/global/firewalls/gce-scale-cluster-master-https]. NAME NETWORK DIRECTION PRIORITY ALLOW DENY DISABLED gce-scale-cluster-master-https gce-scale-cluster INGRESS 1000 tcp:443 False done. Master internal ip is: 10.40.0.2 Generating certs for alternate-names: IP:34.75.174.135,IP:10.40.0.2,IP:10.0.0.1,DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.cluster.local,DNS:gce-scale-cluster-master Creating firewall... ..Created [https://www.googleapis.com/compute/v1/projects/k8s-infra-e2e-scale-5k-project/global/firewalls/gce-scale-cluster-master-etcd]. Creating firewall... ..Created [https://www.googleapis.com/compute/v1/projects/k8s-infra-e2e-scale-5k-project/global/firewalls/gce-scale-cluster-master-konnectivity-server]. NAME NETWORK DIRECTION PRIORITY ALLOW DENY DISABLED gce-scale-cluster-master-etcd gce-scale-cluster INGRESS 1000 tcp:2380,tcp:2381 False done. NAME NETWORK DIRECTION PRIORITY ALLOW DENY DISABLED gce-scale-cluster-master-konnectivity-server gce-scale-cluster INGRESS 1000 tcp:8132 False done. 2022/05/04 12:04:16 [INFO] generating a new CA key and certificate from CSR 2022/05/04 12:04:16 [INFO] generate received request 2022/05/04 12:04:16 [INFO] received CSR 2022/05/04 12:04:16 [INFO] generating key: ecdsa-256 2022/05/04 12:04:16 [INFO] encoded CSR 2022/05/04 12:04:16 [INFO] signed certificate with serial number 680525531986560022980011257512232731298941374353 Generate peer certificates... 2022/05/04 12:04:16 [INFO] generate received request 2022/05/04 12:04:16 [INFO] received CSR 2022/05/04 12:04:16 [INFO] generating key: ecdsa-256 2022/05/04 12:04:16 [INFO] encoded CSR 2022/05/04 12:04:16 [INFO] signed certificate with serial number 462685695291729773139709573157527055387944047756 Generate server certificates... 2022/05/04 12:04:16 [INFO] generate received request 2022/05/04 12:04:16 [INFO] received CSR 2022/05/04 12:04:16 [INFO] generating key: ecdsa-256 2022/05/04 12:04:16 [INFO] encoded CSR 2022/05/04 12:04:16 [INFO] signed certificate with serial number 353604801932546882649566015151477806148942621003 Generate client certificates... 2022/05/04 12:04:16 [INFO] generate received request 2022/05/04 12:04:16 [INFO] received CSR 2022/05/04 12:04:16 [INFO] generating key: ecdsa-256 2022/05/04 12:04:16 [INFO] encoded CSR 2022/05/04 12:04:16 [INFO] signed certificate with serial number 295039245934739402818482805254090397946004319317 2022/05/04 12:04:16 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for websites. For more information see the Baseline Requirements for the Issuance and Management of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org); specifically, section 10.2.3 ("Information Requirements"). WARNING: Some requests generated warnings: - Disk size: '500 GB' is larger than image size: '10 GB'. You might need to resize the root repartition manually if the operating system does not support automatic resizing. See https://cloud.google.com/compute/docs/disks/add-persistent-disk#resize_pd for details. - The resource 'projects/cos-cloud/global/images/cos-85-13310-1308-1' is deprecated. A suggested replacement is 'projects/cos-cloud/global/images/cos-85-13310-1308-6'. ERROR: (gcloud.compute.instances.create) Could not fetch resource: - The zone 'projects/k8s-infra-e2e-scale-5k-project/zones/us-east1-b' does not have enough resources available to fulfill the request. '(resource type:compute)'. Failed to create master instance due to non-retryable error 2022/05/04 12:04:25 process.go:155: Step './hack/e2e-internal/e2e-up.sh' finished in 1m21.479640021s 2022/05/04 12:04:25 e2e.go:571: Dumping logs from nodes to GCS directly at path: gs://k8s-infra-scalability-tests-logs/ci-kubernetes-e2e-gce-scale-correctness/1521822220547002368 2022/05/04 12:04:25 process.go:153: Running: /workspace/log-dump.sh /logs/artifacts gs://k8s-infra-scalability-tests-logs/ci-kubernetes-e2e-gce-scale-correctness/1521822220547002368 Checking for custom logdump instances, if any Using gce provider, skipping check for LOG_DUMP_SSH_KEY and LOG_DUMP_SSH_USER Project: k8s-infra-e2e-scale-5k-project Network Project: k8s-infra-e2e-scale-5k-project Zone: us-east1-b Dumping logs temporarily to '/tmp/tmp.7t2u4VIevR/logs'. Will upload to 'gs://k8s-infra-scalability-tests-logs/ci-kubernetes-e2e-gce-scale-correctness/1521822220547002368' later. Dumping logs from master locally to '/tmp/tmp.7t2u4VIevR/logs' Trying to find master named 'gce-scale-cluster-master' Looking for address 'gce-scale-cluster-master-ip' Looking for address 'gce-scale-cluster-master-internal-ip' Using master: gce-scale-cluster-master (external IP: 34.75.174.135; internal IP: 10.40.0.2) Changing logfiles to be world-readable for download ERROR: (gcloud.compute.ssh) Could not fetch resource: - The resource 'projects/k8s-infra-e2e-scale-5k-project/zones/us-east1-b/instances/gce-scale-cluster-master' was not found ERROR: (gcloud.compute.ssh) Could not fetch resource: - The resource 'projects/k8s-infra-e2e-scale-5k-project/zones/us-east1-b/instances/gce-scale-cluster-master' was not found ERROR: (gcloud.compute.ssh) Could not fetch resource: - The resource 'projects/k8s-infra-e2e-scale-5k-project/zones/us-east1-b/instances/gce-scale-cluster-master' was not found ERROR: (gcloud.compute.ssh) Could not fetch resource: - The resource 'projects/k8s-infra-e2e-scale-5k-project/zones/us-east1-b/instances/gce-scale-cluster-master' was not found ERROR: (gcloud.compute.ssh) Could not fetch resource: - The resource 'projects/k8s-infra-e2e-scale-5k-project/zones/us-east1-b/instances/gce-scale-cluster-master' was not found ERROR: (gcloud.compute.ssh) Could not fetch resource: - The resource 'projects/k8s-infra-e2e-scale-5k-project/zones/us-east1-b/instances/gce-scale-cluster-master' was not found Copying 'kube-apiserver.log kube-apiserver-audit.log kube-scheduler.log kube-controller-manager.log etcd.log etcd-events.log glbc.log cluster-autoscaler.log kube-addon-manager.log konnectivity-server.log fluentd.log kubelet.cov cl2-* startupscript.log kern.log docker/log kubelet.log supervisor/supervisord.log supervisor/kubelet-stdout.log supervisor/kubelet-stderr.log supervisor/docker-stdout.log supervisor/docker-stderr.log' from gce-scale-cluster-master ERROR: (gcloud.compute.instances.get-serial-port-output) Could not fetch serial port output: The resource 'projects/k8s-infra-e2e-scale-5k-project/zones/us-east1-b/instances/gce-scale-cluster-master' was not found ERROR: (gcloud.compute.scp) Could not fetch resource: - The resource 'projects/k8s-infra-e2e-scale-5k-project/zones/us-east1-b/instances/gce-scale-cluster-master' was not found Dumping logs from nodes to GCS directly at 'gs://k8s-infra-scalability-tests-logs/ci-kubernetes-e2e-gce-scale-correctness/1521822220547002368' using logexporter The connection to the server localhost:8080 was refused - did you specify the right host or port? Failed to create logexporter daemonset.. falling back to logdump through SSH The connection to the server localhost:8080 was refused - did you specify the right host or port? Dumping logs for nodes provided as args to dump_nodes() function Changing logfiles to be world-readable for download ERROR: (gcloud.compute.ssh) Could not fetch resource: - The resource 'projects/k8s-infra-e2e-scale-5k-project/zones/us-east1-b/instances/gce-scale-cluster-minion-heapster' was not found ERROR: (gcloud.compute.ssh) Could not fetch resource: - The resource 'projects/k8s-infra-e2e-scale-5k-project/zones/us-east1-b/instances/gce-scale-cluster-minion-heapster' was not found ERROR: (gcloud.compute.ssh) Could not fetch resource: - The resource 'projects/k8s-infra-e2e-scale-5k-project/zones/us-east1-b/instances/gce-scale-cluster-minion-heapster' was not found ERROR: (gcloud.compute.ssh) Could not fetch resource: - The resource 'projects/k8s-infra-e2e-scale-5k-project/zones/us-east1-b/instances/gce-scale-cluster-minion-heapster' was not found ERROR: (gcloud.compute.ssh) Could not fetch resource: - The resource 'projects/k8s-infra-e2e-scale-5k-project/zones/us-east1-b/instances/gce-scale-cluster-minion-heapster' was not found ERROR: (gcloud.compute.ssh) Could not fetch resource: - The resource 'projects/k8s-infra-e2e-scale-5k-project/zones/us-east1-b/instances/gce-scale-cluster-minion-heapster' was not found Copying 'kube-proxy.log containers/konnectivity-agent-*.log fluentd.log node-problem-detector.log kubelet.cov cl2-* startupscript.log kern.log docker/log kubelet.log supervisor/supervisord.log supervisor/kubelet-stdout.log supervisor/kubelet-stderr.log supervisor/docker-stdout.log supervisor/docker-stderr.log' from gce-scale-cluster-minion-heapster ERROR: (gcloud.compute.instances.get-serial-port-output) Could not fetch serial port output: The resource 'projects/k8s-infra-e2e-scale-5k-project/zones/us-east1-b/instances/gce-scale-cluster-minion-heapster' was not found ERROR: (gcloud.compute.scp) Could not fetch resource: - The resource 'projects/k8s-infra-e2e-scale-5k-project/zones/us-east1-b/instances/gce-scale-cluster-minion-heapster' was not found Detecting nodes in the cluster WARNING: The following filter keys were not present in any resource : name, zone WARNING: The following filter keys were not present in any resource : name, zone INSTANCE_GROUPS= NODE_NAMES=gce-scale-cluster-minion-heapster WINDOWS_INSTANCE_GROUPS= WINDOWS_NODE_NAMES= The connection to the server localhost:8080 was refused - did you specify the right host or port? Uploading '/tmp/tmp.7t2u4VIevR/logs' to 'gs://k8s-infra-scalability-tests-logs/ci-kubernetes-e2e-gce-scale-correctness/1521822220547002368' CommandException: One or more URLs matched no objects. Copying file:///tmp/tmp.7t2u4VIevR/logs/nodes/node_names.txt [Content-Type=text/plain]... Copying file:///tmp/tmp.7t2u4VIevR/logs/nodes/kubectl_get_nodes.yaml [Content-Type=application/octet-stream]... Copying file:///tmp/tmp.7t2u4VIevR/logs/gce-scale-cluster-minion-heapster/serial-1.log [Content-Type=application/octet-stream]... Copying file:///tmp/tmp.7t2u4VIevR/logs/gce-scale-cluster-master/serial-1.log [Content-Type=application/octet-stream]... / [0/4 files][ 0.0 B/ 34.0 B] 0% Done / [0/4 files][ 0.0 B/ 34.0 B] 0% Done / [0/4 files][ 0.0 B/ 34.0 B] 0% Done / [0/4 files][ 0.0 B/ 34.0 B] 0% Done / [1/4 files][ 127.0 B/ 34.0 B] 99% Done / [2/4 files][ 127.0 B/ 34.0 B] 99% Done / [3/4 files][ 127.0 B/ 34.0 B] 99% Done / [4/4 files][ 127.0 B/ 34.0 B] 100% Done Operation completed over 4 objects/34.0 B. Detecting nodes in the cluster WARNING: The following filter keys were not present in any resource : name, zone WARNING: The following filter keys were not present in any resource : name, zone INSTANCE_GROUPS= NODE_NAMES=gce-scale-cluster-minion-heapster WINDOWS_INSTANCE_GROUPS= WINDOWS_NODE_NAMES= 2022/05/04 12:07:43 process.go:155: Step '/workspace/log-dump.sh /logs/artifacts gs://k8s-infra-scalability-tests-logs/ci-kubernetes-e2e-gce-scale-correctness/1521822220547002368' finished in 3m18.226944486s 2022/05/04 12:07:43 process.go:153: Running: ./hack/e2e-internal/e2e-down.sh Project: k8s-infra-e2e-scale-5k-project Network Project: k8s-infra-e2e-scale-5k-project Zone: us-east1-b Shutting down test cluster in background. Bringing down cluster using provider: gce ... calling verify-prereqs -c:5: DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead. -c:12: DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead. ... calling verify-kube-binaries ... calling kube-down Project: k8s-infra-e2e-scale-5k-project Network Project: k8s-infra-e2e-scale-5k-project Zone: us-east1-b WARNING: The following filter keys were not present in any resource : name, zone WARNING: The following filter keys were not present in any resource : name, zone INSTANCE_GROUPS= NODE_NAMES=gce-scale-cluster-minion-heapster Bringing down cluster WARNING: The following filter keys were not present in any resource : name WARNING: The following filter keys were not present in any resource : name, zone WARNING: The following filter keys were not present in any resource : name Deleted [https://www.googleapis.com/compute/v1/projects/k8s-infra-e2e-scale-5k-project/zones/us-east1-b/disks/gce-scale-cluster-master-pd]. WARNING: The following filter keys were not present in any resource : name Deleted [https://www.googleapis.com/compute/v1/projects/k8s-infra-e2e-scale-5k-project/global/firewalls/gce-scale-cluster-master-https]. Deleted [https://www.googleapis.com/compute/v1/projects/k8s-infra-e2e-scale-5k-project/global/firewalls/gce-scale-cluster-master-konnectivity-server]. Deleted [https://www.googleapis.com/compute/v1/projects/k8s-infra-e2e-scale-5k-project/global/firewalls/gce-scale-cluster-master-etcd]. Deleted [https://www.googleapis.com/compute/v1/projects/k8s-infra-e2e-scale-5k-project/regions/us-east1/addresses/gce-scale-cluster-master-ip]. Deleted [https://www.googleapis.com/compute/v1/projects/k8s-infra-e2e-scale-5k-project/regions/us-east1/addresses/gce-scale-cluster-master-internal-ip]. WARNING: The following filter keys were not present in any resource : name, zone Deleted [https://www.googleapis.com/compute/v1/projects/k8s-infra-e2e-scale-5k-project/global/firewalls/gce-scale-cluster-default-internal-master]. Deleted [https://www.googleapis.com/compute/v1/projects/k8s-infra-e2e-scale-5k-project/global/firewalls/gce-scale-cluster-default-ssh]. Deleted [https://www.googleapis.com/compute/v1/projects/k8s-infra-e2e-scale-5k-project/global/firewalls/gce-scale-cluster-default-internal-node]. Deleting Cloud NAT router... Deleted [https://www.googleapis.com/compute/v1/projects/k8s-infra-e2e-scale-5k-project/regions/us-east1/routers/gce-scale-cluster-nat-router]. Deleting firewall rules remaining in network gce-scale-cluster: Deleting custom subnet... Deleted [https://www.googleapis.com/compute/v1/projects/k8s-infra-e2e-scale-5k-project/regions/us-east1/subnetworks/gce-scale-cluster-custom-subnet]. Deleted [https://www.googleapis.com/compute/v1/projects/k8s-infra-e2e-scale-5k-project/global/networks/gce-scale-cluster]. W0504 12:09:16.311163 8484 loader.go:221] Config not found: /workspace/.kube/config W0504 12:09:16.431692 8537 loader.go:221] Config not found: /workspace/.kube/config W0504 12:09:16.431879 8537 loader.go:221] Config not found: /workspace/.kube/config Property "clusters.k8s-infra-e2e-scale-5k-project_gce-scale-cluster" unset. W0504 12:09:16.533778 8590 loader.go:221] Config not found: /workspace/.kube/config W0504 12:09:16.533921 8590 loader.go:221] Config not found: /workspace/.kube/config Property "users.k8s-infra-e2e-scale-5k-project_gce-scale-cluster" unset. W0504 12:09:16.633275 8643 loader.go:221] Config not found: /workspace/.kube/config W0504 12:09:16.633416 8643 loader.go:221] Config not found: /workspace/.kube/config Property "users.k8s-infra-e2e-scale-5k-project_gce-scale-cluster-basic-auth" unset. W0504 12:09:16.731298 8696 loader.go:221] Config not found: /workspace/.kube/config W0504 12:09:16.731435 8696 loader.go:221] Config not found: /workspace/.kube/config Property "contexts.k8s-infra-e2e-scale-5k-project_gce-scale-cluster" unset. Cleared config for k8s-infra-e2e-scale-5k-project_gce-scale-cluster from /workspace/.kube/config Done 2022/05/04 12:09:16 process.go:155: Step './hack/e2e-internal/e2e-down.sh' finished in 1m33.161866461s 2022/05/04 12:09:16 process.go:96: Saved XML output to /logs/artifacts/junit_runner.xml. 2022/05/04 12:09:16 main.go:331: Something went wrong: starting e2e cluster: error during ./hack/e2e-internal/e2e-up.sh: exit status 1 Traceback (most recent call last): File "/workspace/scenarios/kubernetes_e2e.py", line 723, in main(parse_args()) File "/workspace/scenarios/kubernetes_e2e.py", line 569, in main mode.start(runner_args) File "/workspace/scenarios/kubernetes_e2e.py", line 228, in start check_env(env, self.command, *args) File "/workspace/scenarios/kubernetes_e2e.py", line 111, in check_env subprocess.check_call(cmd, env=env) File "/usr/lib/python2.7/subprocess.py", line 190, in check_call raise CalledProcessError(retcode, cmd) subprocess.CalledProcessError: Command '('kubetest', '--dump=/logs/artifacts', '--gcp-service-account=/etc/service-account/service-account.json', '--up', '--down', '--test', '--provider=gce', '--cluster=gce-scale-cluster', '--gcp-network=gce-scale-cluster', '--extract=ci/latest-fast', '--extract-ci-bucket=k8s-release-dev', '--gcp-master-image=gci', '--gcp-node-image=gci', '--gcp-node-size=e2-small', '--gcp-nodes=5000', '--gcp-project-type=scalability-scale-project', '--gcp-ssh-proxy-instance-name=gce-scale-cluster-master', '--gcp-zone=us-east1-b', '--ginkgo-parallel=40', '--test_args=--ginkgo.skip=\\[Driver:.gcepd\\]|\\[Serial\\]|\\[Disruptive\\]|\\[Flaky\\]|\\[Feature:.+\\]|\\[DisabledForLargeClusters\\] --minStartupPods=8 --node-schedulable-timeout=90m', '--timeout=240m', '--logexporter-gcs-path=gs://k8s-infra-scalability-tests-logs/ci-kubernetes-e2e-gce-scale-correctness/1521822220547002368')' returned non-zero exit status 1 + EXIT_VALUE=1 + set +o xtrace