/* Copyright 2021 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // Package utils contains Kubeadm utility types. package utils import ( "github.com/blang/semver/v4" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" "sigs.k8s.io/controller-runtime/pkg/conversion" "sigs.k8s.io/controller-runtime/pkg/scheme" bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" "sigs.k8s.io/cluster-api/bootstrap/kubeadm/types/upstreamv1beta2" "sigs.k8s.io/cluster-api/bootstrap/kubeadm/types/upstreamv1beta3" "sigs.k8s.io/cluster-api/bootstrap/kubeadm/types/upstreamv1beta4" "sigs.k8s.io/cluster-api/util/version" ) var ( v1beta2KubeadmVersion = semver.MustParse("1.15.0") v1beta3KubeadmVersion = semver.MustParse("1.22.0") v1beta4KubeadmVersion = semver.MustParse("1.31.0") clusterConfigurationVersionTypeMap = map[schema.GroupVersion]conversion.Convertible{ upstreamv1beta4.GroupVersion: &upstreamv1beta4.ClusterConfiguration{}, upstreamv1beta3.GroupVersion: &upstreamv1beta3.ClusterConfiguration{}, upstreamv1beta2.GroupVersion: &upstreamv1beta2.ClusterConfiguration{}, } clusterStatusVersionTypeMap = map[schema.GroupVersion]conversion.Convertible{ // ClusterStatus has been removed in v1beta3, so we don't need an entry for v1beta3 & v1beta4 upstreamv1beta2.GroupVersion: &upstreamv1beta2.ClusterStatus{}, } initConfigurationVersionTypeMap = map[schema.GroupVersion]conversion.Convertible{ upstreamv1beta4.GroupVersion: &upstreamv1beta4.InitConfiguration{}, upstreamv1beta3.GroupVersion: &upstreamv1beta3.InitConfiguration{}, upstreamv1beta2.GroupVersion: &upstreamv1beta2.InitConfiguration{}, } joinConfigurationVersionTypeMap = map[schema.GroupVersion]conversion.Convertible{ upstreamv1beta4.GroupVersion: &upstreamv1beta4.JoinConfiguration{}, upstreamv1beta3.GroupVersion: &upstreamv1beta3.JoinConfiguration{}, upstreamv1beta2.GroupVersion: &upstreamv1beta2.JoinConfiguration{}, } ) // ConvertibleFromClusterConfiguration defines capabilities of a type that during conversions gets values from ClusterConfiguration. // NOTE: this interface is specifically designed to handle fields migrated from ClusterConfiguration to Init and JoinConfiguration in the kubeadm v1beta4 API version. type ConvertibleFromClusterConfiguration interface { ConvertFromClusterConfiguration(clusterConfiguration *bootstrapv1.ClusterConfiguration) error } // ConvertibleToClusterConfiguration defines capabilities of a type that during conversions sets values to ClusterConfiguration. // NOTE: this interface is specifically designed to handle fields migrated from ClusterConfiguration to Init and JoinConfiguration in the kubeadm v1beta4 API version. type ConvertibleToClusterConfiguration interface { ConvertToClusterConfiguration(clusterConfiguration *bootstrapv1.ClusterConfiguration) error } // KubeVersionToKubeadmAPIGroupVersion maps a Kubernetes version to the correct Kubeadm API Group supported. func KubeVersionToKubeadmAPIGroupVersion(v semver.Version) (schema.GroupVersion, error) { switch { case version.Compare(v, v1beta2KubeadmVersion, version.WithoutPreReleases()) < 0: return schema.GroupVersion{}, errors.New("the bootstrap provider for kubeadm doesn't support Kubernetes version lower than v1.15.0") case version.Compare(v, v1beta3KubeadmVersion, version.WithoutPreReleases()) < 0: // NOTE: All the Kubernetes version >= v1.15 and < v1.22 should use the kubeadm API version v1beta2 return upstreamv1beta2.GroupVersion, nil case version.Compare(v, v1beta4KubeadmVersion, version.WithoutPreReleases()) < 0: // NOTE: All the Kubernetes version >= v1.22 and < v1.31 should use the kubeadm API version v1beta3 return upstreamv1beta3.GroupVersion, nil default: // NOTE: All the Kubernetes version greater or equal to v1.31 (not yet released at the time of writing this code) // are assumed to use the kubeadm API version v1beta4 or to work with Cluster API using it. // This should be reconsidered whenever kubeadm introduces a new API version. return upstreamv1beta4.GroupVersion, nil } } // MarshalClusterConfigurationForVersion converts a Cluster API ClusterConfiguration type to the kubeadm API type // for the given Kubernetes Version. // NOTE: This assumes Kubernetes Version equals to kubeadm version. func MarshalClusterConfigurationForVersion(clusterConfiguration *bootstrapv1.ClusterConfiguration, version semver.Version) (string, error) { return marshalForVersion(nil, clusterConfiguration, version, clusterConfigurationVersionTypeMap) } // MarshalClusterStatusForVersion converts a Cluster API ClusterStatus type to the kubeadm API type // for the given Kubernetes Version. // NOTE: This assumes Kubernetes Version equals to kubeadm version. func MarshalClusterStatusForVersion(clusterStatus *bootstrapv1.ClusterStatus, version semver.Version) (string, error) { return marshalForVersion(nil, clusterStatus, version, clusterStatusVersionTypeMap) } // MarshalInitConfigurationForVersion converts a Cluster API InitConfiguration type to the kubeadm API type // for the given Kubernetes Version. // NOTE: This assumes Kubernetes Version equals to kubeadm version. func MarshalInitConfigurationForVersion(clusterConfiguration *bootstrapv1.ClusterConfiguration, initConfiguration *bootstrapv1.InitConfiguration, version semver.Version) (string, error) { return marshalForVersion(clusterConfiguration, initConfiguration, version, initConfigurationVersionTypeMap) } // MarshalJoinConfigurationForVersion converts a Cluster API JoinConfiguration type to the kubeadm API type // for the given Kubernetes Version. // NOTE: This assumes Kubernetes Version equals to kubeadm version. func MarshalJoinConfigurationForVersion(clusterConfiguration *bootstrapv1.ClusterConfiguration, joinConfiguration *bootstrapv1.JoinConfiguration, version semver.Version) (string, error) { return marshalForVersion(clusterConfiguration, joinConfiguration, version, joinConfigurationVersionTypeMap) } func marshalForVersion(clusterConfiguration *bootstrapv1.ClusterConfiguration, obj conversion.Hub, version semver.Version, kubeadmObjVersionTypeMap map[schema.GroupVersion]conversion.Convertible) (string, error) { kubeadmAPIGroupVersion, err := KubeVersionToKubeadmAPIGroupVersion(version) if err != nil { return "", err } targetKubeadmObj, ok := kubeadmObjVersionTypeMap[kubeadmAPIGroupVersion] if !ok { return "", errors.Errorf("missing KubeadmAPI type mapping for version %s", kubeadmAPIGroupVersion) } targetKubeadmObj = targetKubeadmObj.DeepCopyObject().(conversion.Convertible) if err := targetKubeadmObj.ConvertFrom(obj); err != nil { return "", errors.Wrapf(err, "failed to convert to KubeadmAPI type for version %s", kubeadmAPIGroupVersion) } if convertibleFromClusterConfigurationObj, ok := targetKubeadmObj.(ConvertibleFromClusterConfiguration); ok { if err := convertibleFromClusterConfigurationObj.ConvertFromClusterConfiguration(clusterConfiguration); err != nil { return "", errors.Wrapf(err, "failed to convert from ClusterConfiguration to KubeadmAPI type for version %s", kubeadmAPIGroupVersion) } } codecs, err := getCodecsFor(kubeadmAPIGroupVersion, targetKubeadmObj) if err != nil { return "", err } yaml, err := toYaml(targetKubeadmObj, kubeadmAPIGroupVersion, codecs) if err != nil { return "", errors.Wrapf(err, "failed to generate yaml for the Kubeadm API for version %s", kubeadmAPIGroupVersion) } return string(yaml), nil } func getCodecsFor(gv schema.GroupVersion, obj runtime.Object) (serializer.CodecFactory, error) { sb := &scheme.Builder{GroupVersion: gv} sb.Register(obj) kubeadmScheme, err := sb.Build() if err != nil { return serializer.CodecFactory{}, errors.Wrapf(err, "failed to build scheme for kubeadm types conversions") } return serializer.NewCodecFactory(kubeadmScheme), nil } func toYaml(obj runtime.Object, gv runtime.GroupVersioner, codecs serializer.CodecFactory) ([]byte, error) { info, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), runtime.ContentTypeYAML) if !ok { return []byte{}, errors.Errorf("unsupported media type %q", runtime.ContentTypeYAML) } encoder := codecs.EncoderForVersion(info.Serializer, gv) return runtime.Encode(encoder, obj) } // UnmarshalClusterConfiguration tries to translate a Kubeadm API yaml back to the Cluster API ClusterConfiguration type. // NOTE: The yaml could be any of the known formats for the kubeadm ClusterConfiguration type. func UnmarshalClusterConfiguration(yaml string) (*bootstrapv1.ClusterConfiguration, error) { obj := &bootstrapv1.ClusterConfiguration{} if err := unmarshalFromVersions(yaml, clusterConfigurationVersionTypeMap, nil, obj); err != nil { return nil, err } return obj, nil } // UnmarshalClusterStatus tries to translate a Kubeadm API yaml back to the Cluster API ClusterStatus type. // NOTE: The yaml could be any of the known formats for the kubeadm ClusterStatus type. func UnmarshalClusterStatus(yaml string) (*bootstrapv1.ClusterStatus, error) { obj := &bootstrapv1.ClusterStatus{} if err := unmarshalFromVersions(yaml, clusterStatusVersionTypeMap, nil, obj); err != nil { return nil, err } return obj, nil } // UnmarshalInitConfiguration tries to translate a Kubeadm API yaml back to the InitConfiguration type. // NOTE: The yaml could be any of the known formats for the kubeadm InitConfiguration type. func UnmarshalInitConfiguration(yaml string, clusterConfiguration *bootstrapv1.ClusterConfiguration) (*bootstrapv1.InitConfiguration, error) { obj := &bootstrapv1.InitConfiguration{} if err := unmarshalFromVersions(yaml, initConfigurationVersionTypeMap, clusterConfiguration, obj); err != nil { return nil, err } return obj, nil } // UnmarshalJoinConfiguration tries to translate a Kubeadm API yaml back to the JoinConfiguration type. // NOTE: The yaml could be any of the known formats for the kubeadm JoinConfiguration type. func UnmarshalJoinConfiguration(yaml string, clusterConfiguration *bootstrapv1.ClusterConfiguration) (*bootstrapv1.JoinConfiguration, error) { obj := &bootstrapv1.JoinConfiguration{} if err := unmarshalFromVersions(yaml, joinConfigurationVersionTypeMap, clusterConfiguration, obj); err != nil { return nil, err } return obj, nil } func unmarshalFromVersions(yaml string, kubeadmAPIVersions map[schema.GroupVersion]conversion.Convertible, clusterConfiguration *bootstrapv1.ClusterConfiguration, capiObj conversion.Hub) error { // For each know kubeadm API version for gv, obj := range kubeadmAPIVersions { // Tries conversion from yaml to the corresponding kubeadmObj kubeadmObj := obj.DeepCopyObject() gvk := kubeadmObj.GetObjectKind().GroupVersionKind() codecs, err := getCodecsFor(gv, kubeadmObj) if err != nil { return errors.Wrapf(err, "failed to build scheme for kubeadm types conversions") } _, _, err = codecs.UniversalDeserializer().Decode([]byte(yaml), &gvk, kubeadmObj) if err == nil { if convertibleToClusterConfigurationObj, ok := kubeadmObj.(ConvertibleToClusterConfiguration); ok { if err := convertibleToClusterConfigurationObj.ConvertToClusterConfiguration(clusterConfiguration); err != nil { return errors.Wrapf(err, "failed to convert to ClusterConfiguration from KubeadmAPI type for version %s", gvk) } } // If conversion worked, then converts the kubeadmObj (spoke) back to the Cluster API ClusterConfiguration type (hub). if err := kubeadmObj.(conversion.Convertible).ConvertTo(capiObj); err != nil { return errors.Wrapf(err, "failed to convert kubeadm types to Cluster API types") } return nil } } return errors.New("unknown kubeadm types") }
/* Copyright 2021 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package webhooks import ( "context" "fmt" "net" "strconv" "strings" "time" "github.com/blang/semver/v4" "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" kerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/controllers/external" expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" "sigs.k8s.io/cluster-api/feature" "sigs.k8s.io/cluster-api/internal/contract" "sigs.k8s.io/cluster-api/internal/topology/check" "sigs.k8s.io/cluster-api/internal/topology/variables" "sigs.k8s.io/cluster-api/util/conditions" "sigs.k8s.io/cluster-api/util/version" ) // SetupWebhookWithManager sets up Cluster webhooks. func (webhook *Cluster) SetupWebhookWithManager(mgr ctrl.Manager) error { if webhook.decoder == nil { webhook.decoder = admission.NewDecoder(mgr.GetScheme()) } return ctrl.NewWebhookManagedBy(mgr). For(&clusterv1.Cluster{}). WithDefaulter(webhook). WithValidator(webhook). Complete() } // +kubebuilder:webhook:verbs=create;update;delete,path=/validate-cluster-x-k8s-io-v1beta1-cluster,mutating=false,failurePolicy=fail,matchPolicy=Equivalent,groups=cluster.x-k8s.io,resources=clusters,versions=v1beta1,name=validation.cluster.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 // +kubebuilder:webhook:verbs=create;update,path=/mutate-cluster-x-k8s-io-v1beta1-cluster,mutating=true,failurePolicy=fail,matchPolicy=Equivalent,groups=cluster.x-k8s.io,resources=clusters,versions=v1beta1,name=default.cluster.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 // ClusterCacheReader is a scoped-down interface from ClusterCacheTracker that only allows to get a reader client. type ClusterCacheReader interface { GetReader(ctx context.Context, cluster client.ObjectKey) (client.Reader, error) } // Cluster implements a validating and defaulting webhook for Cluster. type Cluster struct { Client client.Reader ClusterCacheReader ClusterCacheReader decoder admission.Decoder } var _ webhook.CustomDefaulter = &Cluster{} var _ webhook.CustomValidator = &Cluster{} var errClusterClassNotReconciled = errors.New("ClusterClass is not successfully reconciled") // Default satisfies the defaulting webhook interface. func (webhook *Cluster) Default(ctx context.Context, obj runtime.Object) error { // We gather all defaulting errors and return them together. var allErrs field.ErrorList cluster, ok := obj.(*clusterv1.Cluster) if !ok { return apierrors.NewBadRequest(fmt.Sprintf("expected a Cluster but got a %T", obj)) } if cluster.Spec.InfrastructureRef != nil && cluster.Spec.InfrastructureRef.Namespace == "" { cluster.Spec.InfrastructureRef.Namespace = cluster.Namespace } if cluster.Spec.ControlPlaneRef != nil && cluster.Spec.ControlPlaneRef.Namespace == "" { cluster.Spec.ControlPlaneRef.Namespace = cluster.Namespace } // Additional defaulting if the Cluster uses a managed topology. if cluster.Spec.Topology != nil { // Tolerate version strings without a "v" prefix: prepend it if it's not there. if !strings.HasPrefix(cluster.Spec.Topology.Version, "v") { cluster.Spec.Topology.Version = "v" + cluster.Spec.Topology.Version } if cluster.GetClassKey().Name == "" { allErrs = append( allErrs, field.Required( field.NewPath("spec", "topology", "class"), "class cannot be empty", ), ) return apierrors.NewInvalid(clusterv1.GroupVersion.WithKind("Cluster").GroupKind(), cluster.Name, allErrs) } if cluster.Spec.Topology.ControlPlane.MachineHealthCheck != nil && cluster.Spec.Topology.ControlPlane.MachineHealthCheck.MachineHealthCheckClass.RemediationTemplate != nil && cluster.Spec.Topology.ControlPlane.MachineHealthCheck.MachineHealthCheckClass.RemediationTemplate.Namespace == "" { cluster.Spec.Topology.ControlPlane.MachineHealthCheck.MachineHealthCheckClass.RemediationTemplate.Namespace = cluster.Namespace } if cluster.Spec.Topology.Workers != nil { for i := range cluster.Spec.Topology.Workers.MachineDeployments { md := cluster.Spec.Topology.Workers.MachineDeployments[i] if md.MachineHealthCheck != nil && md.MachineHealthCheck.MachineHealthCheckClass.RemediationTemplate != nil && md.MachineHealthCheck.MachineHealthCheckClass.RemediationTemplate.Namespace == "" { md.MachineHealthCheck.MachineHealthCheckClass.RemediationTemplate.Namespace = cluster.Namespace } } } clusterClass, err := webhook.pollClusterClassForCluster(ctx, cluster) if err != nil { // If the ClusterClass can't be found or is not up to date ignore the error. if apierrors.IsNotFound(err) || errors.Is(err, errClusterClassNotReconciled) { return nil } return apierrors.NewInternalError(errors.Wrapf(err, "Cluster %s can't be defaulted. ClusterClass %s can not be retrieved", cluster.Name, cluster.GetClassKey().Name)) } // Validate cluster class variables transitions that may be enforced by CEL validation rules on variables. // If no request found in context, then this has not come via a webhook request, so skip validation of old cluster. var oldCluster *clusterv1.Cluster req, err := admission.RequestFromContext(ctx) if err == nil && len(req.OldObject.Raw) > 0 { oldCluster = &clusterv1.Cluster{} if err := webhook.decoder.DecodeRaw(req.OldObject, oldCluster); err != nil { return apierrors.NewBadRequest(errors.Wrap(err, "failed to decode old cluster object").Error()) } } // Doing both defaulting and validating here prevents a race condition where the ClusterClass could be // different in the defaulting and validating webhook. allErrs = append(allErrs, DefaultAndValidateVariables(ctx, cluster, oldCluster, clusterClass)...) if len(allErrs) > 0 { return apierrors.NewInvalid(clusterv1.GroupVersion.WithKind("Cluster").GroupKind(), cluster.Name, allErrs) } } return nil } // ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type. func (webhook *Cluster) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { cluster, ok := obj.(*clusterv1.Cluster) if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a Cluster but got a %T", obj)) } return webhook.validate(ctx, nil, cluster) } // ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type. func (webhook *Cluster) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { newCluster, ok := newObj.(*clusterv1.Cluster) if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a Cluster but got a %T", newObj)) } oldCluster, ok := oldObj.(*clusterv1.Cluster) if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a Cluster but got a %T", oldObj)) } return webhook.validate(ctx, oldCluster, newCluster) } // ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type. func (webhook *Cluster) ValidateDelete(_ context.Context, _ runtime.Object) (admission.Warnings, error) { return nil, nil } func (webhook *Cluster) validate(ctx context.Context, oldCluster, newCluster *clusterv1.Cluster) (admission.Warnings, error) { var allErrs field.ErrorList var allWarnings admission.Warnings // The Cluster name is used as a label value. This check ensures that names which are not valid label values are rejected. if errs := validation.IsValidLabelValue(newCluster.Name); len(errs) != 0 { for _, err := range errs { allErrs = append( allErrs, field.Invalid( field.NewPath("metadata", "name"), newCluster.Name, fmt.Sprintf("must be a valid label value %s", err), ), ) } } specPath := field.NewPath("spec") if newCluster.Spec.InfrastructureRef != nil && newCluster.Spec.InfrastructureRef.Namespace != newCluster.Namespace { allErrs = append( allErrs, field.Invalid( specPath.Child("infrastructureRef", "namespace"), newCluster.Spec.InfrastructureRef.Namespace, "must match metadata.namespace", ), ) } if newCluster.Spec.ControlPlaneRef != nil && newCluster.Spec.ControlPlaneRef.Namespace != newCluster.Namespace { allErrs = append( allErrs, field.Invalid( specPath.Child("controlPlaneRef", "namespace"), newCluster.Spec.ControlPlaneRef.Namespace, "must match metadata.namespace", ), ) } if newCluster.Spec.ClusterNetwork != nil { // Ensure that the CIDR blocks defined under ClusterNetwork are valid. if newCluster.Spec.ClusterNetwork.Pods != nil { allErrs = append(allErrs, validateCIDRBlocks(specPath.Child("clusterNetwork", "pods", "cidrBlocks"), newCluster.Spec.ClusterNetwork.Pods.CIDRBlocks)...) } if newCluster.Spec.ClusterNetwork.Services != nil { allErrs = append(allErrs, validateCIDRBlocks(specPath.Child("clusterNetwork", "services", "cidrBlocks"), newCluster.Spec.ClusterNetwork.Services.CIDRBlocks)...) } } topologyPath := specPath.Child("topology") // Validate the managed topology, if defined. if newCluster.Spec.Topology != nil { topologyWarnings, topologyErrs := webhook.validateTopology(ctx, oldCluster, newCluster, topologyPath) allWarnings = append(allWarnings, topologyWarnings...) allErrs = append(allErrs, topologyErrs...) } // On update. if oldCluster != nil { // Error if the update moves the cluster from Managed to Unmanaged i.e. the managed topology is removed on update. if oldCluster.Spec.Topology != nil && newCluster.Spec.Topology == nil { allErrs = append(allErrs, field.Forbidden( topologyPath, "cannot be removed from an existing Cluster", )) } } if len(allErrs) > 0 { return allWarnings, apierrors.NewInvalid(clusterv1.GroupVersion.WithKind("Cluster").GroupKind(), newCluster.Name, allErrs) } return allWarnings, nil } func (webhook *Cluster) validateTopology(ctx context.Context, oldCluster, newCluster *clusterv1.Cluster, fldPath *field.Path) (admission.Warnings, field.ErrorList) { var allWarnings admission.Warnings // NOTE: ClusterClass and managed topologies are behind ClusterTopology feature gate flag; the web hook // must prevent the usage of Cluster.Topology in case the feature flag is disabled. if !feature.Gates.Enabled(feature.ClusterTopology) { return allWarnings, field.ErrorList{ field.Forbidden( fldPath, "can be set only if the ClusterTopology feature flag is enabled", ), } } var allErrs field.ErrorList // class should be defined. if newCluster.GetClassKey().Name == "" { allErrs = append( allErrs, field.Required( fldPath.Child("class"), "class cannot be empty", ), ) // Return early if there is no defined class to validate. return allWarnings, allErrs } // version should be valid. if !version.KubeSemver.MatchString(newCluster.Spec.Topology.Version) { allErrs = append( allErrs, field.Invalid( fldPath.Child("version"), newCluster.Spec.Topology.Version, "version must be a valid semantic version", ), ) } // metadata in topology should be valid allErrs = append(allErrs, validateTopologyMetadata(newCluster.Spec.Topology, fldPath)...) // ensure deprecationFrom is not set allErrs = append(allErrs, validateTopologyDefinitionFrom(newCluster.Spec.Topology, fldPath)...) // upgrade concurrency should be a numeric value. if concurrency, ok := newCluster.Annotations[clusterv1.ClusterTopologyUpgradeConcurrencyAnnotation]; ok { concurrencyAnnotationField := field.NewPath("metadata", "annotations", clusterv1.ClusterTopologyUpgradeConcurrencyAnnotation) concurrencyInt, err := strconv.Atoi(concurrency) if err != nil { allErrs = append(allErrs, field.Invalid( concurrencyAnnotationField, concurrency, errors.Wrap(err, "could not parse the value of the annotation").Error(), )) } else if concurrencyInt < 1 { allErrs = append(allErrs, field.Invalid( concurrencyAnnotationField, concurrency, "value cannot be less than 1", )) } } // Get the ClusterClass referenced in the Cluster. clusterClass, warnings, clusterClassPollErr := webhook.validateClusterClassExistsAndIsReconciled(ctx, newCluster) // If the error is anything other than "NotFound" or "NotReconciled" return all errors. if clusterClassPollErr != nil && !(apierrors.IsNotFound(clusterClassPollErr) || errors.Is(clusterClassPollErr, errClusterClassNotReconciled)) { allErrs = append( allErrs, field.InternalError( fldPath.Child("class"), clusterClassPollErr)) return allWarnings, allErrs } // Add the warnings if no error was returned. allWarnings = append(allWarnings, warnings...) // If there's no error validate the Cluster based on the ClusterClass. if clusterClassPollErr == nil { allErrs = append(allErrs, ValidateClusterForClusterClass(newCluster, clusterClass)...) } // Validate the Cluster and associated ClusterClass' autoscaler annotations. allErrs = append(allErrs, validateAutoscalerAnnotationsForCluster(newCluster, clusterClass)...) if oldCluster != nil { // On update // The ClusterClass must exist to proceed with update validation. Return an error if the ClusterClass was // not found. if apierrors.IsNotFound(clusterClassPollErr) { allErrs = append( allErrs, field.InternalError( fldPath.Child("class"), clusterClassPollErr)) return allWarnings, allErrs } // Topology or Class can not be added on update unless ClusterTopologyUnsafeUpdateClassNameAnnotation is set. if oldCluster.Spec.Topology == nil || oldCluster.GetClassKey().Name == "" { if _, ok := newCluster.Annotations[clusterv1.ClusterTopologyUnsafeUpdateClassNameAnnotation]; ok { return allWarnings, allErrs } allErrs = append( allErrs, field.Forbidden( fldPath.Child("class"), "class cannot be set on an existing Cluster", ), ) // return early here if there is no class to compare. return allWarnings, allErrs } inVersion, err := semver.ParseTolerant(newCluster.Spec.Topology.Version) if err != nil { allErrs = append( allErrs, field.Invalid( fldPath.Child("version"), newCluster.Spec.Topology.Version, "version must be a valid semantic version", ), ) } oldVersion, err := semver.ParseTolerant(oldCluster.Spec.Topology.Version) if err != nil { // NOTE: this should never happen. Nevertheless, handling this for extra caution. allErrs = append( allErrs, field.Invalid( fldPath.Child("version"), oldCluster.Spec.Topology.Version, "old version must be a valid semantic version", ), ) } if _, ok := newCluster.GetAnnotations()[clusterv1.ClusterTopologyUnsafeUpdateVersionAnnotation]; ok { log := ctrl.LoggerFrom(ctx) warningMsg := fmt.Sprintf("Skipping version validation for Cluster because annotation %q is set.", clusterv1.ClusterTopologyUnsafeUpdateVersionAnnotation) log.Info(warningMsg) allWarnings = append(allWarnings, warningMsg) } else { if err := webhook.validateTopologyVersion(ctx, fldPath.Child("version"), newCluster.Spec.Topology.Version, inVersion, oldVersion, oldCluster); err != nil { allErrs = append(allErrs, err) } } // If the ClusterClass referenced in the Topology has changed compatibility checks are needed. if oldCluster.GetClassKey() != newCluster.GetClassKey() { // Check to see if the ClusterClass referenced in the old version of the Cluster exists. oldClusterClass, err := webhook.pollClusterClassForCluster(ctx, oldCluster) if err != nil { allErrs = append( allErrs, field.Forbidden( fldPath.Child("class"), fmt.Sprintf("valid ClusterClass with name %q could not be retrieved, change from class %[1]q to class %q cannot be validated. Error: %s", oldCluster.GetClassKey(), newCluster.GetClassKey(), err.Error()))) // Return early with errors if the ClusterClass can't be retrieved. return allWarnings, allErrs } // Check if the new and old ClusterClasses are compatible with one another. allErrs = append(allErrs, check.ClusterClassesAreCompatible(oldClusterClass, clusterClass)...) } } return allWarnings, allErrs } func (webhook *Cluster) validateTopologyVersion(ctx context.Context, fldPath *field.Path, fldValue string, inVersion, oldVersion semver.Version, oldCluster *clusterv1.Cluster) *field.Error { // Version could only be increased. if inVersion.NE(semver.Version{}) && oldVersion.NE(semver.Version{}) && version.Compare(inVersion, oldVersion, version.WithBuildTags()) == -1 { return field.Invalid( fldPath, fldValue, fmt.Sprintf("version cannot be decreased from %q to %q", oldVersion, inVersion), ) } // A +2 minor version upgrade is not allowed. ceilVersion := semver.Version{ Major: oldVersion.Major, Minor: oldVersion.Minor + 2, Patch: 0, } if inVersion.GTE(ceilVersion) { return field.Invalid( fldPath, fldValue, fmt.Sprintf("version cannot be increased from %q to %q", oldVersion, inVersion), ) } // Only check the following cases if the minor version increases by 1 (we already return above for >= 2). ceilVersion = semver.Version{ Major: oldVersion.Major, Minor: oldVersion.Minor + 1, Patch: 0, } // Return early if its not a minor version upgrade. if !inVersion.GTE(ceilVersion) { return nil } allErrs := []error{} // minor version cannot be increased if control plane is upgrading or not yet on the current version if err := validateTopologyControlPlaneVersion(ctx, webhook.Client, oldCluster, oldVersion); err != nil { allErrs = append(allErrs, fmt.Errorf("blocking version update due to ControlPlane version check: %v", err)) } // minor version cannot be increased if MachineDeployments are upgrading or not yet on the current version if err := validateTopologyMachineDeploymentVersions(ctx, webhook.Client, oldCluster, oldVersion); err != nil { allErrs = append(allErrs, fmt.Errorf("blocking version update due to MachineDeployment version check: %v", err)) } // minor version cannot be increased if MachinePools are upgrading or not yet on the current version if err := validateTopologyMachinePoolVersions(ctx, webhook.Client, webhook.ClusterCacheReader, oldCluster, oldVersion); err != nil { allErrs = append(allErrs, fmt.Errorf("blocking version update due to MachinePool version check: %v", err)) } if len(allErrs) > 0 { return field.Invalid( fldPath, fldValue, fmt.Sprintf("minor version update cannot happen at this time: %v", kerrors.NewAggregate(allErrs)), ) } return nil } func validateTopologyControlPlaneVersion(ctx context.Context, ctrlClient client.Reader, oldCluster *clusterv1.Cluster, oldVersion semver.Version) error { cp, err := external.Get(ctx, ctrlClient, oldCluster.Spec.ControlPlaneRef, oldCluster.Namespace) if err != nil { return errors.Wrap(err, "failed to get ControlPlane object") } cpVersionString, err := contract.ControlPlane().Version().Get(cp) if err != nil { return errors.Wrap(err, "failed to get ControlPlane version") } cpVersion, err := semver.ParseTolerant(*cpVersionString) if err != nil { // NOTE: this should never happen. Nevertheless, handling this for extra caution. return errors.New("failed to parse version of ControlPlane") } if cpVersion.NE(oldVersion) { return fmt.Errorf("ControlPlane version %q does not match the current version %q", cpVersion, oldVersion) } provisioning, err := contract.ControlPlane().IsProvisioning(cp) if err != nil { return errors.Wrap(err, "failed to check if ControlPlane is provisioning") } if provisioning { return errors.New("ControlPlane is currently provisioning") } upgrading, err := contract.ControlPlane().IsUpgrading(cp) if err != nil { return errors.Wrap(err, "failed to check if ControlPlane is upgrading") } if upgrading { return errors.New("ControlPlane is still completing a previous upgrade") } return nil } func validateTopologyMachineDeploymentVersions(ctx context.Context, ctrlClient client.Reader, oldCluster *clusterv1.Cluster, oldVersion semver.Version) error { // List all the machine deployments in the current cluster and in a managed topology. // FROM: current_state.go getCurrentMachineDeploymentState mds := &clusterv1.MachineDeploymentList{} err := ctrlClient.List(ctx, mds, client.MatchingLabels{ clusterv1.ClusterNameLabel: oldCluster.Name, clusterv1.ClusterTopologyOwnedLabel: "", }, client.InNamespace(oldCluster.Namespace), ) if err != nil { return errors.Wrap(err, "failed to read MachineDeployments for managed topology") } if len(mds.Items) == 0 { return nil } mdUpgradingNames := []string{} for i := range mds.Items { md := &mds.Items[i] mdVersion, err := semver.ParseTolerant(*md.Spec.Template.Spec.Version) if err != nil { // NOTE: this should never happen. Nevertheless, handling this for extra caution. return errors.Wrapf(err, "failed to parse MachineDeployment's %q version %q", klog.KObj(md), *md.Spec.Template.Spec.Version) } if mdVersion.NE(oldVersion) { mdUpgradingNames = append(mdUpgradingNames, md.Name) continue } upgrading, err := check.IsMachineDeploymentUpgrading(ctx, ctrlClient, md) if err != nil { return errors.Wrap(err, "failed to check if MachineDeployment is upgrading") } if upgrading { mdUpgradingNames = append(mdUpgradingNames, md.Name) } } if len(mdUpgradingNames) > 0 { return fmt.Errorf("there are MachineDeployments still completing a previous upgrade: [%s]", strings.Join(mdUpgradingNames, ", ")) } return nil } func validateTopologyMachinePoolVersions(ctx context.Context, ctrlClient client.Reader, clusterCacheReader ClusterCacheReader, oldCluster *clusterv1.Cluster, oldVersion semver.Version) error { // List all the machine pools in the current cluster and in a managed topology. // FROM: current_state.go getCurrentMachinePoolState mps := &expv1.MachinePoolList{} err := ctrlClient.List(ctx, mps, client.MatchingLabels{ clusterv1.ClusterNameLabel: oldCluster.Name, clusterv1.ClusterTopologyOwnedLabel: "", }, client.InNamespace(oldCluster.Namespace), ) if err != nil { return errors.Wrap(err, "failed to read MachinePools for managed topology") } // Return early if len(mps.Items) == 0 { return nil } wlClient, err := clusterCacheReader.GetReader(ctx, client.ObjectKeyFromObject(oldCluster)) if err != nil { return errors.Wrap(err, "unable to get client for workload cluster") } mpUpgradingNames := []string{} for i := range mps.Items { mp := &mps.Items[i] mpVersion, err := semver.ParseTolerant(*mp.Spec.Template.Spec.Version) if err != nil { // NOTE: this should never happen. Nevertheless, handling this for extra caution. return errors.Wrapf(err, "failed to parse MachinePool's %q version %q", klog.KObj(mp), *mp.Spec.Template.Spec.Version) } if mpVersion.NE(oldVersion) { mpUpgradingNames = append(mpUpgradingNames, mp.Name) continue } upgrading, err := check.IsMachinePoolUpgrading(ctx, wlClient, mp) if err != nil { return errors.Wrap(err, "failed to check if MachinePool is upgrading") } if upgrading { mpUpgradingNames = append(mpUpgradingNames, mp.Name) } } if len(mpUpgradingNames) > 0 { return fmt.Errorf("there are MachinePools still completing a previous upgrade: [%s]", strings.Join(mpUpgradingNames, ", ")) } return nil } func validateMachineHealthChecks(cluster *clusterv1.Cluster, clusterClass *clusterv1.ClusterClass) field.ErrorList { var allErrs field.ErrorList if cluster.Spec.Topology.ControlPlane.MachineHealthCheck != nil { fldPath := field.NewPath("spec", "topology", "controlPlane", "machineHealthCheck") // Validate ControlPlane MachineHealthCheck if defined. if !cluster.Spec.Topology.ControlPlane.MachineHealthCheck.MachineHealthCheckClass.IsZero() { // Ensure ControlPlane does not define a MachineHealthCheck if the ClusterClass does not define MachineInfrastructure. if clusterClass.Spec.ControlPlane.MachineInfrastructure == nil { allErrs = append(allErrs, field.Forbidden( fldPath, "can be set only if spec.controlPlane.machineInfrastructure is set in ClusterClass", )) } allErrs = append(allErrs, validateMachineHealthCheckClass(fldPath, cluster.Namespace, &cluster.Spec.Topology.ControlPlane.MachineHealthCheck.MachineHealthCheckClass)...) } // If MachineHealthCheck is explicitly enabled then make sure that a MachineHealthCheck definition is // available either in the Cluster topology or in the ClusterClass. // (One of these definitions will be used in the controller to create the MachineHealthCheck) // Check if the machineHealthCheck is explicitly enabled in the ControlPlaneTopology. if cluster.Spec.Topology.ControlPlane.MachineHealthCheck.Enable != nil && *cluster.Spec.Topology.ControlPlane.MachineHealthCheck.Enable { // Ensure the MHC is defined in at least one of the ControlPlaneTopology of the Cluster or the ControlPlaneClass of the ClusterClass. if cluster.Spec.Topology.ControlPlane.MachineHealthCheck.MachineHealthCheckClass.IsZero() && clusterClass.Spec.ControlPlane.MachineHealthCheck == nil { allErrs = append(allErrs, field.Forbidden( fldPath.Child("enable"), fmt.Sprintf("cannot be set to %t as MachineHealthCheck definition is not available in the Cluster topology or the ClusterClass", *cluster.Spec.Topology.ControlPlane.MachineHealthCheck.Enable), )) } } } if cluster.Spec.Topology.Workers != nil { for i := range cluster.Spec.Topology.Workers.MachineDeployments { md := cluster.Spec.Topology.Workers.MachineDeployments[i] if md.MachineHealthCheck != nil { fldPath := field.NewPath("spec", "topology", "workers", "machineDeployments").Key(md.Name).Child("machineHealthCheck") // Validate the MachineDeployment MachineHealthCheck if defined. if !md.MachineHealthCheck.MachineHealthCheckClass.IsZero() { allErrs = append(allErrs, validateMachineHealthCheckClass(fldPath, cluster.Namespace, &md.MachineHealthCheck.MachineHealthCheckClass)...) } // If MachineHealthCheck is explicitly enabled then make sure that a MachineHealthCheck definition is // available either in the Cluster topology or in the ClusterClass. // (One of these definitions will be used in the controller to create the MachineHealthCheck) mdClass := machineDeploymentClassOfName(clusterClass, md.Class) if mdClass != nil { // Note: we skip handling the nil case here as it is already handled in previous validations. // Check if the machineHealthCheck is explicitly enabled in the machineDeploymentTopology. if md.MachineHealthCheck.Enable != nil && *md.MachineHealthCheck.Enable { // Ensure the MHC is defined in at least one of the MachineDeploymentTopology of the Cluster or the MachineDeploymentClass of the ClusterClass. if md.MachineHealthCheck.MachineHealthCheckClass.IsZero() && mdClass.MachineHealthCheck == nil { allErrs = append(allErrs, field.Forbidden( fldPath.Child("enable"), fmt.Sprintf("cannot be set to %t as MachineHealthCheck definition is not available in the Cluster topology or the ClusterClass", *md.MachineHealthCheck.Enable), )) } } } } } } return allErrs } // machineDeploymentClassOfName find a MachineDeploymentClass of the given name in the provided ClusterClass. // Returns nil if it can not find one. // TODO: Check if there is already a helper function that can do this. func machineDeploymentClassOfName(clusterClass *clusterv1.ClusterClass, name string) *clusterv1.MachineDeploymentClass { for _, mdClass := range clusterClass.Spec.Workers.MachineDeployments { if mdClass.Class == name { return &mdClass } } return nil } // validateCIDRBlocks ensures the passed CIDR is valid. func validateCIDRBlocks(fldPath *field.Path, cidrs []string) field.ErrorList { var allErrs field.ErrorList for i, cidr := range cidrs { if _, _, err := net.ParseCIDR(cidr); err != nil { allErrs = append(allErrs, field.Invalid( fldPath.Index(i), cidr, err.Error())) } } return allErrs } // DefaultAndValidateVariables defaults and validates variables in the Cluster and MachineDeployment/MachinePool topologies based // on the definitions in the ClusterClass. func DefaultAndValidateVariables(ctx context.Context, cluster, oldCluster *clusterv1.Cluster, clusterClass *clusterv1.ClusterClass) field.ErrorList { var allErrs field.ErrorList allErrs = append(allErrs, DefaultVariables(cluster, clusterClass)...) // Capture variables from old cluster if it is present to be used in validation for transitions that may be specified // via CEL validation rules. var ( oldClusterVariables, oldCPOverrides []clusterv1.ClusterVariable oldMDVariables map[string][]clusterv1.ClusterVariable oldMPVariables map[string][]clusterv1.ClusterVariable ) if oldCluster != nil { oldClusterVariables = oldCluster.Spec.Topology.Variables if oldCluster.Spec.Topology.ControlPlane.Variables != nil { oldCPOverrides = oldCluster.Spec.Topology.ControlPlane.Variables.Overrides } if oldCluster.Spec.Topology.Workers != nil { oldMDVariables = make(map[string][]clusterv1.ClusterVariable, len(oldCluster.Spec.Topology.Workers.MachineDeployments)) for _, md := range oldCluster.Spec.Topology.Workers.MachineDeployments { if md.Variables != nil { oldMDVariables[md.Name] = md.Variables.Overrides } } oldMPVariables = make(map[string][]clusterv1.ClusterVariable, len(oldCluster.Spec.Topology.Workers.MachinePools)) for _, mp := range oldCluster.Spec.Topology.Workers.MachinePools { if mp.Variables != nil { oldMPVariables[mp.Name] = mp.Variables.Overrides } } } } // Variables must be validated in the defaulting webhook. Variable definitions are stored in the ClusterClass status // and are patched in the ClusterClass reconcile. // Validate cluster-wide variables. allErrs = append(allErrs, variables.ValidateClusterVariables( ctx, cluster.Spec.Topology.Variables, oldClusterVariables, clusterClass.Status.Variables, field.NewPath("spec", "topology", "variables"))...) // Validate ControlPlane variable overrides. if cluster.Spec.Topology.ControlPlane.Variables != nil && len(cluster.Spec.Topology.ControlPlane.Variables.Overrides) > 0 { allErrs = append(allErrs, variables.ValidateControlPlaneVariables( ctx, cluster.Spec.Topology.ControlPlane.Variables.Overrides, oldCPOverrides, clusterClass.Status.Variables, field.NewPath("spec", "topology", "controlPlane", "variables", "overrides"))..., ) } if cluster.Spec.Topology.Workers != nil { // Validate MachineDeployment variable overrides. for _, md := range cluster.Spec.Topology.Workers.MachineDeployments { // Continue if there are no variable overrides. if md.Variables == nil || len(md.Variables.Overrides) == 0 { continue } allErrs = append(allErrs, variables.ValidateMachineVariables( ctx, md.Variables.Overrides, oldMDVariables[md.Name], clusterClass.Status.Variables, field.NewPath("spec", "topology", "workers", "machineDeployments").Key(md.Name).Child("variables", "overrides"))..., ) } // Validate MachinePool variable overrides. for _, mp := range cluster.Spec.Topology.Workers.MachinePools { // Continue if there are no variable overrides. if mp.Variables == nil || len(mp.Variables.Overrides) == 0 { continue } allErrs = append(allErrs, variables.ValidateMachineVariables( ctx, mp.Variables.Overrides, oldMPVariables[mp.Name], clusterClass.Status.Variables, field.NewPath("spec", "topology", "workers", "machinePools").Key(mp.Name).Child("variables", "overrides"))..., ) } } return allErrs } // DefaultVariables defaults variables in the Cluster based on information in the ClusterClass. func DefaultVariables(cluster *clusterv1.Cluster, clusterClass *clusterv1.ClusterClass) field.ErrorList { var allErrs field.ErrorList if cluster == nil { return field.ErrorList{field.InternalError(field.NewPath(""), errors.New("Cluster can not be nil"))} } if clusterClass == nil { return field.ErrorList{field.InternalError(field.NewPath(""), errors.New("ClusterClass can not be nil"))} } // Default cluster-wide variables. defaultedVariables, errs := variables.DefaultClusterVariables(cluster.Spec.Topology.Variables, clusterClass.Status.Variables, field.NewPath("spec", "topology", "variables")) if len(errs) > 0 { allErrs = append(allErrs, errs...) } else { cluster.Spec.Topology.Variables = defaultedVariables } // Default ControlPlane variable overrides. if cluster.Spec.Topology.ControlPlane.Variables != nil && len(cluster.Spec.Topology.ControlPlane.Variables.Overrides) > 0 { defaultedVariables, errs := variables.DefaultMachineVariables(cluster.Spec.Topology.ControlPlane.Variables.Overrides, clusterClass.Status.Variables, field.NewPath("spec", "topology", "controlPlane", "variables", "overrides")) if len(errs) > 0 { allErrs = append(allErrs, errs...) } else { cluster.Spec.Topology.ControlPlane.Variables.Overrides = defaultedVariables } } if cluster.Spec.Topology.Workers != nil { // Default MachineDeployment variable overrides. for _, md := range cluster.Spec.Topology.Workers.MachineDeployments { // Continue if there are no variable overrides. if md.Variables == nil || len(md.Variables.Overrides) == 0 { continue } defaultedVariables, errs := variables.DefaultMachineVariables(md.Variables.Overrides, clusterClass.Status.Variables, field.NewPath("spec", "topology", "workers", "machineDeployments").Key(md.Name).Child("variables", "overrides")) if len(errs) > 0 { allErrs = append(allErrs, errs...) } else { md.Variables.Overrides = defaultedVariables } } // Default MachinePool variable overrides. for _, mp := range cluster.Spec.Topology.Workers.MachinePools { // Continue if there are no variable overrides. if mp.Variables == nil || len(mp.Variables.Overrides) == 0 { continue } defaultedVariables, errs := variables.DefaultMachineVariables(mp.Variables.Overrides, clusterClass.Status.Variables, field.NewPath("spec", "topology", "workers", "machinePools").Key(mp.Name).Child("variables", "overrides")) if len(errs) > 0 { allErrs = append(allErrs, errs...) } else { mp.Variables.Overrides = defaultedVariables } } } return allErrs } // ValidateClusterForClusterClass uses information in the ClusterClass to validate the Cluster. func ValidateClusterForClusterClass(cluster *clusterv1.Cluster, clusterClass *clusterv1.ClusterClass) field.ErrorList { var allErrs field.ErrorList if cluster == nil { return field.ErrorList{field.InternalError(field.NewPath(""), errors.New("Cluster can not be nil"))} } if clusterClass == nil { return field.ErrorList{field.InternalError(field.NewPath(""), errors.New("ClusterClass can not be nil"))} } allErrs = append(allErrs, check.MachineDeploymentTopologiesAreValidAndDefinedInClusterClass(cluster, clusterClass)...) allErrs = append(allErrs, check.MachinePoolTopologiesAreValidAndDefinedInClusterClass(cluster, clusterClass)...) // Validate the MachineHealthChecks defined in the cluster topology. allErrs = append(allErrs, validateMachineHealthChecks(cluster, clusterClass)...) return allErrs } // validateClusterClassExistsAndIsReconciled will try to get the ClusterClass referenced in the Cluster. If it does not exist or is not reconciled it will add a warning. // In any other case it will return an error. func (webhook *Cluster) validateClusterClassExistsAndIsReconciled(ctx context.Context, newCluster *clusterv1.Cluster) (*clusterv1.ClusterClass, admission.Warnings, error) { var allWarnings admission.Warnings clusterClass, clusterClassPollErr := webhook.pollClusterClassForCluster(ctx, newCluster) if clusterClassPollErr != nil { // Add a warning if the Class does not exist or if it has not been successfully reconciled. switch { case apierrors.IsNotFound(clusterClassPollErr): allWarnings = append(allWarnings, fmt.Sprintf( "Cluster refers to ClusterClass %s, but this ClusterClass does not exist. "+ "Cluster topology has not been fully validated. "+ "The ClusterClass must be created to reconcile the Cluster", newCluster.GetClassKey()), ) case errors.Is(clusterClassPollErr, errClusterClassNotReconciled): allWarnings = append(allWarnings, fmt.Sprintf( "Cluster refers to ClusterClass %s, but this ClusterClass hasn't been successfully reconciled. "+ "Cluster topology has not been fully validated. "+ "Please take a look at the ClusterClass status", newCluster.GetClassKey()), ) // If there's any other error return a generic warning with the error message. default: allWarnings = append(allWarnings, fmt.Sprintf( "Cluster refers to ClusterClass %s, but this ClusterClass could not be retrieved. "+ "Cluster topology has not been fully validated: %s", newCluster.GetClassKey(), clusterClassPollErr.Error()), ) } } return clusterClass, allWarnings, clusterClassPollErr } // pollClusterClassForCluster will retry getting the ClusterClass referenced in the Cluster for two seconds. func (webhook *Cluster) pollClusterClassForCluster(ctx context.Context, cluster *clusterv1.Cluster) (*clusterv1.ClusterClass, error) { clusterClass := &clusterv1.ClusterClass{} var clusterClassPollErr error _ = wait.PollUntilContextTimeout(ctx, 200*time.Millisecond, 2*time.Second, true, func(ctx context.Context) (bool, error) { if clusterClassPollErr = webhook.Client.Get(ctx, cluster.GetClassKey(), clusterClass); clusterClassPollErr != nil { return false, nil //nolint:nilerr } if clusterClassPollErr = clusterClassIsReconciled(clusterClass); clusterClassPollErr != nil { return false, nil //nolint:nilerr } clusterClassPollErr = nil return true, nil }) if clusterClassPollErr != nil { return nil, clusterClassPollErr } return clusterClass, nil } // clusterClassIsReconciled returns errClusterClassNotReconciled if the ClusterClass has not successfully reconciled or if the // ClusterClass variables have not been successfully reconciled. func clusterClassIsReconciled(clusterClass *clusterv1.ClusterClass) error { // If the clusterClass metadata generation does not match the status observed generation, the ClusterClass has not been successfully reconciled. if clusterClass.Generation != clusterClass.Status.ObservedGeneration { return errClusterClassNotReconciled } // If the clusterClass does not have ClusterClassVariablesReconciled==True, the ClusterClass has not been successfully reconciled. if !conditions.Has(clusterClass, clusterv1.ClusterClassVariablesReconciledCondition) || conditions.IsFalse(clusterClass, clusterv1.ClusterClassVariablesReconciledCondition) { return errClusterClassNotReconciled } return nil } func validateTopologyMetadata(topology *clusterv1.Topology, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList allErrs = append(allErrs, topology.ControlPlane.Metadata.Validate(fldPath.Child("controlPlane", "metadata"))...) if topology.Workers != nil { for _, md := range topology.Workers.MachineDeployments { allErrs = append(allErrs, md.Metadata.Validate( fldPath.Child("workers", "machineDeployments").Key(md.Name).Child("metadata"), )...) } for _, mp := range topology.Workers.MachinePools { allErrs = append(allErrs, mp.Metadata.Validate( fldPath.Child("workers", "machinePools").Key(mp.Name).Child("metadata"), )...) } } return allErrs } func validateTopologyDefinitionFrom(topology *clusterv1.Topology, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList for _, variable := range topology.Variables { if variable.DefinitionFrom != "" { //nolint:staticcheck // Intentionally using the deprecated field here to check that it is not set. allErrs = append(allErrs, field.Invalid( fldPath.Child("variables").Key(variable.Name), string(variable.Value.Raw), fmt.Sprintf("variable %q has DefinitionFrom set", variable.Name)), ) } } if topology.ControlPlane.Variables != nil { for _, variable := range topology.ControlPlane.Variables.Overrides { if variable.DefinitionFrom != "" { //nolint:staticcheck // Intentionally using the deprecated field here to check that it is not set. allErrs = append(allErrs, field.Invalid( fldPath.Child("controlPlane", "variables", "overrides").Key(variable.Name), string(variable.Value.Raw), fmt.Sprintf("variable %q has DefinitionFrom set", variable.Name)), ) } } } if topology.Workers != nil { for _, md := range topology.Workers.MachineDeployments { if md.Variables != nil { for _, variable := range md.Variables.Overrides { if variable.DefinitionFrom != "" { //nolint:staticcheck // Intentionally using the deprecated field here to check that it is not set. allErrs = append(allErrs, field.Invalid( fldPath.Child("workers", "machineDeployments").Key(md.Name).Child("variables", "overrides").Key(variable.Name), string(variable.Value.Raw), fmt.Sprintf("variable %q has DefinitionFrom set", variable.Name)), ) } } } } for _, mp := range topology.Workers.MachinePools { if mp.Variables != nil { for _, variable := range mp.Variables.Overrides { if variable.DefinitionFrom != "" { //nolint:staticcheck // Intentionally using the deprecated field here to check that it is not set. allErrs = append(allErrs, field.Invalid( fldPath.Child("workers", "machinePools").Key(mp.Name).Child("variables", "overrides").Key(variable.Name), string(variable.Value.Raw), fmt.Sprintf("variable %q has DefinitionFrom set", variable.Name)), ) } } } } } return allErrs } // validateAutoscalerAnnotationsForCluster iterates the MachineDeploymentsTopology objects under Workers and ensures the replicas // field and min/max annotations for autoscaler are not set at the same time. Optionally it also checks if a given ClusterClass has // the annotations that may apply to this Cluster. func validateAutoscalerAnnotationsForCluster(cluster *clusterv1.Cluster, clusterClass *clusterv1.ClusterClass) field.ErrorList { var allErrs field.ErrorList if cluster.Spec.Topology == nil || cluster.Spec.Topology.Workers == nil { return allErrs } fldPath := field.NewPath("spec", "topology") for _, mdt := range cluster.Spec.Topology.Workers.MachineDeployments { if mdt.Replicas == nil { continue } for k := range mdt.Metadata.Annotations { if k == clusterv1.AutoscalerMinSizeAnnotation || k == clusterv1.AutoscalerMaxSizeAnnotation { allErrs = append( allErrs, field.Invalid( fldPath.Child("workers", "machineDeployments").Key(mdt.Name).Child("replicas"), mdt.Replicas, fmt.Sprintf("cannot be set for cluster %q in namespace %q if the same MachineDeploymentTopology has autoscaler annotations", cluster.Name, cluster.Namespace), ), ) break } } // Find a matching MachineDeploymentClass for this MachineDeploymentTopology and make sure it does not have // the autoscaler annotations in its Template. Skip this step entirely if clusterClass is nil. if clusterClass == nil { continue } for _, mdc := range clusterClass.Spec.Workers.MachineDeployments { if mdc.Class != mdt.Class { continue } for k := range mdc.Template.Metadata.Annotations { if k == clusterv1.AutoscalerMinSizeAnnotation || k == clusterv1.AutoscalerMaxSizeAnnotation { allErrs = append( allErrs, field.Invalid( fldPath.Child("workers", "machineDeployments").Key(mdt.Name).Child("replicas"), mdt.Replicas, fmt.Sprintf("cannot be set for cluster %q in namespace %q if the source class %q of this MachineDeploymentTopology has autoscaler annotations", cluster.Name, cluster.Namespace, mdt.Class), ), ) break } } } } return allErrs }
/* Copyright 2021 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package webhooks import ( "context" "fmt" "strings" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/api/v1beta1/index" "sigs.k8s.io/cluster-api/feature" "sigs.k8s.io/cluster-api/internal/topology/check" "sigs.k8s.io/cluster-api/internal/topology/names" "sigs.k8s.io/cluster-api/internal/topology/variables" ) func (webhook *ClusterClass) SetupWebhookWithManager(mgr ctrl.Manager) error { return ctrl.NewWebhookManagedBy(mgr). For(&clusterv1.ClusterClass{}). WithDefaulter(webhook). WithValidator(webhook). Complete() } // +kubebuilder:webhook:verbs=create;update;delete,path=/validate-cluster-x-k8s-io-v1beta1-clusterclass,mutating=false,failurePolicy=fail,matchPolicy=Equivalent,groups=cluster.x-k8s.io,resources=clusterclasses,versions=v1beta1,name=validation.clusterclass.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 // +kubebuilder:webhook:verbs=create;update,path=/mutate-cluster-x-k8s-io-v1beta1-clusterclass,mutating=true,failurePolicy=fail,matchPolicy=Equivalent,groups=cluster.x-k8s.io,resources=clusterclasses,versions=v1beta1,name=default.clusterclass.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 // ClusterClass implements a validation and defaulting webhook for ClusterClass. type ClusterClass struct { Client client.Reader } var _ webhook.CustomDefaulter = &ClusterClass{} var _ webhook.CustomValidator = &ClusterClass{} // Default implements defaulting for ClusterClass create and update. func (webhook *ClusterClass) Default(_ context.Context, obj runtime.Object) error { in, ok := obj.(*clusterv1.ClusterClass) if !ok { return apierrors.NewBadRequest(fmt.Sprintf("expected a ClusterClass but got a %T", obj)) } // Default all namespaces in the references to the object namespace. defaultNamespace(in.Spec.Infrastructure.Ref, in.Namespace) defaultNamespace(in.Spec.ControlPlane.Ref, in.Namespace) if in.Spec.ControlPlane.MachineInfrastructure != nil { defaultNamespace(in.Spec.ControlPlane.MachineInfrastructure.Ref, in.Namespace) } if in.Spec.ControlPlane.MachineHealthCheck != nil { defaultNamespace(in.Spec.ControlPlane.MachineHealthCheck.RemediationTemplate, in.Namespace) } for i := range in.Spec.Workers.MachineDeployments { defaultNamespace(in.Spec.Workers.MachineDeployments[i].Template.Bootstrap.Ref, in.Namespace) defaultNamespace(in.Spec.Workers.MachineDeployments[i].Template.Infrastructure.Ref, in.Namespace) if in.Spec.Workers.MachineDeployments[i].MachineHealthCheck != nil { defaultNamespace(in.Spec.Workers.MachineDeployments[i].MachineHealthCheck.RemediationTemplate, in.Namespace) } } for i := range in.Spec.Workers.MachinePools { defaultNamespace(in.Spec.Workers.MachinePools[i].Template.Bootstrap.Ref, in.Namespace) defaultNamespace(in.Spec.Workers.MachinePools[i].Template.Infrastructure.Ref, in.Namespace) } return nil } func defaultNamespace(ref *corev1.ObjectReference, namespace string) { if ref != nil && ref.Namespace == "" { ref.Namespace = namespace } } // ValidateCreate implements validation for ClusterClass create. func (webhook *ClusterClass) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { in, ok := obj.(*clusterv1.ClusterClass) if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a ClusterClass but got a %T", obj)) } return nil, webhook.validate(ctx, nil, in) } // ValidateUpdate implements validation for ClusterClass update. func (webhook *ClusterClass) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { newClusterClass, ok := newObj.(*clusterv1.ClusterClass) if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a ClusterClass but got a %T", newObj)) } oldClusterClass, ok := oldObj.(*clusterv1.ClusterClass) if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a ClusterClass but got a %T", oldObj)) } return nil, webhook.validate(ctx, oldClusterClass, newClusterClass) } // ValidateDelete implements validation for ClusterClass delete. func (webhook *ClusterClass) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { clusterClass, ok := obj.(*clusterv1.ClusterClass) if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a ClusterClass but got a %T", obj)) } clusters, err := webhook.getClustersUsingClusterClass(ctx, clusterClass) if err != nil { return nil, apierrors.NewInternalError(errors.Wrapf(err, "could not retrieve Clusters using ClusterClass")) } if len(clusters) > 0 { // TODO(killianmuldoon): Improve error here to include the names of some clusters using the clusterClass. return nil, apierrors.NewForbidden(clusterv1.GroupVersion.WithResource("ClusterClass").GroupResource(), clusterClass.Name, fmt.Errorf("ClusterClass cannot be deleted because it is used by %d Cluster(s)", len(clusters))) } return nil, nil } func (webhook *ClusterClass) validate(ctx context.Context, oldClusterClass, newClusterClass *clusterv1.ClusterClass) error { // NOTE: ClusterClass and managed topologies are behind ClusterTopology feature gate flag; the web hook // must prevent creating new objects when the feature flag is disabled. if !feature.Gates.Enabled(feature.ClusterTopology) { return field.Forbidden( field.NewPath("spec"), "can be set only if the ClusterTopology feature flag is enabled", ) } var allErrs field.ErrorList // Ensure all references are valid. allErrs = append(allErrs, check.ClusterClassReferencesAreValid(newClusterClass)...) // Ensure all MachineDeployment classes are unique. allErrs = append(allErrs, check.MachineDeploymentClassesAreUnique(newClusterClass)...) // Ensure all MachinePool classes are unique. allErrs = append(allErrs, check.MachinePoolClassesAreUnique(newClusterClass)...) // Ensure MachineHealthChecks are valid. allErrs = append(allErrs, validateMachineHealthCheckClasses(newClusterClass)...) // Ensure NamingStrategies are valid. allErrs = append(allErrs, validateNamingStrategies(newClusterClass)...) // Validate variables. var oldClusterClassVariables []clusterv1.ClusterClassVariable if oldClusterClass != nil { oldClusterClassVariables = oldClusterClass.Spec.Variables } allErrs = append(allErrs, variables.ValidateClusterClassVariables(ctx, oldClusterClassVariables, newClusterClass.Spec.Variables, field.NewPath("spec", "variables"))..., ) // Validate patches. allErrs = append(allErrs, validatePatches(newClusterClass)...) // Validate metadata allErrs = append(allErrs, validateClusterClassMetadata(newClusterClass)...) // If this is an update run additional validation. if oldClusterClass != nil { // Ensure spec changes are compatible. allErrs = append(allErrs, check.ClusterClassesAreCompatible(oldClusterClass, newClusterClass)...) // Retrieve all clusters using the ClusterClass. clusters, err := webhook.getClustersUsingClusterClass(ctx, oldClusterClass) if err != nil { allErrs = append(allErrs, field.InternalError(field.NewPath(""), errors.Wrapf(err, "Clusters using ClusterClass %v can not be retrieved", oldClusterClass.Name))) return apierrors.NewInvalid(clusterv1.GroupVersion.WithKind("ClusterClass").GroupKind(), newClusterClass.Name, allErrs) } // Ensure no MachineDeploymentClass currently in use has been removed from the ClusterClass. allErrs = append(allErrs, webhook.validateRemovedMachineDeploymentClassesAreNotUsed(clusters, oldClusterClass, newClusterClass)...) // Ensure no MachinePoolClass currently in use has been removed from the ClusterClass. allErrs = append(allErrs, webhook.validateRemovedMachinePoolClassesAreNotUsed(clusters, oldClusterClass, newClusterClass)...) // Ensure no MachineHealthCheck currently in use has been removed from the ClusterClass. allErrs = append(allErrs, validateUpdatesToMachineHealthCheckClasses(clusters, oldClusterClass, newClusterClass)...) allErrs = append(allErrs, validateAutoscalerAnnotationsForClusterClass(clusters, newClusterClass)...) } if len(allErrs) > 0 { return apierrors.NewInvalid(clusterv1.GroupVersion.WithKind("ClusterClass").GroupKind(), newClusterClass.Name, allErrs) } return nil } // validateUpdatesToMachineHealthCheckClasses checks if the updates made to MachineHealthChecks are valid. // It makes sure that if a MachineHealthCheck definition is dropped from the ClusterClass then none of the // clusters using the ClusterClass rely on it to create a MachineHealthCheck. // A cluster relies on an MachineHealthCheck in the ClusterClass if in cluster topology MachineHealthCheck // is explicitly enabled and it does not provide a MachineHealthCheckOverride. func validateUpdatesToMachineHealthCheckClasses(clusters []clusterv1.Cluster, oldClusterClass, newClusterClass *clusterv1.ClusterClass) field.ErrorList { var allErrs field.ErrorList // Check if the MachineHealthCheck for the control plane is dropped. if oldClusterClass.Spec.ControlPlane.MachineHealthCheck != nil && newClusterClass.Spec.ControlPlane.MachineHealthCheck == nil { // Make sure that none of the clusters are using this MachineHealthCheck. clustersUsingMHC := []string{} for _, cluster := range clusters { if cluster.Spec.Topology.ControlPlane.MachineHealthCheck != nil && cluster.Spec.Topology.ControlPlane.MachineHealthCheck.Enable != nil && *cluster.Spec.Topology.ControlPlane.MachineHealthCheck.Enable && cluster.Spec.Topology.ControlPlane.MachineHealthCheck.MachineHealthCheckClass.IsZero() { clustersUsingMHC = append(clustersUsingMHC, cluster.Name) } } if len(clustersUsingMHC) != 0 { allErrs = append(allErrs, field.Forbidden( field.NewPath("spec", "controlPlane", "machineHealthCheck"), fmt.Sprintf("MachineHealthCheck cannot be deleted because it is used by Cluster(s) %q", strings.Join(clustersUsingMHC, ",")), )) } } // For each MachineDeploymentClass check if the MachineHealthCheck definition is dropped. for _, newMdClass := range newClusterClass.Spec.Workers.MachineDeployments { oldMdClass := machineDeploymentClassOfName(oldClusterClass, newMdClass.Class) if oldMdClass == nil { // This is a new MachineDeploymentClass. Nothing to do here. continue } // If the MachineHealthCheck is dropped then check that no cluster is using it. if oldMdClass.MachineHealthCheck != nil && newMdClass.MachineHealthCheck == nil { clustersUsingMHC := []string{} for _, cluster := range clusters { if cluster.Spec.Topology.Workers == nil { continue } for _, mdTopology := range cluster.Spec.Topology.Workers.MachineDeployments { if mdTopology.Class == newMdClass.Class { if mdTopology.MachineHealthCheck != nil && mdTopology.MachineHealthCheck.Enable != nil && *mdTopology.MachineHealthCheck.Enable && mdTopology.MachineHealthCheck.MachineHealthCheckClass.IsZero() { clustersUsingMHC = append(clustersUsingMHC, cluster.Name) break } } } } if len(clustersUsingMHC) != 0 { allErrs = append(allErrs, field.Forbidden( field.NewPath("spec", "workers", "machineDeployments").Key(newMdClass.Class).Child("machineHealthCheck"), fmt.Sprintf("MachineHealthCheck cannot be deleted because it is used by Cluster(s) %q", strings.Join(clustersUsingMHC, ",")), )) } } } return allErrs } func (webhook *ClusterClass) validateRemovedMachineDeploymentClassesAreNotUsed(clusters []clusterv1.Cluster, oldClusterClass, newClusterClass *clusterv1.ClusterClass) field.ErrorList { var allErrs field.ErrorList removedClasses := webhook.removedMachineDeploymentClasses(oldClusterClass, newClusterClass) // If no classes have been removed return early as no further checks are needed. if len(removedClasses) == 0 { return nil } // Error if any Cluster using the ClusterClass uses a MachineDeploymentClass that has been removed. for _, c := range clusters { for _, machineDeploymentTopology := range c.Spec.Topology.Workers.MachineDeployments { if removedClasses.Has(machineDeploymentTopology.Class) { // TODO(killianmuldoon): Improve error printing here so large scale changes don't flood the error log e.g. deduplication, only example usages given. // TODO: consider if we get the index of the MachineDeploymentClass being deleted allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "workers", "machineDeployments"), fmt.Sprintf("MachineDeploymentClass %q cannot be deleted because it is used by Cluster %q", machineDeploymentTopology.Class, c.Name), )) } } } return allErrs } func (webhook *ClusterClass) validateRemovedMachinePoolClassesAreNotUsed(clusters []clusterv1.Cluster, oldClusterClass, newClusterClass *clusterv1.ClusterClass) field.ErrorList { var allErrs field.ErrorList removedClasses := webhook.removedMachinePoolClasses(oldClusterClass, newClusterClass) // If no classes have been removed return early as no further checks are needed. if len(removedClasses) == 0 { return nil } // Error if any Cluster using the ClusterClass uses a MachinePoolClass that has been removed. for _, c := range clusters { for _, machinePoolTopology := range c.Spec.Topology.Workers.MachinePools { if removedClasses.Has(machinePoolTopology.Class) { // TODO(killianmuldoon): Improve error printing here so large scale changes don't flood the error log e.g. deduplication, only example usages given. // TODO: consider if we get the index of the MachinePoolClass being deleted allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "workers", "machinePools"), fmt.Sprintf("MachinePoolClass %q cannot be deleted because it is used by Cluster %q", machinePoolTopology.Class, c.Name), )) } } } return allErrs } func (webhook *ClusterClass) removedMachineDeploymentClasses(oldClusterClass, newClusterClass *clusterv1.ClusterClass) sets.Set[string] { removedClasses := sets.Set[string]{} mdClasses := webhook.classNamesFromMDWorkerClass(newClusterClass.Spec.Workers) for _, oldClass := range oldClusterClass.Spec.Workers.MachineDeployments { if !mdClasses.Has(oldClass.Class) { removedClasses.Insert(oldClass.Class) } } return removedClasses } func (webhook *ClusterClass) removedMachinePoolClasses(oldClusterClass, newClusterClass *clusterv1.ClusterClass) sets.Set[string] { removedClasses := sets.Set[string]{} mpClasses := webhook.classNamesFromMPWorkerClass(newClusterClass.Spec.Workers) for _, oldClass := range oldClusterClass.Spec.Workers.MachinePools { if !mpClasses.Has(oldClass.Class) { removedClasses.Insert(oldClass.Class) } } return removedClasses } // classNamesFromMDWorkerClass returns the set of MachineDeployment class names. func (webhook *ClusterClass) classNamesFromMDWorkerClass(w clusterv1.WorkersClass) sets.Set[string] { classes := sets.Set[string]{} for _, class := range w.MachineDeployments { classes.Insert(class.Class) } return classes } // classNamesFromMPWorkerClass returns the set of MachinePool class names. func (webhook *ClusterClass) classNamesFromMPWorkerClass(w clusterv1.WorkersClass) sets.Set[string] { classes := sets.Set[string]{} for _, class := range w.MachinePools { classes.Insert(class.Class) } return classes } func (webhook *ClusterClass) getClustersUsingClusterClass(ctx context.Context, clusterClass *clusterv1.ClusterClass) ([]clusterv1.Cluster, error) { clusters := &clusterv1.ClusterList{} err := webhook.Client.List(ctx, clusters, client.MatchingFields{index.ClusterClassNameField: clusterClass.Name}, client.InNamespace(clusterClass.Namespace), ) if err != nil { return nil, err } return clusters.Items, nil } func getClusterClassVariablesMapWithReverseIndex(clusterClassVariables []clusterv1.ClusterClassVariable) (map[string]*clusterv1.ClusterClassVariable, map[string]int) { variablesMap := map[string]*clusterv1.ClusterClassVariable{} variablesIndexMap := map[string]int{} for i := range clusterClassVariables { variablesMap[clusterClassVariables[i].Name] = &clusterClassVariables[i] variablesIndexMap[clusterClassVariables[i].Name] = i } return variablesMap, variablesIndexMap } func validateMachineHealthCheckClasses(clusterClass *clusterv1.ClusterClass) field.ErrorList { var allErrs field.ErrorList // Validate ControlPlane MachineHealthCheck if defined. if clusterClass.Spec.ControlPlane.MachineHealthCheck != nil { fldPath := field.NewPath("spec", "controlPlane", "machineHealthCheck") allErrs = append(allErrs, validateMachineHealthCheckClass(fldPath, clusterClass.Namespace, clusterClass.Spec.ControlPlane.MachineHealthCheck)...) // Ensure ControlPlane does not define a MachineHealthCheck if it does not define MachineInfrastructure. if clusterClass.Spec.ControlPlane.MachineInfrastructure == nil { allErrs = append(allErrs, field.Forbidden( fldPath.Child("machineInfrastructure"), "can be set only if spec.controlPlane.machineInfrastructure is set", )) } } // Validate MachineDeployment MachineHealthChecks. for _, md := range clusterClass.Spec.Workers.MachineDeployments { if md.MachineHealthCheck == nil { continue } fldPath := field.NewPath("spec", "workers", "machineDeployments").Key(md.Class).Child("machineHealthCheck") allErrs = append(allErrs, validateMachineHealthCheckClass(fldPath, clusterClass.Namespace, md.MachineHealthCheck)...) } return allErrs } func validateNamingStrategies(clusterClass *clusterv1.ClusterClass) field.ErrorList { var allErrs field.ErrorList if clusterClass.Spec.ControlPlane.NamingStrategy != nil && clusterClass.Spec.ControlPlane.NamingStrategy.Template != nil { name, err := names.ControlPlaneNameGenerator(*clusterClass.Spec.ControlPlane.NamingStrategy.Template, "cluster").GenerateName() templateFldPath := field.NewPath("spec", "controlPlane", "namingStrategy", "template") if err != nil { allErrs = append(allErrs, field.Invalid( templateFldPath, *clusterClass.Spec.ControlPlane.NamingStrategy.Template, fmt.Sprintf("invalid ControlPlane name template: %v", err), )) } else { for _, err := range validation.IsDNS1123Subdomain(name) { allErrs = append(allErrs, field.Invalid(templateFldPath, *clusterClass.Spec.ControlPlane.NamingStrategy.Template, err)) } } } for _, md := range clusterClass.Spec.Workers.MachineDeployments { if md.NamingStrategy == nil || md.NamingStrategy.Template == nil { continue } name, err := names.MachineDeploymentNameGenerator(*md.NamingStrategy.Template, "cluster", "mdtopology").GenerateName() templateFldPath := field.NewPath("spec", "workers", "machineDeployments").Key(md.Class).Child("namingStrategy", "template") if err != nil { allErrs = append(allErrs, field.Invalid( templateFldPath, *md.NamingStrategy.Template, fmt.Sprintf("invalid MachineDeployment name template: %v", err), )) } else { for _, err := range validation.IsDNS1123Subdomain(name) { allErrs = append(allErrs, field.Invalid(templateFldPath, *md.NamingStrategy.Template, err)) } } } for _, mp := range clusterClass.Spec.Workers.MachinePools { if mp.NamingStrategy == nil || mp.NamingStrategy.Template == nil { continue } name, err := names.MachinePoolNameGenerator(*mp.NamingStrategy.Template, "cluster", "mptopology").GenerateName() templateFldPath := field.NewPath("spec", "workers", "machinePools").Key(mp.Class).Child("namingStrategy", "template") if err != nil { allErrs = append(allErrs, field.Invalid( templateFldPath, *mp.NamingStrategy.Template, fmt.Sprintf("invalid MachinePool name template: %v", err), )) } else { for _, err := range validation.IsDNS1123Subdomain(name) { allErrs = append(allErrs, field.Invalid(templateFldPath, *mp.NamingStrategy.Template, err)) } } } return allErrs } // validateMachineHealthCheckClass validates the MachineHealthCheckSpec fields defined in a MachineHealthCheckClass. func validateMachineHealthCheckClass(fldPath *field.Path, namepace string, m *clusterv1.MachineHealthCheckClass) field.ErrorList { mhc := clusterv1.MachineHealthCheck{ ObjectMeta: metav1.ObjectMeta{ Namespace: namepace, }, Spec: clusterv1.MachineHealthCheckSpec{ NodeStartupTimeout: m.NodeStartupTimeout, MaxUnhealthy: m.MaxUnhealthy, UnhealthyConditions: m.UnhealthyConditions, UnhealthyRange: m.UnhealthyRange, RemediationTemplate: m.RemediationTemplate, }} return (&MachineHealthCheck{}).validateCommonFields(&mhc, fldPath) } func validateClusterClassMetadata(clusterClass *clusterv1.ClusterClass) field.ErrorList { var allErrs field.ErrorList allErrs = append(allErrs, clusterClass.Spec.ControlPlane.Metadata.Validate(field.NewPath("spec", "controlPlane", "metadata"))...) for _, m := range clusterClass.Spec.Workers.MachineDeployments { allErrs = append(allErrs, m.Template.Metadata.Validate(field.NewPath("spec", "workers", "machineDeployments").Key(m.Class).Child("template", "metadata"))...) } for _, m := range clusterClass.Spec.Workers.MachinePools { allErrs = append(allErrs, m.Template.Metadata.Validate(field.NewPath("spec", "workers", "machinePools").Key(m.Class).Child("template", "metadata"))...) } return allErrs } // validateAutoscalerAnnotationsForClusterClass iterates over a list of Clusters that use a ClusterClass and returns // errors if the ClusterClass contains autoscaler annotations while a Cluster has worker replicas. func validateAutoscalerAnnotationsForClusterClass(clusters []clusterv1.Cluster, newClusterClass *clusterv1.ClusterClass) field.ErrorList { var allErrs field.ErrorList for _, c := range clusters { allErrs = append(allErrs, validateAutoscalerAnnotationsForCluster(&c, newClusterClass)...) } return allErrs }
/* Copyright 2021 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package webhooks import ( "context" "fmt" "strings" "time" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation/field" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/util/labels" "sigs.k8s.io/cluster-api/util/version" ) const defaultNodeDeletionTimeout = 10 * time.Second func (webhook *Machine) SetupWebhookWithManager(mgr ctrl.Manager) error { return ctrl.NewWebhookManagedBy(mgr). For(&clusterv1.Machine{}). WithDefaulter(webhook). WithValidator(webhook). Complete() } // +kubebuilder:webhook:verbs=create;update,path=/validate-cluster-x-k8s-io-v1beta1-machine,mutating=false,failurePolicy=fail,matchPolicy=Equivalent,groups=cluster.x-k8s.io,resources=machines,versions=v1beta1,name=validation.machine.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 // +kubebuilder:webhook:verbs=create;update,path=/mutate-cluster-x-k8s-io-v1beta1-machine,mutating=true,failurePolicy=fail,matchPolicy=Equivalent,groups=cluster.x-k8s.io,resources=machines,versions=v1beta1,name=default.machine.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 // Machine implements a validation and defaulting webhook for Machine. type Machine struct{} var _ webhook.CustomValidator = &Machine{} var _ webhook.CustomDefaulter = &Machine{} // Default implements webhook.Defaulter so a webhook will be registered for the type. func (webhook *Machine) Default(_ context.Context, obj runtime.Object) error { m, ok := obj.(*clusterv1.Machine) if !ok { return apierrors.NewBadRequest(fmt.Sprintf("expected a Machine but got a %T", obj)) } if m.Labels == nil { m.Labels = make(map[string]string) } m.Labels[clusterv1.ClusterNameLabel] = m.Spec.ClusterName if m.Spec.Bootstrap.ConfigRef != nil && m.Spec.Bootstrap.ConfigRef.Namespace == "" { m.Spec.Bootstrap.ConfigRef.Namespace = m.Namespace } if m.Spec.InfrastructureRef.Namespace == "" { m.Spec.InfrastructureRef.Namespace = m.Namespace } if m.Spec.Version != nil && !strings.HasPrefix(*m.Spec.Version, "v") { normalizedVersion := "v" + *m.Spec.Version m.Spec.Version = &normalizedVersion } if m.Spec.NodeDeletionTimeout == nil { m.Spec.NodeDeletionTimeout = &metav1.Duration{Duration: defaultNodeDeletionTimeout} } return nil } // ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type. func (webhook *Machine) ValidateCreate(_ context.Context, obj runtime.Object) (admission.Warnings, error) { m, ok := obj.(*clusterv1.Machine) if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a Machine but got a %T", obj)) } return nil, webhook.validate(nil, m) } // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. func (webhook *Machine) ValidateUpdate(_ context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { oldM, ok := oldObj.(*clusterv1.Machine) if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a Machine but got a %T", oldObj)) } newM, ok := newObj.(*clusterv1.Machine) if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a Machine but got a %T", newObj)) } return nil, webhook.validate(oldM, newM) } // ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type. func (webhook *Machine) ValidateDelete(_ context.Context, _ runtime.Object) (admission.Warnings, error) { return nil, nil } func (webhook *Machine) validate(oldM, newM *clusterv1.Machine) error { var allErrs field.ErrorList specPath := field.NewPath("spec") if newM.Spec.Bootstrap.ConfigRef == nil && newM.Spec.Bootstrap.DataSecretName == nil { // MachinePool Machines don't have a bootstrap configRef, so don't require it. The bootstrap config is instead owned by the MachinePool. if !labels.IsMachinePoolOwned(newM) { allErrs = append( allErrs, field.Required( specPath.Child("bootstrap", "data"), "expected either spec.bootstrap.dataSecretName or spec.bootstrap.configRef to be populated", ), ) } } if newM.Spec.Bootstrap.ConfigRef != nil && newM.Spec.Bootstrap.ConfigRef.Namespace != newM.Namespace { allErrs = append( allErrs, field.Invalid( specPath.Child("bootstrap", "configRef", "namespace"), newM.Spec.Bootstrap.ConfigRef.Namespace, "must match metadata.namespace", ), ) } if newM.Spec.InfrastructureRef.Namespace != newM.Namespace { allErrs = append( allErrs, field.Invalid( specPath.Child("infrastructureRef", "namespace"), newM.Spec.InfrastructureRef.Namespace, "must match metadata.namespace", ), ) } if oldM != nil && oldM.Spec.ClusterName != newM.Spec.ClusterName { allErrs = append( allErrs, field.Forbidden(specPath.Child("clusterName"), "field is immutable"), ) } if newM.Spec.Version != nil { if !version.KubeSemver.MatchString(*newM.Spec.Version) { allErrs = append(allErrs, field.Invalid(specPath.Child("version"), *newM.Spec.Version, "must be a valid semantic version")) } } if len(allErrs) == 0 { return nil } return apierrors.NewInvalid(clusterv1.GroupVersion.WithKind("Machine").GroupKind(), newM.Name, allErrs) }
/* Copyright 2021 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package webhooks import ( "context" "fmt" "strconv" "strings" "github.com/pkg/errors" v1 "k8s.io/api/admission/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/feature" "sigs.k8s.io/cluster-api/util/version" ) func (webhook *MachineDeployment) SetupWebhookWithManager(mgr ctrl.Manager) error { if webhook.decoder == nil { webhook.decoder = admission.NewDecoder(mgr.GetScheme()) } return ctrl.NewWebhookManagedBy(mgr). For(&clusterv1.MachineDeployment{}). WithDefaulter(webhook). WithValidator(webhook). Complete() } // +kubebuilder:webhook:verbs=create;update,path=/validate-cluster-x-k8s-io-v1beta1-machinedeployment,mutating=false,failurePolicy=fail,matchPolicy=Equivalent,groups=cluster.x-k8s.io,resources=machinedeployments,versions=v1beta1,name=validation.machinedeployment.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 // +kubebuilder:webhook:verbs=create;update,path=/mutate-cluster-x-k8s-io-v1beta1-machinedeployment,mutating=true,failurePolicy=fail,matchPolicy=Equivalent,groups=cluster.x-k8s.io,resources=machinedeployments,versions=v1beta1,name=default.machinedeployment.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 // MachineDeployment implements a validation and defaulting webhook for MachineDeployment. type MachineDeployment struct { decoder admission.Decoder } var _ webhook.CustomDefaulter = &MachineDeployment{} var _ webhook.CustomValidator = &MachineDeployment{} // Default implements webhook.CustomDefaulter. func (webhook *MachineDeployment) Default(ctx context.Context, obj runtime.Object) error { m, ok := obj.(*clusterv1.MachineDeployment) if !ok { return apierrors.NewBadRequest(fmt.Sprintf("expected a MachineDeployment but got a %T", obj)) } req, err := admission.RequestFromContext(ctx) if err != nil { return err } dryRun := false if req.DryRun != nil { dryRun = *req.DryRun } var oldMD *clusterv1.MachineDeployment if req.Operation == v1.Update { oldMD = &clusterv1.MachineDeployment{} if err := webhook.decoder.DecodeRaw(req.OldObject, oldMD); err != nil { return errors.Wrapf(err, "failed to decode oldObject to MachineDeployment") } } if m.Labels == nil { m.Labels = make(map[string]string) } m.Labels[clusterv1.ClusterNameLabel] = m.Spec.ClusterName replicas, err := calculateMachineDeploymentReplicas(ctx, oldMD, m, dryRun) if err != nil { return err } m.Spec.Replicas = ptr.To[int32](replicas) if m.Spec.MinReadySeconds == nil { m.Spec.MinReadySeconds = ptr.To[int32](0) } if m.Spec.RevisionHistoryLimit == nil { m.Spec.RevisionHistoryLimit = ptr.To[int32](1) } if m.Spec.ProgressDeadlineSeconds == nil { m.Spec.ProgressDeadlineSeconds = ptr.To[int32](600) } if m.Spec.Selector.MatchLabels == nil { m.Spec.Selector.MatchLabels = make(map[string]string) } if m.Spec.Strategy == nil { m.Spec.Strategy = &clusterv1.MachineDeploymentStrategy{} } if m.Spec.Strategy.Type == "" { m.Spec.Strategy.Type = clusterv1.RollingUpdateMachineDeploymentStrategyType } if m.Spec.Template.Labels == nil { m.Spec.Template.Labels = make(map[string]string) } // Default RollingUpdate strategy only if strategy type is RollingUpdate. if m.Spec.Strategy.Type == clusterv1.RollingUpdateMachineDeploymentStrategyType { if m.Spec.Strategy.RollingUpdate == nil { m.Spec.Strategy.RollingUpdate = &clusterv1.MachineRollingUpdateDeployment{} } if m.Spec.Strategy.RollingUpdate.MaxSurge == nil { ios1 := intstr.FromInt(1) m.Spec.Strategy.RollingUpdate.MaxSurge = &ios1 } if m.Spec.Strategy.RollingUpdate.MaxUnavailable == nil { ios0 := intstr.FromInt(0) m.Spec.Strategy.RollingUpdate.MaxUnavailable = &ios0 } } // If no selector has been provided, add label and selector for the // MachineDeployment's name as a default way of providing uniqueness. if len(m.Spec.Selector.MatchLabels) == 0 && len(m.Spec.Selector.MatchExpressions) == 0 { m.Spec.Selector.MatchLabels[clusterv1.MachineDeploymentNameLabel] = m.Name m.Spec.Template.Labels[clusterv1.MachineDeploymentNameLabel] = m.Name } // Make sure selector and template to be in the same cluster. m.Spec.Selector.MatchLabels[clusterv1.ClusterNameLabel] = m.Spec.ClusterName m.Spec.Template.Labels[clusterv1.ClusterNameLabel] = m.Spec.ClusterName // tolerate version strings without a "v" prefix: prepend it if it's not there if m.Spec.Template.Spec.Version != nil && !strings.HasPrefix(*m.Spec.Template.Spec.Version, "v") { normalizedVersion := "v" + *m.Spec.Template.Spec.Version m.Spec.Template.Spec.Version = &normalizedVersion } return nil } // ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type. func (webhook *MachineDeployment) ValidateCreate(_ context.Context, obj runtime.Object) (admission.Warnings, error) { m, ok := obj.(*clusterv1.MachineDeployment) if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a MachineDeployment but got a %T", obj)) } return nil, webhook.validate(nil, m) } // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. func (webhook *MachineDeployment) ValidateUpdate(_ context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { oldMD, ok := oldObj.(*clusterv1.MachineDeployment) if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a MachineDeployment but got a %T", oldObj)) } newMD, ok := newObj.(*clusterv1.MachineDeployment) if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a MachineDeployment but got a %T", newObj)) } return nil, webhook.validate(oldMD, newMD) } // ValidateDelete implements webhook.Validator so a webhook will be registered for the type. func (webhook *MachineDeployment) ValidateDelete(_ context.Context, _ runtime.Object) (admission.Warnings, error) { return nil, nil } func (webhook *MachineDeployment) validate(oldMD, newMD *clusterv1.MachineDeployment) error { var allErrs field.ErrorList // The MachineDeployment name is used as a label value. This check ensures names which are not be valid label values are rejected. if errs := validation.IsValidLabelValue(newMD.Name); len(errs) != 0 { for _, err := range errs { allErrs = append( allErrs, field.Invalid( field.NewPath("metadata", "name"), newMD.Name, fmt.Sprintf("must be a valid label value: %s", err), ), ) } } specPath := field.NewPath("spec") selector, err := metav1.LabelSelectorAsSelector(&newMD.Spec.Selector) if err != nil { allErrs = append( allErrs, field.Invalid(specPath.Child("selector"), newMD.Spec.Selector, err.Error()), ) } else if !selector.Matches(labels.Set(newMD.Spec.Template.Labels)) { allErrs = append( allErrs, field.Forbidden( specPath.Child("template", "metadata", "labels"), fmt.Sprintf("must match spec.selector %q", selector.String()), ), ) } // MachineSet preflight checks that should be skipped could also be set as annotation on the MachineDeployment // since MachineDeployment annotations are synced to the MachineSet. if feature.Gates.Enabled(feature.MachineSetPreflightChecks) { if err := validateSkippedMachineSetPreflightChecks(newMD); err != nil { allErrs = append(allErrs, err) } } if oldMD != nil && oldMD.Spec.ClusterName != newMD.Spec.ClusterName { allErrs = append( allErrs, field.Forbidden( specPath.Child("clusterName"), "field is immutable", ), ) } if newMD.Spec.Strategy != nil && newMD.Spec.Strategy.RollingUpdate != nil { total := 1 if newMD.Spec.Replicas != nil { total = int(*newMD.Spec.Replicas) } if newMD.Spec.Strategy.RollingUpdate.MaxSurge != nil { if _, err := intstr.GetScaledValueFromIntOrPercent(newMD.Spec.Strategy.RollingUpdate.MaxSurge, total, true); err != nil { allErrs = append( allErrs, field.Invalid(specPath.Child("strategy", "rollingUpdate", "maxSurge"), newMD.Spec.Strategy.RollingUpdate.MaxSurge, fmt.Sprintf("must be either an int or a percentage: %v", err.Error())), ) } } if newMD.Spec.Strategy.RollingUpdate.MaxUnavailable != nil { if _, err := intstr.GetScaledValueFromIntOrPercent(newMD.Spec.Strategy.RollingUpdate.MaxUnavailable, total, true); err != nil { allErrs = append( allErrs, field.Invalid(specPath.Child("strategy", "rollingUpdate", "maxUnavailable"), newMD.Spec.Strategy.RollingUpdate.MaxUnavailable, fmt.Sprintf("must be either an int or a percentage: %v", err.Error())), ) } } } if newMD.Spec.Strategy != nil && newMD.Spec.Strategy.Remediation != nil { total := 1 if newMD.Spec.Replicas != nil { total = int(*newMD.Spec.Replicas) } if newMD.Spec.Strategy.Remediation.MaxInFlight != nil { if _, err := intstr.GetScaledValueFromIntOrPercent(newMD.Spec.Strategy.Remediation.MaxInFlight, total, true); err != nil { allErrs = append( allErrs, field.Invalid(specPath.Child("strategy", "remediation", "maxInFlight"), newMD.Spec.Strategy.Remediation.MaxInFlight.String(), fmt.Sprintf("must be either an int or a percentage: %v", err.Error())), ) } } } if newMD.Spec.Template.Spec.Version != nil { if !version.KubeSemver.MatchString(*newMD.Spec.Template.Spec.Version) { allErrs = append(allErrs, field.Invalid(specPath.Child("template", "spec", "version"), *newMD.Spec.Template.Spec.Version, "must be a valid semantic version")) } } // Validate the metadata of the template. allErrs = append(allErrs, newMD.Spec.Template.ObjectMeta.Validate(specPath.Child("template", "metadata"))...) if len(allErrs) == 0 { return nil } return apierrors.NewInvalid(clusterv1.GroupVersion.WithKind("MachineDeployment").GroupKind(), newMD.Name, allErrs) } // calculateMachineDeploymentReplicas calculates the default value of the replicas field. // The value will be calculated based on the following logic: // * if replicas is already set on newMD, keep the current value // * if the autoscaler min size and max size annotations are set: // - if it's a new MachineDeployment, use min size // - if the replicas field of the old MachineDeployment is < min size, use min size // - if the replicas field of the old MachineDeployment is > max size, use max size // - if the replicas field of the old MachineDeployment is in the (min size, max size) range, keep the value from the oldMD // // * otherwise use 1 // // The goal of this logic is to provide a smoother UX for clusters using the Kubernetes autoscaler. // Note: Autoscaler only takes over control of the replicas field if the replicas value is in the (min size, max size) range. // // We are supporting the following use cases: // * A new MD is created and replicas should be managed by the autoscaler // - If the min size and max size annotations are set, the replicas field is defaulted to the value of the min size // annotation so the autoscaler can take control. // // * An existing MD which initially wasn't controlled by the autoscaler should be later controlled by the autoscaler // - To adopt an existing MD users can use the min size and max size annotations to enable the autoscaler // and to ensure the replicas field is within the (min size, max size) range. Without defaulting based on the annotations, handing over // control to the autoscaler by unsetting the replicas field would lead to the field being set to 1. This could be // very disruptive if the previous value of the replica field is greater than 1. func calculateMachineDeploymentReplicas(ctx context.Context, oldMD *clusterv1.MachineDeployment, newMD *clusterv1.MachineDeployment, dryRun bool) (int32, error) { // If replicas is already set => Keep the current value. if newMD.Spec.Replicas != nil { return *newMD.Spec.Replicas, nil } log := ctrl.LoggerFrom(ctx) // If both autoscaler annotations are set, use them to calculate the default value. minSizeString, hasMinSizeAnnotation := newMD.Annotations[clusterv1.AutoscalerMinSizeAnnotation] maxSizeString, hasMaxSizeAnnotation := newMD.Annotations[clusterv1.AutoscalerMaxSizeAnnotation] if hasMinSizeAnnotation && hasMaxSizeAnnotation { minSize, err := strconv.ParseInt(minSizeString, 10, 32) if err != nil { return 0, errors.Wrapf(err, "failed to caculate MachineDeployment replicas value: could not parse the value of the %q annotation", clusterv1.AutoscalerMinSizeAnnotation) } maxSize, err := strconv.ParseInt(maxSizeString, 10, 32) if err != nil { return 0, errors.Wrapf(err, "failed to caculate MachineDeployment replicas value: could not parse the value of the %q annotation", clusterv1.AutoscalerMaxSizeAnnotation) } // If it's a new MachineDeployment => Use the min size. // Note: This will result in a scale up to get into the range where autoscaler takes over. if oldMD == nil { if !dryRun { log.V(2).Info(fmt.Sprintf("Replica field has been defaulted to %d based on the %s annotation (MD is a new MD)", minSize, clusterv1.AutoscalerMinSizeAnnotation)) } return int32(minSize), nil } // Otherwise we are handing over the control for the replicas field for an existing MachineDeployment // to the autoscaler. switch { // If the old MachineDeployment doesn't have replicas set => Use the min size. // Note: As defaulting always sets the replica field, this case should not be possible // We only have this handling to be 100% safe against panics. case oldMD.Spec.Replicas == nil: if !dryRun { log.V(2).Info(fmt.Sprintf("Replica field has been defaulted to %d based on the %s annotation (old MD didn't have replicas set)", minSize, clusterv1.AutoscalerMinSizeAnnotation)) } return int32(minSize), nil // If the old MachineDeployment replicas are lower than min size => Use the min size. // Note: This will result in a scale up to get into the range where autoscaler takes over. case *oldMD.Spec.Replicas < int32(minSize): if !dryRun { log.V(2).Info(fmt.Sprintf("Replica field has been defaulted to %d based on the %s annotation (old MD had replicas below min size)", minSize, clusterv1.AutoscalerMinSizeAnnotation)) } return int32(minSize), nil // If the old MachineDeployment replicas are higher than max size => Use the max size. // Note: This will result in a scale down to get into the range where autoscaler takes over. case *oldMD.Spec.Replicas > int32(maxSize): if !dryRun { log.V(2).Info(fmt.Sprintf("Replica field has been defaulted to %d based on the %s annotation (old MD had replicas above max size)", maxSize, clusterv1.AutoscalerMaxSizeAnnotation)) } return int32(maxSize), nil // If the old MachineDeployment replicas are between min and max size => Keep the current value. default: if !dryRun { log.V(2).Info(fmt.Sprintf("Replica field has been defaulted to %d based on replicas of the old MachineDeployment (old MD had replicas within min size / max size range)", *oldMD.Spec.Replicas)) } return *oldMD.Spec.Replicas, nil } } // If neither the default nor the autoscaler annotations are set => Default to 1. if !dryRun { log.V(2).Info("Replica field has been defaulted to 1") } return 1, nil }
/* Copyright 2021 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package webhooks import ( "context" "fmt" "time" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/validation/field" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ) var ( // Minimum time allowed for a node to start up. minNodeStartupTimeout = metav1.Duration{Duration: 30 * time.Second} // We allow users to disable the nodeStartupTimeout by setting the duration to 0. disabledNodeStartupTimeout = clusterv1.ZeroDuration ) // SetMinNodeStartupTimeout allows users to optionally set a custom timeout // for the validation webhook. // // This function is mostly used within envtest (integration tests), and should // never be used in a production environment. func SetMinNodeStartupTimeout(d metav1.Duration) { minNodeStartupTimeout = d } func (webhook *MachineHealthCheck) SetupWebhookWithManager(mgr ctrl.Manager) error { return ctrl.NewWebhookManagedBy(mgr). For(&clusterv1.MachineHealthCheck{}). WithDefaulter(webhook). WithValidator(webhook). Complete() } // +kubebuilder:webhook:verbs=create;update,path=/validate-cluster-x-k8s-io-v1beta1-machinehealthcheck,mutating=false,failurePolicy=fail,matchPolicy=Equivalent,groups=cluster.x-k8s.io,resources=machinehealthchecks,versions=v1beta1,name=validation.machinehealthcheck.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 // +kubebuilder:webhook:verbs=create;update,path=/mutate-cluster-x-k8s-io-v1beta1-machinehealthcheck,mutating=true,failurePolicy=fail,matchPolicy=Equivalent,groups=cluster.x-k8s.io,resources=machinehealthchecks,versions=v1beta1,name=default.machinehealthcheck.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 // MachineHealthCheck implements a validation and defaulting webhook for MachineHealthCheck. type MachineHealthCheck struct{} var _ webhook.CustomDefaulter = &MachineHealthCheck{} var _ webhook.CustomValidator = &MachineHealthCheck{} // Default implements webhook.CustomDefaulter so a webhook will be registered for the type. func (webhook *MachineHealthCheck) Default(_ context.Context, obj runtime.Object) error { m, ok := obj.(*clusterv1.MachineHealthCheck) if !ok { return apierrors.NewBadRequest(fmt.Sprintf("expected a MachineHealthCheck but got a %T", obj)) } if m.Labels == nil { m.Labels = make(map[string]string) } m.Labels[clusterv1.ClusterNameLabel] = m.Spec.ClusterName if m.Spec.MaxUnhealthy == nil { defaultMaxUnhealthy := intstr.FromString("100%") m.Spec.MaxUnhealthy = &defaultMaxUnhealthy } if m.Spec.NodeStartupTimeout == nil { m.Spec.NodeStartupTimeout = &clusterv1.DefaultNodeStartupTimeout } if m.Spec.RemediationTemplate != nil && m.Spec.RemediationTemplate.Namespace == "" { m.Spec.RemediationTemplate.Namespace = m.Namespace } return nil } // ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type. func (webhook *MachineHealthCheck) ValidateCreate(_ context.Context, obj runtime.Object) (admission.Warnings, error) { m, ok := obj.(*clusterv1.MachineHealthCheck) if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a MachineHealthCheck but got a %T", obj)) } return nil, webhook.validate(nil, m) } // ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type. func (webhook *MachineHealthCheck) ValidateUpdate(_ context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { oldM, ok := oldObj.(*clusterv1.MachineHealthCheck) if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a MachineHealthCheck but got a %T", oldObj)) } newM, ok := newObj.(*clusterv1.MachineHealthCheck) if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a MachineHealthCheck but got a %T", newObj)) } return nil, webhook.validate(oldM, newM) } // ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type. func (webhook *MachineHealthCheck) ValidateDelete(_ context.Context, _ runtime.Object) (admission.Warnings, error) { return nil, nil } func (webhook *MachineHealthCheck) validate(oldMHC, newMHC *clusterv1.MachineHealthCheck) error { var allErrs field.ErrorList specPath := field.NewPath("spec") // Validate selector parses as Selector selector, err := metav1.LabelSelectorAsSelector(&newMHC.Spec.Selector) if err != nil { allErrs = append( allErrs, field.Invalid(specPath.Child("selector"), newMHC.Spec.Selector, err.Error()), ) } // Validate that the selector isn't empty. if selector != nil && selector.Empty() { allErrs = append( allErrs, field.Required(specPath.Child("selector"), "selector must not be empty"), ) } if clusterName, ok := newMHC.Spec.Selector.MatchLabels[clusterv1.ClusterNameLabel]; ok && clusterName != newMHC.Spec.ClusterName { allErrs = append( allErrs, field.Invalid(specPath.Child("selector"), newMHC.Spec.Selector, "cannot specify a cluster selector other than the one specified by ClusterName")) } if oldMHC != nil && oldMHC.Spec.ClusterName != newMHC.Spec.ClusterName { allErrs = append( allErrs, field.Forbidden(specPath.Child("clusterName"), "field is immutable"), ) } allErrs = append(allErrs, webhook.validateCommonFields(newMHC, specPath)...) if len(allErrs) == 0 { return nil } return apierrors.NewInvalid(clusterv1.GroupVersion.WithKind("MachineHealthCheck").GroupKind(), newMHC.Name, allErrs) } // ValidateCommonFields validates NodeStartupTimeout, MaxUnhealthy, and RemediationTemplate of the MHC. // These are the fields in common with other types which define MachineHealthChecks such as MachineHealthCheckClass and MachineHealthCheckTopology. func (webhook *MachineHealthCheck) validateCommonFields(m *clusterv1.MachineHealthCheck, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList if m.Spec.NodeStartupTimeout != nil && m.Spec.NodeStartupTimeout.Seconds() != disabledNodeStartupTimeout.Seconds() && m.Spec.NodeStartupTimeout.Seconds() < minNodeStartupTimeout.Seconds() { allErrs = append( allErrs, field.Invalid(fldPath.Child("nodeStartupTimeout"), m.Spec.NodeStartupTimeout.String(), "must be at least 30s"), ) } if m.Spec.MaxUnhealthy != nil { if _, err := intstr.GetScaledValueFromIntOrPercent(m.Spec.MaxUnhealthy, 0, false); err != nil { allErrs = append( allErrs, field.Invalid(fldPath.Child("maxUnhealthy"), m.Spec.MaxUnhealthy, fmt.Sprintf("must be either an int or a percentage: %v", err.Error())), ) } } if m.Spec.RemediationTemplate != nil && m.Spec.RemediationTemplate.Namespace != m.Namespace { allErrs = append( allErrs, field.Invalid( fldPath.Child("remediationTemplate", "namespace"), m.Spec.RemediationTemplate.Namespace, "must match metadata.namespace", ), ) } return allErrs }
/* Copyright 2021 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package webhooks import ( "context" "fmt" "strconv" "strings" "github.com/pkg/errors" v1 "k8s.io/api/admission/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/feature" "sigs.k8s.io/cluster-api/util/labels/format" "sigs.k8s.io/cluster-api/util/version" ) func (webhook *MachineSet) SetupWebhookWithManager(mgr ctrl.Manager) error { if webhook.decoder == nil { webhook.decoder = admission.NewDecoder(mgr.GetScheme()) } return ctrl.NewWebhookManagedBy(mgr). For(&clusterv1.MachineSet{}). WithDefaulter(webhook). WithValidator(webhook). Complete() } // +kubebuilder:webhook:verbs=create;update,path=/validate-cluster-x-k8s-io-v1beta1-machineset,mutating=false,failurePolicy=fail,matchPolicy=Equivalent,groups=cluster.x-k8s.io,resources=machinesets,versions=v1beta1,name=validation.machineset.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 // +kubebuilder:webhook:verbs=create;update,path=/mutate-cluster-x-k8s-io-v1beta1-machineset,mutating=true,failurePolicy=fail,matchPolicy=Equivalent,groups=cluster.x-k8s.io,resources=machinesets,versions=v1beta1,name=default.machineset.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 // MachineSet implements a validation and defaulting webhook for MachineSet. type MachineSet struct { decoder admission.Decoder } var _ webhook.CustomDefaulter = &MachineSet{} var _ webhook.CustomValidator = &MachineSet{} // Default sets default MachineSet field values. func (webhook *MachineSet) Default(ctx context.Context, obj runtime.Object) error { m, ok := obj.(*clusterv1.MachineSet) if !ok { return apierrors.NewBadRequest(fmt.Sprintf("expected a MachineSet but got a %T", obj)) } req, err := admission.RequestFromContext(ctx) if err != nil { return err } dryRun := false if req.DryRun != nil { dryRun = *req.DryRun } var oldMS *clusterv1.MachineSet if req.Operation == v1.Update { oldMS = &clusterv1.MachineSet{} if err := webhook.decoder.DecodeRaw(req.OldObject, oldMS); err != nil { return errors.Wrapf(err, "failed to decode oldObject to MachineSet") } } if m.Labels == nil { m.Labels = make(map[string]string) } m.Labels[clusterv1.ClusterNameLabel] = m.Spec.ClusterName replicas, err := calculateMachineSetReplicas(ctx, oldMS, m, dryRun) if err != nil { return err } m.Spec.Replicas = ptr.To[int32](replicas) if m.Spec.DeletePolicy == "" { randomPolicy := string(clusterv1.RandomMachineSetDeletePolicy) m.Spec.DeletePolicy = randomPolicy } if m.Spec.Selector.MatchLabels == nil { m.Spec.Selector.MatchLabels = make(map[string]string) } if m.Spec.Template.Labels == nil { m.Spec.Template.Labels = make(map[string]string) } if len(m.Spec.Selector.MatchLabels) == 0 && len(m.Spec.Selector.MatchExpressions) == 0 { // Note: MustFormatValue is used here as the value of this label will be a hash if the MachineSet name is longer than 63 characters. m.Spec.Selector.MatchLabels[clusterv1.MachineSetNameLabel] = format.MustFormatValue(m.Name) m.Spec.Template.Labels[clusterv1.MachineSetNameLabel] = format.MustFormatValue(m.Name) } if m.Spec.Template.Spec.Version != nil && !strings.HasPrefix(*m.Spec.Template.Spec.Version, "v") { normalizedVersion := "v" + *m.Spec.Template.Spec.Version m.Spec.Template.Spec.Version = &normalizedVersion } return nil } // ValidateCreate implements webhook.Validator so a webhook will be registered for the type. func (webhook *MachineSet) ValidateCreate(_ context.Context, obj runtime.Object) (admission.Warnings, error) { m, ok := obj.(*clusterv1.MachineSet) if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a MachineSet but got a %T", obj)) } return nil, webhook.validate(nil, m) } // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. func (webhook *MachineSet) ValidateUpdate(_ context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { oldMS, ok := oldObj.(*clusterv1.MachineSet) if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a MachineSet but got a %T", oldObj)) } newMS, ok := newObj.(*clusterv1.MachineSet) if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a MachineSet but got a %T", newObj)) } return nil, webhook.validate(oldMS, newMS) } // ValidateDelete implements webhook.Validator so a webhook will be registered for the type. func (webhook *MachineSet) ValidateDelete(_ context.Context, _ runtime.Object) (admission.Warnings, error) { return nil, nil } func (webhook *MachineSet) validate(oldMS, newMS *clusterv1.MachineSet) error { var allErrs field.ErrorList specPath := field.NewPath("spec") selector, err := metav1.LabelSelectorAsSelector(&newMS.Spec.Selector) if err != nil { allErrs = append( allErrs, field.Invalid( specPath.Child("selector"), newMS.Spec.Selector, err.Error(), ), ) } else if !selector.Matches(labels.Set(newMS.Spec.Template.Labels)) { allErrs = append( allErrs, field.Invalid( specPath.Child("template", "metadata", "labels"), newMS.Spec.Template.ObjectMeta.Labels, fmt.Sprintf("must match spec.selector %q", selector.String()), ), ) } if feature.Gates.Enabled(feature.MachineSetPreflightChecks) { if err := validateSkippedMachineSetPreflightChecks(newMS); err != nil { allErrs = append(allErrs, err) } } if oldMS != nil && oldMS.Spec.ClusterName != newMS.Spec.ClusterName { allErrs = append( allErrs, field.Forbidden( specPath.Child("clusterName"), "field is immutable", ), ) } if newMS.Spec.Template.Spec.Version != nil { if !version.KubeSemver.MatchString(*newMS.Spec.Template.Spec.Version) { allErrs = append( allErrs, field.Invalid( specPath.Child("template", "spec", "version"), *newMS.Spec.Template.Spec.Version, "must be a valid semantic version", ), ) } } // Validate the metadata of the template. allErrs = append(allErrs, newMS.Spec.Template.ObjectMeta.Validate(specPath.Child("template", "metadata"))...) if len(allErrs) == 0 { return nil } return apierrors.NewInvalid(clusterv1.GroupVersion.WithKind("MachineSet").GroupKind(), newMS.Name, allErrs) } func validateSkippedMachineSetPreflightChecks(o client.Object) *field.Error { if o == nil { return nil } skip := o.GetAnnotations()[clusterv1.MachineSetSkipPreflightChecksAnnotation] if skip == "" { return nil } supported := sets.New[clusterv1.MachineSetPreflightCheck]( clusterv1.MachineSetPreflightCheckAll, clusterv1.MachineSetPreflightCheckKubeadmVersionSkew, clusterv1.MachineSetPreflightCheckKubernetesVersionSkew, clusterv1.MachineSetPreflightCheckControlPlaneIsStable, ) skippedList := strings.Split(skip, ",") invalid := []clusterv1.MachineSetPreflightCheck{} for i := range skippedList { skipped := clusterv1.MachineSetPreflightCheck(strings.TrimSpace(skippedList[i])) if !supported.Has(skipped) { invalid = append(invalid, skipped) } } if len(invalid) > 0 { return field.Invalid( field.NewPath("metadata", "annotations", clusterv1.MachineSetSkipPreflightChecksAnnotation), invalid, fmt.Sprintf("skipped preflight check(s) must be among: %v", sets.List(supported)), ) } return nil } // calculateMachineSetReplicas calculates the default value of the replicas field. // The value will be calculated based on the following logic: // * if replicas is already set on newMS, keep the current value // * if the autoscaler min size and max size annotations are set: // - if it's a new MachineSet, use min size // - if the replicas field of the old MachineSet is < min size, use min size // - if the replicas field of the old MachineSet is > max size, use max size // - if the replicas field of the old MachineSet is in the (min size, max size) range, keep the value from the oldMS // // * otherwise use 1 // // The goal of this logic is to provide a smoother UX for clusters using the Kubernetes autoscaler. // Note: Autoscaler only takes over control of the replicas field if the replicas value is in the (min size, max size) range. // // We are supporting the following use cases: // * A new MS is created and replicas should be managed by the autoscaler // - If the min size and max size annotations are set, the replicas field is defaulted to the value of the min size // annotation so the autoscaler can take control. // // * An existing MS which initially wasn't controlled by the autoscaler should be later controlled by the autoscaler // - To adopt an existing MS users can use the min size and max size annotations to enable the autoscaler // and to ensure the replicas field is within the (min size, max size) range. Without defaulting based on the annotations, handing over // control to the autoscaler by unsetting the replicas field would lead to the field being set to 1. This could be // very disruptive if the previous value of the replica field is greater than 1. func calculateMachineSetReplicas(ctx context.Context, oldMS *clusterv1.MachineSet, newMS *clusterv1.MachineSet, dryRun bool) (int32, error) { // If replicas is already set => Keep the current value. if newMS.Spec.Replicas != nil { return *newMS.Spec.Replicas, nil } log := ctrl.LoggerFrom(ctx) // If both autoscaler annotations are set, use them to calculate the default value. minSizeString, hasMinSizeAnnotation := newMS.Annotations[clusterv1.AutoscalerMinSizeAnnotation] maxSizeString, hasMaxSizeAnnotation := newMS.Annotations[clusterv1.AutoscalerMaxSizeAnnotation] if hasMinSizeAnnotation && hasMaxSizeAnnotation { minSize, err := strconv.ParseInt(minSizeString, 10, 32) if err != nil { return 0, errors.Wrapf(err, "failed to caculate MachineSet replicas value: could not parse the value of the %q annotation", clusterv1.AutoscalerMinSizeAnnotation) } maxSize, err := strconv.ParseInt(maxSizeString, 10, 32) if err != nil { return 0, errors.Wrapf(err, "failed to caculate MachineSet replicas value: could not parse the value of the %q annotation", clusterv1.AutoscalerMaxSizeAnnotation) } // If it's a new MachineSet => Use the min size. // Note: This will result in a scale up to get into the range where autoscaler takes over. if oldMS == nil { if !dryRun { log.V(2).Info(fmt.Sprintf("Replica field has been defaulted to %d based on the %s annotation (MS is a new MS)", minSize, clusterv1.AutoscalerMinSizeAnnotation)) } return int32(minSize), nil } // Otherwise we are handing over the control for the replicas field for an existing MachineSet // to the autoscaler. switch { // If the old MachineSet doesn't have replicas set => Use the min size. // Note: As defaulting always sets the replica field, this case should not be possible // We only have this handling to be 100% safe against panics. case oldMS.Spec.Replicas == nil: if !dryRun { log.V(2).Info(fmt.Sprintf("Replica field has been defaulted to %d based on the %s annotation (old MS didn't have replicas set)", minSize, clusterv1.AutoscalerMinSizeAnnotation)) } return int32(minSize), nil // If the old MachineSet replicas are lower than min size => Use the min size. // Note: This will result in a scale up to get into the range where autoscaler takes over. case *oldMS.Spec.Replicas < int32(minSize): if !dryRun { log.V(2).Info(fmt.Sprintf("Replica field has been defaulted to %d based on the %s annotation (old MS had replicas below min size)", minSize, clusterv1.AutoscalerMinSizeAnnotation)) } return int32(minSize), nil // If the old MachineSet replicas are higher than max size => Use the max size. // Note: This will result in a scale down to get into the range where autoscaler takes over. case *oldMS.Spec.Replicas > int32(maxSize): if !dryRun { log.V(2).Info(fmt.Sprintf("Replica field has been defaulted to %d based on the %s annotation (old MS had replicas above max size)", maxSize, clusterv1.AutoscalerMaxSizeAnnotation)) } return int32(maxSize), nil // If the old MachineSet replicas are between min and max size => Keep the current value. default: if !dryRun { log.V(2).Info(fmt.Sprintf("Replica field has been defaulted to %d based on replicas of the old MachineSet (old MS had replicas within min size / max size range)", *oldMS.Spec.Replicas)) } return *oldMS.Spec.Replicas, nil } } // If neither the default nor the autoscaler annotations are set => Default to 1. if !dryRun { log.V(2).Info("Replica field has been defaulted to 1") } return 1, nil }
/* Copyright 2021 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package webhooks import ( "encoding/json" "fmt" "strconv" "strings" "text/template" "github.com/Masterminds/sprig/v3" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/feature" ) // validatePatches returns errors if the Patches in the ClusterClass violate any validation rules. func validatePatches(clusterClass *clusterv1.ClusterClass) field.ErrorList { var allErrs field.ErrorList names := sets.Set[string]{} for i, patch := range clusterClass.Spec.Patches { allErrs = append( allErrs, validatePatch(patch, names, clusterClass, field.NewPath("spec", "patches").Index(i))..., ) names.Insert(patch.Name) } return allErrs } func validatePatch(patch clusterv1.ClusterClassPatch, names sets.Set[string], clusterClass *clusterv1.ClusterClass, path *field.Path) field.ErrorList { var allErrs field.ErrorList allErrs = append(allErrs, validatePatchName(patch, names, path)..., ) allErrs = append(allErrs, validatePatchDefinitions(patch, clusterClass, path)..., ) return allErrs } func validatePatchName(patch clusterv1.ClusterClassPatch, names sets.Set[string], path *field.Path) field.ErrorList { var allErrs field.ErrorList if patch.Name == "" { allErrs = append(allErrs, field.Required( path.Child("name"), "patch name must be defined", ), ) } if patch.Name == clusterv1.VariableDefinitionFromInline { allErrs = append(allErrs, field.Required( path.Child("name"), fmt.Sprintf("%q can not be used as the name of a patch", clusterv1.VariableDefinitionFromInline), ), ) } if names.Has(patch.Name) { allErrs = append(allErrs, field.Invalid( path.Child("name"), patch.Name, fmt.Sprintf("patch names must be unique. Patch with name %q is defined more than once", patch.Name), ), ) } return allErrs } func validatePatchDefinitions(patch clusterv1.ClusterClassPatch, clusterClass *clusterv1.ClusterClass, path *field.Path) field.ErrorList { var allErrs field.ErrorList allErrs = append(allErrs, validateEnabledIf(patch.EnabledIf, path.Child("enabledIf"))...) if patch.Definitions == nil && patch.External == nil { allErrs = append(allErrs, field.Required( path, "one of definitions or external must be defined", )) } if patch.Definitions != nil && patch.External != nil { allErrs = append(allErrs, field.Invalid( path, patch, "only one of definitions or external can be defined", )) } if patch.Definitions != nil { for i, definition := range patch.Definitions { allErrs = append(allErrs, validateJSONPatches(definition.JSONPatches, clusterClass.Spec.Variables, path.Child("definitions").Index(i).Child("jsonPatches"))...) allErrs = append(allErrs, validateSelectors(definition.Selector, clusterClass, path.Child("definitions").Index(i).Child("selector"))...) } } if patch.External != nil { if !feature.Gates.Enabled(feature.RuntimeSDK) { allErrs = append(allErrs, field.Forbidden( path.Child("external"), "patch.external can be used only if the RuntimeSDK feature flag is enabled", )) } if patch.External.ValidateExtension == nil && patch.External.GenerateExtension == nil { allErrs = append(allErrs, field.Invalid( path.Child("external"), patch.External, "one of validateExtension and generateExtension must be defined", )) } } return allErrs } // validateSelectors validates if enabledIf is a valid template if it is set. func validateEnabledIf(enabledIf *string, path *field.Path) field.ErrorList { var allErrs field.ErrorList if enabledIf != nil { // Error if template can not be parsed. _, err := template.New("enabledIf").Funcs(sprig.HermeticTxtFuncMap()).Parse(*enabledIf) if err != nil { allErrs = append(allErrs, field.Invalid( path, *enabledIf, fmt.Sprintf("template can not be parsed: %v", err), )) } } return allErrs } // validateSelectors tests to see if the selector matches any template in the ClusterClass. // It returns nil as soon as it finds any matching template and an error if there is no match. func validateSelectors(selector clusterv1.PatchSelector, class *clusterv1.ClusterClass, path *field.Path) field.ErrorList { var allErrs field.ErrorList // Return an error if none of the possible selectors are enabled. if !(selector.MatchResources.InfrastructureCluster || selector.MatchResources.ControlPlane || (selector.MatchResources.MachineDeploymentClass != nil && len(selector.MatchResources.MachineDeploymentClass.Names) > 0) || (selector.MatchResources.MachinePoolClass != nil && len(selector.MatchResources.MachinePoolClass.Names) > 0)) { return append(allErrs, field.Invalid( path, prettyPrint(selector), "no selector enabled", )) } if selector.MatchResources.InfrastructureCluster { if !selectorMatchTemplate(selector, class.Spec.Infrastructure.Ref) { allErrs = append(allErrs, field.Invalid( path.Child("matchResources", "infrastructureCluster"), selector.MatchResources.InfrastructureCluster, "selector is enabled but does not match the infrastructure ref", )) } } if selector.MatchResources.ControlPlane { match := false if selectorMatchTemplate(selector, class.Spec.ControlPlane.Ref) { match = true } if class.Spec.ControlPlane.MachineInfrastructure != nil && selectorMatchTemplate(selector, class.Spec.ControlPlane.MachineInfrastructure.Ref) { match = true } if !match { allErrs = append(allErrs, field.Invalid( path.Child("matchResources", "controlPlane"), selector.MatchResources.ControlPlane, "selector is enabled but matches neither the controlPlane ref nor the controlPlane machineInfrastructure ref", )) } } if selector.MatchResources.MachineDeploymentClass != nil && len(selector.MatchResources.MachineDeploymentClass.Names) > 0 { for i, name := range selector.MatchResources.MachineDeploymentClass.Names { match := false err := validateSelectorName(name, path, "machineDeploymentClass", i) if err != nil { allErrs = append(allErrs, err) break } for _, md := range class.Spec.Workers.MachineDeployments { var matches bool if md.Class == name || name == "*" { matches = true } else if strings.HasPrefix(name, "*") && strings.HasSuffix(md.Class, strings.TrimPrefix(name, "*")) { matches = true } else if strings.HasSuffix(name, "*") && strings.HasPrefix(md.Class, strings.TrimSuffix(name, "*")) { matches = true } if matches { if selectorMatchTemplate(selector, md.Template.Infrastructure.Ref) || selectorMatchTemplate(selector, md.Template.Bootstrap.Ref) { match = true break } } } if !match { allErrs = append(allErrs, field.Invalid( path.Child("matchResources", "machineDeploymentClass", "names").Index(i), name, "selector is enabled but matches neither the bootstrap ref nor the infrastructure ref of a MachineDeployment class", )) } } } if selector.MatchResources.MachinePoolClass != nil && len(selector.MatchResources.MachinePoolClass.Names) > 0 { for i, name := range selector.MatchResources.MachinePoolClass.Names { match := false err := validateSelectorName(name, path, "machinePoolClass", i) if err != nil { allErrs = append(allErrs, err) break } for _, mp := range class.Spec.Workers.MachinePools { var matches bool if mp.Class == name || name == "*" { matches = true } else if strings.HasPrefix(name, "*") && strings.HasSuffix(mp.Class, strings.TrimPrefix(name, "*")) { matches = true } else if strings.HasSuffix(name, "*") && strings.HasPrefix(mp.Class, strings.TrimSuffix(name, "*")) { matches = true } if matches { if selectorMatchTemplate(selector, mp.Template.Infrastructure.Ref) || selectorMatchTemplate(selector, mp.Template.Bootstrap.Ref) { match = true break } } } if !match { allErrs = append(allErrs, field.Invalid( path.Child("matchResources", "machinePoolClass", "names").Index(i), name, "selector is enabled but matches neither the bootstrap ref nor the infrastructure ref of a MachinePool class", )) } } } return allErrs } // validateSelectorName validates if the selector name is valid. func validateSelectorName(name string, path *field.Path, resourceName string, index int) *field.Error { if strings.Contains(name, "*") { // selector can at most have a single * rune if strings.Count(name, "*") > 1 { return field.Invalid( path.Child("matchResources", resourceName, "names").Index(index), name, "selector can at most contain a single \"*\" rune") } // the * rune can appear only at the beginning, or ending of the selector. if strings.Contains(name, "*") && !(strings.HasPrefix(name, "*") || strings.HasSuffix(name, "*")) { // templateMDClass or templateMPClass can only have "*" rune at the start or end of the string return field.Invalid( path.Child("matchResources", resourceName, "names").Index(index), name, "\"*\" rune can only appear at the beginning, or ending of the selector") } // a valid selector without "*" should comply with Kubernetes naming standards. if validation.IsQualifiedName(strings.ReplaceAll(name, "*", "a")) != nil { return field.Invalid( path.Child("matchResources", resourceName, "names").Index(index), name, "selector does not comply with the Kubernetes naming standards") } } return nil } // selectorMatchTemplate returns true if APIVersion and Kind for the given selector match the reference. func selectorMatchTemplate(selector clusterv1.PatchSelector, reference *corev1.ObjectReference) bool { if reference == nil { return false } return selector.Kind == reference.Kind && selector.APIVersion == reference.APIVersion } var validOps = sets.Set[string]{}.Insert("add", "replace", "remove") func validateJSONPatches(jsonPatches []clusterv1.JSONPatch, variables []clusterv1.ClusterClassVariable, path *field.Path) field.ErrorList { var allErrs field.ErrorList variableSet, _ := getClusterClassVariablesMapWithReverseIndex(variables) for i, jsonPatch := range jsonPatches { if !validOps.Has(jsonPatch.Op) { allErrs = append(allErrs, field.NotSupported( path.Index(i).Child("op"), prettyPrint(jsonPatch), sets.List(validOps), )) } if !strings.HasPrefix(jsonPatch.Path, "/spec/") { allErrs = append(allErrs, field.Invalid( path.Index(i).Child("path"), prettyPrint(jsonPatch), "jsonPatch path must start with \"/spec/\"", )) } // Validate that array access is only prepend or append for add and not allowed for replace or remove. allErrs = append(allErrs, validateIndexAccess(jsonPatch, path.Index(i).Child("path"))..., ) // Validate the value and valueFrom fields for the patch. allErrs = append(allErrs, validateJSONPatchValues(jsonPatch, variableSet, path.Index(i))..., ) } return allErrs } func validateJSONPatchValues(jsonPatch clusterv1.JSONPatch, variableSet map[string]*clusterv1.ClusterClassVariable, path *field.Path) field.ErrorList { var allErrs field.ErrorList // move to the next variable if the jsonPatch does not have "replace" or "add" op. Additional validation is not needed. if jsonPatch.Op != "add" && jsonPatch.Op != "replace" { return allErrs } if jsonPatch.Value == nil && jsonPatch.ValueFrom == nil { allErrs = append(allErrs, field.Invalid( path, prettyPrint(jsonPatch), "jsonPatch must define one of value or valueFrom", )) } if jsonPatch.Value != nil && jsonPatch.ValueFrom != nil { allErrs = append(allErrs, field.Invalid( path, prettyPrint(jsonPatch), "jsonPatch can not define both value and valueFrom", )) } // Attempt to marshal the JSON to discover if it is valid. If jsonPatch.Value.Raw is set to nil skip this check // and accept the nil value. if jsonPatch.Value != nil && jsonPatch.Value.Raw != nil { var v interface{} if err := json.Unmarshal(jsonPatch.Value.Raw, &v); err != nil { allErrs = append(allErrs, field.Invalid( path.Child("value"), string(jsonPatch.Value.Raw), "jsonPatch Value is invalid JSON", )) } } if jsonPatch.ValueFrom != nil && jsonPatch.ValueFrom.Template == nil && jsonPatch.ValueFrom.Variable == nil { allErrs = append(allErrs, field.Invalid( path.Child("valueFrom"), prettyPrint(jsonPatch.ValueFrom), "valueFrom must set either template or variable", )) } if jsonPatch.ValueFrom != nil && jsonPatch.ValueFrom.Template != nil && jsonPatch.ValueFrom.Variable != nil { allErrs = append(allErrs, field.Invalid( path.Child("valueFrom"), prettyPrint(jsonPatch.ValueFrom), "valueFrom can not set both template and variable", )) } if jsonPatch.ValueFrom != nil && jsonPatch.ValueFrom.Template != nil { // Error if template can not be parsed. _, err := template.New("valueFrom.template").Funcs(sprig.HermeticTxtFuncMap()).Parse(*jsonPatch.ValueFrom.Template) if err != nil { allErrs = append(allErrs, field.Invalid( path.Child("valueFrom", "template"), *jsonPatch.ValueFrom.Template, fmt.Sprintf("template can not be parsed: %v", err), )) } } // If set validate that the variable is valid. if jsonPatch.ValueFrom != nil && jsonPatch.ValueFrom.Variable != nil { // If the variable is one of the list of builtin variables it's valid. if strings.HasPrefix(*jsonPatch.ValueFrom.Variable, "builtin.") { if _, ok := builtinVariables[*jsonPatch.ValueFrom.Variable]; !ok { allErrs = append(allErrs, field.Invalid( path.Child("valueFrom", "variable"), *jsonPatch.ValueFrom.Variable, "not a defined builtin variable", )) } } else { // Note: We're only validating if the variable name exists without // validating if the whole path is an existing variable. // This could be done by re-using getVariableValue of the json patch // generator but requires a refactoring first. variableName := getVariableName(*jsonPatch.ValueFrom.Variable) if _, ok := variableSet[variableName]; !ok { allErrs = append(allErrs, field.Invalid( path.Child("valueFrom", "variable"), *jsonPatch.ValueFrom.Variable, fmt.Sprintf("variable with name %s cannot be found", *jsonPatch.ValueFrom.Variable), )) } } } return allErrs } func getVariableName(variable string) string { return strings.FieldsFunc(variable, func(r rune) bool { return r == '[' || r == '.' })[0] } // This contains a list of all of the valid builtin variables. // TODO(killianmuldoon): Match this list to controllers/topology/internal/extensions/patches/variables as those structs become available across the code base i.e. public or top-level internal. var builtinVariables = sets.Set[string]{}.Insert( "builtin", // Cluster builtins. "builtin.cluster", "builtin.cluster.name", "builtin.cluster.namespace", "builtin.cluster.uid", // ClusterTopology builtins. "builtin.cluster.topology", "builtin.cluster.topology.class", "builtin.cluster.topology.version", // ClusterNetwork builtins "builtin.cluster.network", "builtin.cluster.network.serviceDomain", "builtin.cluster.network.services", "builtin.cluster.network.pods", "builtin.cluster.network.ipFamily", // ControlPlane builtins. "builtin.controlPlane", "builtin.controlPlane.name", "builtin.controlPlane.replicas", "builtin.controlPlane.version", "builtin.controlPlane.metadata.labels", "builtin.controlPlane.metadata.annotations", // ControlPlane ref builtins. "builtin.controlPlane.machineTemplate.infrastructureRef.name", // MachineDeployment builtins. "builtin.machineDeployment", "builtin.machineDeployment.class", "builtin.machineDeployment.name", "builtin.machineDeployment.replicas", "builtin.machineDeployment.topologyName", "builtin.machineDeployment.version", "builtin.machineDeployment.metadata.labels", "builtin.machineDeployment.metadata.annotations", // MachineDeployment ref builtins. "builtin.machineDeployment.bootstrap.configRef.name", "builtin.machineDeployment.infrastructureRef.name", // MachinePool builtins. "builtin.machinePool", "builtin.machinePool.class", "builtin.machinePool.name", "builtin.machinePool.replicas", "builtin.machinePool.topologyName", "builtin.machinePool.version", "builtin.machinePool.metadata.labels", "builtin.machinePool.metadata.annotations", // MachinePool ref builtins. "builtin.machinePool.bootstrap.configRef.name", "builtin.machinePool.infrastructureRef.name", ) // validateIndexAccess checks to see if the jsonPath is attempting to add an element in the array i.e. access by number // If the operation is add an error is thrown if a number greater than 0 is used as an index. // If the operation is replace an error is thrown if an index is used. func validateIndexAccess(jsonPatch clusterv1.JSONPatch, path *field.Path) field.ErrorList { var allErrs field.ErrorList pathParts := strings.Split(jsonPatch.Path, "/") for _, part := range pathParts { // Check if the path segment is a valid number. If an error is thrown continue to the next segment. index, err := strconv.Atoi(part) if err != nil { continue } // If the operation is add an error is thrown if a number greater than 0 is used as an index. if jsonPatch.Op == "add" && index != 0 { allErrs = append(allErrs, field.Invalid(path, jsonPatch.Path, "arrays can only be accessed using \"0\" (prepend) or \"-\" (append)", )) } // If the jsonPatch operation is replace or remove disallow any number as an element in the path. if jsonPatch.Op == "replace" || jsonPatch.Op == "remove" { allErrs = append(allErrs, field.Invalid(path, jsonPatch.Path, fmt.Sprintf("elements in arrays can not be accessed in a %s operation", jsonPatch.Op), )) } } return allErrs } func prettyPrint(v interface{}) string { b, err := json.MarshalIndent(v, "", " ") if err != nil { return errors.Wrapf(err, "failed to marshal field value").Error() } return string(b) }
/* Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package conditions import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ) // Getter interface defines methods that a Cluster API object should implement in order to // use the conditions package for getting conditions. type Getter interface { client.Object // GetConditions returns the list of conditions for a cluster API object. GetConditions() clusterv1.Conditions } // Get returns the condition with the given type, if the condition does not exist, // it returns nil. func Get(from Getter, t clusterv1.ConditionType) *clusterv1.Condition { conditions := from.GetConditions() if conditions == nil { return nil } for _, condition := range conditions { if condition.Type == t { return &condition } } return nil } // Has returns true if a condition with the given type exists. func Has(from Getter, t clusterv1.ConditionType) bool { return Get(from, t) != nil } // IsTrue is true if the condition with the given type is True, otherwise it returns false // if the condition is not True or if the condition does not exist (is nil). func IsTrue(from Getter, t clusterv1.ConditionType) bool { if c := Get(from, t); c != nil { return c.Status == corev1.ConditionTrue } return false } // IsFalse is true if the condition with the given type is False, otherwise it returns false // if the condition is not False or if the condition does not exist (is nil). func IsFalse(from Getter, t clusterv1.ConditionType) bool { if c := Get(from, t); c != nil { return c.Status == corev1.ConditionFalse } return false } // IsUnknown is true if the condition with the given type is Unknown or if the condition // does not exist (is nil). func IsUnknown(from Getter, t clusterv1.ConditionType) bool { if c := Get(from, t); c != nil { return c.Status == corev1.ConditionUnknown } return true } // GetReason returns a nil safe string of Reason for the condition with the given type. func GetReason(from Getter, t clusterv1.ConditionType) string { if c := Get(from, t); c != nil { return c.Reason } return "" } // GetMessage returns a nil safe string of Message. func GetMessage(from Getter, t clusterv1.ConditionType) string { if c := Get(from, t); c != nil { return c.Message } return "" } // GetSeverity returns the condition Severity or nil if the condition // does not exist (is nil). func GetSeverity(from Getter, t clusterv1.ConditionType) *clusterv1.ConditionSeverity { if c := Get(from, t); c != nil { return &c.Severity } return nil } // GetLastTransitionTime returns the condition Severity or nil if the condition // does not exist (is nil). func GetLastTransitionTime(from Getter, t clusterv1.ConditionType) *metav1.Time { if c := Get(from, t); c != nil { return &c.LastTransitionTime } return nil } // summary returns a Ready condition with the summary of all the conditions existing // on an object. If the object does not have other conditions, no summary condition is generated. // NOTE: The resulting Ready condition will have positive polarity; the conditions we are starting from might have positive or negative polarity. func summary(from Getter, options ...MergeOption) *clusterv1.Condition { conditions := from.GetConditions() mergeOpt := &mergeOptions{} for _, o := range options { o(mergeOpt) } // Identifies the conditions in scope for the Summary by taking all the existing conditions except Ready, // or, if a list of conditions types is specified, only the conditions the condition in that list. conditionsInScope := make([]localizedCondition, 0, len(conditions)) for i := range conditions { c := conditions[i] if c.Type == clusterv1.ReadyCondition { continue } if mergeOpt.conditionTypes != nil { found := false for _, t := range mergeOpt.conditionTypes { if c.Type == t { found = true break } } if !found { continue } } // Keep track of the polarity of the condition we are starting from. polarity := PositivePolarity for _, t := range mergeOpt.negativeConditionTypes { if c.Type == t { polarity = NegativePolarity break } } conditionsInScope = append(conditionsInScope, localizedCondition{ Condition: &c, Polarity: polarity, Getter: from, }) } // If it is required to add a step counter only if a subset of condition exists, check if the conditions // in scope are included in this subset or not. if mergeOpt.addStepCounterIfOnlyConditionTypes != nil { for _, c := range conditionsInScope { found := false for _, t := range mergeOpt.addStepCounterIfOnlyConditionTypes { if c.Type == t { found = true break } } if !found { mergeOpt.addStepCounter = false break } } } // If it is required to add a step counter, determine the total number of conditions defaulting // to the selected conditions or, if defined, to the total number of conditions type to be considered. if mergeOpt.addStepCounter { mergeOpt.stepCounter = len(conditionsInScope) if mergeOpt.conditionTypes != nil { mergeOpt.stepCounter = len(mergeOpt.conditionTypes) } if mergeOpt.addStepCounterIfOnlyConditionTypes != nil { mergeOpt.stepCounter = len(mergeOpt.addStepCounterIfOnlyConditionTypes) } } return merge(conditionsInScope, clusterv1.ReadyCondition, mergeOpt) } // mirrorOptions allows to set options for the mirror operation. type mirrorOptions struct { fallbackTo *bool fallbackReason string fallbackSeverity clusterv1.ConditionSeverity fallbackMessage string } // MirrorOptions defines an option for mirroring conditions. type MirrorOptions func(*mirrorOptions) // WithFallbackValue specify a fallback value to use in case the mirrored condition does not exist; // in case the fallbackValue is false, given values for reason, severity and message will be used. func WithFallbackValue(fallbackValue bool, reason string, severity clusterv1.ConditionSeverity, message string) MirrorOptions { return func(c *mirrorOptions) { c.fallbackTo = &fallbackValue c.fallbackReason = reason c.fallbackSeverity = severity c.fallbackMessage = message } } // mirror mirrors the Ready condition from a dependent object into the target condition; // if the Ready condition does not exist in the source object, no target conditions is generated. // NOTE: Considering that we are mirroring Ready conditions with positive polarity, also the resulting condition will have positive polarity. func mirror(from Getter, targetCondition clusterv1.ConditionType, options ...MirrorOptions) *clusterv1.Condition { mirrorOpt := &mirrorOptions{} for _, o := range options { o(mirrorOpt) } condition := Get(from, clusterv1.ReadyCondition) if mirrorOpt.fallbackTo != nil && condition == nil { switch *mirrorOpt.fallbackTo { case true: condition = TrueCondition(targetCondition) case false: condition = FalseCondition(targetCondition, mirrorOpt.fallbackReason, mirrorOpt.fallbackSeverity, mirrorOpt.fallbackMessage) } } if condition != nil { condition.Type = targetCondition } return condition } // Aggregates all the Ready condition from a list of dependent objects into the target object; // if the Ready condition does not exist in one of the source object, the object is excluded from // the aggregation; if none of the source object have ready condition, no target conditions is generated. // NOTE: Considering that we are aggregating Ready conditions with positive polarity, also the resulting condition will have positive polarity. func aggregate(from []Getter, targetCondition clusterv1.ConditionType, options ...MergeOption) *clusterv1.Condition { conditionsInScope := make([]localizedCondition, 0, len(from)) for i := range from { condition := Get(from[i], clusterv1.ReadyCondition) conditionsInScope = append(conditionsInScope, localizedCondition{ Condition: condition, Polarity: PositivePolarity, Getter: from[i], }) } mergeOpt := &mergeOptions{ addStepCounter: true, stepCounter: len(from), } for _, o := range options { o(mergeOpt) } return merge(conditionsInScope, targetCondition, mergeOpt) }
/* Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package conditions import ( "fmt" "github.com/onsi/gomega" "github.com/onsi/gomega/types" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ) // MatchConditions returns a custom matcher to check equality of clusterv1.Conditions. func MatchConditions(expected clusterv1.Conditions) types.GomegaMatcher { return &matchConditions{ expected: expected, } } type matchConditions struct { expected clusterv1.Conditions } func (m matchConditions) Match(actual interface{}) (success bool, err error) { elems := []interface{}{} for _, condition := range m.expected { elems = append(elems, MatchCondition(condition)) } return gomega.ConsistOf(elems...).Match(actual) } func (m matchConditions) FailureMessage(actual interface{}) (message string) { return fmt.Sprintf("expected\n\t%#v\nto match\n\t%#v\n", actual, m.expected) } func (m matchConditions) NegatedFailureMessage(actual interface{}) (message string) { return fmt.Sprintf("expected\n\t%#v\nto not match\n\t%#v\n", actual, m.expected) } // MatchCondition returns a custom matcher to check equality of clusterv1.Condition. func MatchCondition(expected clusterv1.Condition) types.GomegaMatcher { return &matchCondition{ expected: expected, } } type matchCondition struct { expected clusterv1.Condition } func (m matchCondition) Match(actual interface{}) (success bool, err error) { actualCondition, ok := actual.(clusterv1.Condition) if !ok { return false, fmt.Errorf("actual should be of type Condition") } ok, err = gomega.Equal(m.expected.Type).Match(actualCondition.Type) if !ok { return ok, err } ok, err = gomega.Equal(m.expected.Status).Match(actualCondition.Status) if !ok { return ok, err } ok, err = gomega.Equal(m.expected.Severity).Match(actualCondition.Severity) if !ok { return ok, err } ok, err = gomega.Equal(m.expected.Reason).Match(actualCondition.Reason) if !ok { return ok, err } ok, err = gomega.Equal(m.expected.Message).Match(actualCondition.Message) if !ok { return ok, err } return ok, err } func (m matchCondition) FailureMessage(actual interface{}) (message string) { return fmt.Sprintf("expected\n\t%#v\nto match\n\t%#v\n", actual, m.expected) } func (m matchCondition) NegatedFailureMessage(actual interface{}) (message string) { return fmt.Sprintf("expected\n\t%#v\nto not match\n\t%#v\n", actual, m.expected) }
/* Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package conditions import ( "errors" "github.com/onsi/gomega/format" "github.com/onsi/gomega/types" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ) // HaveSameStateOf matches a condition to have the same state of another. func HaveSameStateOf(expected *clusterv1.Condition) types.GomegaMatcher { return &conditionMatcher{ Expected: expected, } } type conditionMatcher struct { Expected *clusterv1.Condition } func (matcher *conditionMatcher) Match(actual interface{}) (success bool, err error) { actualCondition, ok := actual.(*clusterv1.Condition) if !ok { return false, errors.New("value should be a condition") } return HasSameState(actualCondition, matcher.Expected), nil } func (matcher *conditionMatcher) FailureMessage(actual interface{}) (message string) { return format.Message(actual, "to have the same state of", matcher.Expected) } func (matcher *conditionMatcher) NegatedFailureMessage(actual interface{}) (message string) { return format.Message(actual, "not to have the same state of", matcher.Expected) }
/* Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package conditions import ( "sort" corev1 "k8s.io/api/core/v1" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ) type conditionPolarity string const ( // PositivePolarity describe a condition with positive polarity (Status=True good). PositivePolarity conditionPolarity = "Positive" // NegativePolarity describe a condition with negative polarity (Status=False good). NegativePolarity conditionPolarity = "Negative" ) // localizedCondition defines a condition with the information of the object the conditions // was originated from. type localizedCondition struct { *clusterv1.Condition Polarity conditionPolarity Getter } // merge a list of condition into a single one. // This operation is designed to ensure visibility of the most relevant conditions for defining the // operational state of a component. E.g. If there is one error in the condition list, this one takes // priority over the other conditions and it is should be reflected in the target condition. // // More specifically: // 1. Conditions are grouped by status, severity // 2. The resulting condition groups are sorted according to the following priority: // - P0 - Status=False - PositivePolarity | Status=True - NegativePolarity, Severity=Error // - P1 - Status=False - PositivePolarity | Status=True - NegativePolarity, Severity=Warning // - P2 - Status=False - PositivePolarity | Status=True - NegativePolarity, Severity=Info // - P3 - Status=True - PositivePolarity | Status=False - NegativePolarity // - P4 - Status=Unknown // // 3. The group with highest priority is used to determine status, severity and other info of the target condition. // // Please note that the last operation includes also the task of computing the Reason and the Message for the target // condition; in order to complete such task some trade-off should be made, because there is no a golden rule // for summarizing many Reason/Message into single Reason/Message. // mergeOptions allows the user to adapt this process to the specific needs by exposing a set of merge strategies. // NOTE: Target condition will have positive polarity. func merge(conditions []localizedCondition, targetCondition clusterv1.ConditionType, options *mergeOptions) *clusterv1.Condition { g := getConditionGroups(conditions) if len(g) == 0 { return nil } if g.TopGroup().status == corev1.ConditionTrue { return TrueCondition(targetCondition) } targetReason := getReason(g, options) targetMessage := getMessage(g, options) if g.TopGroup().status == corev1.ConditionFalse { return FalseCondition(targetCondition, targetReason, g.TopGroup().severity, targetMessage) } return UnknownCondition(targetCondition, targetReason, targetMessage) } // getConditionGroups groups a list of conditions according to status, severity values. // Additionally, the resulting groups are sorted by mergePriority. func getConditionGroups(conditions []localizedCondition) conditionGroups { groups := conditionGroups{} for _, condition := range conditions { if condition.Condition == nil { continue } added := false // Identify the groupStatus the condition belongs to. // NOTE: status for the conditions with negative polarity is "negated" so it is possible // to merge conditions with different polarity (conditionGroup is always using positive polarity). groupStatus := condition.Status if condition.Polarity == NegativePolarity { switch groupStatus { case corev1.ConditionFalse: groupStatus = corev1.ConditionTrue case corev1.ConditionTrue: groupStatus = corev1.ConditionFalse case corev1.ConditionUnknown: groupStatus = corev1.ConditionUnknown } } for i := range groups { if groups[i].status == groupStatus && groups[i].severity == condition.Severity { groups[i].conditions = append(groups[i].conditions, condition) added = true break } } if !added { groups = append(groups, conditionGroup{ conditions: []localizedCondition{condition}, status: groupStatus, severity: condition.Severity, }) } } // sort groups by priority sort.Sort(groups) // sorts conditions in the TopGroup so we ensure predictable result for merge strategies. // condition are sorted using the same lexicographic order used by Set; in case two conditions // have the same type, condition are sorted using according to the alphabetical order of the source object name. if len(groups) > 0 { sort.Slice(groups[0].conditions, func(i, j int) bool { a := groups[0].conditions[i] b := groups[0].conditions[j] if a.Type != b.Type { return lexicographicLess(a.Condition, b.Condition) } return a.GetName() < b.GetName() }) } return groups } // conditionGroups provides supports for grouping a list of conditions to be // merged into a single condition. ConditionGroups can be sorted by mergePriority. type conditionGroups []conditionGroup func (g conditionGroups) Len() int { return len(g) } func (g conditionGroups) Less(i, j int) bool { return g[i].mergePriority() < g[j].mergePriority() } func (g conditionGroups) Swap(i, j int) { g[i], g[j] = g[j], g[i] } // TopGroup returns the condition group with the highest mergePriority. func (g conditionGroups) TopGroup() *conditionGroup { if len(g) == 0 { return nil } return &g[0] } // TrueGroup returns the condition group with status True, if any. // Note: conditionGroup is always using positive polarity; the conditions in the group might have positive or negative polarity. func (g conditionGroups) TrueGroup() *conditionGroup { return g.getByStatusAndSeverity(corev1.ConditionTrue, clusterv1.ConditionSeverityNone) } // ErrorGroup returns the condition group with status False and severity Error, if any. // Note: conditionGroup is always using positive polarity; the conditions in the group might have positive or negative polarity. func (g conditionGroups) ErrorGroup() *conditionGroup { return g.getByStatusAndSeverity(corev1.ConditionFalse, clusterv1.ConditionSeverityError) } // WarningGroup returns the condition group with status False and severity Warning, if any. // Note: conditionGroup is always using positive polarity; the conditions in the group might have positive or negative polarity. func (g conditionGroups) WarningGroup() *conditionGroup { return g.getByStatusAndSeverity(corev1.ConditionFalse, clusterv1.ConditionSeverityWarning) } func (g conditionGroups) getByStatusAndSeverity(status corev1.ConditionStatus, severity clusterv1.ConditionSeverity) *conditionGroup { if len(g) == 0 { return nil } for _, group := range g { if group.status == status && group.severity == severity { return &group } } return nil } // conditionGroup define a group of conditions with the same status and severity, // and thus with the same priority when merging into a Ready condition. // Note: conditionGroup is always using positive polarity; the conditions in the group might have positive or negative polarity. type conditionGroup struct { status corev1.ConditionStatus severity clusterv1.ConditionSeverity conditions []localizedCondition } // mergePriority provides a priority value for the status and severity tuple that identifies this // condition group. The mergePriority value allows an easier sorting of conditions groups. // Note: conditionGroup is always using positive polarity. func (g conditionGroup) mergePriority() int { switch g.status { case corev1.ConditionFalse: switch g.severity { case clusterv1.ConditionSeverityError: return 0 case clusterv1.ConditionSeverityWarning: return 1 case clusterv1.ConditionSeverityInfo: return 2 } case corev1.ConditionTrue: return 3 case corev1.ConditionUnknown: return 4 } // this should never happen return 99 }
/* Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package conditions import ( "fmt" "strings" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ) // mergeOptions allows to set strategies for merging a set of conditions into a single condition, // and more specifically for computing the target Reason and the target Message. type mergeOptions struct { conditionTypes []clusterv1.ConditionType negativeConditionTypes []clusterv1.ConditionType addSourceRef bool addStepCounter bool addStepCounterIfOnlyConditionTypes []clusterv1.ConditionType stepCounter int } // MergeOption defines an option for computing a summary of conditions. type MergeOption func(*mergeOptions) // WithConditions instructs merge about the condition types to consider when doing a merge operation; // if this option is not specified, all the conditions (excepts Ready) will be considered. This is required, // so we can provide some guarantees about the semantic of the target condition without worrying about // side effects if someone or something adds custom conditions to the objects. // // NOTE: The order of conditions types defines the priority for determining the Reason and Message for the // target condition. // IMPORTANT: This options works only while generating the Summary condition. func WithConditions(t ...clusterv1.ConditionType) MergeOption { return func(c *mergeOptions) { c.conditionTypes = t } } // WithNegativePolarityConditions instruct merge about which conditions should be considered having negative polarity. // IMPORTANT: this must be a subset of WithConditions. func WithNegativePolarityConditions(t ...clusterv1.ConditionType) MergeOption { return func(c *mergeOptions) { c.negativeConditionTypes = t } } // WithStepCounter instructs merge to add a "x of y completed" string to the message, // where x is the number of conditions with Status=true and y is the number of conditions in scope. func WithStepCounter() MergeOption { return func(c *mergeOptions) { c.addStepCounter = true } } // WithStepCounterIf adds a step counter if the value is true. // This can be used e.g. to add a step counter only if the object is not being deleted. // // IMPORTANT: This options works only while generating the Summary condition. func WithStepCounterIf(value bool) MergeOption { return func(c *mergeOptions) { c.addStepCounter = value } } // WithStepCounterIfOnly ensure a step counter is show only if a subset of condition exists. // This applies for example on Machines, where we want to use // the step counter notation while provisioning the machine, but then we want to move away from this notation // as soon as the machine is provisioned and e.g. a Machine health check condition is generated // // IMPORTANT: This options requires WithStepCounter or WithStepCounterIf to be set. // IMPORTANT: This options works only while generating the Summary condition. func WithStepCounterIfOnly(t ...clusterv1.ConditionType) MergeOption { return func(c *mergeOptions) { c.addStepCounterIfOnlyConditionTypes = t } } // AddSourceRef instructs merge to add info about the originating object to the target Reason. func AddSourceRef() MergeOption { return func(c *mergeOptions) { c.addSourceRef = true } } // getReason returns the reason to be applied to the condition resulting by merging a set of condition groups. // The reason is computed according to the given mergeOptions. func getReason(groups conditionGroups, options *mergeOptions) string { return getFirstReason(groups, options.conditionTypes, options.addSourceRef) } // getFirstReason returns the first reason from the ordered list of conditions in the top group. // If required, the reason gets localized with the source object reference. func getFirstReason(g conditionGroups, order []clusterv1.ConditionType, addSourceRef bool) string { if condition := getFirstCondition(g, order); condition != nil { reason := condition.Reason if addSourceRef { return localizeReason(reason, condition.Getter) } return reason } return "" } // localizeReason adds info about the originating object to the target Reason. func localizeReason(reason string, from Getter) string { if strings.Contains(reason, "@") { return reason } return fmt.Sprintf("%s @ %s/%s", reason, from.GetObjectKind().GroupVersionKind().Kind, from.GetName()) } // getMessage returns the message to be applied to the condition resulting by merging a set of condition groups. // The message is computed according to the given mergeOptions, but in case of errors or warning a // summary of existing errors is automatically added. func getMessage(groups conditionGroups, options *mergeOptions) string { if options.addStepCounter { return getStepCounterMessage(groups, options.stepCounter) } return getFirstMessage(groups, options.conditionTypes) } // getStepCounterMessage returns a message "x of y completed", where x is the number of conditions // with Status=true and y is the number passed to this method. func getStepCounterMessage(groups conditionGroups, to int) string { ct := 0 if trueGroup := groups.TrueGroup(); trueGroup != nil { ct = len(trueGroup.conditions) } return fmt.Sprintf("%d of %d completed", ct, to) } // getFirstMessage returns the message from the ordered list of conditions in the top group. func getFirstMessage(groups conditionGroups, order []clusterv1.ConditionType) string { if condition := getFirstCondition(groups, order); condition != nil { return condition.Message } return "" } // getFirstCondition returns a first condition from the ordered list of conditions in the top group. func getFirstCondition(g conditionGroups, priority []clusterv1.ConditionType) *localizedCondition { topGroup := g.TopGroup() if topGroup == nil { return nil } switch len(topGroup.conditions) { case 0: return nil case 1: return &topGroup.conditions[0] default: for _, p := range priority { for _, c := range topGroup.conditions { if c.Type == p { return &c } } } return &topGroup.conditions[0] } }
/* Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package conditions import ( "reflect" "github.com/google/go-cmp/cmp" "github.com/pkg/errors" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/util" ) // Patch defines a list of operations to change a list of conditions into another. type Patch []PatchOperation // PatchOperation define an operation that changes a single condition. type PatchOperation struct { Before *clusterv1.Condition After *clusterv1.Condition Op PatchOperationType } // PatchOperationType defines patch operation types. type PatchOperationType string const ( // AddConditionPatch defines an add condition patch operation. AddConditionPatch PatchOperationType = "Add" // ChangeConditionPatch defines an change condition patch operation. ChangeConditionPatch PatchOperationType = "Change" // RemoveConditionPatch defines a remove condition patch operation. RemoveConditionPatch PatchOperationType = "Remove" ) // NewPatch returns the Patch required to align source conditions to after conditions. func NewPatch(before Getter, after Getter) (Patch, error) { var patch Patch if util.IsNil(before) { return nil, errors.New("error creating patch: before object is nil") } if util.IsNil(after) { return nil, errors.New("error creating patch: after object is nil") } // Identify AddCondition and ModifyCondition changes. targetConditions := after.GetConditions() for i := range targetConditions { targetCondition := targetConditions[i] currentCondition := Get(before, targetCondition.Type) if currentCondition == nil { patch = append(patch, PatchOperation{Op: AddConditionPatch, After: &targetCondition}) continue } if !reflect.DeepEqual(&targetCondition, currentCondition) { patch = append(patch, PatchOperation{Op: ChangeConditionPatch, After: &targetCondition, Before: currentCondition}) } } // Identify RemoveCondition changes. baseConditions := before.GetConditions() for i := range baseConditions { baseCondition := baseConditions[i] targetCondition := Get(after, baseCondition.Type) if targetCondition == nil { patch = append(patch, PatchOperation{Op: RemoveConditionPatch, Before: &baseCondition}) } } return patch, nil } // applyOptions allows to set strategies for patch apply. type applyOptions struct { ownedConditions []clusterv1.ConditionType forceOverwrite bool } func (o *applyOptions) isOwnedCondition(t clusterv1.ConditionType) bool { for _, i := range o.ownedConditions { if i == t { return true } } return false } // ApplyOption defines an option for applying a condition patch. type ApplyOption func(*applyOptions) // WithOwnedConditions allows to define condition types owned by the controller. // In case of conflicts for the owned conditions, the patch helper will always use the value provided by the controller. func WithOwnedConditions(t ...clusterv1.ConditionType) ApplyOption { return func(c *applyOptions) { c.ownedConditions = t } } // WithForceOverwrite In case of conflicts for the owned conditions, the patch helper will always use the value provided by the controller. func WithForceOverwrite(v bool) ApplyOption { return func(c *applyOptions) { c.forceOverwrite = v } } // Apply executes a three-way merge of a list of Patch. // When merge conflicts are detected (latest deviated from before in an incompatible way), an error is returned. func (p Patch) Apply(latest Setter, options ...ApplyOption) error { if p.IsZero() { return nil } if util.IsNil(latest) { return errors.New("error patching conditions: latest object was nil") } applyOpt := &applyOptions{} for _, o := range options { if util.IsNil(o) { return errors.New("error patching conditions: ApplyOption was nil") } o(applyOpt) } for _, conditionPatch := range p { switch conditionPatch.Op { case AddConditionPatch: // If the conditions is owned, always keep the after value. if applyOpt.forceOverwrite || applyOpt.isOwnedCondition(conditionPatch.After.Type) { Set(latest, conditionPatch.After) continue } // If the condition is already on latest, check if latest and after agree on the change; if not, this is a conflict. if latestCondition := Get(latest, conditionPatch.After.Type); latestCondition != nil { // If latest and after agree on the change, then it is a conflict. if !HasSameState(latestCondition, conditionPatch.After) { return errors.Errorf("error patching conditions: The condition %q was modified by a different process and this caused a merge/AddCondition conflict: %v", conditionPatch.After.Type, cmp.Diff(latestCondition, conditionPatch.After)) } // otherwise, the latest is already as intended. // NOTE: We are preserving LastTransitionTime from the latest in order to avoid altering the existing value. continue } // If the condition does not exists on the latest, add the new after condition. Set(latest, conditionPatch.After) case ChangeConditionPatch: // If the conditions is owned, always keep the after value. if applyOpt.forceOverwrite || applyOpt.isOwnedCondition(conditionPatch.After.Type) { Set(latest, conditionPatch.After) continue } latestCondition := Get(latest, conditionPatch.After.Type) // If the condition does not exist anymore on the latest, this is a conflict. if latestCondition == nil { return errors.Errorf("error patching conditions: The condition %q was deleted by a different process and this caused a merge/ChangeCondition conflict", conditionPatch.After.Type) } // If the condition on the latest is different from the base condition, check if // the after state corresponds to the desired value. If not this is a conflict (unless we should ignore conflicts for this condition type). if !reflect.DeepEqual(latestCondition, conditionPatch.Before) { if !HasSameState(latestCondition, conditionPatch.After) { return errors.Errorf("error patching conditions: The condition %q was modified by a different process and this caused a merge/ChangeCondition conflict: %v", conditionPatch.After.Type, cmp.Diff(latestCondition, conditionPatch.After)) } // Otherwise the latest is already as intended. // NOTE: We are preserving LastTransitionTime from the latest in order to avoid altering the existing value. continue } // Otherwise apply the new after condition. Set(latest, conditionPatch.After) case RemoveConditionPatch: // If the conditions is owned, always keep the after value (condition should be deleted). if applyOpt.forceOverwrite || applyOpt.isOwnedCondition(conditionPatch.Before.Type) { Delete(latest, conditionPatch.Before.Type) continue } // If the condition is still on the latest, check if it is changed in the meantime; // if so then this is a conflict. if latestCondition := Get(latest, conditionPatch.Before.Type); latestCondition != nil { if !HasSameState(latestCondition, conditionPatch.Before) { return errors.Errorf("error patching conditions: The condition %q was modified by a different process and this caused a merge/RemoveCondition conflict: %v", conditionPatch.Before.Type, cmp.Diff(latestCondition, conditionPatch.Before)) } } // Otherwise the latest and after agreed on the delete operation, so there's nothing to change. Delete(latest, conditionPatch.Before.Type) } } return nil } // IsZero returns true if the patch is nil or has no changes. func (p Patch) IsZero() bool { if p == nil { return true } return len(p) == 0 }
/* Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package conditions import ( "fmt" "sort" "time" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ) // Setter interface defines methods that a Cluster API object should implement in order to // use the conditions package for setting conditions. type Setter interface { Getter SetConditions(clusterv1.Conditions) } // Set sets the given condition. // // NOTE: If a condition already exists, the LastTransitionTime is updated only if a change is detected // in any of the following fields: Status, Reason, Severity and Message. func Set(to Setter, condition *clusterv1.Condition) { if to == nil || condition == nil { return } // Check if the new conditions already exists, and change it only if there is a status // transition (otherwise we should preserve the current last transition time)- conditions := to.GetConditions() exists := false for i := range conditions { existingCondition := conditions[i] if existingCondition.Type == condition.Type { exists = true if !HasSameState(&existingCondition, condition) { condition.LastTransitionTime = metav1.NewTime(time.Now().UTC().Truncate(time.Second)) conditions[i] = *condition break } condition.LastTransitionTime = existingCondition.LastTransitionTime break } } // If the condition does not exist, add it, setting the transition time only if not already set if !exists { if condition.LastTransitionTime.IsZero() { condition.LastTransitionTime = metav1.NewTime(time.Now().UTC().Truncate(time.Second)) } conditions = append(conditions, *condition) } // Sorts conditions for convenience of the consumer, i.e. kubectl. sort.Slice(conditions, func(i, j int) bool { return lexicographicLess(&conditions[i], &conditions[j]) }) to.SetConditions(conditions) } // TrueCondition returns a condition with Status=True and the given type. func TrueCondition(t clusterv1.ConditionType) *clusterv1.Condition { return &clusterv1.Condition{ Type: t, Status: corev1.ConditionTrue, } } // TrueConditionWithNegativePolarity returns a condition with negative polarity, Status=True and the given type (Status=True has a negative meaning). func TrueConditionWithNegativePolarity(t clusterv1.ConditionType, reason string, severity clusterv1.ConditionSeverity, messageFormat string, messageArgs ...interface{}) *clusterv1.Condition { return &clusterv1.Condition{ Type: t, Status: corev1.ConditionTrue, Reason: reason, Severity: severity, Message: fmt.Sprintf(messageFormat, messageArgs...), } } // FalseCondition returns a condition with Status=False and the given type. func FalseCondition(t clusterv1.ConditionType, reason string, severity clusterv1.ConditionSeverity, messageFormat string, messageArgs ...interface{}) *clusterv1.Condition { return &clusterv1.Condition{ Type: t, Status: corev1.ConditionFalse, Reason: reason, Severity: severity, Message: fmt.Sprintf(messageFormat, messageArgs...), } } // FalseConditionWithNegativePolarity returns a condition with negative polarity, Status=false and the given type (Status=False has a positive meaning). func FalseConditionWithNegativePolarity(t clusterv1.ConditionType) *clusterv1.Condition { return &clusterv1.Condition{ Type: t, Status: corev1.ConditionFalse, } } // UnknownCondition returns a condition with Status=Unknown and the given type. func UnknownCondition(t clusterv1.ConditionType, reason string, messageFormat string, messageArgs ...interface{}) *clusterv1.Condition { return &clusterv1.Condition{ Type: t, Status: corev1.ConditionUnknown, Reason: reason, Message: fmt.Sprintf(messageFormat, messageArgs...), } } // MarkTrue sets Status=True for the condition with the given type. func MarkTrue(to Setter, t clusterv1.ConditionType) { Set(to, TrueCondition(t)) } // MarkTrueWithNegativePolarity sets Status=True for a condition with negative polarity and the given type (Status=True has a negative meaning). func MarkTrueWithNegativePolarity(to Setter, t clusterv1.ConditionType, reason string, severity clusterv1.ConditionSeverity, messageFormat string, messageArgs ...interface{}) { Set(to, TrueConditionWithNegativePolarity(t, reason, severity, messageFormat, messageArgs...)) } // MarkUnknown sets Status=Unknown for the condition with the given type. func MarkUnknown(to Setter, t clusterv1.ConditionType, reason, messageFormat string, messageArgs ...interface{}) { Set(to, UnknownCondition(t, reason, messageFormat, messageArgs...)) } // MarkFalse sets Status=False for the condition with the given type. func MarkFalse(to Setter, t clusterv1.ConditionType, reason string, severity clusterv1.ConditionSeverity, messageFormat string, messageArgs ...interface{}) { Set(to, FalseCondition(t, reason, severity, messageFormat, messageArgs...)) } // MarkFalseWithNegativePolarity sets Status=False for a condition with negative polarity and the given type (Status=False has a positive meaning). func MarkFalseWithNegativePolarity(to Setter, t clusterv1.ConditionType) { Set(to, FalseConditionWithNegativePolarity(t)) } // SetSummary sets a Ready condition with the summary of all the conditions existing // on an object. If the object does not have other conditions, no summary condition is generated. func SetSummary(to Setter, options ...MergeOption) { Set(to, summary(to, options...)) } // SetMirror creates a new condition by mirroring the Ready condition from a dependent object; // if the Ready condition does not exist in the source object, no target conditions is generated. func SetMirror(to Setter, targetCondition clusterv1.ConditionType, from Getter, options ...MirrorOptions) { Set(to, mirror(from, targetCondition, options...)) } // SetAggregate creates a new condition with the aggregation of all the Ready condition // from a list of dependent objects; if the Ready condition does not exist in one of the source object, // the object is excluded from the aggregation; if none of the source object have ready condition, // no target conditions is generated. func SetAggregate(to Setter, targetCondition clusterv1.ConditionType, from []Getter, options ...MergeOption) { Set(to, aggregate(from, targetCondition, options...)) } // Delete deletes the condition with the given type. func Delete(to Setter, t clusterv1.ConditionType) { if to == nil { return } conditions := to.GetConditions() newConditions := make(clusterv1.Conditions, 0, len(conditions)) for _, condition := range conditions { if condition.Type != t { newConditions = append(newConditions, condition) } } to.SetConditions(newConditions) } // lexicographicLess returns true if a condition is less than another in regard to // the order of conditions designed for convenience of the consumer, i.e. kubectl. // According to this order the Ready condition always goes first, followed by all the other // conditions sorted by Type. func lexicographicLess(i, j *clusterv1.Condition) bool { if i == nil { return true } if j == nil { return false } return (i.Type == clusterv1.ReadyCondition || i.Type < j.Type) && j.Type != clusterv1.ReadyCondition } // HasSameState returns true if a condition has the same state of another; state is defined // by the union of following fields: Type, Status, Reason, Severity and Message (it excludes LastTransitionTime). func HasSameState(i, j *clusterv1.Condition) bool { if i == nil || j == nil { return i == j } return i.Type == j.Type && i.Status == j.Status && i.Reason == j.Reason && i.Severity == j.Severity && i.Message == j.Message }
/* Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package conditions import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/log" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/util" ) // UnstructuredGetter return a Getter object that can read conditions from an Unstructured object. // Important. This method should be used only with types implementing Cluster API conditions. func UnstructuredGetter(u *unstructured.Unstructured) Getter { return &unstructuredWrapper{Unstructured: u} } // UnstructuredSetter return a Setter object that can set conditions from an Unstructured object. // Important. This method should be used only with types implementing Cluster API conditions. func UnstructuredSetter(u *unstructured.Unstructured) Setter { return &unstructuredWrapper{Unstructured: u} } type unstructuredWrapper struct { *unstructured.Unstructured } // GetConditions returns the list of conditions from an Unstructured object. // // NOTE: Due to the constraints of JSON-unmarshal, this operation is to be considered best effort. // In more details: // - Errors during JSON-unmarshal are ignored and a empty collection list is returned. // - It's not possible to detect if the object has an empty condition list or if it does not implement conditions; // in both cases the operation returns an empty slice is returned. // - If the object doesn't implement conditions on under status as defined in Cluster API, // JSON-unmarshal matches incoming object keys to the keys; this can lead to conditions values partially set. func (c *unstructuredWrapper) GetConditions() clusterv1.Conditions { conditions := clusterv1.Conditions{} if err := util.UnstructuredUnmarshalField(c.Unstructured, &conditions, "status", "conditions"); err != nil { return nil } return conditions } // SetConditions set the conditions into an Unstructured object. // // NOTE: Due to the constraints of JSON-unmarshal, this operation is to be considered best effort. // In more details: // - Errors during JSON-unmarshal are ignored and a empty collection list is returned. // - It's not possible to detect if the object has an empty condition list or if it does not implement conditions; // in both cases the operation returns an empty slice is returned. func (c *unstructuredWrapper) SetConditions(conditions clusterv1.Conditions) { v := make([]interface{}, 0, len(conditions)) for i := range conditions { m, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&conditions[i]) if err != nil { log.Log.Error(err, "Failed to convert Condition to unstructured map. This error shouldn't have occurred, please file an issue.", "groupVersionKind", c.GroupVersionKind(), "name", c.GetName(), "namespace", c.GetNamespace()) continue } v = append(v, m) } // unstructured.SetNestedField returns an error only if value cannot be set because one of // the nesting levels is not a map[string]interface{}; this is not the case so the error should never happen here. err := unstructured.SetNestedField(c.Unstructured.Object, v, "status", "conditions") if err != nil { log.Log.Error(err, "Failed to set Conditions on unstructured object. This error shouldn't have occurred, please file an issue.", "groupVersionKind", c.GroupVersionKind(), "name", c.GetName(), "namespace", c.GetNamespace()) } }
/* Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // Package container implements container utility functionality. package container import ( _ "crypto/sha256" // Import the crypto/sha256 algorithm for the docker image parser to work with sha256 hashes. _ "crypto/sha512" // Import the crypto/sha512 algorithm for the docker image parser to work with 384 and 512 sha hashes. "fmt" "path" "regexp" "github.com/distribution/reference" "github.com/pkg/errors" ) var ( ociTagAllowedChars = regexp.MustCompile(`[^-a-zA-Z0-9_\.]`) ) // Image type represents the container image details. type Image struct { Repository string Name string Tag string Digest string } // ImageFromString parses a docker image string into three parts: repo, tag and digest. func ImageFromString(image string) (Image, error) { named, err := reference.ParseNamed(image) if err != nil { return Image{}, fmt.Errorf("couldn't parse image name: %v", err) } var repo, tag, digest string _, nameOnly := path.Split(reference.Path(named)) if nameOnly != "" { // split out the part of the name after the last / lenOfCompleteName := len(named.Name()) repo = named.Name()[:lenOfCompleteName-len(nameOnly)-1] } tagged, ok := named.(reference.Tagged) if ok { tag = tagged.Tag() } digested, ok := named.(reference.Digested) if ok { digest = digested.Digest().String() } return Image{Repository: repo, Name: nameOnly, Tag: tag, Digest: digest}, nil } func (i Image) String() string { // repo/name [ ":" tag ] [ "@" digest ] ref := fmt.Sprintf("%s/%s", i.Repository, i.Name) if i.Tag != "" { ref = fmt.Sprintf("%s:%s", ref, i.Tag) } if i.Digest != "" { ref = fmt.Sprintf("%s@%s", ref, i.Digest) } return ref } // ModifyImageRepository takes an imageName (e.g., repository/image:tag), and returns an image name with updated repository. func ModifyImageRepository(imageName, repositoryName string) (string, error) { image, err := ImageFromString(imageName) if err != nil { return "", errors.Wrap(err, "failed to parse image name") } nameUpdated, err := reference.WithName(path.Join(repositoryName, image.Name)) if err != nil { return "", errors.Wrap(err, "failed to update repository name") } if image.Tag != "" { retagged, err := reference.WithTag(nameUpdated, image.Tag) if err != nil { return "", errors.Wrap(err, "failed to parse image tag") } return reference.FamiliarString(retagged), nil } return "", errors.New("image must be tagged") } // ModifyImageTag takes an imageName (e.g., repository/image:tag), and returns an image name with updated tag. func ModifyImageTag(imageName, tagName string) (string, error) { normalisedTagName := SemverToOCIImageTag(tagName) namedRef, err := reference.ParseNormalizedNamed(imageName) if err != nil { return "", errors.Wrap(err, "failed to parse image name") } // return error if images use digest as version instead of tag if _, isCanonical := namedRef.(reference.Canonical); isCanonical { return "", errors.New("image uses digest as version, cannot update tag ") } // update the image tag with tagName namedTagged, err := reference.WithTag(namedRef, normalisedTagName) if err != nil { return "", errors.Wrap(err, "failed to update image tag") } return reference.TagNameOnly(namedTagged).String(), nil } // ImageTagIsValid ensures that a given image tag is compliant with the OCI spec. func ImageTagIsValid(tagName string) bool { return !ociTagAllowedChars.MatchString(tagName) } // SemverToOCIImageTag is a helper function that replaces all // non-allowed symbols in tag strings with underscores. // Image tag can only contain lowercase and uppercase letters, digits, // underscores, periods and dashes. // Current usage is for CI images where all of symbols except '+' are valid, // but function is for generic usage where input can't be always pre-validated. // Taken from k8s.io/cmd/kubeadm/app/util. func SemverToOCIImageTag(version string) string { return ociTagAllowedChars.ReplaceAllString(version, "_") }
/* Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // Package yaml implements yaml utility functions. package yaml import ( "bufio" "bytes" "io" "os" "strings" "github.com/MakeNowJust/heredoc" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer/streaming" apiyaml "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/yaml" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/util" ) // ExtractClusterReferences returns the references in a Cluster object. func ExtractClusterReferences(out *ParseOutput, c *clusterv1.Cluster) (res []*unstructured.Unstructured) { if c.Spec.InfrastructureRef == nil { return nil } if obj := out.FindUnstructuredReference(c.Spec.InfrastructureRef); obj != nil { res = append(res, obj) } return } // ExtractMachineReferences returns the references in a Machine object. func ExtractMachineReferences(out *ParseOutput, m *clusterv1.Machine) (res []*unstructured.Unstructured) { if obj := out.FindUnstructuredReference(&m.Spec.InfrastructureRef); obj != nil { res = append(res, obj) } if m.Spec.Bootstrap.ConfigRef != nil { if obj := out.FindUnstructuredReference(m.Spec.Bootstrap.ConfigRef); obj != nil { res = append(res, obj) } } return } // ParseOutput is the output given from the Parse function. type ParseOutput struct { Clusters []*clusterv1.Cluster Machines []*clusterv1.Machine MachineSets []*clusterv1.MachineSet MachineDeployments []*clusterv1.MachineDeployment UnstructuredObjects []*unstructured.Unstructured } // Add adds the other ParseOutput slices to this instance. func (p *ParseOutput) Add(o *ParseOutput) *ParseOutput { p.Clusters = append(p.Clusters, o.Clusters...) p.Machines = append(p.Machines, o.Machines...) p.MachineSets = append(p.MachineSets, o.MachineSets...) p.MachineDeployments = append(p.MachineDeployments, o.MachineDeployments...) p.UnstructuredObjects = append(p.UnstructuredObjects, o.UnstructuredObjects...) return p } // FindUnstructuredReference takes in an ObjectReference and tries to find an Unstructured object. func (p *ParseOutput) FindUnstructuredReference(ref *corev1.ObjectReference) *unstructured.Unstructured { for _, obj := range p.UnstructuredObjects { if obj.GroupVersionKind() == ref.GroupVersionKind() && ref.Namespace == obj.GetNamespace() && ref.Name == obj.GetName() { return obj } } return nil } // ParseInput is an input struct for the Parse function. type ParseInput struct { File string } // Parse extracts runtime objects from a file. func Parse(input ParseInput) (*ParseOutput, error) { output := &ParseOutput{} // Open the input file. reader, err := os.Open(input.File) if err != nil { return nil, err } // Create a new decoder. decoder := NewYAMLDecoder(reader) defer decoder.Close() for { u := &unstructured.Unstructured{} _, gvk, err := decoder.Decode(nil, u) if errors.Is(err, io.EOF) { break } if runtime.IsNotRegisteredError(err) { continue } if err != nil { return nil, err } switch gvk.Kind { case "Cluster": obj := &clusterv1.Cluster{} if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil { return nil, errors.Wrapf(err, "cannot convert object to %s", gvk.Kind) } output.Clusters = append(output.Clusters, obj) case "Machine": obj := &clusterv1.Machine{} if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil { return nil, errors.Wrapf(err, "cannot convert object to %s", gvk.Kind) } output.Machines = append(output.Machines, obj) case "MachineSet": obj := &clusterv1.MachineSet{} if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil { return nil, errors.Wrapf(err, "cannot convert object to %s", gvk.Kind) } output.MachineSets = append(output.MachineSets, obj) case "MachineDeployment": obj := &clusterv1.MachineDeployment{} if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil { return nil, errors.Wrapf(err, "cannot convert object to %s", gvk.Kind) } output.MachineDeployments = append(output.MachineDeployments, obj) default: output.UnstructuredObjects = append(output.UnstructuredObjects, u) } } return output, nil } type yamlDecoder struct { reader *apiyaml.YAMLReader decoder runtime.Decoder close func() error } func (d *yamlDecoder) Decode(defaults *schema.GroupVersionKind, into runtime.Object) (runtime.Object, *schema.GroupVersionKind, error) { for { doc, err := d.reader.Read() if err != nil { return nil, nil, err } // Skip over empty documents, i.e. a leading `---` if len(bytes.TrimSpace(doc)) == 0 { continue } return d.decoder.Decode(doc, defaults, into) } } func (d *yamlDecoder) Close() error { return d.close() } // NewYAMLDecoder returns a new streaming Decoded that supports YAML. func NewYAMLDecoder(r io.ReadCloser) streaming.Decoder { return &yamlDecoder{ reader: apiyaml.NewYAMLReader(bufio.NewReader(r)), decoder: scheme.Codecs.UniversalDeserializer(), close: r.Close, } } // ToUnstructured takes a YAML and converts it to a list of Unstructured objects. func ToUnstructured(rawyaml []byte) ([]unstructured.Unstructured, error) { var ret []unstructured.Unstructured reader := apiyaml.NewYAMLReader(bufio.NewReader(bytes.NewReader(rawyaml))) count := 1 for { // Read one YAML document at a time, until io.EOF is returned b, err := reader.Read() if err != nil { if errors.Is(err, io.EOF) { break } return nil, errors.Wrapf(err, "failed to read yaml") } if len(b) == 0 { break } var m map[string]interface{} if err := yaml.Unmarshal(b, &m); err != nil { return nil, errors.Wrapf(err, "failed to unmarshal the %s yaml document: %q", util.Ordinalize(count), string(b)) } var u unstructured.Unstructured u.SetUnstructuredContent(m) // Ignore empty objects. // Empty objects are generated if there are weird things in manifest files like e.g. two --- in a row without a yaml doc in the middle if u.Object == nil { continue } ret = append(ret, u) count++ } return ret, nil } // JoinYaml takes a list of YAML files and join them ensuring // each YAML that the yaml separator goes on a new line by adding \n where necessary. func JoinYaml(yamls ...[]byte) []byte { var yamlSeparator = []byte("---") var cr = []byte("\n") var b [][]byte //nolint:prealloc for _, y := range yamls { if !bytes.HasPrefix(y, cr) { y = append(cr, y...) } if !bytes.HasSuffix(y, cr) { y = append(y, cr...) } b = append(b, y) } r := bytes.Join(b, yamlSeparator) r = bytes.TrimPrefix(r, cr) r = bytes.TrimSuffix(r, cr) return r } // FromUnstructured takes a list of Unstructured objects and converts it into a YAML. func FromUnstructured(objs []unstructured.Unstructured) ([]byte, error) { var ret [][]byte //nolint:prealloc for _, o := range objs { content, err := yaml.Marshal(o.UnstructuredContent()) if err != nil { return nil, errors.Wrapf(err, "failed to marshal yaml for %s, %s/%s", o.GroupVersionKind(), o.GetNamespace(), o.GetName()) } ret = append(ret, content) } return JoinYaml(ret...), nil } // Raw returns un-indented yaml string; it also remove the first empty line, if any. // While writing yaml, always use space instead of tabs for indentation. func Raw(raw string) string { return strings.TrimPrefix(heredoc.Doc(raw), "\n") }