istio 源码 – Citadel 源码分析 （原创）

[TOC]

安全整体架构

From: Istio 安全

源码位于 security ，编译后名称为 citadel。

命令行介绍

Dockerfile istio.io/istio/security/docker/Dockerfile.citadel

FROM scratch

# obtained from debian ca-certs deb using fetch_cacerts.sh
ADD ca-certificates.tgz /
# All containers need a /tmp directory
WORKDIR /tmp/
ADD istio_ca /usr/local/bin/istio_ca

ENTRYPOINT [ "/usr/local/bin/istio_ca", "--self-signed-ca" ]

查看版本：

# kubectl exec -ti istio-citadel-55cdfdd57c-bh7dk -n istio-system -- /usr/local/bin/istio_ca version
Version: 1.0.5
GitRevision: c1707e45e71c75d74bf3a5dec8c7086f32f32fad
User: root@6f6ea1061f2b
Hub: docker.io/istio
GolangVersion: go1.10.4
BuildStatus: Clean

命令行帮助：

# kubectl exec -ti istio-citadel-55cdfdd57c-bh7dk -n istio-system -- /usr/local/bin/istio_ca --help
Istio Certificate Authority (CA)

Usage:
  istio_ca [flags]
  istio_ca [command]

Available Commands:
  help        Help about any command
  probe       Check the liveness or readiness of a locally-running server
  version     Prints out build version information

Flags:
      --append-dns-names                           Append DNS names to the certificates for webhook services. (default true)
      --cert-chain string                          Path to the certificate chain file
      --citadel-storage-namespace string           Namespace where the Citadel pod is running. Will not be used if explicit file or other storage mechanism is specified. (default "istio-system")
      --custom-dns-names string                    The list of account.namespace:customdns names, separated by comma.
      --enable-profiling                           Enabling profiling when monitoring Citadel.
      --grpc-host-identities string                The list of hostnames for istio ca server, separated by comma. (default "istio-ca,istio-citadel")
      --grpc-hostname string                       DEPRECATED, use --grpc-host-identites. (default "istio-ca")
      --grpc-port int                              The port number for Citadel GRPC server. If unspecified, Citadel will not serve GRPC requests. (default 8060)
  -h, --help                                       help for istio_ca
      --key-size int                               Size of generated private key (default 2048)
      --kube-config string                         Specifies path to kubeconfig file. This must be specified when not running inside a Kubernetes pod.
      --listened-namespace string                  Select a namespace for the CA to listen to. If unspecified, Citadel tries to use the ${NAMESPACE} environment variable. If neither is set, Citadel listens to all namespaces.
      --liveness-probe-interval duration           Interval of updating file for the liveness probe.
      --liveness-probe-path string                 Path to the file for the liveness probe.
      --log_as_json                                Whether to format output as JSON or in plain console-friendly format
      --log_caller string                          Comma-separated list of scopes for which to include caller information, scopes can be any of [default, model]
      --log_output_level string                    Comma-separated minimum per-scope logging level of messages to output, in the form of <scope>:<level>,<scope>:<level>,... where scope can be one of [default, model] and level can be one of [debug, info, warn, error, none] (default "default:info")
      --log_rotate string                          The path for the optional rotating log file
      --log_rotate_max_age int                     The maximum age in days of a log file beyond which the file is rotated (0 indicates no limit) (default 30)
      --log_rotate_max_backups int                 The maximum number of log file backups to keep before older files are deleted (0 indicates no limit) (default 1000)
      --log_rotate_max_size int                    The maximum size in megabytes of a log file beyond which the file is rotated (default 104857600)
      --log_stacktrace_level string                Comma-separated minimum per-scope logging level at which stack traces are captured, in the form of <scope>:<level>,<scope:level>,... where scope can be one of [default, model] and level can be one of [debug, info, warn, error, none] (default "default:none")
      --log_target stringArray                     The set of paths where to output the log. This can be any path as well as the special values stdout and stderr (default [stdout])
      --max-workload-cert-ttl duration             The max TTL of issued workload certificates (default 2160h0m0s)
      --monitoring-port int                        The port number for monitoring Citadel. If unspecified, Citadel will disable monitoring. (default 9093)
      --org string                                 Organization for the cert
      --probe-check-interval duration              Interval of checking the liveness of the CA. (default 30s)
      --requested-ca-cert-ttl duration             The requested TTL for the workload (default 8760h0m0s)
      --root-cert string                           Path to the root certificate file
      --self-signed-ca                             Indicates whether to use auto-generated self-signed CA certificate. When set to true, the '--signing-cert' and '--signing-key' options are ignored.
      --self-signed-ca-cert-ttl duration           The TTL of self-signed CA root certificate (default 8760h0m0s)
      --self-signed-ca-org string                  The issuer organization used in self-signed CA certificate (default to k8s.cluster.local) (default "k8s.cluster.local")
      --sign-ca-certs                              Whether Citadel signs certificates for other CAs
      --signing-cert string                        Path to the CA signing certificate file
      --signing-key string                         Path to the CA signing key file
      --upstream-ca-address string                 The IP:port address of the upstream CA. When set, the CA will rely on the upstream Citadel to provision its own certificate.
      --workload-cert-grace-period-ratio float32   The workload certificate rotation grace period, as a ratio of the workload certificate TTL. (default 0.5)
      --workload-cert-min-grace-period duration    The minimum workload certificate rotation grace period. (default 10m0s)
      --workload-cert-ttl duration                 The TTL of issued workload certificates (default 2160h0m0s)

Use "istio_ca [command] --help" for more information about a command.

容器内部启动添加的命令行如下：

- --append-dns-names=true
    - --grpc-port=8060
    - --grpc-hostname=citadel
    - --citadel-storage-namespace=istio-system
    - --custom-dns-names=istio-pilot-service-account.istio-system:istio-pilot.istio-system,istio-ingressgateway-service-account.istio-system:istio-ingressgateway.istio-system
    - --self-signed-ca=true

可以在其运行的 node 节点上通过命令查看

$ /usr/local/bin/istio_ca --self-signed-ca --append-dns-names=true --grpc-port=8060 --grpc-hostname=citadel --citadel-storage-namespace=istio-system --custom-dns-names=istio-pilot-service-account.istio-system:istio-pilot.istio-system,istio-ingressgateway-service-account.istio-system:istio-ingressgateway.istio-system --self-signed-ca=true

istio-citadel 启动的 yaml 文件

# kubectl get pod istio-citadel-55cdfdd57c-bh7dk -n istio-system -o yaml
apiVersion: v1
kind: Pod
metadata:
  annotations:
    scheduler.alpha.kubernetes.io/critical-pod: ""
    sidecar.istio.io/inject: "false"
  creationTimestamp: 2019-01-15T08:24:24Z
  generateName: istio-citadel-55cdfdd57c-
  labels:
    istio: citadel
    pod-template-hash: 55cdfdd57c
  name: istio-citadel-55cdfdd57c-bh7dk
  namespace: istio-system
  ownerReferences:
  - apiVersion: apps/v1
    blockOwnerDeletion: true
    controller: true
    kind: ReplicaSet
    name: istio-citadel-55cdfdd57c
    uid: f6db1f80-189e-11e9-ab53-00163e0c1552
  resourceVersion: "16685125"
  selfLink: /api/v1/namespaces/istio-system/pods/istio-citadel-55cdfdd57c-bh7dk
  uid: f710ae31-189e-11e9-ab53-00163e0c1552
spec:
  affinity:
    nodeAffinity:
      preferredDuringSchedulingIgnoredDuringExecution:
      - preference:
          matchExpressions:
          - key: beta.kubernetes.io/arch
            operator: In
            values:
            - amd64
        weight: 2
      - preference:
          matchExpressions:
          - key: beta.kubernetes.io/arch
            operator: In
            values:
            - ppc64le
        weight: 2
      - preference:
          matchExpressions:
          - key: beta.kubernetes.io/arch
            operator: In
            values:
            - s390x
        weight: 2
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
        - matchExpressions:
          - key: beta.kubernetes.io/arch
            operator: In
            values:
            - amd64
            - ppc64le
            - s390x
  containers:
  - args:
    - --append-dns-names=true
    - --grpc-port=8060
    - --grpc-hostname=citadel
    - --citadel-storage-namespace=istio-system
    - --custom-dns-names=istio-pilot-service-account.istio-system:istio-pilot.istio-system,istio-ingressgateway-service-account.istio-system:istio-ingressgateway.istio-system
    - --self-signed-ca=true
    image: docker.io/istio/citadel:1.0.5
    imagePullPolicy: IfNotPresent
    name: citadel
    resources:
      requests:
        cpu: 10m
    terminationMessagePath: /dev/termination-log
    terminationMessagePolicy: File
    volumeMounts:
    - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
      name: istio-citadel-service-account-token-gdxfk
      readOnly: true
  dnsPolicy: ClusterFirst
  nodeName: node02
  priority: 0
  restartPolicy: Always
  schedulerName: default-scheduler
  securityContext: {}
  serviceAccount: istio-citadel-service-account
  serviceAccountName: istio-citadel-service-account
  terminationGracePeriodSeconds: 30
  tolerations:
  - effect: NoExecute
    key: node.kubernetes.io/not-ready
    operator: Exists
    tolerationSeconds: 10
  - effect: NoExecute
    key: node.kubernetes.io/unreachable
    operator: Exists
    tolerationSeconds: 10
  volumes:
  - name: istio-citadel-service-account-token-gdxfk
    secret:
      defaultMode: 420
      secretName: istio-citadel-service-account-token-gdxfk

代码流程分析

整体架构

istio.io/istio/security/cmd/istio_ca/main.go

// /usr/local/bin/istio_ca 
//  --self-signed-ca 
//  --append-dns-names=true 
//  --grpc-port=8060 
//  --grpc-hostname=citadel 
//  --citadel-storage-namespace=istio-system 
//  --custom-dns-names=istio-pilot-service-account.istio-system:istio-pilot.istio-system,
//     istio-ingressgateway-service-account.istio-system:istio-ingressgateway.istio-system //  --self-signed-ca=true    

rootCmd = &cobra.Command{
        Use:   "istio_ca",
        Short: "Istio Certificate Authority (CA).",
        Args:  cobra.ExactArgs(0),
        Run: func(cmd *cobra.Command, args []string) {
            runCA()
        },
    }

runCA 的主函数流程如下：

func runCA() {
    //...
    // --listened-namespace 设置 CA 监控的 namespace，如果没有指定会从 ${NAMESPACE}  环境变量中获取，如果都没有设置，Citadel 则会监听全部的 namespace.

    if value, exists := os.LookupEnv(cmd.ListenedNamespaceKey); exists {
        // When -namespace is not set, try to read the namespace from environment variable.
        if opts.listenedNamespace == "" {
            opts.listenedNamespace = value
        }
        // Use environment variable for istioCaStorageNamespace if it exists
        opts.istioCaStorageNamespace = value
    }

    // 验证命令行
    verifyCommandLineOptions()

    var webhooks map[string]controller.DNSNameEntry

    // 如果设置了添加 DNS 名字的后缀
    if opts.appendDNSNames {
        webhooks = make(map[string]controller.DNSNameEntry)
        /*
        // ServiceAccount/DNS pair for generating DNS names in certificates.
        // TODO: move it to a configmap later when we have more services to support.
        webhookServiceAccounts = []string{
            "istio-sidecar-injector-service-account",
            "istio-galley-service-account",
        }

        webhookServiceNames = []string{
            "istio-sidecar-injector",
            "istio-galley",
        }
        */
        for i, svcAccount := range webhookServiceAccounts { 
            // istio-sidecar-injector-service-account
            // istio-galley-service-account
            webhooks[svcAccount] = controller.DNSNameEntry{
                ServiceName: webhookServiceNames[i],
                Namespace:   opts.istioCaStorageNamespace, 
                // opts.istioCaStorageNamespace 运行的 namespace，默认为  istio-system
            }
        }

        // ...

    // 创建连接到集群中的 client
    cs := createClientset()

    // 返回 ca.IstioCA，用于管理证书链和签新的证书
    ca := createCA(cs.CoreV1())

    // For workloads in K8s, we apply the configured workload cert TTL.
    // 1. 创建 NewSecretController 来完成对于 API Server 中的 ServiceAccount 和 Secret 的创建
    sc, err := controller.NewSecretController(ca,
        opts.workloadCertTTL,
        opts.workloadCertGracePeriodRatio, opts.workloadCertMinGracePeriod, opts.dualUse,
        cs.CoreV1(), opts.signCACerts, opts.listenedNamespace, webhooks)
    if err != nil {
        fatalf("Failed to create secret controller: %v", err)
    }

    stopCh := make(chan struct{})
    // !!! 运行 NewSecretController
    sc.Run(stopCh)

    // 2. 如果设置了 grpcPort，则启动相关 server
    if opts.grpcPort > 0 {
        // ...
        ch := make(chan struct{})
        // monitor service objects with "alpha.istio.io/kubernetes-serviceaccounts" and
        // "alpha.istio.io/canonical-serviceaccounts" annotations
        // 2.1 NewServiceController
        serviceController := kube.NewServiceController(cs.CoreV1(), opts.listenedNamespace, reg)

        // ServiceController
        serviceController.Run(ch)

        // 2.2 NewServiceAccountController
        // monitor service account objects for istio mesh expansion
        serviceAccountController := kube.NewServiceAccountController(cs.CoreV1(), opts.listenedNamespace, reg)
        serviceAccountController.Run(ch)

        // The CA API uses cert with the max workload cert TTL.
        hostnames := append(strings.Split(opts.grpcHosts, ","), fqdn())
        caServer, startErr := caserver.New(ca, opts.maxWorkloadCertTTL, opts.signCACerts, hostnames, opts.grpcPort, spiffe.GetTrustDomain())
        if startErr != nil {
            fatalf("Failed to create istio ca server: %v", startErr)
        }
        if serverErr := caServer.Run(); serverErr != nil {
            // stop the registry-related controllers
            ch <- struct{}{}

            log.Warnf("Failed to start GRPC server with error: %v", serverErr)
        }
    }

    monitorErrCh := make(chan error)

    // 3. Start the monitoring server.
    if opts.monitoringPort > 0 {
        monitor, mErr := monitoring.NewMonitor(opts.monitoringPort, opts.enableProfiling)
        if mErr != nil {
            fatalf("Unable to setup monitoring: %v", mErr)
        }
        go monitor.Start(monitorErrCh)
        log.Info("Citadel monitor has started.")
        defer monitor.Close()
    }

    log.Info("Citadel has started")

    rotatorErrCh := make(chan error)
    // Start CA client if the upstream CA address is specified.
    if len(opts.cAClientConfig.CAAddress) != 0 {
        config := &opts.cAClientConfig
        config.Env = "onprem"
        config.Platform = "vm"
        config.ForCA = true
        config.CertFile = opts.signingCertFile
        config.KeyFile = opts.signingKeyFile
        config.CertChainFile = opts.certChainFile
        config.RootCertFile = opts.rootCertFile
        config.CSRGracePeriodPercentage = cmd.DefaultCSRGracePeriodPercentage
        config.CSRMaxRetries = cmd.DefaultCSRMaxRetries
        config.CSRInitialRetrialInterval = cmd.DefaultCSRInitialRetrialInterval
        rotator, creationErr := caclient.NewKeyCertBundleRotator(config, ca.GetCAKeyCertBundle())
        if creationErr != nil {
            fatalf("Failed to create key cert bundle rotator: %v", creationErr)
        }

        // 4. rotator 启动
        go rotator.Start(rotatorErrCh)
        log.Info("Key cert bundle rotator has started.")
        defer rotator.Stop()
    }

    // Blocking until receives error.
    for {
        select {
        case <-monitorErrCh:
            fatalf("Monitoring server error: %v", err)
        case <-rotatorErrCh:
            fatalf("Key cert bundle rotator error: %v", err)
        }
    }
}

NewSecretController

SecretController 内部会创建两个 Controller：

1. ServiceAccount 的监听，如果设置了 listened-namespace，则监听该 namespace 下，否则是全部；
2. Secret 的监听，namespace 同上，但是 Controller 只会监听自己创建的类型，即：type:”istio.io/key-and-cert”

实现的主要功能是为 ServiceAccount 创建对应的 Secret，Secret 中设置了相关的证书，在对应的 Pod 启动的时候进行加载；

// NewSecretController returns a pointer to a newly constructed SecretController instance.
func NewSecretController(ca ca.CertificateAuthority, certTTL time.Duration,
    gracePeriodRatio float32, minGracePeriod time.Duration, dualUse bool,
    core corev1.CoreV1Interface, forCA bool, namespace string, dnsNames map[string]DNSNameEntry) (*SecretController, error) {

    //...

    c := &SecretController{
        ca:               ca,
        certTTL:          certTTL,
        gracePeriodRatio: gracePeriodRatio,
        minGracePeriod:   minGracePeriod,
        dualUse:          dualUse,
        core:             core,
        forCA:            forCA,
        dnsNames:         dnsNames,
        monitoring:       newMonitoringMetrics(),
    }

    // 监听特定 namespace 下的 ServiceAccount
    c.saStore, c.saController = cache.NewInformer(saLW, &v1.ServiceAccount{}, time.Minute, rehf)

    istioSecretSelector := fields.SelectorFromSet(map[string]string{"type": IstioSecretType}).String()

    // 监听 type:”istio.io/key-and-cert” 的 secret 
    c.scrtStore, c.scrtController =
        cache.NewInformer(scrtLW, &v1.Secret{}, secretResyncPeriod, cache.ResourceEventHandlerFuncs{
            DeleteFunc: c.scrtDeleted,
            UpdateFunc: c.scrtUpdated,
        })

    // ...
}

// Run starts the SecretController until a value is sent to stopCh.
func (sc *SecretController) Run(stopCh chan struct{}) {
    go sc.scrtController.Run(stopCh)

    // saAdded calls upsertSecret to update and insert secret
    // it throws error if the secret cache is not synchronized, but the secret exists in the system
    cache.WaitForCacheSync(stopCh, sc.scrtController.HasSynced)

    go sc.saController.Run(stopCh)
}

gRPC Server 启动

如果设置了 gRPC 相关的参数，则会启动相关的服务，同上也会启动两个 Controller 和 一个 gRPC Server，Controller 监听的 namespace 由 listened-namespace 设置，同上：

1. NewServiceController ：用于监听添加了注解 alpha.istio.io/kubernetes-serviceaccounts 和 alpha.istio.io/canonical-serviceaccounts 的 Service 对象；从注解中解出来对应的用户名对应的 Reg 注册表的映射关系中，当前 key 和 value 都是相同 c.reg.AddMapping(svcAcct, svcAcct)

   ；

   istio.io/istio/security/pkg/registry/kube/service.go

   // KubeServiceAccountsOnVMAnnotation is to specify the K8s service accounts that are allowed to run
   // this service on the VMs
   KubeServiceAccountsOnVMAnnotation = "alpha.istio.io/kubernetes-serviceaccounts"
   
   // CanonicalServiceAccountsAnnotation is to specify the non-Kubernetes service accounts that
   // are allowed to run this service.
   CanonicalServiceAccountsAnnotation = "alpha.istio.io/canonical-serviceaccounts"

   结构体定义如下：

   // ServiceController monitors the service definition changes in a namespace. If a
   // new service is added with "alpha.istio.io/kubernetes-serviceaccounts" or
   // "alpha.istio.io/canonical-serviceaccounts" annotations enabled,
   // the corresponding service account will be added to the identity registry
   // for whitelisting.
   type ServiceController struct {
   core corev1.CoreV1Interface
   
   // identity registry object
   reg registry.Registry
   
   // controller for service objects
   controller cache.Controller
   }

2. NewServiceAccountController ： 监听 ServiceAccount 对象；对于获取到 sa 信息，生成相对应的 SpiffeID 保存到 Reg 注册表的映射关系中，当前 key 和 value 都是相同 c.reg.DeleteMapping(id, id)

   ；

   结构体定义如下：

   istio.io/istio/security/pkg/registry/kube/serviceaccount.go

   // ServiceAccountController monitors service account definition changes in a namespace.
   // For each service account object, its SpiffeID is added to identity registry for
   // whitelisting purpose.
   type ServiceAccountController struct {
   core corev1.CoreV1Interface
   
   // identity registry object
   reg registry.Registry
   
   // controller for service objects
   controller cache.Controller
   }

3. IstioCAServiceServer

   ：主要提供证书的生成和验证功能；

   istio.io/istio/security/pkg/server/ca/server.go

   // CreateCertificate handles an incoming certificate signing request (CSR). It does
   // authentication and authorization. Upon validated, signs a certificate that:
   // the SAN is the identity of the caller in authentication result.
   // the subject public key is the public key in the CSR.
   // the validity duration is the ValidityDuration in request, or default value if the given duration is invalid.
   // it is signed by the CA signing key.
   func (s *Server) CreateCertificate(ctx context.Context, request *pb.IstioCertificateRequest) (
   *pb.IstioCertificateResponse, error) {
   
   // 根据请求生成对应的证书
   _, _, certChainBytes, rootCertBytes := s.ca.GetCAKeyCertBundle().GetAll()
   cert, signErr := s.ca.Sign(
       []byte(request.Csr), caller.Identities, time.Duration(request.ValidityDuration)*time.Second, false)
   
   respCertChain := []string{string(cert)}
   respCertChain = append(respCertChain, string(rootCertBytes))
   response := &pb.IstioCertificateResponse{
       CertChain: respCertChain,
   }
   log.Debug("CSR successfully signed.")
   
   return response, nil
   }
   
   // HandleCSR handles an incoming certificate signing request (CSR). It does
   // proper validation (e.g. authentication) and upon validated, signs the CSR
   // and returns the resulting certificate. If not approved, reason for refusal
   // to sign is returned as part of the response object.
   // [TODO](myidpt): Deprecate this function.
   func (s *Server) HandleCSR(ctx context.Context, request *pb.CsrRequest) (*pb.CsrResponse, error) {
   
   csr, err := util.ParsePemEncodedCSR(request.CsrPem)
   
   _, err = util.ExtractIDs(csr.Extensions)
   
   // TODO: Call authorizer.
   
   _, _, certChainBytes, _ := s.ca.GetCAKeyCertBundle().GetAll()
   cert, signErr := s.ca.Sign(request.CsrPem, []string{}, time.Duration(request.RequestedTtlMinutes)*time.Minute, s.forCA)
   
   response := &pb.CsrResponse{
       IsApproved: true,
       SignedCert: cert,
       CertChain:  certChainBytes,
   }
   log.Debug("CSR successfully signed.")
   
   return response, nil
   }

Monitor

Monitor 服务主要用于对外输出检查，为 HttpServer, 主要提供 /metrics 和 /version ，如果启用了 enableProfiling 还会启用 /debug/pprof/ 相关的路径；当 Monitor 启动以后， Citadel 则任务已经启动成功，打印以下信息： log.Info("Citadel has started") ；

CA client

如果指定了 upstream CA Server，还会启动一个 CA Client， 创建一个 rotator go routine，定期用于证书的轮转替换；

istio.io/istio/security/pkg/caclient/keycertbundlerotator.go

// Start periodically rotates the KeyCertBundle by interacting with the upstream CA.
// It is a blocking function that should run as a go routine. Thread safe.
func (c *KeyCertBundleRotator) Start(errCh chan<- error) {
    c.stoppedMutex.Lock()
    if !c.stopped {
        errCh <- fmt.Errorf("rotator already started")
        c.stoppedMutex.Unlock()
        return
    }
    c.stopped = false
    c.stoppedMutex.Unlock()

    // Make sure we mark rotator stopped after this method finishes.
    defer func() {
        c.stoppedMutex.Lock()
        c.stopped = true
        c.stoppedMutex.Unlock()
    }()

    for {
        certBytes, _, _, _ := c.keycert.GetAllPem()
        if len(certBytes) != 0 {
            waitTime, ttlErr := c.certUtil.GetWaitTime(certBytes, time.Now())
            if ttlErr != nil {
                log.Errorf("Error getting TTL from cert: %v. Rotate immediately.", ttlErr)
            } else {
                timer := time.NewTimer(waitTime)
                log.Infof("Will rotate key and cert in %v.", waitTime)
                select {
                case <-c.stopCh:
                    return
                case <-timer.C:
                    // Continue in the loop.
                }
            }
        }
        co, coErr := c.keycert.CertOptions()
        if coErr != nil {
            err := fmt.Errorf("failed to extact CertOptions from bundle: %v, abort auto rotation", coErr)
            log.Errora(err)
            errCh <- err
            return
        }
        certBytes, certChainBytes, privKeyBytes, rErr := c.retriever.Retrieve(co)
        if rErr != nil {
            err := fmt.Errorf("error retrieving the key and cert: %v, abort auto rotation", rErr)
            log.Errora(err)
            errCh <- err
            return
        }
        _, _, _, rootCertBytes := c.keycert.GetAllPem()
        if vErr := c.keycert.VerifyAndSetAll(certBytes, privKeyBytes, certChainBytes, rootCertBytes); vErr != nil {
            err := fmt.Errorf("cannot verify the retrieved key and cert: %v, abort auto rotation", vErr)
            log.Errora(err)
            errCh <- err
            return
        }
        log.Infof("Successfully retrieved new key and certs.")
    }
}