docker - Kubernetes 私有(private)注册表证书由未知机构签名

Question

我在 AWS EC2 实例上使用 bitnami kubernetes 镜像。
kubectl 集群信息 说

Kubernetes master is running at https://172.30.0.120:6443
Heapster is running at https://172.30.0.120:6443/api/v1/namespaces/kube-system/services/heapster/proxy
KubeDNS is running at https://172.30.0.120:6443/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy
monitoring-grafana is running at https://172.30.0.120:6443/api/v1/namespaces/kube-system/services/monitoring-grafana/proxy
monitoring-influxdb is running at https://172.30.0.120:6443/api/v1/namespaces/kube-system/services/monitoring-influxdb/proxy

所以我假设我只有一个主节点而没有节点？
我通过部署了一个私有(private)注册表

apiVersion: apps/v1
kind: Deployment
metadata:
  name: registry
  namespace: kube-system
spec:
  selector:
    matchLabels:
      app: registry
  template:
    metadata:
      labels:
        app: registry
        id: "registry"
    spec:
      containers:
      - name: registry
        image: registry:2.7.0
        ports:
        - name: registry-port
          containerPort: 5000
        volumeMounts:
        - mountPath: /var/lib/registry
          name: images
        - mountPath: /certs
          name: certs
        - mountPath: /auth
          name: auth
        env:
        - name: REGISTRY_AUTH
          value: "htpasswd"
        - name: REGISTRY_AUTH_HTPASSWD_REALM
          value: "Registry Realm"
        - name: REGISTRY_AUTH_HTPASSWD_PATH
          value: /auth/htpasswd
        - name: REGISTRY_HTTP_TLS_CERTIFICATE
          value: /certs/domain.crt
        - name: REGISTRY_HTTP_TLS_KEY
          value: /certs/domain.key
      volumes:
      - name: images
        hostPath:
          path: /home/bitnami/registry/images
      - name: certs
        hostPath:
          path: /home/bitnami/docker_reg_certs
      - name: auth
        hostPath:
          path: /home/bitnami/registry/auth

我通过 openssl 生成一个自签名的 ssl-ca 并将其复制到/etc/docker/certs.d/[ec2-insta-domain:port] 和/usr/local/share/ca-certificates/ca.crt 和执行 update-ca-certificates 和 sudo service docker restart。
之后，我使用 docker login [domainname]:5000 登录到我的注册表并输入用户和密码。这项工作。
然后我用 kubectl create secret generic my-secret --from-file=.dockerconfigjson=~/.docker/config.json --type=kubernetes.io/dockerconfigjson 创建一个 docker-registry secret
之后我创建我的部署

kind: Deployment
apiVersion: apps/v1
metadata:
  name: app-config-service
  namespace: kube-system
spec:
  selector:
    matchLabels:
      app: app-config-service
  template:
    metadata:
      labels:
        app: app-config-service
    spec:
      containers:
      - name: app-config-service
        image: [AWS-EC2-DomainName]:5000/cloud/app-config-service:latest
        ports:
        - containerPort: 48000
        env:
        - name: SPRING_PROFILE
          value: docker
        - name: SPRING_BOOT_PORT
          value: "48000"
        - name: KAFKA_BINDER_BROKERS_URL
          value: xxxxxxxxxxx:29095
      imagePullSecrets:
        - name: my-secret

但是，当我尝试使用 kubectl create -f deployment.yml 运行此部署并描述创建的 pod 时，我收到错误无法执行请求:Head https://[domainname]:5000/v2/cloud/app-config-service/manifests/latest: x509: 未知权威签署的证书
注册表的日志显示 2020/07/14 15:54:01 http: TLS handshake error from 10.32.0.1:37943: remote error: tls: bad certificate
如果我尝试使用 curl -X GET https://[domainname]:5000/v2/_catalog 手动连接到私有(private)存储库，它可以工作。
有谁知道我在这里错过了什么或做错了什么？
问候
编辑
我发现了一些有趣的东西。对于我的 kubernetes，我使用容器运行时 containerd github.com/containerd/containerd v1.3.3 d76c121f76a5fc8a462dc64594aea72fe18e1178
我像这样编辑/etc/containered/config.toml

version = 2
root = "/var/lib/containerd"
state = "/run/containerd"
plugin_dir = ""
disabled_plugins = []
required_plugins = []
oom_score = 0

[grpc]
  address = "/run/containerd/containerd.sock"
  tcp_address = ""
  tcp_tls_cert = ""
  tcp_tls_key = ""
  uid = 0
  gid = 0
  max_recv_message_size = 16777216
  max_send_message_size = 16777216

[ttrpc]
  address = ""
  uid = 0
  gid = 0

[debug]
  address = ""
  uid = 0
  gid = 0
  level = ""

[metrics]
  address = ""
  grpc_histogram = false

[cgroup]
  path = ""

[timeouts]
  "io.containerd.timeout.shim.cleanup" = "5s"
  "io.containerd.timeout.shim.load" = "5s"
  "io.containerd.timeout.shim.shutdown" = "3s"
  "io.containerd.timeout.task.state" = "2s"

[plugins]
  [plugins."io.containerd.gc.v1.scheduler"]
    pause_threshold = 0.02
    deletion_threshold = 0
    mutation_threshold = 100
    schedule_delay = "0s"
    startup_delay = "100ms"
  [plugins."io.containerd.grpc.v1.cri"]
    disable_tcp_service = true
    stream_server_address = "127.0.0.1"
    stream_server_port = "0"
    stream_idle_timeout = "4h0m0s"
    enable_selinux = false
    sandbox_image = "k8s.gcr.io/pause:3.1"
    stats_collect_period = 10
    systemd_cgroup = false
    enable_tls_streaming = false
    max_container_log_line_size = 16384
    disable_cgroup = false
    disable_apparmor = false
    restrict_oom_score_adj = false
    max_concurrent_downloads = 3
    disable_proc_mount = false
    [plugins."io.containerd.grpc.v1.cri".containerd]
      snapshotter = "overlayfs"
      default_runtime_name = "runc"
      no_pivot = false
      [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
        runtime_type = ""
        runtime_engine = ""
        runtime_root = ""
        privileged_without_host_devices = false
      [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime]
        runtime_type = ""
        runtime_engine = ""
        runtime_root = ""
        privileged_without_host_devices = false
      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
          runtime_type = "io.containerd.runc.v1"
          runtime_engine = ""
          runtime_root = ""
          privileged_without_host_devices = false
    [plugins."io.containerd.grpc.v1.cri".cni]
      bin_dir = "/opt/cni/bin"
      conf_dir = "/etc/cni/net.d"
      max_conf_num = 1
      conf_template = ""
    [plugins."io.containerd.grpc.v1.cri".registry]
    [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
      [plugins."io.containerd.grpc.va.cri".registry.mirrors."ip:5000"]
        endpoint = ["http://ip:5000"]
        [plugins."io.containerd.gprc.v1.cri".registry.auths]
      [plugins."io.containerd.grpc.va.cri".registry.mirrors."172.30.0.120:5000"]
        endpoint = ["http://172.30.0.120:5000"]
      [plugins."io.containerd.grpc.va.cri".registry.mirrors."domainname:5000"]
        endpoint = ["http://domainame:5000"]
      [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
        endpoint = ["https://registry-1.docker.io"]
    [plugins."io.containerd.gprc.v1.cri".registry.auths."http://ip:5000"]
      username = "admin"
      password = "admin"
    [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
      tls_cert_file = ""
      tls_key_file = ""
  [plugins."io.containerd.internal.v1.opt"]
    path = "/opt/containerd"
  [plugins."io.containerd.internal.v1.restart"]
    interval = "10s"
  [plugins."io.containerd.metadata.v1.bolt"]
    content_sharing_policy = "shared"
  [plugins."io.containerd.monitor.v1.cgroups"]
    no_prometheus = false
  [plugins."io.containerd.runtime.v1.linux"]
    shim = "containerd-shim"
    runtime = "runc"
    runtime_root = ""
    no_shim = false
    shim_debug = false
  [plugins."io.containerd.runtime.v2.task"]
    platforms = ["linux/amd64"]
  [plugins."io.containerd.service.v1.diff-service"]
    default = ["walking"]
  [plugins."io.containerd.snapshotter.v1.devmapper"]
    root_path = ""
    pool_name = ""
    base_image_size = ""

但是如果我调用 crictl info 输出会显示以下内容

...
...
 },
    "registry": {
      "mirrors": {
        "docker.io": {
          "endpoint": [
            "https://registry-1.docker.io"
          ]
        }
      },
...
...

所以 config.toml对我的 kubernetes 使用的注册表没有影响吗？有任何想法吗？

Answer 1

看起来你创造了 secret my-secret在默认命名空间但部署 app-config-service正在运行 kube-system命名空间，因此不能引用 my-secret使用 imagePullSecrets .创建 secret my-secret在 kube-system命名空间应该可以解决这个问题。