azure - 使用 Terraform 增加 AKS 的 default_node_pool 中的 max_pod 数量时，必须重新创建集群本身

Question

1. 使用 Terraform，我们将 AKS 的 default_node_pool 的 max_pod 数量调整为 20 -> 30。
2. network_policy 和 network_plugin 是“azure”

代码如下。

• var.tf

variable "system_rg" {
  type        = string
  default     = "aks-test-resourcegroup"
}

variable "location" {
  type        = string
  default     = "Korea Central"
}

###################
# k8s cluster
###################
variable "cluster_name" {
  default     = "Test-AKS"
}

variable "aks_version" {
  type    = string
  default = "1.25.5"
}

variable "private_cluster_enabled" {
  type    = string
  default = "true"
}

variable "private_cluster_public_fqdn_enabled" {
  type    = string
  default = "true"
}

variable "private_dns_zone_id" {
  type    = string
  default = "None"
}

variable "sku_tier" {
  type    = string
  default = "Free"
}
################### 
# default_node_pool
###################
variable "only_critical_addons_enabled" {
  type        = string
  default     = "true"
}

variable "temporary_name_for_rotation" {
  type    = string
  default = "tempsys01"
}

variable "orchestrator_version" {
  type    = string
  default = "1.25.5"
}

variable "agents_count" {
  type    = number
  default = "3"
}

variable "agents_size" {
  type    = string
  default = "Standard_D4s_v5"
}

variable "os_disk_size_gb" {
  description = "The size of the OS Disk which should be used for each agent in the Node Pool. Changing this forces a new resource to be created."
  type        = number
  default     = 256
}

variable "max_pods" {
  description = "The maximum number of pods that can run on each agent. Changing this forces a new resource to be created."
  type        = number
  default     = "30" # 20 => 30
}
###################
# linux_profile
###################
variable "admin_username" {
  type    = string
  default = "azureuser"
}

variable "ssh_public_key" {
  type        = string
  default     = ""
}

################### 
# network_profile
###################
variable "service_cidr" {
  type    = string
  default = "10.254.0.0/24"
}

variable "dns_service_ip" {
  type    = string
  default = "10.254.0.10"
}

variable "docker_bridge_cidr" {
  type    = string
  default = "172.17.0.1/16"
}

# ###############################
# # user_node_pool
# ###############################
variable "usernodepoo_vm" {
  description = "VM of AKS Cluster"
  type        = map(any)
  default = {
    vm1 = {
      user_agents_name         = "upool01"
      user_agents_size         = "Standard_D4s_v5"
      user_agents_count        = "4"
      user_agents_os_disk_size = "256" 
      max_pods                 = "20"
      orchestrator_version     = "1.25.5"
    }
  }
}

• 集群.tf

############################################################
# AKS Cluster
############################################################

resource "azurerm_kubernetes_cluster" "aks" {
  name                                = var.cluster_name
  location                            = var.location
  resource_group_name                 = data.azurerm_resource_group.aks-rg.name 
  node_resource_group                 = "${var.system_rg}-node"
  dns_prefix                          = var.cluster_name
  kubernetes_version                  = var.aks_version
  private_cluster_enabled             = var.private_cluster_enabled
  private_cluster_public_fqdn_enabled = var.private_cluster_public_fqdn_enabled 
  private_dns_zone_id                 = var.private_dns_zone_id
  sku_tier                            = var.sku_tier

  default_node_pool {
    name                         = "syspool01"
    vm_size                      = var.agents_size
    os_disk_size_gb              = var.os_disk_size_gb
    node_count                   = var.agents_count
    vnet_subnet_id               = data.azurerm_subnet.subnet.id
    zones                        = [1, 2, 3]
    kubelet_disk_type            = "OS"
    os_sku                       = "Ubuntu"
    os_disk_type                 = "Managed"
    ultra_ssd_enabled            = "false"
    max_pods                     = var.max_pods
    only_critical_addons_enabled = var.only_critical_addons_enabled 
    temporary_name_for_rotation  = var.temporary_name_for_rotation 
    orchestrator_version         = var.aks_version
  }

  linux_profile {
    admin_username = var.admin_username

    ssh_key {
      key_data = replace(coalesce("${var.ssh_public_key}", tls_private_key.ssh[0].public_key_openssh), "\n", "")
    }
  }

  network_profile {
    network_plugin    = "azure"
    network_policy    = "azure"
    load_balancer_sku = "standard"
    outbound_type     = "userDefinedRouting"
    service_cidr      = var.service_cidr
    dns_service_ip    = var.dns_service_ip
  }

  tags = {
    Environment = "${var.tag}"
  }

  identity {
    type = "SystemAssigned"
  }
}

## usernodepool
resource "azurerm_kubernetes_cluster_node_pool" "usernodepool" {
  for_each = var.usernodepoo_vm

  name                  = each.value.user_agents_name
  kubernetes_cluster_id = azurerm_kubernetes_cluster.aks.id
  vm_size               = each.value.user_agents_size
  os_disk_size_gb       = each.value.user_agents_os_disk_size 
  node_count            = each.value.user_agents_count
  vnet_subnet_id        = data.azurerm_subnet.subnet.id
  zones                 = [1, 2, 3]
  mode                  = "User"
  kubelet_disk_type     = "OS"
  os_sku                = "Ubuntu"
  os_disk_type          = "Managed"
  ultra_ssd_enabled     = "false"
  max_pods              = each.value.max_pods
  orchestrator_version  = each.value.orchestrator_version
}

应用此 Terraform 代码将尝试重新创建整个集群。有没有办法防止这种情况，只增加 max_pods 的数量？

我尝试如下设置，但结果是一样的。

resource "azurerm_kubernetes_cluster" "aks" {
...
  lifecycle {
    prevent_destroy = true
  }
}

│ Error: Instance cannot be destroyed
│
│   on cluster.tf line 63:
│   63: resource "azurerm_kubernetes_cluster" "aks" {
│
│ Resource azurerm_kubernetes_cluster.aks has lifecycle.prevent_destroy set, but the plan calls for this resource to be destroyed. To avoid this error and continue with the plan, either disable lifecycle.prevent_destroy or reduce the
│ scope of the plan using the -target flag.

Answer 1

Applying this Terraform code will attempt to recreate the entire cluster. Is there a way to prevent this and just increase the number of max_pods?

防止重新创建整个集群并仅更新 max_pods 值。您可以使用 Terraform 生命周期 配置 block 来管理更新期间资源的行为。

以下是使用相同代码更新 max_pods 且不破坏现有 AKS 集群 的示例代码

provider "azurerm" {
  features {}
}
resource "azurerm_resource_group" "aksdemo-rg" {
  name     = "demo-rg-aks-test"
  location = "West Europe"
}
resource "azurerm_kubernetes_cluster" "hellaks" {
  name                = "example-aks1"
  location            = azurerm_resource_group.aksdemo-rg.location
  resource_group_name = azurerm_resource_group.aksdemo-rg.name
  dns_prefix          = "exampleaks1"

  default_node_pool {
    name       = "default"
    node_count = 3
    max_pods   = 30
    vm_size    = "Standard_D2_v2"
    temporary_name_for_rotation = "exampleaks1temp"
  }
  identity {
    type = "SystemAssigned"
  }
  tags = {
    Environment = "Production"
  }
    lifecycle {
    prevent_destroy = true
  }
}

地形计划

Terraform 应用:

输出: