azure - Terraform - 如何在 AKS 集群上安装 AZURE CNI 并设置 POD IP 范围？

Question

我正在尝试使用 terraform 在 azure 中创建 AKS 集群。我的要求如下:

1. 创建站点到站点 VPN 连接，其中网关位于 172.30.0.0/16 范围的子网中 - 此操作已完成

2. 使用 Azure CNI 安装 Azure AKS 集群，并且 Pod 应该位于 VPN CIDR (172.30.0.0/16) 范围内。

这是我的地形代码。我读到，如果您使用 azure 作为 network_policy 和 network_plugin，则无法设置 pod_cidr - source

那么我该如何执行此操作，以便我的 POD 可以通过站点到站点 VPN 访问本地网络？

resource "azurerm_kubernetes_cluster" "k8s_cluster" {
  lifecycle {
    ignore_changes = [
      default_node_pool[0].node_count
    ]
    prevent_destroy = false
  }

  name                = var.cluster_name
  location            = var.location
  resource_group_name = var.rg_name
  dns_prefix          = var.dns_prefix
  kubernetes_version  = var.kubernetes_version
  # node_resource_group = var.resource_group_name

  default_node_pool {
    name                  = var.default_node_pool.name
    node_count            = var.default_node_pool.node_count
    max_count             = var.default_node_pool.max_count
    min_count             = var.default_node_pool.min_count
    vm_size               = var.default_node_pool.vm_size
    os_disk_size_gb       = var.default_node_pool.os_disk_size_gb
    # vnet_subnet_id        = var.vnet_subnet_id
    max_pods              = var.default_node_pool.max_pods
    type                  = var.default_node_pool.agent_pool_type
    enable_node_public_ip = var.default_node_pool.enable_node_public_ip
    enable_auto_scaling   = var.default_node_pool.enable_auto_scaling

    tags = merge(var.common_tags)
  }

  identity {
    type = var.identity
  }

  network_profile {
    network_plugin     = var.network_plugin #azure
    network_policy     = var.network_policy #"azure"
    load_balancer_sku  = var.load_balancer_sku #"standard"
    # pod_cidr  = var.pod_cidr | When network_plugin is set to azure - the vnet_subnet_id field in the default_node_pool block must be set and pod_cidr must not be set.
  }
  

  tags = merge(var.common_tags)
}

# AKS cluster related variables
cluster_name       = "test-cluster"
dns_prefix         = "testjana"
kubernetes_version = "1.22.15"
default_node_pool = {
  name                  = "masternp" # for system pods
  node_count            = 1
  vm_size               = "standard_e4bds_v5" # 4 vcpu and 32 Gb of memory
  enable_auto_scaling   = false
  enable_node_public_ip = false
  min_count             = null
  max_count             = null
  max_pods              = 100
  os_disk_size_gb       = 80
  agent_pool_type       = "VirtualMachineScaleSets"
}
admin_username    = "jananathadmin"
ssh_public_key    = "public_key"
identity          = "SystemAssigned"
network_plugin    = "azure"
network_policy    = "azure"
load_balancer_sku = "standard"

Answer 1

默认情况下，AKS 中的所有 POD 都会相互通信，当我们想要限制流量时，可以使用网络策略来允许或拒绝 Pod 之间的流量。

这是教程 link

使用下面的代码片段通过 terraform 复制了相同的内容，将集群与 azure CNI 和 vnet 网关连接起来，该网关通过站点到站点 VPN 将我们的本地环境链接到 azure。

第一步:主tf文件如下

resource "azurerm_resource_group" "example" {
  name     = "*****-****"
  location = "East US"
}

resource "azurerm_role_assignment" "role_acrpull" {
  scope                            = azurerm_container_registry.acr.id
  role_definition_name             = "AcrPull"
  principal_id                     = azurerm_kubernetes_cluster.demo.kubelet_identity.0.object_id
}

resource "azurerm_container_registry" "acr" {
  name                = "acrswarna"
  resource_group_name = azurerm_resource_group.example.name
  location            = azurerm_resource_group.example.location
  sku                 = "Standard"
  admin_enabled       = false
}
resource "azurerm_virtual_network" "puvnet" {
    name                = "Publics_VNET"
   resource_group_name = azurerm_resource_group.example.name
  location            = azurerm_resource_group.example.location
  address_space       = ["10.19.0.0/16"]
}
resource "azurerm_subnet" "example" {
  name                 = "GatewaySubnet"
   resource_group_name = azurerm_resource_group.example.name
  virtual_network_name = azurerm_virtual_network.puvnet.name
  address_prefixes     = ["10.19.3.0/24"]
}
resource "azurerm_subnet" "osubnet" {
  name                 = "Outer_Subnet"
  resource_group_name = azurerm_resource_group.example.name
  address_prefixes     = ["10.19.1.0/24"]
  virtual_network_name = azurerm_virtual_network.puvnet.name
}

resource "azurerm_kubernetes_cluster" "demo" {
  name                = "demo-aksnew"
  location            = azurerm_resource_group.example.location
  resource_group_name = azurerm_resource_group.example.name
  dns_prefix          = "demo-aks"

  default_node_pool {
    name                = "default"
    node_count          = 2
    vm_size             = "standard_e4bds_v5"
    type                = "VirtualMachineScaleSets"
    enable_auto_scaling = false
    min_count             = null
    max_count             = null
    max_pods              = 100
    //vnet_subnet_id = azurerm_subnet.osubnet.id
  }
  
  identity {
    type = "SystemAssigned"
  }

  network_profile {
    network_plugin    = "azure"
    load_balancer_sku = "standard"
    network_policy    = "azure"
  }

  tags = {
    Environment = "Development"
  }
}
resource "azurerm_public_ip" "example" {
  name                = "pips-firewall"
   resource_group_name = azurerm_resource_group.example.name
  location            = azurerm_resource_group.example.location
   allocation_method = "Static"
  sku                 = "Standard"
}
resource "azurerm_virtual_network_gateway" "example" {
  name                = "test"
  location            = azurerm_resource_group.example.location
  resource_group_name = azurerm_resource_group.example.name

  type     = "Vpn"
  vpn_type = "RouteBased"

  active_active = false
  enable_bgp    = false
  sku           = "VpnGw1"

  ip_configuration {
    name                          = "vnetGatewayConfig"
    public_ip_address_id          = azurerm_public_ip.example.id
    private_ip_address_allocation = "Dynamic"
    subnet_id                     = azurerm_subnet.example.id
  }

  vpn_client_configuration {
    address_space = ["172.30.0.0/16"]

    root_certificate {
      name = "******-****-ID-Root-CA"

      public_cert_data = <<EOF
   **Use certificate here**
   EOF
    }
    revoked_certificate {
      name       = "*****-Global-Root-CA"
      thumbprint = "****************"
    }
  }
}

注意:在上面的代码上自行更新根证书配置。

提供tf文件如下

terraform {
  required_providers {
    azurerm = {
      source  = "hashicorp/azurerm"
      version = "=3.0.0"
    }
  }

}
provider "azurerm" {
  features {}
  
  skip_provider_registration = true
}

运行时

 terraform plan 
 terraform apply -auto-approve

Vnet 和子网配置

虚拟网络网关配置如下。

在集群上部署示例 Pod