Terraform:InvalidParameterException:PutSubscriptionFilter 操作无法与 vendor es 的 destinationArn 一起使用

Question

我正在尝试复制 AWS 控制台功能，我只需单击一个日志组，然后选择 ES 上的流，然后选择我的托管 ES 之一。

我以为 aws_cloudwatch_log_subscription_filter是我一直在寻找的。尽管医生说只有 Kinesis 和 Lambda，但我心想“也许文档是旧的/不完整的”

所以我尝试了

resource "aws_cloudwatch_log_subscription_filter" "whatever" {                                                                                                                                                                      
  name            = "lambda_logs_to_es"                                                                                                                                                                           
  role_arn        = aws_iam_role.my_lamda_role.arn                                                                                                                                                                            
  log_group_name  = aws_cloudwatch_log_group.my_log_group.name                                                                                                                                                                
  filter_pattern  = ""                                                                                                                                                                                                                       
  destination_arn = "arn:aws:es:eu-west-3:585399047133:domain/my-es"                                                                                                                          
}    

不出所料，它告诉我

Terraform  InvalidParameterException: PutSubscriptionFilter operation cannot work with destinationArn for vendor es

那么 terraform 是否有一种简单的方法来复制此功能，而无需编写我自己的 lambda，将我的其他 lambda(lambda-ception)的日志转发到 ES？ (这是我选择 AWS 的 ES 而不是自己安装的原因之一，感觉它会更好地与其他 AWS 服务集成)？

即是否已经(如果可能由 AWS 支持)lambda 和 terrafor 模块使用所述 lambda 来执行此功能？

Answer 1

稍后我将使用更多代码编辑答案，但长话短说:

没有什么神奇的，您在 AWS 控制台中执行的单击操作，实际上创建了一个将日志转发到 ES 的 lambda，亚马逊托管 Elasticsearch 没有神奇的直接链接。

所以我实际上最终做了

#
# Create a lambda that will forward other lambda's logs
# to the VPC's elasticsearch, this whole configuration can be done
# manually though the "stream to elasticsearch" option when you
# choose where to stream a log group of cloudwatch
#

resource "aws_iam_role" "logs_to_es_lambda_role" {
  name = "logs_to_es_lambda_${local.env_type}"

  assume_role_policy = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Action": "sts:AssumeRole",
      "Principal": {
        "Service": "lambda.amazonaws.com"
      },
      "Effect": "Allow",
      "Sid": ""
    }
  ]
}
EOF
}

resource "aws_iam_role_policy_attachment" "attach_basic_role_to_logs_to_es" {
  role = aws_iam_role.logs_to_es_lambda_role.id
  # this policy permits to log in cloudwatch
  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
}

resource "aws_iam_role_policy_attachment" "access_vpc_role_to_logs_to_es" {
  role = aws_iam_role.logs_to_es_lambda_role.id
  # this policy permits the lambda to have access to the vpc (as the Elastic search
  # is only accessible from there
  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole"
}

resource "aws_cloudwatch_log_group" "logs_to_es_log_group" {
  name              = "/aws/lambda/logs_to_es_lambda_role_${local.env_type}"
  retention_in_days = 7
}

resource "aws_security_group" "logs_to_es" {
  description = "allow all egress network for the lambda logs_to_es"
  vpc_id      = module.vpc.vpc_id
  name        = "lambda-${local.env_type}-logs-to-es"

  egress {
    protocol    = "-1"
    from_port   = 0
    to_port     = 0
    cidr_blocks = ["0.0.0.0/0"]
  }

  tags = {
    Name               = "lambda-${local.env_type}-logs-to-es"
    ManagedByTerraform = "true"
    EnvironmentType    = "${local.env_type}"
  }
}


data "archive_file" "logs_to_es_zip" {
  type        = "zip"
  source_file = "lambdas/logs_to_elasticsearch/logs_to_elasticsearch.js"
  output_path = "lambdas/logs_to_elasticsearch/logs_to_elasticsearch.zip"
}

resource "aws_lambda_function" "logs_to_es" {

  filename         = data.archive_file.logs_to_es_zip.output_path
  source_code_hash = data.archive_file.logs_to_es_zip.output_base64sha256
  function_name    = "logs_to_es_${local.env_type}"

  role    = aws_iam_role.logs_to_es_lambda_role.arn
  handler = "logs_to_elasticsearch.handler"

  timeout = 100

  runtime = "nodejs10.x"

  description = "${local.env_type} script that forwards cloudwatch logs of that VPC's lambda to ES"

  vpc_config {
    subnet_ids = module.vpc.private_subnets
    security_group_ids = [
      aws_security_group.logs_to_es.id
    ]
  }

  environment {
    variables = {
      ELASTICSEARCH_ENDPOINT = aws_elasticsearch_domain.technical_logs.endpoint
    }
  }

  tags = {
    Environment        = local.env_type
    ManagedByTerraform = "true"
  }

  depends_on = [
    "aws_iam_role_policy_attachment.attach_basic_role_to_logs_to_es",
    "aws_iam_role_policy_attachment.access_vpc_role_to_logs_to_es",
    "aws_cloudwatch_log_group.logs_to_es_log_group",
  ]
}

lambda 的代码是

#
# Create a lambda that will forward other lambda's logs
# to the VPC's elasticsearch, this whole configuration can be done
# manually though the "stream to elasticsearch" option when you
# choose where to stream a log group of cloudwatch
#

resource "aws_iam_role" "logs_to_es_lambda_role" {
  name = "logs_to_es_lambda_${local.env_type}"

  assume_role_policy = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Action": "sts:AssumeRole",
      "Principal": {
        "Service": "lambda.amazonaws.com"
      },
      "Effect": "Allow",
      "Sid": ""
    }
  ]
}
EOF
}

resource "aws_iam_role_policy_attachment" "attach_basic_role_to_logs_to_es" {
  role = aws_iam_role.logs_to_es_lambda_role.id
  # this policy permits to log in cloudwatch
  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
}

resource "aws_iam_role_policy_attachment" "access_vpc_role_to_logs_to_es" {
  role = aws_iam_role.logs_to_es_lambda_role.id
  # this policy permits the lambda to have access to the vpc (as the Elastic search
  # is only accessible from there
  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole"
}

resource "aws_cloudwatch_log_group" "logs_to_es_log_group" {
  name              = "/aws/lambda/logs_to_es_lambda_role_${local.env_type}"
  retention_in_days = 7
}

resource "aws_security_group" "logs_to_es" {
  description = "allow all egress network for the lambda logs_to_es"
  vpc_id      = module.vpc.vpc_id
  name        = "lambda-${local.env_type}-logs-to-es"

  egress {
    protocol    = "-1"
    from_port   = 0
    to_port     = 0
    cidr_blocks = ["0.0.0.0/0"]
  }

  tags = {
    Name               = "lambda-${local.env_type}-logs-to-es"
    ManagedByTerraform = "true"
    EnvironmentType    = "${local.env_type}"
  }
}


data "archive_file" "logs_to_es_zip" {
  type        = "zip"
  source_file = "lambdas/logs_to_elasticsearch/logs_to_elasticsearch.js"
  output_path = "lambdas/logs_to_elasticsearch/logs_to_elasticsearch.zip"
}

resource "aws_lambda_function" "logs_to_es" {

  filename         = data.archive_file.logs_to_es_zip.output_path
  source_code_hash = data.archive_file.logs_to_es_zip.output_base64sha256
  function_name    = "logs_to_es_${local.env_type}"

  role    = aws_iam_role.logs_to_es_lambda_role.arn
  handler = "logs_to_elasticsearch.handler"

  timeout = 100

  runtime = "nodejs10.x"

  description = "${local.env_type} script that forwards cloudwatch logs of that VPC's lambda to ES"

  vpc_config {
    subnet_ids = module.vpc.private_subnets
    security_group_ids = [
      aws_security_group.logs_to_es.id
    ]
  }

  environment {
    variables = {
      ELASTICSEARCH_ENDPOINT = aws_elasticsearch_domain.technical_logs.endpoint
    }
  }

  tags = {
    Environment        = local.env_type
    ManagedByTerraform = "true"
  }

  depends_on = [
    "aws_iam_role_policy_attachment.attach_basic_role_to_logs_to_es",
    "aws_iam_role_policy_attachment.access_vpc_role_to_logs_to_es",
    "aws_cloudwatch_log_group.logs_to_es_log_group",
  ]
}

然后要使用它，您只需将该 lambda ARN 放入 aws_cloudwatch_log_subscription_filter资源。