Skip to content

Commit

Permalink
AWS: add EKS cluster for kOps CI
Browse files Browse the repository at this point in the history
Related to:
  - kubernetes#5127

Add an EKS cluster that will be used as build cluster for kOps

Signed-off-by: Arnaud Meukam <[email protected]>
  • Loading branch information
ameukam committed Aug 8, 2023
1 parent 0e96fb2 commit 80c8e2f
Show file tree
Hide file tree
Showing 3 changed files with 202 additions and 6 deletions.
190 changes: 190 additions & 0 deletions infra/aws/terraform/kops-infra-ci/eks.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,190 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

module "eks" {
providers = { aws = aws.kops-infra-ci }
source = "terraform-aws-modules/eks/aws"
version = "19.16.0"

cluster_name = "${local.prefix}-prow-build"
cluster_version = var.eks_version
cluster_endpoint_public_access = true

cluster_ip_family = "ipv4"

vpc_id = module.vpc.vpc_id
subnet_ids = module.vpc.private_subnets
control_plane_subnet_ids = module.vpc.intra_subnets

cluster_enabled_log_types = [
"audit",
"authenticator",
"api",
"controllerManager",
"scheduler"
]

cloudwatch_log_group_retention_in_days = 30

cluster_addons = {
coredns = {
most_recent = true
resolve_conflicts = "OVERWRITE"
}
kube-proxy = {
most_recent = true
}
vpc-cni = {
most_recent = true
resolve_conflicts = "OVERWRITE"
service_account_role_arn = module.vpc_cni_irsa.iam_role_arn
}
aws-ebs-csi-driver = {
most_recent = true
resolve_conflicts = "OVERWRITE"
service_account_role_arn = module.ebs_csi_irsa.iam_role_arn
}
}

eks_managed_node_group_defaults = {
ami_type = "AL2_x86_64"
instance_types = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"]

iam_role_attach_cni_policy = true
}

eks_managed_node_groups = {
prow-build = {
name = "prow-build"
description = "EKS managed node group used to run kops jobs"
use_name_prefix = true

dataplane_wait_duration = "600s"

subnet_ids = module.vpc.private_subnets

min_size = 3
max_size = 100
desired_size = 3

# Force version update if existing pods are unable to be drained due to a PodDisruptionBudget issue
force_update_version = true
update_config = {
max_unavailable = 1
}

capacity_type = "ON_DEMAND"
instance_types = ["r6i.2xlarge"]
ami_type = "BOTTLEROCKET_x86_64"
platform = "bottlerocket"

ebs_optimized = true
enable_monitoring = true

block_device_mappings = {
xvda = {
device_name = "/dev/xvda"
ebs = {
volume_size = 100
volume_type = "gp3"
iops = "3000" #https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-optimized.html
encrypted = false
delete_on_termination = true
}
}
}

metadata_options = {
http_endpoint = "enabled"
http_tokens = "required"
http_put_response_hop_limit = 1
instance_metadata_tags = "enabled"
}

iam_role_additional_policies = {
AmazonEC2ContainerRegistryReadOnly = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
}

tags = merge(var.tags, {
"region" = "${data.aws_region.current.name}"
})
}
}

tags = merge(var.tags, {
"region" = "${data.aws_region.current.name}"
})
}


module "vpc_cni_irsa" {
providers = { aws = aws.kops-infra-ci }
source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
version = "~> 5.0"

role_name_prefix = "VPC-CNI-IRSA"
attach_vpc_cni_policy = true
vpc_cni_enable_ipv4 = true
# We use IPv4-based EKS cluster, so we don't need this
vpc_cni_enable_ipv6 = false

oidc_providers = {
main = {
provider_arn = module.eks.oidc_provider_arn
namespace_service_accounts = ["kube-system:aws-node"]
}
}

tags = var.tags
}

module "ebs_csi_irsa" {
providers = { aws = aws.kops-infra-ci }
source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
version = "~> 5.0"

role_name_prefix = "EBS-CSI-IRSA"
attach_ebs_csi_policy = true

oidc_providers = {
main = {
provider_arn = module.eks.oidc_provider_arn
namespace_service_accounts = ["kube-system:ebs-csi-controller-sa"]
}
}

tags = var.tags
}

module "cluster_autoscaler_irsa_role" {
providers = { aws = aws.kops-infra-ci }
source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
version = "~> 5.0"

role_name = "cluster-autoscaler"
attach_cluster_autoscaler_policy = true
cluster_autoscaler_cluster_ids = [module.eks.cluster_name]

oidc_providers = {
main = {
provider_arn = module.eks.oidc_provider_arn
namespace_service_accounts = ["kube-system:cluster-autoscaler"]
}
}

tags = var.tags
}
12 changes: 9 additions & 3 deletions infra/aws/terraform/kops-infra-ci/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,23 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

variable "eks_version" {
type = string
default = "1.26"
}

variable "tags" {
type = map(string)
default = {
"managed-by" = "Terraform",
"group" = "sig-cluster-lifecycle",
"managed-by" = "Terraform",
"group" = "sig-cluster-lifecycle",
"subproject" = "kops"
"githubRepo" = "git.k8s.io/k8s.io"
}
}

variable "region" {
type = string
type = string
default = "us-east-2"
}

Expand Down
6 changes: 3 additions & 3 deletions infra/aws/terraform/kops-infra-ci/vpc.tf
Original file line number Diff line number Diff line change
Expand Up @@ -48,14 +48,14 @@ resource "aws_vpc_ipam_pool" "main" {


resource "aws_vpc_ipam_pool_cidr" "main" {
provider = aws.kops-infra-ci
provider = aws.kops-infra-ci
ipam_pool_id = aws_vpc_ipam_pool.main.id
cidr = var.vpc_cidr
}

resource "aws_vpc_ipam_preview_next_cidr" "main" {
provider = aws.kops-infra-ci
ipam_pool_id = aws_vpc_ipam_pool.main.id
provider = aws.kops-infra-ci
ipam_pool_id = aws_vpc_ipam_pool.main.id

netmask_length = 20 // a 18 netmask length is considered as too big for the CIDR pool
}
Expand Down

0 comments on commit 80c8e2f

Please sign in to comment.