terraform-aws-kubernetes

3 years ago · 1f07dec79c
commit 1f07dec79c
24 changed files with 21171 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@ -0,0 +1,8 @@
 # Terraform state files
 *.tfstate
 *.tfstate.backup
 *.tfstate.*.backup
 *.tfstate.lock.info
 schojak.tfvars
 .terraform
 jakub.tfvars
--- a/201
+++ b/201
@ -0,0 +1,201 @@
                                 Apache License
                           Version 2.0, January 2004
                        http://www.apache.org/licenses/
   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
   1. Definitions.
      "License" shall mean the terms and conditions for use, reproduction,
      and distribution as defined by Sections 1 through 9 of this document.
      "Licensor" shall mean the copyright owner or entity authorized by
      the copyright owner that is granting the License.
      "Legal Entity" shall mean the union of the acting entity and all
      other entities that control, are controlled by, or are under common
      control with that entity. For the purposes of this definition,
      "control" means (i) the power, direct or indirect, to cause the
      direction or management of such entity, whether by contract or
      otherwise, or (ii) ownership of fifty percent (50%) or more of the
      outstanding shares, or (iii) beneficial ownership of such entity.
      "You" (or "Your") shall mean an individual or Legal Entity
      exercising permissions granted by this License.
      "Source" form shall mean the preferred form for making modifications,
      including but not limited to software source code, documentation
      source, and configuration files.
      "Object" form shall mean any form resulting from mechanical
      transformation or translation of a Source form, including but
      not limited to compiled object code, generated documentation,
      and conversions to other media types.
      "Work" shall mean the work of authorship, whether in Source or
      Object form, made available under the License, as indicated by a
      copyright notice that is included in or attached to the work
      (an example is provided in the Appendix below).
      "Derivative Works" shall mean any work, whether in Source or Object
      form, that is based on (or derived from) the Work and for which the
      editorial revisions, annotations, elaborations, or other modifications
      represent, as a whole, an original work of authorship. For the purposes
      of this License, Derivative Works shall not include works that remain
      separable from, or merely link (or bind by name) to the interfaces of,
      the Work and Derivative Works thereof.
      "Contribution" shall mean any work of authorship, including
      the original version of the Work and any modifications or additions
      to that Work or Derivative Works thereof, that is intentionally
      submitted to Licensor for inclusion in the Work by the copyright owner
      or by an individual or Legal Entity authorized to submit on behalf of
      the copyright owner. For the purposes of this definition, "submitted"
      means any form of electronic, verbal, or written communication sent
      to the Licensor or its representatives, including but not limited to
      communication on electronic mailing lists, source code control systems,
      and issue tracking systems that are managed by, or on behalf of, the
      Licensor for the purpose of discussing and improving the Work, but
      excluding communication that is conspicuously marked or otherwise
      designated in writing by the copyright owner as "Not a Contribution."
      "Contributor" shall mean Licensor and any individual or Legal Entity
      on behalf of whom a Contribution has been received by Licensor and
      subsequently incorporated within the Work.
   2. Grant of Copyright License. Subject to the terms and conditions of
      this License, each Contributor hereby grants to You a perpetual,
      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
      copyright license to reproduce, prepare Derivative Works of,
      publicly display, publicly perform, sublicense, and distribute the
      Work and such Derivative Works in Source or Object form.
   3. Grant of Patent License. Subject to the terms and conditions of
      this License, each Contributor hereby grants to You a perpetual,
      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
      (except as stated in this section) patent license to make, have made,
      use, offer to sell, sell, import, and otherwise transfer the Work,
      where such license applies only to those patent claims licensable
      by such Contributor that are necessarily infringed by their
      Contribution(s) alone or by combination of their Contribution(s)
      with the Work to which such Contribution(s) was submitted. If You
      institute patent litigation against any entity (including a
      cross-claim or counterclaim in a lawsuit) alleging that the Work
      or a Contribution incorporated within the Work constitutes direct
      or contributory patent infringement, then any patent licenses
      granted to You under this License for that Work shall terminate
      as of the date such litigation is filed.
   4. Redistribution. You may reproduce and distribute copies of the
      Work or Derivative Works thereof in any medium, with or without
      modifications, and in Source or Object form, provided that You
      meet the following conditions:
      (a) You must give any other recipients of the Work or
          Derivative Works a copy of this License; and
      (b) You must cause any modified files to carry prominent notices
          stating that You changed the files; and
      (c) You must retain, in the Source form of any Derivative Works
          that You distribute, all copyright, patent, trademark, and
          attribution notices from the Source form of the Work,
          excluding those notices that do not pertain to any part of
          the Derivative Works; and
      (d) If the Work includes a "NOTICE" text file as part of its
          distribution, then any Derivative Works that You distribute must
          include a readable copy of the attribution notices contained
          within such NOTICE file, excluding those notices that do not
          pertain to any part of the Derivative Works, in at least one
          of the following places: within a NOTICE text file distributed
          as part of the Derivative Works; within the Source form or
          documentation, if provided along with the Derivative Works; or,
          within a display generated by the Derivative Works, if and
          wherever such third-party notices normally appear. The contents
          of the NOTICE file are for informational purposes only and
          do not modify the License. You may add Your own attribution
          notices within Derivative Works that You distribute, alongside
          or as an addendum to the NOTICE text from the Work, provided
          that such additional attribution notices cannot be construed
          as modifying the License.
      You may add Your own copyright statement to Your modifications and
      may provide additional or different license terms and conditions
      for use, reproduction, or distribution of Your modifications, or
      for any such Derivative Works as a whole, provided Your use,
      reproduction, and distribution of the Work otherwise complies with
      the conditions stated in this License.
   5. Submission of Contributions. Unless You explicitly state otherwise,
      any Contribution intentionally submitted for inclusion in the Work
      by You to the Licensor shall be under the terms and conditions of
      this License, without any additional terms or conditions.
      Notwithstanding the above, nothing herein shall supersede or modify
      the terms of any separate license agreement you may have executed
      with Licensor regarding such Contributions.
   6. Trademarks. This License does not grant permission to use the trade
      names, trademarks, service marks, or product names of the Licensor,
      except as required for reasonable and customary use in describing the
      origin of the Work and reproducing the content of the NOTICE file.
   7. Disclaimer of Warranty. Unless required by applicable law or
      agreed to in writing, Licensor provides the Work (and each
      Contributor provides its Contributions) on an "AS IS" BASIS,
      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
      implied, including, without limitation, any warranties or conditions
      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
      PARTICULAR PURPOSE. You are solely responsible for determining the
      appropriateness of using or redistributing the Work and assume any
      risks associated with Your exercise of permissions under this License.
   8. Limitation of Liability. In no event and under no legal theory,
      whether in tort (including negligence), contract, or otherwise,
      unless required by applicable law (such as deliberate and grossly
      negligent acts) or agreed to in writing, shall any Contributor be
      liable to You for damages, including any direct, indirect, special,
      incidental, or consequential damages of any character arising as a
      result of this License or out of the use or inability to use the
      Work (including but not limited to damages for loss of goodwill,
      work stoppage, computer failure or malfunction, or any and all
      other commercial damages or losses), even if such Contributor
      has been advised of the possibility of such damages.
   9. Accepting Warranty or Additional Liability. While redistributing
      the Work or Derivative Works thereof, You may choose to offer,
      and charge a fee for, acceptance of support, warranty, indemnity,
      or other liability obligations and/or rights consistent with this
      License. However, in accepting such obligations, You may act only
      on Your own behalf and on Your sole responsibility, not on behalf
      of any other Contributor, and only if You agree to indemnify,
      defend, and hold each Contributor harmless for any liability
      incurred by, or claims asserted against, such Contributor by reason
      of your accepting any such warranty or additional liability.
   END OF TERMS AND CONDITIONS
   APPENDIX: How to apply the Apache License to your work.
      To apply the Apache License to your work, attach the following
      boilerplate notice, with the fields enclosed by brackets "{}"
      replaced with your own identifying information. (Don't include
      the brackets!)  The text should be enclosed in the appropriate
      comment syntax for the file format. We also recommend that a
      file or class name and description of purpose be included on the
      same "printed page" as the copyright notice for easier
      identification within third-party archives.
   Copyright 2017 Jakub Scholz
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at
       http://www.apache.org/licenses/LICENSE-2.0
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
--- a/README.md
+++ b/README.md
@ -0,0 +1,110 @@
 # AWS Kubernetes
 AWS Kubernetes is a Kubernetes cluster deployed using [Kubeadm](https://kubernetes.io/docs/admin/kubeadm/) tool. It provides full integration with AWS. It is able to handle ELB load balancers, EBS disks, Route53 domains etc.
 <!-- TOC -->
 - [AWS Kubernetes](#aws-kubernetes)
    - [Updates](#updates)
    - [Prerequisites and dependencies](#prerequisites-and-dependencies)
    - [Including the module](#including-the-module)
    - [Add-ons](#addons)
    - [Custom add-ons](#custom-addons)
    - [Tagging](#tagging)
 <!-- /TOC -->
 ## Updates
 * *2.10.2022* Update to Kubernetes 1.25.2 + update add-ons
 * *26.8.2022* Update to Kubernetes 1.25.0 + Calico upgrade
 * *22.8.2022* Update to Kubernetes 1.24.4
 * *16.7.2022* Update to Kubernetes 1.24.3
 * *27.6.2022* Update to Kubernetes 1.24.2
 * *11.6.2022* Update to Kubernetes 1.24.1 + update add-ons + remove dependency on the template provider
 * *8.5.2022* Update to Kubernetes 1.24.0 + update add-ons
 * *23.3.2022* Update to Kubernetes 1.23.5 + update add-ons
 * *19.2.2022* Update to Kubernetes 1.23.4
 * *12.2.2022* Update to Kubernetes 1.23.2
 * *29.12.2021* Update to Kubernetes 1.23.1
 * *11.12.2021* Update to Kubernetes 1.23.0
 ## Prerequisites and dependencies
 * AWS Kubernetes deploys into existing VPC / public subnet. If you don't have your VPC / subnet yet, you can use [this](https://github.com/scholzj/terraform-aws-vpc) module to create one.
  * The VPC / subnet should be properly linked with Internet Gateway (IGW) and should have DNS and DHCP enabled.
  * Hosted DNS zone configured in Route53 (in case the zone is private you have to use IP address to copy `kubeconfig` and access the cluster).
 * To deploy AWS Kubernetes there are no other dependencies apart from [Terraform](https://www.terraform.io). Kubeadm is used only on the EC2 hosts and doesn't have to be installed locally.
 ## Including the module
 Although it can be run on its own, the main value is that it can be included into another Terraform configuration.
 ```hcl
 module "kubernetes" {
  source = "scholzj/kubernetes/aws"
  aws_region    = "eu-central-1"
  cluster_name  = "aws-kubernetes"
  master_instance_type = "t2.medium"
  worker_instance_type = "t2.medium"
  ssh_public_key = "~/.ssh/id_rsa.pub"
  ssh_access_cidr = ["0.0.0.0/0"]
  api_access_cidr = ["0.0.0.0/0"]
  min_worker_count = 3
  max_worker_count = 6
  hosted_zone = "my-domain.com"
  hosted_zone_private = false
  master_subnet_id = "subnet-8a3517f8"
  worker_subnet_ids = [		
      "subnet-8a3517f8",
      "subnet-9b7853f7",
      "subnet-8g9sdfv8"
  ]
  # Tags
  tags = {
    Application = "AWS-Kubernetes"
  }
  # Tags in a different format for Auto Scaling Group
  tags2 = [
    {
      key                 = "Application"
      value               = "AWS-Kubernetes"
      propagate_at_launch = true
    }
  ]
  addons = [
    "https://raw.githubusercontent.com/scholzj/terraform-aws-kubernetes/master/addons/storage-class.yaml",
    "https://raw.githubusercontent.com/scholzj/terraform-aws-kubernetes/master/addons/heapster.yaml",
    "https://raw.githubusercontent.com/scholzj/terraform-aws-kubernetes/master/addons/dashboard.yaml",
    "https://raw.githubusercontent.com/scholzj/terraform-aws-kubernetes/master/addons/external-dns.yaml",
    "https://raw.githubusercontent.com/scholzj/terraform-aws-kubernetes/master/addons/autoscaler.yaml"
  ]
 }
 ```
 An example of how to include this can be found in the [examples](examples/) dir.
 ## Add-ons
 Currently, following add-ons are supported:
 * Kubernetes dashboard
 * Heapster for resource monitoring
 * Storage class and CSI driver for automatic provisioning of persistent volumes
 * External DNS (Replaces Route53 mapper)
 * Ingress
 * Autoscaler
 The add-ons will be installed automatically based on the Terraform variables. 
 ## Custom add-ons
 Custom add-ons can be added if needed. For every URL in the `addons` list, the initialization scripts will automatically call `kubectl -f apply <Addon URL>` to deploy it. The cluster is using RBAC. So the custom add-ons have to be *RBAC ready*.
 ## Tagging
 If you need to tag resources created by your Kubernetes cluster (EBS volumes, ELB load balancers etc.) check [this AWS Lambda function which can do the tagging](https://github.com/scholzj/aws-kubernetes-tagging-lambda).
--- a/addons/autoscaler.yaml
+++ b/addons/autoscaler.yaml
@ -0,0 +1,207 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: cluster-autoscaler
  namespace: kube-system
  labels:
    k8s-addon: cluster-autoscaler.addons.k8s.io
    k8s-app: cluster-autoscaler
 spec:
  replicas: 1
  selector:
    matchLabels:
      k8s-app: cluster-autoscaler
  template:
    metadata:
      labels:
        k8s-addon: cluster-autoscaler.addons.k8s.io
        k8s-app: cluster-autoscaler
      annotations:
        scheduler.alpha.kubernetes.io/tolerations: '[{"key":"dedicated", "value":"master"}]'
    spec:
      serviceAccountName: cluster-autoscaler
      containers:
        - name: cluster-autoscaler
          image: k8s.gcr.io/autoscaling/cluster-autoscaler:v1.21.0
          resources:
            limits:
              cpu: 100m
              memory: 300Mi
            requests:
              cpu: 100m
              memory: 300Mi
          command:
            - ./cluster-autoscaler
            - --cloud-provider=aws
            - --skip-nodes-with-local-storage=false
            - --nodes=${ASG_MIN_NODES}:${ASG_MAX_NODES}:${ASG_NAME}
            - --alsologtostderr
          env:
            - name: AWS_REGION
              value: ${AWS_REGION}
          volumeMounts:
            - name: ssl-certs
              mountPath: /etc/ssl/certs/ca-certificates.crt
              readOnly: true
      volumes:
        - name: ssl-certs
          hostPath:
            path: /etc/ssl/certs/ca-certificates.crt
      nodeSelector:
        node-role.kubernetes.io/control-plane: ""
      tolerations:
        - key: "node-role.kubernetes.io/master"
          effect: NoSchedule
 ---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
  labels:
    k8s-addon: cluster-autoscaler.addons.k8s.io
    k8s-app: cluster-autoscaler
  name: cluster-autoscaler
  namespace: kube-system
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
  name: cluster-autoscaler
  labels:
    k8s-addon: cluster-autoscaler.addons.k8s.io
    k8s-app: cluster-autoscaler
 rules:
 - apiGroups:
  - ""
  resources:
  - events
  - endpoints
  verbs:
  - create
  - patch
 - apiGroups:
  - ""
  resources:
  - endpoints
  resourceNames:
  - cluster-autoscaler
  verbs:
  - get
  - update
 - apiGroups:
  - ""
  resources:
  - pods
  - services
  - replicasets
  - replicationcontrollers
  - persistentvolumeclaims
  - persistentvolumes
  verbs:
  - get
  - watch
  - list
 - apiGroups:
  - ""
  resources:
  - nodes
  verbs:
  - get
  - watch
  - list
  - update
 - apiGroups:
  - ""
  resources:
  - pods/status
  verbs:
  - update
 - apiGroups:
  - extensions
  resources:
  - replicasets
  - daemonsets
  verbs:
  - watch
  - list
 - apiGroups:
  - apps
  resources:
  - statefulsets
  verbs:
  - watch
  - list
 - apiGroups:
  - policy
  resources:
  - poddisruptionbudgets
  verbs:
  - watch
  - list
 - apiGroups: 
  - storage.k8s.io
  resources: 
  - storageclasses
  verbs: 
  - get
  - list
  - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
  name: cluster-autoscaler
  namespace: kube-system
  labels:
    k8s-addon: cluster-autoscaler.addons.k8s.io
    k8s-app: cluster-autoscaler
 rules:
 - apiGroups:
  - ""
  resources:
  - configmaps
  verbs:
  - create
 - apiGroups:
  - ""
  resources:
  - configmaps
  resourceNames:
  - cluster-autoscaler-status
  verbs:
  - delete
  - get
  - update
  - patch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
  name: cluster-autoscaler
  labels:
    k8s-addon: cluster-autoscaler.addons.k8s.io
    k8s-app: cluster-autoscaler
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-autoscaler
 subjects:
  - kind: ServiceAccount
    name: cluster-autoscaler
    namespace: kube-system
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: cluster-autoscaler
  namespace: kube-system
  labels:
    k8s-addon: cluster-autoscaler.addons.k8s.io
    k8s-app: cluster-autoscaler
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: cluster-autoscaler
 subjects:
  - kind: ServiceAccount
    name: cluster-autoscaler
    namespace: kube-system
--- a/addons/csi-driver.yaml
+++ b/addons/csi-driver.yaml
@ -0,0 +1,526 @@
 ---
 # Source: aws-ebs-csi-driver/templates/serviceaccount-csi-node.yaml
 apiVersion: v1
 kind: ServiceAccount
 metadata:
  name: ebs-csi-node-sa
  namespace: kube-system
  labels:
    app.kubernetes.io/name: aws-ebs-csi-driver
 ---
 # Source: aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml
 apiVersion: v1
 kind: ServiceAccount
 metadata:
  name: ebs-csi-controller-sa
  namespace: kube-system
  labels:
    app.kubernetes.io/name: aws-ebs-csi-driver
  #Enable if EKS IAM for SA is used
  #annotations:
  #  eks.amazonaws.com/role-arn: arn:aws:iam::586565787010:role/ebs-csi-role
 ---
 # Source: aws-ebs-csi-driver/templates/clusterrole-attacher.yaml
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
  name: ebs-external-attacher-role
  labels:
    app.kubernetes.io/name: aws-ebs-csi-driver
 rules:
  - apiGroups: [ "" ]
    resources: [ "persistentvolumes" ]
    verbs: [ "get", "list", "watch", "update", "patch" ]
  - apiGroups: [ "" ]
    resources: [ "nodes" ]
    verbs: [ "get", "list", "watch" ]
  - apiGroups: [ "csi.storage.k8s.io" ]
    resources: [ "csinodeinfos" ]
    verbs: [ "get", "list", "watch" ]
  - apiGroups: [ "storage.k8s.io" ]
    resources: [ "volumeattachments" ]
    verbs: [ "get", "list", "watch", "update", "patch" ]
  - apiGroups: [ "storage.k8s.io" ]
    resources: [ "volumeattachments/status" ]
    verbs: [ "patch" ]
 ---
 # Source: aws-ebs-csi-driver/templates/clusterrole-csi-node.yaml
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
  name: ebs-csi-node-role
  labels:
    app.kubernetes.io/name: aws-ebs-csi-driver
 rules:
  - apiGroups: [""]
    resources: ["nodes"]
    verbs: ["get"]
 ---
 # Source: aws-ebs-csi-driver/templates/clusterrole-provisioner.yaml
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
  name: ebs-external-provisioner-role
  labels:
    app.kubernetes.io/name: aws-ebs-csi-driver
 rules:
  - apiGroups: [ "" ]
    resources: [ "persistentvolumes" ]
    verbs: [ "get", "list", "watch", "create", "delete" ]
  - apiGroups: [ "" ]
    resources: [ "persistentvolumeclaims" ]
    verbs: [ "get", "list", "watch", "update" ]
  - apiGroups: [ "storage.k8s.io" ]
    resources: [ "storageclasses" ]
    verbs: [ "get", "list", "watch" ]
  - apiGroups: [ "" ]
    resources: [ "events" ]
    verbs: [ "list", "watch", "create", "update", "patch" ]
  - apiGroups: [ "snapshot.storage.k8s.io" ]
    resources: [ "volumesnapshots" ]
    verbs: [ "get", "list" ]
  - apiGroups: [ "snapshot.storage.k8s.io" ]
    resources: [ "volumesnapshotcontents" ]
    verbs: [ "get", "list" ]
  - apiGroups: [ "storage.k8s.io" ]
    resources: [ "csinodes" ]
    verbs: [ "get", "list", "watch" ]
  - apiGroups: [ "" ]
    resources: [ "nodes" ]
    verbs: [ "get", "list", "watch" ]
  - apiGroups: [ "coordination.k8s.io" ]
    resources: [ "leases" ]
    verbs: [ "get", "watch", "list", "delete", "update", "create" ]
  - apiGroups: [ "storage.k8s.io" ]
    resources: [ "volumeattachments" ]
    verbs: [ "get", "list", "watch" ]
 ---
 # Source: aws-ebs-csi-driver/templates/clusterrole-resizer.yaml
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
  name: ebs-external-resizer-role
  labels:
    app.kubernetes.io/name: aws-ebs-csi-driver
 rules:
  # The following rule should be uncommented for plugins that require secrets
  # for provisioning.
  # - apiGroups: [""]
  #   resources: ["secrets"]
  #   verbs: ["get", "list", "watch"]
  - apiGroups: [ "" ]
    resources: [ "persistentvolumes" ]
    verbs: [ "get", "list", "watch", "update", "patch" ]
  - apiGroups: [ "" ]
    resources: [ "persistentvolumeclaims" ]
    verbs: [ "get", "list", "watch" ]
  - apiGroups: [ "" ]
    resources: [ "persistentvolumeclaims/status" ]
    verbs: [ "update", "patch" ]
  - apiGroups: [ "storage.k8s.io" ]
    resources: [ "storageclasses" ]
    verbs: [ "get", "list", "watch" ]
  - apiGroups: [ "" ]
    resources: [ "events" ]
    verbs: [ "list", "watch", "create", "update", "patch" ]
  - apiGroups: [ "" ]
    resources: [ "pods" ]
    verbs: [ "get", "list", "watch" ]
 ---
 # Source: aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
  name: ebs-external-snapshotter-role
  labels:
    app.kubernetes.io/name: aws-ebs-csi-driver
 rules:
  - apiGroups: [ "" ]
    resources: [ "events" ]
    verbs: [ "list", "watch", "create", "update", "patch" ]
  - apiGroups: [ "" ]
    resources: [ "secrets" ]
    verbs: [ "get", "list" ]
  - apiGroups: [ "snapshot.storage.k8s.io" ]
    resources: [ "volumesnapshotclasses" ]
    verbs: [ "get", "list", "watch" ]
  - apiGroups: [ "snapshot.storage.k8s.io" ]
    resources: [ "volumesnapshotcontents" ]
    verbs: [ "create", "get", "list", "watch", "update", "delete" ]
  - apiGroups: [ "snapshot.storage.k8s.io" ]
    resources: [ "volumesnapshotcontents/status" ]
    verbs: [ "update" ]
 ---
 # Source: aws-ebs-csi-driver/templates/clusterrolebinding-attacher.yaml
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
  name: ebs-csi-attacher-binding
  labels:
    app.kubernetes.io/name: aws-ebs-csi-driver
 subjects:
  - kind: ServiceAccount
    name: ebs-csi-controller-sa
    namespace: kube-system
 roleRef:
  kind: ClusterRole
  name: ebs-external-attacher-role
  apiGroup: rbac.authorization.k8s.io
 ---
 # Source: aws-ebs-csi-driver/templates/clusterrolebinding-csi-node.yaml
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
  name: ebs-csi-node-getter-binding
  labels:
    app.kubernetes.io/name: aws-ebs-csi-driver
 subjects:
  - kind: ServiceAccount
    name: ebs-csi-node-sa
    namespace: kube-system
 roleRef:
  kind: ClusterRole
  name: ebs-csi-node-role
  apiGroup: rbac.authorization.k8s.io
 ---
 # Source: aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
  name: ebs-csi-resizer-binding
  labels:
    app.kubernetes.io/name: aws-ebs-csi-driver
 subjects:
  - kind: ServiceAccount
    name: ebs-csi-controller-sa
    namespace: kube-system
 roleRef:
  kind: ClusterRole
  name: ebs-external-resizer-role
  apiGroup: rbac.authorization.k8s.io
 ---
 # Source: aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
  name: ebs-csi-snapshotter-binding
  labels:
    app.kubernetes.io/name: aws-ebs-csi-driver
 subjects:
  - kind: ServiceAccount
    name: ebs-csi-controller-sa
    namespace: kube-system
 roleRef:
  kind: ClusterRole
  name: ebs-external-snapshotter-role
  apiGroup: rbac.authorization.k8s.io
 ---
 # Source: aws-ebs-csi-driver/templates/clusterrolebinding-provisioner.yaml
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
  name: ebs-csi-provisioner-binding
  labels:
    app.kubernetes.io/name: aws-ebs-csi-driver
 subjects:
  - kind: ServiceAccount
    name: ebs-csi-controller-sa
    namespace: kube-system
 roleRef:
  kind: ClusterRole
  name: ebs-external-provisioner-role
  apiGroup: rbac.authorization.k8s.io
 ---
 # Source: aws-ebs-csi-driver/templates/csidriver.yaml
 apiVersion: storage.k8s.io/v1
 kind: CSIDriver
 metadata:
  name: ebs.csi.aws.com
  labels:
    app.kubernetes.io/name: aws-ebs-csi-driver
 spec:
  attachRequired: true
  podInfoOnMount: false
 ---
 # Source: aws-ebs-csi-driver/templates/controller.yaml
 # Controller Service
 kind: Deployment
 apiVersion: apps/v1
 metadata:
  name: ebs-csi-controller
  namespace: kube-system
  labels:
    app.kubernetes.io/name: aws-ebs-csi-driver
 spec:
  replicas: 1
  selector:
    matchLabels:
      app: ebs-csi-controller
      app.kubernetes.io/name: aws-ebs-csi-driver
  template:
    metadata:
      labels:
        app: ebs-csi-controller
        app.kubernetes.io/name: aws-ebs-csi-driver
    spec:
      nodeSelector:
        node-role.kubernetes.io/control-plane: ""
      tolerations:
        - key: "node-role.kubernetes.io/master"
          effect: NoSchedule
      serviceAccountName: ebs-csi-controller-sa
      priorityClassName: system-cluster-critical
      containers:
        - name: ebs-plugin
          image: public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver:v1.6.2
          imagePullPolicy: IfNotPresent
          args:
            # - {all,controller,node} # specify the driver mode
            - --endpoint=$(CSI_ENDPOINT)
            - --logtostderr
            - --v=2
          env:
            - name: CSI_ENDPOINT
              value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
            - name: CSI_NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            # - name: AWS_ACCESS_KEY_ID
            #   valueFrom:
            #     secretKeyRef:
            #       name: aws-secret
            #       key: key_id
            #       optional: true
            # - name: AWS_SECRET_ACCESS_KEY
            #   valueFrom:
            #     secretKeyRef:
            #       name: aws-secret
            #       key: access_key
            #       optional: true
          volumeMounts:
            - name: socket-dir
              mountPath: /var/lib/csi/sockets/pluginproxy/
          ports:
            - name: healthz
              containerPort: 9808
              protocol: TCP
          livenessProbe:
            httpGet:
              path: /healthz
              port: healthz
            initialDelaySeconds: 10
            timeoutSeconds: 3
            periodSeconds: 10
            failureThreshold: 5
          readinessProbe:
            httpGet:
              path: /healthz
              port: healthz
            initialDelaySeconds: 10
            timeoutSeconds: 3
            periodSeconds: 10
            failureThreshold: 5
        - name: csi-provisioner
          image: k8s.gcr.io/sig-storage/csi-provisioner:v2.1.1
          imagePullPolicy: IfNotPresent
          args:
            - --csi-address=$(ADDRESS)
            - --v=2
            - --feature-gates=Topology=true
            - --extra-create-metadata
            - --leader-election=true
            - --default-fstype=ext4
          env:
            - name: ADDRESS
              value: /var/lib/csi/sockets/pluginproxy/csi.sock
          volumeMounts:
            - name: socket-dir
              mountPath: /var/lib/csi/sockets/pluginproxy/
        - name: csi-attacher
          image: k8s.gcr.io/sig-storage/csi-attacher:v3.1.0
          imagePullPolicy: IfNotPresent
          args:
            - --csi-address=$(ADDRESS)
            - --v=2
            - --leader-election=true
          env:
            - name: ADDRESS
              value: /var/lib/csi/sockets/pluginproxy/csi.sock
          volumeMounts:
            - name: socket-dir
              mountPath: /var/lib/csi/sockets/pluginproxy/
        - name: csi-snapshotter
          image: k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.3
          imagePullPolicy: IfNotPresent
          args:
            - --csi-address=$(ADDRESS)
            - --leader-election=true
          env:
            - name: ADDRESS
              value: /var/lib/csi/sockets/pluginproxy/csi.sock
          volumeMounts:
            - name: socket-dir
              mountPath: /var/lib/csi/sockets/pluginproxy/
        - name: csi-resizer
          image: k8s.gcr.io/sig-storage/csi-resizer:v1.1.0
          imagePullPolicy: IfNotPresent
          args:
            - --csi-address=$(ADDRESS)
            - --v=2
          env:
            - name: ADDRESS
              value: /var/lib/csi/sockets/pluginproxy/csi.sock
          volumeMounts:
            - name: socket-dir
              mountPath: /var/lib/csi/sockets/pluginproxy/
        - name: liveness-probe
          image: k8s.gcr.io/sig-storage/livenessprobe:v2.4.0
          imagePullPolicy: IfNotPresent
          args:
            - --csi-address=/csi/csi.sock
          volumeMounts:
            - name: socket-dir
              mountPath: /csi
      volumes:
        - name: socket-dir
          emptyDir: {}
 ---
 # Source: aws-ebs-csi-driver/templates/node.yaml
 # Node Service
 kind: DaemonSet
 apiVersion: apps/v1
 metadata:
  name: ebs-csi-node
  namespace: kube-system
  labels:
    app.kubernetes.io/name: aws-ebs-csi-driver
 spec:
  selector:
    matchLabels:
      app: ebs-csi-node
      app.kubernetes.io/name: aws-ebs-csi-driver
  updateStrategy:
    rollingUpdate:
      maxUnavailable: 10%
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: ebs-csi-node
        app.kubernetes.io/name: aws-ebs-csi-driver
    spec:
      serviceAccountName: ebs-csi-node-sa
      priorityClassName: system-node-critical
      containers:
        - name: ebs-plugin
          securityContext:
            privileged: true
          image: public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver:v1.6.2
          imagePullPolicy: IfNotPresent
          args:
            - node
            - --endpoint=$(CSI_ENDPOINT)
            - --logtostderr
            - --v=2
          env:
            - name: CSI_ENDPOINT
              value: unix:/csi/csi.sock
            - name: CSI_NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
          volumeMounts:
            - name: kubelet-dir
              mountPath: /var/lib/kubelet
              mountPropagation: "Bidirectional"
            - name: plugin-dir
              mountPath: /csi
            - name: device-dir
              mountPath: /dev
          ports:
            - name: healthz
              containerPort: 9808
              protocol: TCP
          livenessProbe:
            httpGet:
              path: /healthz
              port: healthz
            initialDelaySeconds: 10
            timeoutSeconds: 3
            periodSeconds: 10
            failureThreshold: 5
        - name: node-driver-registrar
          image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.1.0
          imagePullPolicy: IfNotPresent
          args:
            - --csi-address=$(ADDRESS)
            - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
            - --v=2
          env:
            - name: ADDRESS
              value: /csi/csi.sock
            - name: DRIVER_REG_SOCK_PATH
              value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock
          volumeMounts:
            - name: plugin-dir
              mountPath: /csi
            - name: registration-dir
              mountPath: /registration
        - name: liveness-probe
          image: k8s.gcr.io/sig-storage/livenessprobe:v2.4.0
          imagePullPolicy: IfNotPresent
          args:
            - --csi-address=/csi/csi.sock
          volumeMounts:
            - name: plugin-dir
              mountPath: /csi
      volumes:
        - name: kubelet-dir
          hostPath:
            path: /var/lib/kubelet
            type: Directory
        - name: plugin-dir
          hostPath:
            path: /var/lib/kubelet/plugins/ebs.csi.aws.com/
            type: DirectoryOrCreate
        - name: registration-dir
          hostPath:
            path: /var/lib/kubelet/plugins_registry/
            type: Directory
        - name: device-dir
          hostPath:
            path: /dev
            type: Directory
 ---
 # Source: aws-ebs-csi-driver/templates/poddisruptionbudget-controller.yaml
 apiVersion: policy/v1
 kind: PodDisruptionBudget
 metadata:
  name: ebs-csi-controller
  namespace: kube-system
  labels:
    app.kubernetes.io/name: aws-ebs-csi-driver
 spec:
  selector:
    matchLabels:
      app: ebs-csi-controller
      app.kubernetes.io/name: aws-ebs-csi-driver
  maxUnavailable: 1
--- a/addons/dashboard.yaml
+++ b/addons/dashboard.yaml
@ -0,0 +1,226 @@
 # Copyright 2017 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 apiVersion: v1
 kind: ServiceAccount
 metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kube-system
 ---
 kind: Service
 apiVersion: v1
 metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kube-system
 spec:
  ports:
    - port: 443
      targetPort: 8443
  selector:
    k8s-app: kubernetes-dashboard
 ---
 apiVersion: v1
 kind: Secret
 metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard-certs
  namespace: kube-system
 type: Opaque
 ---
 apiVersion: v1
 kind: Secret
 metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard-csrf
  namespace: kube-system
 type: Opaque
 data:
  csrf: ""
 ---
 apiVersion: v1
 kind: Secret
 metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard-key-holder
  namespace: kube-system
 type: Opaque
 ---
 kind: ConfigMap
 apiVersion: v1
 metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard-settings
  namespace: kube-system
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
  name: kubernetes-dashboard
  namespace: kube-system
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
 subjects:
  - kind: ServiceAccount
    name: kubernetes-dashboard
    namespace: kube-system
 ---
 kind: Deployment
 apiVersion: apps/v1
 metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kube-system
 spec:
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      k8s-app: kubernetes-dashboard
  template:
    metadata:
      labels:
        k8s-app: kubernetes-dashboard
    spec:
      containers:
        - name: kubernetes-dashboard
          image: kubernetesui/dashboard:v2.7.0
          imagePullPolicy: Always
          ports:
            - containerPort: 8443
              protocol: TCP
          args:
            - --auto-generate-certificates
            - --enable-skip-login
            - --disable-settings-authorizer
            # Uncomment the following line to manually specify Kubernetes API server Host
            # If not specified, Dashboard will attempt to auto discover the API server and connect
            # to it. Uncomment only if the default does not work.
            # - --apiserver-host=http://my-address:port
          volumeMounts:
            - name: kubernetes-dashboard-certs
              mountPath: /certs
              # Create on-disk volume to store exec logs
            - mountPath: /tmp
              name: tmp-volume
          livenessProbe:
            httpGet:
              scheme: HTTPS
              path: /
              port: 8443
            initialDelaySeconds: 30
            timeoutSeconds: 30
      volumes:
        - name: kubernetes-dashboard-certs
          secret:
            secretName: kubernetes-dashboard-certs
        - name: tmp-volume
          emptyDir: {}
      serviceAccountName: kubernetes-dashboard
      # Comment the following tolerations if Dashboard must not be deployed on master
      tolerations:
        - key: node-role.kubernetes.io/master
          effect: NoSchedule
 ---
 kind: Service
 apiVersion: v1
 metadata:
  labels:
    k8s-app: dashboard-metrics-scraper
  name: dashboard-metrics-scraper
  namespace: kube-system
 spec:
  ports:
    - port: 8000
      targetPort: 8000
  selector:
    k8s-app: dashboard-metrics-scraper
 ---
 kind: Deployment
 apiVersion: apps/v1
 metadata:
  labels:
    k8s-app: dashboard-metrics-scraper
  name: dashboard-metrics-scraper
  namespace: kube-system
 spec:
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      k8s-app: dashboard-metrics-scraper
  template:
    metadata:
      labels:
        k8s-app: dashboard-metrics-scraper
    spec:
      containers:
        - name: dashboard-metrics-scraper
          image: kubernetesui/metrics-scraper:v1.0.8
          ports:
            - containerPort: 8000
              protocol: TCP
          livenessProbe:
            httpGet:
              scheme: HTTP
              path: /
              port: 8000
            initialDelaySeconds: 30
            timeoutSeconds: 30
          volumeMounts:
          - mountPath: /tmp
            name: tmp-volume
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            runAsUser: 1001
            runAsGroup: 2001
      serviceAccountName: kubernetes-dashboard
      nodeSelector:
        "kubernetes.io/os": linux
      # Comment the following tolerations if Dashboard must not be deployed on master
      tolerations:
        - key: node-role.kubernetes.io/master
          effect: NoSchedule
      volumes:
        - name: tmp-volume
          emptyDir: {}
--- a/addons/external-dns.yaml
+++ b/addons/external-dns.yaml
@ -0,0 +1,89 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
  labels:
    k8s-app: external-dns
    k8s-addon: external-dns.alpha.kubernetes.io
  name: external-dns
  namespace: kube-system
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
  name: external-dns
 rules:
 - apiGroups: [""]
  resources: ["services"]
  verbs: ["get","watch","list"]
 - apiGroups: [""]
  resources: ["endpoints"]
  verbs: ["get","watch","list"]
 - apiGroups: [""]
  resources: ["pods"]
  verbs: ["get","watch","list"]
 - apiGroups: ["networking.k8s.io"] 
  resources: ["ingresses"] 
  verbs: ["get","watch","list"]
 - apiGroups: ["extensions"] 
  resources: ["ingresses"] 
  verbs: ["get","watch","list"]
 - apiGroups: [""]
  resources: ["nodes"]
  verbs: ["get","watch","list"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
  name: external-dns
  labels:
    k8s-app: external-dns
    k8s-addon: external-dns.alpha.kubernetes.io
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: external-dns
 subjects:
 - kind: ServiceAccount
  name: external-dns
  namespace: kube-system
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: external-dns
  namespace: kube-system
  labels:
    app: external-dns
    k8s-addon: external-dns.alpha.kubernetes.io
 spec:
  strategy:
    type: Recreate
  replicas: 1
  selector:
    matchLabels:
      app: external-dns
      k8s-addon: external-dns.alpha.kubernetes.io
  template:
    metadata:
      labels:
        app: external-dns
        k8s-addon: external-dns.alpha.kubernetes.io
    spec:
      serviceAccountName: external-dns
      containers:
      - name: external-dns
        image: k8s.gcr.io/external-dns/external-dns:v0.12.2
        args:
        - --source=service
        - --source=ingress
        - --provider=aws
        - --registry=txt
        - --txt-owner-id=${CLUSTER_NAME}
        - --interval=5m # Default value (1m) causes "Rate Exceed" on Route 53 due to too many API call
        #- --domain-filter={{ dns_zone }}. # will make ExternalDNS see only the hosted zones matching provided domain, omit to process all available hosted zones
        #- --policy=upsert-only # would prevent ExternalDNS from deleting any records, omit to enable full synchronization
      nodeSelector:
        node-role.kubernetes.io/control-plane: ""
      tolerations:
        - key: "node-role.kubernetes.io/master"
          effect: NoSchedule
--- a/addons/heapster.yaml
+++ b/addons/heapster.yaml
@ -0,0 +1,68 @@
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
  name: heapster
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:heapster
 subjects:
 - kind: ServiceAccount
  name: heapster
  namespace: kube-system
 ---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
  name: heapster
  namespace: kube-system
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: heapster
  namespace: kube-system
 spec:
  replicas: 1
  selector:
    matchLabels:
      task: monitoring
      k8s-app: heapster
  template:
    metadata:
      labels:
        task: monitoring
        k8s-app: heapster
    spec:
      serviceAccountName: heapster
      containers:
      - name: heapster
        image: gcr.io/google_containers/heapster-amd64:v1.5.4
        imagePullPolicy: IfNotPresent
        command:
        - /heapster
        - --source=kubernetes:https://kubernetes.default
      nodeSelector:
        node-role.kubernetes.io/master: ""
      tolerations:
        - key: "node-role.kubernetes.io/master"
          effect: NoSchedule
 ---
 apiVersion: v1
 kind: Service
 metadata:
  labels:
    task: monitoring
    # For use as a Cluster add-on (https://github.com/kubernetes/kubernetes/tree/master/cluster/addons)
    # If you are NOT using this as an addon, you should comment out this line.
    kubernetes.io/cluster-service: 'true'
    kubernetes.io/name: Heapster
  name: heapster
  namespace: kube-system
 spec:
  ports:
  - port: 80
    targetPort: 8082
  selector:
    k8s-app: heapster
--- a/addons/ingress.yaml
+++ b/addons/ingress.yaml
@ -0,0 +1,416 @@
 ---
 apiVersion: v1
 kind: Namespace
 metadata:
  name: ingress-nginx
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: default-http-backend
  labels:
    app: default-http-backend
  namespace: ingress-nginx
 spec:
  replicas: 1
  selector:
    matchLabels:
      app: default-http-backend
  template:
    metadata:
      labels:
        app: default-http-backend
    spec:
      terminationGracePeriodSeconds: 60
      containers:
      - name: default-http-backend
        # Any image is permissible as long as:
        # 1. It serves a 404 page at /
        # 2. It serves 200 on a /healthz endpoint
        image: registry.k8s.io/defaultbackend-amd64:1.5
        livenessProbe:
          httpGet:
            path: /healthz
            port: 8080
            scheme: HTTP
          initialDelaySeconds: 30
          timeoutSeconds: 5
        ports:
        - containerPort: 8080
        resources:
          limits:
            cpu: 10m
            memory: 20Mi
          requests:
            cpu: 10m
            memory: 20Mi
 ---
 apiVersion: v1
 kind: Service
 metadata:
  name: default-http-backend
  namespace: ingress-nginx
  labels:
    app: default-http-backend
 spec:
  ports:
  - port: 80
    targetPort: 8080
  selector:
    app: default-http-backend
 ---
 kind: ConfigMap
 apiVersion: v1
 metadata:
  name: nginx-configuration
  namespace: ingress-nginx
  labels:
    app: ingress-nginx
 data:
  use-proxy-protocol: "true"
 ---
 kind: ConfigMap
 apiVersion: v1
 metadata:
  name: tcp-services
  namespace: ingress-nginx
 ---
 kind: ConfigMap
 apiVersion: v1
 metadata:
  name: udp-services
  namespace: ingress-nginx
 ---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
  name: nginx-ingress-serviceaccount
  namespace: ingress-nginx
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
  name: nginx-ingress-clusterrole
 rules:
  - apiGroups:
    - ""
    resources:
    - configmaps
    - endpoints
    - nodes
    - pods
    - secrets
    - namespaces
    verbs:
    - list
    - watch
  - apiGroups:
    - coordination.k8s.io
    resources:
    - leases
    verbs:
    - list
    - watch
  - apiGroups:
    - ""
    resources:
    - nodes
    verbs:
    - get
  - apiGroups:
    - ""
    resources:
    - services
    verbs:
    - get
    - list
    - watch
  - apiGroups:
    - networking.k8s.io
    resources:
    - ingresses
    verbs:
    - get
    - list
    - watch
  - apiGroups:
    - ""
    resources:
    - events
    verbs:
    - create
    - patch
  - apiGroups:
    - networking.k8s.io
    resources:
    - ingresses/status
    verbs:
    - update
  - apiGroups:
    - networking.k8s.io
    resources:
    - ingressclasses
    verbs:
    - get
    - list
    - watch
  - apiGroups:
    - discovery.k8s.io
    resources:
    - endpointslices
    verbs:
    - list
    - watch
    - get
 ---
 kind: Service
 apiVersion: v1
 metadata:
  name: ingress-nginx
  namespace: ingress-nginx
  labels:
    app: ingress-nginx
  annotations:
    # Enable PROXY protocol
    service.beta.kubernetes.io/aws-load-balancer-proxy-protocol: '*'
    # Increase the ELB idle timeout to avoid issues with WebSockets or Server-Sent Events.
    service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: '3600'
 spec:
  type: LoadBalancer
  selector:
    app: ingress-nginx
  ports:
  - name: http
    port: 80
    targetPort: http
  - name: https
    port: 443
    targetPort: https
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
  name: nginx-ingress-role
  namespace: ingress-nginx
 rules:
  - apiGroups:
    - ""
    resources:
    - namespaces
    verbs:
    - get
  - apiGroups:
    - ""
    resources:
    - configmaps
    - pods
    - secrets
    - endpoints
    verbs:
    - get
    - list
    - watch
  - apiGroups:
    - ""
    resources:
    - services
    verbs:
    - get
    - list
    - watch
  - apiGroups:
    - networking.k8s.io
    resources:
    - ingresses
    verbs:
    - get
    - list
    - watch
  - apiGroups:
    - networking.k8s.io
    resources:
    - ingresses/status
    verbs:
    - update
  - apiGroups:
    - networking.k8s.io
    resources:
    - ingressclasses
    verbs:
    - get
    - list
    - watch
  - apiGroups:
    - ""
    resourceNames:
    - ingress-controller-leader
    resources:
    - configmaps
    verbs:
    - get
    - update
  - apiGroups:
    - ""
    resources:
    - configmaps
    verbs:
    - create
  - apiGroups:
    - coordination.k8s.io
    resourceNames:
    - ingress-controller-leader
    resources:
    - leases
    verbs:
    - get
    - update
  - apiGroups:
    - coordination.k8s.io
    resources:
    - leases
    verbs:
    - create
  - apiGroups:
    - ""
    resources:
    - events
    verbs:
    - create
    - patch
  - apiGroups:
    - discovery.k8s.io
    resources:
    - endpointslices
    verbs:
    - list
    - watch
    - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: nginx-ingress-role-nisa-binding
  namespace: ingress-nginx
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: nginx-ingress-role
 subjects:
  - kind: ServiceAccount
    name: nginx-ingress-serviceaccount
    namespace: ingress-nginx
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
  name: nginx-ingress-clusterrole-nisa-binding
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: nginx-ingress-clusterrole
 subjects:
  - kind: ServiceAccount
    name: nginx-ingress-serviceaccount
    namespace: ingress-nginx
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: nginx-ingress-controller
  namespace: ingress-nginx 
 spec:
  replicas: 1
  selector:
    matchLabels:
      app: ingress-nginx
  template:
    metadata:
      labels:
        app: ingress-nginx
      annotations:
        prometheus.io/port: '10254'
        prometheus.io/scrape: 'true'
    spec:
      serviceAccountName: nginx-ingress-serviceaccount
      containers:
        - name: nginx-ingress-controller
          image: registry.k8s.io/ingress-nginx/controller:v1.4.0
          args:
            - /nginx-ingress-controller
            - --default-backend-service=$(POD_NAMESPACE)/default-http-backend
            - --configmap=$(POD_NAMESPACE)/nginx-configuration
            - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
            - --udp-services-configmap=$(POD_NAMESPACE)/udp-services
            - --publish-service=$(POD_NAMESPACE)/ingress-nginx
            - --annotations-prefix=nginx.ingress.kubernetes.io
            - --enable-ssl-passthrough
            - --controller-class=k8s.io/ingress-nginx
            - --election-id=ingress-controller-leader
          securityContext:
            capabilities:
                drop:
                - ALL
                add:
                - NET_BIND_SERVICE
            # www-data -> 33
            runAsUser: 101
          env:
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
          ports:
          - name: http
            containerPort: 80
          - name: https
            containerPort: 443
          livenessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            initialDelaySeconds: 10
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          readinessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
 ---
 apiVersion: networking.k8s.io/v1
 kind: IngressClass
 metadata:
  labels:
    app: ingress-nginx
  name: nginx
  # Currently non-namespaced only
  #namespace: ingress-nginx
  annotations:
    ingressclass.kubernetes.io/is-default-class: "true"
 spec:
  controller: k8s.io/ingress-nginx
--- a/addons/metrics-server.yaml
+++ b/addons/metrics-server.yaml
@ -0,0 +1,190 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
  labels:
    k8s-app: metrics-server
  name: metrics-server
  namespace: kube-system
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
  name: system:aggregated-metrics-reader
  labels:
    k8s-app: metrics-server
    rbac.authorization.k8s.io/aggregate-to-view: "true"
    rbac.authorization.k8s.io/aggregate-to-edit: "true"
    rbac.authorization.k8s.io/aggregate-to-admin: "true"
 rules:
 - apiGroups: ["metrics.k8s.io"]
  resources: ["pods", "nodes"]
  verbs: ["get", "list", "watch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  labels:
    k8s-app: metrics-server
  name: metrics-server-auth-reader
  namespace: kube-system
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: extension-apiserver-authentication-reader
 subjects:
  - kind: ServiceAccount
    name: metrics-server
    namespace: kube-system
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
  labels:
    k8s-app: metrics-server
  name: metrics-server:system:auth-delegator
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:auth-delegator
 subjects:
  - kind: ServiceAccount
    name: metrics-server
    namespace: kube-system
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
  labels:
    k8s-app: metrics-server
  name: system:metrics-server
 rules:
  - apiGroups: [""]
    resources:
      - nodes/metrics
    verbs:
      - get
  - apiGroups: [""]
    resources:
      - pods
      - nodes
    verbs:
      - get
      - list
      - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
  labels:
    k8s-app: metrics-server
  name: system:metrics-server
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:metrics-server
 subjects:
  - kind: ServiceAccount
    name: metrics-server
    namespace: kube-system
 ---
 apiVersion: v1
 kind: Service
 metadata:
  labels:
    k8s-app: metrics-server
  name: metrics-server
  namespace: kube-system
 spec:
  ports:
  - name: https
    port: 443
    protocol: TCP
    targetPort: https
  selector:
    k8s-app: metrics-server
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  labels:
    k8s-app: metrics-server
  name: metrics-server
  namespace: kube-system
 spec:
  selector:
    matchLabels:
      k8s-app: metrics-server
  strategy:
    rollingUpdate:
      maxUnavailable: 0
  template:
    metadata:
      labels:
        k8s-app: metrics-server
    spec:
      containers:
      - args:
          - --cert-dir=/tmp
          - --secure-port=10250
          - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
          - --kubelet-use-node-status-port
          - --metric-resolution=15s
          - --kubelet-insecure-tls
        image: k8s.gcr.io/metrics-server/metrics-server:v0.6.1
        imagePullPolicy: IfNotPresent
        livenessProbe:
          failureThreshold: 3
          httpGet:
            path: /livez
            port: https
            scheme: HTTPS
          periodSeconds: 10
        name: metrics-server
        resources:
          requests:
            cpu: 100m
            memory: 200Mi
        ports:
        - containerPort: 10250
          name: https
          protocol: TCP
        readinessProbe:
          failureThreshold: 3
          httpGet:
            path: /readyz
            port: https
            scheme: HTTPS
          periodSeconds: 10
          initialDelaySeconds: 20
        securityContext:
          readOnlyRootFilesystem: true
          runAsNonRoot: true
          runAsUser: 1000
          allowPrivilegeEscalation: false
        volumeMounts:
        - mountPath: /tmp
          name: tmp-dir
      nodeSelector:
        kubernetes.io/os: linux
      priorityClassName: system-cluster-critical
      serviceAccountName: metrics-server
      volumes:
      # mount in tmp so we can safely use from-scratch images and/or read-only containers
      - emptyDir: {}
        name: tmp-dir
 ---
 apiVersion: apiregistration.k8s.io/v1
 kind: APIService
 metadata:
  labels:
    k8s-app: metrics-server
  name: v1beta1.metrics.k8s.io
 spec:
  group: metrics.k8s.io
  groupPriorityMinimum: 100
  insecureSkipTLSVerify: true
  service:
    name: metrics-server
    namespace: kube-system
  version: v1beta1
  versionPriority: 100
--- a/addons/storage-class.yaml
+++ b/addons/storage-class.yaml
@ -0,0 +1,43 @@
 ---
 kind: StorageClass
 apiVersion: storage.k8s.io/v1
 metadata:
  name: aws-gp3
  annotations:
    storageclass.kubernetes.io/is-default-class: "true"
 provisioner: ebs.csi.aws.com
 volumeBindingMode: WaitForFirstConsumer
 parameters:
  type: gp3
  encrypted: "true"
  csi.storage.k8s.io/fstype: xfs
 reclaimPolicy: Delete
 allowVolumeExpansion: true
 ---
 kind: StorageClass
 apiVersion: storage.k8s.io/v1
 metadata:
  name: aws-gp2
 provisioner: ebs.csi.aws.com
 volumeBindingMode: WaitForFirstConsumer
 parameters:
  type: gp2
  encrypted: "true"
  csi.storage.k8s.io/fstype: xfs
 reclaimPolicy: Delete
 allowVolumeExpansion: true
 ---
 kind: StorageClass
 apiVersion: storage.k8s.io/v1
 metadata:
  name: aws-st1
 provisioner: ebs.csi.aws.com
 volumeBindingMode: WaitForFirstConsumer
 parameters:
  type: st1
  encrypted: "true"
  csi.storage.k8s.io/fstype: xfs
 reclaimPolicy: Delete
 allowVolumeExpansion: true
--- a/calico/calico-cr.yaml
+++ b/calico/calico-cr.yaml
@ -0,0 +1,27 @@
 # This section includes base Calico installation configuration.
 # For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.Installation
 apiVersion: operator.tigera.io/v1
 kind: Installation
 metadata:
  name: default
  namespace: calico
 spec:
  # Configures Calico networking.
  calicoNetwork:
    # Note: The ipPools section cannot be modified post-install.
    ipPools:
    - blockSize: 26
      cidr: 192.168.0.0/16
      encapsulation: VXLANCrossSubnet
      natOutgoing: Enabled
      nodeSelector: all()
 ---
 # This section configures the Calico API server.
 # For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.APIServer
 apiVersion: operator.tigera.io/v1
 kind: APIServer 
 metadata: 
  name: default 
 spec: {}
--- a/calico/calico-operator.yaml
+++ b/calico/calico-operator.yaml
--- a/examples/README.md
+++ b/examples/README.md
@ -0,0 +1,7 @@
 # Examples
 This examples shows how to include this module. To run this example, execute:
 ```
 $ terraform init
 $ terraform apply
 ```
--- a/examples/main.tf
+++ b/examples/main.tf
@ -0,0 +1,46 @@
 module "kubernetes" {
  source = "scholzj/kubernetes/aws"
  aws_region           = "eu-central-1"
  cluster_name         = "aws-kubernetes"
  master_instance_type = "t2.medium"
  worker_instance_type = "t2.medium"
  ssh_public_key       = "~/.ssh/id_rsa.pub"
  ssh_access_cidr      = ["0.0.0.0/0"]
  api_access_cidr      = ["0.0.0.0/0"]
  min_worker_count     = 3
  max_worker_count     = 6
  hosted_zone          = "my-domain.com"
  hosted_zone_private  = false
  master_subnet_id = "subnet-8a3517f8"
  worker_subnet_ids = [
    "subnet-8a3517f8",
    "subnet-9b7853f7",
    "subnet-8g9sdfv8",
  ]
  # Tags
  tags = {
    Application = "AWS-Kubernetes"
  }
  # Tags in a different format for Auto Scaling Group
  tags2 = [
    {
      key                 = "Application"
      value               = "AWS-Kubernetes"
      propagate_at_launch = true
    },
  ]
  addons = [
    "https://raw.githubusercontent.com/scholzj/terraform-aws-kubernetes/master/addons/storage-class.yaml",
    "https://raw.githubusercontent.com/scholzj/terraform-aws-kubernetes/master/addons/csi-driver.yaml",
    "https://raw.githubusercontent.com/scholzj/terraform-aws-kubernetes/master/addons/metrics-server.yaml",
    "https://raw.githubusercontent.com/scholzj/terraform-aws-kubernetes/master/addons/dashboard.yaml",
    "https://raw.githubusercontent.com/scholzj/terraform-aws-kubernetes/master/addons/external-dns.yaml",
    "https://raw.githubusercontent.com/scholzj/terraform-aws-kubernetes/master/addons/autoscaler.yaml",
  ]
 }
--- a/examples/versions.tf
+++ b/examples/versions.tf
@ -0,0 +1,4 @@
 terraform {
  required_version = ">= 0.12"
 }
--- a/main.tf
+++ b/main.tf
@ -0,0 +1,373 @@
 #####
 # AWS Prodvider
 #####
 # Retrieve AWS credentials from env variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
 provider "aws" {
  region = var.aws_region
 }
 #####
 # Generate kubeadm token
 #####
 module "kubeadm-token" {
  source = "scholzj/kubeadm-token/random"
 }
 #####
 # IAM roles
 #####
 # Master nodes
 resource "aws_iam_policy" "master_policy" {
  name        = "${var.cluster_name}-master"
  path        = "/"
  description = "Policy for role ${var.cluster_name}-master"
  policy      = file("${path.module}/template/master-policy.json.tpl")
 }
 resource "aws_iam_role" "master_role" {
  name = "${var.cluster_name}-master"
  assume_role_policy = <<EOF
 {
  "Version": "2012-10-17",
  "Statement": [
    {
      "Action": "sts:AssumeRole",
      "Principal": {
        "Service": "ec2.amazonaws.com"
      },
      "Effect": "Allow",
      "Sid": ""
    }
  ]
 }
 EOF
 }
 resource "aws_iam_policy_attachment" "master-attach" {
  name = "master-attachment"
  roles = [aws_iam_role.master_role.name]
  policy_arn = aws_iam_policy.master_policy.arn
 }
 resource "aws_iam_instance_profile" "master_profile" {
  name = "${var.cluster_name}-master"
  role = aws_iam_role.master_role.name
 }
 # Worker nodes
 resource "aws_iam_policy" "node_policy" {
  name = "${var.cluster_name}-node"
  path = "/"
  description = "Policy for role ${var.cluster_name}-node"
  policy = file("${path.module}/template/node-policy.json.tpl")
 }
 resource "aws_iam_role" "node_role" {
  name = "${var.cluster_name}-node"
  assume_role_policy = <<EOF
 {
  "Version": "2012-10-17",
  "Statement": [
    {
      "Action": "sts:AssumeRole",
      "Principal": {
        "Service": "ec2.amazonaws.com"
      },
      "Effect": "Allow",
      "Sid": ""
    }
  ]
 }
 EOF
 }
 resource "aws_iam_policy_attachment" "node-attach" {
  name       = "node-attachment"
  roles      = [aws_iam_role.node_role.name]
  policy_arn = aws_iam_policy.node_policy.arn
 }
 resource "aws_iam_instance_profile" "node_profile" {
  name = "${var.cluster_name}-node"
  role = aws_iam_role.node_role.name
 }
 #####
 # Security Group
 #####
 # Find VPC details based on Master subnet
 data "aws_subnet" "cluster_subnet" {
  id = var.master_subnet_id
 }
 resource "aws_security_group" "kubernetes" {
  vpc_id = data.aws_subnet.cluster_subnet.vpc_id
  name   = var.cluster_name
  tags = merge(
    {
      "Name"                                               = var.cluster_name
      format("kubernetes.io/cluster/%v", var.cluster_name) = "owned"
    },
    var.tags,
  )
 }
 # Allow outgoing connectivity
 resource "aws_security_group_rule" "allow_all_outbound_from_kubernetes" {
  type              = "egress"
  from_port         = 0
  to_port           = 0
  protocol          = "-1"
  cidr_blocks       = ["0.0.0.0/0"]
  security_group_id = aws_security_group.kubernetes.id
 }
 # Allow SSH connections only from specific CIDR (TODO)
 resource "aws_security_group_rule" "allow_ssh_from_cidr" {
  count     = length(var.ssh_access_cidr)
  type      = "ingress"
  from_port = 22
  to_port   = 22
  protocol  = "tcp"
  # TF-UPGRADE-TODO: In Terraform v0.10 and earlier, it was sometimes necessary to
  # force an interpolation expression to be interpreted as a list by wrapping it
  # in an extra set of list brackets. That form was supported for compatibilty in
  # v0.11, but is no longer supported in Terraform v0.12.
  #
  # If the expression in the following list itself returns a list, remove the
  # brackets to avoid interpretation as a list of lists. If the expression
  # returns a single list item then leave it as-is and remove this TODO comment.
  cidr_blocks       = [var.ssh_access_cidr[count.index]]
  security_group_id = aws_security_group.kubernetes.id
 }
 # Allow the security group members to talk with each other without restrictions
 resource "aws_security_group_rule" "allow_cluster_crosstalk" {
  type                     = "ingress"
  from_port                = 0
  to_port                  = 0
  protocol                 = "-1"
  source_security_group_id = aws_security_group.kubernetes.id
  security_group_id        = aws_security_group.kubernetes.id
 }
 # Allow API connections only from specific CIDR (TODO)
 resource "aws_security_group_rule" "allow_api_from_cidr" {
  count     = length(var.api_access_cidr)
  type      = "ingress"
  from_port = 6443
  to_port   = 6443
  protocol  = "tcp"
  # TF-UPGRADE-TODO: In Terraform v0.10 and earlier, it was sometimes necessary to
  # force an interpolation expression to be interpreted as a list by wrapping it
  # in an extra set of list brackets. That form was supported for compatibilty in
  # v0.11, but is no longer supported in Terraform v0.12.
  #
  # If the expression in the following list itself returns a list, remove the
  # brackets to avoid interpretation as a list of lists. If the expression
  # returns a single list item then leave it as-is and remove this TODO comment.
  cidr_blocks       = [var.api_access_cidr[count.index]]
  security_group_id = aws_security_group.kubernetes.id
 }
 ##########
 # Bootstraping scripts
 ##########
 data "cloudinit_config" "master_cloud_init" {
  gzip          = true
  base64_encode = true
  part {
    filename     = "init-aws-kubernete-master.sh"
    content_type = "text/x-shellscript"
    content      = templatefile("${path.module}/scripts/init-aws-kubernetes-master.sh", { kubeadm_token = module.kubeadm-token.token, dns_name = "${var.cluster_name}.${var.hosted_zone}", ip_address = aws_eip.master.public_ip, cluster_name = var.cluster_name, addons = join(" ", var.addons), aws_region = var.aws_region, asg_name = "${var.cluster_name}-nodes", asg_min_nodes = var.min_worker_count, asg_max_nodes = var.max_worker_count, aws_subnets = join(" ", concat(var.worker_subnet_ids, [var.master_subnet_id])) } )
  }
 }
 data "cloudinit_config" "node_cloud_init" {
  gzip          = true
  base64_encode = true
  part {
    filename     = "init-aws-kubernetes-node.sh"
    content_type = "text/x-shellscript"
    content      = templatefile("${path.module}/scripts/init-aws-kubernetes-node.sh", { kubeadm_token = module.kubeadm-token.token, master_ip = aws_eip.master.public_ip, master_private_ip = aws_instance.master.private_ip, dns_name = "${var.cluster_name}.${var.hosted_zone}" } )
  }
 }
 ##########
 # Keypair
 ##########
 resource "aws_key_pair" "keypair" {
  key_name   = var.cluster_name
  public_key = file(var.ssh_public_key)
 }
 #####
 # AMI image
 #####
 data "aws_ami" "centos7" {
  most_recent = true
  owners      = ["aws-marketplace"]
  filter {
    name   = "product-code"
    values = ["aw0evgkw8e5c1q413zgy5pjce", "cvugziknvmxgqna9noibqnnsy"]
  }
  filter {
    name   = "architecture"
    values = ["x86_64"]
  }
  filter {
    name   = "virtualization-type"
    values = ["hvm"]
  }
 }
 #####
 # Master - EC2 instance
 #####
 resource "aws_eip" "master" {
  vpc = true
 }
 resource "aws_instance" "master" {
  instance_type = var.master_instance_type
  ami = data.aws_ami.centos7.id
  key_name = aws_key_pair.keypair.key_name
  subnet_id = var.master_subnet_id
  associate_public_ip_address = false
  vpc_security_group_ids = [
    aws_security_group.kubernetes.id,
  ]
  iam_instance_profile = aws_iam_instance_profile.master_profile.name
  user_data = data.cloudinit_config.master_cloud_init.rendered
  tags = merge(
    {
      "Name"                                               = join("-", [var.cluster_name, "master"])
      format("kubernetes.io/cluster/%v", var.cluster_name) = "owned"
    },
    var.tags,
  )
  root_block_device {
    volume_type           = "gp2"
    volume_size           = "50"
    delete_on_termination = true
  }
  lifecycle {
    ignore_changes = [
      ami,
      user_data,
      associate_public_ip_address,
    ]
  }
 }
 resource "aws_eip_association" "master_assoc" {
  instance_id   = aws_instance.master.id
  allocation_id = aws_eip.master.id
 }
 #####
 # Nodes
 #####
 resource "aws_launch_configuration" "nodes" {
  name_prefix          = "${var.cluster_name}-nodes-"
  image_id             = data.aws_ami.centos7.id
  instance_type        = var.worker_instance_type
  key_name             = aws_key_pair.keypair.key_name
  iam_instance_profile = aws_iam_instance_profile.node_profile.name
  security_groups = [
    aws_security_group.kubernetes.id,
  ]
  associate_public_ip_address = var.public_worker
  user_data = data.cloudinit_config.node_cloud_init.rendered
  root_block_device {
    volume_type           = "gp2"
    volume_size           = "50"
    delete_on_termination = true
  }
  lifecycle {
    create_before_destroy = true
    ignore_changes        = [user_data]
  }
 }
 resource "aws_autoscaling_group" "nodes" {
  vpc_zone_identifier = var.worker_subnet_ids
  name                 = "${var.cluster_name}-nodes"
  max_size             = var.max_worker_count
  min_size             = var.min_worker_count
  desired_capacity     = var.min_worker_count
  launch_configuration = aws_launch_configuration.nodes.name
  tags = concat(
    [{
      key                 = "kubernetes.io/cluster/${var.cluster_name}"
      value               = "owned"
      propagate_at_launch = true
    },
    {
      key                 = "Name"
      value               = "${var.cluster_name}-node"
      propagate_at_launch = true
    }],
    var.tags2,
  )
  lifecycle {
    ignore_changes = [desired_capacity]
  }
 }
 #####
 # DNS record
 #####
 data "aws_route53_zone" "dns_zone" {
  name         = "${var.hosted_zone}."
  private_zone = var.hosted_zone_private
 }
 resource "aws_route53_record" "master" {
  zone_id = data.aws_route53_zone.dns_zone.zone_id
  name    = "${var.cluster_name}.${var.hosted_zone}"
  type    = "A"
  records = [aws_eip.master.public_ip]
  ttl     = 300
 }
--- a/outputs.tf
+++ b/outputs.tf
@ -0,0 +1,29 @@
 #####
 # Output
 #####
 output "ssh_user" {
  description = "SSH user to download kubeconfig file"
  value       = "centos"
 }
 output "public_ip" {
  description = "Cluster IP address"
  value       = aws_eip.master.public_ip
 }
 output "dns" {
  description = "Cluster DNS address"
  value       = aws_route53_record.master.fqdn
 }
 output "kubeconfig_dns" {
  description = "Path to the the kubeconfig file using DNS address"
  value       = "/home/centos/kubeconfig"
 }
 output "kubeconfig_ip" {
  description = "Path to the kubeconfig file using IP address"
  value       = "/home/centos/kubeconfig_ip"
 }
--- a/scripts/init-aws-kubernetes-master.sh
+++ b/scripts/init-aws-kubernetes-master.sh
@ -0,0 +1,223 @@
 #!/bin/bash
 exec &> /var/log/init-aws-kubernetes-master.log
 set -o verbose
 set -o errexit
 set -o pipefail
 export KUBEADM_TOKEN=${kubeadm_token}
 export DNS_NAME=${dns_name}
 export IP_ADDRESS=${ip_address}
 export CLUSTER_NAME=${cluster_name}
 export ASG_NAME=${asg_name}
 export ASG_MIN_NODES="${asg_min_nodes}"
 export ASG_MAX_NODES="${asg_max_nodes}"
 export AWS_REGION=${aws_region}
 export AWS_SUBNETS="${aws_subnets}"
 export ADDONS="${addons}"
 export KUBERNETES_VERSION="1.25.2"
 # Set this only after setting the defaults
 set -o nounset
 # We needed to match the hostname expected by kubeadm an the hostname used by kubelet
 LOCAL_IP_ADDRESS=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
 FULL_HOSTNAME="$(curl -s http://169.254.169.254/latest/meta-data/hostname)"
 # Make DNS lowercase
 DNS_NAME=$(echo "$DNS_NAME" | tr 'A-Z' 'a-z')
 # Install AWS CLI client
 yum install -y epel-release
 yum install -y python3-pip
 pip3 install --upgrade awscli
 ########################################
 ########################################
 # Tag subnets
 ########################################
 ########################################
 for SUBNET in $AWS_SUBNETS
 do
  aws ec2 create-tags --resources $SUBNET --tags Key=kubernetes.io/cluster/$CLUSTER_NAME,Value=shared --region $AWS_REGION
 done
 ########################################
 ########################################
 # Disable SELinux
 ########################################
 ########################################
 # setenforce returns non zero if already SE Linux is already disabled
 is_enforced=$(getenforce)
 if [[ $is_enforced != "Disabled" ]]; then
  setenforce 0
  sed -i 's/SELINUX=enforcing/SELINUX=permissive/g' /etc/selinux/config
 fi
 ########################################
 ########################################
 # Install containerd
 ########################################
 ########################################
 cat <<EOF | tee /etc/modules-load.d/containerd.conf
 overlay
 br_netfilter
 EOF
 modprobe overlay
 modprobe br_netfilter
 # Setup required sysctl params, these persist across reboots.
 cat <<EOF | tee /etc/sysctl.d/99-kubernetes-cri.conf
 net.bridge.bridge-nf-call-iptables  = 1
 net.ipv4.ip_forward                 = 1
 net.bridge.bridge-nf-call-ip6tables = 1
 EOF
 # Apply sysctl params without reboot
 sysctl --system
 yum install -y yum-utils curl gettext device-mapper-persistent-data lvm2
 yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
 sudo yum install -y containerd.io
 mkdir -p /etc/containerd
 containerd config default > /etc/containerd/config.toml
 sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml
 systemctl restart containerd
 systemctl enable containerd
 ########################################
 ########################################
 # Install Kubernetes components
 ########################################
 ########################################
 sudo cat <<EOF > /etc/yum.repos.d/kubernetes.repo
 [kubernetes]
 name=Kubernetes
 baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
 enabled=1
 gpgcheck=0
 repo_gpgcheck=0
 gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
 exclude=kubelet kubeadm kubectl
 EOF
 yum install -y kubelet-$KUBERNETES_VERSION kubeadm-$KUBERNETES_VERSION kubernetes-cni --disableexcludes=kubernetes
 # Start services
 systemctl enable kubelet
 systemctl start kubelet
 # Fix certificates file on CentOS
 if cat /etc/*release | grep ^NAME= | grep CentOS ; then
    rm -rf /etc/ssl/certs/ca-certificates.crt/
    cp /etc/ssl/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt
 fi
 ########################################
 ########################################
 # Initialize the Kube cluster
 ########################################
 ########################################
 # Initialize the master
 cat >/tmp/kubeadm.yaml <<EOF
 ---
 apiVersion: kubeadm.k8s.io/v1beta3
 kind: InitConfiguration
 bootstrapTokens:
  - groups:
      - system:bootstrappers:kubeadm:default-node-token
    token: $KUBEADM_TOKEN
    ttl: 0s
    usages:
      - signing
      - authentication
 nodeRegistration:
  criSocket: unix:///var/run/containerd/containerd.sock
  imagePullPolicy: IfNotPresent
  kubeletExtraArgs:
    cgroup-driver: systemd
    cloud-provider: aws
    read-only-port: "10255"
  name: $FULL_HOSTNAME
  taints:
    - effect: NoSchedule
      key: node-role.kubernetes.io/master
 localAPIEndpoint:
  advertiseAddress: $LOCAL_IP_ADDRESS
  bindPort: 6443
 ---
 apiVersion: kubeadm.k8s.io/v1beta3
 kind: ClusterConfiguration
 apiServer:
  certSANs:
    - $DNS_NAME
    - $IP_ADDRESS
    - $LOCAL_IP_ADDRESS
    - $FULL_HOSTNAME
  extraArgs:
    cloud-provider: aws
  timeoutForControlPlane: 5m0s
 certificatesDir: /etc/kubernetes/pki
 clusterName: kubernetes
 controllerManager:
  extraArgs:
    cloud-provider: aws
 dns: {}
 etcd:
  local:
    dataDir: /var/lib/etcd
 kubernetesVersion: v$KUBERNETES_VERSION
 networking:
  dnsDomain: cluster.local
  podSubnet: 192.168.0.0/16
  serviceSubnet: 10.96.0.0/12
 scheduler: {}
 ---
 EOF
 kubeadm reset --force
 kubeadm init --config /tmp/kubeadm.yaml
 # Use the local kubectl config for further kubectl operations
 export KUBECONFIG=/etc/kubernetes/admin.conf
 # Install calico
 kubectl create -f https://raw.githubusercontent.com/scholzj/terraform-aws-minikube/master/calico/calico-operator.yaml
 kubectl create -f https://raw.githubusercontent.com/scholzj/terraform-aws-minikube/master/calico/calico-cr.yaml
 ########################################
 ########################################
 # Create user and kubeconfig files
 ########################################
 ########################################
 # Allow the user to administer the cluster
 kubectl create clusterrolebinding admin-cluster-binding --clusterrole=cluster-admin --user=admin
 # Prepare the kubectl config file for download to client (IP address)
 export KUBECONFIG_OUTPUT=/home/centos/kubeconfig_ip
 kubeadm kubeconfig user --client-name admin --config /tmp/kubeadm.yaml > $KUBECONFIG_OUTPUT
 chown centos:centos $KUBECONFIG_OUTPUT
 chmod 0600 $KUBECONFIG_OUTPUT
 cp /home/centos/kubeconfig_ip /home/centos/kubeconfig
 sed -i "s/server: https:\/\/.*:6443/server: https:\/\/$IP_ADDRESS:6443/g" /home/centos/kubeconfig_ip
 sed -i "s/server: https:\/\/.*:6443/server: https:\/\/$DNS_NAME:6443/g" /home/centos/kubeconfig
 chown centos:centos /home/centos/kubeconfig
 chmod 0600 /home/centos/kubeconfig
 ########################################
 ########################################
 # Install addons
 ########################################
 ########################################
 for ADDON in $ADDONS
 do
  curl $ADDON | envsubst > /tmp/addon.yaml
  kubectl apply -f /tmp/addon.yaml
  rm /tmp/addon.yaml
 done
--- a/scripts/init-aws-kubernetes-node.sh
+++ b/scripts/init-aws-kubernetes-node.sh
@ -0,0 +1,140 @@
 #!/bin/bash
 exec &> /var/log/init-aws-kubernetes-node.log
 set -o verbose
 set -o errexit
 set -o pipefail
 export KUBEADM_TOKEN=${kubeadm_token}
 export MASTER_IP=${master_private_ip}
 export DNS_NAME=${dns_name}
 export KUBERNETES_VERSION="1.25.2"
 # Set this only after setting the defaults
 set -o nounset
 # We to match the hostname expected by kubeadm an the hostname used by kubelet
 LOCAL_IP_ADDRESS=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
 FULL_HOSTNAME="$(curl -s http://169.254.169.254/latest/meta-data/hostname)"
 # Make DNS lowercase
 DNS_NAME=$(echo "$DNS_NAME" | tr 'A-Z' 'a-z')
 ########################################
 ########################################
 # Disable SELinux
 ########################################
 ########################################
 # setenforce returns non zero if already SE Linux is already disabled
 is_enforced=$(getenforce)
 if [[ $is_enforced != "Disabled" ]]; then
  setenforce 0
  sed -i 's/SELINUX=enforcing/SELINUX=permissive/g' /etc/selinux/config
 fi
 ########################################
 ########################################
 # Install containerd
 ########################################
 ########################################
 cat <<EOF | tee /etc/modules-load.d/containerd.conf
 overlay
 br_netfilter
 EOF
 modprobe overlay
 modprobe br_netfilter
 # Setup required sysctl params, these persist across reboots.
 cat <<EOF | tee /etc/sysctl.d/99-kubernetes-cri.conf
 net.bridge.bridge-nf-call-iptables  = 1
 net.ipv4.ip_forward                 = 1
 net.bridge.bridge-nf-call-ip6tables = 1
 EOF
 # Apply sysctl params without reboot
 sysctl --system
 yum install -y yum-utils curl gettext device-mapper-persistent-data lvm2
 yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
 sudo yum install -y containerd.io
 mkdir -p /etc/containerd
 containerd config default > /etc/containerd/config.toml
 sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml
 systemctl restart containerd
 systemctl enable containerd
 ########################################
 ########################################
 # Install docker
 ########################################
 ########################################
 # yum install -y yum-utils curl gettext device-mapper-persistent-data lvm2 docker
 # # Start services
 # systemctl enable docker
 # systemctl start docker
 # # Set settings needed by Docker
 # sysctl net.bridge.bridge-nf-call-iptables=1
 # sysctl net.bridge.bridge-nf-call-ip6tables=1
 ########################################
 ########################################
 # Install Kubernetes components
 ########################################
 ########################################
 sudo cat <<EOF > /etc/yum.repos.d/kubernetes.repo
 [kubernetes]
 name=Kubernetes
 baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
 enabled=1
 gpgcheck=0
 repo_gpgcheck=0
 gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
 exclude=kubelet kubeadm kubectl
 EOF
 yum install -y kubelet-$KUBERNETES_VERSION kubeadm-$KUBERNETES_VERSION kubernetes-cni --disableexcludes=kubernetes
 # Start services
 systemctl enable kubelet
 systemctl start kubelet
 # Fix certificates file on CentOS
 if cat /etc/*release | grep ^NAME= | grep CentOS ; then
    rm -rf /etc/ssl/certs/ca-certificates.crt/
    cp /etc/ssl/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt
 fi
 ########################################
 ########################################
 # Initialize the Kube node
 ########################################
 ########################################
 cat >/tmp/kubeadm.yaml <<EOF
 ---
 apiVersion: kubeadm.k8s.io/v1beta3
 kind: JoinConfiguration
 discovery:
  bootstrapToken:
    apiServerEndpoint: $MASTER_IP:6443
    token: $KUBEADM_TOKEN
    unsafeSkipCAVerification: true
  timeout: 5m0s
  tlsBootstrapToken: $KUBEADM_TOKEN
 nodeRegistration:
  kubeletExtraArgs:
    cloud-provider: aws
    read-only-port: "10255"
    cgroup-driver: systemd
  name: $FULL_HOSTNAME
 ---
 EOF
 kubeadm reset --force
 kubeadm join --config /tmp/kubeadm.yaml
--- a/template/master-policy.json.tpl
+++ b/template/master-policy.json.tpl
@ -0,0 +1,32 @@
 {
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "ec2:*",
                "elasticloadbalancing:*",
                "ecr:GetAuthorizationToken",
                "ecr:BatchCheckLayerAvailability",
                "ecr:GetDownloadUrlForLayer",
                "ecr:GetRepositoryPolicy",
                "ecr:DescribeRepositories",
                "ecr:ListImages",
                "ecr:BatchGetImage",
                "route53:GetHostedZone",
                "route53:ListHostedZones",
                "route53:ListHostedZonesByName",
                "route53:ChangeResourceRecordSets",
                "route53:ListResourceRecordSets",
                "route53:GetChange",
                "autoscaling:DescribeAutoScalingGroups",
                "autoscaling:DescribeAutoScalingInstances",
                "autoscaling:DescribeTags",
                "autoscaling:SetDesiredCapacity",
                "autoscaling:TerminateInstanceInAutoScalingGroup",
                "autoscaling:DescribeLaunchConfigurations"
                ],
            "Resource": "*"
        }
    ]
 }
--- a/template/node-policy.json.tpl
+++ b/template/node-policy.json.tpl
@ -0,0 +1,19 @@
 {
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "ec2:Describe*",
                "ecr:GetAuthorizationToken",
                "ecr:BatchCheckLayerAvailability",
                "ecr:GetDownloadUrlForLayer",
                "ecr:GetRepositoryPolicy",
                "ecr:DescribeRepositories",
                "ecr:ListImages",
                "ecr:BatchGetImage"
                ],
            "Resource": "*"
        }
    ]
 }
--- a/variables.tf
+++ b/variables.tf
@ -0,0 +1,86 @@
 variable "aws_region" {
  description = "Region where Cloud Formation is created"
  default     = "eu-central-1"
 }
 variable "cluster_name" {
  description = "Name of the AWS Kubernetes cluster - will be used to name all created resources"
 }
 variable "tags" {
  description = "Tags used for the AWS resources created by this template"
  type        = map(string)
 }
 variable "tags2" {
  description = "Tags in format used for the AWS Autoscaling Group"
  type        = list(object({key = string, value = string, propagate_at_launch = bool}))
 }
 variable "addons" {
  description = "list of YAML files with Kubernetes addons which should be installed"
  type        = list(string)
 }
 variable "master_instance_type" {
  description = "Type of instance for master"
  default     = "t2.medium"
 }
 variable "worker_instance_type" {
  description = "Type of instance for workers"
  default     = "t2.medium"
 }
 variable "master_subnet_id" {
  description = "The subnet-id to be used for the master instance. Master can be only in single subnet. All subnets have to belong to the same VPC."
 }
 variable "worker_subnet_ids" {
  description = "The subnet-ids to be used for the worker instances. Workers can be in multiple subnets. Worker subnets can contain also the master subnet. If you want to run workers in different subnet(s) than master you have to tag the subnets with kubernetes.io/cluster/{cluster_name}=shared.  All subnets have to belong to the same VPC."
  type        = list(string)
 }
 variable "min_worker_count" {
  description = "Minimal number of worker nodes"
 }
 variable "max_worker_count" {
  description = "Maximal number of worker nodes"
 }
 variable "public_worker" {
  description = "Weither or not the workers should have a public IP"
  default     = true
 }
 variable "ssh_public_key" {
  description = "Path to the pulic part of SSH key which should be used for the instance"
  default     = "~/.ssh/id_rsa.pub"
 }
 variable "hosted_zone" {
  description = "Hosted zone to be used for the alias"
 }
 variable "hosted_zone_private" {
  description = "Is the hosted zone public or private"
  default     = false
 }
 variable "ssh_access_cidr" {
  description = "List of CIDRs from which SSH access is allowed"
  type        = list(string)
  default = [
    "0.0.0.0/0",
  ]
 }
 variable "api_access_cidr" {
  description = "List of CIDRs from which API access is allowed"
  type        = list(string)
  default = [
    "0.0.0.0/0",
  ]
 }
--- a/versions.tf
+++ b/versions.tf
@ -0,0 +1,4 @@
 terraform {
  required_version = ">= 0.12"
 }