forked from kube-hetzner/terraform-hcloud-kube-hetzner
-
Notifications
You must be signed in to change notification settings - Fork 0
/
agents.tf
197 lines (164 loc) · 7.25 KB
/
agents.tf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
module "agents" {
source = "./modules/host"
providers = {
hcloud = hcloud,
}
for_each = local.agent_nodes
name = "${var.use_cluster_name_in_node_name ? "${var.cluster_name}-" : ""}${each.value.nodepool_name}"
base_domain = var.base_domain
ssh_keys = length(var.ssh_hcloud_key_label) > 0 ? concat([local.hcloud_ssh_key_id], data.hcloud_ssh_keys.keys_by_selector[0].ssh_keys.*.id) : [local.hcloud_ssh_key_id]
ssh_port = var.ssh_port
ssh_public_key = var.ssh_public_key
ssh_private_key = var.ssh_private_key
ssh_additional_public_keys = length(var.ssh_hcloud_key_label) > 0 ? concat(var.ssh_additional_public_keys, data.hcloud_ssh_keys.keys_by_selector[0].ssh_keys.*.public_key) : var.ssh_additional_public_keys
firewall_ids = [hcloud_firewall.k3s.id]
placement_group_id = var.placement_group_disable ? null : hcloud_placement_group.agent[floor(index(keys(local.agent_nodes), each.key) / 10)].id
location = each.value.location
server_type = each.value.server_type
backups = each.value.backups
ipv4_subnet_id = hcloud_network_subnet.agent[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0]].id
packages_to_install = local.packages_to_install
dns_servers = var.dns_servers
k3s_registries = var.k3s_registries
k3s_registries_update_script = local.k3s_registries_update_script
opensuse_microos_mirror_link = var.opensuse_microos_mirror_link
private_ipv4 = cidrhost(hcloud_network_subnet.agent[[for i, v in var.agent_nodepools : i if v.name == each.value.nodepool_name][0]].ip_range, each.value.index + 101)
labels = merge(local.labels, local.labels_agent_node)
automatically_upgrade_os = var.automatically_upgrade_os
depends_on = [
hcloud_network_subnet.agent,
hcloud_placement_group.agent
]
}
resource "null_resource" "agents" {
for_each = local.agent_nodes
triggers = {
agent_id = module.agents[each.key].id
}
connection {
user = "root"
private_key = var.ssh_private_key
agent_identity = local.ssh_agent_identity
host = module.agents[each.key].ipv4_address
port = var.ssh_port
}
# Generating k3s agent config file
provisioner "file" {
content = yamlencode({
node-name = module.agents[each.key].name
server = "https://${var.use_control_plane_lb ? hcloud_load_balancer_network.control_plane.*.ip[0] : module.control_planes[keys(module.control_planes)[0]].private_ipv4_address}:6443"
token = random_password.k3s_token.result
kubelet-arg = local.kubelet_arg
flannel-iface = local.flannel_iface
node-ip = module.agents[each.key].private_ipv4_address
node-label = each.value.labels
node-taint = each.value.taints
})
destination = "/tmp/config.yaml"
}
# Install k3s agent
provisioner "remote-exec" {
inline = local.install_k3s_agent
}
# Start the k3s agent and wait for it to have started
provisioner "remote-exec" {
inline = [
"/etc/cloud/rename_interface.sh",
"systemctl start k3s-agent 2> /dev/null",
<<-EOT
timeout 120 bash <<EOF
until systemctl status k3s-agent > /dev/null; do
systemctl start k3s-agent 2> /dev/null
echo "Waiting for the k3s agent to start..."
sleep 2
done
EOF
EOT
]
}
depends_on = [
null_resource.first_control_plane,
hcloud_network_subnet.agent
]
}
resource "hcloud_volume" "longhorn_volume" {
for_each = { for k, v in local.agent_nodes : k => v if((lookup(v, "longhorn_volume_size", 0) >= 10) && (lookup(v, "longhorn_volume_size", 0) <= 10000) && var.enable_longhorn) }
labels = {
provisioner = "terraform"
cluster = var.cluster_name
scope = "longhorn"
}
name = "${var.cluster_name}-longhorn-${module.agents[each.key].name}"
size = lookup(local.agent_nodes[each.key], "longhorn_volume_size", 0)
server_id = module.agents[each.key].id
automount = true
format = var.longhorn_fstype
}
resource "null_resource" "configure_longhorn_volume" {
for_each = { for k, v in local.agent_nodes : k => v if((lookup(v, "longhorn_volume_size", 0) >= 10) && (lookup(v, "longhorn_volume_size", 0) <= 10000) && var.enable_longhorn) }
triggers = {
agent_id = module.agents[each.key].id
}
# Start the k3s agent and wait for it to have started
provisioner "remote-exec" {
inline = [
"mkdir /var/longhorn >/dev/null 2>&1",
"mount -o discard,defaults ${hcloud_volume.longhorn_volume[each.key].linux_device} /var/longhorn",
"${var.longhorn_fstype == "ext4" ? "resize2fs" : "xfs_growfs"} ${hcloud_volume.longhorn_volume[each.key].linux_device}",
"echo '${hcloud_volume.longhorn_volume[each.key].linux_device} /var/longhorn ${var.longhorn_fstype} discard,nofail,defaults 0 0' >> /etc/fstab"
]
}
connection {
user = "root"
private_key = var.ssh_private_key
agent_identity = local.ssh_agent_identity
host = module.agents[each.key].ipv4_address
port = var.ssh_port
}
depends_on = [
hcloud_volume.longhorn_volume
]
}
resource "hcloud_floating_ip" "agents" {
for_each = { for k, v in local.agent_nodes : k => v if coalesce(lookup(v, "floating_ip"), false) }
type = "ipv4"
labels = local.labels
home_location = each.value.location
}
resource "hcloud_floating_ip_assignment" "agents" {
for_each = { for k, v in local.agent_nodes : k => v if coalesce(lookup(v, "floating_ip"), false) }
floating_ip_id = hcloud_floating_ip.agents[each.key].id
server_id = module.agents[each.key].id
depends_on = [
null_resource.agents
]
}
resource "null_resource" "configure_floating_ip" {
for_each = { for k, v in local.agent_nodes : k => v if coalesce(lookup(v, "floating_ip"), false) }
triggers = {
agent_id = module.agents[each.key].id
floating_ip_id = hcloud_floating_ip.agents[each.key].id
}
provisioner "remote-exec" {
inline = [
"echo \"BOOTPROTO='static'\nSTARTMODE='auto'\nIPADDR=${hcloud_floating_ip.agents[each.key].ip_address}/32\nIPADDR_1=${module.agents[each.key].ipv4_address}/32\" > /etc/sysconfig/network/ifcfg-eth0",
"echo \"172.31.1.1 - 255.255.255.255 eth0\ndefault 172.31.1.1 - eth0 src ${hcloud_floating_ip.agents[each.key].ip_address}\" > /etc/sysconfig/network/ifroute-eth0",
"ip addr add ${hcloud_floating_ip.agents[each.key].ip_address}/32 dev eth0",
"ip route replace default via 172.31.1.1 dev eth0 src ${hcloud_floating_ip.agents[each.key].ip_address}",
# its important: floating IP should be first on the interface IP list
# move main IP to the second position
"ip addr del ${module.agents[each.key].ipv4_address}/32 dev eth0",
"ip addr add ${module.agents[each.key].ipv4_address}/32 dev eth0",
]
}
connection {
user = "root"
private_key = var.ssh_private_key
agent_identity = local.ssh_agent_identity
host = module.agents[each.key].ipv4_address
port = var.ssh_port
}
depends_on = [
hcloud_floating_ip_assignment.agents
]
}