· 9 years ago · Nov 01, 2016, 08:04 PM
1$ ls
2cloud-config.yaml instaces.tf run
3$ terraform -v
4Terraform v0.7.7
5$ cat cloud-config.yaml
6
7#cloud-config
8
9ssh_authorized_keys:
10 - "ssh-rsa 1"
11 - "ssh-rsa 2"
12coreos:
13 update:
14 reboot-strategy: etcd-lock
15 locksmith:
16 window-start: Wed 04:00
17 window-length: 1h
18 etcd2:
19 discovery: "https://discovery.etcd.io/5aa752c2da1883df8a28fd877068bdae"
20 advertise-client-urls: "http://$private_ipv4:2379"
21 initial-advertise-peer-urls: "http://$private_ipv4:2380"
22 listen-client-urls: "http://0.0.0.0:2379"
23 listen-peer-urls: "http://$private_ipv4:2380"
24 fleet:
25 public-ip: "$public_ipv4"
26 flannel:
27 etcd_prefix: "/coreos.com/network2"
28 units:
29 - name: etcd2.service
30 command: start
31 - name: fleet.service
32 command: start
33 - name: flanneld.service
34 drop-ins:
35 - name: 50-network-config.conf
36 content: |
37 [Service]
38 ExecStartPre=/usr/bin/etcdctl set /coreos.com/network2/config '{"Network": "10.0.0.0/8", "SubnetLen": 20, "SubnetMin": "10.10.0.0", "SubnetMax": "10.99.0.0", "Backend": { "Type": "vxlan"}}'
39 command: start
40 - name: nameservers.network
41 content: |
42 [Network]
43 DNS=8.8.8.8
44
45$ cat instances.tf
46
47variable "key_name" {}
48variable "amid" {}
49variable "skey" {}
50variable "akey" {}
51variable "region" {}
52variable "purpose" {}
53variable "vm_name" {}
54variable "vm_type" {}
55variable "rootgb" {}
56variable "count" {}
57variable "my_security_groups" { type = "list" }
58variable "pg" { default = "" }
59variable "az" { default = "" }
60
61provider "aws" {
62 access_key = "${var.akey}"
63 secret_key = "${var.skey}"
64 region = "${var.region}"
65}
66
67resource "aws_instance" "multipurpose-instances" {
68 ami = "${var.amid}"
69 availability_zone = "${var.az}"
70 instance_type = "${var.vm_type}"
71 count = "${var.count}"
72 key_name = "${var.key_name}"
73 security_groups = "${var.my_security_groups}"
74 placement_group = "${var.pg}"
75 user_data = "${file("cloud-config.yaml")}"
76 root_block_device {
77 volume_size = "${var.rootgb}"
78 }
79 tags {
80 Name = "${format("${var.vm_name}-${var.purpose}-%03d", count.index + 1)}"
81 Purpose = "${var.purpose}"
82 }
83}
84
85$ cat run
86
87#!/bin/bash
88terraform apply -var "purpose=$1" \
89 -var "az=eu-central-1b" \
90 -var "region=eu-central-1" \
91 -var "akey=some_key" \
92 -var "skey=some_key" \
93 -var "amid=ami-27877c48" \
94 -var "key_name=some_key" \
95 -var "count=3" \
96 -var "rootgb=20" \
97 -var "vm_name=$1" \
98 -var "vm_type=c4.large" \
99 -var 'my_security_groups=["AllowSSH","AllowPings","ok_all_internal"]'
100
101$ bash run illskillz
102
103aws_instance.multipurpose-instances.1: Creating...
104 ami: "" => "ami-27877c48"
105 availability_zone: "" => "eu-central-1b"
106 ebs_block_device.#: "" => "<computed>"
107 ephemeral_block_device.#: "" => "<computed>"
108 instance_state: "" => "<computed>"
109 instance_type: "" => "c4.large"
110 key_name: "" => "dima_r_key_frankfurt"
111 network_interface_id: "" => "<computed>"
112 placement_group: "" => "<computed>"
113 private_dns: "" => "<computed>"
114 private_ip: "" => "<computed>"
115 public_dns: "" => "<computed>"
116 public_ip: "" => "<computed>"
117 root_block_device.#: "" => "1"
118 root_block_device.0.delete_on_termination: "" => "true"
119 root_block_device.0.iops: "" => "<computed>"
120 root_block_device.0.volume_size: "" => "20"
121 root_block_device.0.volume_type: "" => "<computed>"
122 security_groups.#: "" => "3"
123 security_groups.3449970610: "" => "ok_all_internal"
124 security_groups.3654605775: "" => "AllowSSH"
125 security_groups.469536343: "" => "AllowPings"
126 source_dest_check: "" => "true"
127 subnet_id: "" => "<computed>"
128 tags.%: "" => "2"
129 tags.Name: "" => "illskillz-illskillz-002"
130 tags.Purpose: "" => "illskillz"
131 tenancy: "" => "<computed>"
132 user_data: "" => "7ee22904da7b5db4ee987f5f46f0214c48563bc8"
133 vpc_security_group_ids.#: "" => "<computed>"
134aws_instance.multipurpose-instances.2: Creating...
135 ami: "" => "ami-27877c48"
136 availability_zone: "" => "eu-central-1b"
137 ebs_block_device.#: "" => "<computed>"
138 ephemeral_block_device.#: "" => "<computed>"
139 instance_state: "" => "<computed>"
140 instance_type: "" => "c4.large"
141 key_name: "" => "dima_r_key_frankfurt"
142 network_interface_id: "" => "<computed>"
143 placement_group: "" => "<computed>"
144 private_dns: "" => "<computed>"
145 private_ip: "" => "<computed>"
146 public_dns: "" => "<computed>"
147 public_ip: "" => "<computed>"
148 root_block_device.#: "" => "1"
149 root_block_device.0.delete_on_termination: "" => "true"
150 root_block_device.0.iops: "" => "<computed>"
151 root_block_device.0.volume_size: "" => "20"
152 root_block_device.0.volume_type: "" => "<computed>"
153 security_groups.#: "" => "3"
154 security_groups.3449970610: "" => "ok_all_internal"
155 security_groups.3654605775: "" => "AllowSSH"
156 security_groups.469536343: "" => "AllowPings"
157 source_dest_check: "" => "true"
158 subnet_id: "" => "<computed>"
159 tags.%: "" => "2"
160 tags.Name: "" => "illskillz-illskillz-003"
161 tags.Purpose: "" => "illskillz"
162 tenancy: "" => "<computed>"
163 user_data: "" => "7ee22904da7b5db4ee987f5f46f0214c48563bc8"
164 vpc_security_group_ids.#: "" => "<computed>"
165aws_instance.multipurpose-instances.0: Creating...
166 ami: "" => "ami-27877c48"
167 availability_zone: "" => "eu-central-1b"
168 ebs_block_device.#: "" => "<computed>"
169 ephemeral_block_device.#: "" => "<computed>"
170 instance_state: "" => "<computed>"
171 instance_type: "" => "c4.large"
172 key_name: "" => "dima_r_key_frankfurt"
173 network_interface_id: "" => "<computed>"
174 placement_group: "" => "<computed>"
175 private_dns: "" => "<computed>"
176 private_ip: "" => "<computed>"
177 public_dns: "" => "<computed>"
178 public_ip: "" => "<computed>"
179 root_block_device.#: "" => "1"
180 root_block_device.0.delete_on_termination: "" => "true"
181 root_block_device.0.iops: "" => "<computed>"
182 root_block_device.0.volume_size: "" => "20"
183 root_block_device.0.volume_type: "" => "<computed>"
184 security_groups.#: "" => "3"
185 security_groups.3449970610: "" => "ok_all_internal"
186 security_groups.3654605775: "" => "AllowSSH"
187 security_groups.469536343: "" => "AllowPings"
188 source_dest_check: "" => "true"
189 subnet_id: "" => "<computed>"
190 tags.%: "" => "2"
191 tags.Name: "" => "illskillz-illskillz-001"
192 tags.Purpose: "" => "illskillz"
193 tenancy: "" => "<computed>"
194 user_data: "" => "7ee22904da7b5db4ee987f5f46f0214c48563bc8"
195 vpc_security_group_ids.#: "" => "<computed>"
196aws_instance.multipurpose-instances.1: Still creating... (10s elapsed)
197aws_instance.multipurpose-instances.2: Still creating... (10s elapsed)
198aws_instance.multipurpose-instances.0: Still creating... (10s elapsed)
199aws_instance.multipurpose-instances.1: Creation complete
200aws_instance.multipurpose-instances.2: Creation complete
201aws_instance.multipurpose-instances.0: Still creating... (20s elapsed)
202aws_instance.multipurpose-instances.0: Creation complete
203
204Apply complete! Resources: 3 added, 0 changed, 0 destroyed.
205
206The state of your infrastructure has been saved to the path
207below. This state is required to modify and destroy your
208infrastructure, so keep it safe. To inspect the complete state
209use the `terraform show` command.
210
211State path: terraform.tfstate
212
213$ ssh core@coreos1
214CoreOS stable (1185.3.0)
215
216$ etcdctl cluster-health
217member 72a48c858004fdf is healthy: got healthy result from http://172.31.1.45:2379
218member 3a0b5e53f4b8943a is healthy: got healthy result from http://172.31.5.235:2379
219member bde0d421e981123d is healthy: got healthy result from http://172.31.10.174:2379
220
221$ etcdctl member list
22272a48c858004fdf: name=dbd2d30730454454acdd31e4138b0dd6 peerURLs=http://172.31.1.45:2380 clientURLs=http://172.31.1.45:2379 isLeader=false
2233a0b5e53f4b8943a: name=9f41a88f181947b09954839847f3092a peerURLs=http://172.31.5.235:2380 clientURLs=http://172.31.5.235:2379 isLeader=false
224bde0d421e981123d: name=4e2ac14e50704ba69c06e317ed7dd6c9 peerURLs=http://172.31.10.174:2380 clientURLs=http://172.31.10.174:2379 isLeader=true
225
226sudo systemctl status etcd2
227â— etcd2.service - etcd2
228 Loaded: loaded (/usr/lib/systemd/system/etcd2.service; disabled; vendor preset: disabled)
229 Drop-In: /run/systemd/system/etcd2.service.d
230 └─10-oem.conf, 20-cloudinit.conf
231 Active: active (running) since Tue 2016-11-01 19:45:30 UTC; 47s ago
232 Main PID: 1020 (etcd2)
233 Tasks: 8
234 Memory: 25.1M
235 CPU: 382ms
236 CGroup: /system.slice/etcd2.service
237 └─1020 /usr/bin/etcd2
238
239Nov 01 19:45:30 ip-172-31-1-45.eu-central-1.compute.internal systemd[1]: Started etcd2.
240Nov 01 19:45:30 ip-172-31-1-45.eu-central-1.compute.internal etcd2[1020]: added local member 72a48c858004fdf [http://172.31.1.45:2380] to cluster 839c65049e6b536
241Nov 01 19:45:30 ip-172-31-1-45.eu-central-1.compute.internal etcd2[1020]: added member 3a0b5e53f4b8943a [http://172.31.5.235:2380] to cluster 839c65049e6b536
242Nov 01 19:45:30 ip-172-31-1-45.eu-central-1.compute.internal etcd2[1020]: added member bde0d421e981123d [http://172.31.10.174:2380] to cluster 839c65049e6b536
243Nov 01 19:45:30 ip-172-31-1-45.eu-central-1.compute.internal etcd2[1020]: 72a48c858004fdf [term: 1] received a MsgVote message with higher term from bde0d421e981123d [term: 2]
244Nov 01 19:45:30 ip-172-31-1-45.eu-central-1.compute.internal etcd2[1020]: 72a48c858004fdf became follower at term 2
245Nov 01 19:45:30 ip-172-31-1-45.eu-central-1.compute.internal etcd2[1020]: 72a48c858004fdf [logterm: 1, index: 3, vote: 0] voted for bde0d421e981123d [logterm: 1, index: 3] at term 2
246Nov 01 19:45:30 ip-172-31-1-45.eu-central-1.compute.internal etcd2[1020]: raft.node: 72a48c858004fdf elected leader bde0d421e981123d at term 2
247Nov 01 19:45:31 ip-172-31-1-45.eu-central-1.compute.internal etcd2[1020]: set the initial cluster version to 2.3
248Nov 01 19:45:31 ip-172-31-1-45.eu-central-1.compute.internal etcd2[1020]: published {Name:dbd2d30730454454acdd31e4138b0dd6 ClientURLs:[http://172.31.1.45:2379]} to cluster 839c65049e6b536
249
250$ sudo systemctl status flanneld
251â— flanneld.service - Network fabric for containers
252 Loaded: loaded (/usr/lib/systemd/system/flanneld.service; disabled; vendor preset: disabled)
253 Drop-In: /etc/systemd/system/flanneld.service.d
254 └─50-network-config.conf
255 Active: active (running) since Tue 2016-11-01 19:45:45 UTC; 39s ago
256 Docs: https://github.com/coreos/flannel
257 Process: 1225 ExecStartPost=/usr/bin/rkt run --net=host --stage1-path=/usr/lib/rkt/stage1-images/stage1-fly.aci --insecure-options=image --volume runvol,kind=host,source=/run,readOnly=false --mount volume=runv
258 Process: 1118 ExecStartPre=/usr/bin/etcdctl set /coreos.com/network2/config {"Network": "10.0.0.0/8", "SubnetLen": 20, "SubnetMin": "10.10.0.0", "SubnetMax": "10.99.0.0", "Backend": { "Type": "vxlan"}} (code=e
259 Process: 1112 ExecStartPre=/usr/bin/mkdir -p ${ETCD_SSL_DIR} (code=exited, status=0/SUCCESS)
260 Process: 1109 ExecStartPre=/usr/bin/mkdir -p /run/flannel (code=exited, status=0/SUCCESS)
261 Process: 1105 ExecStartPre=/sbin/modprobe ip_tables (code=exited, status=0/SUCCESS)
262 Main PID: 1149 (flanneld)
263 Tasks: 8
264 Memory: 97.3M
265 CPU: 1.872s
266 CGroup: /system.slice/flanneld.service
267 └─1149 /opt/bin/flanneld --ip-masq=true
268
269Nov 01 19:45:44 ip-172-31-1-45.eu-central-1.compute.internal rkt[1149]: I1101 19:45:44.860392 01149 manager.go:246] Lease acquired: 10.11.48.0/20
270Nov 01 19:45:44 ip-172-31-1-45.eu-central-1.compute.internal rkt[1149]: I1101 19:45:44.860605 01149 network.go:58] Watching for L3 misses
271Nov 01 19:45:44 ip-172-31-1-45.eu-central-1.compute.internal rkt[1149]: I1101 19:45:44.860640 01149 network.go:66] Watching for new subnet leases
272Nov 01 19:45:44 ip-172-31-1-45.eu-central-1.compute.internal rkt[1225]: image: using image from file /usr/lib/rkt/stage1-images/stage1-fly.aci
273Nov 01 19:45:45 ip-172-31-1-45.eu-central-1.compute.internal rkt[1149]: I1101 19:45:45.068422 01149 network.go:153] Handling initial subnet events
274Nov 01 19:45:45 ip-172-31-1-45.eu-central-1.compute.internal rkt[1149]: I1101 19:45:45.068449 01149 device.go:163] calling GetL2List() dev.link.Index: 3
275Nov 01 19:45:45 ip-172-31-1-45.eu-central-1.compute.internal rkt[1149]: I1101 19:45:45.068635 01149 device.go:168] calling NeighAdd: 172.31.5.235, 3a:d0:59:ec:0f:ab
276Nov 01 19:45:45 ip-172-31-1-45.eu-central-1.compute.internal rkt[1149]: I1101 19:45:45.068706 01149 device.go:168] calling NeighAdd: 172.31.10.174, 46:a0:96:3d:63:84
277Nov 01 19:45:45 ip-172-31-1-45.eu-central-1.compute.internal rkt[1225]: image: using image from local store for image name quay.io/coreos/flannel:v0.6.2-amd64
278Nov 01 19:45:45 ip-172-31-1-45.eu-central-1.compute.internal systemd[1]: Started Network fabric for containers.
279
280$ ifconfig
281eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 9001
282 inet 172.31.1.45 netmask 255.255.240.0 broadcast 172.31.15.255
283 inet6 fe80::4ee:67ff:feb3:7443 prefixlen 64 scopeid 0x20<link>
284 ether 06:ee:67:b3:74:43 txqueuelen 1000 (Ethernet)
285 RX packets 9216 bytes 9311589 (8.8 MiB)
286 RX errors 0 dropped 0 overruns 0 frame 0
287 TX packets 3795 bytes 361999 (353.5 KiB)
288 TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
289
290flannel.1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 8951
291 inet 10.11.48.0 netmask 255.0.0.0 broadcast 0.0.0.0
292 inet6 fe80::48d5:b5ff:fe96:3ec7 prefixlen 64 scopeid 0x20<link>
293 ether 4a:d5:b5:96:3e:c7 txqueuelen 0 (Ethernet)
294 RX packets 0 bytes 0 (0.0 B)
295 RX errors 0 dropped 0 overruns 0 frame 0
296 TX packets 0 bytes 0 (0.0 B)
297 TX errors 0 dropped 8 overruns 0 carrier 0 collisions 0
298
299lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
300 inet 127.0.0.1 netmask 255.0.0.0
301 inet6 ::1 prefixlen 128 scopeid 0x10<host>
302 loop txqueuelen 1 (Local Loopback)
303 RX packets 464 bytes 93183 (90.9 KiB)
304 RX errors 0 dropped 0 overruns 0 frame 0
305 TX packets 464 bytes 93183 (90.9 KiB)
306 TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
307
308$ cat /run/flannel_docker_opts.env
309DOCKER_OPT_BIP="--bip=10.11.48.1/20"
310DOCKER_OPT_IPMASQ="--ip-masq=false"
311DOCKER_OPT_MTU="--mtu=8951"
312
313$ sudo systemctl start docker
314
315$ ifconfig
316docker0: flags=4099<UP,BROADCAST,MULTICAST> mtu 1500
317 inet 10.11.48.1 netmask 255.255.240.0 broadcast 0.0.0.0
318 ether 02:42:fc:49:f8:3d txqueuelen 0 (Ethernet)
319 RX packets 0 bytes 0 (0.0 B)
320 RX errors 0 dropped 0 overruns 0 frame 0
321 TX packets 0 bytes 0 (0.0 B)
322 TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
323
324eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 9001
325 inet 172.31.1.45 netmask 255.255.240.0 broadcast 172.31.15.255
326 inet6 fe80::4ee:67ff:feb3:7443 prefixlen 64 scopeid 0x20<link>
327 ether 06:ee:67:b3:74:43 txqueuelen 1000 (Ethernet)
328 RX packets 10690 bytes 9454711 (9.0 MiB)
329 RX errors 0 dropped 0 overruns 0 frame 0
330 TX packets 5197 bytes 504825 (492.9 KiB)
331 TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
332
333flannel.1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 8951
334 inet 10.11.48.0 netmask 255.0.0.0 broadcast 0.0.0.0
335 inet6 fe80::48d5:b5ff:fe96:3ec7 prefixlen 64 scopeid 0x20<link>
336 ether 4a:d5:b5:96:3e:c7 txqueuelen 0 (Ethernet)
337 RX packets 0 bytes 0 (0.0 B)
338 RX errors 0 dropped 0 overruns 0 frame 0
339 TX packets 0 bytes 0 (0.0 B)
340 TX errors 0 dropped 8 overruns 0 carrier 0 collisions 0
341
342lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
343 inet 127.0.0.1 netmask 255.0.0.0
344 inet6 ::1 prefixlen 128 scopeid 0x10<host>
345 loop txqueuelen 1 (Local Loopback)
346 RX packets 617 bytes 128455 (125.4 KiB)
347 RX errors 0 dropped 0 overruns 0 frame 0
348 TX packets 617 bytes 128455 (125.4 KiB)
349 TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
350
351$ docker run -d --name=cassie -h cassie --restart=always cassandra:latest
352all ok here
353
354$ curl http://127.0.0.1:2379/v2/keys/coreos.com/network2/config
355{"action":"get","node":{"key":"/coreos.com/network2/config","value":"{\"Network\": \"10.0.0.0/8\", \"SubnetLen\": 20, \"SubnetMin\": \"10.10.0.0\", \"SubnetMax\": \"10.99.0.0\", \"Backend\": { \"Type\": \"vxlan\"}}","modifiedIndex":16,"createdIndex":16}}
356
357$ docker logs cassie
358all ok here
359
360$ update_engine_client -update
361I1101 19:49:17.170145 1709 update_engine_client.cc:247] Initiating update check and install.
362I1101 19:49:17.174758 1709 update_engine_client.cc:252] Waiting for update to complete.
363LAST_CHECKED_TIME=0
364PROGRESS=0.000000
365CURRENT_OP=UPDATE_STATUS_IDLE
366NEW_VERSION=0.0.0.0
367NEW_SIZE=0
368E1101 19:49:22.181495 1709 update_engine_client.cc:190] Update failed.
369
370$ sudo systemctl status fleet
371â— fleet.service - fleet daemon
372 Loaded: loaded (/usr/lib/systemd/system/fleet.service; disabled; vendor preset: disabled)
373 Drop-In: /run/systemd/system/fleet.service.d
374 └─20-cloudinit.conf
375 Active: active (running) since Tue 2016-11-01 19:45:30 UTC; 4min 26s ago
376 Main PID: 1095 (fleetd)
377 Tasks: 7
378 Memory: 22.4M
379 CPU: 2.090s
380 CGroup: /system.slice/fleet.service
381 └─1095 /usr/bin/fleetd
382
383Nov 01 19:45:30 ip-172-31-1-45.eu-central-1.compute.internal systemd[1]: Started fleet daemon.
384Nov 01 19:45:31 ip-172-31-1-45.eu-central-1.compute.internal fleetd[1095]: INFO fleetd.go:64: Starting fleetd version 0.11.8
385Nov 01 19:45:31 ip-172-31-1-45.eu-central-1.compute.internal fleetd[1095]: INFO fleetd.go:170: No provided or default config file found - proceeding without
386Nov 01 19:45:31 ip-172-31-1-45.eu-central-1.compute.internal fleetd[1095]: INFO server.go:160: Establishing etcd connectivity
387Nov 01 19:45:31 ip-172-31-1-45.eu-central-1.compute.internal fleetd[1095]: INFO server.go:171: Starting server components
388Nov 01 19:45:31 ip-172-31-1-45.eu-central-1.compute.internal fleetd[1095]: INFO engine.go:79: Engine leader is 4e2ac14e50704ba69c06e317ed7dd6c9
389Nov 01 19:48:12 ip-172-31-1-45.eu-central-1.compute.internal fleetd[1095]: ERROR server.go:192: Server monitor triggered: Monitor timed out before successful heartbeat
390Nov 01 19:48:12 ip-172-31-1-45.eu-central-1.compute.internal fleetd[1095]: INFO server.go:160: Establishing etcd connectivity
391Nov 01 19:48:13 ip-172-31-1-45.eu-central-1.compute.internal fleetd[1095]: INFO server.go:171: Starting server components
392
393$ docker ps
394CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
3954332a3a6d815 cassandra:latest "/docker-entrypoint.s" 2 minutes ago Up 2 minutes 7000-7001/tcp, 7199/tcp, 9042/tcp, 9160/tcp cassie
396
397$ sudo reboot
398Connection to 52.57.214.223 closed by remote host.
399
400Forever and ever. No network on that host again