· 7 years ago · Sep 15, 2018, 10:56 AM
1{
2 "AWSTemplateFormatVersion" : "2010-09-09",
3
4 "Description": "Creates an dynamically scaling pollster collector installation with a public S3 bucket.",
5
6 "Parameters": {
7 "KeyName": {
8 "Description": "_REQUIRED_ - key for SSH access.",
9 "Type": "String"
10 },
11 "ArtifactURL": {
12 "Description": "_REQUIRED_ - URL to instance artifact.",
13 "Type": "String"
14 },
15 "MinGroupSize": {
16 "Description": "(default: 0) - Minimum size for the pollster autoscaling group.",
17 "Type": "Number",
18 "Default": 0
19 },
20 "MaxGroupSize": {
21 "Description": "(default: 4) - Maximum size for the pollster autoscaling group.",
22 "Type": "Number",
23 "Default": 4
24 },
25 "ScaleUpSize": {
26 "Description": "(default: 5000) - Scale up when there are this many messages in the queue.",
27 "Type": "Number",
28 "Default": 5000
29 },
30 "ScaleDownSize": {
31 "Description": "(default: 1000) - Scale down when there are this many messages in the queue.",
32 "Type": "Number",
33 "Default": 1000
34 },
35 "PollsterCollectorProcesses": {
36 "Default": 1,
37 "Description": "(default: 1) - This is the number of collector processes to run on the instances.",
38 "Type": "Number"
39 },
40 "InstanceType": {
41 "Description": "(default: m1.small) - instance type.",
42 "Type": "String",
43 "AllowedValues": ["t1.micro",
44 "m1.small",
45 "m1.large",
46 "m1.xlarge",
47 "m2.xlarge",
48 "m2.2xlarge",
49 "m2.4xlarge",
50 "c1.medium",
51 "c1.xlarge",
52 "cc1.4xlarge",
53 "cc2.8xlarge",
54 "cg1.4xlarge" ],
55 "Default": "m1.small",
56 "ConstraintDescription": "must select a valid instance type."
57 }
58 },
59
60 "Mappings": {
61 "AWSInstanceType2Arch": {
62 "t1.micro": { "Arch": "32" },
63 "m1.small": { "Arch": "32" },
64 "m1.large": { "Arch": "64" },
65 "m1.xlarge": { "Arch": "64" },
66 "m2.xlarge": { "Arch": "64" },
67 "m2.2xlarge": { "Arch": "64" },
68 "m2.4xlarge": { "Arch": "64" },
69 "c1.medium": { "Arch": "32" },
70 "c1.xlarge": { "Arch": "64"},
71 "cc1.4xlarge": { "Arch": "64"},
72 "cc2.8xlarge": { "Arch": "64"},
73 "cg1.4xlarge": { "Arch": "64"}
74 },
75 "AWSRegionArch2AMI": {
76 "us-east-1": { "32": "ami-31814f58", "64": "ami-1b814f72" },
77 "us-west-2": { "32": "ami-38fe7308", "64": "ami-30fe7300" },
78 "us-west-1": { "32": "ami-11d68a54", "64": "ami-1bd68a5e" },
79 "eu-west-1": { "32": "ami-973b06e3", "64": "ami-953b06e1" },
80 "ap-southeast-1": { "32": "ami-b4b0cae6", "64": "ami-beb0caec" },
81 "ap-northeast-1": { "32": "ami-0644f007", "64": "ami-0a44f00b" },
82 "sa-east-1": { "32": "ami-3e3be423", "64": "ami-3c3be421" }
83
84 }
85 },
86 "Resources": {
87 "PollsterQueue": {
88 "Type": "AWS::SQS::Queue"
89 },
90 "PollsterS3Bucket": {
91 "Type" : "AWS::S3::Bucket",
92 "DeletionPolicy" : "Retain",
93 "Properties" : {
94 "AccessControl" : "PublicRead"
95 }
96 },
97 "PollsterClientUser": {
98 "Type" : "AWS::IAM::User",
99 "Properties" : {
100 "Policies": [
101 {
102 "PolicyName": "PollsterClientPolicy",
103 "PolicyDocument":
104 { "Statement": [
105 {
106 "Effect": "Allow",
107 "Action": "sqs:ListQueues",
108 "Resource": "*"
109 },
110 {
111 "Effect": "Allow",
112 "Action": "sqs:SendMessage",
113 "Resource": { "Fn::GetAtt": ["PollsterQueue", "Arn"]}
114 }
115 ]
116 }
117 }
118 ]
119 }
120 },
121 "PollsterClientKeys": {
122 "Type": "AWS::IAM::AccessKey",
123 "Properties": {
124 "UserName": { "Ref": "PollsterClientUser" }
125 }
126 },
127 "PollsterUser": {
128 "Type" : "AWS::IAM::User",
129 "Properties" : {
130 "Policies": [
131 {
132 "PolicyName": "PollsterCollectorPolicy",
133 "PolicyDocument":
134 { "Statement": [
135 {
136 "Effect": "Allow",
137 "Action": "sqs:ListQueues",
138 "Resource": "*"
139 },
140 {
141 "Effect": "Allow",
142 "Action": "sqs:*",
143 "Resource": [ { "Fn::GetAtt": ["PollsterQueue", "Arn"]},
144 { "Fn::Join" : [ "", [{ "Fn::GetAtt": ["PollsterQueue", "Arn"]}, "/*"]]}]
145 },
146 {
147 "Effect": "Allow",
148 "Action": "s3:*",
149 "Resource": [ {"Fn::Join" : [ "", [ "arn:aws:s3:::", { "Ref": "PollsterS3Bucket"}]]},
150 {"Fn::Join" : [ "", [ "arn:aws:s3:::", { "Ref": "PollsterS3Bucket"}, "/*"]]}]
151 },
152 {
153 "Effect": "Allow",
154 "Action": "dynamodb:*",
155 "Resource": "*"
156 }
157 ]
158 }
159 }
160 ]
161 }
162 },
163 "PollsterKeys": {
164 "Type": "AWS::IAM::AccessKey",
165 "Properties": {
166 "UserName": { "Ref": "PollsterUser" }
167 }
168 },
169 "CFNInitUser" : {
170 "Type" : "AWS::IAM::User",
171 "Properties" : {
172 "Policies": [{
173 "PolicyName": "AccessForCFNInit",
174 "PolicyDocument" : {
175 "Statement": [{
176 "Effect" : "Allow",
177 "Action" : "cloudformation:DescribeStackResource",
178 "Resource" : "*"
179 }]
180 }
181 }]
182 }
183 },
184 "CFNKeys": {
185 "Type": "AWS::IAM::AccessKey",
186 "Properties": {
187 "UserName": { "Ref": "CFNInitUser" }
188 }
189 },
190 "PollsterSecurityGroup": {
191 "Type": "AWS::EC2::SecurityGroup",
192 "Properties": {
193 "GroupDescription": "Group for pollster collectors",
194 "SecurityGroupIngress": [
195 {"IpProtocol" : "tcp", "FromPort": "22", "ToPort": "22", "CidrIp": "0.0.0.0/0"}
196 ]
197 }
198 },
199 "PollsterScalingGroup" : {
200 "Type": "AWS::AutoScaling::AutoScalingGroup",
201 "Properties": {
202 "AvailabilityZones": { "Fn::GetAZs" : ""},
203 "LaunchConfigurationName": {"Ref": "PollsterLaunchConfig"},
204 "MinSize": {"Ref": "MinGroupSize"},
205 "MaxSize": {"Ref": "MaxGroupSize"}
206 }
207 },
208 "PollsterScaleUpPolicy": {
209 "Type": "AWS::AutoScaling::ScalingPolicy",
210 "Properties": {
211 "AdjustmentType": "ChangeInCapacity",
212 "AutoScalingGroupName": {"Ref": "PollsterScalingGroup"},
213 "Cooldown": "300",
214 "ScalingAdjustment": "1"
215 }
216 },
217 "PollsterScaleDownPolicy": {
218 "Type": "AWS::AutoScaling::ScalingPolicy",
219 "Properties": {
220 "AdjustmentType": "ChangeInCapacity",
221 "AutoScalingGroupName": {"Ref": "PollsterScalingGroup"},
222 "Cooldown": "300",
223 "ScalingAdjustment": "-1"
224 }
225 },
226 "PollsterQueueAlarmHigh": {
227 "Type": "AWS::CloudWatch::Alarm",
228 "Properties": {
229 "AlarmDescription": "Scale up if there are more than 50 messages in the queue for 10 minutes",
230 "MetricName": "ApproximateNumberOfMessagesVisible",
231 "Namespace": "AWS/SQS",
232 "Statistic": "Average",
233 "Period": "300",
234 "EvaluationPeriods": "2",
235 "Threshold": {"Ref": "ScaleUpSize"},
236 "AlarmActions": [ {"Ref": "PollsterScaleUpPolicy"}],
237 "Dimensions": [
238 {
239 "Name": "QueueName",
240 "Value": { "Fn::GetAtt": ["PollsterQueue", "QueueName"]}
241 }
242 ],
243 "ComparisonOperator": "GreaterThanThreshold"
244 }
245 },
246 "PollsterQueueAlarmLow": {
247 "Type": "AWS::CloudWatch::Alarm",
248 "Properties": {
249 "AlarmDescription": "Scale Down if there are less than 10 messages in the queue for 10 minutes",
250 "MetricName": "ApproximateNumberOfMessagesVisible",
251 "Namespace": "AWS/SQS",
252 "Statistic": "Average",
253 "Period": "300",
254 "EvaluationPeriods": "2",
255 "Threshold": {"Ref": "ScaleDownSize"},
256 "AlarmActions": [ {"Ref": "PollsterScaleDownPolicy"}],
257 "Dimensions": [
258 {
259 "Name": "QueueName",
260 "Value": { "Fn::GetAtt": ["PollsterQueue", "QueueName"]}
261 }
262 ],
263 "ComparisonOperator": "LessThanThreshold"
264 }
265 },
266 "PollsterLaunchConfig": {
267 "Type": "AWS::AutoScaling::LaunchConfiguration",
268 "Metadata":{
269 "AWS::CloudFormation::Init": {
270 "config":{
271 "packages":{
272 "yum": {
273 "puppet": [],
274 "ruby-devel": [],
275 "gcc": [],
276 "make": [],
277 "rubygems": []
278 },
279 "rubygems":{
280 "json": []
281 }
282 },
283 "sources": {
284 "/var/cg":{"Ref":"ArtifactURL"}
285 },
286 "files":{
287 "/etc/init.d/cg-cf-init": {
288 "content": "#!/bin/bash \nif [ -x /var/cg/devops/init.sh ]; then /var/cg/devops/init.sh >> /var/log/cg-cf-init.log 2>&1 ; else /var/cg/*/devops/init.sh >> /var/log/cg-cf-init.log 2>&1; fi",
289 "mode": "100755",
290 "owner": "root",
291 "group": "wheel"
292 }
293 },
294 "services": {
295 "sysvinit":{
296 "cg-cf-init": {
297 "ensureRunning": "true",
298 "files": ["/etc/init.d/cg-cf-init"],
299 "sources": ["/var/cg"],
300 "packages": {"yum": ["puppet", "ruby-devel", "gcc", "make", "rubygems"],
301 "rubygems": ["json"]
302 }
303 }
304 }
305 }
306 },
307 "Puppet": {
308 "aws_region": { "Ref" : "AWS::Region" },
309 "pollster_queue": { "Fn::GetAtt": ["PollsterQueue", "QueueName"]},
310 "pollster_bucket": { "Ref": "PollsterS3Bucket"},
311 "pollster_access_key": {"Ref": "PollsterKeys"},
312 "pollster_secret_key": {"Fn::GetAtt": ["PollsterKeys", "SecretAccessKey"]},
313 "pollster_collector_procs": {"Ref": "PollsterCollectorProcesses"},
314 "pollster_public": "True"
315 }
316 }
317 },
318 "Properties": {
319 "InstanceType": {"Ref": "InstanceType"},
320 "SecurityGroups": [{"Ref": "PollsterSecurityGroup"}],
321 "ImageId": { "Fn::FindInMap": ["AWSRegionArch2AMI", {"Ref": "AWS::Region"},
322 {"Fn::FindInMap": ["AWSInstanceType2Arch", { "Ref": "InstanceType"}, "Arch"]}]},
323 "KeyName": {"Ref": "KeyName" },
324 "UserData" : { "Fn::Base64" : { "Fn::Join" : ["", [
325 "#!/bin/bash\n",
326 "/opt/aws/bin/cfn-init --region ", { "Ref" : "AWS::Region" },
327 " -s ", { "Ref" : "AWS::StackName" }, " -r PollsterLaunchConfig ",
328 " --access-key ", { "Ref" : "CFNKeys" },
329 " --secret-key ", { "Fn::GetAtt" : ["CFNKeys", "SecretAccessKey"]}, "\n"]]}}}
330 }
331 },
332 "Outputs": {
333 "PollsterQueueName": {
334 "Description": "Name of the Pollster SQS Queue",
335 "Value": { "Fn::GetAtt": ["PollsterQueue", "QueueName"] }
336 },
337 "PollsterQueueURL": {
338 "Description": "URL of the Pollster SQS Queue",
339 "Value": { "Ref": "PollsterQueue" }
340 },
341 "PollsterClientAccessKey": {
342 "Description": "IAM access key for Pollster the client user (This user can only send messages to the queue).",
343 "Value": {"Ref": "PollsterClientKeys"}
344 },
345 "PollsterClientSecretKey": {
346 "Description": "IAM secret key for Pollster the client user (This user can only send messages to the queue).",
347 "Value": {"Fn::GetAtt": ["PollsterClientKeys", "SecretAccessKey"]}
348 },
349 "PollsterAWSRegion": {
350 "Description": "AWS region where pollster resources reside.",
351 "Value": { "Ref" : "AWS::Region" }
352 },
353 "PollsterS3Bucket": {
354 "Description": "Name of the S3 bucket where datapoints are being stored.",
355 "Value": { "Ref": "PollsterS3Bucket"}
356 }
357 }
358}