· 6 years ago · Feb 14, 2020, 03:56 PM
1#!/usr/bin/env python
2
3'''
4EC2 external inventory script
5=================================
6
7Generates inventory that Ansible can understand by making API request to
8AWS EC2 a the Boto library.
9
10NOTE: This script assumes Ansible is being executed where the environment
11variables needed for Boto have already been set:
12 export AWS_ACCESS_KEY_ID='AK123'
13 export AWS_SECRET_ACCESS_KEY='abc123'
14
15Optional region environment variable if region is 'auto'
16
17This script also assumes that there is an ec2.ini file alongside it. To specify a
18different path to ec2.ini, define the EC2_INI_PATH environment variable:
19
20 export EC2_INI_PATH=/path/to/my_ec2.ini
21
22If you're using eucalyptus you need to set the above variables and
23you need to define:
24
25 export EC2_URL=http://hostname_of_your_cc:port/services/Eucalyptus
26
27If you're using boto profiles (requires boto>=2.24.0) you can choose a profile
28using the --boto-profile command line argument (e.g. ec2.py --boto-profile prod) or using
29the AWS_PROFILE variable:
30
31 AWS_PROFILE=prod ansible-playbook -i ec2.py myplaybook.yml
32
33For more details, see: http://docs.pythonboto.org/en/latest/boto_config_tut.html
34
35When run against a specific host, this script returns the following variables:
36- ec2_ami_launch_index
37- ec2_architecture
38- ec2_association
39- ec2_attachTime
40- ec2_attachment
41- ec2_attachmentId
42- ec2_block_devices
43- ec2_client_token
44- ec2_deleteOnTermination
45- ec2_description
46- ec2_deviceIndex
47- ec2_dns_name
48- ec2_eventsSet
49- ec2_group_name
50- ec2_hypervisor
51- ec2_id
52- ec2_image_id
53- ec2_instanceState
54- ec2_instance_type
55- ec2_ipOwnerId
56- ec2_ip_address
57- ec2_item
58- ec2_kernel
59- ec2_key_name
60- ec2_launch_time
61- ec2_monitored
62- ec2_monitoring
63- ec2_networkInterfaceId
64- ec2_ownerId
65- ec2_persistent
66- ec2_placement
67- ec2_platform
68- ec2_previous_state
69- ec2_private_dns_name
70- ec2_private_ip_address
71- ec2_publicIp
72- ec2_public_dns_name
73- ec2_ramdisk
74- ec2_reason
75- ec2_region
76- ec2_requester_id
77- ec2_root_device_name
78- ec2_root_device_type
79- ec2_security_group_ids
80- ec2_security_group_names
81- ec2_shutdown_state
82- ec2_sourceDestCheck
83- ec2_spot_instance_request_id
84- ec2_state
85- ec2_state_code
86- ec2_state_reason
87- ec2_status
88- ec2_subnet_id
89- ec2_tenancy
90- ec2_virtualization_type
91- ec2_vpc_id
92
93These variables are pulled out of a boto.ec2.instance object. There is a lack of
94consistency with variable spellings (camelCase and underscores) since this
95just loops through all variables the object exposes. It is preferred to use the
96ones with underscores when multiple exist.
97
98In addition, if an instance has AWS tags associated with it, each tag is a new
99variable named:
100- ec2_tag_[Key] = [Value]
101
102Security groups are comma-separated in 'ec2_security_group_ids' and
103'ec2_security_group_names'.
104
105When destination_format and destination_format_tags are specified
106the destination_format can be built from the instance tags and attributes.
107The behavior will first check the user defined tags, then proceed to
108check instance attributes, and finally if neither are found 'nil' will
109be used instead.
110
111'my_instance': {
112 'region': 'us-east-1', # attribute
113 'availability_zone': 'us-east-1a', # attribute
114 'private_dns_name': '172.31.0.1', # attribute
115 'ec2_tag_deployment': 'blue', # tag
116 'ec2_tag_clusterid': 'ansible', # tag
117 'ec2_tag_Name': 'webserver', # tag
118 ...
119}
120
121Inside of the ec2.ini file the following settings are specified:
122...
123destination_format: {0}-{1}-{2}-{3}
124destination_format_tags: Name,clusterid,deployment,private_dns_name
125...
126
127These settings would produce a destination_format as the following:
128'webserver-ansible-blue-172.31.0.1'
129'''
130
131# (c) 2012, Peter Sankauskas
132#
133# This file is part of Ansible,
134#
135# Ansible is free software: you can redistribute it and/or modify
136# it under the terms of the GNU General Public License as published by
137# the Free Software Foundation, either version 3 of the License, or
138# (at your option) any later version.
139#
140# Ansible is distributed in the hope that it will be useful,
141# but WITHOUT ANY WARRANTY; without even the implied warranty of
142# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
143# GNU General Public License for more details.
144#
145# You should have received a copy of the GNU General Public License
146# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
147
148######################################################################
149
150import sys
151import os
152import argparse
153import re
154from time import time
155import boto
156from boto import ec2
157from boto import rds
158from boto import elasticache
159from boto import route53
160from boto import sts
161import six
162
163from ansible.module_utils import ec2 as ec2_utils
164
165HAS_BOTO3 = False
166try:
167 import boto3 # noqa
168 HAS_BOTO3 = True
169except ImportError:
170 pass
171
172from six.moves import configparser
173from collections import defaultdict
174
175try:
176 import json
177except ImportError:
178 import simplejson as json
179
180DEFAULTS = {
181 'all_elasticache_clusters': 'False',
182 'all_elasticache_nodes': 'False',
183 'all_elasticache_replication_groups': 'False',
184 'all_instances': 'False',
185 'all_rds_instances': 'False',
186 'aws_access_key_id': None,
187 'aws_secret_access_key': None,
188 'aws_security_token': None,
189 'boto_profile': None,
190 'cache_max_age': '300',
191 'cache_path': '~/.ansible/tmp',
192 'destination_variable': 'public_dns_name',
193 'elasticache': 'True',
194 'eucalyptus': 'False',
195 'eucalyptus_host': None,
196 'expand_csv_tags': 'False',
197 'group_by_ami_id': 'True',
198 'group_by_availability_zone': 'True',
199 'group_by_aws_account': 'False',
200 'group_by_elasticache_cluster': 'True',
201 'group_by_elasticache_engine': 'True',
202 'group_by_elasticache_parameter_group': 'True',
203 'group_by_elasticache_replication_group': 'True',
204 'group_by_instance_id': 'True',
205 'group_by_instance_state': 'False',
206 'group_by_instance_type': 'True',
207 'group_by_key_pair': 'True',
208 'group_by_platform': 'True',
209 'group_by_rds_engine': 'True',
210 'group_by_rds_parameter_group': 'True',
211 'group_by_region': 'True',
212 'group_by_route53_names': 'True',
213 'group_by_security_group': 'True',
214 'group_by_tag_keys': 'True',
215 'group_by_tag_none': 'True',
216 'group_by_vpc_id': 'True',
217 'hostname_variable': None,
218 'iam_role': None,
219 'include_rds_clusters': 'False',
220 'nested_groups': 'False',
221 'pattern_exclude': None,
222 'pattern_include': None,
223 'rds': 'False',
224 'regions': 'all',
225 'regions_exclude': 'us-gov-west-1, cn-north-1',
226 'replace_dash_in_groups': 'True',
227 'route53': 'False',
228 'route53_excluded_zones': '',
229 'route53_hostnames': None,
230 'stack_filters': 'False',
231 'vpc_destination_variable': 'ip_address'
232}
233
234
235class Ec2Inventory(object):
236
237 def _empty_inventory(self):
238 return {"_meta": {"hostvars": {}}}
239
240 def __init__(self):
241 ''' Main execution path '''
242
243 # Inventory grouped by instance IDs, tags, security groups, regions,
244 # and availability zones
245 self.inventory = self._empty_inventory()
246
247 self.aws_account_id = None
248
249 # Index of hostname (address) to instance ID
250 self.index = {}
251
252 # Boto profile to use (if any)
253 self.boto_profile = None
254
255 # AWS credentials.
256 self.credentials = {}
257
258 # Read settings and parse CLI arguments
259 self.parse_cli_args()
260 self.read_settings()
261
262 # Make sure that profile_name is not passed at all if not set
263 # as pre 2.24 boto will fall over otherwise
264 if self.boto_profile:
265 if not hasattr(boto.ec2.EC2Connection, 'profile_name'):
266 self.fail_with_error("boto version must be >= 2.24 to use profile")
267
268 # Cache
269 if self.args.refresh_cache:
270 self.do_api_calls_update_cache()
271 elif not self.is_cache_valid():
272 self.do_api_calls_update_cache()
273
274 # Data to print
275 if self.args.host:
276 data_to_print = self.get_host_info()
277
278 elif self.args.list:
279 # Display list of instances for inventory
280 if self.inventory == self._empty_inventory():
281 data_to_print = self.get_inventory_from_cache()
282 else:
283 data_to_print = self.json_format_dict(self.inventory, True)
284
285 print(data_to_print)
286
287 def is_cache_valid(self):
288 ''' Determines if the cache files have expired, or if it is still valid '''
289
290 if os.path.isfile(self.cache_path_cache):
291 mod_time = os.path.getmtime(self.cache_path_cache)
292 current_time = time()
293 if (mod_time + self.cache_max_age) > current_time:
294 if os.path.isfile(self.cache_path_index):
295 return True
296
297 return False
298
299 def read_settings(self):
300 ''' Reads the settings from the ec2.ini file '''
301
302 scriptbasename = __file__
303 scriptbasename = os.path.basename(scriptbasename)
304 scriptbasename = scriptbasename.replace('.py', '')
305
306 defaults = {
307 'ec2': {
308 'ini_fallback': os.path.join(os.path.dirname(__file__), 'ec2.ini'),
309 'ini_path': os.path.join(os.path.dirname(__file__), '%s.ini' % scriptbasename)
310 }
311 }
312
313 if six.PY3:
314 config = configparser.ConfigParser(DEFAULTS)
315 else:
316 config = configparser.SafeConfigParser(DEFAULTS)
317 ec2_ini_path = os.environ.get('EC2_INI_PATH', defaults['ec2']['ini_path'])
318 ec2_ini_path = os.path.expanduser(os.path.expandvars(ec2_ini_path))
319
320 if not os.path.isfile(ec2_ini_path):
321 ec2_ini_path = os.path.expanduser(defaults['ec2']['ini_fallback'])
322
323 if os.path.isfile(ec2_ini_path):
324 config.read(ec2_ini_path)
325
326 # Add empty sections if they don't exist
327 try:
328 config.add_section('ec2')
329 except configparser.DuplicateSectionError:
330 pass
331
332 try:
333 config.add_section('credentials')
334 except configparser.DuplicateSectionError:
335 pass
336
337 # is eucalyptus?
338 self.eucalyptus = config.getboolean('ec2', 'eucalyptus')
339 self.eucalyptus_host = config.get('ec2', 'eucalyptus_host')
340
341 # Regions
342 self.regions = []
343 configRegions = config.get('ec2', 'regions')
344 if (configRegions == 'all'):
345 if self.eucalyptus_host:
346 self.regions.append(boto.connect_euca(host=self.eucalyptus_host).region.name, **self.credentials)
347 else:
348 configRegions_exclude = config.get('ec2', 'regions_exclude')
349
350 for regionInfo in ec2.regions():
351 if regionInfo.name not in configRegions_exclude:
352 self.regions.append(regionInfo.name)
353 else:
354 self.regions = configRegions.split(",")
355 if 'auto' in self.regions:
356 env_region = os.environ.get('AWS_REGION')
357 if env_region is None:
358 env_region = os.environ.get('AWS_DEFAULT_REGION')
359 self.regions = [env_region]
360
361 # Destination addresses
362 self.destination_variable = config.get('ec2', 'destination_variable')
363 self.vpc_destination_variable = config.get('ec2', 'vpc_destination_variable')
364 self.hostname_variable = config.get('ec2', 'hostname_variable')
365
366 if config.has_option('ec2', 'destination_format') and \
367 config.has_option('ec2', 'destination_format_tags'):
368 self.destination_format = config.get('ec2', 'destination_format')
369 self.destination_format_tags = config.get('ec2', 'destination_format_tags').split(',')
370 else:
371 self.destination_format = None
372 self.destination_format_tags = None
373
374 # Route53
375 self.route53_enabled = config.getboolean('ec2', 'route53')
376 self.route53_hostnames = config.get('ec2', 'route53_hostnames')
377
378 self.route53_excluded_zones = []
379 self.route53_excluded_zones = [a for a in config.get('ec2', 'route53_excluded_zones').split(',') if a]
380
381 # Include RDS instances?
382 self.rds_enabled = config.getboolean('ec2', 'rds')
383
384 # Include RDS cluster instances?
385 self.include_rds_clusters = config.getboolean('ec2', 'include_rds_clusters')
386
387 # Include ElastiCache instances?
388 self.elasticache_enabled = config.getboolean('ec2', 'elasticache')
389
390 # Return all EC2 instances?
391 self.all_instances = config.getboolean('ec2', 'all_instances')
392
393 # Instance states to be gathered in inventory. Default is 'running'.
394 # Setting 'all_instances' to 'yes' overrides this option.
395 ec2_valid_instance_states = [
396 'pending',
397 'running',
398 'shutting-down',
399 'terminated',
400 'stopping',
401 'stopped'
402 ]
403 self.ec2_instance_states = []
404 if self.all_instances:
405 self.ec2_instance_states = ec2_valid_instance_states
406 elif config.has_option('ec2', 'instance_states'):
407 for instance_state in config.get('ec2', 'instance_states').split(','):
408 instance_state = instance_state.strip()
409 if instance_state not in ec2_valid_instance_states:
410 continue
411 self.ec2_instance_states.append(instance_state)
412 else:
413 self.ec2_instance_states = ['running']
414
415 # Return all RDS instances? (if RDS is enabled)
416 self.all_rds_instances = config.getboolean('ec2', 'all_rds_instances')
417
418 # Return all ElastiCache replication groups? (if ElastiCache is enabled)
419 self.all_elasticache_replication_groups = config.getboolean('ec2', 'all_elasticache_replication_groups')
420
421 # Return all ElastiCache clusters? (if ElastiCache is enabled)
422 self.all_elasticache_clusters = config.getboolean('ec2', 'all_elasticache_clusters')
423
424 # Return all ElastiCache nodes? (if ElastiCache is enabled)
425 self.all_elasticache_nodes = config.getboolean('ec2', 'all_elasticache_nodes')
426
427 # boto configuration profile (prefer CLI argument then environment variables then config file)
428 self.boto_profile = self.args.boto_profile or \
429 os.environ.get('AWS_PROFILE') or \
430 config.get('ec2', 'boto_profile')
431
432 # AWS credentials (prefer environment variables)
433 if not (self.boto_profile or os.environ.get('AWS_ACCESS_KEY_ID') or
434 os.environ.get('AWS_PROFILE')):
435
436 aws_access_key_id = config.get('credentials', 'aws_access_key_id')
437 aws_secret_access_key = config.get('credentials', 'aws_secret_access_key')
438 aws_security_token = config.get('credentials', 'aws_security_token')
439
440 if aws_access_key_id:
441 self.credentials = {
442 'aws_access_key_id': aws_access_key_id,
443 'aws_secret_access_key': aws_secret_access_key
444 }
445 if aws_security_token:
446 self.credentials['security_token'] = aws_security_token
447
448 # Cache related
449 cache_dir = os.path.expanduser(config.get('ec2', 'cache_path'))
450 if self.boto_profile:
451 cache_dir = os.path.join(cache_dir, 'profile_' + self.boto_profile)
452 if not os.path.exists(cache_dir):
453 os.makedirs(cache_dir)
454
455 cache_name = 'ansible-ec2'
456 cache_id = self.boto_profile or os.environ.get('AWS_ACCESS_KEY_ID', self.credentials.get('aws_access_key_id'))
457 if cache_id:
458 cache_name = '%s-%s' % (cache_name, cache_id)
459 cache_name += '-' + str(abs(hash(__file__)))[1:7]
460 self.cache_path_cache = os.path.join(cache_dir, "%s.cache" % cache_name)
461 self.cache_path_index = os.path.join(cache_dir, "%s.index" % cache_name)
462 self.cache_max_age = config.getint('ec2', 'cache_max_age')
463
464 self.expand_csv_tags = config.getboolean('ec2', 'expand_csv_tags')
465
466 # Configure nested groups instead of flat namespace.
467 self.nested_groups = config.getboolean('ec2', 'nested_groups')
468
469 # Replace dash or not in group names
470 self.replace_dash_in_groups = config.getboolean('ec2', 'replace_dash_in_groups')
471
472 # IAM role to assume for connection
473 self.iam_role = config.get('ec2', 'iam_role')
474
475 # Configure which groups should be created.
476
477 group_by_options = [a for a in DEFAULTS if a.startswith('group_by')]
478 for option in group_by_options:
479 setattr(self, option, config.getboolean('ec2', option))
480
481 # Do we need to just include hosts that match a pattern?
482 self.pattern_include = config.get('ec2', 'pattern_include')
483 if self.pattern_include:
484 self.pattern_include = re.compile(self.pattern_include)
485
486 # Do we need to exclude hosts that match a pattern?
487 self.pattern_exclude = config.get('ec2', 'pattern_exclude')
488 if self.pattern_exclude:
489 self.pattern_exclude = re.compile(self.pattern_exclude)
490
491 # Do we want to stack multiple filters?
492 self.stack_filters = config.getboolean('ec2', 'stack_filters')
493
494 # Instance filters (see boto and EC2 API docs). Ignore invalid filters.
495 self.ec2_instance_filters = []
496
497 if config.has_option('ec2', 'instance_filters'):
498 filters = config.get('ec2', 'instance_filters')
499
500 if self.stack_filters and '&' in filters:
501 self.fail_with_error("AND filters along with stack_filter enabled is not supported.\n")
502
503 filter_sets = [f for f in filters.split(',') if f]
504
505 for filter_set in filter_sets:
506 filters = {}
507 filter_set = filter_set.strip()
508 for instance_filter in filter_set.split("&"):
509 instance_filter = instance_filter.strip()
510 if not instance_filter or '=' not in instance_filter:
511 continue
512 filter_key, filter_value = [x.strip() for x in instance_filter.split('=', 1)]
513 if not filter_key:
514 continue
515 filters[filter_key] = filter_value
516 self.ec2_instance_filters.append(filters.copy())
517
518 def parse_cli_args(self):
519 ''' Command line argument processing '''
520
521 parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on EC2')
522 parser.add_argument('--list', action='store_true', default=True,
523 help='List instances (default: True)')
524 parser.add_argument('--host', action='store',
525 help='Get all the variables about a specific instance')
526 parser.add_argument('--refresh-cache', action='store_true', default=False,
527 help='Force refresh of cache by making API requests to EC2 (default: False - use cache files)')
528 parser.add_argument('--profile', '--boto-profile', action='store', dest='boto_profile',
529 help='Use boto profile for connections to EC2')
530 self.args = parser.parse_args()
531
532 def do_api_calls_update_cache(self):
533 ''' Do API calls to each region, and save data in cache files '''
534
535 if self.route53_enabled:
536 self.get_route53_records()
537
538 for region in self.regions:
539 self.get_instances_by_region(region)
540 if self.rds_enabled:
541 self.get_rds_instances_by_region(region)
542 if self.elasticache_enabled:
543 self.get_elasticache_clusters_by_region(region)
544 self.get_elasticache_replication_groups_by_region(region)
545 if self.include_rds_clusters:
546 self.include_rds_clusters_by_region(region)
547
548 self.write_to_cache(self.inventory, self.cache_path_cache)
549 self.write_to_cache(self.index, self.cache_path_index)
550
551 def connect(self, region):
552 ''' create connection to api server'''
553 if self.eucalyptus:
554 conn = boto.connect_euca(host=self.eucalyptus_host, **self.credentials)
555 conn.APIVersion = '2010-08-31'
556 else:
557 conn = self.connect_to_aws(ec2, region)
558 return conn
559
560 def boto_fix_security_token_in_profile(self, connect_args):
561 ''' monkey patch for boto issue boto/boto#2100 '''
562 profile = 'profile ' + self.boto_profile
563 if boto.config.has_option(profile, 'aws_security_token'):
564 connect_args['security_token'] = boto.config.get(profile, 'aws_security_token')
565 return connect_args
566
567 def connect_to_aws(self, module, region):
568 connect_args = self.credentials
569
570 # only pass the profile name if it's set (as it is not supported by older boto versions)
571 if self.boto_profile:
572 connect_args['profile_name'] = self.boto_profile
573 self.boto_fix_security_token_in_profile(connect_args)
574
575 if self.iam_role:
576 sts_conn = sts.connect_to_region(region, **connect_args)
577 role = sts_conn.assume_role(self.iam_role, 'ansible_dynamic_inventory')
578 connect_args['aws_access_key_id'] = role.credentials.access_key
579 connect_args['aws_secret_access_key'] = role.credentials.secret_key
580 connect_args['security_token'] = role.credentials.session_token
581
582 conn = module.connect_to_region(region, **connect_args)
583 # connect_to_region will fail "silently" by returning None if the region name is wrong or not supported
584 if conn is None:
585 self.fail_with_error("region name: %s likely not supported, or AWS is down. connection to region failed." % region)
586 return conn
587
588 def get_instances_by_region(self, region):
589 ''' Makes an AWS EC2 API call to the list of instances in a particular
590 region '''
591
592 try:
593 conn = self.connect(region)
594 reservations = []
595 if self.ec2_instance_filters:
596 if self.stack_filters:
597 filters_dict = {}
598 for filters in self.ec2_instance_filters:
599 filters_dict.update(filters)
600 reservations.extend(conn.get_all_instances(filters=filters_dict))
601 else:
602 for filters in self.ec2_instance_filters:
603 reservations.extend(conn.get_all_instances(filters=filters))
604 else:
605 reservations = conn.get_all_instances()
606
607 # Pull the tags back in a second step
608 # AWS are on record as saying that the tags fetched in the first `get_all_instances` request are not
609 # reliable and may be missing, and the only way to guarantee they are there is by calling `get_all_tags`
610 instance_ids = []
611 for reservation in reservations:
612 instance_ids.extend([instance.id for instance in reservation.instances])
613
614 max_filter_value = 199
615 tags = []
616 for i in range(0, len(instance_ids), max_filter_value):
617 tags.extend(conn.get_all_tags(filters={'resource-type': 'instance', 'resource-id': instance_ids[i:i + max_filter_value]}))
618
619 tags_by_instance_id = defaultdict(dict)
620 for tag in tags:
621 tags_by_instance_id[tag.res_id][tag.name] = tag.value
622
623 if (not self.aws_account_id) and reservations:
624 self.aws_account_id = reservations[0].owner_id
625
626 for reservation in reservations:
627 for instance in reservation.instances:
628 instance.tags = tags_by_instance_id[instance.id]
629 self.add_instance(instance, region)
630
631 except boto.exception.BotoServerError as e:
632 if e.error_code == 'AuthFailure':
633 error = self.get_auth_error_message()
634 else:
635 backend = 'Eucalyptus' if self.eucalyptus else 'AWS'
636 error = "Error connecting to %s backend.\n%s" % (backend, e.message)
637 self.fail_with_error(error, 'getting EC2 instances')
638
639 def tags_match_filters(self, tags):
640 ''' return True if given tags match configured filters '''
641 if not self.ec2_instance_filters:
642 return True
643
644 for filters in self.ec2_instance_filters:
645 for filter_name, filter_value in filters.items():
646 if filter_name[:4] != 'tag:':
647 continue
648 filter_name = filter_name[4:]
649 if filter_name not in tags:
650 if self.stack_filters:
651 return False
652 continue
653 if isinstance(filter_value, list):
654 if self.stack_filters and tags[filter_name] not in filter_value:
655 return False
656 if not self.stack_filters and tags[filter_name] in filter_value:
657 return True
658 if isinstance(filter_value, six.string_types):
659 if self.stack_filters and tags[filter_name] != filter_value:
660 return False
661 if not self.stack_filters and tags[filter_name] == filter_value:
662 return True
663
664 return self.stack_filters
665
666 def get_rds_instances_by_region(self, region):
667 ''' Makes an AWS API call to the list of RDS instances in a particular
668 region '''
669
670 if not HAS_BOTO3:
671 self.fail_with_error("Working with RDS instances requires boto3 - please install boto3 and try again",
672 "getting RDS instances")
673
674 client = ec2_utils.boto3_inventory_conn('client', 'rds', region, **self.credentials)
675 db_instances = client.describe_db_instances()
676
677 try:
678 conn = self.connect_to_aws(rds, region)
679 if conn:
680 marker = None
681 while True:
682 instances = conn.get_all_dbinstances(marker=marker)
683 marker = instances.marker
684 for index, instance in enumerate(instances):
685 # Add tags to instances.
686 instance.arn = db_instances['DBInstances'][index]['DBInstanceArn']
687 tags = client.list_tags_for_resource(ResourceName=instance.arn)['TagList']
688 instance.tags = {}
689 for tag in tags:
690 instance.tags[tag['Key']] = tag['Value']
691 if self.tags_match_filters(instance.tags):
692 self.add_rds_instance(instance, region)
693 if not marker:
694 break
695 except boto.exception.BotoServerError as e:
696 error = e.reason
697
698 if e.error_code == 'AuthFailure':
699 error = self.get_auth_error_message()
700 elif e.error_code == "OptInRequired":
701 error = "RDS hasn't been enabled for this account yet. " \
702 "You must either log in to the RDS service through the AWS console to enable it, " \
703 "or set 'rds = False' in ec2.ini"
704 elif not e.reason == "Forbidden":
705 error = "Looks like AWS RDS is down:\n%s" % e.message
706 self.fail_with_error(error, 'getting RDS instances')
707
708 def include_rds_clusters_by_region(self, region):
709 if not HAS_BOTO3:
710 self.fail_with_error("Working with RDS clusters requires boto3 - please install boto3 and try again",
711 "getting RDS clusters")
712
713 client = ec2_utils.boto3_inventory_conn('client', 'rds', region, **self.credentials)
714
715 marker, clusters = '', []
716 while marker is not None:
717 resp = client.describe_db_clusters(Marker=marker)
718 clusters.extend(resp["DBClusters"])
719 marker = resp.get('Marker', None)
720
721 account_id = boto.connect_iam().get_user().arn.split(':')[4]
722 c_dict = {}
723 for c in clusters:
724 # remove these datetime objects as there is no serialisation to json
725 # currently in place and we don't need the data yet
726 if 'EarliestRestorableTime' in c:
727 del c['EarliestRestorableTime']
728 if 'LatestRestorableTime' in c:
729 del c['LatestRestorableTime']
730
731 if not self.ec2_instance_filters:
732 matches_filter = True
733 else:
734 matches_filter = False
735
736 try:
737 # arn:aws:rds:<region>:<account number>:<resourcetype>:<name>
738 tags = client.list_tags_for_resource(
739 ResourceName='arn:aws:rds:' + region + ':' + account_id + ':cluster:' + c['DBClusterIdentifier'])
740 c['Tags'] = tags['TagList']
741
742 if self.ec2_instance_filters:
743 for filters in self.ec2_instance_filters:
744 for filter_key, filter_values in filters.items():
745 # get AWS tag key e.g. tag:env will be 'env'
746 tag_name = filter_key.split(":", 1)[1]
747 # Filter values is a list (if you put multiple values for the same tag name)
748 matches_filter = any(d['Key'] == tag_name and d['Value'] in filter_values for d in c['Tags'])
749
750 if matches_filter:
751 # it matches a filter, so stop looking for further matches
752 break
753
754 if matches_filter:
755 break
756
757 except Exception as e:
758 if e.message.find('DBInstanceNotFound') >= 0:
759 # AWS RDS bug (2016-01-06) means deletion does not fully complete and leave an 'empty' cluster.
760 # Ignore errors when trying to find tags for these
761 pass
762
763 # ignore empty clusters caused by AWS bug
764 if len(c['DBClusterMembers']) == 0:
765 continue
766 elif matches_filter:
767 c_dict[c['DBClusterIdentifier']] = c
768
769 self.inventory['db_clusters'] = c_dict
770
771 def get_elasticache_clusters_by_region(self, region):
772 ''' Makes an AWS API call to the list of ElastiCache clusters (with
773 nodes' info) in a particular region.'''
774
775 # ElastiCache boto module doesn't provide a get_all_instances method,
776 # that's why we need to call describe directly (it would be called by
777 # the shorthand method anyway...)
778 try:
779 conn = self.connect_to_aws(elasticache, region)
780 if conn:
781 # show_cache_node_info = True
782 # because we also want nodes' information
783 response = conn.describe_cache_clusters(None, None, None, True)
784
785 except boto.exception.BotoServerError as e:
786 error = e.reason
787
788 if e.error_code == 'AuthFailure':
789 error = self.get_auth_error_message()
790 elif e.error_code == "OptInRequired":
791 error = "ElastiCache hasn't been enabled for this account yet. " \
792 "You must either log in to the ElastiCache service through the AWS console to enable it, " \
793 "or set 'elasticache = False' in ec2.ini"
794 elif not e.reason == "Forbidden":
795 error = "Looks like AWS ElastiCache is down:\n%s" % e.message
796 self.fail_with_error(error, 'getting ElastiCache clusters')
797
798 try:
799 # Boto also doesn't provide wrapper classes to CacheClusters or
800 # CacheNodes. Because of that we can't make use of the get_list
801 # method in the AWSQueryConnection. Let's do the work manually
802 clusters = response['DescribeCacheClustersResponse']['DescribeCacheClustersResult']['CacheClusters']
803
804 except KeyError as e:
805 error = "ElastiCache query to AWS failed (unexpected format)."
806 self.fail_with_error(error, 'getting ElastiCache clusters')
807
808 for cluster in clusters:
809 self.add_elasticache_cluster(cluster, region)
810
811 def get_elasticache_replication_groups_by_region(self, region):
812 ''' Makes an AWS API call to the list of ElastiCache replication groups
813 in a particular region.'''
814
815 # ElastiCache boto module doesn't provide a get_all_instances method,
816 # that's why we need to call describe directly (it would be called by
817 # the shorthand method anyway...)
818 try:
819 conn = self.connect_to_aws(elasticache, region)
820 if conn:
821 response = conn.describe_replication_groups()
822
823 except boto.exception.BotoServerError as e:
824 error = e.reason
825
826 if e.error_code == 'AuthFailure':
827 error = self.get_auth_error_message()
828 if not e.reason == "Forbidden":
829 error = "Looks like AWS ElastiCache [Replication Groups] is down:\n%s" % e.message
830 self.fail_with_error(error, 'getting ElastiCache clusters')
831
832 try:
833 # Boto also doesn't provide wrapper classes to ReplicationGroups
834 # Because of that we can't make use of the get_list method in the
835 # AWSQueryConnection. Let's do the work manually
836 replication_groups = response['DescribeReplicationGroupsResponse']['DescribeReplicationGroupsResult']['ReplicationGroups']
837
838 except KeyError as e:
839 error = "ElastiCache [Replication Groups] query to AWS failed (unexpected format)."
840 self.fail_with_error(error, 'getting ElastiCache clusters')
841
842 for replication_group in replication_groups:
843 self.add_elasticache_replication_group(replication_group, region)
844
845 def get_auth_error_message(self):
846 ''' create an informative error message if there is an issue authenticating'''
847 errors = ["Authentication error retrieving ec2 inventory."]
848 if None in [os.environ.get('AWS_ACCESS_KEY_ID'), os.environ.get('AWS_SECRET_ACCESS_KEY')]:
849 errors.append(' - No AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY environment vars found')
850 else:
851 errors.append(' - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment vars found but may not be correct')
852
853 boto_paths = ['/etc/boto.cfg', '~/.boto', '~/.aws/credentials']
854 boto_config_found = [p for p in boto_paths if os.path.isfile(os.path.expanduser(p))]
855 if len(boto_config_found) > 0:
856 errors.append(" - Boto configs found at '%s', but the credentials contained may not be correct" % ', '.join(boto_config_found))
857 else:
858 errors.append(" - No Boto config found at any expected location '%s'" % ', '.join(boto_paths))
859
860 return '\n'.join(errors)
861
862 def fail_with_error(self, err_msg, err_operation=None):
863 '''log an error to std err for ansible-playbook to consume and exit'''
864 if err_operation:
865 err_msg = 'ERROR: "{err_msg}", while: {err_operation}'.format(
866 err_msg=err_msg, err_operation=err_operation)
867 sys.stderr.write(err_msg)
868 sys.exit(1)
869
870 def get_instance(self, region, instance_id):
871 conn = self.connect(region)
872
873 reservations = conn.get_all_instances([instance_id])
874 for reservation in reservations:
875 for instance in reservation.instances:
876 return instance
877
878 def add_instance(self, instance, region):
879 ''' Adds an instance to the inventory and index, as long as it is
880 addressable '''
881
882 # Only return instances with desired instance states
883 if instance.state not in self.ec2_instance_states:
884 return
885
886 # Select the best destination address
887 # When destination_format and destination_format_tags are specified
888 # the following code will attempt to find the instance tags first,
889 # then the instance attributes next, and finally if neither are found
890 # assign nil for the desired destination format attribute.
891 if self.destination_format and self.destination_format_tags:
892 dest_vars = []
893 inst_tags = getattr(instance, 'tags')
894 for tag in self.destination_format_tags:
895 if tag in inst_tags:
896 dest_vars.append(inst_tags[tag])
897 elif hasattr(instance, tag):
898 dest_vars.append(getattr(instance, tag))
899 else:
900 dest_vars.append('nil')
901
902 dest = self.destination_format.format(*dest_vars)
903 elif instance.subnet_id:
904 dest = getattr(instance, self.vpc_destination_variable, None)
905 if dest is None:
906 dest = getattr(instance, 'tags').get(self.vpc_destination_variable, None)
907 else:
908 dest = getattr(instance, self.destination_variable, None)
909 if dest is None:
910 dest = getattr(instance, 'tags').get(self.destination_variable, None)
911
912 if not dest:
913 # Skip instances we cannot address (e.g. private VPC subnet)
914 return
915
916 # Set the inventory name
917 hostname = None
918 if self.hostname_variable:
919 if self.hostname_variable.startswith('tag_'):
920 hostname = instance.tags.get(self.hostname_variable[4:], None)
921 else:
922 hostname = getattr(instance, self.hostname_variable)
923
924 # set the hostname from route53
925 if self.route53_enabled and self.route53_hostnames:
926 route53_names = self.get_instance_route53_names(instance)
927 for name in route53_names:
928 if name.endswith(self.route53_hostnames):
929 hostname = name
930
931 # If we can't get a nice hostname, use the destination address
932 if not hostname:
933 hostname = dest
934 # to_safe strips hostname characters like dots, so don't strip route53 hostnames
935 elif self.route53_enabled and self.route53_hostnames and hostname.endswith(self.route53_hostnames):
936 hostname = hostname.lower()
937 else:
938 hostname = self.to_safe(hostname).lower()
939
940 # if we only want to include hosts that match a pattern, skip those that don't
941 if self.pattern_include and not self.pattern_include.match(hostname):
942 return
943
944 # if we need to exclude hosts that match a pattern, skip those
945 if self.pattern_exclude and self.pattern_exclude.match(hostname):
946 return
947
948 # Add to index
949 self.index[hostname] = [region, instance.id]
950
951 # Inventory: Group by instance ID (always a group of 1)
952 if self.group_by_instance_id:
953 self.inventory[instance.id] = [hostname]
954 if self.nested_groups:
955 self.push_group(self.inventory, 'instances', instance.id)
956
957 # Inventory: Group by region
958 if self.group_by_region:
959 self.push(self.inventory, region, hostname)
960 if self.nested_groups:
961 self.push_group(self.inventory, 'regions', region)
962
963 # Inventory: Group by availability zone
964 if self.group_by_availability_zone:
965 self.push(self.inventory, instance.placement, hostname)
966 if self.nested_groups:
967 if self.group_by_region:
968 self.push_group(self.inventory, region, instance.placement)
969 self.push_group(self.inventory, 'zones', instance.placement)
970
971 # Inventory: Group by Amazon Machine Image (AMI) ID
972 if self.group_by_ami_id:
973 ami_id = self.to_safe(instance.image_id)
974 self.push(self.inventory, ami_id, hostname)
975 if self.nested_groups:
976 self.push_group(self.inventory, 'images', ami_id)
977
978 # Inventory: Group by instance type
979 if self.group_by_instance_type:
980 type_name = self.to_safe('type_' + instance.instance_type)
981 self.push(self.inventory, type_name, hostname)
982 if self.nested_groups:
983 self.push_group(self.inventory, 'types', type_name)
984
985 # Inventory: Group by instance state
986 if self.group_by_instance_state:
987 state_name = self.to_safe('instance_state_' + instance.state)
988 self.push(self.inventory, state_name, hostname)
989 if self.nested_groups:
990 self.push_group(self.inventory, 'instance_states', state_name)
991
992 # Inventory: Group by platform
993 if self.group_by_platform:
994 if instance.platform:
995 platform = self.to_safe('platform_' + instance.platform)
996 else:
997 platform = self.to_safe('platform_undefined')
998 self.push(self.inventory, platform, hostname)
999 if self.nested_groups:
1000 self.push_group(self.inventory, 'platforms', platform)
1001
1002 # Inventory: Group by key pair
1003 if self.group_by_key_pair and instance.key_name:
1004 key_name = self.to_safe('key_' + instance.key_name)
1005 self.push(self.inventory, key_name, hostname)
1006 if self.nested_groups:
1007 self.push_group(self.inventory, 'keys', key_name)
1008
1009 # Inventory: Group by VPC
1010 if self.group_by_vpc_id and instance.vpc_id:
1011 vpc_id_name = self.to_safe('vpc_id_' + instance.vpc_id)
1012 self.push(self.inventory, vpc_id_name, hostname)
1013 if self.nested_groups:
1014 self.push_group(self.inventory, 'vpcs', vpc_id_name)
1015
1016 # Inventory: Group by security group
1017 if self.group_by_security_group:
1018 try:
1019 for group in instance.groups:
1020 key = self.to_safe("security_group_" + group.name)
1021 self.push(self.inventory, key, hostname)
1022 if self.nested_groups:
1023 self.push_group(self.inventory, 'security_groups', key)
1024 except AttributeError:
1025 self.fail_with_error('\n'.join(['Package boto seems a bit older.',
1026 'Please upgrade boto >= 2.3.0.']))
1027
1028 # Inventory: Group by AWS account ID
1029 if self.group_by_aws_account:
1030 self.push(self.inventory, self.aws_account_id, hostname)
1031 if self.nested_groups:
1032 self.push_group(self.inventory, 'accounts', self.aws_account_id)
1033
1034 # Inventory: Group by tag keys
1035 if self.group_by_tag_keys:
1036 for k, v in instance.tags.items():
1037 if self.expand_csv_tags and v and ',' in v:
1038 values = map(lambda x: x.strip(), v.split(','))
1039 else:
1040 values = [v]
1041
1042 for v in values:
1043 if v:
1044 key = self.to_safe("tag_" + k + "=" + v)
1045 else:
1046 key = self.to_safe("tag_" + k)
1047 self.push(self.inventory, key, hostname)
1048 if self.nested_groups:
1049 self.push_group(self.inventory, 'tags', self.to_safe("tag_" + k))
1050 if v:
1051 self.push_group(self.inventory, self.to_safe("tag_" + k), key)
1052
1053 # Inventory: Group by Route53 domain names if enabled
1054 if self.route53_enabled and self.group_by_route53_names:
1055 route53_names = self.get_instance_route53_names(instance)
1056 for name in route53_names:
1057 self.push(self.inventory, name, hostname)
1058 if self.nested_groups:
1059 self.push_group(self.inventory, 'route53', name)
1060
1061 # Global Tag: instances without tags
1062 if self.group_by_tag_none and len(instance.tags) == 0:
1063 self.push(self.inventory, 'tag_none', hostname)
1064 if self.nested_groups:
1065 self.push_group(self.inventory, 'tags', 'tag_none')
1066
1067 # Global Tag: tag all EC2 instances
1068 self.push(self.inventory, 'ec2', hostname)
1069
1070 self.inventory["_meta"]["hostvars"][hostname] = self.get_host_info_dict_from_instance(instance)
1071 self.inventory["_meta"]["hostvars"][hostname]['ansible_host'] = dest
1072
1073 def add_rds_instance(self, instance, region):
1074 ''' Adds an RDS instance to the inventory and index, as long as it is
1075 addressable '''
1076
1077 # Only want available instances unless all_rds_instances is True
1078 if not self.all_rds_instances and instance.status != 'available':
1079 return
1080
1081 # Select the best destination address
1082 dest = instance.endpoint[0]
1083
1084 if not dest:
1085 # Skip instances we cannot address (e.g. private VPC subnet)
1086 return
1087
1088 # Set the inventory name
1089 hostname = None
1090 if self.hostname_variable:
1091 if self.hostname_variable.startswith('tag_'):
1092 hostname = instance.tags.get(self.hostname_variable[4:], None)
1093 else:
1094 hostname = getattr(instance, self.hostname_variable)
1095
1096 # If we can't get a nice hostname, use the destination address
1097 if not hostname:
1098 hostname = dest
1099
1100 hostname = self.to_safe(hostname).lower()
1101
1102 # Add to index
1103 self.index[hostname] = [region, instance.id]
1104
1105 # Inventory: Group by instance ID (always a group of 1)
1106 if self.group_by_instance_id:
1107 self.inventory[instance.id] = [hostname]
1108 if self.nested_groups:
1109 self.push_group(self.inventory, 'instances', instance.id)
1110
1111 # Inventory: Group by region
1112 if self.group_by_region:
1113 self.push(self.inventory, region, hostname)
1114 if self.nested_groups:
1115 self.push_group(self.inventory, 'regions', region)
1116
1117 # Inventory: Group by availability zone
1118 if self.group_by_availability_zone:
1119 self.push(self.inventory, instance.availability_zone, hostname)
1120 if self.nested_groups:
1121 if self.group_by_region:
1122 self.push_group(self.inventory, region, instance.availability_zone)
1123 self.push_group(self.inventory, 'zones', instance.availability_zone)
1124
1125 # Inventory: Group by instance type
1126 if self.group_by_instance_type:
1127 type_name = self.to_safe('type_' + instance.instance_class)
1128 self.push(self.inventory, type_name, hostname)
1129 if self.nested_groups:
1130 self.push_group(self.inventory, 'types', type_name)
1131
1132 # Inventory: Group by VPC
1133 if self.group_by_vpc_id and instance.subnet_group and instance.subnet_group.vpc_id:
1134 vpc_id_name = self.to_safe('vpc_id_' + instance.subnet_group.vpc_id)
1135 self.push(self.inventory, vpc_id_name, hostname)
1136 if self.nested_groups:
1137 self.push_group(self.inventory, 'vpcs', vpc_id_name)
1138
1139 # Inventory: Group by security group
1140 if self.group_by_security_group:
1141 try:
1142 if instance.security_group:
1143 key = self.to_safe("security_group_" + instance.security_group.name)
1144 self.push(self.inventory, key, hostname)
1145 if self.nested_groups:
1146 self.push_group(self.inventory, 'security_groups', key)
1147
1148 except AttributeError:
1149 self.fail_with_error('\n'.join(['Package boto seems a bit older.',
1150 'Please upgrade boto >= 2.3.0.']))
1151 # Inventory: Group by tag keys
1152 if self.group_by_tag_keys:
1153 for k, v in instance.tags.items():
1154 if self.expand_csv_tags and v and ',' in v:
1155 values = map(lambda x: x.strip(), v.split(','))
1156 else:
1157 values = [v]
1158
1159 for v in values:
1160 if v:
1161 key = self.to_safe("tag_" + k + "=" + v)
1162 else:
1163 key = self.to_safe("tag_" + k)
1164 self.push(self.inventory, key, hostname)
1165 if self.nested_groups:
1166 self.push_group(self.inventory, 'tags', self.to_safe("tag_" + k))
1167 if v:
1168 self.push_group(self.inventory, self.to_safe("tag_" + k), key)
1169
1170 # Inventory: Group by engine
1171 if self.group_by_rds_engine:
1172 self.push(self.inventory, self.to_safe("rds_" + instance.engine), hostname)
1173 if self.nested_groups:
1174 self.push_group(self.inventory, 'rds_engines', self.to_safe("rds_" + instance.engine))
1175
1176 # Inventory: Group by parameter group
1177 if self.group_by_rds_parameter_group:
1178 self.push(self.inventory, self.to_safe("rds_parameter_group_" + instance.parameter_group.name), hostname)
1179 if self.nested_groups:
1180 self.push_group(self.inventory, 'rds_parameter_groups', self.to_safe("rds_parameter_group_" + instance.parameter_group.name))
1181
1182 # Global Tag: instances without tags
1183 if self.group_by_tag_none and len(instance.tags) == 0:
1184 self.push(self.inventory, 'tag_none', hostname)
1185 if self.nested_groups:
1186 self.push_group(self.inventory, 'tags', 'tag_none')
1187
1188 # Global Tag: all RDS instances
1189 self.push(self.inventory, 'rds', hostname)
1190
1191 self.inventory["_meta"]["hostvars"][hostname] = self.get_host_info_dict_from_instance(instance)
1192 self.inventory["_meta"]["hostvars"][hostname]['ansible_host'] = dest
1193
1194 def add_elasticache_cluster(self, cluster, region):
1195 ''' Adds an ElastiCache cluster to the inventory and index, as long as
1196 it's nodes are addressable '''
1197
1198 # Only want available clusters unless all_elasticache_clusters is True
1199 if not self.all_elasticache_clusters and cluster['CacheClusterStatus'] != 'available':
1200 return
1201
1202 # Select the best destination address
1203 if 'ConfigurationEndpoint' in cluster and cluster['ConfigurationEndpoint']:
1204 # Memcached cluster
1205 dest = cluster['ConfigurationEndpoint']['Address']
1206 is_redis = False
1207 else:
1208 # Redis sigle node cluster
1209 # Because all Redis clusters are single nodes, we'll merge the
1210 # info from the cluster with info about the node
1211 dest = cluster['CacheNodes'][0]['Endpoint']['Address']
1212 is_redis = True
1213
1214 if not dest:
1215 # Skip clusters we cannot address (e.g. private VPC subnet)
1216 return
1217
1218 # Add to index
1219 self.index[dest] = [region, cluster['CacheClusterId']]
1220
1221 # Inventory: Group by instance ID (always a group of 1)
1222 if self.group_by_instance_id:
1223 self.inventory[cluster['CacheClusterId']] = [dest]
1224 if self.nested_groups:
1225 self.push_group(self.inventory, 'instances', cluster['CacheClusterId'])
1226
1227 # Inventory: Group by region
1228 if self.group_by_region and not is_redis:
1229 self.push(self.inventory, region, dest)
1230 if self.nested_groups:
1231 self.push_group(self.inventory, 'regions', region)
1232
1233 # Inventory: Group by availability zone
1234 if self.group_by_availability_zone and not is_redis:
1235 self.push(self.inventory, cluster['PreferredAvailabilityZone'], dest)
1236 if self.nested_groups:
1237 if self.group_by_region:
1238 self.push_group(self.inventory, region, cluster['PreferredAvailabilityZone'])
1239 self.push_group(self.inventory, 'zones', cluster['PreferredAvailabilityZone'])
1240
1241 # Inventory: Group by node type
1242 if self.group_by_instance_type and not is_redis:
1243 type_name = self.to_safe('type_' + cluster['CacheNodeType'])
1244 self.push(self.inventory, type_name, dest)
1245 if self.nested_groups:
1246 self.push_group(self.inventory, 'types', type_name)
1247
1248 # Inventory: Group by VPC (information not available in the current
1249 # AWS API version for ElastiCache)
1250
1251 # Inventory: Group by security group
1252 if self.group_by_security_group and not is_redis:
1253
1254 # Check for the existence of the 'SecurityGroups' key and also if
1255 # this key has some value. When the cluster is not placed in a SG
1256 # the query can return None here and cause an error.
1257 if 'SecurityGroups' in cluster and cluster['SecurityGroups'] is not None:
1258 for security_group in cluster['SecurityGroups']:
1259 key = self.to_safe("security_group_" + security_group['SecurityGroupId'])
1260 self.push(self.inventory, key, dest)
1261 if self.nested_groups:
1262 self.push_group(self.inventory, 'security_groups', key)
1263
1264 # Inventory: Group by engine
1265 if self.group_by_elasticache_engine and not is_redis:
1266 self.push(self.inventory, self.to_safe("elasticache_" + cluster['Engine']), dest)
1267 if self.nested_groups:
1268 self.push_group(self.inventory, 'elasticache_engines', self.to_safe(cluster['Engine']))
1269
1270 # Inventory: Group by parameter group
1271 if self.group_by_elasticache_parameter_group:
1272 self.push(self.inventory, self.to_safe("elasticache_parameter_group_" + cluster['CacheParameterGroup']['CacheParameterGroupName']), dest)
1273 if self.nested_groups:
1274 self.push_group(self.inventory, 'elasticache_parameter_groups', self.to_safe(cluster['CacheParameterGroup']['CacheParameterGroupName']))
1275
1276 # Inventory: Group by replication group
1277 if self.group_by_elasticache_replication_group and 'ReplicationGroupId' in cluster and cluster['ReplicationGroupId']:
1278 self.push(self.inventory, self.to_safe("elasticache_replication_group_" + cluster['ReplicationGroupId']), dest)
1279 if self.nested_groups:
1280 self.push_group(self.inventory, 'elasticache_replication_groups', self.to_safe(cluster['ReplicationGroupId']))
1281
1282 # Global Tag: all ElastiCache clusters
1283 self.push(self.inventory, 'elasticache_clusters', cluster['CacheClusterId'])
1284
1285 host_info = self.get_host_info_dict_from_describe_dict(cluster)
1286
1287 self.inventory["_meta"]["hostvars"][dest] = host_info
1288
1289 # Add the nodes
1290 for node in cluster['CacheNodes']:
1291 self.add_elasticache_node(node, cluster, region)
1292
1293 def add_elasticache_node(self, node, cluster, region):
1294 ''' Adds an ElastiCache node to the inventory and index, as long as
1295 it is addressable '''
1296
1297 # Only want available nodes unless all_elasticache_nodes is True
1298 if not self.all_elasticache_nodes and node['CacheNodeStatus'] != 'available':
1299 return
1300
1301 # Select the best destination address
1302 dest = node['Endpoint']['Address']
1303
1304 if not dest:
1305 # Skip nodes we cannot address (e.g. private VPC subnet)
1306 return
1307
1308 node_id = self.to_safe(cluster['CacheClusterId'] + '_' + node['CacheNodeId'])
1309
1310 # Add to index
1311 self.index[dest] = [region, node_id]
1312
1313 # Inventory: Group by node ID (always a group of 1)
1314 if self.group_by_instance_id:
1315 self.inventory[node_id] = [dest]
1316 if self.nested_groups:
1317 self.push_group(self.inventory, 'instances', node_id)
1318
1319 # Inventory: Group by region
1320 if self.group_by_region:
1321 self.push(self.inventory, region, dest)
1322 if self.nested_groups:
1323 self.push_group(self.inventory, 'regions', region)
1324
1325 # Inventory: Group by availability zone
1326 if self.group_by_availability_zone:
1327 self.push(self.inventory, cluster['PreferredAvailabilityZone'], dest)
1328 if self.nested_groups:
1329 if self.group_by_region:
1330 self.push_group(self.inventory, region, cluster['PreferredAvailabilityZone'])
1331 self.push_group(self.inventory, 'zones', cluster['PreferredAvailabilityZone'])
1332
1333 # Inventory: Group by node type
1334 if self.group_by_instance_type:
1335 type_name = self.to_safe('type_' + cluster['CacheNodeType'])
1336 self.push(self.inventory, type_name, dest)
1337 if self.nested_groups:
1338 self.push_group(self.inventory, 'types', type_name)
1339
1340 # Inventory: Group by VPC (information not available in the current
1341 # AWS API version for ElastiCache)
1342
1343 # Inventory: Group by security group
1344 if self.group_by_security_group:
1345
1346 # Check for the existence of the 'SecurityGroups' key and also if
1347 # this key has some value. When the cluster is not placed in a SG
1348 # the query can return None here and cause an error.
1349 if 'SecurityGroups' in cluster and cluster['SecurityGroups'] is not None:
1350 for security_group in cluster['SecurityGroups']:
1351 key = self.to_safe("security_group_" + security_group['SecurityGroupId'])
1352 self.push(self.inventory, key, dest)
1353 if self.nested_groups:
1354 self.push_group(self.inventory, 'security_groups', key)
1355
1356 # Inventory: Group by engine
1357 if self.group_by_elasticache_engine:
1358 self.push(self.inventory, self.to_safe("elasticache_" + cluster['Engine']), dest)
1359 if self.nested_groups:
1360 self.push_group(self.inventory, 'elasticache_engines', self.to_safe("elasticache_" + cluster['Engine']))
1361
1362 # Inventory: Group by parameter group (done at cluster level)
1363
1364 # Inventory: Group by replication group (done at cluster level)
1365
1366 # Inventory: Group by ElastiCache Cluster
1367 if self.group_by_elasticache_cluster:
1368 self.push(self.inventory, self.to_safe("elasticache_cluster_" + cluster['CacheClusterId']), dest)
1369
1370 # Global Tag: all ElastiCache nodes
1371 self.push(self.inventory, 'elasticache_nodes', dest)
1372
1373 host_info = self.get_host_info_dict_from_describe_dict(node)
1374
1375 if dest in self.inventory["_meta"]["hostvars"]:
1376 self.inventory["_meta"]["hostvars"][dest].update(host_info)
1377 else:
1378 self.inventory["_meta"]["hostvars"][dest] = host_info
1379
1380 def add_elasticache_replication_group(self, replication_group, region):
1381 ''' Adds an ElastiCache replication group to the inventory and index '''
1382
1383 # Only want available clusters unless all_elasticache_replication_groups is True
1384 if not self.all_elasticache_replication_groups and replication_group['Status'] != 'available':
1385 return
1386
1387 # Skip clusters we cannot address (e.g. private VPC subnet or clustered redis)
1388 if replication_group['NodeGroups'][0]['PrimaryEndpoint'] is None or \
1389 replication_group['NodeGroups'][0]['PrimaryEndpoint']['Address'] is None:
1390 return
1391
1392 # Select the best destination address (PrimaryEndpoint)
1393 dest = replication_group['NodeGroups'][0]['PrimaryEndpoint']['Address']
1394
1395 # Add to index
1396 self.index[dest] = [region, replication_group['ReplicationGroupId']]
1397
1398 # Inventory: Group by ID (always a group of 1)
1399 if self.group_by_instance_id:
1400 self.inventory[replication_group['ReplicationGroupId']] = [dest]
1401 if self.nested_groups:
1402 self.push_group(self.inventory, 'instances', replication_group['ReplicationGroupId'])
1403
1404 # Inventory: Group by region
1405 if self.group_by_region:
1406 self.push(self.inventory, region, dest)
1407 if self.nested_groups:
1408 self.push_group(self.inventory, 'regions', region)
1409
1410 # Inventory: Group by availability zone (doesn't apply to replication groups)
1411
1412 # Inventory: Group by node type (doesn't apply to replication groups)
1413
1414 # Inventory: Group by VPC (information not available in the current
1415 # AWS API version for replication groups
1416
1417 # Inventory: Group by security group (doesn't apply to replication groups)
1418 # Check this value in cluster level
1419
1420 # Inventory: Group by engine (replication groups are always Redis)
1421 if self.group_by_elasticache_engine:
1422 self.push(self.inventory, 'elasticache_redis', dest)
1423 if self.nested_groups:
1424 self.push_group(self.inventory, 'elasticache_engines', 'redis')
1425
1426 # Global Tag: all ElastiCache clusters
1427 self.push(self.inventory, 'elasticache_replication_groups', replication_group['ReplicationGroupId'])
1428
1429 host_info = self.get_host_info_dict_from_describe_dict(replication_group)
1430
1431 self.inventory["_meta"]["hostvars"][dest] = host_info
1432
1433 def get_route53_records(self):
1434 ''' Get and store the map of resource records to domain names that
1435 point to them. '''
1436
1437 if self.boto_profile:
1438 r53_conn = route53.Route53Connection(profile_name=self.boto_profile)
1439 else:
1440 r53_conn = route53.Route53Connection()
1441 all_zones = r53_conn.get_zones()
1442
1443 route53_zones = [zone for zone in all_zones if zone.name[:-1] not in self.route53_excluded_zones]
1444
1445 self.route53_records = {}
1446
1447 for zone in route53_zones:
1448 rrsets = r53_conn.get_all_rrsets(zone.id)
1449
1450 for record_set in rrsets:
1451 record_name = record_set.name
1452
1453 if record_name.endswith('.'):
1454 record_name = record_name[:-1]
1455
1456 for resource in record_set.resource_records:
1457 self.route53_records.setdefault(resource, set())
1458 self.route53_records[resource].add(record_name)
1459
1460 def get_instance_route53_names(self, instance):
1461 ''' Check if an instance is referenced in the records we have from
1462 Route53. If it is, return the list of domain names pointing to said
1463 instance. If nothing points to it, return an empty list. '''
1464
1465 instance_attributes = ['public_dns_name', 'private_dns_name',
1466 'ip_address', 'private_ip_address']
1467
1468 name_list = set()
1469
1470 for attrib in instance_attributes:
1471 try:
1472 value = getattr(instance, attrib)
1473 except AttributeError:
1474 continue
1475
1476 if value in self.route53_records:
1477 name_list.update(self.route53_records[value])
1478
1479 return list(name_list)
1480
1481 def get_host_info_dict_from_instance(self, instance):
1482 instance_vars = {}
1483 for key in vars(instance):
1484 value = getattr(instance, key)
1485 key = self.to_safe('ec2_' + key)
1486
1487 # Handle complex types
1488 # state/previous_state changed to properties in boto in https://github.com/boto/boto/commit/a23c379837f698212252720d2af8dec0325c9518
1489 if key == 'ec2__state':
1490 instance_vars['ec2_state'] = instance.state or ''
1491 instance_vars['ec2_state_code'] = instance.state_code
1492 elif key == 'ec2__previous_state':
1493 instance_vars['ec2_previous_state'] = instance.previous_state or ''
1494 instance_vars['ec2_previous_state_code'] = instance.previous_state_code
1495 elif isinstance(value, (int, bool)):
1496 instance_vars[key] = value
1497 elif isinstance(value, six.string_types):
1498 instance_vars[key] = value.strip()
1499 elif value is None:
1500 instance_vars[key] = ''
1501 elif key == 'ec2_region':
1502 instance_vars[key] = value.name
1503 elif key == 'ec2__placement':
1504 instance_vars['ec2_placement'] = value.zone
1505 elif key == 'ec2_tags':
1506 for k, v in value.items():
1507 if self.expand_csv_tags and ',' in v:
1508 v = list(map(lambda x: x.strip(), v.split(',')))
1509 key = self.to_safe('ec2_tag_' + k)
1510 instance_vars[key] = v
1511 elif key == 'ec2_groups':
1512 group_ids = []
1513 group_names = []
1514 for group in value:
1515 group_ids.append(group.id)
1516 group_names.append(group.name)
1517 instance_vars["ec2_security_group_ids"] = ','.join([str(i) for i in group_ids])
1518 instance_vars["ec2_security_group_names"] = ','.join([str(i) for i in group_names])
1519 elif key == 'ec2_block_device_mapping':
1520 instance_vars["ec2_block_devices"] = {}
1521 for k, v in value.items():
1522 instance_vars["ec2_block_devices"][os.path.basename(k)] = v.volume_id
1523 else:
1524 pass
1525 # TODO Product codes if someone finds them useful
1526 # print key
1527 # print type(value)
1528 # print value
1529
1530 instance_vars[self.to_safe('ec2_account_id')] = self.aws_account_id
1531
1532 return instance_vars
1533
1534 def get_host_info_dict_from_describe_dict(self, describe_dict):
1535 ''' Parses the dictionary returned by the API call into a flat list
1536 of parameters. This method should be used only when 'describe' is
1537 used directly because Boto doesn't provide specific classes. '''
1538
1539 # I really don't agree with prefixing everything with 'ec2'
1540 # because EC2, RDS and ElastiCache are different services.
1541 # I'm just following the pattern used until now to not break any
1542 # compatibility.
1543
1544 host_info = {}
1545 for key in describe_dict:
1546 value = describe_dict[key]
1547 key = self.to_safe('ec2_' + self.uncammelize(key))
1548
1549 # Handle complex types
1550
1551 # Target: Memcached Cache Clusters
1552 if key == 'ec2_configuration_endpoint' and value:
1553 host_info['ec2_configuration_endpoint_address'] = value['Address']
1554 host_info['ec2_configuration_endpoint_port'] = value['Port']
1555
1556 # Target: Cache Nodes and Redis Cache Clusters (single node)
1557 if key == 'ec2_endpoint' and value:
1558 host_info['ec2_endpoint_address'] = value['Address']
1559 host_info['ec2_endpoint_port'] = value['Port']
1560
1561 # Target: Redis Replication Groups
1562 if key == 'ec2_node_groups' and value:
1563 host_info['ec2_endpoint_address'] = value[0]['PrimaryEndpoint']['Address']
1564 host_info['ec2_endpoint_port'] = value[0]['PrimaryEndpoint']['Port']
1565 replica_count = 0
1566 for node in value[0]['NodeGroupMembers']:
1567 if node['CurrentRole'] == 'primary':
1568 host_info['ec2_primary_cluster_address'] = node['ReadEndpoint']['Address']
1569 host_info['ec2_primary_cluster_port'] = node['ReadEndpoint']['Port']
1570 host_info['ec2_primary_cluster_id'] = node['CacheClusterId']
1571 elif node['CurrentRole'] == 'replica':
1572 host_info['ec2_replica_cluster_address_' + str(replica_count)] = node['ReadEndpoint']['Address']
1573 host_info['ec2_replica_cluster_port_' + str(replica_count)] = node['ReadEndpoint']['Port']
1574 host_info['ec2_replica_cluster_id_' + str(replica_count)] = node['CacheClusterId']
1575 replica_count += 1
1576
1577 # Target: Redis Replication Groups
1578 if key == 'ec2_member_clusters' and value:
1579 host_info['ec2_member_clusters'] = ','.join([str(i) for i in value])
1580
1581 # Target: All Cache Clusters
1582 elif key == 'ec2_cache_parameter_group':
1583 host_info["ec2_cache_node_ids_to_reboot"] = ','.join([str(i) for i in value['CacheNodeIdsToReboot']])
1584 host_info['ec2_cache_parameter_group_name'] = value['CacheParameterGroupName']
1585 host_info['ec2_cache_parameter_apply_status'] = value['ParameterApplyStatus']
1586
1587 # Target: Almost everything
1588 elif key == 'ec2_security_groups':
1589
1590 # Skip if SecurityGroups is None
1591 # (it is possible to have the key defined but no value in it).
1592 if value is not None:
1593 sg_ids = []
1594 for sg in value:
1595 sg_ids.append(sg['SecurityGroupId'])
1596 host_info["ec2_security_group_ids"] = ','.join([str(i) for i in sg_ids])
1597
1598 # Target: Everything
1599 # Preserve booleans and integers
1600 elif isinstance(value, (int, bool)):
1601 host_info[key] = value
1602
1603 # Target: Everything
1604 # Sanitize string values
1605 elif isinstance(value, six.string_types):
1606 host_info[key] = value.strip()
1607
1608 # Target: Everything
1609 # Replace None by an empty string
1610 elif value is None:
1611 host_info[key] = ''
1612
1613 else:
1614 # Remove non-processed complex types
1615 pass
1616
1617 return host_info
1618
1619 def get_host_info(self):
1620 ''' Get variables about a specific host '''
1621
1622 if len(self.index) == 0:
1623 # Need to load index from cache
1624 self.load_index_from_cache()
1625
1626 if self.args.host not in self.index:
1627 # try updating the cache
1628 self.do_api_calls_update_cache()
1629 if self.args.host not in self.index:
1630 # host might not exist anymore
1631 return self.json_format_dict({}, True)
1632
1633 (region, instance_id) = self.index[self.args.host]
1634
1635 instance = self.get_instance(region, instance_id)
1636 return self.json_format_dict(self.get_host_info_dict_from_instance(instance), True)
1637
1638 def push(self, my_dict, key, element):
1639 ''' Push an element onto an array that may not have been defined in
1640 the dict '''
1641 group_info = my_dict.setdefault(key, [])
1642 if isinstance(group_info, dict):
1643 host_list = group_info.setdefault('hosts', [])
1644 host_list.append(element)
1645 else:
1646 group_info.append(element)
1647
1648 def push_group(self, my_dict, key, element):
1649 ''' Push a group as a child of another group. '''
1650 parent_group = my_dict.setdefault(key, {})
1651 if not isinstance(parent_group, dict):
1652 parent_group = my_dict[key] = {'hosts': parent_group}
1653 child_groups = parent_group.setdefault('children', [])
1654 if element not in child_groups:
1655 child_groups.append(element)
1656
1657 def get_inventory_from_cache(self):
1658 ''' Reads the inventory from the cache file and returns it as a JSON
1659 object '''
1660
1661 with open(self.cache_path_cache, 'r') as f:
1662 json_inventory = f.read()
1663 return json_inventory
1664
1665 def load_index_from_cache(self):
1666 ''' Reads the index from the cache file sets self.index '''
1667
1668 with open(self.cache_path_index, 'rb') as f:
1669 self.index = json.load(f)
1670
1671 def write_to_cache(self, data, filename):
1672 ''' Writes data in JSON format to a file '''
1673
1674 json_data = self.json_format_dict(data, True)
1675 with open(filename, 'w') as f:
1676 f.write(json_data)
1677
1678 def uncammelize(self, key):
1679 temp = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', key)
1680 return re.sub('([a-z0-9])([A-Z])', r'\1_\2', temp).lower()
1681
1682 def to_safe(self, word):
1683 ''' Converts 'bad' characters in a string to underscores so they can be used as Ansible groups '''
1684 regex = r"[^A-Za-z0-9\_"
1685 if not self.replace_dash_in_groups:
1686 regex += r"\-"
1687 return re.sub(regex + "]", "_", word)
1688
1689 def json_format_dict(self, data, pretty=False):
1690 ''' Converts a dict to a JSON object and dumps it as a formatted
1691 string '''
1692
1693 if pretty:
1694 return json.dumps(data, sort_keys=True, indent=2)
1695 else:
1696 return json.dumps(data)
1697
1698
1699if __name__ == '__main__':
1700 # Run the script
1701 Ec2Inventory()