Package starcluster :: Module config
[hide private]
[frames] | no frames]

Source Code for Module starcluster.config

  1  #!/usr/bin/env python 
  2  import os 
  3  import sys 
  4  import urllib 
  5  import ConfigParser 
  6   
  7  from starcluster.cluster import Cluster 
  8  from starcluster import static  
  9  from starcluster import awsutils  
 10  from starcluster import utils 
 11  from starcluster.utils import AttributeDict 
 12  from starcluster import exception  
 13   
 14  from starcluster.logger import log 
15 16 -def get_easy_s3():
17 """ 18 Factory for EasyS3 class that attempts to load AWS credentials from 19 the StarCluster config file. Returns an EasyS3 object if 20 successful. 21 """ 22 cfg = StarClusterConfig(); cfg.load() 23 return cfg.get_easy_s3()
24
25 -def get_easy_ec2():
26 """ 27 Factory for EasyEC2 class that attempts to load AWS credentials from 28 the StarCluster config file. Returns an EasyEC2 object if 29 successful. 30 """ 31 cfg = StarClusterConfig(); cfg.load() 32 return cfg.get_easy_ec2()
33
34 35 -def get_config(config_file=None, cache=False):
36 """Factory for StarClusterConfig object""" 37 return StarClusterConfig(config_file, cache)
38
39 -class StarClusterConfig(object):
40 """ 41 Loads StarCluster configuration settings defined in config_file 42 which defaults to ~/.starclustercfg 43 44 Settings are available as follows: 45 46 cfg = StarClusterConfig() 47 or 48 cfg = StarClusterConfig('/path/to/my/config.cfg') 49 cfg.load() 50 aws_info = cfg.aws 51 cluster_cfg = cfg.clusters['mycluster'] 52 key_cfg = cfg.keys['gsg-keypair'] 53 print cluster_cfg 54 """ 55 56 global_settings = static.GLOBAL_SETTINGS 57 aws_settings = static.AWS_SETTINGS 58 key_settings = static.KEY_SETTINGS 59 volume_settings = static.EBS_VOLUME_SETTINGS 60 plugin_settings = static.PLUGIN_SETTINGS 61 cluster_settings = static.CLUSTER_SETTINGS 62 63 # until i can find a way to query AWS for instance types... 64 instance_types = static.INSTANCE_TYPES 65
66 - def __init__(self, config_file=None, cache=False):
67 self.cfg_file = config_file 68 self.type_validators = { 69 int: self._get_int, 70 str: self._get_string, 71 bool: self._get_bool, 72 } 73 self._config = None 74 self.globals = AttributeDict() 75 self.aws = AttributeDict() 76 self.clusters = AttributeDict() 77 self.keys = AttributeDict() 78 self.vols = AttributeDict() 79 self.plugins = AttributeDict() 80 self.cache = cache
81
82 - def _get_urlfp(self, url):
83 log.debug("Loading url: %s" % url) 84 import socket 85 try: 86 fp = urllib.urlopen(url) 87 if fp.getcode() == 404: 88 raise exception.ConfigError("url %s does not exist" % url) 89 fp.name = url 90 return fp 91 except IOError,e: 92 raise exception.ConfigError("error loading config from url %s\n%s" 93 % (url,e))
94
95 - def _get_fp(self, config_file):
96 if not os.path.isdir(static.STARCLUSTER_CFG_DIR): 97 os.makedirs(static.STARCLUSTER_CFG_DIR) 98 cfg_file = config_file or static.STARCLUSTER_CFG_FILE 99 log.debug("Loading file: %s" % cfg_file) 100 if os.path.exists(cfg_file): 101 if not os.path.isfile(cfg_file): 102 raise exception.ConfigError('config %s exists but is not a regular file' % 103 cfg_file) 104 else: 105 raise exception.ConfigNotFound( 106 ("config file %s does not exist\n") % 107 cfg_file, cfg_file, 108 ) 109 return open(cfg_file)
110
111 - def _get_bool(self, config, section, option):
112 try: 113 opt = config.getboolean(section,option) 114 return opt 115 except ConfigParser.NoSectionError,e: 116 pass 117 except ConfigParser.NoOptionError,e: 118 pass 119 except ValueError,e: 120 raise exception.ConfigError( 121 "Expected True/False value for setting %s in section [%s]" % (option,section))
122
123 - def _get_int(self, config, section, option):
124 try: 125 opt = config.getint(section,option) 126 return opt 127 except ConfigParser.NoSectionError,e: 128 pass 129 except ConfigParser.NoOptionError,e: 130 pass 131 except ValueError,e: 132 raise exception.ConfigError( 133 "Expected integer value for setting %s in section [%s]" % (option,section))
134
135 - def _get_string(self, config, section, option):
136 try: 137 opt = config.get(section,option) 138 return opt 139 except ConfigParser.NoSectionError,e: 140 pass 141 except ConfigParser.NoOptionError,e: 142 pass
143
144 - def __load_config(self):
145 """ 146 Populates self._config with a new ConfigParser instance 147 """ 148 cfg = self.cfg_file 149 if utils.is_url(cfg): 150 cfg = self._get_urlfp(cfg) 151 else: 152 cfg = self._get_fp(cfg) 153 try: 154 cp = ConfigParser.ConfigParser() 155 cp.readfp(cfg) 156 return cp 157 except ConfigParser.MissingSectionHeaderError,e: 158 raise exception.ConfigHasNoSections(cfg.name) 159 except ConfigParser.ParsingError,e: 160 raise exception.ConfigError(e)
161
162 - def reload(self):
163 """ 164 Reloads the configuration file 165 """ 166 self._config = self.__load_config() 167 self.load()
168 169 @property
170 - def config(self):
171 cfg = self.cfg_file 172 if self._config is None: 173 self._config = self.__load_config() 174 return self._config
175
176 - def load_settings(self, section_name, settings, store):
177 section = self.config._sections.get(section_name) 178 if not section: 179 raise exception.ConfigSectionMissing('Missing section %s in config'\ 180 % section_name) 181 store.update(section) 182 section_conf = store 183 for setting in settings: 184 requirements = settings[setting] 185 name = setting 186 func = self.type_validators.get(requirements[0]) 187 required = requirements[1]; 188 default = requirements[2] 189 value = func(self.config, section_name, name) 190 if value is not None: 191 section_conf[name.lower()] = value
192
193 - def check_required(self, section_name, settings, store):
194 section_conf = store 195 for setting in settings: 196 name = setting 197 requirements = settings[setting] 198 required = requirements[1]; 199 value = section_conf.get(name.lower()) 200 if value is None and required: 201 raise exception.ConfigError('missing required option %s in section "%s"' % 202 (name.lower(), section_name))
203
204 - def load_defaults(self, settings, store):
205 section_conf = store 206 for setting in settings: 207 name = setting.lower(); default = settings[setting][2] 208 if section_conf.get(name, None) is None: 209 if default: 210 log.debug('%s setting not specified. Defaulting to %s' % (name, default)) 211 section_conf[name] = default
212
213 - def load_extends_variables(self, section_name, store):
214 section = store[section_name] 215 extends = section['extends'] = section.get('extends') 216 default = section.get('default') 217 if extends is None: 218 return 219 log.debug('%s extends %s' % (section_name, extends)) 220 extensions = [section] 221 while True: 222 extends = section.get('extends',None) 223 if extends: 224 try: 225 section = store[extends] 226 extensions.insert(0, section) 227 except KeyError,e: 228 log.warn("can't extend non-existent section %s" % extends) 229 break 230 else: 231 break 232 transform = AttributeDict() 233 for extension in extensions: 234 transform.update(extension) 235 # do not inherit default setting from other cluster sections 236 transform.update(dict(default=default)) 237 store[section_name] = transform
238
239 - def load_keypairs(self, section_name, store):
240 cluster_section = store 241 keyname = cluster_section.get('keyname') 242 if not keyname: 243 return 244 keypair = self.keys.get(keyname) 245 if keypair is None: 246 raise exception.ConfigError("keypair %s not defined in config" % keyname) 247 cluster_section['keyname'] = keyname 248 cluster_section['key_location'] = keypair.get('key_location')
249
250 - def load_volumes(self, section_name, store):
251 cluster_section = store 252 volumes = cluster_section.get('volumes') 253 if not volumes or isinstance(volumes, AttributeDict): 254 return 255 vols = AttributeDict() 256 cluster_section['volumes'] = vols 257 volumes = [vol.strip() for vol in volumes.split(',')] 258 for volume in volumes: 259 if not self.vols.has_key(volume): 260 raise exception.ConfigError("volume %s not defined in config" % volume) 261 vol = self.vols.get(volume) 262 vols[volume] = vol
263
264 - def load_plugins(self, section_name, store):
265 cluster_section = store 266 plugins = cluster_section.get('plugins') 267 if not plugins or isinstance(plugins, list): 268 return 269 plugs = [] 270 cluster_section['plugins'] = plugs 271 plugins = [plugin.strip() for plugin in plugins.split(',')] 272 for plugin in plugins: 273 if self.plugins.has_key(plugin): 274 p = self.plugins.get(plugin) 275 p['__name__'] = p['__name__'].split()[-1] 276 plugs.append(p) 277 else: 278 raise exception.ConfigError("plugin %s not defined in config" % plugin)
279
280 - def load(self):
281 log.debug('Loading config') 282 try: 283 self.load_settings('global', self.global_settings, self.globals) 284 except exception.ConfigSectionMissing,e: 285 pass 286 try: 287 self.load_settings('aws info', self.aws_settings, self.aws) 288 self.check_required('aws info', self.aws_settings, self.aws) 289 except exception.ConfigSectionMissing,e: 290 log.warn("no [aws info] section found in config") 291 log.warn("attempting to load credentials from environment...") 292 self.aws.update(self.get_aws_from_environ()) 293 keys = [section.split()[1] for section in self.config.sections() if 294 section.startswith('key')] 295 for key in keys: 296 self.keys[key] = AttributeDict() 297 section_name = 'key ' + key 298 self.load_settings(section_name, self.key_settings, self.keys[key]) 299 self.check_required(section_name, self.key_settings, self.keys[key]) 300 vols = [section.split()[1] for section in self.config.sections() if 301 section.startswith('volume')] 302 for vol in vols: 303 self.vols[vol] = AttributeDict() 304 section_name = 'volume ' + vol 305 self.load_settings(section_name, self.volume_settings, 306 self.vols[vol]) 307 self.load_defaults(self.volume_settings, self.vols[vol]) 308 self.check_required(section_name, self.volume_settings, 309 self.vols[vol]) 310 plugins = [section.split()[1] for section in self.config.sections() if 311 section.startswith('plugin')] 312 for plugin in plugins: 313 self.plugins[plugin] = AttributeDict() 314 section_name = 'plugin ' + plugin 315 self.load_settings(section_name, self.plugin_settings, 316 self.plugins[plugin]) 317 self.check_required(section_name, self.plugin_settings, 318 self.plugins[plugin]) 319 clusters = [section.split()[1] for section in self.config.sections() if 320 section.startswith('cluster')] 321 for cluster in clusters: 322 self.clusters[cluster] = AttributeDict() 323 section_name = 'cluster ' + cluster 324 self.load_settings(section_name, self.cluster_settings, 325 self.clusters[cluster]) 326 for cluster in clusters: 327 section_name = 'cluster ' + cluster 328 self.load_extends_variables(cluster, self.clusters) 329 self.load_defaults(self.cluster_settings, self.clusters[cluster]) 330 self.load_keypairs(cluster, self.clusters[cluster]) 331 self.load_volumes(cluster, self.clusters[cluster]) 332 self.load_plugins(cluster, self.clusters[cluster]) 333 self.check_required(section_name, self.cluster_settings, 334 self.clusters[cluster])
335
336 - def get_aws_from_environ(self):
337 """Returns AWS credentials defined in the user's shell 338 environment.""" 339 awscreds = {} 340 for key in static.AWS_SETTINGS: 341 if os.environ.has_key(key): 342 awscreds[key.lower()] = os.environ.get(key) 343 return awscreds
344
345 - def get_aws_credentials(self):
346 """ 347 Returns AWS credentials defined in the configuration 348 file. Defining any of the AWS settings in the environment 349 overrides the configuration file. 350 """ 351 # first override with environment settings if they exist 352 self.aws.update(self.get_aws_from_environ()) 353 return self.aws
354
355 - def get_cluster_names(self):
356 return self.clusters
357
358 - def get_cluster_template(self, template_name, tag_name=None):
359 """ 360 Returns Cluster instance configured with the settings in the config file. 361 362 template_name is the name of a cluster section defined in the config 363 364 tag_name, if specified, will be passed to Cluster instance as cluster_tag 365 """ 366 try: 367 kwargs = {} 368 if tag_name: 369 kwargs.update(dict(cluster_tag=tag_name)) 370 kwargs.update(**self.aws) 371 kwargs.update(self.clusters[template_name]) 372 clust = Cluster(**kwargs) 373 return clust 374 except KeyError,e: 375 raise exception.ClusterTemplateDoesNotExist(template_name)
376
377 - def get_default_cluster_template(self, tag_name=None):
378 """ 379 Returns the cluster template with "DEFAULT=True" in the config 380 If more than one found, raises MultipleDefaultTemplates exception. 381 If no cluster template has "DEFAULT=True", raises NoDefaultTemplateFound 382 exception. 383 """ 384 default = self.globals.get('default_template') 385 if not default: 386 raise exception.NoDefaultTemplateFound(options=self.clusters.keys()) 387 if not self.clusters.has_key(default): 388 raise exception.ClusterTemplateDoesNotExist(default) 389 return default
390
391 - def get_clusters(self):
392 clusters = [] 393 for cluster in self.clusters: 394 clusters.append(self.get_cluster_template(cluster)) 395 return clusters
396
397 - def get_plugin(self, plugin):
398 try: 399 return self.plugins[plugin] 400 except KeyError: 401 raise exception.PluginNotFound(plugin)
402
403 - def get_key(self, keyname):
404 try: 405 return self.keys[keyname] 406 except KeyError: 407 raise exception.KeyNotFound(keyname)
408
409 - def get_easy_s3(self):
410 """ 411 Factory for EasyEC2 class that attempts to load AWS credentials from 412 the StarCluster config file. Returns an EasyS3 object if 413 successful. 414 """ 415 aws = self.get_aws_credentials() 416 try: 417 s3 = awsutils.EasyS3(**aws) 418 return s3 419 except TypeError,e: 420 raise exception.ConfigError("no aws credentials found")
421
422 - def get_easy_ec2(self):
423 """ 424 Factory for EasyEC2 class that attempts to load AWS credentials from 425 the StarCluster config file. Returns an EasyEC2 object if 426 successful. 427 """ 428 aws = self.get_aws_credentials() 429 try: 430 ec2 = awsutils.EasyEC2(**aws) 431 return ec2 432 except TypeError,e: 433 raise exception.ConfigError("no aws credentials found")
434 435 if __name__ == "__main__": 436 from pprint import pprint 437 cfg = StarClusterConfig(); cfg.load() 438 pprint(cfg.aws) 439 pprint(cfg.clusters) 440 pprint(cfg.keys) 441 pprint(cfg.vols) 442