79
def vpn_ping(address, port, timeout=0.05, session_id=None):
80
"""Sends a vpn negotiation packet and returns the server session.
82
Returns False on a failure. Basic packet structure is below.
84
Client packet (14 bytes)::
89
x = packet identifier 0x38
90
cli_id = 64 bit identifier
91
? = unknown, probably flags/padding
93
Server packet (26 bytes)::
95
+-+--------+-----+--------+----+
96
|x| srv_id |?????| cli_id |????|
97
+-+--------+-----+--------+----+
98
x = packet identifier 0x40
99
cli_id = 64 bit identifier
100
? = unknown, probably flags/padding
101
bit 9 was 1 and the rest were 0 in testing
104
if session_id is None:
105
session_id = random.randint(0, 0xffffffffffffffff)
106
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
107
data = struct.pack('!BQxxxxxx', 0x38, session_id)
108
sock.sendto(data, (address, port))
109
sock.settimeout(timeout)
111
received = sock.recv(2048)
112
except socket.timeout:
116
fmt = '!BQxxxxxQxxxx'
117
if len(received) != struct.calcsize(fmt):
118
print struct.calcsize(fmt)
120
(identifier, server_sess, client_sess) = struct.unpack(fmt, received)
121
if identifier == 0x40 and client_sess == session_id:
125
71
def fetchfile(url, target):
126
LOG.debug(_('Fetching %s') % url)
127
execute('curl', '--fail', url, '-o', target)
130
def execute(*cmd, **kwargs):
132
Helper method to execute command with optional retry.
134
:cmd Passed to subprocess.Popen.
135
:process_input Send to opened process.
136
:check_exit_code Defaults to 0. Raise exception.ProcessExecutionError
137
unless program exits with this code.
138
:delay_on_retry True | False. Defaults to True. If set to True, wait a
139
short amount of time before retrying.
140
:attempts How many times to retry cmd.
141
:run_as_root True | False. Defaults to False. If set to True,
142
the command is prefixed by the command specified
143
in the root_helper FLAG.
145
:raises exception.Error on receiving unknown arguments
146
:raises exception.ProcessExecutionError
149
process_input = kwargs.pop('process_input', None)
150
check_exit_code = kwargs.pop('check_exit_code', 0)
151
delay_on_retry = kwargs.pop('delay_on_retry', True)
152
attempts = kwargs.pop('attempts', 1)
153
run_as_root = kwargs.pop('run_as_root', False)
155
raise exception.Error(_('Got unknown keyword args '
156
'to utils.execute: %r') % kwargs)
159
cmd = shlex.split(FLAGS.root_helper) + list(cmd)
165
LOG.debug(_('Running cmd (subprocess): %s'), ' '.join(cmd))
166
_PIPE = subprocess.PIPE # pylint: disable=E1101
167
obj = subprocess.Popen(cmd,
173
if process_input is not None:
174
result = obj.communicate(process_input)
176
result = obj.communicate()
177
obj.stdin.close() # pylint: disable=E1101
178
_returncode = obj.returncode # pylint: disable=E1101
180
LOG.debug(_('Result was %s') % _returncode)
181
if type(check_exit_code) == types.IntType \
182
and _returncode != check_exit_code:
183
(stdout, stderr) = result
184
raise exception.ProcessExecutionError(
185
exit_code=_returncode,
190
except exception.ProcessExecutionError:
194
LOG.debug(_('%r failed. Retrying.'), cmd)
196
greenthread.sleep(random.randint(20, 200) / 100.0)
198
# NOTE(termie): this appears to be necessary to let the subprocess
199
# call clean something up in between calls, without
200
# it two execute calls in a row hangs the second one
204
def ssh_execute(ssh, cmd, process_input=None,
205
addl_env=None, check_exit_code=True):
206
LOG.debug(_('Running cmd (SSH): %s'), ' '.join(cmd))
72
logging.debug("Fetching %s" % url)
74
# fp = open(target, "wb")
75
# c.setopt(c.URL, url)
76
# c.setopt(c.WRITEDATA, fp)
80
execute("curl --fail %s -o %s" % (url, target))
83
def execute(cmd, process_input=None, addl_env=None, check_exit_code=True):
84
logging.debug("Running cmd: %s", cmd)
85
env = os.environ.copy()
208
raise exception.Error(_('Environment not supported over SSH'))
211
# This is (probably) fixable if we need it...
212
raise exception.Error(_('process_input not supported over SSH'))
214
stdin_stream, stdout_stream, stderr_stream = ssh.exec_command(cmd)
215
channel = stdout_stream.channel
217
#stdin.write('process_input would go here')
220
# NOTE(justinsb): This seems suspicious...
221
# ...other SSH clients have buffering issues with this approach
222
stdout = stdout_stream.read()
223
stderr = stderr_stream.read()
226
exit_status = channel.recv_exit_status()
228
# exit_status == -1 if no exit code was returned
229
if exit_status != -1:
230
LOG.debug(_('Result was %s') % exit_status)
231
if check_exit_code and exit_status != 0:
232
raise exception.ProcessExecutionError(exit_code=exit_status,
237
return (stdout, stderr)
88
obj = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE,
89
stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
91
if process_input != None:
92
result = obj.communicate(process_input)
94
result = obj.communicate()
97
logging.debug("Result was %s" % (obj.returncode))
98
if check_exit_code and obj.returncode != 0:
99
(stdout, stderr) = result
100
raise ProcessExecutionError(exit_code=obj.returncode,
241
108
return os.path.join(os.path.dirname(__file__), s)
246
return os.path.abspath(nova.__file__).split('nova/__init__.py')[0]
249
def default_flagfile(filename='nova.conf', args=None):
111
def default_flagfile(filename='nova.conf'):
253
113
if arg.find('flagfile') != -1:
282
144
return '%s-%s' % (topic, ''.join(choices))
285
# Default symbols to use for passwords. Avoids visually confusing characters.
287
DEFAULT_PASSWORD_SYMBOLS = ('23456789' # Removed: 0,1
288
'ABCDEFGHJKLMNPQRSTUVWXYZ' # Removed: I, O
289
'abcdefghijkmnopqrstuvwxyz') # Removed: l
293
EASIER_PASSWORD_SYMBOLS = ('23456789' # Removed: 0, 1
294
'ABCDEFGHJKLMNPQRSTUVWXYZ') # Removed: I, O
297
def usage_from_instance(instance_ref, **kw):
299
project_id=instance_ref['project_id'],
300
user_id=instance_ref['user_id'],
301
instance_id=instance_ref['id'],
302
instance_type=instance_ref['instance_type']['name'],
303
instance_type_id=instance_ref['instance_type_id'],
304
display_name=instance_ref['display_name'],
305
created_at=str(instance_ref['created_at']),
306
launched_at=str(instance_ref['launched_at']) \
307
if instance_ref['launched_at'] else '',
308
image_ref=instance_ref['image_ref'])
309
usage_info.update(kw)
313
def generate_password(length=20, symbols=DEFAULT_PASSWORD_SYMBOLS):
314
"""Generate a random password from the supplied symbols.
316
Believed to be reasonably secure (with a reasonable password length!)
319
r = random.SystemRandom()
320
return ''.join([r.choice(symbols) for _i in xrange(length)])
148
mac = [0x02, 0x16, 0x3e,
149
random.randint(0x00, 0x7f),
150
random.randint(0x00, 0xff),
151
random.randint(0x00, 0xff)]
152
return ':'.join(map(lambda x: "%02x" % x, mac))
323
155
def last_octet(address):
324
return int(address.split('.')[-1])
156
return int(address.split(".")[-1])
160
"""Returns the actual ip of the local machine."""
161
if getattr(FLAGS, 'fake_tests', None):
164
csock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
165
csock.connect(('8.8.8.8', 80))
166
(addr, port) = csock.getsockname()
169
except socket.gaierror as ex:
170
logging.warn("Couldn't get IP, using 127.0.0.1 %s", ex)
327
174
def get_my_linklocal(interface):
175
if getattr(FLAGS, 'fake_tests', None):
329
if_str = execute('ip', '-f', 'inet6', '-o', 'addr', 'show', interface)
330
condition = '\s+inet6\s+([0-9a-f:]+)/\d+\s+scope\s+link'
178
if_str = execute("ifconfig %s" % interface)
179
condition = "\s+inet6\s+addr:\s+([0-9a-f:]+/\d+)\s+Scope:Link"
331
180
links = [re.search(condition, x) for x in if_str[0].split('\n')]
332
181
address = [w.group(1) for w in links if w is not None]
333
182
if address[0] is not None:
334
183
return address[0]
336
raise exception.Error(_('Link Local address is not found.:%s')
338
except Exception as ex:
339
raise exception.Error(_("Couldn't get Link Local IP of %(interface)s"
340
" :%(ex)s") % locals())
344
"""Overridable version of utils.utcnow."""
345
if utcnow.override_time:
346
return utcnow.override_time
347
return datetime.datetime.utcnow()
350
utcnow.override_time = None
353
def is_older_than(before, seconds):
354
"""Return True if before is older than seconds."""
355
return utcnow() - before > datetime.timedelta(seconds=seconds)
359
"""Timestamp version of our utcnow function."""
360
return time.mktime(utcnow().timetuple())
363
def set_time_override(override_time=datetime.datetime.utcnow()):
364
"""Override utils.utcnow to return a constant time."""
365
utcnow.override_time = override_time
368
def advance_time_delta(timedelta):
369
"""Advance overriden time using a datetime.timedelta."""
370
assert(not utcnow.override_time is None)
371
utcnow.override_time += timedelta
374
def advance_time_seconds(seconds):
375
"""Advance overriden time by seconds."""
376
advance_time_delta(datetime.timedelta(0, seconds))
379
def clear_time_override():
380
"""Remove the overridden time."""
381
utcnow.override_time = None
384
def strtime(at=None, fmt=PERFECT_TIME_FORMAT):
385
"""Returns formatted utcnow."""
186
except RuntimeError as ex:
187
logging.warn("Couldn't get Link Local IP of %s :%s", interface, ex)
191
def to_global_ipv6(prefix, mac):
192
mac64 = netaddr.EUI(mac).eui64().words
193
int_addr = int(''.join(['%02x' % i for i in mac64]), 16)
194
mac64_addr = netaddr.IPAddress(int_addr)
195
maskIP = netaddr.IPNetwork(prefix).ip
196
return (mac64_addr ^ netaddr.IPAddress('::0200:0:0:0') | maskIP).format()
199
def to_mac(ipv6_address):
200
address = netaddr.IPAddress(ipv6_address)
201
mask1 = netaddr.IPAddress("::ffff:ffff:ffff:ffff")
202
mask2 = netaddr.IPAddress("::0200:0:0:0")
203
mac64 = netaddr.EUI(int(address & mask1 ^ mask2)).words
204
return ":".join(["%02x" % i for i in mac64[0:3] + mac64[5:8]])
207
def isotime(at=None):
388
return at.strftime(fmt)
391
def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT):
392
"""Turn a formatted time back into a datetime."""
393
return datetime.datetime.strptime(timestr, fmt)
396
def isotime(at=None):
397
"""Returns iso formatted utcnow."""
398
return strtime(at, ISO_TIME_FORMAT)
209
at = datetime.datetime.utcnow()
210
return at.strftime(TIME_FORMAT)
401
213
def parse_isotime(timestr):
402
"""Turn an iso formatted time back into a datetime."""
403
return parse_strtime(timestr, ISO_TIME_FORMAT)
214
return datetime.datetime.strptime(timestr, TIME_FORMAT)
406
217
def parse_mailmap(mailmap='.mailmap'):
531
289
if isinstance(value, unicode):
532
return value.encode('utf-8')
290
return value.encode("utf-8")
533
291
assert isinstance(value, str)
537
def to_primitive(value, convert_instances=False, level=0):
538
"""Convert a complex object into primitives.
540
Handy for JSON serialization. We can optionally handle instances,
541
but since this is a recursive function, we could have cyclical
544
To handle cyclical data structures we could track the actual objects
545
visited in a set, but not all objects are hashable. Instead we just
546
track the depth of the object inspections and don't go too deep.
548
Therefore, convert_instances=True is lossy ... be aware.
551
nasty = [inspect.ismodule, inspect.isclass, inspect.ismethod,
552
inspect.isfunction, inspect.isgeneratorfunction,
553
inspect.isgenerator, inspect.istraceback, inspect.isframe,
554
inspect.iscode, inspect.isbuiltin, inspect.isroutine,
558
return unicode(value)
563
# The try block may not be necessary after the class check above,
564
# but just in case ...
566
if type(value) is type([]) or type(value) is type((None,)):
569
o.append(to_primitive(v, convert_instances=convert_instances,
572
elif type(value) is type({}):
574
for k, v in value.iteritems():
575
o[k] = to_primitive(v, convert_instances=convert_instances,
578
elif isinstance(value, datetime.datetime):
580
elif hasattr(value, 'iteritems'):
581
return to_primitive(dict(value.iteritems()),
582
convert_instances=convert_instances,
584
elif hasattr(value, '__iter__'):
585
return to_primitive(list(value), level)
586
elif convert_instances and hasattr(value, '__dict__'):
587
# Likely an instance of something. Watch for cycles.
588
# Ignore class member vars.
589
return to_primitive(value.__dict__,
590
convert_instances=convert_instances,
595
# Class objects are tricky since they may define something like
596
# __iter__ defined but it isn't callable as list().
597
return unicode(value)
602
return json.dumps(value)
605
return json.dumps(to_primitive(value))
617
anyjson._modules.append(("nova.utils", "dumps", TypeError,
618
"loads", ValueError))
619
anyjson.force_implementation("nova.utils")
625
class _NoopContextManager(object):
629
def __exit__(self, exc_type, exc_val, exc_tb):
633
def synchronized(name, external=False):
634
"""Synchronization decorator.
636
Decorating a method like so:
637
@synchronized('mylock')
638
def foo(self, *args):
641
ensures that only one thread will execute the bar method at a time.
643
Different methods can share the same lock:
644
@synchronized('mylock')
645
def foo(self, *args):
648
@synchronized('mylock')
649
def bar(self, *args):
652
This way only one of either foo or bar can be executing at a time.
654
The external keyword argument denotes whether this lock should work across
655
multiple processes. This means that if two different workers both run a
656
a method decorated with @synchronized('mylock', external=True), only one
657
of them will execute at a time.
663
def inner(*args, **kwargs):
664
# NOTE(soren): If we ever go natively threaded, this will be racy.
665
# See http://stackoverflow.com/questions/5390569/dyn\
666
# amically-allocating-and-destroying-mutexes
667
if name not in _semaphores:
668
_semaphores[name] = semaphore.Semaphore()
669
sem = _semaphores[name]
670
LOG.debug(_('Attempting to grab semaphore "%(lock)s" for method '
671
'"%(method)s"...' % {'lock': name,
672
'method': f.__name__}))
675
LOG.debug(_('Attempting to grab file lock "%(lock)s" for '
676
'method "%(method)s"...' %
677
{'lock': name, 'method': f.__name__}))
678
lock_file_path = os.path.join(FLAGS.lock_path,
679
'nova-%s.lock' % name)
680
lock = lockfile.FileLock(lock_file_path)
682
lock = _NoopContextManager()
685
retval = f(*args, **kwargs)
687
# If no-one else is waiting for it, delete it.
688
# See note about possible raciness above.
689
if not sem.balance < 1:
690
del _semaphores[name]
697
def get_from_path(items, path):
698
"""Returns a list of items matching the specified path.
700
Takes an XPath-like expression e.g. prop1/prop2/prop3, and for each item
701
in items, looks up items[prop1][prop2][prop3]. Like XPath, if any of the
702
intermediate results are lists it will treat each list item individually.
703
A 'None' in items or any child expressions will be ignored, this function
704
will not throw because of None (anywhere) in items. The returned list
705
will contain no None values.
709
raise exception.Error('Invalid mini_xpath')
711
(first_token, sep, remainder) = path.partition('/')
713
if first_token == '':
714
raise exception.Error('Invalid mini_xpath')
721
if not isinstance(items, types.ListType):
722
# Wrap single objects in a list
728
get_method = getattr(item, 'get', None)
729
if get_method is None:
731
child = get_method(first_token)
734
if isinstance(child, types.ListType):
735
# Flatten intermediate lists
739
results.append(child)
745
return get_from_path(results, remainder)
748
def flatten_dict(dict_, flattened=None):
749
"""Recursively flatten a nested dictionary."""
750
flattened = flattened or {}
751
for key, value in dict_.iteritems():
752
if hasattr(value, 'iteritems'):
753
flatten_dict(value, flattened)
755
flattened[key] = value
759
def partition_dict(dict_, keys):
760
"""Return two dicts, one with `keys` the other with everything else."""
763
for key, value in dict_.iteritems():
765
intersection[key] = value
767
difference[key] = value
768
return intersection, difference
771
def map_dict_keys(dict_, key_map):
772
"""Return a dict in which the dictionaries keys are mapped to new keys."""
774
for key, value in dict_.iteritems():
775
mapped_key = key_map[key] if key in key_map else key
776
mapped[mapped_key] = value
780
def subset_dict(dict_, keys):
781
"""Return a dict that only contains a subset of keys."""
782
subset = partition_dict(dict_, keys)[0]
786
def check_isinstance(obj, cls):
787
"""Checks that obj is of type cls, and lets PyLint infer types."""
788
if isinstance(obj, cls):
790
raise Exception(_('Expected object of type: %s') % (str(cls)))
791
# TODO(justinsb): Can we make this better??
792
return cls() # Ugly PyLint hack
795
def parse_server_string(server_str):
797
Parses the given server_string and returns a list of host and port.
798
If it's not a combination of host part and port, the port element
799
is a null string. If the input is invalid expression, return a null
803
# First of all, exclude pure IPv6 address (w/o port).
804
if netaddr.valid_ipv6(server_str):
805
return (server_str, '')
807
# Next, check if this is IPv6 address with a port number combination.
808
if server_str.find("]:") != -1:
809
(address, port) = server_str.replace('[', '', 1).split(']:')
810
return (address, port)
812
# Third, check if this is a combination of an address and a port
813
if server_str.find(':') == -1:
814
return (server_str, '')
816
# This must be a combination of an address and a port
817
(address, port) = server_str.split(':')
818
return (address, port)
821
LOG.debug(_('Invalid server_string: %s' % server_str))
829
def is_uuid_like(val):
830
"""For our purposes, a UUID is a string in canoical form:
832
aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa
834
if not isinstance(val, basestring):
836
return (len(val) == 36) and (val.count('-') == 4)
839
def bool_from_str(val):
840
"""Convert a string representation of a bool into a bool value"""
845
return True if int(val) else False
847
return val.lower() == 'true'
850
def is_valid_ipv4(address):
851
"""valid the address strictly as per format xxx.xxx.xxx.xxx.
852
where xxx is a value between 0 and 255.
854
parts = address.split(".")
859
if not 0 <= int(item) <= 255:
867
""" If the Flags.monkey_patch set as True,
868
this functuion patches a decorator
869
for all functions in specified modules.
870
You can set decorators for each modules
871
using FLAGS.monkey_patch_modules.
872
The format is "Module path:Decorator function".
873
Example: 'nova.api.ec2.cloud:nova.notifier.api.notify_decorator'
875
Parameters of the decorator is as follows.
876
(See nova.notifier.api.notify_decorator)
878
name - name of the function
879
function - object of the function
881
# If FLAGS.monkey_patch is not True, this function do nothing.
882
if not FLAGS.monkey_patch:
884
# Get list of modules and decorators
885
for module_and_decorator in FLAGS.monkey_patch_modules:
886
module, decorator_name = module_and_decorator.split(':')
887
# import decorator function
888
decorator = import_class(decorator_name)
890
# Retrieve module information using pyclbr
891
module_data = pyclbr.readmodule_ex(module)
892
for key in module_data.keys():
893
# set the decorator for the class methods
894
if isinstance(module_data[key], pyclbr.Class):
895
clz = import_class("%s.%s" % (module, key))
896
for method, func in inspect.getmembers(clz, inspect.ismethod):
897
setattr(clz, method,\
898
decorator("%s.%s.%s" % (module, key, method), func))
899
# set the decorator for the function
900
if isinstance(module_data[key], pyclbr.Function):
901
func = import_class("%s.%s" % (module, key))
902
setattr(sys.modules[module], key,\
903
decorator("%s.%s" % (module, key), func))
906
def convert_to_list_dict(lst, label):
907
"""Convert a value or list into a list of dicts"""
910
if not isinstance(lst, list):
912
return [{label: x} for x in lst]
added
removed
nova/utils.py
