« back to all changes in this revision
Viewing changes to oslo/vmware/rw_handles.py
- browse files at revision 1
- compare with another revision
- download diff
- download tarball
- view history from revision 1
- Committer: Package Import Robot
- Author(s): Thomas Goirand
- Date: 2014-03-05 15:29:17 UTC
- Revision ID: package-import@ubuntu.com-20140305152917-9n6zp4cktcwyr3ul
Tags: upstream-0.2
Import upstream version 0.2
- files added:
- doc
- doc/source
- oslo
- oslo/vmware
- oslo/vmware/common
- oslo/vmware/openstack
- oslo/vmware/openstack/common
- tests
added
removed
oslo/vmware/rw_handles.py
1
# Copyright (c) 2014 VMware, Inc.
2
# All Rights Reserved.
3
#
4
# Licensed under the Apache License, Version 2.0 (the "License"); you may
5
# not use this file except in compliance with the License. You may obtain
6
# a copy of the License at
7
#
8
# http://www.apache.org/licenses/LICENSE-2.0
9
#
10
# Unless required by applicable law or agreed to in writing, software
11
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
# License for the specific language governing permissions and limitations
14
# under the License.
15
16
"""
17
Classes defining read and write handles for image transfer.
18
19
This module defines various classes for reading and writing files including
20
VMDK files in VMware servers. It also contains a class to read images from
21
glance server.
22
"""
23
24
import httplib
25
import logging
26
import socket
27
import urllib
28
import urllib2
29
import urlparse
30
31
import netaddr
32
33
from oslo.vmware import exceptions
34
from oslo.vmware.openstack.common.gettextutils import _
35
from oslo.vmware import vim_util
36
37
38
LOG = logging.getLogger(__name__)
39
40
READ_CHUNKSIZE = 65536
41
USER_AGENT = 'OpenStack-ESX-Adapter'
42
43
44
class FileHandle(object):
45
"""Base class for VMware server file (including VMDK) access over HTTP.
46
47
This class wraps a backing file handle and provides utility methods
48
for various sub-classes.
49
"""
50
51
def __init__(self, file_handle):
52
"""Initializes the file handle.
53
54
:param _file_handle: backing file handle
55
"""
56
self._eof = False
57
self._file_handle = file_handle
58
59
def close(self):
60
"""Close the file handle."""
61
try:
62
self._file_handle.close()
63
except Exception:
64
LOG.warn(_("Error occurred while closing the file handle"),
65
exc_info=True)
66
67
def __del__(self):
68
"""Close the file handle on garbage collection."""
69
self.close()
70
71
def _build_vim_cookie_header(self, vim_cookies):
72
"""Build ESX host session cookie header."""
73
cookie_header = ""
74
for vim_cookie in vim_cookies:
75
cookie_header = vim_cookie.name + '=' + vim_cookie.value
76
break
77
return cookie_header
78
79
def write(self, data):
80
"""Write data to the file.
81
82
:param data: data to be written
83
:raises: NotImplementedError
84
"""
85
raise NotImplementedError()
86
87
def read(self, chunk_size):
88
"""Read a chunk of data.
89
90
:param chunk_size: read chunk size
91
:raises: NotImplementedError
92
"""
93
raise NotImplementedError()
94
95
def get_size(self):
96
"""Get size of the file to be read.
97
98
:raises: NotImplementedError
99
"""
100
raise NotImplementedError()
101
102
def _is_valid_ipv6(self, address):
103
"""Checks whether the given host address is a valid IPv6 address."""
104
try:
105
return netaddr.valid_ipv6(address)
106
except Exception:
107
return False
108
109
def _get_soap_url(self, scheme, host):
110
"""Returns the IPv4/v6 compatible SOAP URL for the given host."""
111
if self._is_valid_ipv6(host):
112
return '%s://[%s]' % (scheme, host)
113
return '%s://%s' % (scheme, host)
114
115
def _fix_esx_url(self, url, host):
116
"""Fix netloc in the case of an ESX host.
117
118
In the case of an ESX host, the netloc is set to '*' in the URL
119
returned in HttpNfcLeaseInfo. It should be replaced with host name
120
or IP address.
121
"""
122
urlp = urlparse.urlparse(url)
123
if urlp.netloc == '*':
124
scheme, netloc, path, params, query, fragment = urlp
125
url = urlparse.urlunparse((scheme,
126
host,
127
path,
128
params,
129
query,
130
fragment))
131
return url
132
133
def _find_vmdk_url(self, lease_info, host):
134
"""Find the URL corresponding to a VMDK file in lease info."""
135
LOG.debug(_("Finding VMDK URL from lease info."))
136
url = None
137
for deviceUrl in lease_info.deviceUrl:
138
if deviceUrl.disk:
139
url = self._fix_esx_url(deviceUrl.url, host)
140
break
141
if not url:
142
excep_msg = _("Could not retrieve VMDK URL from lease info.")
143
LOG.error(excep_msg)
144
raise exceptions.VimException(excep_msg)
145
LOG.debug(_("Found VMDK URL: %s from lease info."), url)
146
return url
147
148
149
class FileWriteHandle(FileHandle):
150
"""Write handle for a file in VMware server."""
151
152
def __init__(self, host, data_center_name, datastore_name, cookies,
153
file_path, file_size, scheme='https'):
154
"""Initializes the write handle with given parameters.
155
156
:param host: ESX/VC server IP address[:port] or host name[:port]
157
:param data_center_name: name of the data center in the case of a VC
158
server
159
:param datastore_name: name of the datastore where the file is stored
160
:param cookies: cookies to build the vim cookie header
161
:param file_path: datastore path where the file is written
162
:param file_size: size of the file in bytes
163
:param scheme: protocol-- http or https
164
:raises: VimConnectionException, ValueError
165
"""
166
soap_url = self._get_soap_url(scheme, host)
167
param_list = {'dcPath': data_center_name, 'dsName': datastore_name}
168
self._url = '%s/folder/%s' % (soap_url, file_path)
169
self._url = self._url + '?' + urllib.urlencode(param_list)
170
171
self.conn = self._create_connection(self._url,
172
file_size,
173
cookies)
174
FileHandle.__init__(self, self.conn)
175
176
def _create_connection(self, url, file_size, cookies):
177
"""Create HTTP connection to write to the file with given URL."""
178
LOG.debug(_("Creating HTTP connection to write to file with "
179
"size = %(file_size)d and URL = %(url)s."),
180
{'file_size': file_size,
181
'url': url})
182
_urlparse = urlparse.urlparse(url)
183
scheme, netloc, path, params, query, fragment = _urlparse
184
185
try:
186
if scheme == 'http':
187
conn = httplib.HTTPConnection(netloc)
188
elif scheme == 'https':
189
conn = httplib.HTTPSConnection(netloc)
190
else:
191
excep_msg = _("Invalid scheme: %s.") % scheme
192
LOG.error(excep_msg)
193
raise ValueError(excep_msg)
194
195
conn.putrequest('PUT', path + '?' + query)
196
conn.putheader('User-Agent', USER_AGENT)
197
conn.putheader('Content-Length', file_size)
198
conn.putheader('Cookie', self._build_vim_cookie_header(cookies))
199
conn.endheaders()
200
LOG.debug(_("Created HTTP connection to write to file with "
201
"URL = %s."), url)
202
return conn
203
except (httplib.InvalidURL, httplib.CannotSendRequest,
204
httplib.CannotSendHeader) as excep:
205
excep_msg = _("Error occurred while creating HTTP connection "
206
"to write to file with URL = %s.") % url
207
LOG.exception(excep_msg)
208
raise exceptions.VimConnectionException(excep_msg, excep)
209
210
def write(self, data):
211
"""Write data to the file.
212
213
:param data: data to be written
214
:raises: VimConnectionException, VimException
215
"""
216
LOG.debug(_("Writing data to %s."), self._url)
217
try:
218
self._file_handle.send(data)
219
except (socket.error, httplib.NotConnected) as excep:
220
excep_msg = _("Connection error occurred while writing data to"
221
" %s.") % self._url
222
LOG.exception(excep_msg)
223
raise exceptions.VimConnectionException(excep_msg, excep)
224
except Exception as excep:
225
# TODO(vbala) We need to catch and raise specific exceptions
226
# related to connection problems, invalid request and invalid
227
# arguments.
228
excep_msg = _("Error occurred while writing data to"
229
" %s.") % self._url
230
LOG.exception(excep_msg)
231
raise exceptions.VimException(excep_msg, excep)
232
233
def close(self):
234
"""Get the response and close the connection."""
235
LOG.debug(_("Closing write handle for %s."), self._url)
236
try:
237
self.conn.getresponse()
238
except Exception:
239
LOG.warn(_("Error occurred while reading the HTTP response."),
240
exc_info=True)
241
super(FileWriteHandle, self).close()
242
LOG.debug(_("Closed write handle for %s."), self._url)
243
244
def __str__(self):
245
return "File write handle for %s" % self._url
246
247
248
class VmdkWriteHandle(FileHandle):
249
"""VMDK write handle based on HttpNfcLease.
250
251
This class creates a vApp in the specified resource pool and uploads the
252
virtual disk contents.
253
"""
254
255
def __init__(self, session, host, rp_ref, vm_folder_ref, import_spec,
256
vmdk_size):
257
"""Initializes the VMDK write handle with input parameters.
258
259
:param session: valid API session to ESX/VC server
260
:param host: ESX/VC server IP address[:port] or host name[:port]
261
:param rp_ref: resource pool into which the backing VM is imported
262
:param vm_folder_ref: VM folder in ESX/VC inventory to use as parent
263
of backing VM
264
:param import_spec: import specification of the backing VM
265
:param vmdk_size: size of the backing VM's VMDK file
266
:raises: VimException, VimFaultException, VimAttributeException,
267
VimSessionOverLoadException, VimConnectionException,
268
ValueError
269
"""
270
self._session = session
271
self._vmdk_size = vmdk_size
272
self._bytes_written = 0
273
274
# Get lease and its info for vApp import
275
self._lease = self._create_and_wait_for_lease(session,
276
rp_ref,
277
import_spec,
278
vm_folder_ref)
279
LOG.debug(_("Invoking VIM API for reading info of lease: %s."),
280
self._lease)
281
lease_info = session.invoke_api(vim_util,
282
'get_object_property',
283
session.vim,
284
self._lease,
285
'info')
286
287
# Find VMDK URL where data is to be written
288
self._url = self._find_vmdk_url(lease_info, host)
289
self._vm_ref = lease_info.entity
290
291
# Create HTTP connection to write to VMDK URL
292
self._conn = self._create_connection(session, self._url, vmdk_size)
293
FileHandle.__init__(self, self._conn)
294
295
def get_imported_vm(self):
296
""""Get managed object reference of the VM created for import."""
297
return self._vm_ref
298
299
def _create_and_wait_for_lease(self, session, rp_ref, import_spec,
300
vm_folder_ref):
301
"""Create and wait for HttpNfcLease lease for vApp import."""
302
LOG.debug(_("Creating HttpNfcLease lease for vApp import into resource"
303
" pool: %s."),
304
rp_ref)
305
lease = session.invoke_api(session.vim,
306
'ImportVApp',
307
rp_ref,
308
spec=import_spec,
309
folder=vm_folder_ref)
310
LOG.debug(_("Lease: %(lease)s obtained for vApp import into resource"
311
" pool %(rp_ref)s."),
312
{'lease': lease,
313
'rp_ref': rp_ref})
314
session.wait_for_lease_ready(lease)
315
return lease
316
317
def _create_connection(self, session, url, vmdk_size):
318
"""Create HTTP connection to write to VMDK file."""
319
LOG.debug(_("Creating HTTP connection to write to VMDK file with "
320
"size = %(vmdk_size)d and URL = %(url)s."),
321
{'vmdk_size': vmdk_size,
322
'url': url})
323
cookies = session.vim.client.options.transport.cookiejar
324
_urlparse = urlparse.urlparse(url)
325
scheme, netloc, path, params, query, fragment = _urlparse
326
327
try:
328
if scheme == 'http':
329
conn = httplib.HTTPConnection(netloc)
330
elif scheme == 'https':
331
conn = httplib.HTTPSConnection(netloc)
332
else:
333
excep_msg = _("Invalid scheme: %s.") % scheme
334
LOG.error(excep_msg)
335
raise ValueError(excep_msg)
336
337
if query:
338
path = path + '?' + query
339
conn.putrequest('PUT', path)
340
conn.putheader('User-Agent', USER_AGENT)
341
conn.putheader('Content-Length', str(vmdk_size))
342
conn.putheader('Overwrite', 't')
343
conn.putheader('Cookie', self._build_vim_cookie_header(cookies))
344
conn.putheader('Content-Type', 'binary/octet-stream')
345
conn.endheaders()
346
LOG.debug(_("Created HTTP connection to write to VMDK file with "
347
"URL = %s."),
348
url)
349
return conn
350
except (httplib.InvalidURL, httplib.CannotSendRequest,
351
httplib.CannotSendHeader) as excep:
352
excep_msg = _("Error occurred while creating HTTP connection "
353
"to write to VMDK file with URL = %s.") % url
354
LOG.exception(excep_msg)
355
raise exceptions.VimConnectionException(excep_msg, excep)
356
357
def write(self, data):
358
"""Write data to the file.
359
360
:param data: data to be written
361
:raises: VimConnectionException, VimException
362
"""
363
LOG.debug(_("Writing data to VMDK file with URL = %s."), self._url)
364
365
try:
366
self._file_handle.send(data)
367
self._bytes_written += len(data)
368
LOG.debug(_("Total %(bytes_written)d bytes written to VMDK file "
369
"with URL = %(url)s."),
370
{'bytes_written': self._bytes_written,
371
'url': self._url})
372
except (socket.error, httplib.NotConnected) as excep:
373
excep_msg = _("Connection error occurred while writing data to"
374
" %s.") % self._url
375
LOG.exception(excep_msg)
376
raise exceptions.VimConnectionException(excep_msg, excep)
377
except Exception as excep:
378
# TODO(vbala) We need to catch and raise specific exceptions
379
# related to connection problems, invalid request and invalid
380
# arguments.
381
excep_msg = _("Error occurred while writing data to"
382
" %s.") % self._url
383
LOG.exception(excep_msg)
384
raise exceptions.VimException(excep_msg, excep)
385
386
def update_progress(self):
387
"""Updates progress to lease.
388
389
This call back to the lease is essential to keep the lease alive
390
across long running write operations.
391
392
:raises: VimException, VimFaultException, VimAttributeException,
393
VimSessionOverLoadException, VimConnectionException
394
"""
395
percent = int(float(self._bytes_written) / self._vmdk_size * 100)
396
LOG.debug(_("Calling VIM API to update write progress of VMDK file"
397
" with URL = %(url)s to %(percent)d%%."),
398
{'url': self._url,
399
'percent': percent})
400
try:
401
self._session.invoke_api(self._session.vim,
402
'HttpNfcLeaseProgress',
403
self._lease,
404
percent=percent)
405
LOG.debug(_("Updated write progress of VMDK file with "
406
"URL = %(url)s to %(percent)d%%."),
407
{'url': self._url,
408
'percent': percent})
409
except exceptions.VimException as excep:
410
LOG.exception(_("Error occurred while updating the write progress "
411
"of VMDK file with URL = %s."),
412
self._url)
413
raise excep
414
415
def close(self):
416
"""Releases the lease and close the connection.
417
418
:raises: VimException, VimFaultException, VimAttributeException,
419
VimSessionOverLoadException, VimConnectionException
420
"""
421
LOG.debug(_("Getting lease state for %s."), self._url)
422
try:
423
state = self._session.invoke_api(vim_util,
424
'get_object_property',
425
self._session.vim,
426
self._lease,
427
'state')
428
LOG.debug(_("Lease for %(url)s is in state: %(state)s."),
429
{'url': self._url,
430
'state': state})
431
if state == 'ready':
432
LOG.debug(_("Releasing lease for %s."), self._url)
433
self._session.invoke_api(self._session.vim,
434
'HttpNfcLeaseComplete',
435
self._lease)
436
LOG.debug(_("Lease for %s released."), self._url)
437
else:
438
LOG.debug(_("Lease for %(url)s is in state: %(state)s; no "
439
"need to release."),
440
{'url': self._url,
441
'state': state})
442
except exceptions.VimException:
443
LOG.warn(_("Error occurred while releasing the lease for %s."),
444
self._url,
445
exc_info=True)
446
super(VmdkWriteHandle, self).close()
447
LOG.debug(_("Closed VMDK write handle for %s."), self._url)
448
449
def __str__(self):
450
return "VMDK write handle for %s" % self._url
451
452
453
class VmdkReadHandle(FileHandle):
454
"""VMDK read handle based on HttpNfcLease."""
455
456
def __init__(self, session, host, vm_ref, vmdk_path, vmdk_size):
457
"""Initializes the VMDK read handle with the given parameters.
458
459
During the read (export) operation, the VMDK file is converted to a
460
stream-optimized sparse disk format. Therefore, the size of the VMDK
461
file read may be smaller than the actual VMDK size.
462
463
:param session: valid api session to ESX/VC server
464
:param host: ESX/VC server IP address[:port] or host name[:port]
465
:param vm_ref: managed object reference of the backing VM whose VMDK
466
is to be exported
467
:param vmdk_path: path of the VMDK file to be exported
468
:param vmdk_size: actual size of the VMDK file
469
:raises: VimException, VimFaultException, VimAttributeException,
470
VimSessionOverLoadException, VimConnectionException
471
"""
472
self._session = session
473
self._vmdk_size = vmdk_size
474
self._bytes_read = 0
475
476
# Obtain lease for VM export
477
self._lease = self._create_and_wait_for_lease(session, vm_ref)
478
LOG.debug(_("Invoking VIM API for reading info of lease: %s."),
479
self._lease)
480
lease_info = session.invoke_api(vim_util,
481
'get_object_property',
482
session.vim,
483
self._lease,
484
'info')
485
486
# find URL of the VMDK file to be read and open connection
487
self._url = self._find_vmdk_url(lease_info, host)
488
self._conn = self._create_connection(session, self._url)
489
FileHandle.__init__(self, self._conn)
490
491
def _create_and_wait_for_lease(self, session, vm_ref):
492
"""Create and wait for HttpNfcLease lease for VM export."""
493
LOG.debug(_("Creating HttpNfcLease lease for exporting VM: %s."),
494
vm_ref)
495
lease = session.invoke_api(session.vim, 'ExportVm', vm_ref)
496
LOG.debug(_("Lease: %(lease)s obtained for exporting VM: %(vm_ref)s."),
497
{'lease': lease,
498
'vm_ref': vm_ref})
499
session.wait_for_lease_ready(lease)
500
return lease
501
502
def _create_connection(self, session, url):
503
LOG.debug(_("Opening URL: %s for reading."), url)
504
try:
505
cookies = session.vim.client.options.transport.cookiejar
506
headers = {'User-Agent': USER_AGENT,
507
'Cookie': self._build_vim_cookie_header(cookies)}
508
request = urllib2.Request(url, None, headers)
509
conn = urllib2.urlopen(request)
510
LOG.debug(_("URL: %s opened for reading."), url)
511
return conn
512
except Exception as excep:
513
# TODO(vbala) We need to catch and raise specific exceptions
514
# related to connection problems, invalid request and invalid
515
# arguments.
516
excep_msg = _("Error occurred while opening URL: %s for "
517
"reading.") % url
518
LOG.exception(excep_msg)
519
raise exceptions.VimException(excep_msg, excep)
520
521
def read(self, chunk_size):
522
"""Read a chunk of data from the VMDK file.
523
524
:param chunk_size: size of read chunk
525
:returns: the data
526
:raises: VimException
527
"""
528
LOG.debug(_("Reading data from VMDK file with URL = %s."), self._url)
529
530
try:
531
data = self._file_handle.read(READ_CHUNKSIZE)
532
self._bytes_read += len(data)
533
LOG.debug(_("Total %(bytes_read)d bytes read from VMDK file "
534
"with URL = %(url)s."),
535
{'bytes_read': self._bytes_read,
536
'url': self._url})
537
return data
538
except Exception as excep:
539
# TODO(vbala) We need to catch and raise specific exceptions
540
# related to connection problems, invalid request and invalid
541
# arguments.
542
excep_msg = _("Error occurred while reading data from"
543
" %s.") % self._url
544
LOG.exception(excep_msg)
545
raise exceptions.VimException(excep_msg, excep)
546
547
def update_progress(self):
548
"""Updates progress to lease.
549
550
This call back to the lease is essential to keep the lease alive
551
across long running read operations.
552
553
:raises: VimException, VimFaultException, VimAttributeException,
554
VimSessionOverLoadException, VimConnectionException
555
"""
556
percent = int(float(self._bytes_read) / self._vmdk_size * 100)
557
LOG.debug(_("Calling VIM API to update read progress of VMDK file"
558
" with URL = %(url)s to %(percent)d%%."),
559
{'url': self._url,
560
'percent': percent})
561
try:
562
self._session.invoke_api(self._session.vim,
563
'HttpNfcLeaseProgress',
564
self._lease,
565
percent=percent)
566
LOG.debug(_("Updated read progress of VMDK file with "
567
"URL = %(url)s to %(percent)d%%."),
568
{'url': self._url,
569
'percent': percent})
570
except exceptions.VimException as excep:
571
LOG.exception(_("Error occurred while updating the read progress "
572
"of VMDK file with URL = %s."),
573
self._url)
574
raise excep
575
576
def close(self):
577
"""Releases the lease and close the connection.
578
579
:raises: VimException, VimFaultException, VimAttributeException,
580
VimSessionOverLoadException, VimConnectionException
581
"""
582
LOG.debug(_("Getting lease state for %s."), self._url)
583
try:
584
state = self._session.invoke_api(vim_util,
585
'get_object_property',
586
self._session.vim,
587
self._lease,
588
'state')
589
LOG.debug(_("Lease for %(url)s is in state: %(state)s."),
590
{'url': self._url,
591
'state': state})
592
if state == 'ready':
593
LOG.debug(_("Releasing lease for %s."), self._url)
594
self._session.invoke_api(self._session.vim,
595
'HttpNfcLeaseComplete',
596
self._lease)
597
LOG.debug(_("Lease for %s released."), self._url)
598
else:
599
LOG.debug(_("Lease for %(url)s is in state: %(state)s; no "
600
"need to release."),
601
{'url': self._url,
602
'state': state})
603
except exceptions.VimException:
604
LOG.warn(_("Error occurred while releasing the lease for %s."),
605
self._url,
606
exc_info=True)
607
raise
608
super(VmdkReadHandle, self).close()
609
LOG.debug(_("Closed VMDK read handle for %s."), self._url)
610
611
def __str__(self):
612
return "VMDK read handle for %s" % self._url
613
614
615
class ImageReadHandle(object):
616
"""Read handle for glance images."""
617
618
def __init__(self, glance_read_iter):
619
"""Initializes the read handle with given parameters.
620
621
:param glance_read_iter: iterator to read data from glance image
622
"""
623
self._glance_read_iter = glance_read_iter
624
self._iter = self.get_next()
625
626
def read(self, chunk_size):
627
"""Read an item from the image data iterator.
628
629
The input chunk size is ignored since the client ImageBodyIterator
630
uses its own chunk size.
631
"""
632
try:
633
data = self._iter.next()
634
LOG.debug(_("Read %d bytes from the image iterator."), len(data))
635
return data
636
except StopIteration:
637
LOG.debug(_("Completed reading data from the image iterator."))
638
return ""
639
640
def get_next(self):
641
"""Get the next item from the image iterator."""
642
for data in self._glance_read_iter:
643
yield data
644
645
def close(self):
646
"""Close the read handle.
647
648
This is a NOP.
649
"""
650
pass
651
652
def __str__(self):
653
return "Image read handle"
Loggerhead is a web-based interface for Breezy
Version: 2.0.1