« back to all changes in this revision
Viewing changes to lib/pyfits/fitsrec.py
- browse files at revision 17
- compare with another revision
- download diff
- download tarball
- view history from revision 17
- Committer: Package Import Robot
- Author(s): Aurelien Jarno
- Date: 2013-12-07 16:18:48 UTC
- mfrom: (1.1.11)
- Revision ID: package-import@ubuntu.com-20131207161848-mcw0saz0iprjhbju
Tags: 1:3.2-1
* New upstream version.
* Bump Standards-Version to 3.9.5 (no changes).
* Remove build-depends on zlib1g-dev and remove patches/01-zlib.diff.
* Add build-depends on libcfitsio3-dev and add
patches/01-system-cfitsio.diff.
* Update debian/copyright.
* Install upstream changelog now that it is provided in the upstream
tarball.
* Don't compress the binary packages with xz, it's no the dpkg's default.
* Bump Standards-Version to 3.9.5 (no changes).
* Remove build-depends on zlib1g-dev and remove patches/01-zlib.diff.
* Add build-depends on libcfitsio3-dev and add
patches/01-system-cfitsio.diff.
* Update debian/copyright.
* Install upstream changelog now that it is provided in the upstream
tarball.
* Don't compress the binary packages with xz, it's no the dpkg's default.
- files added:
- .pc/01-system-cfitsio.diff
- cextern
- cextern/cfitsio
- files removed:
- .pc/01-zlib.diff
- .pc/01-zlib.diff/src
- files modified:
- .pc/applied-patches
added
removed
lib/pyfits/fitsrec.py
1
import copy
1
2
import operator
2
import sys
3
3
import warnings
4
4
import weakref
5
5
6
6
import numpy as np
7
7
8
from pyfits.column import (ASCIITNULL, FITS2NUMPY, Column, ColDefs, _FormatX,
9
_FormatP, _VLF, _get_index, _wrapx, _unwrapx,
10
_convert_ascii_format)
11
from pyfits.util import _array_from_file, decode_ascii, lazyproperty
8
from numpy import char as chararray
9
10
from pyfits.column import (ASCIITNULL, FITS2NUMPY, ASCII2NUMPY, ASCII2STR,
11
ColDefs, _AsciiColDefs, _FormatX, _FormatP, _VLF,
12
_get_index, _wrapx, _unwrapx, _makep,
13
_convert_ascii_format, Delayed)
14
from pyfits.util import decode_ascii, lazyproperty
12
15
13
16
14
17
class FITS_record(object):
70
73
raise KeyError("Key '%s' does not exist." % key)
71
74
elif isinstance(key, slice):
72
75
return type(self)(self.array, self.row, key.start, key.stop,
73
key.step, self)
76
key.step, self)
74
77
else:
75
78
indx = self._get_index(key)
76
79
175
178
self._convert = [None] * len(self.dtype.names)
176
179
self._heapoffset = 0
177
180
self._heapsize = 0
178
self._file = None
179
self._buffer = None
180
181
self._coldefs = None
181
182
self._gap = 0
183
self._uint = False
182
184
self.names = list(self.dtype.names)
183
185
self.formats = None
184
186
return self
191
193
self._convert = obj._convert
192
194
self._heapoffset = obj._heapoffset
193
195
self._heapsize = obj._heapsize
194
self._file = obj._file
195
self._buffer = obj._buffer
196
196
self._coldefs = obj._coldefs
197
197
self._nfields = obj._nfields
198
198
self._gap = obj._gap
199
self._uint = obj._uint
199
200
self.names = obj.names
200
201
self.formats = obj.formats
201
202
else:
206
207
207
208
self._heapoffset = getattr(obj, '_heapoffset', 0)
208
209
self._heapsize = getattr(obj, '_heapsize', 0)
209
self._file = getattr(obj, '_file', None)
210
self._buffer = getattr(obj, '_buffer', None)
211
210
212
211
self._coldefs = None
213
self._gap = 0
212
self._gap = getattr(obj, '_gap', 0)
213
self._uint = getattr(obj, '_uint', False)
214
214
215
215
# Bypass setattr-based assignment to fields; see #86
216
216
self.names = list(obj.dtype.names)
228
228
self._coldefs = ColDefs(self)
229
229
self.formats = self._coldefs.formats
230
230
231
@classmethod
232
def from_columns(cls, columns, nrows=0, fill=False):
233
"""
234
Given a ColDefs object of unknown origin, initialize a new FITS_rec
235
object.
236
237
This was originally part of the new_table function in the table module
238
but was moved into a class method since most of its functionality
239
always had more to do with initializing a FITS_rec object than anything
240
else, and much of it also overlapped with FITS_rec._scale_back.
241
242
Parameters
243
----------
244
columns : sequence of Columns or a ColDefs
245
The columns from which to create the table data. If these
246
columns have data arrays attached that data may be used in
247
initializing the new table. Otherwise the input columns
248
will be used as a template for a new table with the requested
249
number of rows.
250
251
nrows : int
252
Number of rows in the new table. If the input columns have data
253
associated with them, the size of the largest input column is used.
254
Otherwise the default is 0.
255
256
fill : bool
257
If `True`, will fill all cells with zeros or blanks. If
258
`False`, copy the data from input, undefined cells will still
259
be filled with zeros/blanks.
260
"""
261
262
# read the delayed data
263
for idx in range(len(columns)):
264
arr = columns._arrays[idx]
265
if isinstance(arr, Delayed):
266
if arr.hdu.data is None:
267
columns._arrays[idx] = None
268
else:
269
columns._arrays[idx] = np.rec.recarray.field(arr.hdu.data,
270
arr.field)
271
272
# use the largest column shape as the shape of the record
273
if nrows == 0:
274
for arr in columns._arrays:
275
if arr is not None:
276
dim = arr.shape[0]
277
else:
278
dim = 0
279
if dim > nrows:
280
nrows = dim
281
282
raw_data = np.empty(columns.dtype.itemsize * nrows, dtype=np.uint8)
283
raw_data.fill(ord(columns._padding_byte))
284
data = np.recarray(nrows, dtype=columns.dtype, buf=raw_data).view(cls)
285
286
# Previously this assignment was made from hdu.columns, but that's a
287
# bug since if a _TableBaseHDU has a FITS_rec in its .data attribute
288
# the _TableBaseHDU.columns property is actually returned from
289
# .data._coldefs, so this assignment was circular! Don't make that
290
# mistake again.
291
# All of this is an artifact of the fragility of the FITS_rec class,
292
# and that it can't just be initialized by columns...
293
data._coldefs = columns
294
data.formats = columns.formats
295
296
# If fill is True we don't copy anything from the column arrays. We're
297
# just using them as a template, and returning a table filled with
298
# zeros/blanks
299
if fill:
300
return data
301
302
# Otherwise we have to fill the recarray with data from the input
303
# columns
304
for idx in range(len(columns)):
305
# For each column in the ColDef object, determine the number of
306
# rows in that column. This will be either the number of rows in
307
# the ndarray associated with the column, or the number of rows
308
# given in the call to this function, which ever is smaller. If
309
# the input FILL argument is true, the number of rows is set to
310
# zero so that no data is copied from the original input data.
311
arr = columns._arrays[idx]
312
313
if arr is None:
314
array_size = 0
315
else:
316
array_size = len(arr)
317
318
n = min(array_size, nrows)
319
320
# TODO: At least *some* of this logic is mostly redundant with the
321
# _convert_foo methods in this class; see if we can eliminate some
322
# of that duplication.
323
324
if not n:
325
# The input column had an empty array, so just use the fill
326
# value
327
continue
328
329
field = np.rec.recarray.field(data, idx)
330
fitsformat = columns.formats[idx]
331
recformat = columns._recformats[idx]
332
333
outarr = field[:n]
334
inarr = arr[:n]
335
336
if isinstance(recformat, _FormatX):
337
# Data is a bit array
338
if inarr.shape[-1] == recformat.repeat:
339
_wrapx(inarr, outarr, recformat.repeat)
340
continue
341
elif isinstance(recformat, _FormatP):
342
data._convert[idx] = _makep(inarr, field, recformat,
343
nrows=nrows)
344
continue
345
# TODO: Find a better way of determining that the column is meant
346
# to be FITS L formatted
347
elif recformat[-2:] == FITS2NUMPY['L'] and inarr.dtype == bool:
348
# column is boolean
349
# The raw data field should be filled with either 'T' or 'F'
350
# (not 0). Use 'F' as a default
351
field[:] = ord('F')
352
# Also save the original boolean array in data._converted so
353
# that it doesn't have to be re-converted
354
data._convert[idx] = np.zeros(field.shape, dtype=bool)
355
data._convert[idx][:n] = inarr
356
# TODO: Maybe this step isn't necessary at all if _scale_back
357
# will handle it?
358
inarr = np.where(inarr == False, ord('F'), ord('T'))
359
elif (columns[idx]._physical_values and
360
columns[idx]._pseudo_unsigned_ints):
361
# Temporary hack...
362
bzero = columns[idx].bzero
363
data._convert[idx] = np.zeros(field.shape, dtype=inarr.dtype)
364
data._convert[idx][:n] = inarr
365
if n < nrows:
366
# Pre-scale rows below the input data
367
field[n:] = -bzero
368
369
inarr = inarr - bzero
370
elif isinstance(columns, _AsciiColDefs):
371
# Regardless whether the format is character or numeric, if the
372
# input array contains characters then it's already in the raw
373
# format for ASCII tables
374
if fitsformat._pseudo_logical:
375
# Hack to support converting from 8-bit T/F characters
376
# Normally the column array is a chararray of 1 character
377
# strings, but we need to view it as a normal ndarray of
378
# 8-bit ints to fill it with ASCII codes for 'T' and 'F'
379
outarr = field.view(np.uint8, np.ndarray)[:n]
380
elif not isinstance(arr, chararray.chararray):
381
# Fill with the appropriate blanks for the column format
382
data._convert[idx] = np.zeros(nrows, dtype=arr.dtype)
383
outarr = data._convert[idx][:n]
384
385
outarr[:] = inarr
386
continue
387
388
if inarr.shape != outarr.shape:
389
if inarr.dtype != outarr.dtype:
390
inarr = inarr.view(outarr.dtype)
391
392
# This is a special case to handle input arrays with
393
# non-trivial TDIMn.
394
# By design each row of the outarray is 1-D, while each row of
395
# the input array may be n-D
396
if outarr.ndim > 1:
397
# The normal case where the first dimension is the rows
398
inarr_rowsize = inarr[0].size
399
inarr = inarr.reshape((n, inarr_rowsize))
400
outarr[:, :inarr_rowsize] = inarr
401
else:
402
# Special case for strings where the out array only has one
403
# dimension (the second dimension is rolled up into the
404
# strings
405
outarr[:n] = inarr.ravel()
406
else:
407
outarr[:] = inarr
408
409
return data
410
231
411
def __repr__(self):
232
412
return np.recarray.__repr__(self)
233
413
303
483
if isinstance(value, FITS_record):
304
484
for idx in range(self._nfields):
305
485
self.field(self.names[idx])[row] = value.field(self.names[idx])
306
elif isinstance(value, (tuple, list)):
486
elif isinstance(value, (tuple, list, np.void)):
307
487
if self._nfields == len(value):
308
488
for idx in range(self._nfields):
309
489
self.field(idx)[row] = value[idx]
320
500
def __setslice__(self, start, end, value):
321
501
self[slice(start, end)] = value
322
502
503
def copy(self, order='C'):
504
"""
505
The Numpy documentation lies; ndarray.copy is not equivalent to
506
np.copy. Differences include that it re-views the copied array
507
as self's ndarray subclass, as though it were taking a slice;
508
this means __array_finalize__ is called and the copy shares all
509
the array attributes (including ._convert!). So we need to make
510
a deep copy of all those attributes so that the two arrays truly do
511
not share any data.
512
"""
513
514
try:
515
new = super(FITS_rec, self).copy(order=order)
516
except TypeError:
517
# This will probably occur if the order argument is not supported,
518
# such as on Numpy 1.5; in other words we're just going to ask
519
# forgiveness rather than check the Numpy version explicitly.
520
new = super(FITS_rec, self).copy()
521
522
new.__dict__ = copy.deepcopy(self.__dict__)
523
return new
524
323
525
@property
324
526
def columns(self):
325
527
"""
340
542
# contains a reference to the ._coldefs object of the original data;
341
543
# this can lead to a circular reference; see ticket #49
342
544
base = self
343
while isinstance(base, FITS_rec) and \
344
isinstance(base.base, np.recarray):
545
while (isinstance(base, FITS_rec) and
546
isinstance(base.base, np.recarray)):
345
547
base = base.base
346
548
# base could still be a FITS_rec in some cases, so take care to
347
549
# use rec.recarray.field to avoid a potential infinite
349
551
field = np.recarray.field(base, indx)
350
552
351
553
if self._convert[indx] is None:
352
# for X format
353
if isinstance(recformat, _FormatX):
354
_nx = recformat._nx
355
dummy = np.zeros(self.shape + (_nx,), dtype=np.bool_)
356
_unwrapx(field, dummy, _nx)
357
self._convert[indx] = dummy
358
return self._convert[indx]
359
360
(_str, _bool, _number, _scale, _zero, bscale, bzero, dim) = \
361
self._get_scale_factors(indx)
362
363
# for P format
364
554
if isinstance(recformat, _FormatP):
365
dummy = _VLF([None] * len(self), dtype=recformat.dtype)
366
raw_data = self._get_raw_data()
367
if raw_data is None:
368
raise IOError(
369
"Could not find heap data for the %r variable-length "
370
"array column." % self.names[indx])
371
for i in range(len(self)):
372
offset = field[i, 1] + self._heapoffset
373
count = field[i, 0]
374
375
if recformat.dtype == 'a':
376
dt = np.dtype(recformat.dtype + str(1))
377
arr_len = count * dt.itemsize
378
da = raw_data[offset:offset + arr_len].view(dt)
379
da = np.char.array(da.view(dtype=dt), itemsize=count)
380
dummy[i] = decode_ascii(da)
381
else:
382
dt = np.dtype(recformat.dtype)
383
arr_len = count * dt.itemsize
384
dummy[i] = raw_data[offset:offset + arr_len].view(dt)
385
dummy[i].dtype = dummy[i].dtype.newbyteorder('>')
386
387
# scale by TSCAL and TZERO
388
if _scale or _zero:
389
for i in range(len(self)):
390
dummy[i][:] = dummy[i] * bscale + bzero
391
392
# Boolean (logical) column
393
if recformat.dtype == FITS2NUMPY['L']:
394
for i in range(len(self)):
395
dummy[i] = np.equal(dummy[i], ord('T'))
396
397
self._convert[indx] = dummy
398
return self._convert[indx]
399
400
# ASCII table, convert strings to numbers
401
if not _str and self._coldefs._tbtype == 'TableHDU':
402
_fmap = {'I': np.int32, 'F': np.float32, 'E': np.float32,
403
'D': np.float64}
404
_type = _fmap[self._coldefs.formats[indx][0]]
405
406
# if the string = TNULL, return ASCIITNULL
407
nullval = self._coldefs.nulls[indx].strip().encode('ascii')
408
dummy = field.replace('D'.encode('ascii'),
409
'E'.encode('ascii'))
410
dummy = np.where(dummy.strip() == nullval, str(ASCIITNULL),
411
dummy)
412
try:
413
dummy = np.array(dummy, dtype=_type)
414
except ValueError, e:
415
raise ValueError(
416
'%s; the header may be missing the necessary TNULL%d '
417
'keyword or the table contains invalid data' %
418
(e, indx + 1))
419
420
self._convert[indx] = dummy
421
else:
422
dummy = field
423
424
# Test that the dimensions given in dim are sensible; otherwise
425
# display a warning and ignore them
426
if dim:
427
# See if the dimensions already match, if not, make sure the
428
# number items will fit in the specified dimensions
429
if dummy.ndim > 1:
430
actual_shape = dummy[0].shape
431
if _str:
432
actual_shape = (dummy[0].itemsize,) + actual_shape
555
# for P format
556
converted = self._convert_p(indx, field, recformat)
557
else:
558
# Handle all other column data types which are fixed-width
559
# fields
560
converted = self._convert_other(indx, field, recformat)
561
562
self._convert[indx] = converted
563
return converted
564
565
return self._convert[indx]
566
567
def _convert_x(self, field, recformat):
568
"""Convert a raw table column to a bit array as specified by the
569
FITS X format.
570
"""
571
572
dummy = np.zeros(self.shape + (recformat.repeat,), dtype=np.bool_)
573
_unwrapx(field, dummy, recformat.repeat)
574
return dummy
575
576
def _convert_p(self, indx, field, recformat):
577
"""Convert a raw table column of FITS P or Q format descriptors
578
to a VLA column with the array data returned from the heap.
579
"""
580
581
dummy = _VLF([None] * len(self), dtype=recformat.dtype)
582
raw_data = self._get_raw_data()
583
584
if raw_data is None:
585
raise IOError(
586
"Could not find heap data for the %r variable-length "
587
"array column." % self.names[indx])
588
589
for idx in xrange(len(self)):
590
offset = field[idx, 1] + self._heapoffset
591
count = field[idx, 0]
592
593
if recformat.dtype == 'a':
594
dt = np.dtype(recformat.dtype + str(1))
595
arr_len = count * dt.itemsize
596
da = raw_data[offset:offset + arr_len].view(dt)
597
da = np.char.array(da.view(dtype=dt), itemsize=count)
598
dummy[idx] = decode_ascii(da)
599
else:
600
dt = np.dtype(recformat.dtype)
601
arr_len = count * dt.itemsize
602
dummy[idx] = raw_data[offset:offset + arr_len].view(dt)
603
dummy[idx].dtype = dummy[idx].dtype.newbyteorder('>')
604
# Each array in the field may now require additional
605
# scaling depending on the other scaling parameters
606
# TODO: The same scaling parameters apply to every
607
# array in the column so this is currently very slow; we
608
# really only need to check once whether any scaling will
609
# be necessary and skip this step if not
610
# TODO: Test that this works for X format; I don't think
611
# that it does--the recformat variable only applies to the P
612
# format not the X format
613
dummy[idx] = self._convert_other(indx, dummy[idx], recformat)
614
615
return dummy
616
617
def _convert_ascii(self, indx, field):
618
"""Special handling for ASCII table columns to convert columns
619
containing numeric types to actual numeric arrays from the string
620
representation.
621
"""
622
623
format = self._coldefs.formats[indx]
624
recformat = ASCII2NUMPY[format[0]]
625
# if the string = TNULL, return ASCIITNULL
626
nullval = str(self._coldefs.nulls[indx]).strip().encode('ascii')
627
if len(nullval) > format.width:
628
nullval = nullval[:format.width]
629
dummy = field.replace('D'.encode('ascii'), 'E'.encode('ascii'))
630
dummy = np.where(dummy.strip() == nullval, str(ASCIITNULL), dummy)
631
632
try:
633
dummy = np.array(dummy, dtype=recformat)
634
except ValueError, e:
635
raise ValueError(
636
'%s; the header may be missing the necessary TNULL%d '
637
'keyword or the table contains invalid data' % (e, indx + 1))
638
639
return dummy
640
641
def _convert_other(self, indx, field, recformat):
642
"""Perform conversions on any other fixed-width column data types.
643
644
This may not perform any conversion at all if it's not necessary, in
645
which case the original column array is returned.
646
"""
647
648
if isinstance(recformat, _FormatX):
649
# special handling for the X format
650
return self._convert_x(field, recformat)
651
652
(_str, _bool, _number, _scale, _zero, bscale, bzero, dim) = \
653
self._get_scale_factors(indx)
654
655
# ASCII table, convert strings to numbers
656
# TODO:
657
# For now, check that these are ASCII columns by checking the coldefs
658
# type; in the future all columns (for binary tables, ASCII tables, or
659
# otherwise) should "know" what type they are already and how to handle
660
# converting their data from FITS format to native format and vice
661
# versa...
662
if not _str and isinstance(self._coldefs, _AsciiColDefs):
663
field = self._convert_ascii(indx, field)
664
665
# Test that the dimensions given in dim are sensible; otherwise
666
# display a warning and ignore them
667
if dim:
668
# See if the dimensions already match, if not, make sure the
669
# number items will fit in the specified dimensions
670
if field.ndim > 1:
671
actual_shape = field[0].shape
672
if _str:
673
actual_shape = (field[0].itemsize,) + actual_shape
674
else:
675
actual_shape = len(field[0])
676
677
if dim == actual_shape:
678
# The array already has the correct dimensions, so we
679
# ignore dim and don't convert
680
dim = None
681
else:
682
nitems = reduce(operator.mul, dim)
683
if _str:
684
actual_nitems = field.itemsize
433
685
else:
434
actual_shape = len(dummy[0])
435
if dim == actual_shape:
436
# The array already has the correct dimensions, so we
437
# ignore dim and don't convert
686
actual_nitems = field.shape[1]
687
if nitems > actual_nitems:
688
warnings.warn(
689
'TDIM%d value %s does not fit with the size of '
690
'the array items (%d). TDIM%d will be ignored.'
691
% (indx + 1, self._coldefs.dims[indx],
692
actual_nitems, indx + 1))
438
693
dim = None
694
695
# further conversion for both ASCII and binary tables
696
# For now we've made columns responsible for *knowing* whether their
697
# data has been scaled, but we make the FITS_rec class responsible for
698
# actually doing the scaling
699
# TODO: This also needs to be fixed in the effort to make Columns
700
# responsible for scaling their arrays to/from FITS native values
701
column = self._coldefs[indx]
702
if (_number and (_scale or _zero) and not column._physical_values):
703
# This is to handle pseudo unsigned ints in table columns
704
# TODO: For now this only really works correctly for binary tables
705
# Should it work for ASCII tables as well?
706
if self._uint:
707
if bzero == 2**15 and 'I' in self._coldefs.formats[indx]:
708
field = np.array(field, dtype=np.uint16)
709
elif bzero == 2**31 and 'J' in self._coldefs.formats[indx]:
710
field = np.array(field, dtype=np.uint32)
711
elif bzero == 2**63 and 'K' in self._coldefs.formats[indx]:
712
field = np.array(field, dtype=np.uint64)
713
bzero64 = np.uint64(2 ** 63)
439
714
else:
440
nitems = reduce(operator.mul, dim)
441
if _str:
442
actual_nitems = dummy.itemsize
715
field = np.array(field, dtype=np.float64)
716
else:
717
field = np.array(field, dtype=np.float64)
718
719
if _scale:
720
np.multiply(field, bscale, field)
721
if _zero:
722
if self._uint and 'K' in self._coldefs.formats[indx]:
723
# There is a chance of overflow, so be careful
724
test_overflow = field.copy()
725
try:
726
test_overflow += bzero64
727
except OverflowError:
728
warnings.warn(
729
"Overflow detected while applying TZERO{0:d}. "
730
"Returning unscaled data.".format(indx))
443
731
else:
444
actual_nitems = dummy.shape[1]
445
if nitems > actual_nitems:
446
warnings.warn(
447
'TDIM%d value %s does not fit with the size of '
448
'the array items (%d). TDIM%d will be ignored.'
449
% (indx + 1, self._coldefs.dims[indx],
450
actual_nitems, indx + 1))
451
dim = None
452
453
# further conversion for both ASCII and binary tables
454
if _number and (_scale or _zero):
455
456
# only do the scaling the first time and store it in _convert
457
self._convert[indx] = np.array(dummy, dtype=np.float64)
458
if _scale:
459
np.multiply(self._convert[indx], bscale,
460
self._convert[indx])
461
if _zero:
462
self._convert[indx] += bzero
463
elif _bool and dummy.dtype != bool:
464
self._convert[indx] = np.equal(dummy, ord('T'))
465
elif _str:
466
try:
467
self._convert[indx] = decode_ascii(dummy)
468
except UnicodeDecodeError:
469
pass
470
471
if dim:
472
nitems = reduce(operator.mul, dim)
473
if self._convert[indx] is None:
474
self._convert[indx] = dummy[:,:nitems]
475
if _str:
476
fmt = self._convert[indx].dtype.char
477
dtype = ('|%s%d' % (fmt, dim[-1]), dim[:-1])
478
self._convert[indx].dtype = dtype
732
field = test_overflow
479
733
else:
480
self._convert[indx].shape = (dummy.shape[0],) + dim
481
482
if self._convert[indx] is not None:
483
return self._convert[indx]
484
else:
485
return dummy
734
field += bzero
735
elif _bool and field.dtype != bool:
736
field = np.equal(field, ord('T'))
737
elif _str:
738
try:
739
field = decode_ascii(field)
740
except UnicodeDecodeError:
741
pass
742
743
if dim:
744
# Apply the new field item dimensions
745
nitems = reduce(operator.mul, dim)
746
if field.ndim > 1:
747
field = field[:, :nitems]
748
if _str:
749
fmt = field.dtype.char
750
dtype = ('|%s%d' % (fmt, dim[-1]), dim[:-1])
751
field.dtype = dtype
752
else:
753
field.shape = (field.shape[0],) + dim
754
755
return field
486
756
487
757
def _clone(self, shape):
488
758
"""
525
795
`indx` is the index of the field.
526
796
"""
527
797
528
if self._coldefs._tbtype == 'BinTableHDU':
798
if isinstance(self._coldefs, _AsciiColDefs):
799
_str = self._coldefs.formats[indx][0] == 'A'
800
_bool = False # there is no boolean in ASCII table
801
else:
529
802
_str = 'a' in self._coldefs._recformats[indx]
803
# TODO: Determine a better way to determine if the column is bool
804
# formatted
530
805
_bool = self._coldefs._recformats[indx][-2:] == FITS2NUMPY['L']
531
else:
532
_str = self._coldefs.formats[indx][0] == 'A'
533
_bool = False # there is no boolean in ASCII table
534
_number = not(_bool or _str)
806
807
_number = not (_bool or _str)
535
808
bscale = self._coldefs.bscales[indx]
536
809
bzero = self._coldefs.bzeros[indx]
537
_scale = bscale not in ['', None, 1]
538
_zero = bzero not in ['', None, 0]
810
_scale = bscale not in ('', None, 1)
811
_zero = bzero not in ('', None, 0)
539
812
# ensure bscale/bzero are numbers
540
813
if not _scale:
541
814
bscale = 1
551
824
Update the parent array, using the (latest) scaled array.
552
825
"""
553
826
554
_fmap = {'A': 's', 'I': 'd', 'J': 'd', 'F': 'f', 'E': 'E', 'D': 'E'}
555
# calculate the starting point and width of each field for ASCII table
556
# TODO: Ick--fix this _tbtype usage eventually...
557
if self._coldefs._tbtype == 'TableHDU':
558
loc = self._coldefs.starts
559
widths = []
560
561
idx = 0
562
for idx in range(len(self.dtype.names)):
563
f = _convert_ascii_format(self._coldefs.formats[idx])
564
widths.append(f[1])
565
loc.append(loc[-1] + super(FITS_rec, self).field(idx).itemsize)
566
567
827
for indx in range(len(self.dtype.names)):
568
828
recformat = self._coldefs._recformats[indx]
569
829
field = super(FITS_rec, self).field(indx)
572
832
continue
573
833
574
834
if isinstance(recformat, _FormatX):
575
_wrapx(self._convert[indx], field, recformat._nx)
835
_wrapx(self._convert[indx], field, recformat.repeat)
576
836
continue
577
837
578
(_str, _bool, _number, _scale, _zero, bscale, bzero, dim) = \
838
_str, _bool, _number, _scale, _zero, bscale, bzero, _ = \
579
839
self._get_scale_factors(indx)
580
840
581
841
# add the location offset of the heap area for each
587
847
self._heapsize = 0
588
848
589
849
field[:] = 0 # reset
590
npts = map(len, self._convert[indx])
850
npts = [len(arr) for arr in self._convert[indx]]
591
851
852
# Irritatingly, this can return a different dtype than just
853
# doing np.dtype(recformat.dtype); but this returns the results
854
# that we want. For example if recformat.dtype is 'a' we want
855
# an array of characters.
592
856
dtype = np.array([], dtype=recformat.dtype).dtype
593
857
field[:len(npts), 0] = npts
594
858
field[1:, 1] = (np.add.accumulate(field[:-1, 0]) *
598
862
599
863
# conversion for both ASCII and binary tables
600
864
if _number or _str:
601
if _number and (_scale or _zero):
865
column = self._coldefs[indx]
866
if _number and (_scale or _zero) and column._physical_values:
602
867
dummy = self._convert[indx].copy()
603
868
if _zero:
604
869
dummy -= bzero
605
870
if _scale:
606
871
dummy /= bscale
872
# This will set the raw values in the recarray back to
873
# their non-physical storage values, so the column should
874
# be mark is not scaled
875
column._physical_values = False
607
876
elif _str:
608
877
dummy = self._convert[indx]
609
elif self._coldefs._tbtype == 'TableHDU':
878
elif isinstance(self._coldefs, _AsciiColDefs):
610
879
dummy = self._convert[indx]
611
880
else:
612
881
continue
613
882
614
883
# ASCII table, convert numbers to strings
615
if self._coldefs._tbtype == 'TableHDU':
884
if isinstance(self._coldefs, _AsciiColDefs):
885
starts = self._coldefs.starts[:]
886
spans = self._coldefs.spans
616
887
format = self._coldefs.formats[indx].strip()
617
lead = self._coldefs.starts[indx] - loc[indx]
888
889
# The the index of the "end" column of the record, beyond
890
# which we can't write
891
end = super(FITS_rec, self).field(-1).itemsize
892
starts.append(end + starts[-1])
893
894
if indx > 0:
895
lead = (starts[indx] - starts[indx - 1] -
896
spans[indx - 1])
897
else:
898
lead = 0
899
618
900
if lead < 0:
619
raise ValueError(
620
'Column `%s` starting point overlaps to the '
621
'previous column.' % indx + 1)
622
trail = (loc[indx + 1] - widths[indx] -
623
self._coldefs.starts[indx])
901
warnings.warn(
902
'Column %r starting point overlaps the '
903
'previous column.' % (indx + 1))
904
905
trail = starts[indx + 1] - starts[indx] - spans[indx]
906
624
907
if trail < 0:
625
raise ValueError(
626
'Column `%s` ending point overlaps to the next '
627
'column.' % indx + 1)
908
warnings.warn(
909
'Column %r ending point overlaps the next '
910
'column.' % (indx + 1))
911
912
# TODO: It would be nice if these string column formatting
913
# details were left to a specialized class, as is the case
914
# with FormatX and FormatP
628
915
if 'A' in format:
629
916
_pc = '%-'
630
917
else:
631
918
_pc = '%'
632
919
633
fmt = ''.join([(' ' * lead), _pc, format[1:],
634
_fmap[format[0]], (' ' * trail)])
920
fmt = ''.join([_pc, format[1:], ASCII2STR[format[0]],
921
(' ' * trail)])
635
922
636
923
# not using numarray.strings's num2char because the
637
924
# result is not allowed to expand (as C/Python does).
638
925
for jdx in range(len(dummy)):
639
926
x = fmt % dummy[jdx]
640
if len(x) > (loc[indx + 1] - loc[indx]):
927
if len(x) > starts[indx + 1] - starts[indx]:
641
928
raise ValueError(
642
"Number `%s` does not fit into the output's "
643
"itemsize of %s." % (x, widths[indx]))
929
"Value %r does not fit into the output's "
930
"itemsize of %s." % (x, spans[indx]))
644
931
else:
645
932
field[jdx] = x
646
933
# Replace exponent separator in floating point numbers
Loggerhead is a web-based interface for Breezy
Version: 2.0.1