3
# Copyright © 2009-2010 Pierre Raybaut
4
# Licensed under the terms of the MIT License
5
# (see spyderlib/__init__.py for details)
10
Note: 'load' functions has to return a dictionary from which a globals()
11
namespace may be updated
14
from __future__ import with_statement
28
with warnings.catch_warnings():
29
warnings.simplefilter("ignore")
30
import scipy.io as spio
31
except AttributeError:
32
# Python 2.5: warnings.catch_warnings was introduced in Python 2.6
33
import scipy.io as spio
34
def load_matlab(filename):
36
out = spio.loadmat(filename, struct_as_record=True,
38
for key, value in out.iteritems():
39
if isinstance(value, np.ndarray):
41
out[key] = value.tolist()
42
# The following would be needed if squeeze_me=False:
43
# elif value.shape == (1,):
45
# elif value.shape == (1, 1):
46
# out[key] = value[0][0]
48
except Exception, error:
49
return None, str(error)
50
def save_matlab(data, filename):
52
spio.savemat(filename, data, oned_as='row')
53
except Exception, error:
62
def load_array(filename):
64
name = osp.splitext(osp.basename(filename))[0]
65
return {name: np.load(filename)}, None
66
except Exception, error:
67
return None, str(error)
68
def __save_array(data, basename, index):
69
"""Save numpy array"""
70
fname = basename + '_%04d.npy' % index
78
from spyderlib.pil_patch import Image
79
if sys.byteorder == 'little':
86
"I": ('%si4' % _ENDIAN, None),
87
"F": ('%sf4' % _ENDIAN, None),
88
"I;16": ('|u2', None),
89
"I;16S": ('%si2' % _ENDIAN, None),
97
def __image_to_array(filename):
98
img = Image.open(filename)
100
dtype, extra = DTYPES[img.mode]
102
raise RuntimeError("%s mode is not supported" % img.mode)
103
shape = (img.size[1], img.size[0])
104
if extra is not None:
106
return np.array(img.getdata(), dtype=np.dtype(dtype)).reshape(shape)
107
def load_image(filename):
109
name = osp.splitext(osp.basename(filename))[0]
110
return {name: __image_to_array(filename)}, None
111
except Exception, error:
112
return None, str(error)
117
def save_dictionary(data, filename):
118
"""Save dictionary in a single file .spydata file"""
119
filename = osp.abspath(filename)
120
old_cwd = os.getcwdu()
121
os.chdir(osp.dirname(filename))
125
if load_array is not None:
126
# Saving numpy arrays with np.save
127
arr_fname = osp.splitext(filename)[0]
128
for name in data.keys():
129
if isinstance(data[name], np.ndarray) and data[name].size > 0:
130
# Saving arrays at data root
131
fname = __save_array(data[name], arr_fname,
133
saved_arrays[(name, None)] = osp.basename(fname)
135
elif isinstance(data[name], (list, dict)):
136
# Saving arrays nested in lists or dictionaries
137
if isinstance(data[name], list):
138
iterator = enumerate(data[name])
140
iterator = data[name].iteritems()
142
for index, value in iterator:
143
if isinstance(value, np.ndarray) and value.size > 0:
144
fname = __save_array(value, arr_fname,
146
saved_arrays[(name, index)] = osp.basename(fname)
147
to_remove.append(index)
148
for index in sorted(to_remove, reverse=True):
149
data[name].pop(index)
151
data['__saved_arrays__'] = saved_arrays
152
pickle_filename = osp.splitext(filename)[0]+'.pickle'
153
cPickle.dump(data, file(pickle_filename, 'w'))
154
tar = tarfile.open(filename, "w")
155
for fname in [pickle_filename]+[fn for fn in saved_arrays.itervalues()]:
156
tar.add(osp.basename(fname))
160
data.pop('__saved_arrays__')
161
except (RuntimeError, cPickle.PicklingError, TypeError), error:
162
error_message = unicode(error)
166
def load_dictionary(filename):
167
"""Load dictionary from .spydata file"""
168
filename = osp.abspath(filename)
169
old_cwd = os.getcwdu()
170
os.chdir(osp.dirname(filename))
174
tar = tarfile.open(filename, "r")
176
pickle_filename = osp.splitext(filename)[0]+'.pickle'
177
data = cPickle.loads(file(pickle_filename, 'U').read())
179
if load_array is not None:
180
# Loading numpy arrays saved with np.save
182
saved_arrays = data.pop('__saved_arrays__')
183
for (name, index), fname in saved_arrays.iteritems():
184
arr = np.load( osp.join(osp.dirname(filename), fname) )
187
elif isinstance(data[name], dict):
188
data[name][index] = arr
190
data[name].insert(index, arr)
193
for fname in [pickle_filename]+[fn for fn in saved_arrays.itervalues()]:
195
except (EOFError, ValueError), error:
196
error_message = unicode(error)
198
return data, error_message
201
from spyderlib.baseconfig import get_conf_path, STDERR
203
SAVED_CONFIG_FILES = ('.inspector', '.onlinehelp', '.path', '.pylint.results',
204
'.spyder.ini', '.temp.py', '.temp.spydata', 'template.py',
205
'.projects', '.history.py', '.history_internal.py',
206
'.spyderproject', '.ropeproject', '.workingdir',
207
'monitor.log', 'monitor_debug.log', 'rope.log')
210
"""Remove all config files"""
211
print >>STDERR, "*** Reset Spyder settings to defaults ***"
212
for fname in SAVED_CONFIG_FILES:
213
cfg_fname = get_conf_path(fname)
214
if osp.isfile(cfg_fname):
216
elif osp.isdir(cfg_fname):
217
shutil.rmtree(cfg_fname)
220
print >>STDERR, "removing:", cfg_fname
222
def save_session(filename):
223
"""Save Spyder session"""
224
local_fname = get_conf_path(osp.basename(filename))
225
filename = osp.abspath(filename)
226
old_cwd = os.getcwdu()
227
os.chdir(get_conf_path())
230
tar = tarfile.open(local_fname, "w")
231
for fname in SAVED_CONFIG_FILES:
232
if osp.isfile(fname):
235
shutil.move(local_fname, filename)
236
except Exception, error:
237
error_message = unicode(error)
241
def load_session(filename):
242
"""Load Spyder session"""
243
filename = osp.abspath(filename)
244
old_cwd = os.getcwdu()
245
os.chdir(osp.dirname(filename))
249
tar = tarfile.open(filename, "r")
250
extracted_files = tar.getnames()
252
# Rename original config files
253
for fname in extracted_files:
254
orig_name = get_conf_path(fname)
255
bak_name = get_conf_path(fname+'.bak')
256
if osp.isfile(bak_name):
258
if osp.isfile(orig_name):
259
os.rename(orig_name, bak_name)
264
for fname in extracted_files:
265
shutil.move(fname, get_conf_path(fname))
267
except Exception, error:
268
error_message = unicode(error)
270
# Restore original config files
271
for fname in extracted_files:
272
orig_name = get_conf_path(fname)
273
bak_name = get_conf_path(fname+'.bak')
274
if osp.isfile(orig_name):
276
if osp.isfile(bak_name):
277
os.rename(bak_name, orig_name)
280
# Removing backup config files
281
for fname in extracted_files:
282
bak_name = get_conf_path(fname+'.bak')
283
if osp.isfile(bak_name):
290
from spyderlib.baseconfig import _
292
class IOFunctions(object):
294
self.load_extensions = None
295
self.save_extensions = None
296
self.load_filters = None
297
self.save_filters = None
298
self.load_funcs = None
299
self.save_funcs = None
302
iofuncs = self.get_internal_funcs()+self.get_3rd_party_funcs()
310
for ext, name, loadfunc, savefunc in iofuncs:
311
filter_str = unicode(name + " (*%s)" % ext)
312
if loadfunc is not None:
313
load_filters.append(filter_str)
314
load_extensions[filter_str] = ext
315
load_funcs[ext] = loadfunc
317
if savefunc is not None:
318
save_extensions[filter_str] = ext
319
save_filters.append(filter_str)
320
save_funcs[ext] = savefunc
321
load_filters.insert(0, unicode(_("Supported files")+" (*"+\
322
" *".join(load_ext)+")"))
323
self.load_filters = "\n".join(load_filters)
324
self.save_filters = "\n".join(save_filters)
325
self.load_funcs = load_funcs
326
self.save_funcs = save_funcs
327
self.load_extensions = load_extensions
328
self.save_extensions = save_extensions
330
def get_internal_funcs(self):
332
('.spydata', _("Spyder data files"),
333
load_dictionary, save_dictionary),
334
('.npy', _("NumPy arrays"), load_array, None),
335
('.mat', _("Matlab files"), load_matlab, save_matlab),
336
('.csv', _("CSV text files"), 'import_wizard', None),
337
('.txt', _("Text files"), 'import_wizard', None),
338
('.jpg', _("JPEG images"), load_image, None),
339
('.png', _("PNG images"), load_image, None),
340
('.gif', _("GIF images"), load_image, None),
341
('.tif', _("TIFF images"), load_image, None),
344
def get_3rd_party_funcs(self):
346
from spyderlib.otherplugins import get_spyderplugins_mods
347
for mod in get_spyderplugins_mods(prefix='io_', extension='.py'):
349
other_funcs.append((mod.FORMAT_EXT, mod.FORMAT_NAME,
350
mod.FORMAT_LOAD, mod.FORMAT_SAVE))
351
except AttributeError, error:
352
print >>STDERR, "%s: %s" % (mod, str(error))
355
def save(self, data, filename):
356
ext = osp.splitext(filename)[1].lower()
357
if ext in self.save_funcs:
358
return self.save_funcs[ext](data, filename)
360
return _("<b>Unsupported file type '%s'</b>") % ext
362
def load(self, filename):
363
ext = osp.splitext(filename)[1].lower()
364
if ext in self.load_funcs:
365
return self.load_funcs[ext](filename)
367
return None, _("<b>Unsupported file type '%s'</b>") % ext
369
iofunctions = IOFunctions()
373
def save_auto(data, filename):
374
"""Save data into filename, depending on file extension"""
378
if __name__ == "__main__":
380
testdict = {'d': 1, 'a': np.random.rand(10, 10), 'b': [1, 2]}
381
testdate = datetime.date(1945, 5, 8)
382
example = {'str': 'kjkj kj k j j kj k jkj',
384
'list': [1, 3, [4, 5, 6], 'kjkj', None],
385
'tuple': ([1, testdate, testdict], 'kjkj', None),
388
'array': np.random.rand(4000, 400),
389
'empty_array': np.array([]),
391
'datetime': datetime.datetime(1945, 5, 8),
395
save_dictionary(example, "test.spydata")
396
print " Data saved in %.3f seconds" % (time.time()-t0)
398
example2, ok = load_dictionary("test.spydata")
399
print "Data loaded in %.3f seconds" % (time.time()-t0)
400
# for key in example:
401
# print key, ":", example[key] == example2[key]
3
# Copyright © 2009-2010 Pierre Raybaut
4
# Licensed under the terms of the MIT License
5
# (see spyderlib/__init__.py for details)
10
Note: 'load' functions has to return a dictionary from which a globals()
11
namespace may be updated
14
from __future__ import with_statement
28
with warnings.catch_warnings():
29
warnings.simplefilter("ignore")
30
import scipy.io as spio
31
except AttributeError:
32
# Python 2.5: warnings.catch_warnings was introduced in Python 2.6
33
import scipy.io as spio
34
def load_matlab(filename):
36
out = spio.loadmat(filename, struct_as_record=True,
38
for key, value in out.iteritems():
39
if isinstance(value, np.ndarray):
41
out[key] = value.tolist()
42
# The following would be needed if squeeze_me=False:
43
# elif value.shape == (1,):
45
# elif value.shape == (1, 1):
46
# out[key] = value[0][0]
48
except Exception, error:
49
return None, str(error)
50
def save_matlab(data, filename):
52
spio.savemat(filename, data, oned_as='row')
53
except Exception, error:
62
def load_array(filename):
64
name = osp.splitext(osp.basename(filename))[0]
65
return {name: np.load(filename)}, None
66
except Exception, error:
67
return None, str(error)
68
def __save_array(data, basename, index):
69
"""Save numpy array"""
70
fname = basename + '_%04d.npy' % index
78
from spyderlib.pil_patch import Image
79
if sys.byteorder == 'little':
86
"I": ('%si4' % _ENDIAN, None),
87
"F": ('%sf4' % _ENDIAN, None),
88
"I;16": ('|u2', None),
89
"I;16S": ('%si2' % _ENDIAN, None),
97
def __image_to_array(filename):
98
img = Image.open(filename)
100
dtype, extra = DTYPES[img.mode]
102
raise RuntimeError("%s mode is not supported" % img.mode)
103
shape = (img.size[1], img.size[0])
104
if extra is not None:
106
return np.array(img.getdata(), dtype=np.dtype(dtype)).reshape(shape)
107
def load_image(filename):
109
name = osp.splitext(osp.basename(filename))[0]
110
return {name: __image_to_array(filename)}, None
111
except Exception, error:
112
return None, str(error)
117
def save_dictionary(data, filename):
118
"""Save dictionary in a single file .spydata file"""
119
filename = osp.abspath(filename)
120
old_cwd = os.getcwdu()
121
os.chdir(osp.dirname(filename))
125
if load_array is not None:
126
# Saving numpy arrays with np.save
127
arr_fname = osp.splitext(filename)[0]
128
for name in data.keys():
129
if isinstance(data[name], np.ndarray) and data[name].size > 0:
130
# Saving arrays at data root
131
fname = __save_array(data[name], arr_fname,
133
saved_arrays[(name, None)] = osp.basename(fname)
135
elif isinstance(data[name], (list, dict)):
136
# Saving arrays nested in lists or dictionaries
137
if isinstance(data[name], list):
138
iterator = enumerate(data[name])
140
iterator = data[name].iteritems()
142
for index, value in iterator:
143
if isinstance(value, np.ndarray) and value.size > 0:
144
fname = __save_array(value, arr_fname,
146
saved_arrays[(name, index)] = osp.basename(fname)
147
to_remove.append(index)
148
for index in sorted(to_remove, reverse=True):
149
data[name].pop(index)
151
data['__saved_arrays__'] = saved_arrays
152
pickle_filename = osp.splitext(filename)[0]+'.pickle'
153
cPickle.dump(data, file(pickle_filename, 'w'))
154
tar = tarfile.open(filename, "w")
155
for fname in [pickle_filename]+[fn for fn in saved_arrays.itervalues()]:
156
tar.add(osp.basename(fname))
160
data.pop('__saved_arrays__')
161
except (RuntimeError, cPickle.PicklingError, TypeError), error:
162
error_message = unicode(error)
166
def load_dictionary(filename):
167
"""Load dictionary from .spydata file"""
168
filename = osp.abspath(filename)
169
old_cwd = os.getcwdu()
170
os.chdir(osp.dirname(filename))
174
tar = tarfile.open(filename, "r")
176
pickle_filename = osp.splitext(filename)[0]+'.pickle'
177
data = cPickle.loads(file(pickle_filename, 'U').read())
179
if load_array is not None:
180
# Loading numpy arrays saved with np.save
182
saved_arrays = data.pop('__saved_arrays__')
183
for (name, index), fname in saved_arrays.iteritems():
184
arr = np.load( osp.join(osp.dirname(filename), fname) )
187
elif isinstance(data[name], dict):
188
data[name][index] = arr
190
data[name].insert(index, arr)
193
for fname in [pickle_filename]+[fn for fn in saved_arrays.itervalues()]:
195
except (EOFError, ValueError), error:
196
error_message = unicode(error)
198
return data, error_message
201
from spyderlib.baseconfig import get_conf_path, STDERR
203
SAVED_CONFIG_FILES = ('.inspector', '.onlinehelp', '.path', '.pylint.results',
204
'.spyder.ini', '.temp.py', '.temp.spydata', 'template.py',
205
'.projects', '.history.py', '.history_internal.py',
206
'.spyderproject', '.ropeproject', '.workingdir',
207
'monitor.log', 'monitor_debug.log', 'rope.log')
210
"""Remove all config files"""
211
print >>STDERR, "*** Reset Spyder settings to defaults ***"
212
for fname in SAVED_CONFIG_FILES:
213
cfg_fname = get_conf_path(fname)
214
if osp.isfile(cfg_fname):
216
elif osp.isdir(cfg_fname):
217
shutil.rmtree(cfg_fname)
220
print >>STDERR, "removing:", cfg_fname
222
def save_session(filename):
223
"""Save Spyder session"""
224
local_fname = get_conf_path(osp.basename(filename))
225
filename = osp.abspath(filename)
226
old_cwd = os.getcwdu()
227
os.chdir(get_conf_path())
230
tar = tarfile.open(local_fname, "w")
231
for fname in SAVED_CONFIG_FILES:
232
if osp.isfile(fname):
235
shutil.move(local_fname, filename)
236
except Exception, error:
237
error_message = unicode(error)
241
def load_session(filename):
242
"""Load Spyder session"""
243
filename = osp.abspath(filename)
244
old_cwd = os.getcwdu()
245
os.chdir(osp.dirname(filename))
249
tar = tarfile.open(filename, "r")
250
extracted_files = tar.getnames()
252
# Rename original config files
253
for fname in extracted_files:
254
orig_name = get_conf_path(fname)
255
bak_name = get_conf_path(fname+'.bak')
256
if osp.isfile(bak_name):
258
if osp.isfile(orig_name):
259
os.rename(orig_name, bak_name)
264
for fname in extracted_files:
265
shutil.move(fname, get_conf_path(fname))
267
except Exception, error:
268
error_message = unicode(error)
270
# Restore original config files
271
for fname in extracted_files:
272
orig_name = get_conf_path(fname)
273
bak_name = get_conf_path(fname+'.bak')
274
if osp.isfile(orig_name):
276
if osp.isfile(bak_name):
277
os.rename(bak_name, orig_name)
280
# Removing backup config files
281
for fname in extracted_files:
282
bak_name = get_conf_path(fname+'.bak')
283
if osp.isfile(bak_name):
290
from spyderlib.baseconfig import _
292
class IOFunctions(object):
294
self.load_extensions = None
295
self.save_extensions = None
296
self.load_filters = None
297
self.save_filters = None
298
self.load_funcs = None
299
self.save_funcs = None
302
iofuncs = self.get_internal_funcs()+self.get_3rd_party_funcs()
310
for ext, name, loadfunc, savefunc in iofuncs:
311
filter_str = unicode(name + " (*%s)" % ext)
312
if loadfunc is not None:
313
load_filters.append(filter_str)
314
load_extensions[filter_str] = ext
315
load_funcs[ext] = loadfunc
317
if savefunc is not None:
318
save_extensions[filter_str] = ext
319
save_filters.append(filter_str)
320
save_funcs[ext] = savefunc
321
load_filters.insert(0, unicode(_("Supported files")+" (*"+\
322
" *".join(load_ext)+")"))
323
self.load_filters = "\n".join(load_filters)
324
self.save_filters = "\n".join(save_filters)
325
self.load_funcs = load_funcs
326
self.save_funcs = save_funcs
327
self.load_extensions = load_extensions
328
self.save_extensions = save_extensions
330
def get_internal_funcs(self):
332
('.spydata', _("Spyder data files"),
333
load_dictionary, save_dictionary),
334
('.npy', _("NumPy arrays"), load_array, None),
335
('.mat', _("Matlab files"), load_matlab, save_matlab),
336
('.csv', _("CSV text files"), 'import_wizard', None),
337
('.txt', _("Text files"), 'import_wizard', None),
338
('.jpg', _("JPEG images"), load_image, None),
339
('.png', _("PNG images"), load_image, None),
340
('.gif', _("GIF images"), load_image, None),
341
('.tif', _("TIFF images"), load_image, None),
344
def get_3rd_party_funcs(self):
346
from spyderlib.otherplugins import get_spyderplugins_mods
347
for mod in get_spyderplugins_mods(prefix='io_', extension='.py'):
349
other_funcs.append((mod.FORMAT_EXT, mod.FORMAT_NAME,
350
mod.FORMAT_LOAD, mod.FORMAT_SAVE))
351
except AttributeError, error:
352
print >>STDERR, "%s: %s" % (mod, str(error))
355
def save(self, data, filename):
356
ext = osp.splitext(filename)[1].lower()
357
if ext in self.save_funcs:
358
return self.save_funcs[ext](data, filename)
360
return _("<b>Unsupported file type '%s'</b>") % ext
362
def load(self, filename):
363
ext = osp.splitext(filename)[1].lower()
364
if ext in self.load_funcs:
365
return self.load_funcs[ext](filename)
367
return None, _("<b>Unsupported file type '%s'</b>") % ext
369
iofunctions = IOFunctions()
373
def save_auto(data, filename):
374
"""Save data into filename, depending on file extension"""
378
if __name__ == "__main__":
380
testdict = {'d': 1, 'a': np.random.rand(10, 10), 'b': [1, 2]}
381
testdate = datetime.date(1945, 5, 8)
382
example = {'str': 'kjkj kj k j j kj k jkj',
384
'list': [1, 3, [4, 5, 6], 'kjkj', None],
385
'tuple': ([1, testdate, testdict], 'kjkj', None),
388
'array': np.random.rand(4000, 400),
389
'empty_array': np.array([]),
391
'datetime': datetime.datetime(1945, 5, 8),
395
save_dictionary(example, "test.spydata")
396
print " Data saved in %.3f seconds" % (time.time()-t0)
398
example2, ok = load_dictionary("test.spydata")
399
print "Data loaded in %.3f seconds" % (time.time()-t0)
400
# for key in example:
401
# print key, ":", example[key] == example2[key]
added
removed
spyderlib/utils/iofuncs.py
