« back to all changes in this revision
Viewing changes to tests/actions.py
- browse files at revision 1176.1.62
- compare with another revision
- download diff
- download tarball
- view history from revision 1176.1.62
- Committer: Juho Vepsäläinen
- Date: 2009-09-15 12:39:24 UTC
- mto: This revision was merged to the branch mainline in revision 1306.
- Revision ID: bebraw@gmail.com-20090915123924-h9gu98dzdf3sztkq
Reverted to revision 1234.
- files added:
- phatch/lib/gettextFix.py
- tools
- files modified:
- .bzrignore
added
removed
tests/actions.py
1
1
#!/usr/bin/python
2
2
3
# Phatch - Photo Batch Processor
4
# Copyright (C) 2009 Nadia Alramli
5
#
6
# This program is free software: you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation, either version 3 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful,
12
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
# GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program. If not, see http://www.gnu.org/licenses/
18
#
19
# Phatch recommends SPE (http://pythonide.stani.be) for editing python files.
20
#
21
# Follows PEP8
22
3
23
import glob
4
import os
24
import os.path
5
25
import sys
26
import pprint
27
import logging
28
import subprocess
6
29
import time
7
import logging
30
import shutil
8
31
import optparse
9
from subprocess import Popen, PIPE
10
import pprint
11
12
DISABLE_ACTIONS = ['save', 'save tags', 'rename']
13
14
#import gettext
15
#gettext.install('test')
16
17
def get_actions(path):
18
"""Get a dictionary that maps action names to action classes"""
19
action_names = [
20
os.path.basename(action_file).partition('.')[0]
21
for action_file in glob.glob(os.path.join(path, '*.py'))
22
]
23
return dict(
24
(name, __import__('actions.%s' % name, name, fromlist=['actions']).Action())
25
for name in action_names
26
if name != '__init__'
27
)
28
29
30
def get_actionlist(actions):
31
"""Get actionlist structure from a list of actions"""
32
data = [action.dump() for action in actions]
33
return {'description': '', 'actions': data}
34
35
36
def build_library_actionlists(library_path, in_path, out_path):
37
"""Build library actionlists"""
38
if not os.path.exists(in_path):
39
os.makedirs(in_path)
40
result = []
41
actionlists = [
42
(os.path.basename(actionlist_file).partition('.')[0], eval(open(actionlist_file).read()))
43
for actionlist_file in glob.glob(os.path.join(library_path, '*.phatch'))
44
]
45
for name, actionlist in actionlists:
46
save_actions = [
47
action
48
for action in actionlist['actions']
49
if action['label'] == 'Save'
32
import gettext
33
from collections import defaultdict
34
from PIL import Image
35
36
37
DISABLE = ['rename']
38
TEMPLATE ="Errors: %(errors)s\nMissing: %(missing)s\nCorrupted: %(corrupted)s\nMismatch: %(mismatch)s\nNew: %(new)s"
39
40
41
gettext.install('test')
42
def system_path(path):
43
"""Convert a path string into the correct form"""
44
return os.path.abspath(os.path.normpath(path))
45
46
sys.path.insert(0, system_path('..'))
47
48
class PhatchTest(object):
49
50
DEFAULT_INPUT = system_path('images/input')
51
DEFAULT_OUTPUT = system_path('images/output')
52
DEFAULT_LOG = system_path('logs')
53
DEFAULT_REPORT = system_path('report')
54
ACTIONS_PATH = system_path('../phatch/actions')
55
LIBRARY_PATH = system_path('../data/actionlists')
56
OUT_ACTIONS_PATH = system_path('test_actionlists')
57
PHATCH_PATH = system_path('../phatch/phatch.py')
58
59
def __init__(self, input=DEFAULT_INPUT, output=DEFAULT_OUTPUT,
60
verbose=False, log=DEFAULT_LOG, compare=None):
61
"""Build a dictionary of actions"""
62
self.output = output
63
self.input = input
64
logging.basicConfig(
65
level=logging.DEBUG,
66
filename=log,
67
filemode='w',
68
format='%(asctime)s %(levelname)s:%(message)s'
69
)
70
self.clean()
71
self.compare = compare
72
create(self.OUT_ACTIONS_PATH)
73
create(self.output)
74
action_names = [
75
file_name(action_file)
76
for action_file in glob.glob(os.path.join(self.ACTIONS_PATH, '*.py'))
50
77
]
51
for save_action in save_actions:
52
save_action['fields']['In'] = '%s/<subfolder>' % out_path
53
save_action['fields']['File Name'] = '<filename>_%s' % name
54
result.append(write_file(os.path.join(
55
in_path,
56
'%s.phatch' % name
57
),
58
actionlist
59
))
60
return result
61
62
63
def build_actionlists(actions, in_path, out_path):
64
"""Build actionlists"""
65
actionlists = []
66
if not os.path.exists(in_path):
67
os.makedirs(in_path)
68
save_action = actions['save']
69
save_action.set_field(
70
'In',
71
'%s/<subfolder>' % out_path
72
)
73
for name, action in actions.iteritems():
74
if name in DISABLE_ACTIONS:
75
continue
76
choices = []
77
possible_choices(action, choices)
78
for choice in choices:
79
filename = [name]
80
for fname, fvalue in choice.iteritems():
81
action.set_field(fname, fvalue)
82
filename.append('%s=%s' % (fname.lower(), str(fvalue).lower()))
83
filename = '_'.join(filename)
84
save_action.set_field(
78
self.actions = dict(
79
(name, __import__('phatch.actions.%s' % name, name, fromlist=['phatch.actions']).Action())
80
for name in action_names
81
if name != '__init__' and name not in DISABLE
82
)
83
self.by_tag = {}
84
for name, action in self.actions.iteritems():
85
for tag in action.tags:
86
if tag in self.by_tag:
87
self.by_tag[tag][name] = action
88
else:
89
self.by_tag[tag] = {name: action}
90
self.file_actions = dict(
91
(name, action)
92
for name, action in self.actions.iteritems()
93
if action.valid_last
94
)
95
self.library = dict(
96
(file_name(actionlist_file), eval(open(actionlist_file).read()))
97
for actionlist_file in glob.glob(os.path.join(self.LIBRARY_PATH, '*.phatch'))
98
)
99
self.verbose = verbose
100
101
102
def clean(self):
103
if os.path.exists(self.output):
104
shutil.rmtree(self.output)
105
if os.path.exists(self.OUT_ACTIONS_PATH):
106
shutil.rmtree(self.OUT_ACTIONS_PATH)
107
108
109
def build_test(self, name, actionlist):
110
file_actions = [action for action in actionlist if action.valid_last]
111
for index, file_action in enumerate(file_actions):
112
file_action.set_field(
113
'In',
114
'%s' % self.output
115
)
116
file_action.set_field(
85
117
'File Name',
86
'<filename>_%s' % filename
87
)
88
actionlists.append(write_file(
89
os.path.join(
90
in_path,
91
'%s.phatch' % filename
92
),
93
get_actionlist([action, save_action])
94
))
95
return actionlists
96
97
98
def possible_choices(action, choices):
118
'<filename>_%s' % name_index(name, index)
119
)
120
return dump(actionlist)
121
122
123
def get_output_filename(self, filename, file_action):
124
try:
125
ftype = file_action.get_field_string('As')
126
if ftype == '<type>':
127
ftype = '%(type)s'
128
except KeyError:
129
ftype = '%(type)s'
130
return '%%(filename)s_%s.%s' % (filename, ftype)
131
132
133
def generate_actionlists(self, save='save', actions=None):
134
print 'Generating actionlists...'
135
actionlists = {}
136
if not actions:
137
actions = self.actions
138
for name, action in actions.iteritems():
139
if name in self.file_actions:
140
actionlist = [action]
141
file_action = action
142
else:
143
file_action = self.actions[save]
144
actionlist = [action, file_action]
145
choices = possible_choices(action)
146
for choice in choices:
147
set_action_fields(action, choice)
148
cname = choice_name(choice)
149
if cname:
150
filename = '%s_%s' % (name, cname)
151
else:
152
filename = name
153
filename = shorten(filename)
154
actionlist_data = self.build_test(filename, actionlist)
155
path = os.path.join(self.OUT_ACTIONS_PATH, '%s.phatch' % filename)
156
out_filename = self.get_output_filename(filename, file_action)
157
write_file(path, actionlist_data)
158
actionlists[filename] = {
159
'path': path,
160
'filename': out_filename
161
}
162
return actionlists
163
164
165
def generate_library_actionlists(self):
166
"""Build library actionlists"""
167
print 'Generating library actionlists...'
168
actionlists = {}
169
for name, actionlist in self.library.iteritems():
170
file_actions = [
171
action
172
for action in actionlist['actions']
173
if action['label'].lower() in self.file_actions
174
]
175
for index, file_action in enumerate(file_actions):
176
file_action['fields']['In'] = '%s' % self.output
177
file_action['fields']['File Name'] = '<filename>_%s' % name_index(name, index)
178
ftype = file_action['fields']['As']
179
if ftype == '<type>':
180
ftype = '%(type)s'
181
path = os.path.join(self.OUT_ACTIONS_PATH, '%s.phatch' % name)
182
out_filename = '%%(filename)s_%s.%s' % (name, ftype)
183
write_file(path, actionlist)
184
actionlists[name] = {
185
'path': path,
186
'filename': out_filename
187
}
188
return actionlists
189
190
191
def generate_custom_actionlists(self, action1, action2):
192
print 'Generating save actionlists...'
193
actionlists = {}
194
for choice1 in possible_choices(action1):
195
set_action_fields(action1, choice1)
196
cname1 = choice_name(choice1)
197
for choice2 in possible_choices(action2):
198
set_action_fields(action2, choice2)
199
cname2 = choice_name(choice2)
200
filename = '_'.join(
201
[action1.label.replace(' ', '-'), cname1,
202
action2.label.replace(' ', '-'), cname2]
203
)
204
actionlist_data = self.build_test(filename, [action1, action2])
205
path = os.path.join(self.OUT_ACTIONS_PATH, '%s.phatch' % filename)
206
out_filename = self.get_output_filename(filename, action2)
207
write_file(path, actionlist_data)
208
actionlists[filename] = {
209
'path': path,
210
'filename': out_filename
211
}
212
return actionlists
213
214
215
def execute_actionlists(self, actionlists=None):
216
start = time.time()
217
result = {
218
'errors': [],
219
'mismatch': [],
220
'new': [],
221
'corrupted': [],
222
'missing': []
223
}
224
try:
225
if not actionlists:
226
actionlists = phatch.generate_library_actionlists()
227
actionlists.update(phatch.generate_actionlists())
228
total = len(actionlists)
229
for i, name in enumerate(sorted(actionlists)):
230
if self.verbose:
231
print name.title().center(50, '-')
232
else:
233
sys.stdout.write(
234
'\rRunning %s/%s %s' % (
235
i + 1,
236
total,
237
name[:50].ljust(50)
238
)
239
)
240
sys.stdout.flush()
241
if not phatch.execute(actionlists[name]['path']):
242
result['errors'].append(name)
243
if self.verbose:
244
print
245
246
print '\nChecking all output images exist...'
247
if os.path.isfile(self.input):
248
images = [tuple(os.path.basename(self.input).rsplit('.', 1))]
249
else:
250
images = [tuple(image.rsplit('.', 1)) for image in os.listdir(self.input)]
251
filenames = []
252
for name, actionlist in actionlists.iteritems():
253
for image_name, image_type in images:
254
filename = actionlist['filename'] % {
255
'filename': image_name,
256
'type': image_type
257
}
258
path = os.path.join(self.output, filename)
259
if not os.path.exists(path):
260
result['missing'].append(filename)
261
else:
262
filenames.append(filename)
263
total = len(filenames)
264
if self.compare:
265
print '\nComparing results...'
266
for i, filename in enumerate(filenames):
267
sys.stdout.write(
268
'\rComparing %s/%s %s' % (
269
i + 1,
270
total,
271
filename[:50].ljust(50)
272
)
273
)
274
sys.stdout.flush()
275
try:
276
im1 = Image.open(os.path.join(self.output, filename))
277
except IOError:
278
result['corrupted'].append(filename)
279
continue
280
if not self.compare:
281
continue
282
try:
283
im2 = Image.open(os.path.join(self.compare, filename))
284
except IOError:
285
result['new'].append(filename)
286
continue
287
288
if not compare(im1, im2):
289
result['mismatch'].append(filename)
290
except KeyboardInterrupt:
291
print 'Stopped'
292
banner('Stats')
293
final_result = {}
294
for key, value in result.iteritems():
295
if value and isinstance(value, list):
296
value = '\n\t'.join(value)
297
else:
298
value = 'None'
299
final_result[key] = value
300
output = TEMPLATE % final_result
301
output += '\n%s' % ', '.join('%s: %s' % (key, len(value)) for key, value in result.iteritems())
302
f = open(self.DEFAULT_REPORT, 'w')
303
f.write(output)
304
f.close()
305
print output
306
print 'The report was saved in: %s' % self.DEFAULT_REPORT
307
print 'Execution took %.2f seconds' % (time.time() - start)
308
309
310
def execute(self, actionlist):
311
"""Execute the actionlist in phatch"""
312
if self.verbose:
313
stdout = None
314
else:
315
stdout = subprocess.PIPE
316
output, error = subprocess.Popen(
317
['python', self.PHATCH_PATH, '-cv', actionlist, self.input],
318
stdout=stdout, stderr=subprocess.PIPE
319
).communicate()
320
logs_file = open(system_path(os.path.expanduser('~/.cache/phatch/log')))
321
logs = logs_file.read()
322
logs_file.close()
323
324
if logs:
325
logging.error(
326
'Executing actionlist: %s generated the following logs:\n%s\n'
327
% (actionlist, logs)
328
)
329
if error:
330
logging.error(
331
'Executing actionlist: %s generated the following error:\n%s\n'
332
% (actionlist, error)
333
)
334
return not(error or logs)
335
336
337
def shorten(x):
338
return x.replace('True', '1').replace('False', '0')\
339
.replace('Background', 'Bg').replace('Rotation', 'Rot')\
340
.replace('Options', 'Op').replace('Position', 'Pos')\
341
.replace('Orientation', 'Or').replace('Vertical', 'Ver')\
342
.replace('Horizontal', 'Hor').replace('Transformation', 'Tr')\
343
.replace('Advanced', 'Adv').replace('Thumbnail', 'Thumb')\
344
.replace('Automatic', 'Auto').replace(' (use exif orientation)', '')\
345
.replace('Rotate', 'Rot').replace('Update', 'Upd')\
346
.replace('-Corner', '')
347
348
349
def possible_choices(action):
350
choices = []
351
possible_choices_helper(action, choices)
352
return choices
353
354
355
def possible_choices_helper(action, choices):
99
356
"""Generate all possible action choices"""
100
357
if hasattr(action, 'get_relevant_field_labels'):
101
358
relevant = action.get_relevant_field_labels()
129
386
if option == default:
130
387
continue
131
388
action.set_field(fname, option)
132
possible_choices(action, choices)
389
possible_choices_helper(action, choices)
390
391
392
def set_action_fields(action, fields):
393
for field_name, field_value in fields.iteritems():
394
action.set_field(field_name, field_value)
395
396
397
def dump(actions):
398
data = [action.dump() for action in actions]
399
return {'description': '', 'actions': data}
133
400
134
401
135
402
def write_file(path, data):
140
407
)
141
408
f.write(pprint.pformat(data))
142
409
f.close()
143
return path
144
145
146
def execute(input, actionlist):
147
"""Execute the actionlist in phatch"""
148
output, error = Popen(
149
['python', '../phatch/phatch.py', '-c', actionlist, input],
150
stdout=open('logs', 'a'), stderr=open('logs', 'a')
151
).communicate()
152
if output:
153
logging.info(
154
'Executing actionlist: %s generated the following output:\n%s'
155
% (actionlist, output)
156
)
157
if error:
158
logging.error(
159
'Executing actionlist: %s generated the following error:\n%s'
160
% (actionlist, error)
161
)
410
411
412
def create(path):
413
"""Create a path if it doesn't already exist"""
414
if not os.path.exists(path):
415
os.makedirs(path)
416
417
def file_name(path):
418
"""Get the file name from path without its extension"""
419
return os.path.basename(path).partition('.')[0]
420
421
def name_index(name, index):
422
if index:
423
return '%s_%s' % (name, index)
424
return name
425
426
def choice_name(choice):
427
result = '_'.join(
428
'%s=%s' % (
429
field_name.strip().replace(' ', '-'),
430
('%s' % field_value).strip().replace(' ', '-')
431
)
432
for field_name, field_value in choice.iteritems()
433
)
434
return result
435
436
def compare(im1, im2):
437
return list(im1.getdata()) == list(im2.getdata())
438
439
def banner(title, width=50):
440
print
441
print '*' * 50
442
print '*%s*' % title.center(48, ' ')
443
print '*' * 50
162
444
163
445
164
446
if __name__ == '__main__':
165
# Hack to be able to import phatch
166
sys.path.insert(0, '../phatch')
167
168
# Logging
169
logging.basicConfig(
170
filename='logs',
171
level=logging.DEBUG,
172
format='%(levelname)s: %(message)s'
173
)
174
175
447
# Option parser
176
448
parser = optparse.OptionParser()
177
449
parser.add_option(
178
450
'-i', '--input',
179
default='images/source',
451
default=PhatchTest.DEFAULT_INPUT,
180
452
help='Image input folder [default: %default]'
181
453
)
182
454
parser.add_option(
183
455
'-o', '--output',
184
default='images/output',
456
default=PhatchTest.DEFAULT_OUTPUT,
185
457
help='Image output folder [default: %default]'
186
458
)
187
459
parser.add_option(
460
'-v', '--verbose',
461
action='store_true',
462
default=False,
463
help='Verbose output'
464
)
465
parser.add_option(
188
466
'-a', '--actionlists',
189
467
action='store_true',
190
468
default=False,
191
469
help='Generate actionlists only'
192
470
)
193
471
parser.add_option(
194
'-l', '--library',
195
action='store_true',
196
default=False,
197
help='Run library actionlists tests only'
472
'-l', '--log',
473
default=PhatchTest.DEFAULT_LOG,
474
help='Log file path [default: %default]'
475
)
476
parser.add_option(
477
'-t', '--tag',
478
default=None,
479
help='Test by tag'
480
)
481
parser.add_option(
482
'-c', '--custom',
483
type="string",
484
nargs=2,
485
default=None,
486
help='Test custom action'
487
)
488
parser.add_option(
489
'--compare',
490
default=None,
491
help='Compare results'
198
492
)
199
493
options, args = parser.parse_args()
200
options.output = os.path.abspath(options.output)
201
start = time.time()
202
203
# Getting actions
204
actions = get_actions('../phatch/actions')
205
206
# Generating actionlists
207
try:
208
print 'Generating library actionlists...',
209
actionlists = build_library_actionlists(
210
'../data/actionlists',
211
'test_actionlists',
212
options.output
213
)
214
print 'Done'
215
216
if not options.library:
217
sys.stdout.write('Generating all possible actionlists...')
218
sys.stdout.flush()
219
actionlists.extend(
220
build_actionlists(actions, 'test_actionlists', '<root>/output')
494
phatch = PhatchTest(
495
input=system_path(options.input),
496
output=system_path(options.output),
497
verbose=options.verbose,
498
log=options.log,
499
compare=options.compare
500
)
501
if options.custom:
502
actionlists = phatch.generate_custom_actionlists(
503
action1=phatch.actions[options.custom[0]],
504
action2=phatch.actions[options.custom[1]],
505
)
506
elif options.tag == 'library':
507
actionlists = phatch.generate_library_actionlists()
508
elif options.tag == 'save':
509
actionlists = phatch.generate_custom_actionlists(
510
action1=phatch.actions['convert_mode'],
511
action2=phatch.actions['save'],
512
)
513
elif options.tag:
514
try:
515
actions = phatch.by_tag[options.tag]
516
except KeyError:
517
raise Exception(
518
'Tag %s is not available. Select one of:\n%s' % (
519
options.tag,
520
', '.join(sorted(phatch.by_tag.keys() + ['library', 'save']))
521
)
221
522
)
222
print 'Done'
223
if not options.actionlists:
224
total = len(actionlists)
225
for i, name in enumerate(actionlists):
226
sys.stdout.write(' Running %s/%s\r' % (i, total))
227
sys.stdout.flush()
228
execute(options.input, name)
229
except KeyboardInterrupt:
230
print '\nStopped'
231
print 'Execution took %.2f seconds' % (time.time() - start)
232
233
523
print 'Testing the following actions:'
524
print ', '.join(actions.keys())
525
actionlists = phatch.generate_actionlists(actions=actions)
526
else:
527
actionlists = phatch.generate_library_actionlists()
528
actionlists.update(phatch.generate_actionlists())
529
actionlists.update(phatch.generate_custom_actionlists(
530
action1=phatch.actions['convert_mode'],
531
action2=phatch.actions['save'],
532
))
533
if not options.actionlists:
534
phatch.execute_actionlists(actionlists)
Loggerhead is a web-based interface for Breezy
Version: 2.0.1