1
import shutil, time, os, sys, json, tempfile, copy, shlex, atexit, subprocess, hashlib, cPickle, re
1
import shutil, time, os, sys, json, tempfile, copy, shlex, atexit, subprocess, hashlib, cPickle, re, errno
2
2
from subprocess import Popen, PIPE, STDOUT
3
3
from tempfile import mkstemp
4
4
from distutils.spawn import find_executable
5
5
import jsrun, cache, tempfiles
6
from response_file import create_response_file
7
7
import logging, platform
41
41
# emscripten.py supports reading args from a response file instead of cmdline.
42
42
# Use .rsp to avoid cmdline length limitations on Windows.
43
43
if len(args) >= 2 and args[1].endswith("emscripten.py"):
44
self.response_filename = create_response_file(args[2:], TEMP_DIR)
45
args = args[0:2] + ['@' + self.response_filename]
44
response_filename = response_file.create_response_file(args[2:], TEMP_DIR)
45
args = args[0:2] + ['@' + response_filename]
48
48
# Call the process with fixed streams.
49
49
self.process = subprocess.Popen(args, bufsize, executable, self.stdin_, self.stdout_, self.stderr_, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags)
50
self.pid = self.process.pid
50
51
except Exception, e:
51
52
logging.error('\nsubprocess.Popen(args=%s) failed! Exception %s\n' % (' '.join(args), str(e)))
78
79
return self.process.kill()
82
# Clean up the temporary response file that was used to spawn this process, so that we don't leave temp files around.
83
tempfiles.try_delete(self.response_filename)
85
pass # Mute all exceptions in dtor, particularly if we didn't use a response file, self.response_filename doesn't exist.
87
# Install our replacement Popen handler if we are running on Windows to avoid python spawn process function.
91
81
__rootpath__ = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
92
82
def path_from_root(*pathelems):
93
83
return os.path.join(__rootpath__, *pathelems)
187
177
logging.StreamHandler.emit = add_coloring_to_emit_ansi(logging.StreamHandler.emit)
189
# Emscripten configuration is done through the EM_CONFIG environment variable.
190
# If the string value contained in this environment variable contains newline
191
# separated definitions, then these definitions will be used to configure
179
# Emscripten configuration is done through the --em-config command line option or
180
# the EM_CONFIG environment variable. If the specified string value contains newline
181
# or semicolon-separated definitions, then these definitions will be used to configure
192
182
# Emscripten. Otherwise, the string is understood to be a path to a settings
193
183
# file that contains the required definitions.
195
EM_CONFIG = os.environ.get('EM_CONFIG')
186
EM_CONFIG = sys.argv[sys.argv.index('--em-config')+1]
187
# Emscripten compiler spawns other processes, which can reimport shared.py, so make sure that
188
# those child processes get the same configuration file by setting it to the currently active environment.
189
os.environ['EM_CONFIG'] = EM_CONFIG
191
EM_CONFIG = os.environ.get('EM_CONFIG')
193
if EM_CONFIG and not os.path.isfile(EM_CONFIG):
194
if EM_CONFIG.startswith('-'):
195
raise Exception('Passed --em-config without an argument. Usage: --em-config /path/to/.emscripten or --em-config EMSCRIPTEN_ROOT=/path/;LLVM_ROOT=/path;...')
196
if not '=' in EM_CONFIG:
197
raise Exception('File ' + EM_CONFIG + ' passed to --em-config does not exist!')
199
EM_CONFIG = EM_CONFIG.replace(';', '\n') + '\n'
196
201
if not EM_CONFIG:
197
202
EM_CONFIG = '~/.emscripten'
198
203
if '\n' in EM_CONFIG:
250
255
logging.error('Error in evaluating %s (at %s): %s, text: %s' % (EM_CONFIG, CONFIG_FILE, str(e), config_text))
261
EM_POPEN_WORKAROUND = os.environ.get('EM_POPEN_WORKAROUND')
263
# Install our replacement Popen handler if we are running on Windows to avoid python spawn process function.
264
# nb. This is by default disabled since it has the adverse effect of buffering up all logging messages, which makes
265
# builds look unresponsive (messages are printed only after the whole build finishes). Whether this workaround is needed
266
# seems to depend on how the host application that invokes emcc has set up its stdout and stderr.
267
if EM_POPEN_WORKAROUND and os.name == 'nt':
268
logging.debug('Installing Popen workaround handler to avoid bug http://bugs.python.org/issue3905')
255
273
EXPECTED_LLVM_VERSION = (3,2)
275
actual_clang_version = None
277
def get_clang_version():
278
global actual_clang_version
279
if actual_clang_version is None:
280
actual_clang_version = Popen([CLANG, '-v'], stderr=PIPE).communicate()[1].split('\n')[0].split(' ')[2]
281
return actual_clang_version
257
283
def check_clang_version():
258
expected = 'clang version ' + '.'.join(map(str, EXPECTED_LLVM_VERSION))
259
actual = Popen([CLANG, '-v'], stderr=PIPE).communicate()[1].split('\n')[0]
284
expected = '.'.join(map(str, EXPECTED_LLVM_VERSION))
285
actual = get_clang_version()
260
286
if expected in actual:
262
288
logging.warning('LLVM version appears incorrect (seeing "%s", expected "%s")' % (actual, expected))
268
294
except Exception, e:
269
295
logging.warning('Could not verify LLVM version: %s' % str(e))
297
def check_fastcomp():
299
llc_version_info = Popen([LLVM_COMPILER, '--version'], stdout=PIPE).communicate()[0]
300
pre, targets = llc_version_info.split('Registered Targets:')
301
if 'js' not in targets or 'JavaScript (asm.js, emscripten) backend' not in targets:
302
logging.critical('fastcomp in use, but LLVM has not been built with the JavaScript backend as a target, llc reports:')
303
print >> sys.stderr, '==========================================================================='
304
print >> sys.stderr, llc_version_info,
305
print >> sys.stderr, '==========================================================================='
309
logging.warning('cound not check fastcomp: %s' % str(e))
271
312
EXPECTED_NODE_VERSION = (0,8,0)
273
314
def check_node_version():
275
316
node = listify(NODE_JS)
276
317
actual = Popen(node + ['--version'], stdout=PIPE).communicate()[0].strip()
277
version = tuple(map(int, actual.replace('v', '').split('.')))
318
version = tuple(map(int, actual.replace('v', '').replace('-pre', '').split('.')))
278
319
if version >= EXPECTED_NODE_VERSION:
280
321
logging.warning('node version appears too old (seeing "%s", expected "%s")' % (actual, 'v' + ('.'.join(map(str, EXPECTED_NODE_VERSION)))))
304
345
# we re-check sanity when the settings are changed)
305
346
# We also re-check sanity and clear the cache when the version changes
307
EMSCRIPTEN_VERSION = '1.5.6'
348
EMSCRIPTEN_VERSION = '1.9.0'
309
350
def generate_sanity():
310
return EMSCRIPTEN_VERSION + '|' + get_llvm_target() + '|' + LLVM_ROOT
351
return EMSCRIPTEN_VERSION + '|' + get_llvm_target() + '|' + LLVM_ROOT + '|' + get_clang_version()
312
353
def check_sanity(force=False):
315
356
if not CONFIG_FILE:
316
if not force: return # config stored directly in EM_CONFIG => skip sanity checks
357
return # config stored directly in EM_CONFIG => skip sanity checks
318
359
settings_mtime = os.stat(CONFIG_FILE).st_mtime
319
360
sanity_file = CONFIG_FILE + '_sanity'
336
377
force = False # the check actually failed, so definitely write out the sanity file, to avoid others later seeing failures too
338
# some warning, not fatal checks - do them even if EM_IGNORE_SANITY is on
379
# some warning, mostly not fatal checks - do them even if EM_IGNORE_SANITY is on
339
380
check_llvm_version()
340
381
check_node_version()
382
if os.environ.get('EMCC_FAST_COMPILER') == '1':
383
fastcomp_ok = check_fastcomp()
342
385
if os.environ.get('EM_IGNORE_SANITY'):
343
386
logging.info('EM_IGNORE_SANITY set, ignoring sanity checks')
354
397
logging.critical('Node.js (%s) does not seem to work, check the paths in %s' % (NODE_JS, EM_CONFIG))
357
for cmd in [CLANG, LINK_CMD[0], LLVM_AR, LLVM_OPT, LLVM_AS, LLVM_DIS, LLVM_NM]:
400
for cmd in [CLANG, LINK_CMD[0], LLVM_AR, LLVM_OPT, LLVM_AS, LLVM_DIS, LLVM_NM, LLVM_INTERPRETER]:
358
401
if not os.path.exists(cmd) and not os.path.exists(cmd + '.exe'): # .exe extension required for Windows
359
402
logging.critical('Cannot find %s, check the paths in %s' % (cmd, EM_CONFIG))
405
if os.environ.get('EMCC_FAST_COMPILER') == '1':
407
logging.critical('failing sanity checks due to previous fastcomp failure')
363
411
subprocess.call([JAVA, '-version'], stdout=PIPE, stderr=PIPE)
454
502
# Temp dir. Create a random one, unless EMCC_DEBUG is set, in which case use TEMP_DIR/emscripten_temp
504
def safe_ensure_dirs(dirname):
508
# Ignore error for already existing dirname
509
if e.errno != errno.EEXIST:
511
# FIXME: Notice that this will result in a false positive,
512
# should the dirname be a file! There seems to no way to
513
# handle this atomically in Python 2.x.
514
# There is an additional option for Python 3.x, though.
456
516
class Configuration:
457
517
def __init__(self, environ=os.environ):
458
518
self.DEBUG = environ.get('EMCC_DEBUG')
479
539
self.EMSCRIPTEN_TEMP_DIR = self.CANONICAL_TEMP_DIR
480
if not os.path.exists(self.EMSCRIPTEN_TEMP_DIR):
481
os.makedirs(self.EMSCRIPTEN_TEMP_DIR)
540
safe_ensure_dirs(self.EMSCRIPTEN_TEMP_DIR)
482
541
except Exception, e:
483
logging.debug(e + 'Could not create canonical temp dir. Check definition of TEMP_DIR in ~/.emscripten')
542
logging.error(str(e) + 'Could not create canonical temp dir. Check definition of TEMP_DIR in ~/.emscripten')
485
544
def get_temp_files(self):
486
545
return tempfiles.TempFiles(
917
def build_library(name, build_dir, output_dir, generated_libs, configure=['sh', './configure'], configure_args=[], make=['make'], make_args=['-j', '2'], cache=None, cache_name=None, copy_project=False, env_init={}, source_dir=None, native=False):
974
def build_library(name, build_dir, output_dir, generated_libs, configure=['sh', './configure'], configure_args=[], make=['make'], make_args='help', cache=None, cache_name=None, copy_project=False, env_init={}, source_dir=None, native=False):
918
975
''' Build a library into a .bc file. We build the .bc file once and cache it for all our tests. (We cache in
919
976
memory since the test directory is destroyed and recreated for each test. Note that we cache separately
920
977
for different compilers).
1017
1076
temp_dir = os.path.join(EMSCRIPTEN_TEMP_DIR, 'ar_output_' + str(os.getpid()) + '_' + str(len(temp_dirs)))
1018
1077
temp_dirs.append(temp_dir)
1019
if not os.path.exists(temp_dir):
1020
os.makedirs(temp_dir)
1078
safe_ensure_dirs(temp_dir)
1021
1079
os.chdir(temp_dir)
1022
1080
contents = filter(lambda x: len(x) > 0, Popen([LLVM_AR, 't', f], stdout=PIPE).communicate()[0].split('\n'))
1023
1081
#print >> sys.stderr, ' considering archive', f, ':', contents
1027
1085
for content in contents: # ar will silently fail if the directory for the file does not exist, so make all the necessary directories
1028
1086
dirname = os.path.dirname(content)
1029
if dirname and not os.path.exists(dirname):
1030
os.makedirs(dirname)
1031
Popen([LLVM_AR, 'x', f], stdout=PIPE).communicate() # if absolute paths, files will appear there. otherwise, in this directory
1088
safe_ensure_dirs(dirname)
1089
Popen([LLVM_AR, 'xo', f], stdout=PIPE).communicate() # if absolute paths, files will appear there. otherwise, in this directory
1032
1090
contents = map(lambda content: os.path.join(temp_dir, content), contents)
1033
1091
contents = filter(os.path.exists, map(os.path.abspath, contents))
1034
1092
added_contents = set()
1109
1167
# @param opt Either an integer, in which case it is the optimization level (-O1, -O2, etc.), or a list of raw
1110
1168
# optimization passes passed to llvm opt
1112
def llvm_opt(filename, opts):
1170
def llvm_opt(filename, opts, out=None):
1113
1171
if type(opts) is int:
1114
1172
opts = Building.pick_llvm_opts(opts)
1115
1173
#opts += ['-debug-pass=Arguments']
1174
if get_clang_version() == '3.4' and not Settings.SIMD:
1175
opts += ['-disable-loop-vectorization', '-disable-slp-vectorization'] # llvm 3.4 has these on by default
1116
1176
logging.debug('emcc: LLVM opts: ' + str(opts))
1117
output = Popen([LLVM_OPT, filename] + opts + ['-o', filename + '.opt.bc'], stdout=PIPE).communicate()[0]
1118
assert os.path.exists(filename + '.opt.bc'), 'Failed to run llvm optimizations: ' + output
1119
shutil.move(filename + '.opt.bc', filename)
1177
target = out or (filename + '.opt.bc')
1178
output = Popen([LLVM_OPT, filename] + opts + ['-o', target], stdout=PIPE).communicate()[0]
1179
assert os.path.exists(target), 'Failed to run llvm optimizations: ' + output
1181
shutil.move(filename + '.opt.bc', filename)
1122
1184
def llvm_opts(filename): # deprecated version, only for test runner. TODO: remove
1206
1268
# Run Emscripten
1207
1269
Settings.RELOOPER = Cache.get_path('relooper.js')
1208
1270
settings = Settings.serialize()
1209
compiler_output = jsrun.timeout_run(Popen([PYTHON, EMSCRIPTEN, filename + ('.o.ll' if append_ext else ''), '-o', filename + '.o.js'] + settings + extra_args, stdout=PIPE), None, 'Compiling')
1271
args = settings + extra_args
1273
args = ['@' + response_file.create_response_file(args, TEMP_DIR)]
1274
cmdline = [PYTHON, EMSCRIPTEN, filename + ('.o.ll' if append_ext else ''), '-o', filename + '.o.js'] + args
1275
if jsrun.TRACK_PROCESS_SPAWNS:
1276
logging.info('Executing emscripten.py compiler with cmdline "' + ' '.join(cmdline) + '"')
1277
compiler_output = jsrun.timeout_run(Popen(cmdline, stdout=PIPE), None, 'Compiling')
1210
1278
#print compiler_output
1212
1280
# Detect compilation crashes and errors
1353
1423
'-jar', CLOSURE_COMPILER,
1354
1424
'--compilation_level', 'ADVANCED_OPTIMIZATIONS',
1355
1425
'--language_in', 'ECMASCRIPT5',
1426
'--externs', CLOSURE_EXTERNS,
1356
1427
#'--variable_map_output_file', filename + '.vars',
1357
1428
'--js', filename, '--js_output_file', filename + '.cc.js']
1358
1429
if pretty: args += ['--formatting', 'PRETTY_PRINT']
1413
1488
emcc_debug = os.environ.get('EMCC_DEBUG')
1414
1489
if emcc_debug: del os.environ['EMCC_DEBUG']
1416
def make(opt_level):
1491
emcc_leave_inputs_raw = os.environ.get('EMCC_LEAVE_INPUTS_RAW')
1492
if emcc_leave_inputs_raw: del os.environ['EMCC_LEAVE_INPUTS_RAW']
1494
def make(opt_level, reloop):
1417
1495
raw = relooper + '.raw.js'
1418
1496
Building.emcc(os.path.join('relooper', 'Relooper.cpp'), ['-I' + os.path.join('relooper'), '--post-js',
1419
1497
os.path.join('relooper', 'emscripten', 'glue.js'),
1420
'--memory-init-file', '0',
1421
'-s', 'TOTAL_MEMORY=67108864',
1498
'--memory-init-file', '0', '-s', 'RELOOP=%d' % reloop,
1422
1499
'-s', 'EXPORTED_FUNCTIONS=["_rl_set_output_buffer","_rl_make_output_buffer","_rl_new_block","_rl_delete_block","_rl_block_add_branch_to","_rl_new_relooper","_rl_delete_relooper","_rl_relooper_add_block","_rl_relooper_calculate","_rl_relooper_render", "_rl_set_asm_js_mode"]',
1423
1500
'-s', 'DEFAULT_LIBRARY_FUNCS_TO_INCLUDE=["memcpy", "memset", "malloc", "free", "puts"]',
1424
1501
'-s', 'RELOOPER="' + relooper + '"',
1425
1502
'-O' + str(opt_level), '--closure', '0'], raw)
1426
1503
f = open(relooper, 'w')
1427
1504
f.write("// Relooper, (C) 2012 Alon Zakai, MIT license, https://github.com/kripken/Relooper\n")
1428
f.write("var Relooper = (function() {\n")
1505
f.write("var Relooper = (function(Module) {\n")
1429
1506
f.write(open(raw).read())
1430
1507
f.write('\n return Module.Relooper;\n')
1508
f.write('})(RelooperModule);\n')
1434
1511
# bootstrap phase 1: generate unrelooped relooper, for which we do not need a relooper (so we cannot recurse infinitely in this function)
1435
1512
logging.info(' bootstrap phase 1')
1437
1514
# bootstrap phase 2: generate relooped relooper, using the unrelooped relooper (we see relooper.js exists so we cannot recurse infinitely in this function)
1438
1515
logging.info(' bootstrap phase 2')
1440
1517
logging.info('bootstrapping relooper succeeded')
1441
1518
logging.info('=======================================')
1445
1522
if emcc_debug: os.environ['EMCC_DEBUG'] = emcc_debug
1523
if emcc_leave_inputs_raw: os.environ['EMCC_LEAVE_INPUTS_RAW'] = emcc_leave_inputs_raw
1447
1525
logging.error('bootstrapping relooper failed. You may need to manually create relooper.js by compiling it, see src/relooper/emscripten')
1526
try_delete(relooper) # do not leave a phase-1 version if phase 2 broke
1530
def ensure_struct_info(info_path):
1531
if os.path.exists(info_path): return
1534
import gen_struct_info
1535
gen_struct_info.main(['-qo', info_path, path_from_root('src/struct_info.json')])
1451
1538
def preprocess(infile, outfile):
1461
1548
text = m.groups(0)[0]
1462
1549
assert text.count('(') == 1 and text.count(')') == 1, 'must have simple expressions in emscripten_jcache_printf calls, no parens'
1463
1550
assert text.count('"') == 2, 'must have simple expressions in emscripten_jcache_printf calls, no strings as varargs parameters'
1551
if os.environ.get('EMCC_FAST_COMPILER') == '1': # fake it in fastcomp
1552
return text.replace('emscripten_jcache_printf', 'printf')
1464
1553
start = text.index('(')
1465
1554
end = text.rindex(')')
1466
1555
args = text[start+1:end].split(',')
1490
1579
def to_nice_ident(ident): # limited version of the JS function toNiceIdent
1491
return ident.replace('%', '$').replace('@', '_')
1580
return ident.replace('%', '$').replace('@', '_').replace('.', '_')
1583
def make_initializer(sig, settings=None):
1584
settings = settings or Settings
1587
elif sig == 'f' and settings.get('PRECISE_F32'):
1588
return 'Math_fround(0)'
1593
def make_coercion(value, sig, settings=None):
1594
settings = settings or Settings
1597
elif sig == 'f' and settings.get('PRECISE_F32'):
1598
return 'Math_fround(' + value + ')'
1599
elif sig == 'd' or sig == 'f':
1494
1605
def make_extcall(sig, named=True):