14
file_packager.py TARGET [--preload A [B..]] [--embed C [D..]] [--compress COMPRESSION_DATA] [--pre-run] [--crunch[=X]] [--js-output=OUTPUT.js] [--no-force]
16
--pre-run Will generate wrapper code that does preloading in Module.preRun. This is necessary if you add this
17
code before the main file has been loading, which includes necessary components like addRunDependency.
14
file_packager.py TARGET [--preload A [B..]] [--embed C [D..]] [--exclude E [F..]] [--compress COMPRESSION_DATA] [--crunch[=X]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--no-heap-copy]
19
16
--crunch=X Will compress dxt files to crn with quality level X. The crunch commandline tool must be present
20
17
and CRUNCH should be defined in ~/.emscripten that points to it. JS crunch decompressing code will
31
28
--use-preload-cache Stores package in IndexedDB so that subsequent loads don't need to do XHR. Checks package version.
30
--no-heap-copy If specified, the preloaded filesystem is not copied inside the Emscripten HEAP, but kept in a separate typed array outside it.
31
The default, if this is not specified, is to embed the VFS inside the HEAP, so that mmap()ing files in it is a no-op.
32
Passing this flag optimizes for fread() usage, omitting it optimizes for mmap() usage.
35
36
* The file packager generates unix-style file paths. So if you are on windows and a file is accessed at
45
46
from shared import Compression, execute, suffix, unsuffixed
46
47
from subprocess import Popen, PIPE, STDOUT
48
50
if len(sys.argv) == 1:
49
print '''Usage: file_packager.py TARGET [--preload A...] [--embed B...] [--compress COMPRESSION_DATA] [--pre-run] [--crunch[=X]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache]
51
print '''Usage: file_packager.py TARGET [--preload A...] [--embed B...] [--exclude C...] [--compress COMPRESSION_DATA] [--crunch[=X]] [--js-output=OUTPUT.js] [--no-force] [--use-preload-cache] [--no-heap-copy]
50
52
See the source for more details.'''
65
67
AV_WORKAROUND = 0 # Set to 1 to randomize file order and add some padding, to work around silly av false positives
70
excluded_patterns = []
70
72
has_preloaded = False
78
# If set to True, IndexedDB (IDBFS in library_idbfs.js) is used to locally cache VFS XHR so that subsequent
79
# page loads can read the data from the offline cache instead.
77
80
use_preload_cache = False
81
# If set to True, the blob received from XHR is moved to the Emscripten HEAP, optimizing for mmap() performance.
82
# If set to False, the XHR blob is kept intact, and fread()s etc. are performed directly to that data. This optimizes for minimal memory usage and fread() performance.
79
for arg in sys.argv[1:]:
85
for arg in sys.argv[2:]:
80
86
if arg == '--preload':
83
87
has_preloaded = True
85
89
elif arg == '--embed':
91
elif arg == '--exclude':
89
93
elif arg == '--compress':
90
95
Compression.on = True
94
elif arg == '--pre-run':
99
97
elif arg == '--no-force':
101
100
elif arg == '--use-preload-cache':
102
101
use_preload_cache = True
103
elif arg == '--no-heap-copy':
103
106
elif arg.startswith('--js-output'):
104
107
jsoutput = arg.split('=')[1] if '=' in arg else None
105
109
elif arg.startswith('--crunch'):
106
110
from shared import CRUNCH
107
111
crunch = arg.split('=')[1] if '=' in arg else '128'
111
113
elif arg.startswith('--plugin'):
112
114
plugin = open(arg.split('=')[1], 'r').read()
113
115
eval(plugin) # should append itself to plugins
117
elif in_preload or in_embed:
117
elif leading == 'preload' or leading == 'embed':
122
120
srcpath, dstpath = arg.split('@') # User is specifying destination filename explicitly.
126
124
data_files.append({ 'srcpath': srcpath, 'dstpath': dstpath, 'mode': mode })
128
126
print >> sys.stderr, 'Warning: ' + arg + ' does not exist, ignoring.'
127
elif leading == 'exclude':
128
excluded_patterns.append(arg)
129
elif leading == 'compress':
130
if compress_cnt == 1:
131
131
Compression.encoder = arg
133
elif in_compress == 2:
133
elif compress_cnt == 2:
134
134
Compression.decoder = arg
136
elif in_compress == 3:
136
elif compress_cnt == 3:
137
137
Compression.js_name = arg
140
print >> sys.stderr, 'Unknown parameter:', arg
140
143
if (not force) and len(data_files) == 0:
141
144
has_preloaded = False
148
if (typeof Module === 'undefined') Module = eval('(function() { try { return Module || {} } catch(e) { return {} } })()');
149
if (!Module.expectedDataFileDownloads) {
150
Module.expectedDataFileDownloads = 0;
151
Module.finishedDataFileDownloads = 0;
153
Module.expectedDataFileDownloads++;
166
# The packager should never preload/embed any directories that have a component starting with '.' in them,
167
# or if the file is hidden (Win32). Note that this filter ONLY applies to directories. Explicitly specified single files
168
# are always preloaded/embedded, even if they start with a '.'.
169
def should_ignore(filename):
170
if has_hidden_attribute(filename):
176
# The packager should never preload/embed files if the file is hidden (Win32).
177
# or it matches any pattern specified in --exclude
178
def should_ignore(fullname):
179
if has_hidden_attribute(fullname):
173
components = filename.replace('\\\\', '/').replace('\\', '/').split('/')
175
if c.startswith('.') and c != '.' and c != '..':
182
for p in excluded_patterns:
183
if fnmatch.fnmatch(fullname, p):
181
189
# rootpathsrc: The path name of the root directory on the local FS we are adding to emscripten virtual FS.
182
190
# rootpathdst: The name we want to make the source path available on the emscripten virtual FS.
183
191
mode, rootpathsrc, rootpathdst = arg
184
193
for name in names:
185
194
fullname = os.path.join(dirname, name)
186
if not os.path.isdir(fullname):
187
if should_ignore(fullname):
189
print >> sys.stderr, 'Skipping hidden file "' + fullname + '" from inclusion in the emscripten virtual file system.'
195
if should_ignore(fullname):
197
print >> sys.stderr, 'Skipping file "' + fullname + '" from inclusion in the emscripten virtual file system.'
199
new_names.append(name)
200
if not os.path.isdir(fullname):
191
201
dstpath = os.path.join(rootpathdst, os.path.relpath(fullname, rootpathsrc)) # Convert source filename relative to root directory of target FS.
192
data_files.append({ 'srcpath': fullname, 'dstpath': dstpath, 'mode': mode })
202
new_data_files.append({ 'srcpath': fullname, 'dstpath': dstpath, 'mode': mode })
204
names.extend(new_names)
194
207
for file_ in data_files:
195
if os.path.isdir(file_['srcpath']):
196
os.path.walk(file_['srcpath'], add, [file_['mode'], file_['srcpath'], file_['dstpath']])
197
data_files = filter(lambda file_: not os.path.isdir(file_['srcpath']), data_files)
208
if not should_ignore(file_['srcpath']):
209
if os.path.isdir(file_['srcpath']):
210
os.path.walk(file_['srcpath'], add, [file_['mode'], file_['srcpath'], file_['dstpath']])
212
new_data_files.append(file_)
213
data_files = filter(lambda file_: not os.path.isdir(file_['srcpath']), new_data_files)
214
if len(data_files) == 0:
215
print >> sys.stderr, 'Nothing to do!'
199
218
# Absolutize paths, and check that they make sense
200
219
curr_abspath = os.path.abspath(os.getcwd())
345
364
send: function() {},
346
365
onload: function() {
347
366
var byteArray = this.byteArray.subarray(this.start, this.end);
349
var ddsHeader = byteArray.subarray(0, 128);
351
requestDecrunch(this.name, byteArray.subarray(128), function(ddsData) {
352
byteArray = new Uint8Array(ddsHeader.length + ddsData.length);
353
byteArray.set(ddsHeader, 0);
354
byteArray.set(ddsData, 128);
355
that.finish(byteArray);
358
368
this.finish(byteArray);
361
371
finish: function(byteArray) {
366
376
if (that.audio) {
367
377
Module['removeRunDependency']('fp ' + that.name); // workaround for chromium bug 124926 (still no audio with this, but at least we don't hang)
369
Runtime.warn('Preloading file ' + that.name + ' failed');
379
Module.printErr('Preloading file ' + that.name + ' failed');
371
381
}, false, true); // canOwn this data in the filesystem, it is a slide into the heap that will never change
372
382
this.requests[this.name] = null;
385
''' % ('' if not crunch else '''
387
var ddsHeader = byteArray.subarray(0, 128);
389
requestDecrunch(this.name, byteArray.subarray(128), function(ddsData) {
390
byteArray = new Uint8Array(ddsHeader.length + ddsData.length);
391
byteArray.set(ddsHeader, 0);
392
byteArray.set(ddsData, 128);
393
that.finish(byteArray);
396
''', '' if not crunch else '''
378
401
for file_ in data_files:
413
436
if has_preloaded:
414
437
# Get the big archive and split it up
416
440
// copy the entire loaded file into a spot in the heap. Files will refer to slices in that. They cannot be freed though.
417
441
var ptr = Module['_malloc'](byteArray.length);
418
442
Module['HEAPU8'].set(byteArray, ptr);
419
443
DataRequest.prototype.byteArray = Module['HEAPU8'].subarray(ptr, ptr+byteArray.length);
447
// Reuse the bytearray from the XHR as the source for file reads.
448
DataRequest.prototype.byteArray = byteArray;
421
450
for file_ in data_files:
422
451
if file_['mode'] == 'preload':
423
452
use_data += ' DataRequest.prototype.requests["%s"].onload();\n' % (file_['dstpath'])
434
463
package_uuid = uuid.uuid4();
464
remote_package_name = os.path.basename(Compression.compressed_name(data_target) if Compression.on else data_target)
436
if (!Module.expectedDataFileDownloads) {
437
Module.expectedDataFileDownloads = 0;
438
Module.finishedDataFileDownloads = 0;
467
if (typeof window === 'object') {
468
PACKAGE_PATH = window['encodeURIComponent'](window.location.pathname.toString().substring(0, window.location.pathname.toString().lastIndexOf('/')) + '/');
471
PACKAGE_PATH = encodeURIComponent(location.pathname.toString().substring(0, location.pathname.toString().lastIndexOf('/')) + '/');
440
Module.expectedDataFileDownloads++;
442
var PACKAGE_PATH = window['encodeURIComponent'](window.location.pathname.toString().substring(0, window.location.pathname.toString().lastIndexOf('/')) + '/');
443
473
var PACKAGE_NAME = '%s';
444
474
var REMOTE_PACKAGE_NAME = '%s';
445
475
var PACKAGE_UUID = '%s';
446
''' % (data_target, os.path.basename(Compression.compressed_name(data_target) if Compression.on else data_target), package_uuid)
476
''' % (data_target, remote_package_name, package_uuid)
448
478
if use_preload_cache:
540
570
function fetchRemotePackage(packageName, callback, errback) {
541
571
var xhr = new XMLHttpRequest();
542
572
xhr.open('GET', packageName, true);
566
596
total = Math.ceil(total * Module.expectedDataFileDownloads/num);
567
Module['setStatus']('Downloading data... (' + loaded + '/' + total + ')');
597
if (Module['setStatus']) Module['setStatus']('Downloading data... (' + loaded + '/' + total + ')');
568
598
} else if (!Module.dataFileDownloads) {
569
Module['setStatus']('Downloading data...');
599
if (Module['setStatus']) Module['setStatus']('Downloading data...');
572
602
xhr.onload = function(event) {
609
function handleError(error) {
610
console.error('package error:', error);
579
615
function processPackageData(arrayBuffer) {
580
616
Module.finishedDataFileDownloads++;
581
617
assert(arrayBuffer, 'Loading data file failed.');
586
622
Module['addRunDependency']('datafile_%s');
588
function handleError(error) {
589
console.error('package error:', error);
591
623
''' % (use_data, data_target) # use basename because from the browser's point of view, we need to find the datafile in the same dir as the html file
594
if (!Module.preloadResults)
595
Module.preloadResults = {};
626
if (!Module.preloadResults) Module.preloadResults = {};
598
629
if use_preload_cache:
631
662
if (Module['setStatus']) Module['setStatus']('Downloading...');
665
# Not using preload cache, so we might as well start the xhr ASAP, potentially before JS parsing of the main codebase if it's after us.
666
# Only tricky bit is the fetch is async, but also when runWithFS is called is async, so we handle both orderings.
668
var fetched = null, fetchedCallback = null;
669
fetchRemotePackage('%s', function(data) {
670
if (fetchedCallback) {
671
fetchedCallback(data);
672
fetchedCallback = null;
677
''' % os.path.basename(Compression.compressed_name(data_target) if Compression.on else data_target)
635
680
Module.preloadResults[PACKAGE_NAME] = {fromCache: false};
636
fetchRemotePackage(REMOTE_PACKAGE_NAME, processPackageData, handleError);
682
processPackageData(fetched);
685
fetchedCallback = processPackageData;
641
if (typeof Module == 'undefined') Module = {};
642
if (!Module['preRun']) Module['preRun'] = [];
643
Module["preRun"].push(function() {
690
function runWithFS() {
695
if (Module['calledRun']) {
698
if (!Module['preRun']) Module['preRun'] = [];
699
Module["preRun"].push(runWithFS); // FS is not initialized yet, wait for it
added
removed
tools/file_packager.py
