Skip to content

[file_packager] Convert preload caching functions to async. NFC #24885

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Aug 8, 2025
Merged

[file_packager] Convert preload caching functions to async. NFC #24885

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9b0f421
[test] Improve test_preload_caching_indexeddb_name. NFC
sbc100 Aug 8, 2025
02b795e
[file_packager] Convert checkCachedPackage to async/await. NFC
sbc100 Aug 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 11 additions & 9 deletions test/test_browser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -617,7 +617,7 @@ def test_preload_caching(self, extra_size):

def test_preload_caching_indexeddb_name(self):
self.set_setting('EXIT_RUNTIME')
create_file('somefile.txt', '''load me right before running the code please''')
create_file('somefile.txt', 'load me right before running the code please')

def make_main(path):
print(path)
Expand All @@ -640,20 +640,21 @@ def make_main(path):
int result = 0;

assert(strcmp("load me right before", buf) == 0);
return checkPreloadResults();
int num_cached = checkPreloadResults();
printf("got %%d preloadResults from cache\n", num_cached);
return num_cached;
}
''' % path)

create_file('test.js', '''
addToLibrary({
checkPreloadResults: function() {
checkPreloadResults: () => {
var cached = 0;
var packages = Object.keys(Module['preloadResults']);
packages.forEach(function(package) {
var fromCache = Module['preloadResults'][package]['fromCache'];
if (fromCache)
++ cached;
});
for (var result of Object.values(Module['preloadResults'])) {
if (result['fromCache']) {
cached++;
}
}
return cached;
}
});
Expand All @@ -663,6 +664,7 @@ def make_main(path):
self.run_process([FILE_PACKAGER, 'somefile.data', '--use-preload-cache', '--indexedDB-name=testdb', '--preload', 'somefile.txt', '--js-output=' + 'somefile.js'])
self.compile_btest('main.c', ['--js-library', 'test.js', '--pre-js', 'somefile.js', '-o', 'page.html', '-sFORCE_FILESYSTEM'], reporting=Reporting.JS_ONLY)
self.run_browser('page.html', '/report_result?exit:0')
print("Re-running ..")
self.run_browser('page.html', '/report_result?exit:1')

def test_multifile(self):
Expand Down
200 changes: 103 additions & 97 deletions tools/file_packager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -805,6 +805,7 @@ def generate_js(data_target, data_files, metadata):
var DB_VERSION = 1;
var METADATA_STORE_NAME = 'METADATA';
var PACKAGE_STORE_NAME = 'PACKAGES';

async function openDatabase() {
if (typeof indexedDB == 'undefined') {
throw 'using IndexedDB to cache data can only be done on a web page or in a web worker';
Expand Down Expand Up @@ -838,67 +839,68 @@ def generate_js(data_target, data_files, metadata):
// We set the chunk size to 64MB to stay well-below the limit
var CHUNK_SIZE = 64 * 1024 * 1024;

function cacheRemotePackage(
db,
packageName,
packageData,
packageMeta,
callback,
errback
) {
async function cacheRemotePackage(db, packageName, packageData, packageMeta) {
var transactionPackages = db.transaction([PACKAGE_STORE_NAME], IDB_RW);
var packages = transactionPackages.objectStore(PACKAGE_STORE_NAME);
var chunkSliceStart = 0;
var nextChunkSliceStart = 0;
var chunkCount = Math.ceil(packageData.byteLength / CHUNK_SIZE);
var finishedChunks = 0;
for (var chunkId = 0; chunkId < chunkCount; chunkId++) {
nextChunkSliceStart += CHUNK_SIZE;
var putPackageRequest = packages.put(
packageData.slice(chunkSliceStart, nextChunkSliceStart),
`package/${packageName}/${chunkId}`
);
chunkSliceStart = nextChunkSliceStart;
putPackageRequest.onsuccess = (event) => {
finishedChunks++;
if (finishedChunks == chunkCount) {
var transaction_metadata = db.transaction(
[METADATA_STORE_NAME],
IDB_RW
);
var metadata = transaction_metadata.objectStore(METADATA_STORE_NAME);
var putMetadataRequest = metadata.put(
{
'uuid': packageMeta.uuid,
'chunkCount': chunkCount
},
`metadata/${packageName}`
);
putMetadataRequest.onsuccess = (event) => callback(packageData);
putMetadataRequest.onerror = (error) => errback(error);
}
};
putPackageRequest.onerror = (error) => errback(error);
}

return new Promise((resolve, reject) => {
for (var chunkId = 0; chunkId < chunkCount; chunkId++) {
nextChunkSliceStart += CHUNK_SIZE;
var putPackageRequest = packages.put(
packageData.slice(chunkSliceStart, nextChunkSliceStart),
`package/${packageName}/${chunkId}`
);
chunkSliceStart = nextChunkSliceStart;
putPackageRequest.onsuccess = (event) => {
finishedChunks++;
if (finishedChunks == chunkCount) {
var transaction_metadata = db.transaction(
[METADATA_STORE_NAME],
IDB_RW
);
var metadata = transaction_metadata.objectStore(METADATA_STORE_NAME);
var putMetadataRequest = metadata.put(
{
'uuid': packageMeta.uuid,
'chunkCount': chunkCount
},
`metadata/${packageName}`
);
putMetadataRequest.onsuccess = (event) => resolve(packageData);
putMetadataRequest.onerror = reject;
}
};
putPackageRequest.onerror = reject;
}
});
}

/* Check if there's a cached package, and if so whether it's the latest available */
function checkCachedPackage(db, packageName, callback, errback) {
/*
* Check if there's a cached package, and if so whether it's the latest available.
* Resolves to the cached metadata, or `null` if it is missing or out-of-date.
*/
async function checkCachedPackage(db, packageName) {
var transaction = db.transaction([METADATA_STORE_NAME], IDB_RO);
var metadata = transaction.objectStore(METADATA_STORE_NAME);
var getRequest = metadata.get(`metadata/${packageName}`);
getRequest.onsuccess = (event) => {
var result = event.target.result;
if (!result) {
return callback(false, null);
} else {
return callback(PACKAGE_UUID === result['uuid'], result);
return new Promise((resolve, reject) => {
getRequest.onsuccess = (event) => {
var result = event.target.result;
if (result && PACKAGE_UUID === result['uuid']) {
resolve(result);
} else {
resolve(null);
}
}
};
getRequest.onerror = (error) => errback(error);
getRequest.onerror = reject;
});
}

function fetchCachedPackage(db, packageName, metadata, callback, errback) {
async function fetchCachedPackage(db, packageName, metadata) {
var transaction = db.transaction([PACKAGE_STORE_NAME], IDB_RO);
var packages = transaction.objectStore(PACKAGE_STORE_NAME);

Expand All @@ -907,41 +909,43 @@ def generate_js(data_target, data_files, metadata):
var chunkCount = metadata['chunkCount'];
var chunks = new Array(chunkCount);

for (var chunkId = 0; chunkId < chunkCount; chunkId++) {
var getRequest = packages.get(`package/${packageName}/${chunkId}`);
getRequest.onsuccess = (event) => {
if (!event.target.result) {
errback(new Error(`CachedPackageNotFound for: ${packageName}`));
return;
}
// If there's only 1 chunk, there's nothing to concatenate it with so we can just return it now
if (chunkCount == 1) {
callback(event.target.result);
} else {
chunksDone++;
totalSize += event.target.result.byteLength;
chunks.push(event.target.result);
if (chunksDone == chunkCount) {
if (chunksDone == 1) {
callback(event.target.result);
} else {
var tempTyped = new Uint8Array(totalSize);
var byteOffset = 0;
for (var chunkId in chunks) {
var buffer = chunks[chunkId];
tempTyped.set(new Uint8Array(buffer), byteOffset);
byteOffset += buffer.byteLength;
buffer = undefined;
return new Promise((resolve, reject) => {
for (var chunkId = 0; chunkId < chunkCount; chunkId++) {
var getRequest = packages.get(`package/${packageName}/${chunkId}`);
getRequest.onsuccess = (event) => {
if (!event.target.result) {
reject(`CachedPackageNotFound for: ${packageName}`);
return;
}
// If there's only 1 chunk, there's nothing to concatenate it with so we can just return it now
if (chunkCount == 1) {
resolve(event.target.result);
} else {
chunksDone++;
totalSize += event.target.result.byteLength;
chunks.push(event.target.result);
if (chunksDone == chunkCount) {
if (chunksDone == 1) {
resolve(event.target.result);
} else {
var tempTyped = new Uint8Array(totalSize);
var byteOffset = 0;
for (var chunkId in chunks) {
var buffer = chunks[chunkId];
tempTyped.set(new Uint8Array(buffer), byteOffset);
byteOffset += buffer.byteLength;
buffer = undefined;
}
chunks = undefined;
resolve(tempTyped.buffer);
tempTyped = undefined;
}
chunks = undefined;
callback(tempTyped.buffer);
tempTyped = undefined;
}
}
}
};
getRequest.onerror = (error) => errback(error);
}
};
getRequest.onerror = reject;
}
});
}\n'''

# add Node.js support code, if necessary
Expand Down Expand Up @@ -1044,24 +1048,26 @@ def generate_js(data_target, data_files, metadata):
};

openDatabase()
.then((db) => checkCachedPackage(db, PACKAGE_PATH + PACKAGE_NAME,
(useCached, metadata) => {
Module['preloadResults'][PACKAGE_NAME] = {fromCache: useCached};
if (useCached) {
fetchCachedPackage(db, PACKAGE_PATH + PACKAGE_NAME, metadata, processPackageData, preloadFallback);
} else {
fetchRemotePackage(REMOTE_PACKAGE_NAME, REMOTE_PACKAGE_SIZE,
(packageData) => {
cacheRemotePackage(db, PACKAGE_PATH + PACKAGE_NAME, packageData, {uuid:PACKAGE_UUID}, processPackageData,
(error) => {
console.error(error);
processPackageData(packageData);
});
}
, preloadFallback);
}
}, preloadFallback))
.catch(preloadFallback);
.then((db) => {
checkCachedPackage(db, PACKAGE_PATH + PACKAGE_NAME)
.then((cachedData) => {
Module['preloadResults'][PACKAGE_NAME] = {fromCache: !!cachedData};
if (cachedData) {
fetchCachedPackage(db, PACKAGE_PATH + PACKAGE_NAME, cachedData).then(processPackageData);
} else {
fetchRemotePackage(REMOTE_PACKAGE_NAME, REMOTE_PACKAGE_SIZE,
(packageData) => {
cacheRemotePackage(db, PACKAGE_PATH + PACKAGE_NAME, packageData, {uuid:PACKAGE_UUID})
.then(processPackageData)
.catch((error) => {
console.error(error);
processPackageData(packageData);
});
}
, preloadFallback);
}
})
}).catch(preloadFallback);

Module['setStatus']?.('Downloading...');\n'''
else:
Expand Down