Skip to content

Transformers.js line items shouldn't load resources in the benchmark #238

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kmiller68 merged 2 commits into from
Nov 13, 2025
Merged

Transformers.js line items shouldn't load resources in the benchmark #238

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
19b5b04
Transformers.js line items shouldn't load resources in the benchmark
kmiller68 Nov 12, 2025
1f6ad81
Always initialize preloads to an empty array.
kmiller68 Nov 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 28 additions & 24 deletions JetStreamDriver.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -560,20 +560,35 @@ const BenchmarkState = Object.freeze({


class Scripts {
constructor() {
constructor(preloads) {
this.scripts = [];

let preloadsCode = "";
let resourcesCode = "";
for (let { name, resource, blobURLOrPath } of preloads) {
console.assert(name?.length > 0, "Invalid preload name.");
console.assert(resource?.length > 0, "Invalid preload resource.");
console.assert(blobURLOrPath?.length > 0, "Invalid preload data.");
preloadsCode += `${JSON.stringify(name)}: "${blobURLOrPath}",\n`;
resourcesCode += `${JSON.stringify(resource)}: "${blobURLOrPath}",\n`;
}
// Expose a globalThis.JetStream object to the workload. We use
// a proxy to prevent prototype access and throw on unknown properties.
this.add(`
const throwOnAccess = (name) => new Proxy({}, {
get(target, property, receiver) {
throw new Error(name + "." + property + " is not defined.");
}
});
});
globalThis.JetStream = {
__proto__: throwOnAccess("JetStream"),
preload: {
__proto__: throwOnAccess("JetStream.preload"),
${preloadsCode}
},
resources: {
__proto__: throwOnAccess("JetStream.preload"),
${resourcesCode}
},
};
`);
Expand Down Expand Up @@ -631,8 +646,8 @@ class Scripts {
}

class ShellScripts extends Scripts {
constructor() {
super();
constructor(preloads) {
super(preloads);
this.prefetchedResources = Object.create(null);;
}

Expand Down Expand Up @@ -698,8 +713,8 @@ class ShellScripts extends Scripts {
}

class BrowserScripts extends Scripts {
constructor() {
super();
constructor(preloads) {
super(preloads);
this.add("window.onerror = top.currentReject;");
}

Expand Down Expand Up @@ -738,7 +753,7 @@ class Benchmark {
this.isAsync = !!plan.isAsync;
this.allowUtf16 = !!plan.allowUtf16;
this.scripts = null;
this.preloads = null;
this.preloads = [];
this.shellPrefetchedResources = null;
this.results = [];
this._state = BenchmarkState.READY;
Expand Down Expand Up @@ -898,7 +913,7 @@ class Benchmark {
if (this.isDone)
throw new Error(`Cannot run Benchmark ${this.name} twice`);
this._state = BenchmarkState.PREPARE;
const scripts = isInBrowser ? new BrowserScripts() : new ShellScripts();
const scripts = isInBrowser ? new BrowserScripts(this.preloads) : new ShellScripts(this.preloads);

if (!!this.plan.deterministicRandom)
scripts.addDeterministicRandom()
Expand All @@ -908,15 +923,6 @@ class Benchmark {
if (this.shellPrefetchedResources) {
scripts.addPrefetchedResources(this.shellPrefetchedResources);
}
if (this.plan.preload) {
let preloadCode = "";
for (let [ variableName, blobURLOrPath ] of this.preloads) {
console.assert(variableName?.length > 0, "Invalid preload name.");
console.assert(blobURLOrPath?.length > 0, "Invalid preload data.");
preloadCode += `JetStream.preload[${JSON.stringify(variableName)}] = "${blobURLOrPath}";\n`;
}
scripts.add(preloadCode);
}

const prerunCode = this.prerunCode;
if (prerunCode)
Expand Down Expand Up @@ -1067,12 +1073,11 @@ class Benchmark {
}));

if (this.plan.preload) {
this.preloads = [];
for (const [name, resource] of Object.entries(this.plan.preload)) {
promises.push(this.loadBlob("preload", name, resource).then((blobData) => {
if (!globalThis.allIsGood)
return;
this.preloads.push([ blobData.prop, blobData.blobURL ]);
this.preloads.push({ name: blobData.prop, resource: blobData.resource, blobURLOrPath: blobData.blobURL });
this.updateCounter();
}).catch((error) => {
// We'll try again later in retryPrefetchResourceForBrowser(). Don't throw an error.
Expand All @@ -1099,7 +1104,7 @@ class Benchmark {
if (type == "preload") {
if (this.failedPreloads && this.failedPreloads[blobData.prop]) {
this.failedPreloads[blobData.prop] = false;
this.preloads.push([ blobData.prop, blobData.blobURL ]);
this.preloads.push({ name: blobData.prop, resource: blobData.resource, blobURLOrPath: blobData.blobURL });
counter.failedPreloadResources--;
}
}
Expand All @@ -1112,7 +1117,7 @@ class Benchmark {
if (!globalThis.allIsGood)
return;
if (blobData.type == "preload")
this.preloads.push([ blobData.prop, blobData.blobURL ]);
this.preloads.push({ name: blobData.prop, resource: blobData.resource, blobURLOrPath: blobData.blobURL });
this.updateCounter();
});

Expand Down Expand Up @@ -1150,8 +1155,7 @@ class Benchmark {
console.assert(this.scripts === null, "This initialization should be called only once.");
this.scripts = this.plan.files.map(file => shellFileLoader.load(file));

console.assert(this.preloads === null, "This initialization should be called only once.");
this.preloads = [];
console.assert(this.preloads.length === 0, "This initialization should be called only once.");
this.shellPrefetchedResources = Object.create(null);
if (!this.plan.preload) {
return;
Expand All @@ -1170,7 +1174,7 @@ class Benchmark {
this.shellPrefetchedResources[resource] = bytes;
}

this.preloads.push([name, resource]);
this.preloads.push({ name, resource, blobURLOrPath: resource });
}
}

Expand Down
15 changes: 9 additions & 6 deletions transformersjs/benchmark.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,14 @@ globalThis.URL = URL;

// Polyfill fetch for shell-compatibility and to cache / preload model weights etc.
let preload = { /* Initialized in init() below due to async. */ };
const originalFetch = globalThis.fetch ?? function(url) {
throw new Error("no fetch available");
}
globalThis.fetch = async function(url) {

async function redirectingFetch(url) {
// DEBUG
// console.log('fetch', url);

if (url.startsWith("./"))
url = JetStream.resources[url];

// Redirect some paths to cached/preloaded resources.
if (preload[url]) {
return {
Expand All @@ -38,8 +39,7 @@ globalThis.fetch = async function(url) {
};
}

// This should only be called in the browser, where fetch() is available.
return originalFetch(url);
throw new Error(`Unexpected resource requested in benchmark: ${url}`);
};

// JetStream benchmark harness. Reuse for two different Transformers.js tasks.
Expand All @@ -66,6 +66,9 @@ class Benchmark {
// DEBUG
// console.log('inputFile', this.inputFile.byteLength, 'bytes');
}

// After we have loaded everything close the door behind us to make sure no other network requests happen.
globalThis.fetch = redirectingFetch;
}

async runIteration() {
Expand Down
Loading