parallelize script loads

This commit is contained in:
Brian Beck 2025-12-02 22:06:20 -08:00
parent 9d3554de02
commit 2f23934de0
17 changed files with 561 additions and 68 deletions

View file

@ -601,6 +601,15 @@ export function createBuiltins(
console.debug(
`exec(${JSON.stringify(pathString)}): preparing to execute…`,
);
// Engine requires an extension - check for a '.' in the filename
if (!pathString.includes(".")) {
console.error(
`exec: invalid script file name ${JSON.stringify(pathString)}.`,
);
return false;
}
const normalizedPath = normalizePath(pathString);
const rt = runtime();
const { executedScripts, scripts } = rt.state;

View file

@ -74,7 +74,7 @@ export interface RunServerResult {
export function runServer(options: RunServerOptions): RunServerResult {
const { missionName, missionType, runtimeOptions, onMissionLoadDone } =
options;
const { signal } = runtimeOptions ?? {};
const { signal, fileSystem } = runtimeOptions ?? {};
const runtime = createRuntime({
...runtimeOptions,
@ -87,16 +87,36 @@ export function runServer(options: RunServerOptions): RunServerResult {
const gameTypeName = `${missionType}Game`;
const gameTypeScript = `scripts/${gameTypeName}.cs`;
const ready = (async () => {
const ready = (async function createServer() {
try {
// Load all required scripts
const serverScript = await runtime.loadFromPath("scripts/server.cs");
signal?.throwIfAborted();
// These are dynamic exec() calls in server.cs since their paths are
// computed based on the game type and mission. So, we need to load them
// ahead of time so they're available to execute.
await runtime.loadFromPath(gameTypeScript);
// server.cs has a glob loop that does: findFirstFile("scripts/*Game.cs")
// and then exec()s each result dynamically. Since we can't statically
// analyze dynamic exec paths, we need to either preload all game scripts
// in the same way (so they're available when exec() is called) or just
// exec() the ones we know we need...
//
// To load them all, do:
// if (fileSystem) {
// const gameScripts = fileSystem.findFiles("scripts/*Game.cs");
// await Promise.all(
// gameScripts.map((path) => runtime.loadFromPath(path)),
// );
// signal?.throwIfAborted();
// }
await runtime.loadFromPath("scripts/DefaultGame.cs");
signal?.throwIfAborted();
try {
await runtime.loadFromPath(gameTypeScript);
} catch (err) {
// It's OK if that one fails. Not every game type needs its own script.
}
signal?.throwIfAborted();
// Also preload the mission file (another dynamic exec path)
await runtime.loadFromPath(`missions/${missionName}.mis`);
signal?.throwIfAborted();
@ -112,7 +132,7 @@ export function runServer(options: RunServerOptions): RunServerResult {
// which we added specifically to solve this problem.
if (onMissionLoadDone) {
runtime.$.onMethodCalled(
gameTypeName,
"DefaultGame",
"missionLoadDone",
onMissionLoadDone,
);

View file

@ -1353,7 +1353,9 @@ describe("TorqueScript Runtime", () => {
function createLoader(
files: Record<string, string>,
): (path: string) => Promise<string | null> {
return async (path: string) => files[path.toLowerCase()] ?? null;
// Loader normalizes paths itself (slashes + lowercase)
return async (path: string) =>
files[path.replace(/\\/g, "/").toLowerCase()] ?? null;
}
it("executes scripts via exec()", async () => {
@ -1419,6 +1421,202 @@ describe("TorqueScript Runtime", () => {
expect(runtime.$g.get("Loaded")).toBe(true);
});
describe("path normalization for state tracking", () => {
it("treats different cases as the same file for loaded scripts", async () => {
let loadCount = 0;
const runtime = createRuntime({
loadScript: async (path) => {
loadCount++;
if (path.toLowerCase() === "scripts/test.cs") {
return "$LoadCount = $LoadCount + 1;";
}
return null;
},
});
// Load with different cases - should only call loader once
await runtime.loadFromPath("scripts/test.cs");
await runtime.loadFromPath("Scripts/Test.cs");
await runtime.loadFromPath("SCRIPTS/TEST.CS");
expect(loadCount).toBe(1);
});
it("treats different slashes as the same file for loaded scripts", async () => {
let loadCount = 0;
const runtime = createRuntime({
loadScript: async (path) => {
loadCount++;
if (path.replace(/\\/g, "/").toLowerCase() === "scripts/test.cs") {
return "$LoadCount = $LoadCount + 1;";
}
return null;
},
});
// Load with different slash styles - should only call loader once
await runtime.loadFromPath("scripts/test.cs");
await runtime.loadFromPath("scripts\\test.cs");
expect(loadCount).toBe(1);
});
it("treats different cases as the same file for failed scripts", async () => {
let loadCount = 0;
const runtime = createRuntime({
loadScript: async () => {
loadCount++;
return null; // Script not found
},
});
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
// Try to load with different cases - should only call loader once
const script = await runtime.loadFromSource(`
exec("scripts/missing.cs");
exec("Scripts/Missing.cs");
exec("SCRIPTS/MISSING.CS");
`);
script.execute();
warnSpy.mockRestore();
// Loader should only be called once since subsequent attempts
// see it's already in failedScripts
expect(loadCount).toBe(1);
});
it("treats different slashes as the same file for failed scripts", async () => {
let loadCount = 0;
const runtime = createRuntime({
loadScript: async () => {
loadCount++;
return null; // Script not found
},
});
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
// Try to load with different slash styles
const script = await runtime.loadFromSource(`
exec("scripts/missing.cs");
exec("scripts\\\\missing.cs");
`);
script.execute();
warnSpy.mockRestore();
expect(loadCount).toBe(1);
});
it("treats different cases as the same file for executed scripts", async () => {
const runtime = createRuntime({
loadScript: createLoader({
"scripts/counter.cs": "$Counter = $Counter + 1;",
}),
});
runtime.$g.set("Counter", 0);
const script = await runtime.loadFromSource(`
exec("scripts/counter.cs");
exec("Scripts/Counter.cs");
exec("SCRIPTS/COUNTER.CS");
`);
script.execute();
// Script should only execute once despite different cases
expect(runtime.$g.get("Counter")).toBe(1);
});
it("treats different slashes as the same file for executed scripts", async () => {
const runtime = createRuntime({
loadScript: createLoader({
"scripts/counter.cs": "$Counter = $Counter + 1;",
}),
});
runtime.$g.set("Counter", 0);
const script = await runtime.loadFromSource(`
exec("scripts/counter.cs");
exec("scripts\\\\counter.cs");
`);
script.execute();
// Script should only execute once despite different slashes
expect(runtime.$g.get("Counter")).toBe(1);
});
it("caches sequential loads with different cases", async () => {
let loadCount = 0;
const runtime = createRuntime({
loadScript: async (path) => {
loadCount++;
if (path.toLowerCase() === "scripts/test.cs") {
return "$Loaded = true;";
}
return null;
},
});
// Load same file with different cases sequentially
await runtime.loadFromPath("scripts/test.cs");
await runtime.loadFromPath("Scripts/Test.cs");
await runtime.loadFromPath("SCRIPTS/TEST.CS");
// Should only load once since cached after first load
expect(loadCount).toBe(1);
});
it("caches sequential loads with different slashes", async () => {
let loadCount = 0;
const runtime = createRuntime({
loadScript: async (path) => {
loadCount++;
if (path.replace(/\\/g, "/").toLowerCase() === "scripts/test.cs") {
return "$Loaded = true;";
}
return null;
},
});
// Load same file with different slashes sequentially
await runtime.loadFromPath("scripts/test.cs");
await runtime.loadFromPath("scripts\\test.cs");
// Should only load once since cached after first load
expect(loadCount).toBe(1);
});
it("handles combined case and slash differences", async () => {
let loadCount = 0;
const runtime = createRuntime({
loadScript: async (path) => {
loadCount++;
if (path.replace(/\\/g, "/").toLowerCase() === "scripts/test.cs") {
return "$Counter = $Counter + 1;";
}
return null;
},
});
runtime.$g.set("Counter", 0);
const script = await runtime.loadFromSource(`
exec("scripts/test.cs");
exec("Scripts\\\\Test.cs");
exec("SCRIPTS/TEST.CS");
exec("scripts\\\\TEST.CS");
`);
script.execute();
// All variations should be treated as the same file
expect(loadCount).toBe(1);
expect(runtime.$g.get("Counter")).toBe(1);
});
});
it("returns false when script is not found", async () => {
const runtime = createRuntime({
loadScript: async () => null,
@ -1462,6 +1660,24 @@ describe("TorqueScript Runtime", () => {
expect(runtime.$g.get("Result")).toBe(false);
});
it("returns false and logs error when exec called without extension", async () => {
// Engine requires file extension (matches tribes2-engine/TorqueSDK-1.2 behavior)
const runtime = createRuntime();
const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
const debugSpy = vi.spyOn(console, "debug").mockImplementation(() => {});
const script = await runtime.loadFromSource(
'$Result = exec("scripts/noextension");',
);
script.execute();
expect(errorSpy).toHaveBeenCalledWith(
'exec: invalid script file name "scripts/noextension".',
);
errorSpy.mockRestore();
debugSpy.mockRestore();
expect(runtime.$g.get("Result")).toBe(false);
});
it("shares globals between exec'd scripts", async () => {
const runtime = createRuntime({
loadScript: createLoader({
@ -1659,6 +1875,177 @@ describe("TorqueScript Runtime", () => {
expect(loadCount).toBe(1); // Only loaded once
});
it("handles diamond dependencies (parallel deduplication)", async () => {
// A depends on B and C, both B and C depend on D
// D should only be loaded once
let dLoadCount = 0;
const sources: Record<string, string> = {
"scripts/a.cs": 'exec("scripts/b.cs"); exec("scripts/c.cs"); $A = 1;',
"scripts/b.cs": 'exec("scripts/d.cs"); $B = 1;',
"scripts/c.cs": 'exec("scripts/d.cs"); $C = 1;',
"scripts/d.cs": "$D = 1;",
};
const runtime = createRuntime({
loadScript: async (path) => {
if (path === "scripts/d.cs") dLoadCount++;
return sources[path] ?? null;
},
});
const script = await runtime.loadFromPath("scripts/a.cs");
script.execute();
expect(dLoadCount).toBe(1); // D loaded only once despite being dep of both B and C
expect(runtime.$g.get("A")).toBe(1);
expect(runtime.$g.get("B")).toBe(1);
expect(runtime.$g.get("C")).toBe(1);
expect(runtime.$g.get("D")).toBe(1);
});
it("handles longer cycles not involving main script (A → B → C → B)", async () => {
const sources: Record<string, string> = {
"scripts/a.cs": 'exec("scripts/b.cs"); $A = 1;',
"scripts/b.cs": 'exec("scripts/c.cs"); $B = 1;',
"scripts/c.cs": 'exec("scripts/b.cs"); $C = 1;', // Cycle back to B
};
const runtime = createRuntime({
loadScript: async (path) => sources[path] ?? null,
});
// Should not hang
const script = await runtime.loadFromPath("scripts/a.cs");
script.execute();
expect(runtime.$g.get("A")).toBe(1);
expect(runtime.$g.get("B")).toBe(1);
expect(runtime.$g.get("C")).toBe(1);
});
it("handles multiple scripts depending on each other in complex patterns", async () => {
// A → [B, C], B → [D, E], C → [E, F], D → G, E → G, F → G
// G should only be loaded once
let gLoadCount = 0;
const sources: Record<string, string> = {
"scripts/a.cs": 'exec("scripts/b.cs"); exec("scripts/c.cs"); $A = 1;',
"scripts/b.cs": 'exec("scripts/d.cs"); exec("scripts/e.cs"); $B = 1;',
"scripts/c.cs": 'exec("scripts/e.cs"); exec("scripts/f.cs"); $C = 1;',
"scripts/d.cs": 'exec("scripts/g.cs"); $D = 1;',
"scripts/e.cs": 'exec("scripts/g.cs"); $E = 1;',
"scripts/f.cs": 'exec("scripts/g.cs"); $F = 1;',
"scripts/g.cs": "$G = 1;",
};
const runtime = createRuntime({
loadScript: async (path) => {
if (path === "scripts/g.cs") gLoadCount++;
return sources[path] ?? null;
},
});
const script = await runtime.loadFromPath("scripts/a.cs");
script.execute();
expect(gLoadCount).toBe(1); // G loaded only once
expect(runtime.$g.get("A")).toBe(1);
expect(runtime.$g.get("G")).toBe(1);
});
it("continues loading other scripts when one fails", async () => {
const sources: Record<string, string> = {
"scripts/main.cs":
'exec("scripts/good.cs"); exec("scripts/bad.cs"); exec("scripts/also-good.cs"); $Main = 1;',
"scripts/good.cs": "$Good = 1;",
// bad.cs is missing
"scripts/also-good.cs": "$AlsoGood = 1;",
};
const runtime = createRuntime({
loadScript: async (path) => sources[path] ?? null,
});
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
const script = await runtime.loadFromPath("scripts/main.cs");
script.execute();
warnSpy.mockRestore();
// Good scripts should still be loaded and executed
expect(runtime.$g.get("Main")).toBe(1);
expect(runtime.$g.get("Good")).toBe(1);
expect(runtime.$g.get("AlsoGood")).toBe(1);
});
it("handles parse errors gracefully without blocking other scripts", async () => {
const sources: Record<string, string> = {
"scripts/main.cs":
'exec("scripts/good.cs"); exec("scripts/bad-syntax.cs"); $Main = 1;',
"scripts/good.cs": "$Good = 1;",
"scripts/bad-syntax.cs": "this is not valid { syntax",
};
const runtime = createRuntime({
loadScript: async (path) => sources[path] ?? null,
});
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
const script = await runtime.loadFromPath("scripts/main.cs");
script.execute();
warnSpy.mockRestore();
expect(runtime.$g.get("Main")).toBe(1);
expect(runtime.$g.get("Good")).toBe(1);
});
it("respects pre-aborted signal when loading dependencies", async () => {
const controller = new AbortController();
controller.abort(); // Abort before starting
let loaderCalled = false;
const runtime = createRuntime({
loadScript: async () => {
loaderCalled = true;
return "$X = 1;";
},
signal: controller.signal,
});
// Load a script that has dependencies - should throw when trying to load deps
await expect(
runtime.loadFromSource('exec("scripts/dep.cs"); $Main = 1;'),
).rejects.toThrow();
expect(loaderCalled).toBe(false); // Loader should never be called
});
it("loads scripts in parallel (not serially)", async () => {
const loadOrder: string[] = [];
const sources: Record<string, string> = {
"scripts/main.cs":
'exec("scripts/a.cs"); exec("scripts/b.cs"); exec("scripts/c.cs");',
"scripts/a.cs": "$A = 1;",
"scripts/b.cs": "$B = 1;",
"scripts/c.cs": "$C = 1;",
};
const runtime = createRuntime({
loadScript: async (path) => {
loadOrder.push(`start:${path}`);
// Small delay to allow interleaving
await new Promise((resolve) => setTimeout(resolve, 10));
loadOrder.push(`end:${path}`);
return sources[path] ?? null;
},
});
await runtime.loadFromPath("scripts/main.cs");
// If parallel: starts happen before all ends
// If serial: each end would happen before the next start
const aStart = loadOrder.indexOf("start:scripts/a.cs");
const bStart = loadOrder.indexOf("start:scripts/b.cs");
const cStart = loadOrder.indexOf("start:scripts/c.cs");
const aEnd = loadOrder.indexOf("end:scripts/a.cs");
// All starts should happen before any of them ends (parallel behavior)
expect(bStart).toBeLessThan(aEnd);
expect(cStart).toBeLessThan(aEnd);
});
});
describe("path option", () => {

View file

@ -1066,67 +1066,85 @@ export function createRuntime(
}
async function loadDependencies(
ast: Program,
loading: Set<string>,
includePreload: boolean = false,
scriptsToLoad: string[],
loadingPromises: Map<string, Promise<void>>,
ancestors: Set<string>,
): Promise<void> {
const loader = options.loadScript;
if (!loader) {
// No loader, can't resolve dependencies
if (ast.execScriptPaths.length > 0) {
if (scriptsToLoad.length > 0) {
console.warn(
`Script has exec() calls but no loadScript provided:`,
ast.execScriptPaths,
scriptsToLoad,
);
}
return;
}
// Combine static exec() paths with preload scripts (on first call only)
const scriptsToLoad = includePreload
? [...ast.execScriptPaths, ...(options.preloadScripts ?? [])]
: ast.execScriptPaths;
for (const ref of scriptsToLoad) {
async function loadSingleScript(ref: string): Promise<void> {
options.signal?.throwIfAborted();
const normalized = normalizePath(ref);
// Skip if already loaded, failed, or currently loading (cycle detection)
// Skip if already loaded or failed
if (
state.scripts.has(normalized) ||
state.failedScripts.has(normalized) ||
loading.has(normalized)
state.failedScripts.has(normalized)
) {
continue;
return;
}
loading.add(normalized);
const source = await loader(ref);
if (source == null) {
console.warn(`Script not found: ${ref}`);
state.failedScripts.add(normalized);
loading.delete(normalized);
continue;
// If this script is an ancestor in our load chain, it's a cycle - skip
// (awaiting would cause deadlock)
if (ancestors.has(normalized)) {
return;
}
let depAst: Program;
try {
depAst = parse(source, { filename: ref });
} catch (err) {
console.warn(`Failed to parse script: ${ref}`, err);
state.failedScripts.add(normalized);
loading.delete(normalized);
continue;
// If already loading from a parallel branch, wait for it to complete
const existingPromise = loadingPromises.get(normalized);
if (existingPromise) {
await existingPromise;
return;
}
// Recursively load this script's dependencies first
await loadDependencies(depAst, loading);
const loadPromise = (async () => {
// Pass original path to loader - it handles its own normalization
const source = await loader(ref);
if (source == null) {
console.warn(`Script not found: ${ref}`);
state.failedScripts.add(normalized);
return;
}
// Store the parsed AST
state.scripts.set(normalized, depAst);
loading.delete(normalized);
let depAst: Program;
try {
depAst = parse(source, { filename: ref });
} catch (err) {
console.warn(`Failed to parse script: ${ref}`, err);
state.failedScripts.add(normalized);
return;
}
// Add this script to ancestors for nested loads (cycle detection)
const newAncestors = new Set(ancestors);
newAncestors.add(normalized);
// Recursively load this script's dependencies in parallel
await loadDependencies(
depAst.execScriptPaths,
loadingPromises,
newAncestors,
);
// Store the parsed AST
state.scripts.set(normalized, depAst);
})();
loadingPromises.set(normalized, loadPromise);
await loadPromise;
}
// Load all scripts in parallel
await Promise.all(scriptsToLoad.map(loadSingleScript));
}
async function loadFromPath(path: string): Promise<LoadedScript> {
@ -1135,12 +1153,12 @@ export function createRuntime(
throw new Error("loadFromPath requires loadScript option to be set");
}
// Check if already loaded (avoid unnecessary fetch)
const normalized = normalizePath(path);
if (state.scripts.has(normalized)) {
return createLoadedScript(state.scripts.get(normalized)!, path);
}
// Pass original path to loader - it handles its own normalization
const source = await loader(path);
if (source == null) {
throw new Error(`Script not found: ${path}`);
@ -1172,14 +1190,21 @@ export function createRuntime(
ast: Program,
loadOptions?: LoadScriptOptions,
): Promise<LoadedScript> {
// Load dependencies (include preload scripts on initial load)
const loading = new Set<string>();
const loadingPromises = new Map<string, Promise<void>>();
const ancestors = new Set<string>();
if (loadOptions?.path) {
const normalized = normalizePath(loadOptions.path);
loading.add(normalized);
// Mark the main script as loaded to prevent cycles back to it
state.scripts.set(normalized, ast);
ancestors.add(normalized);
}
await loadDependencies(ast, loading, true);
// Load dependencies and any preload scripts
const scriptsToLoad = [
...ast.execScriptPaths,
...(options.preloadScripts ?? []),
];
await loadDependencies(scriptsToLoad, loadingPromises, ancestors);
return createLoadedScript(ast, loadOptions?.path);
}

View file

@ -11,19 +11,20 @@ export function createScriptLoader(): ScriptLoader {
try {
url = getUrlForPath(path);
} catch (err) {
console.warn(`Script not in manifest: ${path}`, err);
console.warn(`Script not in manifest: ${path} (${err})`);
return null;
}
try {
const response = await fetch(url);
if (!response.ok) {
console.warn(`Script fetch failed: ${path} (${response.status})`);
console.error(`Script fetch failed: ${path} (${response.status})`);
return null;
}
return await response.text();
} catch (err) {
console.warn(`Script fetch error: ${path}`, err);
console.error(`Script fetch error: ${path}`);
console.error(err);
return null;
}
};

View file

@ -133,6 +133,12 @@ export class CaseInsensitiveSet {
}
}
export function normalizePath(path: string): string {
return path.replace(/\\/g, "/").toLowerCase();
/** Normalize path separators only (backslashes to forward slashes). */
export function normalizeSlashes(path: string): string {
return path.replace(/\\/g, "/");
}
/** Normalize path for use as a cache key (slashes + lowercase). */
export function normalizePath(path: string): string {
return normalizeSlashes(path).toLowerCase();
}