factorio Lua datagen

.gitignore

@@ -0,0 +1,2 @@
+node_modules/
+build/

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "serpent"]
+	path = serpent
+	url = https://github.com/pkulchenko/serpent.git

index.js

@@ -0,0 +1,108 @@
+const { readdirSync: readDir, readFileSync } = require("fs");
+const { readFile, writeFile, mkdir } = require("fs/promises");
+const homeDir = require("os").homedir()
+const { join: joinPath, relative: relativePath, sep: pathSeparator } = require("path");
+const { LuaFactory } = require('wasmoon')
+
+
+const FACTORIO_DATA_DIR = process.env["FACTORIO_DATA_DIR"] ?? `${homeDir}/.steam/steam/steamapps/common/Factorio/data`;
+const LOCALE = process.env["LOCALE"] ?? "en";
+const MODS = "core;base" + (process.env["MODS"] ?? 
+	((process.env["SPACE_AGE"] ?? "true") == "true" ? ";quality;elevated-rails;space-age" : ""));
+
+const mods = MODS.split(";").filter(v => v);
+
+const readLocaleData = mod => readFileSync(joinPath(FACTORIO_DATA_DIR, mod, "locale", LOCALE, mod + ".cfg"), "utf-8")
+	.trim()
+	.split("\n")
+	.filter(v => v)
+	.map(v => v.split(/=(.*)/, 2))
+	.flatMap((category = null, ([ k, v ]) => k.startsWith("[") ? (category = k, []) : { key: k, value: v, category }))
+	.reduce((m, { key, value, category }) => ((m.get(category) ?? m.set(category, new Map()).get(category)).set(key, value), m), new Map());
+
+const defines = [...readFileSync(joinPath(FACTORIO_DATA_DIR, "..", "doc-html", "defines.html"), "utf8")
+	.matchAll(/id="(defines\..*?)"/g)]
+	.map(v => v[1])
+	.flatMap((v, i) => i === 0 ? ["defines", v] : v) // prepend 'defines'
+	.map(v => [...v.matchAll(/([a-zA-Z0-9_]+)|\['(.+?)'\]/g)].map(([ , a, b ]) => a ?? b))
+	.reduce((acc, keys, idx) => {
+		let v = acc;
+		let lastKey = keys.pop();
+		keys.forEach(k => (typeof v[k] != "object" ? v[k] = {} : null, v = v[k]));
+		v[lastKey] = idx;
+		return acc;
+	}, {})
+	.defines;
+
+const loadModFactory = async () => { 
+	const luaFactory = new LuaFactory();
+	const lua = await luaFactory.createEngine();
+
+	await luaFactory.mountFile("serpent.lua", await readFile(joinPath("serpent", "src", "serpent.lua")));
+
+	await lua.doString("serpent = require('serpent')");
+	await Promise.all(readDir(joinPath(FACTORIO_DATA_DIR), { recursive: true, withFileTypes: true })
+		.filter(v => v.isFile() && v.name.endsWith(".lua"))
+		.map(async file => {
+			const data = await readFile(joinPath(file.parentPath, file.name));
+			if (file.parentPath.endsWith("lualib")) {
+				await luaFactory.mountFile(file.name, data);
+			}
+			const rPath = relativePath(joinPath(FACTORIO_DATA_DIR), joinPath(file.parentPath, file.name))
+				.split(pathSeparator);
+			const luaModule = rPath.shift();
+			// console.log(joinPath(`__${luaModule}__`, ...rPath))
+			await luaFactory.mountFile(joinPath(`__${luaModule}__`, ...rPath), data);
+		}));
+		
+	lua.global.set("defines", defines);
+			
+	await lua.doString("math.atan2 = function(x, y) return math.atan(y / x) end");
+	await lua.doString("require('dataloader')");
+
+	return async mod => {
+		const files = readDir(joinPath(FACTORIO_DATA_DIR, mod), { recursive: true, withFileTypes: true });
+		await lua.doString(files
+			.filter(v => v.isFile() && v.name.endsWith(".lua") && !v.parentPath.endsWith("lualib") && v.parentPath.indexOf("prototype") >= 0)
+			.map(file => relativePath(joinPath(FACTORIO_DATA_DIR, mod), joinPath(file.parentPath, file.name)).replace(/\.lua$/, ""))
+			.flatMap(file => [ file, file.replaceAll(pathSeparator, '.') ])
+			.reduce((s, p) => s + `package.loaded['${p}'] = nil\n`, ""));
+
+		await Promise.all(files
+			.filter(v => v.isFile() && v.name.endsWith(".lua") && !v.parentPath.endsWith("lualib"))
+			.map(async file => {
+				const moduleName = relativePath(joinPath(FACTORIO_DATA_DIR, mod), joinPath(file.parentPath, file.name))
+				return luaFactory.mountFile(moduleName, 
+					(mod === "space-age" && moduleName === "prototypes/ambient-sounds.lua" ? "require('__space-age__/sound/ambient/space/interlude-6/interlude-6')" : "") + // no clue why this fix is needed
+						await readFile(joinPath(file.parentPath, file.name)),
+				)
+			}));
+
+		console.log(`processing data, ${mod}`);
+		await lua.doFile("data.lua");
+
+		// enabling these results in a weird failure
+//		if (files.some(v => v.name === "data-updates.lua")) {
+//			console.log("\t data updates");
+//			await lua.doFile("data-updates.lua");
+//		}
+//
+//		if (files.some(v => v.name === "data-final-fixes.lua")) {
+//			console.log("\t data final fixes");
+//			await lua.doFile("data-final-fixes.lua");
+//		}
+
+		return lua.global.get("data");
+	}
+}
+
+console.log("loading mods:", mods);
+
+loadModFactory().then(async loadMod => {
+	let data = null;
+	for (const mod of mods) {
+		data = await loadMod(mod);
+	}
+	await mkdir("build", { recursive: true });
+	await writeFile(joinPath("build", "out.json"), JSON.stringify(data.raw));
+});

package.json

@@ -0,0 +1,5 @@
+{
+  "dependencies": {
+    "wasmoon": "^1.16.0"
+  }
+}

serpent

@@ -0,0 +1 @@
+Subproject commit 139fc18263bc5ffecc1729147891f1eb383046bf

yarn.lock

@@ -0,0 +1,15 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"@types/emscripten@1.39.10":
+  version "1.39.10"
+  resolved "https://registry.yarnpkg.com/@types/emscripten/-/emscripten-1.39.10.tgz#da6e58a6171b46a41d3694f812d845d515c77e18"
+  integrity sha512-TB/6hBkYQJxsZHSqyeuO1Jt0AB/bW6G7rHt9g7lML7SOF6lbgcHvw/Lr+69iqN0qxgXLhWKScAon73JNnptuDw==
+
+wasmoon@^1.16.0:
+  version "1.16.0"
+  resolved "https://registry.yarnpkg.com/wasmoon/-/wasmoon-1.16.0.tgz#aa39fc18bee8427db04ae07bfacdc332eed0c7af"
+  integrity sha512-FlRLb15WwAOz1A9OQDbf6oOKKSiefi5VK0ZRF2wgH9xk3o5SnU11tNPaOnQuAh1Ucr66cwwvVXaeVRaFdRBt5g==
+  dependencies:
+    "@types/emscripten" "1.39.10"