feat(web): stream zip conversion and release library packaging
This commit is contained in:
boykopovar
parent
34766c8236
commit
78df1e3244
|
|
@ -37,3 +37,6 @@
|
|||
[submodule "3rdparty/libusb-cmake"]
|
||||
path = 3rdparty/libusb-cmake
|
||||
url = https://github.com/libusb/libusb-cmake.git
|
||||
[submodule "3rdparty/zip.js"]
|
||||
path = 3rdparty/zip.js
|
||||
url = https://github.com/gildas-lormeau/zip.js.git
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
Subproject commit b608211a036192b665b5fd19589fa2454a315017
|
||||
|
|
@ -19,3 +19,10 @@ set_target_properties(relinker_wasm PROPERTIES
|
|||
CXX_EXTENSIONS OFF
|
||||
RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/web/native"
|
||||
)
|
||||
|
||||
find_package(Python3 REQUIRED COMPONENTS Interpreter)
|
||||
add_custom_target(relinker_web
|
||||
COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/tools/build_web.py" --build "${CMAKE_BINARY_DIR}"
|
||||
DEPENDS relinker_wasm
|
||||
VERBATIM
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,215 @@
|
|||
import { BlobReader, Uint8ArrayReader, ZipReader, ZipWriter, configure } from "./vendor/zip/zip-no-worker.js";
|
||||
|
||||
configure({ useWebWorkers: false, useCompressionStream: false, chunkSize: 65536 });
|
||||
|
||||
export const MEMORY_LIMIT = 256 * 1024 * 1024;
|
||||
export const CONVERSION_LIMIT = 256 * 1024 * 1024;
|
||||
export const ICON_LIMIT = 16 * 1024 * 1024;
|
||||
const MAX_ENTRIES = 100000;
|
||||
|
||||
class BoundedBlobReader extends BlobReader {
|
||||
async readUint8Array(offset, length) {
|
||||
if (length > 16 * 1024 * 1024) throw new Error("ZIP directory metadata exceeds 16 MiB");
|
||||
return super.readUint8Array(offset, length);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
export function validPath(name) {
|
||||
if (!name || name.includes("\\") || /[\x00-\x1f<>:"|?*]/u.test(name)) throw new Error(`Unsafe ZIP path: ${name}`);
|
||||
const parts = name.replace(/\/$/u, "").split("/");
|
||||
if (parts.some(part => !part || part === "." || part === ".." || /[. ]$/u.test(part) || /^(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)/iu.test(part))) throw new Error(`Unsafe ZIP path: ${name}`);
|
||||
return name;
|
||||
}
|
||||
|
||||
function safeSize(value) {
|
||||
if (!Number.isSafeInteger(value) || value < 0) throw new Error("ZIP contains an unsupported file size");
|
||||
return value;
|
||||
}
|
||||
|
||||
async function signature(entry) {
|
||||
if (entry.uncompressedSize < 4) return false;
|
||||
const stop = new AbortController();
|
||||
const header = new Uint8Array(4);
|
||||
let filled = 0;
|
||||
try {
|
||||
await entry.getData(new WritableStream({
|
||||
write(chunk) {
|
||||
const length = Math.min(4 - filled, chunk.length);
|
||||
header.set(chunk.subarray(0, length), filled);
|
||||
filled += length;
|
||||
if (filled === 4) stop.abort();
|
||||
}
|
||||
}), { signal: stop.signal, checkSignature: false });
|
||||
} catch (error) {
|
||||
if (!stop.signal.aborted || filled !== 4) throw error;
|
||||
}
|
||||
return header[0] === 0x7f && header[1] === 69 && header[2] === 76 && header[3] === 70;
|
||||
}
|
||||
|
||||
async function boundedBytes(entry, limit) {
|
||||
if (entry.uncompressedSize > limit) throw new Error(`${entry.filename} exceeds the conversion memory limit`);
|
||||
let size = 0;
|
||||
const chunks = [];
|
||||
await entry.getData(new WritableStream({
|
||||
write(chunk) {
|
||||
size += chunk.byteLength;
|
||||
if (size > limit || size > entry.uncompressedSize) throw new Error(`Unexpected expanded size: ${entry.filename}`);
|
||||
chunks.push(chunk);
|
||||
}
|
||||
}), { checkSignature: true });
|
||||
if (size !== entry.uncompressedSize) throw new Error(`Incorrect expanded size: ${entry.filename}`);
|
||||
const bytes = new Uint8Array(size);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.length; }
|
||||
return bytes;
|
||||
}
|
||||
|
||||
export class Archive {
|
||||
constructor(file) {
|
||||
this.reader = new ZipReader(new BoundedBlobReader(file));
|
||||
this.entries = [];
|
||||
}
|
||||
|
||||
async inspect(report) {
|
||||
const paths = new Map();
|
||||
let total = 0;
|
||||
for await (const entry of this.reader.getEntriesGenerator()) {
|
||||
if (this.entries.length >= MAX_ENTRIES) throw new Error("ZIP contains more than 100,000 entries");
|
||||
validPath(entry.filename);
|
||||
const key = entry.filename.replace(/\/$/u, "").toLowerCase();
|
||||
if (paths.has(key)) throw new Error(`Duplicate ZIP path: ${entry.filename}`);
|
||||
if (entry.encrypted) throw new Error(`Encrypted ZIP entries are not supported: ${entry.filename}`);
|
||||
if (!entry.directory && entry.compressionMethod !== 0 && entry.compressionMethod !== 8) throw new Error(`Unsupported ZIP compression: ${entry.filename}`);
|
||||
const mode = (entry.externalFileAttributes >>> 16) & 0xf000;
|
||||
if (mode && mode !== 0x8000 && mode !== 0x4000) throw new Error(`Links and special files are not supported: ${entry.filename}`);
|
||||
paths.set(key, entry.directory);
|
||||
total = safeSize(total + safeSize(entry.uncompressedSize));
|
||||
this.entries.push(entry);
|
||||
}
|
||||
if (!this.entries.some(entry => !entry.directory)) throw new Error("The ZIP is empty");
|
||||
for (const entry of this.entries) {
|
||||
const parts = entry.filename.split("/");
|
||||
for (let index = 1; index < parts.length; index++) {
|
||||
if (paths.get(parts.slice(0, index).join("/").toLowerCase()) === false) throw new Error(`A file is also used as a folder: ${entry.filename}`);
|
||||
}
|
||||
}
|
||||
let prefix = "";
|
||||
const files = this.entries.filter(entry => !entry.directory);
|
||||
while (files.every(entry => entry.filename.slice(prefix.length).includes("/"))) {
|
||||
const first = files[0].filename.slice(prefix.length).split("/")[0];
|
||||
if (["sce_sys", "sce_module"].includes(first) || !this.entries.every(entry => entry.filename === prefix || entry.filename.startsWith(prefix + first + "/"))) break;
|
||||
prefix += first + "/";
|
||||
}
|
||||
this.prefix = prefix;
|
||||
this.files = this.entries.filter(entry => entry.filename.startsWith(prefix) && entry.filename !== prefix).map(entry => ({ entry, name: entry.filename.slice(prefix.length) }));
|
||||
const executables = [];
|
||||
this.modules = [];
|
||||
for (let index = 0; index < this.files.length; index++) {
|
||||
const item = this.files[index];
|
||||
report({ phase: "Inspecting archive", input: item.name, completed: index, total: this.files.length });
|
||||
if (item.entry.directory || item.name.startsWith("sce_sys/")) continue;
|
||||
const elf = await signature(item.entry);
|
||||
if (item.name.startsWith("sce_module/")) {
|
||||
if (item.name.endsWith(".guest.prx")) throw new Error("Use the original archive, without previously converted .guest.prx files");
|
||||
if (elf) this.modules.push(item);
|
||||
else if (/\.(elf|prx|sprx|suprx|debug_prx)$/iu.test(item.name)) throw new Error(`The module is not a decrypted ELF: ${item.name}`);
|
||||
} else if (elf) executables.push(item);
|
||||
}
|
||||
if (executables.length !== 1) throw new Error(`Expected exactly one executable ELF outside sce_module; found ${executables.length}`);
|
||||
this.executable = executables[0];
|
||||
if (this.executable.name.includes("/")) throw new Error("Place the executable next to sce_sys and sce_module, not inside another folder");
|
||||
this.icon = this.files.find(item => item.name === "sce_sys/icon0.png" && !item.entry.directory);
|
||||
if (this.icon && this.icon.entry.uncompressedSize > ICON_LIMIT) throw new Error("sce_sys/icon0.png exceeds 16 MiB");
|
||||
this.selectedBytes = [this.executable, ...this.modules, ...(this.icon ? [this.icon] : [])].reduce((sum, item) => sum + item.entry.uncompressedSize, 0);
|
||||
if (this.selectedBytes > CONVERSION_LIMIT) throw new Error("Executable, modules and icon exceed the 256 MiB conversion limit. Use the native relinker for this app.");
|
||||
const warnings = [];
|
||||
if (!this.files.some(item => item.name.startsWith("sce_sys/"))) warnings.push("sce_sys is missing. The Windows executable will have no app icon.");
|
||||
else if (!this.icon) warnings.push("sce_sys/icon0.png is missing. The Windows executable will have no app icon.");
|
||||
if (!this.files.some(item => item.name.startsWith("sce_module/"))) warnings.push("sce_module is missing. If the original app included this folder, leaving it out can prevent the app from starting.");
|
||||
this.totalBytes = total;
|
||||
return { name: this.executable.name, files: files.length, modules: this.modules.length, totalBytes: total, selectedBytes: this.selectedBytes, root: prefix, warnings };
|
||||
}
|
||||
|
||||
async conversionInput(target, toIntel, report) {
|
||||
let remaining = CONVERSION_LIMIT;
|
||||
const read = async item => {
|
||||
report({ phase: "Reading conversion input", input: item.name });
|
||||
const bytes = await boundedBytes(item.entry, remaining);
|
||||
remaining -= bytes.length;
|
||||
return bytes;
|
||||
};
|
||||
const name = this.executable.name;
|
||||
const stem = name.includes(".") ? name.slice(0, name.lastIndexOf(".")) : name;
|
||||
const outputName = (stem || "app") + (target === "windows" ? ".exe" : ".elf");
|
||||
const input = { name, outputName, target, toIntel, executable: await read(this.executable), modules: [], icon: null };
|
||||
for (const item of this.modules) input.modules.push({ name: item.name, bytes: await read(item) });
|
||||
if (this.icon && target === "windows") input.icon = await read(this.icon);
|
||||
return input;
|
||||
}
|
||||
|
||||
async close() { await this.reader.close(); }
|
||||
}
|
||||
|
||||
export function memoryEstimate(archiveBytes, selectedBytes, libraryBytes, entries) {
|
||||
return safeSize(Math.ceil(archiveBytes * 1.02) + selectedBytes * 4 + libraryBytes + entries * 1024 + 8 * 1024 * 1024);
|
||||
}
|
||||
|
||||
async function appendEntry(writer, name, entry, report) {
|
||||
const stream = new TransformStream();
|
||||
let produced = 0;
|
||||
const limit = new TransformStream({ transform(chunk, controller) {
|
||||
produced += chunk.length;
|
||||
if (produced > entry.uncompressedSize) throw new Error(`Unexpected expanded size: ${name}`);
|
||||
controller.enqueue(chunk);
|
||||
}, flush() {
|
||||
if (produced !== entry.uncompressedSize) throw new Error(`Incorrect expanded size: ${name}`);
|
||||
} });
|
||||
await Promise.all([
|
||||
entry.getData(stream.writable, { checkSignature: true }),
|
||||
writer.add(name, stream.readable.pipeThrough(limit), {
|
||||
versionMadeBy: 0x31e, externalFileAttributes: entry.externalFileAttributes,
|
||||
onprogress: completed => report({ phase: "Compressing ZIP", input: name, completed, total: entry.uncompressedSize })
|
||||
})
|
||||
]);
|
||||
}
|
||||
|
||||
export async function packageArchive(archive, converted, outputName, libraryFile, destination, report) {
|
||||
const libraries = new ZipReader(new BoundedBlobReader(libraryFile));
|
||||
const writer = new ZipWriter(destination, { level: 9, zip64: true, bufferedWrite: false, useWebWorkers: false, useCompressionStream: false, preventClose: true });
|
||||
const names = new Set();
|
||||
const claim = name => {
|
||||
validPath(name);
|
||||
const key = name.replace(/\/$/u, "").toLowerCase();
|
||||
if (names.has(key)) throw new Error(`Output path collision: ${name}`);
|
||||
names.add(key);
|
||||
};
|
||||
try {
|
||||
const libraryEntries = await libraries.getEntries();
|
||||
if (!libraryEntries.length || libraryEntries.length > 4096) throw new Error("Invalid library archive");
|
||||
for (const entry of libraryEntries) {
|
||||
if (!entry.filename.startsWith("libs/") || entry.encrypted || ((entry.externalFileAttributes >>> 16) & 0xf000) === 0xa000) throw new Error("Invalid library archive layout");
|
||||
claim(entry.filename);
|
||||
}
|
||||
for (const item of archive.files) claim("app0/" + item.name);
|
||||
claim(outputName);
|
||||
for (const module of converted.modules) {
|
||||
if (!module.name.startsWith("app0/sce_module/")) throw new Error("Invalid converted module path");
|
||||
claim(module.name);
|
||||
}
|
||||
const binary = { versionMadeBy: 0x31e, externalFileAttributes: (0o100755 << 16) >>> 0 };
|
||||
await writer.add(outputName, new Uint8ArrayReader(converted.executable), binary);
|
||||
for (const module of converted.modules) await writer.add(module.name, new Uint8ArrayReader(module.bytes), binary);
|
||||
for (const entry of libraryEntries) {
|
||||
if (entry.directory) await writer.add(entry.filename, undefined, { directory: true });
|
||||
else await appendEntry(writer, entry.filename, entry, report);
|
||||
}
|
||||
for (const { entry, name } of archive.files) {
|
||||
if (entry.directory) await writer.add("app0/" + name, undefined, { directory: true });
|
||||
else await appendEntry(writer, "app0/" + name, entry, report);
|
||||
}
|
||||
await writer.close();
|
||||
} finally {
|
||||
await libraries.close();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import path from "node:path";
|
||||
|
||||
const site = name => pathToFileURL(path.join(process.argv[2], name)).href;
|
||||
const { Archive, memoryEstimate, MEMORY_LIMIT, packageArchive } = await import(site("archive.mjs"));
|
||||
const { BlobReader, BlobWriter, Uint8ArrayReader, Uint8ArrayWriter, ZipReader, ZipWriter } = await import(site("vendor/zip/zip-no-worker.js"));
|
||||
const { default: createRelinker } = await import(site("native/relinker_wasm.mjs"));
|
||||
const native = await createRelinker();
|
||||
const executable = new Uint8Array(await readFile(path.join(process.argv[3], "main.elf")));
|
||||
const module = new Uint8Array(await readFile(path.join(process.argv[3], "module.elf")));
|
||||
const resource = new Uint8Array(2 * 1024 * 1024).fill(65);
|
||||
|
||||
async function zip(entries) {
|
||||
const output = new BlobWriter("application/zip");
|
||||
const writer = new ZipWriter(output, { zip64: true, level: 9, useWebWorkers: false, useCompressionStream: false });
|
||||
for (const [name, bytes, options] of entries) await writer.add(name, bytes ? new Uint8ArrayReader(bytes) : undefined, options);
|
||||
return writer.close();
|
||||
}
|
||||
|
||||
const source = await zip([["title/", null, { directory: true }], ["title/program.dat", executable], ["title/sce_module/library.elf", module], ["title/sce_sys/param.json", new TextEncoder().encode("{}")], ["title/content/resource.bin", resource]]);
|
||||
for (const target of ["windows", "linux"]) {
|
||||
const archive = new Archive(source);
|
||||
const summary = await archive.inspect(() => {});
|
||||
assert.equal(summary.name, "program.dat");
|
||||
assert.equal(summary.root, "title/");
|
||||
assert.equal(summary.modules, 1);
|
||||
assert.equal(summary.selectedBytes, executable.length + module.length);
|
||||
const input = await archive.conversionInput(target, false, () => {});
|
||||
assert.equal(input.modules.length, 1);
|
||||
assert.deepEqual(input.executable, executable);
|
||||
assert.equal(input.modules[0].name, "sce_module/library.elf");
|
||||
const events = [];
|
||||
const converted = native.Convert(input, event => events.push(event));
|
||||
assert.equal(converted.ok, true, converted.error);
|
||||
assert.equal(converted.executable[0], target === "windows" ? 77 : 127);
|
||||
assert(events.some(event => event.stage === 6 && event.completed > 0));
|
||||
const libraries = await zip([["libs/system.prx", new Uint8Array([1, 2, 3])]]);
|
||||
const chunks = [];
|
||||
let maxChunk = 0;
|
||||
const destination = new WritableStream({ async write(chunk) {
|
||||
maxChunk = Math.max(maxChunk, chunk.length);
|
||||
chunks.push(chunk.slice());
|
||||
await new Promise(resolve => setTimeout(resolve, 1));
|
||||
} });
|
||||
await packageArchive(archive, converted, input.outputName, libraries, destination, () => {});
|
||||
await destination.close();
|
||||
const reader = new ZipReader(new BlobReader(new Blob(chunks)));
|
||||
const entries = await reader.getEntries();
|
||||
const byName = new Map(entries.map(entry => [entry.filename, entry]));
|
||||
assert(byName.has(input.outputName));
|
||||
assert(byName.has("libs/system.prx"));
|
||||
assert(byName.has("app0/sce_module/library.elf.guest.prx"));
|
||||
assert.deepEqual(await byName.get("app0/program.dat").getData(new Uint8ArrayWriter(), { checkSignature: true }), executable);
|
||||
assert.deepEqual(await byName.get("app0/content/resource.bin").getData(new Uint8ArrayWriter(), { checkSignature: true }), resource);
|
||||
assert.equal((byName.get(input.outputName).externalFileAttributes >>> 16) & 0o777, 0o755);
|
||||
assert(maxChunk < resource.length);
|
||||
await reader.close();
|
||||
await archive.close();
|
||||
console.log(`${target}: conversion, original files, modules, permissions and streamed ZIP verified`);
|
||||
}
|
||||
|
||||
for (const [entries, error] of [
|
||||
[[["one", executable], ["two", executable]], /exactly one/u],
|
||||
[[["readme", new Uint8Array([1])]], /exactly one/u],
|
||||
[[["app", executable], ["..\/escape", resource]], /Unsafe ZIP/u],
|
||||
[[["app", executable], ["APP", resource]], /Duplicate ZIP/u],
|
||||
[[["app", executable], ["folder", resource], ["folder/file", resource]], /also used as a folder/u],
|
||||
[[["app", executable], ["link", resource, { versionMadeBy: 0x31e, externalFileAttributes: (0o120777 << 16) >>> 0 }]], /Links and special/u]
|
||||
]) {
|
||||
const archive = new Archive(await zip(entries));
|
||||
await assert.rejects(archive.inspect(() => {}), error);
|
||||
await archive.close();
|
||||
}
|
||||
const missing = new Archive(await zip([["any-name", executable]]));
|
||||
assert.equal((await missing.inspect(() => {})).warnings.length, 2);
|
||||
await missing.close();
|
||||
assert(memoryEstimate(200 * 1024 ** 3, 1024, 1024, 100) > MEMORY_LIMIT);
|
||||
const invalid = native.Convert({ target: "unsupported" }, () => {});
|
||||
assert.equal(invalid.ok, false);
|
||||
console.log("Archive validation, missing-folder warnings and large-download budget verified");
|
||||
|
|
@ -0,0 +1,95 @@
|
|||
import { Archive, MEMORY_LIMIT, memoryEstimate, packageArchive } from "./archive.mjs";
|
||||
|
||||
let archive;
|
||||
let busy = false;
|
||||
let pendingWrite;
|
||||
let lastReport = 0;
|
||||
let lastPhase = "";
|
||||
|
||||
function report(event) {
|
||||
const phase = event.phase ?? String(event.stage);
|
||||
const now = performance.now();
|
||||
if (!event.message && phase === lastPhase && event.completed !== event.total && now - lastReport < 80) return;
|
||||
lastReport = now;
|
||||
lastPhase = phase;
|
||||
self.postMessage({ type: "progress", ...event });
|
||||
}
|
||||
|
||||
async function downloadLibraries(target) {
|
||||
report({ phase: "Downloading system libraries" });
|
||||
const response = await fetch("./release.json", { cache: "no-cache" });
|
||||
if (!response.ok) throw new Error(`Cannot load release manifest (${response.status})`);
|
||||
const release = await response.json();
|
||||
const asset = release.platforms[target];
|
||||
if (!asset || !/^libraries\/(windows|linux)-[a-f0-9]+\.zip$/u.test(asset.path) || !/^[a-f0-9]{64}$/u.test(asset.sha256)) throw new Error("Invalid release manifest");
|
||||
const latestResponse = await fetch("https://api.github.com/repos/boykopovar/AnyPS5/releases/latest", { headers: { Accept: "application/vnd.github+json" } });
|
||||
if (!latestResponse.ok) throw new Error(`Cannot verify the latest release (${latestResponse.status}). Try again later.`);
|
||||
const latest = await latestResponse.json();
|
||||
if (latest.tag_name !== release.tag) throw new Error("A new release is being published to this site. Try again after the Pages build finishes.");
|
||||
const download = await fetch(asset.path);
|
||||
if (!download.ok) throw new Error(`Cannot download system libraries (${download.status})`);
|
||||
const bytes = await download.arrayBuffer();
|
||||
if (bytes.byteLength !== asset.size) throw new Error("System library download is incomplete");
|
||||
const digest = [...new Uint8Array(await crypto.subtle.digest("SHA-256", bytes))].map(byte => byte.toString(16).padStart(2, "0")).join("");
|
||||
if (digest !== asset.sha256) throw new Error("System library checksum mismatch");
|
||||
return { file: new Blob([bytes]), release, expandedBytes: asset.expandedBytes };
|
||||
}
|
||||
|
||||
self.onmessage = async ({ data }) => {
|
||||
if (data.type === "written") {
|
||||
if (!pendingWrite) return;
|
||||
const resolve = pendingWrite;
|
||||
pendingWrite = null;
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
if (busy) return;
|
||||
busy = true;
|
||||
try {
|
||||
if (data.type === "inspect") {
|
||||
if (archive) await archive.close();
|
||||
archive = new Archive(data.file);
|
||||
const summary = await archive.inspect(report);
|
||||
self.postMessage({ type: "inspected", summary });
|
||||
} else if (data.type === "convert") {
|
||||
if (!archive) throw new Error("Select an archive first");
|
||||
const libraries = await downloadLibraries(data.target);
|
||||
const estimate = memoryEstimate(archive.totalBytes, archive.selectedBytes, libraries.expandedBytes, archive.entries.length);
|
||||
if (data.mode === "memory" && estimate > MEMORY_LIMIT) throw new Error("This result exceeds the 256 MiB memory-download budget. Use Save directly to disk in desktop Chrome or Edge.");
|
||||
report({ phase: "Loading converter" });
|
||||
const { default: createRelinker } = await import("./native/relinker_wasm.mjs");
|
||||
const native = await createRelinker();
|
||||
let input = await archive.conversionInput(data.target, data.toIntel, report);
|
||||
const outputName = input.outputName;
|
||||
const converted = native.Convert(input, report);
|
||||
input = null;
|
||||
if (!converted.ok) throw new Error(converted.error);
|
||||
const chunks = [];
|
||||
let size = 0;
|
||||
const destination = new WritableStream({
|
||||
async write(chunk) {
|
||||
size += chunk.byteLength;
|
||||
if (data.mode === "memory") {
|
||||
if (size > MEMORY_LIMIT) throw new Error("The compressed ZIP exceeded 256 MiB. Use Save directly to disk.");
|
||||
chunks.push(chunk.slice());
|
||||
} else if (data.mode === "disk") {
|
||||
const bytes = chunk.slice();
|
||||
await new Promise(resolve => {
|
||||
pendingWrite = resolve;
|
||||
self.postMessage({ type: "write", bytes }, [bytes.buffer]);
|
||||
});
|
||||
} else throw new Error("Unknown save mode");
|
||||
}
|
||||
});
|
||||
await packageArchive(archive, converted, outputName, libraries.file, destination, report);
|
||||
await destination.close();
|
||||
self.postMessage({ type: "complete", blob: data.mode === "memory" ? new Blob(chunks, { type: "application/zip" }) : null, outputName, size, release: libraries.release.tag });
|
||||
await archive.close();
|
||||
archive = null;
|
||||
} else throw new Error("Unknown worker request");
|
||||
} catch (error) {
|
||||
self.postMessage({ type: "error", message: error instanceof Error ? error.message : String(error) });
|
||||
} finally {
|
||||
busy = false;
|
||||
}
|
||||
};
|
||||
|
|
@ -0,0 +1,114 @@
|
|||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import urllib.request
|
||||
import zipfile
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def copy_sources(output):
|
||||
source = ROOT / "demo/web"
|
||||
for file in source.iterdir():
|
||||
if file.is_file():
|
||||
shutil.copy2(file, output / file.name)
|
||||
library = ROOT / "3rdparty/zip.js/lib"
|
||||
pending = [library / "zip-no-worker.js"]
|
||||
copied = set()
|
||||
while pending:
|
||||
file = pending.pop().resolve()
|
||||
if file in copied:
|
||||
continue
|
||||
if not file.is_relative_to(library.resolve()):
|
||||
raise ValueError(f"ZIP source import escapes library: {file}")
|
||||
text = file.read_text(encoding="utf-8")
|
||||
copied.add(file)
|
||||
destination = output / "vendor/zip" / file.relative_to(library.resolve())
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(file, destination)
|
||||
for name in re.findall(r"(?:from\s+|import\s*)[\"'](\.[^\"']+)[\"']", text):
|
||||
pending.append(file.parent / name)
|
||||
shutil.copy2(ROOT / "3rdparty/zip.js/LICENSE", output / "vendor/zip/LICENSE")
|
||||
(output / ".nojekyll").touch()
|
||||
|
||||
|
||||
def request(url):
|
||||
headers = {"User-Agent": "AnyPS5-Pages"}
|
||||
token = os.environ.get("GITHUB_TOKEN")
|
||||
if token and url.startswith("https://api.github.com/"):
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
return urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=60)
|
||||
|
||||
|
||||
def publish_libraries(output):
|
||||
with request("https://api.github.com/repos/boykopovar/AnyPS5/releases/latest") as response:
|
||||
release = json.load(response)
|
||||
tag = release["tag_name"]
|
||||
if not re.fullmatch(r"v[0-9A-Za-z][0-9A-Za-z._-]*", tag):
|
||||
raise ValueError("Invalid release tag")
|
||||
manifest = {"tag": tag, "url": release["html_url"], "platforms": {}}
|
||||
destination = output / "libraries"
|
||||
destination.mkdir(exist_ok=True)
|
||||
for platform in ("windows", "linux"):
|
||||
name = f"prx-{platform}-{tag}.zip"
|
||||
matches = [asset for asset in release["assets"] if asset["name"] == name]
|
||||
if len(matches) != 1:
|
||||
raise ValueError(f"Missing or ambiguous release asset: {name}")
|
||||
asset = matches[0]
|
||||
temporary = destination / f"{platform}.download"
|
||||
try:
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
with request(asset["browser_download_url"]) as response, temporary.open("wb") as target:
|
||||
while chunk := response.read(1024 * 1024):
|
||||
size += len(chunk)
|
||||
if size > asset["size"] or size > 512 * 1024 * 1024:
|
||||
raise ValueError("Release archive exceeds expected size")
|
||||
digest.update(chunk)
|
||||
target.write(chunk)
|
||||
if size != asset["size"]:
|
||||
raise ValueError("Incomplete release archive")
|
||||
checksum = digest.hexdigest()
|
||||
if asset.get("digest") and asset["digest"] != "sha256:" + checksum:
|
||||
raise ValueError("Release archive checksum mismatch")
|
||||
with zipfile.ZipFile(temporary) as archive:
|
||||
files = archive.infolist()
|
||||
if not files or len(files) > 4096:
|
||||
raise ValueError("Invalid library archive")
|
||||
for file in files:
|
||||
if not file.filename.startswith("libs/") or any(part in ("..", ".", "") for part in file.filename.rstrip("/").split("/")) or "\\" in file.filename:
|
||||
raise ValueError("Invalid library archive layout")
|
||||
corrupt = archive.testzip()
|
||||
if corrupt:
|
||||
raise ValueError(f"Corrupt release entry: {corrupt}")
|
||||
expanded = sum(file.file_size for file in files)
|
||||
path = f"libraries/{platform}-{checksum}.zip"
|
||||
temporary.replace(output / path)
|
||||
manifest["platforms"][platform] = {"path": path, "size": size, "expandedBytes": expanded, "sha256": checksum}
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
(output / "release.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
|
||||
print(f"Published library assets from {tag}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--build", type=Path, required=True)
|
||||
parser.add_argument("--libraries", action="store_true")
|
||||
args = parser.parse_args()
|
||||
output = args.build.resolve() / "web"
|
||||
for name in ("relinker_wasm.mjs", "relinker_wasm.wasm"):
|
||||
if not (output / "native" / name).is_file():
|
||||
raise ValueError(f"Build relinker_wasm first: missing {name}")
|
||||
copy_sources(output)
|
||||
if args.libraries:
|
||||
publish_libraries(output)
|
||||
print(output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
import argparse
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT / "core/relinker/relinker/tests"))
|
||||
from test_guest_intel_trampolines import main_fixture
|
||||
from test_guest_module_directories import module_with_symbol
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--build", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
with tempfile.TemporaryDirectory(prefix="anyps5-web-") as directory:
|
||||
fixture = Path(directory)
|
||||
(fixture / "main.elf").write_bytes(main_fixture())
|
||||
(fixture / "module.elf").write_bytes(module_with_symbol(True))
|
||||
subprocess.run(["node", str(ROOT / "demo/web/tests/archive.test.mjs"), str(args.build.resolve() / "web"), str(fixture)], check=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Reference in New Issue