|
| 1 | +/** |
| 2 | + * In-place tar rewriting and stored-gzip emission over a decompressed buffer. |
| 3 | + * |
| 4 | + * The platform-binary build decompresses the upstream ONCE into a single buffer |
| 5 | + * (within the Worker budget, the packument path already does this via |
| 6 | + * buildMetaLight), rewrites `package/package.json` in place, then re-emits the |
| 7 | + * buffer as gzip "stored" blocks. Because the buffer is fully in memory while it |
| 8 | + * is framed and uploaded, no decompressor is running during the slow R2 part |
| 9 | + * uploads, so nothing buffers ahead, which is what made the previous streaming |
| 10 | + * build spike past the 128MB limit (Cloudflare 1102) under upload backpressure. |
| 11 | + */ |
| 12 | + |
| 13 | +import { HttpError } from '../httpError' |
| 14 | + |
| 15 | +const BLOCK = 512 |
| 16 | +const NAME_LEN = 100 |
| 17 | +const SIZE_OFF = 124 |
| 18 | +const SIZE_LEN = 12 |
| 19 | +const CHKSUM_OFF = 148 |
| 20 | +const CHKSUM_LEN = 8 |
| 21 | +const MAGIC_OFF = 257 |
| 22 | +const PREFIX_OFF = 345 |
| 23 | +const PREFIX_LEN = 155 |
| 24 | + |
| 25 | +const textDecoder = new TextDecoder() |
| 26 | + |
| 27 | +function isZeroBlock(tar: Uint8Array, off: number): boolean { |
| 28 | + for (let i = off; i < off + BLOCK; i++) if (tar[i] !== 0) return false |
| 29 | + return true |
| 30 | +} |
| 31 | + |
| 32 | +function readCStr(tar: Uint8Array, off: number, len: number): string { |
| 33 | + let end = 0 |
| 34 | + while (end < len && tar[off + end] !== 0) end++ |
| 35 | + return textDecoder.decode(tar.subarray(off, off + end)) |
| 36 | +} |
| 37 | + |
| 38 | +function readName(tar: Uint8Array, off: number): string { |
| 39 | + const name = readCStr(tar, off, NAME_LEN) |
| 40 | + const isUstar = |
| 41 | + tar[off + MAGIC_OFF] === 0x75 && |
| 42 | + tar[off + MAGIC_OFF + 1] === 0x73 && |
| 43 | + tar[off + MAGIC_OFF + 2] === 0x74 && |
| 44 | + tar[off + MAGIC_OFF + 3] === 0x61 && |
| 45 | + tar[off + MAGIC_OFF + 4] === 0x72 |
| 46 | + if (isUstar) { |
| 47 | + const prefix = readCStr(tar, off + PREFIX_OFF, PREFIX_LEN) |
| 48 | + if (prefix) return `${prefix}/${name}` |
| 49 | + } |
| 50 | + return name |
| 51 | +} |
| 52 | + |
| 53 | +function readSize(tar: Uint8Array, off: number): number { |
| 54 | + if (tar[off + SIZE_OFF] & 0x80) { |
| 55 | + throw new HttpError(422, 'Unsupported base-256 size field in tarball') |
| 56 | + } |
| 57 | + let value = 0 |
| 58 | + for (let i = off + SIZE_OFF; i < off + SIZE_OFF + SIZE_LEN; i++) { |
| 59 | + const c = tar[i] |
| 60 | + if (c === 0 || c === 0x20) continue |
| 61 | + value = value * 8 + (c - 0x30) |
| 62 | + } |
| 63 | + return value |
| 64 | +} |
| 65 | + |
| 66 | +/** Rewrite the header in place for a new (smaller-or-equal) entry size. */ |
| 67 | +function writeHeader(tar: Uint8Array, off: number, newSize: number): void { |
| 68 | + const octal = newSize.toString(8).padStart(SIZE_LEN - 1, '0') |
| 69 | + for (let i = 0; i < SIZE_LEN - 1; i++) { |
| 70 | + tar[off + SIZE_OFF + i] = octal.charCodeAt(i) |
| 71 | + } |
| 72 | + tar[off + SIZE_OFF + SIZE_LEN - 1] = 0 |
| 73 | + |
| 74 | + for (let i = 0; i < CHKSUM_LEN; i++) tar[off + CHKSUM_OFF + i] = 0x20 |
| 75 | + let sum = 0 |
| 76 | + for (let i = 0; i < BLOCK; i++) sum += tar[off + i] |
| 77 | + const chk = sum.toString(8).padStart(6, '0') |
| 78 | + for (let i = 0; i < 6; i++) tar[off + CHKSUM_OFF + i] = chk.charCodeAt(i) |
| 79 | + tar[off + CHKSUM_OFF + 6] = 0 |
| 80 | + tar[off + CHKSUM_OFF + 7] = 0x20 |
| 81 | +} |
| 82 | + |
| 83 | +/** |
| 84 | + * Overwrite each `replaceNames` entry in `tar` with `replaceWith(originalData)`, |
| 85 | + * in place, preserving the tar layout (the replacement is zero-padded to the |
| 86 | + * entry's existing block allocation, so the large binary after it never moves). |
| 87 | + * Throws if a replacement does not fit (true only for an unexpectedly large |
| 88 | + * package.json) or if an entry name fails `validateName`. |
| 89 | + */ |
| 90 | +export function rewriteTarEntryInPlace( |
| 91 | + tar: Uint8Array, |
| 92 | + replaceNames: Set<string>, |
| 93 | + replaceWith: (data: Uint8Array) => Uint8Array, |
| 94 | + validateName?: (name: string) => void, |
| 95 | +): void { |
| 96 | + let off = 0 |
| 97 | + while (off + BLOCK <= tar.length) { |
| 98 | + if (isZeroBlock(tar, off)) break // end-of-archive |
| 99 | + const name = readName(tar, off) |
| 100 | + validateName?.(name) |
| 101 | + const size = readSize(tar, off) |
| 102 | + const padded = Math.ceil(size / BLOCK) * BLOCK |
| 103 | + |
| 104 | + if (replaceNames.has(name)) { |
| 105 | + const replacement = replaceWith(tar.subarray(off + BLOCK, off + BLOCK + size)) |
| 106 | + if (replacement.length > padded) { |
| 107 | + throw new HttpError( |
| 108 | + 500, |
| 109 | + 'Rewritten package.json does not fit the tarball entry', |
| 110 | + ) |
| 111 | + } |
| 112 | + tar.set(replacement, off + BLOCK) |
| 113 | + tar.fill(0, off + BLOCK + replacement.length, off + BLOCK + padded) |
| 114 | + writeHeader(tar, off, replacement.length) |
| 115 | + } |
| 116 | + off += BLOCK + padded |
| 117 | + } |
| 118 | +} |
| 119 | + |
| 120 | +const CRC_TABLE = (() => { |
| 121 | + const table = new Uint32Array(256) |
| 122 | + for (let n = 0; n < 256; n++) { |
| 123 | + let c = n |
| 124 | + for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1 |
| 125 | + table[n] = c >>> 0 |
| 126 | + } |
| 127 | + return table |
| 128 | +})() |
| 129 | + |
| 130 | +/** |
| 131 | + * Emit `tar` as a valid gzip using "stored" (uncompressed) deflate blocks, |
| 132 | + * calling `emit` for each output chunk. Re-emitting stored blocks is near-zero |
| 133 | + * CPU and emits views into the (in-memory) input, so the only copies are made by |
| 134 | + * the consumer (e.g. assembling R2 multipart parts). `emit` is awaited so the |
| 135 | + * consumer can apply backpressure. |
| 136 | + */ |
| 137 | +export async function emitStoredGzip( |
| 138 | + tar: Uint8Array, |
| 139 | + emit: (chunk: Uint8Array) => Promise<void>, |
| 140 | +): Promise<void> { |
| 141 | + // gzip header: magic, deflate method, no flags, no mtime, OS unknown. |
| 142 | + await emit(new Uint8Array([0x1f, 0x8b, 0x08, 0, 0, 0, 0, 0, 0, 0xff])) |
| 143 | + |
| 144 | + let crc = 0xffffffff |
| 145 | + for (let off = 0; off < tar.length; off += 65535) { |
| 146 | + const n = Math.min(65535, tar.length - off) |
| 147 | + const seg = tar.subarray(off, off + n) |
| 148 | + for (let i = 0; i < n; i++) { |
| 149 | + crc = (crc >>> 8) ^ CRC_TABLE[(crc ^ seg[i]) & 0xff] |
| 150 | + } |
| 151 | + // Stored block header: BFINAL/BTYPE byte (0) + LEN + ~LEN, both LE. |
| 152 | + await emit(new Uint8Array([0, n & 0xff, (n >> 8) & 0xff, ~n & 0xff, (~n >> 8) & 0xff])) |
| 153 | + await emit(seg) |
| 154 | + } |
| 155 | + |
| 156 | + // Final empty stored block, then CRC32 + ISIZE (both little-endian). |
| 157 | + await emit(new Uint8Array([1, 0, 0, 0xff, 0xff])) |
| 158 | + const trailer = new Uint8Array(8) |
| 159 | + const dv = new DataView(trailer.buffer) |
| 160 | + dv.setUint32(0, (crc ^ 0xffffffff) >>> 0, true) |
| 161 | + dv.setUint32(4, tar.length >>> 0, true) |
| 162 | + await emit(trailer) |
| 163 | +} |
0 commit comments