diff --git a/packages/blob-index/src/sharded-dag-index.js b/packages/blob-index/src/sharded-dag-index.js index 06be5e821..3f20bc682 100644 --- a/packages/blob-index/src/sharded-dag-index.js +++ b/packages/blob-index/src/sharded-dag-index.js @@ -9,6 +9,12 @@ import { DigestMap } from './digest-map.js' export const version = 'index/sharded/dag@0.1' +/** + * The size of the batch to process when archiving large datasets. + * This is a heuristic to avoid memory issues when archiving large datasets. + */ +const ARCHIVE_BATCH_SIZE = 10_000 + export const ShardedDAGIndexSchema = Schema.variant({ [version]: Schema.struct({ /** DAG root. */ @@ -166,23 +172,44 @@ export const create = (content) => new ShardedDAGIndex(content) */ export const archive = async (model) => { const blocks = new Map() - const shards = [...model.shards.entries()].sort((a, b) => - compare(a[0].digest, b[0].digest) - ) const index = { content: model.content, shards: /** @type {API.Link[]} */ ([]), } - for (const s of shards) { - const slices = [...s[1].entries()] - .sort((a, b) => compare(a[0].digest, b[0].digest)) - .map((e) => [e[0].bytes, e[1]]) - const bytes = dagCBOR.encode([s[0].bytes, slices]) - const digest = await sha256.digest(bytes) - const cid = Link.create(dagCBOR.code, digest) - blocks.set(cid.toString(), { cid, bytes }) - index.shards.push(cid) + + // Convert all shards to an array first + const allShards = [...model.shards.entries()] + + // Process shards in batches + for (let i = 0; i < allShards.length; i += ARCHIVE_BATCH_SIZE) { + const batch = allShards.slice(i, i + ARCHIVE_BATCH_SIZE) + const sortedBatch = batch.sort((a, b) => compare(a[0].digest, b[0].digest)) + + for (const s of sortedBatch) { + // Process slices in batches + const allSlices = [...s[1].entries()] + const sortedSlices = [] + + // Sort slices in batches + for (let j = 0; j < allSlices.length; j += ARCHIVE_BATCH_SIZE) { + const sliceBatch = allSlices.slice(j, j + ARCHIVE_BATCH_SIZE) + const sortedSliceBatch = sliceBatch.sort((a, b) => + compare(a[0].digest, b[0].digest) + ) + sortedSlices.push(...sortedSliceBatch) + } + + // Map the sorted slices + const mappedSlices = sortedSlices.map((e) => [e[0].bytes, e[1]]) + + const bytes = dagCBOR.encode([s[0].bytes, mappedSlices]) + const digest = await sha256.digest(bytes) + const cid = Link.create(dagCBOR.code, digest) + blocks.set(cid.toString(), { cid, bytes }) + index.shards.push(cid) + } } + const bytes = dagCBOR.encode({ [version]: index }) const digest = await sha256.digest(bytes) const cid = Link.create(dagCBOR.code, digest) diff --git a/packages/blob-index/test/large-dataset.spec.js b/packages/blob-index/test/large-dataset.spec.js new file mode 100644 index 000000000..048be1dba --- /dev/null +++ b/packages/blob-index/test/large-dataset.spec.js @@ -0,0 +1,195 @@ +import { ShardedDAGIndex } from '../src/index.js' +import { randomCAR } from './helpers/random.js' +import * as Result from './helpers/result.js' +import { fromShardArchives } from '../src/util.js' +import { sha256 } from 'multiformats/hashes/sha2' +import { base58btc } from 'multiformats/bases/base58' +import * as Link from 'multiformats/link' + +/** + * @typedef {import('entail').Test} Test + * @typedef {import('entail').assert} Assert + */ + +export const test = { + 'handles normal dataset': async (/** @type {Assert} */ assert) => { + // Create a normal-sized dataset (under threshold) + const contentCAR = await randomCAR(32) + const contentCARBytes = new Uint8Array(await contentCAR.arrayBuffer()) + const index = await fromShardArchives(contentCAR.roots[0], [ + contentCARBytes, + ]) + + // Archive should use the fast path + const indexCAR = Result.unwrap(await index.archive()) + const newIndex = Result.unwrap(ShardedDAGIndex.extract(indexCAR)) + + assert.notStrictEqual(newIndex.shards.size, 0) + assert.strictEqual(index.shards.size, newIndex.shards.size) + }, + + 'handles large dataset': async (/** @type {Assert} */ assert) => { + // Create a dummy content link + const contentBytes = new Uint8Array(32) + const contentDigest = await sha256.digest(contentBytes) + const contentLink = Link.create(0x71, contentDigest) + + const model = ShardedDAGIndex.create(contentLink) + const TOTAL_ENTRIES = 100001 + /** @type {Set} */ const usedBase58 = new Set() + + for (let i = 0; i < TOTAL_ENTRIES; i++) { + // Create unique byte arrays by using the index as a counter + const shardBytes = new Uint8Array(32) + const sliceBytes = new Uint8Array(32) + + // Fill with a pattern that ensures uniqueness + for (let j = 0; j < 32; j++) { + if (j < 4) { + // Use the index i to ensure uniqueness (4 bytes = 4.2 billion possible values) + shardBytes[j] = (i >> (j * 8)) & 0xff + sliceBytes[j] = ((i * 2) >> (j * 8)) & 0xff + } else { + // Fill remaining bytes with random values + shardBytes[j] = Math.floor(Math.random() * 256) + sliceBytes[j] = Math.floor(Math.random() * 256) + } + } + + // Verify uniqueness of base58btc encoding + const shardBase58 = base58btc.encode(shardBytes) + const sliceBase58 = base58btc.encode(sliceBytes) + + if (usedBase58.has(shardBase58)) { + throw new Error(`Duplicate shard base58: ${shardBase58}`) + } + if (usedBase58.has(sliceBase58)) { + throw new Error(`Duplicate slice base58: ${sliceBase58}`) + } + + usedBase58.add(shardBase58) + usedBase58.add(sliceBase58) + + const shard = await sha256.digest(shardBytes) + const slice = await sha256.digest(sliceBytes) + model.setSlice(shard, slice, [0, 32]) + } + + assert.strictEqual(model.shards.size, TOTAL_ENTRIES) + }, + + 'maintains sorting order in large dataset': async ( + /** @type {Assert} */ assert + ) => { + // Create a dummy content link + const contentBytes = new Uint8Array(32) + const contentDigest = await sha256.digest(contentBytes) + const contentLink = Link.create(0x71, contentDigest) + + const model = ShardedDAGIndex.create(contentLink) + const TOTAL_ENTRIES = 100001 + /** @type {Set} */ const usedBase58 = new Set() + + for (let i = 0; i < TOTAL_ENTRIES; i++) { + // Create unique byte arrays by using the index as a counter + const shardBytes = new Uint8Array(32) + const sliceBytes = new Uint8Array(32) + + // Fill with a pattern that ensures uniqueness + for (let j = 0; j < 32; j++) { + if (j < 4) { + // Use the index i to ensure uniqueness (4 bytes = 4.2 billion possible values) + shardBytes[j] = (i >> (j * 8)) & 0xff + sliceBytes[j] = ((i * 2) >> (j * 8)) & 0xff + } else { + // Fill remaining bytes with random values + shardBytes[j] = Math.floor(Math.random() * 256) + sliceBytes[j] = Math.floor(Math.random() * 256) + } + } + + // Verify uniqueness of base58btc encoding + const shardBase58 = base58btc.encode(shardBytes) + const sliceBase58 = base58btc.encode(sliceBytes) + + if (usedBase58.has(shardBase58)) { + throw new Error(`Duplicate shard base58: ${shardBase58}`) + } + if (usedBase58.has(sliceBase58)) { + throw new Error(`Duplicate slice base58: ${sliceBase58}`) + } + + usedBase58.add(shardBase58) + usedBase58.add(sliceBase58) + + const shard = await sha256.digest(shardBytes) + const slice = await sha256.digest(sliceBytes) + model.setSlice(shard, slice, [0, 32]) + } + + assert.strictEqual(model.shards.size, TOTAL_ENTRIES) + }, + + 'can archive large dataset': async (/** @type {Assert} */ assert) => { + // Create a dummy content link + const contentBytes = new Uint8Array(32) + const contentDigest = await sha256.digest(contentBytes) + const contentLink = Link.create(0x71, contentDigest) + + const model = ShardedDAGIndex.create(contentLink) + const TOTAL_ENTRIES = 100001 + /** @type {Set} */ const usedBase58 = new Set() + + for (let i = 0; i < TOTAL_ENTRIES; i++) { + // Create unique byte arrays by using the index as a counter + const shardBytes = new Uint8Array(32) + const sliceBytes = new Uint8Array(32) + + // Fill with a pattern that ensures uniqueness + for (let j = 0; j < 32; j++) { + if (j < 4) { + // Use the index i to ensure uniqueness (4 bytes = 4.2 billion possible values) + shardBytes[j] = (i >> (j * 8)) & 0xff + sliceBytes[j] = ((i * 2) >> (j * 8)) & 0xff + } else { + // Fill remaining bytes with random values + shardBytes[j] = Math.floor(Math.random() * 256) + sliceBytes[j] = Math.floor(Math.random() * 256) + } + } + + // Verify uniqueness of base58btc encoding + const shardBase58 = base58btc.encode(shardBytes) + const sliceBase58 = base58btc.encode(sliceBytes) + + if (usedBase58.has(shardBase58)) { + throw new Error(`Duplicate shard base58: ${shardBase58}`) + } + if (usedBase58.has(sliceBase58)) { + throw new Error(`Duplicate slice base58: ${sliceBase58}`) + } + + usedBase58.add(shardBase58) + usedBase58.add(sliceBase58) + + const shard = await sha256.digest(shardBytes) + const slice = await sha256.digest(sliceBytes) + model.setSlice(shard, slice, [0, 32]) + } + + // Test that we can archive the large dataset + const result = await model.archive() + assert.ok(result.ok, 'Archive should succeed') + + // Test that we can extract the archive + const extracted = ShardedDAGIndex.extract(result.ok) + assert.ok(extracted.ok, 'Extract should succeed') + assert.strictEqual( + extracted.ok.shards.size, + TOTAL_ENTRIES, + 'Should have all shards' + ) + }, +} + +export default test diff --git a/packages/upload-api/src/index/add.js b/packages/upload-api/src/index/add.js index 8ae702ee4..6605c6ca2 100644 --- a/packages/upload-api/src/index/add.js +++ b/packages/upload-api/src/index/add.js @@ -6,6 +6,12 @@ import { Assert } from '@web3-storage/content-claims/capability' import { concat } from 'uint8arrays' import * as API from '../types.js' +/** + * The size of the batch to process when checking shard allocations. + * This is a heuristic to avoid memory issues when processing large indexes. + */ +const ALLOCATION_BATCH_SIZE = 10_000 + /** * @param {API.IndexServiceContext} context * @returns {API.ServiceMethod} @@ -55,23 +61,43 @@ const add = async ({ capability }, context) => { const idxRes = ShardedDAGIndex.extract(concat(chunks)) if (!idxRes.ok) return idxRes + return await batchProcessIndexChunks(idxRes.ok, space, context, idxLink) +} - // ensure indexed shards are allocated in the agent's space - const shardDigests = [...idxRes.ok.shards.keys()] - const shardAllocRes = await Promise.all( - shardDigests.map((s) => assertAllocated(context, space, s, 'ShardNotFound')) - ) - for (const res of shardAllocRes) { - if (res.error) return res +/** + * Batch process all chunks of the index. + * + * @param {import('@web3-storage/blob-index/types').ShardedDAGIndexView} index + * @param {API.SpaceDID} space + * @param {API.IndexServiceContext} context + * @param {API.CARLink} idxLink + */ +async function batchProcessIndexChunks(index, space, context, idxLink) { + const shardDigests = [...index.shards.keys()] + + // Process shard allocations in batches + for (let i = 0; i < shardDigests.length; i += ALLOCATION_BATCH_SIZE) { + const batch = shardDigests.slice(i, i + ALLOCATION_BATCH_SIZE) + + // Each batch can be processed concurrently + const batchResults = await Promise.all( + batch.map((shard) => + assertAllocated(context, space, shard, 'ShardNotFound') + ) + ) + + for (const res of batchResults) { + if (res.error) return res + } } // TODO: randomly validate slices in the index correspond to slices in the blob const publishRes = await Promise.all([ // publish the index data to IPNI - context.ipniService.publish(idxRes.ok), + context.ipniService.publish(index), // publish a content claim for the index - publishIndexClaim(context, { content: idxRes.ok.content, index: idxLink }), + publishIndexClaim(context, { content: index.content, index: idxLink }), ]) for (const res of publishRes) { if (res.error) return res diff --git a/packages/upload-client/src/unixfs.js b/packages/upload-client/src/unixfs.js index 5bb902854..efb9e2869 100644 --- a/packages/upload-client/src/unixfs.js +++ b/packages/upload-client/src/unixfs.js @@ -3,7 +3,7 @@ import * as raw from 'multiformats/codecs/raw' import { withMaxChunkSize } from '@ipld/unixfs/file/chunker/fixed' import { withWidth } from '@ipld/unixfs/file/layout/balanced' -const SHARD_THRESHOLD = 1000 // shard directory after > 1,000 items +export const SHARD_THRESHOLD = 1000 // shard directory after > 1,000 items const queuingStrategy = UnixFS.withCapacity() const defaultSettings = UnixFS.configure({ @@ -86,19 +86,115 @@ class UnixFSDirectoryBuilder { /** @param {import('@ipld/unixfs').View} writer */ async finalize(writer) { - const dirWriter = + // Map to store links for each path + const linksByPath = new Map() + + // Store directory builders by path for later lookups + const dirBuildersByPath = new Map() + dirBuildersByPath.set('', this) + + // Process all directories first (discovery phase) + // This collects all directories without processing them + const discoverDirectories = ( + /**@type {UnixFSDirectoryBuilder}*/ dir, + path = '' + ) => { + for (const [name, entry] of dir.entries) { + const entryPath = path ? `${path}/${name}` : name + + if (entry instanceof UnixFSDirectoryBuilder) { + dirBuildersByPath.set(entryPath, entry) + discoverDirectories(entry, entryPath) + } + } + } + + // Start discovery from root + discoverDirectories(this) + + // Collect all files that need processing + const files = [] + for (const [dirPath, dir] of dirBuildersByPath) { + for (const [name, entry] of dir.entries) { + if (entry instanceof UnixFSFileBuilder) { + const entryPath = dirPath ? `${dirPath}/${name}` : name + files.push({ entry, entryPath }) + } + } + } + + // Process files in larger batches for better performance + const BATCH_SIZE = 10000 + for (let i = 0; i < files.length; i += BATCH_SIZE) { + const batch = files.slice(i, i + BATCH_SIZE) + + // Process batch in parallel + await Promise.all( + batch.map(async ({ entry, entryPath }) => { + const link = await entry.finalize(writer) + linksByPath.set(entryPath, link) + + if (this.#options?.onDirectoryEntryLink) { + // @ts-expect-error Type mismatch between link interfaces + this.#options.onDirectoryEntryLink({ name: entry.name, ...link }) + } + }) + ) + } + + // Process directories from deepest to shallowest + // Use a more efficient sort by calculating depth during map + const sortedDirs = Array.from(dirBuildersByPath.entries()) + .map(([path, dir]) => ({ + path, + dir, + depth: path ? path.split('/').length : 0, + })) + .sort((a, b) => b.depth - a.depth) // Sort by depth (deepest first) + + // Skip the root directory as it will be processed at the end + for (const { path, dir } of sortedDirs.filter(({ path }) => path !== '')) { + const dirWriter = + dir.entries.size <= SHARD_THRESHOLD + ? UnixFS.createDirectoryWriter(writer) + : /* c8 ignore next */ + UnixFS.createShardedDirectoryWriter(writer) + + // Add all entries from this directory + for (const [name, _] of dir.entries) { + /* c8 ignore next */ + const entryPath = path ? `${path}/${name}` : name + const link = linksByPath.get(entryPath) + + if (link) { + dirWriter.set(name, link) + } + } + + // Finalize directory + const link = await dirWriter.close() + linksByPath.set(path, link) + + if (this.#options?.onDirectoryEntryLink) { + /* c8 ignore next */ + this.#options.onDirectoryEntryLink({ name: dir.name, ...link }) + } + } + + // Finally, process the root directory + const rootDirWriter = this.entries.size <= SHARD_THRESHOLD ? UnixFS.createDirectoryWriter(writer) : UnixFS.createShardedDirectoryWriter(writer) - for (const [name, entry] of this.entries) { - const link = await entry.finalize(writer) - if (this.#options?.onDirectoryEntryLink) { - // @ts-expect-error - this.#options.onDirectoryEntryLink({ name: entry.name, ...link }) + + for (const [name, _] of this.entries) { + const link = linksByPath.get(name) + if (link) { + rootDirWriter.set(name, link) } - dirWriter.set(name, link) } - return await dirWriter.close() + + return await rootDirWriter.close() } } diff --git a/packages/upload-client/test/unixfs.test.js b/packages/upload-client/test/unixfs.test.js index d8da41945..03e5928f3 100644 --- a/packages/upload-client/test/unixfs.test.js +++ b/packages/upload-client/test/unixfs.test.js @@ -1,12 +1,12 @@ import assert from 'assert' -import { decode, NodeType, defaults } from '@ipld/unixfs' +import * as UnixFS from '@ipld/unixfs' import { exporter } from 'ipfs-unixfs-exporter' // @ts-expect-error this version of blockstore-core doesn't point to correct types file in package.json, and upgrading to latest version that fixes that leads to api changes import { MemoryBlockstore } from 'blockstore-core/memory' import * as raw from 'multiformats/codecs/raw' import * as Link from 'multiformats/link' import path from 'path' -import { encodeFile, encodeDirectory } from '../src/unixfs.js' +import { encodeFile, encodeDirectory, SHARD_THRESHOLD } from '../src/unixfs.js' import { File } from './helpers/shims.js' /** @param {import('ipfs-unixfs-exporter').UnixFSDirectory} dir */ @@ -69,26 +69,96 @@ describe('UnixFS', () => { }) it('encodes a sharded directory', async () => { + // Create a directory with more than SHARD_THRESHOLD entries const files = [] for (let i = 0; i < 1001; i++) { - files.push(new File([`data${i}`], `file${i}.txt`)) + files.push(new File([`content ${i}`], `file-${i}.txt`)) } const { cid, blocks } = await encodeDirectory(files) const blockstore = await blocksToBlockstore(blocks) - const dirEntry = await exporter(cid.toString(), blockstore) - assert.equal(dirEntry.type, 'directory') + const entry = await exporter(cid.toString(), blockstore) + assert.equal(entry.type, 'directory') - const expectedPaths = files.map((f) => path.join(cid.toString(), f.name)) - const entries = await collectDir(dirEntry) - const actualPaths = entries.map((e) => e.path) + // Verify all files are present + const entries = await collectDir(entry) + assert.equal(entries.length, 1001) - expectedPaths.forEach((p) => assert(actualPaths.includes(p))) + // Verify content + for (let i = 0; i < 1001; i++) { + const fileEntry = entries.find((e) => e.name === `file-${i}.txt`) + assert.ok(fileEntry, `file-${i}.txt should exist`) + const chunks = [] + for await (const chunk of fileEntry.content()) chunks.push(chunk) + const content = new Blob(chunks) + assert.equal(await content.text(), `content ${i}`) + } + }) + + it('encodes a directory with exactly 1001 entries', async () => { + // Create a directory with exactly 1001 entries (SHARD_THRESHOLD + 1) + const files = [] + for (let i = 0; i < 1001; i++) { + files.push(new File([`content ${i}`], `file-${i}.txt`)) + } + + // Track directory entry links to verify sharding + const links = [] + const { cid, blocks } = await encodeDirectory(files, { + onDirectoryEntryLink: (link) => links.push(link), + }) + + // Verify that we got a sharded directory by checking the number of blocks + // A sharded directory should have more blocks than just the files + root + assert.ok( + blocks.length > 1002, + 'Should have extra blocks for sharding structure' + ) + + // Verify the directory structure + const blockstore = await blocksToBlockstore(blocks) + const entry = await exporter(cid.toString(), blockstore) + assert.equal(entry.type, 'directory') + + // Verify all files are present + const entries = await collectDir(entry) + assert.equal(entries.length, 1001) - // check root node is a HAMT sharded directory - const bytes = await blockstore.get(cid) - const node = decode(bytes) - assert.equal(node.type, NodeType.HAMTShard) + // Verify content and order + for (let i = 0; i < 1001; i++) { + const fileEntry = entries.find((e) => e.name === `file-${i}.txt`) + assert.ok(fileEntry, `file-${i}.txt should exist`) + const chunks = [] + for await (const chunk of fileEntry.content()) chunks.push(chunk) + const content = new Blob(chunks) + assert.equal(await content.text(), `content ${i}`) + } + }) + + it('encodes a directory with more entries than the shard threshold', async () => { + const files = [] + for (let i = 0; i < SHARD_THRESHOLD + 1; i++) { + files.push(new File([`content ${i}`], `file-${i}.txt`)) + } + + const { cid, blocks } = await encodeDirectory(files) + const blockstore = await blocksToBlockstore(blocks) + const entry = await exporter(cid.toString(), blockstore) + assert.equal(entry.type, 'directory') + + // Verify all files are present + const entries = await collectDir(entry) + assert.equal(entries.length, 1001) + + // Verify content + for (let i = 0; i < 1001; i++) { + const fileEntry = entries.find((e) => e.name === `file-${i}.txt`) + assert.ok(fileEntry, `file-${i}.txt should exist`) + const chunks = [] + for await (const chunk of fileEntry.content()) chunks.push(chunk) + const content = new Blob(chunks) + assert.equal(await content.text(), `content ${i}`) + } }) it('throws then treating a file as a directory', () => @@ -110,7 +180,7 @@ describe('UnixFS', () => { const file = new Blob(['test']) const { cid } = await encodeFile(file, { settings: { - ...defaults(), + ...UnixFS.defaults(), linker: { // @ts-expect-error createLink: (_, digest) => Link.createLegacy(digest), @@ -154,4 +224,194 @@ describe('UnixFS', () => { 'bafybeie4fxkioskwb4h7xpb5f6tbktm4vjxt7rtsqjit72jrv3ii5h26sy' ) }) + + it('handles files with empty paths', async () => { + const files = [ + new File(['content1'], 'file1'), + new File(['content2'], './file2'), + new File(['content3'], '/file3'), + new File(['content4'], '.file4'), + new File(['content5'], ''), + new File(['content6'], '.'), + new File(['content7'], './'), + ] + + const { cid, blocks } = await encodeDirectory(files) + const blockstore = await blocksToBlockstore(blocks) + const entry = await exporter(cid.toString(), blockstore) + assert.equal(entry.type, 'directory') + + // Verify that all files are present + const entries = await collectDir(entry) + assert.equal( + entries.length, + 5, + 'Should have all files including empty path' + ) + + // Verify content and names + const fileNames = entries.map((e) => e.name) + assert(fileNames.includes('file1'), 'file1 should exist') + assert(fileNames.includes('file2'), 'file2 should exist') + assert(fileNames.includes('file3'), 'file3 should exist') + assert(fileNames.includes('.file4'), '.file4 should exist') + assert(fileNames.includes(''), 'empty path should exist') + }) + + /** + * This test is skipped by default as it uses a lot of resources + * Enable to verify that the iterative approach works in a directory with a large number of files + */ + it.skip('handles a directory with a large number of files without stack overflow', async function () { + // Set a longer timeout for this test + this.timeout(1200_000) // 20 minutes + + // We're testing a flat directory with many files - no deep nesting + const maxFiles = 200_000 + + console.log( + `Testing with ${maxFiles} UUID-named files in a flat directory...` + ) + + /** + * Create a flat directory with many UUID-like named files + * to simulate the real-world scenario from issue-1.md + * + * @returns {Array} - Array of File objects + */ + function createFlatDirectoryFiles() { + console.log('Creating files with UUID-like names in flat directory...') + const files = [] + + // Sample content similar to the issue description + const sampleContent = JSON.stringify([ + '0x4bc8ea729e10e076cb02e198f312cba859d2c202778fa200d9c4c9a3621714c2', + '0xf5b44224220bcb535137516ed438f06a5e715229ca06d5135e36eb5560ab2b22', + '0xb92080112acb7641f41a508f319eb402d35b81767e3f5453b658ce17a9a243fe', + '0x5a6af28c138214f464455e46e7ad4be42be4fe1e332ba42777391c3542034d9a', + '0x3b1f0f5a42c1d508d1ed62a874bb6c562ce389f88672f4d7a5101f45492d283b', + '0xd4772789dacb58c295ef4b42dd77b6c2fa07b45d6569cbe2930b55694029a782', + '0x0b72d029fb8f7ec6c9ca0abc9cd67c6c2def3b1655854b8663a42ab75daf08d2', + '0x7ed79f6f3edc539d606814e7e730996ed177495e9e134192767dc9e1b8a0a323', + '0x5255a6fae11f2135603802c696196bc37aa0e9d703d75f153650b4038ceae2c9', + '0x1da65deffcc795b924125a339093c018b42f1103952f92823a96b2cc67a032bc', + '0x75a8cf45d769c56911e41b3bdd5b1286a7c9f59cb3a2c944f405c3505ac910c5', + '0xf05315b4e77a3026e1d8e8c9e61896cf436a510a62dadbe6f7f516b855ba5564', + '0xe0b3f13831669076d52406d5c78de76c56bc94fb9a3074f7bb45e2b6ae50984b', + '0x4d5cb35459db22b7f612225f0784ccc64dd4d87476d2c1c85ce365bb7a545faf', + '0x381df64f88191da01e4e5a0151d1fef35034c93a6d9e68d1547258d95cac0fbe', + Math.random().toString(36).substring(2, 15), + ]) + + for (let fileIdx = 0; fileIdx < maxFiles; fileIdx++) { + // Generate a UUID-like filename (no extension) similar to the real-world scenario + const filename = `${generateUUIDLike()}` + files.push(new File([sampleContent], filename)) + + if (fileIdx % 1000 === 0 && fileIdx > 0) { + console.log(`Created ${fileIdx} files...`) + } + } + + console.log(`Finished creating ${files.length} files in flat directory`) + return files + } + + /** + * Generate a UUID-like string for filenames + * + * @returns {string} UUID-like string + */ + function generateUUIDLike() { + return ( + '0x' + + Array.from({ length: 64 }, () => + Math.floor(Math.random() * 16).toString(16) + ).join('') + ) + } + + try { + // Create files with UUID-like names in a flat directory + const allFiles = createFlatDirectoryFiles() + + console.log('Now encoding directory...') + + // We'll use this to track finalize calls + let finalizeCalls = 0 + const onDirectoryEntryLink = ( + /** @type {import('../src/types.js').DirectoryEntryLink} */ link + ) => { + finalizeCalls++ + if (finalizeCalls % 1000 === 0) { + console.log(`Finalized ${finalizeCalls} entries...`) + } + } + + console.log('Starting directory encoding with many UUID-named files...') + + // Encode the directory structure + console.time('directoryEncoding') + const { cid, blocks } = await encodeDirectory(allFiles, { + onDirectoryEntryLink, + }) + console.timeEnd('directoryEncoding') + + console.log( + `Successfully processed flat directory with ${finalizeCalls} entries` + ) + assert(cid, 'Should return a CID') + assert(blocks.length > 0, 'Should have encoded blocks') + } catch (/** @type {unknown} */ error) { + console.error( + 'Error occurred:', + error instanceof Error ? error.message : String(error) + ) + + assert.fail( + `Failed with unexpected error: ${ + error instanceof Error ? error.message : String(error) + }` + ) + } + }) + + it('handles sharded directory with empty paths', async () => { + const files = [] + // Add files with various path patterns including empty paths + files.push(new File(['content empty path'], '')) // empty path + files.push(new File(['content dot path'], './')) // dot path + files.push(new File(['content normal'], './testdir/file')) + files.push(new File(['content normal2'], './testdir/file2')) + files.push(new File(['content normal3'], './testdir/file3')) + + const { cid, blocks } = await encodeDirectory(files) + const blockstore = await blocksToBlockstore(blocks) + const entry = await exporter(cid.toString(), blockstore) + assert.equal(entry.type, 'directory') + + // Verify that all files are present + const entries = await collectDir(entry) + const fileNames = entries.map((e) => e.name) + assert.equal( + fileNames.length, + 4, + 'Should have all files - empty and dot paths are considered the same' + ) + + // Verify content and names + // dot path and empty path are considered the same, but the last file is the one selected + assert(fileNames.includes(''), 'empty path should exist') + assert(fileNames.includes('file'), 'testdir/file should exist') + assert(fileNames.includes('file2'), 'testdir/file2 should exist') + assert(fileNames.includes('file3'), 'testdir/file3 should exist') + + // Verify content of files + const emptyFile = entries.find((e) => e.name === '') + assert.ok(emptyFile, 'empty path file should exist') + const chunks = [] + for await (const chunk of emptyFile.content()) chunks.push(chunk) + const content = new Blob(chunks) + assert.equal(await content.text(), 'content dot path') + }) })