diff --git a/common/build.gradle.kts b/common/build.gradle.kts index 175da5a..b7d2342 100644 --- a/common/build.gradle.kts +++ b/common/build.gradle.kts @@ -41,6 +41,7 @@ dependencies { api("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.9.0") api("org.jetbrains.kotlinx:kotlinx-serialization-json:1.7.0") api("org.jetbrains.kotlinx:atomicfu:0.26.0") + testImplementation("org.jetbrains.kotlin:kotlin-test-junit:2.0.21") } tasks { diff --git a/common/src/main/kotlin/com/github/zly2006/xbackup/BackupDatabaseService.kt b/common/src/main/kotlin/com/github/zly2006/xbackup/BackupDatabaseService.kt index 7eaa69a..02ee4ca 100644 --- a/common/src/main/kotlin/com/github/zly2006/xbackup/BackupDatabaseService.kt +++ b/common/src/main/kotlin/com/github/zly2006/xbackup/BackupDatabaseService.kt @@ -14,6 +14,7 @@ import org.jetbrains.exposed.sql.transactions.TransactionManager import org.jetbrains.exposed.sql.transactions.experimental.newSuspendedTransaction import org.jetbrains.exposed.sql.transactions.transaction import org.slf4j.LoggerFactory +import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream import java.io.File import java.io.IOException @@ -61,6 +62,7 @@ class BackupDatabaseService( BackupEntryBackupTable, withLogs = false ) + ensureBlobRefTable() } } @@ -118,6 +120,12 @@ class BackupDatabaseService( val entry = reference("entry", BackupEntryTable, ReferenceOption.CASCADE).index() } + object BlobRefTable : Table("blob_refs") { + val hash = varchar("hash", 255) + val refCount = integer("ref_count") + override val primaryKey = PrimaryKey(hash) + } + @Serializable data class BackupEntry( override val id: Int, @@ -145,6 +153,10 @@ class BackupDatabaseService( suspend fun getInputStreamInternal(service: BackupDatabaseService): InputStream? { val blob = service.getBlobFile(hash) if (!blob.exists()) { + if (hash == EMPTY_BLOB_HASH) { + service.ensureEmptyBlobExists() + return ByteArrayInputStream(ByteArray(0)) + } return null } try { @@ -378,6 +390,7 @@ class BackupDatabaseService( it[this.backup] = backup.id it[this.entry] = entry.id } + incrementBlobRef(entry.hash) } // recheck val entryList = backup.entries.filter { @@ -408,12 +421,12 @@ class BackupDatabaseService( suspend fun deleteBackupInternal(backup: IBackup) { syncDbQuery { backup.entries.forEach { entry -> + decrementBlobRef(entry.hash) if (BackupEntryBackupTable.selectAll().where { BackupEntryBackupTable.entry eq entry.id and (BackupEntryBackupTable.backup neq backup.id) }.empty() ) { - getBlobFile(entry.hash).toFile().delete() BackupEntryTable.deleteWhere { id eq entry.id } @@ -567,9 +580,12 @@ class BackupDatabaseService( override fun check(backup: IBackup): Boolean { var valid = true backup.entries.forEach { - val blobFile = getBlobFile(it.hash) if (it.isDirectory) return@forEach - else if (!blobFile.exists()) { + if (it.hash == EMPTY_BLOB_HASH) { + ensureEmptyBlobExists() + } + val blobFile = getBlobFile(it.hash) + if (!blobFile.exists()) { log.error("Blob not found for file ${it.path}, hash: ${it.hash}") valid = false } @@ -701,13 +717,11 @@ class BackupDatabaseService( suspend fun deleteUnusedBlobs(): Int { val used = dbQuery { - val column = Substring(BackupEntryTable.hash, intLiteral(3), intLiteral(30)) - BackupEntryTable.select( - column // 32 -2 = 30 - ).withDistinct(true).map { row -> row[column] } - }.toSet() + BlobRefTable.selectAll().map { row -> row[BlobRefTable.hash].drop(2) }.toSet() + } + val emptyBlobName = EMPTY_BLOB_HASH.drop(2) val unused = getBlobFile("").toFile().walk().filter { it.isFile }.filterNot { - it.name in used + it.name == emptyBlobName || it.name in used }.toList() log.info("Deleting ${unused.size} unused blobs") unused.forEach { @@ -746,7 +760,83 @@ class BackupDatabaseService( TransactionManager.closeAndUnregister(database) } + private fun Transaction.blobRefsTableExists(): Boolean { + return exec("SELECT 1 FROM sqlite_master WHERE type='table' AND name='blob_refs'") { rs -> + rs.next() + } == true + } + + private fun Transaction.ensureBlobRefTable() { + if (blobRefsTableExists()) { + return + } + SchemaUtils.create(BlobRefTable) + val countCol = BackupEntryBackupTable.id.count() + (BackupEntryBackupTable innerJoin BackupEntryTable) + .select(BackupEntryTable.hash, countCol) + .where { + (BackupEntryTable.hash neq "") and (BackupEntryTable.hash neq EMPTY_BLOB_HASH) + } + .groupBy(BackupEntryTable.hash) + .forEach { row -> + BlobRefTable.insert { + it[hash] = row[BackupEntryTable.hash] + it[refCount] = row[countCol].toInt() + } + } + log.info("Initialized blob_refs from existing backups") + } + + private fun Transaction.tracksBlobRef(hash: String): Boolean { + return hash.isNotEmpty() && hash != EMPTY_BLOB_HASH + } + + private fun Transaction.incrementBlobRef(hash: String) { + if (!tracksBlobRef(hash)) return + val existing = BlobRefTable.selectAll().where { BlobRefTable.hash eq hash }.firstOrNull() + if (existing == null) { + BlobRefTable.insert { + it[this.hash] = hash + it[refCount] = 1 + } + } else { + val next = existing[BlobRefTable.refCount] + 1 + BlobRefTable.update({ BlobRefTable.hash eq hash }) { + it[refCount] = next + } + } + } + + private fun Transaction.decrementBlobRef(hash: String) { + if (!tracksBlobRef(hash)) return + val existing = BlobRefTable.selectAll().where { BlobRefTable.hash eq hash }.firstOrNull() ?: return + val next = existing[BlobRefTable.refCount] - 1 + if (next <= 0) { + getBlobFile(hash).toFile().delete() + BlobRefTable.deleteWhere { BlobRefTable.hash eq hash } + } else { + BlobRefTable.update({ BlobRefTable.hash eq hash }) { + it[refCount] = next + } + } + } + + internal fun ensureEmptyBlobExists() { + val blob = getBlobFile(EMPTY_BLOB_HASH) + if (!blob.exists()) { + blob.createParentDirectories() + blob.writeBytes(ByteArray(0)) + } + } + + internal fun blobRefCount(hash: String): Int? = transaction { + BlobRefTable.selectAll().where { BlobRefTable.hash eq hash } + .firstOrNull()?.get(BlobRefTable.refCount) + } + companion object { + const val EMPTY_BLOB_HASH = "d41d8cd98f00b204e9800998ecf8427e" + private fun ResultRow.toBackup(): Backup { val id = this[BackupTable.id].value val entries = BackupEntryBackupTable.select(BackupEntryBackupTable.entry).where { diff --git a/common/src/test/kotlin/IncrementalPruneReproduceTest.kt b/common/src/test/kotlin/IncrementalPruneReproduceTest.kt new file mode 100644 index 0000000..f434e18 --- /dev/null +++ b/common/src/test/kotlin/IncrementalPruneReproduceTest.kt @@ -0,0 +1,214 @@ +import com.github.zly2006.xbackup.BackupDatabaseService +import com.github.zly2006.xbackup.BackupDatabaseService.Companion.EMPTY_BLOB_HASH +import com.github.zly2006.xbackup.Config +import kotlinx.coroutines.runBlocking +import org.jetbrains.exposed.sql.Database +import org.jetbrains.exposed.sql.transactions.transaction +import org.junit.Test +import org.sqlite.SQLiteConfig +import org.sqlite.SQLiteDataSource +import java.nio.file.Files +import kotlin.io.path.createParentDirectories +import kotlin.io.path.exists +import kotlin.io.path.fileSize +import kotlin.io.path.writeBytes +import kotlin.io.path.writeText +import kotlin.test.assertEquals +import kotlin.test.assertNull +import kotlin.test.assertTrue + +class IncrementalPruneReproduceTest { + @Test + fun backfillsBlobRefsWhenTableMissing() = runBlocking { + val env = setupService() + val world = env.world + world.resolve("level.dat").writeBytes(ByteArray(2048) { 1 }) + world.resolve("poi/empty.mca").createParentDirectories().writeBytes(ByteArray(0)) + setStableMtime(world) + val created = env.service.createBackup(world, "full") + val backup = env.service.getBackupInternal(created.backId)!! + val contentHash = backup.entries.first { it.path == "level.dat" }.hash + + transaction(env.database) { + exec("DROP TABLE blob_refs") + } + env.service.close() + val database = connectDatabase(env.tmp) + val reopened = BackupDatabaseService( + env.world, + database, + env.blob, + Config() + ) + assertEquals(1, reopened.blobRefCount(contentHash)) + assertNull(reopened.blobRefCount(EMPTY_BLOB_HASH)) + reopened.close() + } + + @Test + fun trustsExistingBlobRefsOnStartup() = runBlocking { + val env = setupService() + val world = env.world + world.resolve("level.dat").writeBytes(ByteArray(2048) { 1 }) + setStableMtime(world) + val created = env.service.createBackup(world, "full") + val hash = env.service.getBackupInternal(created.backId)!!.entries.first { !it.isDirectory }.hash + assertEquals(1, env.service.blobRefCount(hash)) + + transaction(env.database) { + exec("UPDATE blob_refs SET ref_count = 999 WHERE hash = '$hash'") + } + val reopened = BackupDatabaseService(env.world, env.database, env.blob, Config()) + assertEquals(999, reopened.blobRefCount(hash)) + reopened.close() + } + + @Test + fun unchangedPathIncrementalsSurviveDeletingFirstBackup() = runBlocking { + val env = setupService() + val world = env.world + val service = env.service + + world.resolve("level.dat").writeBytes(ByteArray(2048) { 1 }) + world.resolve("region/r.0.0.mca").createParentDirectories() + .writeBytes(ByteArray(4096) { 2 }) + world.resolve("dup_a.txt").writeText("shared-content") + world.resolve("dup_b.txt").writeText("shared-content") + setStableMtime(world) + + val full = service.createBackup(world, "full") + world.resolve("region/r.0.0.mca").writeBytes(ByteArray(4096) { 3 }) + setStableMtime(world) + val inc1 = service.createBackup(world, "inc1") + world.resolve("level.dat").writeBytes(ByteArray(2048) { 4 }) + setStableMtime(world) + val inc2 = service.createBackup(world, "inc2") + + service.deleteBackupInternal(service.getBackupInternal(full.backId)!!) + val valid1 = service.check(service.getBackupInternal(inc1.backId)!!) + val valid2 = service.check(service.getBackupInternal(inc2.backId)!!) + assertTrue(valid1 && valid2, "unchanged-path incrementals should stay valid after deleting first backup") + } + + @Test + fun emptyPoiEntitiesNeverEnterBlobRefs() = runBlocking { + val env = setupService() + val world = env.world + val service = env.service + + world.resolve("level.dat").writeBytes(ByteArray(2048) { 1 }) + world.resolve("poi/r.-29.-26.mca").createParentDirectories().writeBytes(ByteArray(0)) + world.resolve("poi/r.-42.-23.mca").writeBytes(ByteArray(0)) + world.resolve("entities/r.-51.41.mca").createParentDirectories().writeBytes(ByteArray(0)) + setStableMtime(world) + + val full = service.createBackup(world, "full") + assertNull(service.blobRefCount(EMPTY_BLOB_HASH)) + assertTrue(service.getBlobFile(EMPTY_BLOB_HASH).exists()) + + world.resolve("poi/r.-29.-26.mca").writeBytes(ByteArray(4096) { 9 }) + setStableMtime(world) + val inc = service.createBackup(world, "inc-poi-filled") + + service.deleteBackupInternal(service.getBackupInternal(full.backId)!!) + assertTrue(service.check(service.getBackupInternal(inc.backId)!!)) + assertTrue(service.getBlobFile(EMPTY_BLOB_HASH).exists()) + assertNull(service.blobRefCount(EMPTY_BLOB_HASH)) + } + + @Test + fun renamedFileKeepsSharedHashBlob() = runBlocking { + val env = setupService() + val world = env.world + val service = env.service + + val payload = ByteArray(2048) { 7 } + world.resolve("old_name.dat").writeBytes(payload) + setStableMtime(world) + + val full = service.createBackup(world, "full") + val hash = service.getBackupInternal(full.backId)!!.entries.first { it.path == "old_name.dat" }.hash + assertEquals(1, service.blobRefCount(hash)) + + world.resolve("old_name.dat").toFile().delete() + world.resolve("new_name.dat").writeBytes(payload) + setStableMtime(world) + val inc = service.createBackup(world, "inc-renamed") + assertEquals(2, service.blobRefCount(hash)) + + service.deleteBackupInternal(service.getBackupInternal(full.backId)!!) + assertEquals(1, service.blobRefCount(hash)) + assertTrue(service.check(service.getBackupInternal(inc.backId)!!)) + assertTrue(service.getBlobFile(hash).exists()) + } + + @Test + fun restoreRecreatesMissingEmptyBlob() = runBlocking { + val env = setupService() + val world = env.world + val service = env.service + world.resolve("level.dat").writeBytes(ByteArray(2048) { 1 }) + world.resolve("poi/empty.mca").createParentDirectories().writeBytes(ByteArray(0)) + setStableMtime(world) + val created = service.createBackup(world, "full") + + service.getBlobFile(EMPTY_BLOB_HASH).toFile().delete() + assertTrue(!service.getBlobFile(EMPTY_BLOB_HASH).exists()) + + val restoreDir = env.tmp.resolve("restore") + Files.createDirectories(restoreDir) + service.restore(created.backId, restoreDir) { false } + + assertTrue(service.getBlobFile(EMPTY_BLOB_HASH).exists()) + assertEquals(0, restoreDir.resolve("poi/empty.mca").fileSize()) + } + + @Test + fun deleteUnusedBlobsSkipsEmptyHash() = runBlocking { + val env = setupService() + val world = env.world + val service = env.service + world.resolve("level.dat").writeBytes(ByteArray(2048) { 1 }) + world.resolve("poi/empty.mca").createParentDirectories().writeBytes(ByteArray(0)) + setStableMtime(world) + service.createBackup(world, "full") + assertTrue(service.getBlobFile(EMPTY_BLOB_HASH).exists()) + + service.deleteUnusedBlobs() + assertTrue(service.getBlobFile(EMPTY_BLOB_HASH).exists()) + } + + private fun setupService(): Env { + val tmp = Files.createTempDirectory("xb-prune") + val world = tmp.resolve("world") + val blob = tmp.resolve("blob") + Files.createDirectories(world) + Files.createDirectories(blob) + val database = connectDatabase(tmp) + return Env(tmp, world, blob.normalize().toAbsolutePath(), database, BackupDatabaseService(world, database, blob.normalize().toAbsolutePath(), Config())) + } + + private fun connectDatabase(tmp: java.nio.file.Path): Database { + return Database.connect( + SQLiteDataSource( + SQLiteConfig().apply { enforceForeignKeys(true) } + ).apply { + url = "jdbc:sqlite:${tmp.resolve("x_backup.db")}" + } + ) + } + + private fun setStableMtime(world: java.nio.file.Path) { + val t = System.currentTimeMillis() + val stable = if (t % 1000L == 0L) t + 1 else t + world.toFile().walk().filter { it.isFile }.forEach { it.setLastModified(stable) } + } + + private data class Env( + val tmp: java.nio.file.Path, + val world: java.nio.file.Path, + val blob: java.nio.file.Path, + val database: Database, + val service: BackupDatabaseService, + ) +}