diff --git a/Sources/tart/Commands/Save.swift b/Sources/tart/Commands/Save.swift new file mode 100644 index 00000000..0f25bfa4 --- /dev/null +++ b/Sources/tart/Commands/Save.swift @@ -0,0 +1,63 @@ +import ArgumentParser +import Foundation + +struct Save: AsyncParsableCommand { + static var configuration = CommandConfiguration(abstract: "Save a VM to an OCI archive file") + + @Argument(help: "local VM name", completion: .custom(completeMachines)) + var localName: String + + @Argument(help: "output archive path", completion: .file()) + var path: String + + @Option(help: "concurrency for disk layer compression") + var concurrency: UInt = 4 + + @Option(name: [.customLong("label")], help: ArgumentHelp("additional metadata to attach to the OCI image configuration in key=value format", + discussion: "Can be specified multiple times to attach multiple labels.")) + var labels: [String] = [] + + @Option(help: "tag to assign to the saved image (default: latest)") + var tag: String? + + func run() async throws { + let localVMDir = try VMStorageHelper.open(localName) + let lock = try localVMDir.lock() + if try !lock.trylock() { + throw RuntimeError.VMIsRunning(localName) + } + + let resolvedPath: String + if path.hasPrefix("/") { + resolvedPath = path + } else { + resolvedPath = FileManager.default.currentDirectoryPath + "/" + path + } + + try await localVMDir.saveToArchive( + path: resolvedPath, + concurrency: concurrency, + labels: parseLabels(), + tag: tag + ) + } + + func parseLabels() -> [String: String] { + var result = [String: String]() + + for label in labels { + let parts = label.trimmingCharacters(in: .whitespaces).split(separator: "=", maxSplits: 1, omittingEmptySubsequences: false) + + let key = parts.count > 0 ? String(parts[0]) : "" + let value = parts.count > 1 ? String(parts[1]) : "" + + if key.isEmpty { + continue + } + + result[key] = value + } + + return result + } +} diff --git a/Sources/tart/OCI/BlobStorage.swift b/Sources/tart/OCI/BlobStorage.swift new file mode 100644 index 00000000..c99ae323 --- /dev/null +++ b/Sources/tart/OCI/BlobStorage.swift @@ -0,0 +1,7 @@ +import Foundation + +protocol BlobStorage { + func pushBlob(fromData: Data, chunkSizeMb: Int, digest: String?) async throws -> String + func blobExists(_ digest: String) async throws -> Bool + func pushManifest(reference: String, manifest: OCIManifest) async throws -> String +} diff --git a/Sources/tart/OCI/Layerizer/Disk.swift b/Sources/tart/OCI/Layerizer/Disk.swift index 051f5439..2734b959 100644 --- a/Sources/tart/OCI/Layerizer/Disk.swift +++ b/Sources/tart/OCI/Layerizer/Disk.swift @@ -1,6 +1,6 @@ import Foundation protocol Disk { - static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, concurrency: UInt, progress: Progress) async throws -> [OCIManifestLayer] + static func push(diskURL: URL, registry: any BlobStorage, chunkSizeMb: Int, concurrency: UInt, progress: Progress) async throws -> [OCIManifestLayer] static func pull(registry: Registry, diskLayers: [OCIManifestLayer], diskURL: URL, concurrency: UInt, progress: Progress, localLayerCache: LocalLayerCache?, deduplicate: Bool) async throws } diff --git a/Sources/tart/OCI/Layerizer/DiskV2.swift b/Sources/tart/OCI/Layerizer/DiskV2.swift index 2239985d..5cdabd5a 100644 --- a/Sources/tart/OCI/Layerizer/DiskV2.swift +++ b/Sources/tart/OCI/Layerizer/DiskV2.swift @@ -22,7 +22,7 @@ class DiskV2: Disk { private static let holeGranularityBytes = 4 * 1024 * 1024 private static let zeroChunk = Data(count: holeGranularityBytes) - static func push(diskURL: URL, registry: Registry, chunkSizeMb: Int, concurrency: UInt, progress: Progress) async throws -> [OCIManifestLayer] { + static func push(diskURL: URL, registry: any BlobStorage, chunkSizeMb: Int, concurrency: UInt, progress: Progress) async throws -> [OCIManifestLayer] { var pushedLayers: [(index: Int, pushedLayer: OCIManifestLayer)] = [] // Open the disk file diff --git a/Sources/tart/OCI/Manifest.swift b/Sources/tart/OCI/Manifest.swift index 897bd973..c4664355 100644 --- a/Sources/tart/OCI/Manifest.swift +++ b/Sources/tart/OCI/Manifest.swift @@ -4,6 +4,10 @@ import Foundation let ociManifestMediaType = "application/vnd.oci.image.manifest.v1+json" let ociConfigMediaType = "application/vnd.oci.image.config.v1+json" +// Docker manifest and config media types (schema v2) +let dockerManifestMediaType = "application/vnd.docker.distribution.manifest.v2+json" +let dockerConfigMediaType = "application/vnd.docker.container.image.v1+json" + // Layer media types let configMediaType = "application/vnd.cirruslabs.tart.config.v1" let diskV2MediaType = "application/vnd.cirruslabs.tart.disk.v2" diff --git a/Sources/tart/OCI/OCIArchiveWriter.swift b/Sources/tart/OCI/OCIArchiveWriter.swift new file mode 100644 index 00000000..6c814d47 --- /dev/null +++ b/Sources/tart/OCI/OCIArchiveWriter.swift @@ -0,0 +1,119 @@ +import Foundation + +class OCIArchiveWriter { + private let tmpDir: URL + private let blobsDir: URL + private let lock: FileLock + private var manifestDigest: String? + private var manifestSize: Int? + private var manifestReferences: [String] = [] + private var manifestData: Data? + + init() throws { + tmpDir = try Config().tartTmpDir.appendingPathComponent(UUID().uuidString) + blobsDir = tmpDir.appendingPathComponent("blobs/sha256") + try FileManager.default.createDirectory(at: blobsDir, withIntermediateDirectories: true) + lock = try FileLock(lockURL: tmpDir) + if try !lock.trylock() { + throw RuntimeError.Generic("failed to lock archive staging directory") + } + } + + deinit { + try? lock.unlock() + try? FileManager.default.removeItem(at: tmpDir) + } +} + +extension OCIArchiveWriter: BlobStorage { + func pushBlob(fromData: Data, chunkSizeMb: Int, digest: String?) async throws -> String { + let resolvedDigest = digest ?? Digest.hash(fromData) + let hex = resolvedDigest.replacingOccurrences(of: "sha256:", with: "") + let blobPath = blobsDir.appendingPathComponent(hex) + try fromData.write(to: blobPath) + return resolvedDigest + } + + func blobExists(_ digest: String) async throws -> Bool { + let hex = digest.replacingOccurrences(of: "sha256:", with: "") + let blobPath = blobsDir.appendingPathComponent(hex) + return FileManager.default.fileExists(atPath: blobPath.path) + } + + func pushManifest(reference: String, manifest: OCIManifest) async throws -> String { + if let existingDigest = manifestDigest, let existingData = manifestData { + let newData = try manifest.toJSON() + if newData == existingData { + manifestReferences.append(reference) + return existingDigest + } + } + + let data = try manifest.toJSON() + let digest = Digest.hash(data) + let hex = digest.replacingOccurrences(of: "sha256:", with: "") + let blobPath = blobsDir.appendingPathComponent(hex) + try data.write(to: blobPath) + manifestDigest = digest + manifestSize = data.count + manifestData = data + manifestReferences.append(reference) + return digest + } + + func finalize(path: String, tag: String? = nil) throws { + guard let manifestDigest = manifestDigest, let manifestSize = manifestSize else { + throw RuntimeError.Generic("no manifest was pushed") + } + + let ociLayoutData = try JSONSerialization.data(withJSONObject: ["imageLayoutVersion": "1.0.0"]) + try ociLayoutData.write(to: tmpDir.appendingPathComponent("oci-layout")) + + var manifests: [[String: Any]] = [] + + let baseDescriptor: [String: Any] = [ + "mediaType": ociManifestMediaType, + "digest": manifestDigest, + "size": manifestSize, + ] + + let refs = manifestReferences.isEmpty + ? (tag.map { [$0] } ?? ["latest"]) + : manifestReferences + + for ref in refs { + var entry = baseDescriptor + entry["annotations"] = [ + "org.opencontainers.image.ref.name": ref + ] + manifests.append(entry) + } + + let index: [String: Any] = [ + "schemaVersion": 2, + "manifests": manifests + ] + + let indexData = try JSONSerialization.data(withJSONObject: index, options: [.prettyPrinted, .sortedKeys]) + try indexData.write(to: tmpDir.appendingPathComponent("index.json")) + + let absolutePath = URL(fileURLWithPath: path).path + + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/tar") + process.arguments = ["-cf", absolutePath, "-C", tmpDir.path, "."] + + let pipe = Pipe() + process.standardError = pipe + + try process.run() + process.waitUntilExit() + + if process.terminationStatus != 0 { + let errorData = pipe.fileHandleForReading.readDataToEndOfFile() + throw RuntimeError.Generic( + "creating OCI archive failed: \(String(data: errorData, encoding: .utf8) ?? "unknown error")" + ) + } + } +} diff --git a/Sources/tart/OCI/Registry.swift b/Sources/tart/OCI/Registry.swift index 25d13cb5..d26c3c5e 100644 --- a/Sources/tart/OCI/Registry.swift +++ b/Sources/tart/OCI/Registry.swift @@ -110,7 +110,7 @@ struct TokenResponse: Decodable, Authentication { } } -class Registry { +class Registry: BlobStorage { private let baseURL: URL let namespace: String let credentialsProviders: [CredentialsProvider] diff --git a/Sources/tart/Root.swift b/Sources/tart/Root.swift index bb2ed3f5..45f95719 100644 --- a/Sources/tart/Root.swift +++ b/Sources/tart/Root.swift @@ -27,6 +27,7 @@ struct Root: AsyncParsableCommand { Export.self, Prune.self, Rename.self, + Save.self, Stop.self, Delete.self, FQN.self, diff --git a/Sources/tart/VMDirectory+OCIArchive.swift b/Sources/tart/VMDirectory+OCIArchive.swift new file mode 100644 index 00000000..c3369891 --- /dev/null +++ b/Sources/tart/VMDirectory+OCIArchive.swift @@ -0,0 +1,48 @@ +import Foundation + +extension VMDirectory { + func saveToArchive(path: String, concurrency: UInt, labels: [String: String] = [:], tag: String? = nil) async throws { + let archive = try OCIArchiveWriter() + + var layers = [OCIManifestLayer]() + + let config = try VMConfig(fromURL: configURL) + var labels = labels + labels[diskFormatLabel] = config.diskFormat.rawValue + let configJSON = try JSONEncoder().encode(config) + defaultLogger.appendNewLine("saving config...") + let configDigest = try await archive.pushBlob(fromData: configJSON, chunkSizeMb: 0, digest: nil) + layers.append(OCIManifestLayer(mediaType: configMediaType, size: configJSON.count, digest: configDigest)) + + let diskSize = try FileManager.default.attributesOfItem(atPath: diskURL.path)[.size] as! Int64 + defaultLogger.appendNewLine("saving disk... this will take a while...") + let progress = Progress(totalUnitCount: diskSize) + ProgressObserver(progress).log(defaultLogger) + + layers.append(contentsOf: try await DiskV2.push(diskURL: diskURL, registry: archive, chunkSizeMb: 0, concurrency: concurrency, progress: progress)) + + defaultLogger.appendNewLine("saving NVRAM...") + let nvram = try FileHandle(forReadingFrom: nvramURL).readToEnd()! + let nvramDigest = try await archive.pushBlob(fromData: nvram, chunkSizeMb: 0, digest: nil) + layers.append(OCIManifestLayer(mediaType: nvramMediaType, size: nvram.count, digest: nvramDigest)) + + let ociConfigContainer = OCIConfig.ConfigContainer(Labels: labels) + let ociConfigJSON = try OCIConfig(architecture: config.arch, os: config.os, config: ociConfigContainer).toJSON() + let ociConfigDigest = try await archive.pushBlob(fromData: ociConfigJSON, chunkSizeMb: 0, digest: nil) + + let manifest = OCIManifest( + config: OCIManifestConfig(size: ociConfigJSON.count, digest: ociConfigDigest), + layers: layers, + uncompressedDiskSize: UInt64(diskSize), + uploadDate: Date() + ) + + let tagRef = tag ?? "latest" + defaultLogger.appendNewLine("saving manifest...") + _ = try await archive.pushManifest(reference: tagRef, manifest: manifest) + + try archive.finalize(path: path, tag: tagRef) + + defaultLogger.appendNewLine("saved to \(path)") + } +}