diff --git a/Example/Example/RealityKitViewController.swift b/Example/Example/RealityKitViewController.swift index 5439d6b..61fa4e9 100644 --- a/Example/Example/RealityKitViewController.swift +++ b/Example/Example/RealityKitViewController.swift @@ -1,6 +1,7 @@ import Combine import UIKit import RealityKit +import simd internal import VRMKit internal import VRMRealityKit @@ -9,8 +10,10 @@ final class RealityKitViewController: UIViewController, UIGestureRecognizerDeleg private var arView: ARView? private var updateSubscription: Cancellable? private var loadedEntity: VRMEntity? + private var loadedAnchor: AnchorEntity? private var cameraAnchor: AnchorEntity? private var cameraEntity: PerspectiveCamera? + private var lightEntity: DirectionalLight? private var expressionSegmentedControl: UISegmentedControl? private var orbitYaw: Float = 0 private var orbitPitch: Float = -0.1 @@ -18,11 +21,12 @@ final class RealityKitViewController: UIViewController, UIGestureRecognizerDeleg private var orbitTarget = SIMD3(0, 0.8, 0) private var currentModel: VRMExampleModel = .alicia private var currentExpression: ExampleExpression = .neutral + private var isMToonEnabled = true override func viewDidLoad() { super.viewDidLoad() title = "RealityKit" - view.backgroundColor = .black + view.backgroundColor = .white setUpARView() setUpUI() loadVRM(model: .alicia) @@ -31,7 +35,7 @@ final class RealityKitViewController: UIViewController, UIGestureRecognizerDeleg private func setUpARView() { let arView = ARView(frame: .zero, cameraMode: .nonAR, automaticallyConfigureSession: false) arView.translatesAutoresizingMaskIntoConstraints = false - arView.environment.background = .color(.black) + arView.environment.background = .color(.white) view.addSubview(arView) NSLayoutConstraint.activate([ @@ -61,12 +65,25 @@ final class RealityKitViewController: UIViewController, UIGestureRecognizerDeleg expressionSegmentedControl.translatesAutoresizingMaskIntoConstraints = false view.addSubview(expressionSegmentedControl) self.expressionSegmentedControl = expressionSegmentedControl - + + let mtoonLabel = UILabel() + mtoonLabel.text = "MToon" + let mtoonSwitch = UISwitch() + mtoonSwitch.isOn = isMToonEnabled + mtoonSwitch.addTarget(self, action: #selector(mtoonChanged(_:)), for: .valueChanged) + let mtoonControl = UIStackView(arrangedSubviews: [mtoonLabel, mtoonSwitch]) + mtoonControl.axis = .horizontal + mtoonControl.spacing = 8 + mtoonControl.translatesAutoresizingMaskIntoConstraints = false + view.addSubview(mtoonControl) + NSLayoutConstraint.activate([ segmentedControl.centerXAnchor.constraint(equalTo: view.centerXAnchor), segmentedControl.bottomAnchor.constraint(equalTo: view.safeAreaLayoutGuide.bottomAnchor, constant: -50), expressionSegmentedControl.centerXAnchor.constraint(equalTo: view.centerXAnchor), - expressionSegmentedControl.bottomAnchor.constraint(equalTo: segmentedControl.topAnchor, constant: -20) + expressionSegmentedControl.bottomAnchor.constraint(equalTo: segmentedControl.topAnchor, constant: -20), + mtoonControl.centerXAnchor.constraint(equalTo: view.centerXAnchor), + mtoonControl.bottomAnchor.constraint(equalTo: expressionSegmentedControl.topAnchor, constant: -16) ]) } @@ -82,29 +99,40 @@ final class RealityKitViewController: UIViewController, UIGestureRecognizerDeleg loadedEntity?.setExampleExpression(currentExpression, value: 1.0) } + @objc private func mtoonChanged(_ sender: UISwitch) { + isMToonEnabled = sender.isOn + loadVRM(model: currentModel) + } + private func loadVRM(model: VRMExampleModel) { guard let arView = arView else { return } currentModel = model updateExpressionLabels() - if let loadedEntity = loadedEntity { + if let loadedEntity { loadedEntity.entity.removeFromParent() self.loadedEntity = nil } + if let loadedAnchor { + arView.scene.removeAnchor(loadedAnchor) + self.loadedAnchor = nil + } do { - let loader = try VRMEntityLoader(named: model.rawValue) + let loader = try VRMEntityLoader(named: model.rawValue, isMToonEnabled: isMToonEnabled) let vrmEntity = try loader.loadEntity() + vrmEntity.setMToonLightDirection(RealityKitExampleLighting.direction) let anchor = AnchorEntity(world: .zero) vrmEntity.entity.transform.translation = SIMD3(0, -1.0, -1.5) anchor.addChild(vrmEntity.entity) arView.scene.addAnchor(anchor) + setUpLight(in: arView) normalizeScale(for: vrmEntity.entity) updateOrbitTarget(for: vrmEntity.entity, adjustDistance: false) updateCameraTransform() - + let neck = vrmEntity.humanoid.node(for: .neck) let leftArm: Entity? let rightArm: Entity? @@ -116,7 +144,7 @@ final class RealityKitViewController: UIViewController, UIGestureRecognizerDeleg leftArm = vrmEntity.humanoid.node(for: .leftUpperArm) rightArm = vrmEntity.humanoid.node(for: .rightUpperArm) } - + let neckRotation = simd_quatf(angle: 20 * .pi / 180, axis: SIMD3(0, 0, 1)) let armRotation = simd_quatf(angle: 40 * .pi / 180, axis: SIMD3(0, 0, 1)) if let neck { @@ -129,17 +157,18 @@ final class RealityKitViewController: UIViewController, UIGestureRecognizerDeleg rightArm.transform.rotation = rightArm.transform.rotation * armRotation } vrmEntity.setExampleExpression(currentExpression, value: 1.0) - + loadedEntity = vrmEntity - + loadedAnchor = anchor + let rotationOffset = model.initialRotation var time: TimeInterval = 0 updateSubscription = arView.scene.subscribe(to: SceneEvents.Update.self) { [weak self] event in guard let loadedEntity = self?.loadedEntity else { return } - + time += event.deltaTime - + let cycle = time.truncatingRemainder(dividingBy: 1.0) let angle: Float if cycle < 0.5 { @@ -149,10 +178,10 @@ final class RealityKitViewController: UIViewController, UIGestureRecognizerDeleg let progress = Float(cycle - 0.5) / 0.5 angle = -0.5 + 0.5 * progress } - + loadedEntity.entity.transform.rotation = simd_quatf(angle: rotationOffset + angle, axis: SIMD3(0, 1, 0)) - - loadedEntity.update(at: event.deltaTime) + + loadedEntity.update(deltaTime: event.deltaTime) } } catch { print(error) @@ -182,6 +211,19 @@ final class RealityKitViewController: UIViewController, UIGestureRecognizerDeleg updateCameraTransform() } + private func setUpLight(in arView: ARView) { + if lightEntity != nil { return } + let lightAnchor = AnchorEntity(world: .zero) + let light = DirectionalLight() + light.light.intensity = 1200 + light.look(at: .zero, + from: -RealityKitExampleLighting.direction, + relativeTo: nil) + lightAnchor.addChild(light) + arView.scene.addAnchor(lightAnchor) + lightEntity = light + } + private func setUpGestures() { guard let arView = arView else { return } @@ -249,7 +291,7 @@ final class RealityKitViewController: UIViewController, UIGestureRecognizerDeleg @objc private func handlePan(_ gesture: UIPanGestureRecognizer) { guard let arView = arView, let cameraEntity = cameraEntity else { return } let translation = gesture.translation(in: arView) - let panSpeed: Float = 0.002 * orbitDistance + let panSpeed = Float(0.002) * orbitDistance let transform = cameraEntity.transform.matrix let right = SIMD3(transform.columns.0.x, transform.columns.0.y, transform.columns.0.z) @@ -275,3 +317,7 @@ final class RealityKitViewController: UIViewController, UIGestureRecognizerDeleg return true } } + +private enum RealityKitExampleLighting { + static let direction = simd_normalize(SIMD3(0, 0, -1)) +} diff --git a/Example/Example/ViewController.swift b/Example/Example/ViewController.swift index 519f4bc..7705c44 100644 --- a/Example/Example/ViewController.swift +++ b/Example/Example/ViewController.swift @@ -1,5 +1,6 @@ import UIKit import SceneKit +import simd internal import VRMSceneKit class ViewController: UIViewController { @@ -12,12 +13,12 @@ class ViewController: UIViewController { scnView.backgroundColor = UIColor.black } } - + private var vrmNode: VRMNode? private var expressionSegmentedControl: UISegmentedControl? private var currentModel: VRMExampleModel = .alicia private var currentExpression: ExampleExpression = .neutral - + override func viewDidLoad() { super.viewDidLoad() setupUI() @@ -31,7 +32,7 @@ class ViewController: UIViewController { segmentedControl.addTarget(self, action: #selector(segmentChanged(_:)), for: .valueChanged) segmentedControl.translatesAutoresizingMaskIntoConstraints = false view.addSubview(segmentedControl) - + let expressionItems = ExampleExpression.allCases.map { $0.displayName(for: currentModel) } let expressionSegmentedControl = UISegmentedControl(items: expressionItems) expressionSegmentedControl.selectedSegmentIndex = 0 @@ -39,11 +40,11 @@ class ViewController: UIViewController { expressionSegmentedControl.translatesAutoresizingMaskIntoConstraints = false view.addSubview(expressionSegmentedControl) self.expressionSegmentedControl = expressionSegmentedControl - + NSLayoutConstraint.activate([ segmentedControl.centerXAnchor.constraint(equalTo: view.centerXAnchor), segmentedControl.bottomAnchor.constraint(equalTo: view.safeAreaLayoutGuide.bottomAnchor, constant: -50), - + expressionSegmentedControl.centerXAnchor.constraint(equalTo: view.centerXAnchor), expressionSegmentedControl.bottomAnchor.constraint(equalTo: segmentedControl.topAnchor, constant: -20) ]) @@ -53,7 +54,7 @@ class ViewController: UIViewController { let model = VRMExampleModel.allCases[sender.selectedSegmentIndex] loadVRM(model: model) } - + @objc private func expressionSegmentChanged(_ sender: UISegmentedControl) { let expression = ExampleExpression.allCases[sender.selectedSegmentIndex] vrmNode?.setExampleExpression(currentExpression, value: 0.0) @@ -72,12 +73,11 @@ class ViewController: UIViewController { scnView.delegate = self let node = scene.vrmNode self.vrmNode = node - + let rotationOffset = CGFloat(model.initialRotation) node.eulerAngles = SCNVector3(0, rotationOffset, 0) - node.setExampleExpression(currentExpression, value: 1.0) - + node.humanoid.node(for: .neck)?.eulerAngles = SCNVector3(0, 0, 20 * CGFloat.pi / 180) let leftArm: SCNNode? let rightArm: SCNNode? @@ -91,7 +91,7 @@ class ViewController: UIViewController { } leftArm?.eulerAngles = SCNVector3(0, 0, 40 * CGFloat.pi / 180) rightArm?.eulerAngles = SCNVector3(0, 0, 40 * CGFloat.pi / 180) - + node.runAction(SCNAction.repeatForever(SCNAction.sequence([ SCNAction.rotateBy(x: 0, y: -0.5, z: 0, duration: 0.5), SCNAction.rotateBy(x: 0, y: 0.5, z: 0, duration: 0.5), @@ -120,9 +120,21 @@ class ViewController: UIViewController { cameraNode.position = SCNVector3(0, 0.8, -1.6) cameraNode.rotation = SCNVector4(0, 1, 0, Float.pi) + + let lightNode = SCNNode() + lightNode.light = SCNLight() + lightNode.light?.type = .directional + lightNode.light?.intensity = 1200 + lightNode.simdPosition = -SceneKitExampleLighting.direction + lightNode.look(at: SCNVector3Zero) + scene.rootNode.addChildNode(lightNode) } } +private enum SceneKitExampleLighting { + static let direction = simd_normalize(SIMD3(0.35, 0.55, 0.75)) +} + @available(*, deprecated, message: "Deprecated. Use VRMRealityKit instead.") extension ViewController: SCNSceneRendererDelegate { nonisolated func renderer(_ renderer: SCNSceneRenderer, updateAtTime time: TimeInterval) { diff --git a/Example/MacExample/ContentView.swift b/Example/MacExample/ContentView.swift index 1e15c2a..b448ab3 100644 --- a/Example/MacExample/ContentView.swift +++ b/Example/MacExample/ContentView.swift @@ -20,7 +20,10 @@ struct ContentView: View { @State private var selectedRenderer: MacExampleRenderer = .realityKit @State private var selectedModel: MacExampleModel = .alicia @State private var selectedExpression: MacExampleExpression = .neutral - + @State private var isMToonEnabled = true + @State private var hasShownSceneKit = false + @State private var hasShownRealityKit = true + var body: some View { VStack { HStack { @@ -44,18 +47,40 @@ struct ContentView: View { } } .pickerStyle(.segmented) + + Toggle("MToon", isOn: $isMToonEnabled) + .toggleStyle(.switch) + .disabled(selectedRenderer != .realityKit) } .padding([.top, .horizontal]) - switch selectedRenderer { - case .sceneKit: - SceneKitRendererView(viewModel: sceneKitViewModel, - selectedModel: selectedModel, - selectedExpression: selectedExpression) - case .realityKit: - RealityKitRendererView(viewModel: realityKitViewModel, - selectedModel: selectedModel, - selectedExpression: selectedExpression) + ZStack { + if hasShownSceneKit { + SceneKitRendererView(viewModel: sceneKitViewModel, + selectedModel: selectedModel, + selectedExpression: selectedExpression) + .opacity(selectedRenderer == .sceneKit ? 1 : 0) + .allowsHitTesting(selectedRenderer == .sceneKit) + .zIndex(selectedRenderer == .sceneKit ? 1 : 0) + } + + if hasShownRealityKit { + RealityKitRendererView(viewModel: realityKitViewModel, + selectedModel: selectedModel, + selectedExpression: selectedExpression, + isMToonEnabled: isMToonEnabled) + .opacity(selectedRenderer == .realityKit ? 1 : 0) + .allowsHitTesting(selectedRenderer == .realityKit) + .zIndex(selectedRenderer == .realityKit ? 1 : 0) + } + } + .onChange(of: selectedRenderer) { _, renderer in + switch renderer { + case .sceneKit: + hasShownSceneKit = true + case .realityKit: + hasShownRealityKit = true + } } } .frame(minWidth: 800, minHeight: 600) @@ -66,14 +91,26 @@ private struct RealityKitRendererView: View { let viewModel: RealityKitContentViewModel let selectedModel: MacExampleModel let selectedExpression: MacExampleExpression + let isMToonEnabled: Bool + + private var loadConfiguration: RealityKitLoadConfiguration { + RealityKitLoadConfiguration(model: selectedModel, isMToonEnabled: isMToonEnabled) + } var body: some View { RealityView { content in - content.add(viewModel.rootEntity) + content.add(viewModel.makeRenderRootEntity()) } + .background(Color.white) .frame(maxWidth: .infinity, maxHeight: .infinity) - .task(id: selectedModel) { - await viewModel.loadEntity(model: selectedModel, expression: selectedExpression) + .task(id: loadConfiguration) { + await viewModel.loadEntity(model: selectedModel, + expression: selectedExpression, + isMToonEnabled: isMToonEnabled, + forceReload: true) + } + .onAppear { + viewModel.resumeUpdates() } .onChange(of: selectedExpression) { _, expression in viewModel.setExpression(expression) @@ -89,6 +126,11 @@ private struct RealityKitRendererView: View { } } +private struct RealityKitLoadConfiguration: Hashable { + let model: MacExampleModel + let isMToonEnabled: Bool +} + private struct SceneKitRendererView: View { let viewModel: SceneKitContentViewModel let selectedModel: MacExampleModel @@ -100,6 +142,9 @@ private struct SceneKitRendererView: View { .task(id: selectedModel) { await viewModel.loadScene(model: selectedModel, expression: selectedExpression) } + .onAppear { + viewModel.resumeUpdates() + } .onChange(of: selectedExpression) { _, expression in viewModel.setExpression(expression) } @@ -127,44 +172,78 @@ private struct ErrorMessageView: View { @MainActor @Observable final class RealityKitContentViewModel { - let rootEntity = Entity() + private var rootEntity = Entity() private(set) var errorMessage: String? private var vrmEntity: VRMEntity? + private var cameraEntity: PerspectiveCamera? + private var lightEntity: DirectionalLight? private var time: TimeInterval = 0 private var lastUpdateTime: Date? private var currentModel: MacExampleModel = .alicia private var currentExpression: MacExampleExpression = .neutral - + private var orbitDistance: Float = 2 + private var orbitTarget = SIMD3(0, 0.8, 0) + let updateTimer = Timer.publish(every: 1.0 / 60.0, on: .main, in: .common).autoconnect() - - func loadEntity(model: MacExampleModel, expression: MacExampleExpression) async { + + func makeRenderRootEntity() -> Entity { + let nextRootEntity = Entity() + if let cameraEntity { + nextRootEntity.addChild(cameraEntity) + } + if let lightEntity { + nextRootEntity.addChild(lightEntity) + } + if let vrmEntity { + nextRootEntity.addChild(vrmEntity.entity) + } + rootEntity = nextRootEntity + return nextRootEntity + } + + func loadEntity( + model: MacExampleModel, + expression: MacExampleExpression, + isMToonEnabled: Bool, + forceReload: Bool = false + ) async { + if !forceReload, currentModel == model, let vrmEntity { + apply(expression, to: vrmEntity) + currentExpression = expression + resumeUpdates() + return + } + + await Task.yield() + guard !Task.isCancelled else { return } + do { errorMessage = nil - if let vrmEntity { - vrmEntity.entity.removeFromParent() - self.vrmEntity = nil - } - let loader = try VRMEntityLoader(named: model.rawValue) - let vrmEntity = try loader.loadEntity() - - vrmEntity.entity.transform.translation = SIMD3(0, -1, 0) - vrmEntity.entity.transform.rotation = simd_quatf(angle: model.initialRotation, axis: SIMD3(0, 1, 0)) - rootEntity.addChild(vrmEntity.entity) - - // Adjust pose - let neck = vrmEntity.humanoid.node(for: .neck) + let loader = try VRMEntityLoader(named: model.rawValue, isMToonEnabled: isMToonEnabled) + let nextVRMEntity = try loader.loadEntity() + + nextVRMEntity.entity.transform.translation = SIMD3(0, -1, 0) + nextVRMEntity.entity.transform.rotation = simd_quatf(angle: model.initialRotation, axis: SIMD3(0, 1, 0)) + nextVRMEntity.setMToonLightDirection(MacExampleLighting.realityKitDirection) + setUpCamera() + setUpLight() + rootEntity.addChild(nextVRMEntity.entity) + normalizeScale(for: nextVRMEntity.entity) + updateCameraTransform() + + let neck = nextVRMEntity.humanoid.node(for: .neck) let leftArm: Entity? let rightArm: Entity? - switch vrmEntity.vrm { + switch nextVRMEntity.vrm { case .v1: - leftArm = vrmEntity.humanoid.node(for: .leftShoulder) - rightArm = vrmEntity.humanoid.node(for: .rightShoulder) + leftArm = nextVRMEntity.humanoid.node(for: .leftShoulder) + rightArm = nextVRMEntity.humanoid.node(for: .rightShoulder) case .v0: - leftArm = vrmEntity.humanoid.node(for: .leftUpperArm) - rightArm = vrmEntity.humanoid.node(for: .rightUpperArm) + leftArm = nextVRMEntity.humanoid.node(for: .leftUpperArm) + rightArm = nextVRMEntity.humanoid.node(for: .rightUpperArm) } - + let neckRotation = simd_quatf(angle: 20 * .pi / 180, axis: SIMD3(0, 0, 1)) let armRotation = simd_quatf(angle: 40 * .pi / 180, axis: SIMD3(0, 0, 1)) if let neck { @@ -176,12 +255,15 @@ final class RealityKitContentViewModel { if let rightArm { rightArm.transform.rotation = rightArm.transform.rotation * armRotation } - vrmEntity.setExampleExpression(expression, value: 1.0) - - self.vrmEntity = vrmEntity + apply(expression, to: nextVRMEntity) + + let previousVRMEntity = self.vrmEntity + self.vrmEntity = nextVRMEntity + previousVRMEntity?.entity.removeFromParent() self.currentModel = model self.currentExpression = expression - self.lastUpdateTime = Date() + self.time = 0 + resumeUpdates() } catch { errorMessage = error.localizedDescription print("VRM Load Error: \(error)") @@ -190,21 +272,24 @@ final class RealityKitContentViewModel { func setExpression(_ expression: MacExampleExpression) { guard expression != currentExpression else { return } - vrmEntity?.setExampleExpression(currentExpression, value: 0.0) currentExpression = expression - vrmEntity?.setExampleExpression(expression, value: 1.0) + guard let vrmEntity else { return } + apply(expression, to: vrmEntity) } - + + func resumeUpdates() { + lastUpdateTime = Date() + } + func update() { guard let vrmEntity else { return } - + let now = Date() let deltaTime = lastUpdateTime.map { now.timeIntervalSince($0) } ?? (1.0 / 60.0) lastUpdateTime = now - + time += deltaTime - - // An animation that sways left and right + let cycle = time.truncatingRemainder(dividingBy: 1.0) let angle: Float if cycle < 0.5 { @@ -214,10 +299,65 @@ final class RealityKitContentViewModel { let progress = Float(cycle - 0.5) / 0.5 angle = -0.5 + 0.5 * progress } - + vrmEntity.entity.transform.rotation = simd_quatf(angle: currentModel.initialRotation + angle, axis: SIMD3(0, 1, 0)) - vrmEntity.update(at: deltaTime) + vrmEntity.update(deltaTime: deltaTime) + } + + private func setUpLight() { + if lightEntity == nil { + let light = DirectionalLight() + light.light.intensity = 1200 + rootEntity.addChild(light) + lightEntity = light + } + lightEntity?.look(at: .zero, + from: -MacExampleLighting.realityKitDirection, + relativeTo: nil) + } + + private func setUpCamera() { + if cameraEntity == nil { + let camera = PerspectiveCamera() + rootEntity.addChild(camera) + cameraEntity = camera + } + updateCameraTransform() + } + + private func normalizeScale(for entity: Entity) { + let bounds = entity.visualBounds(relativeTo: nil) + let height = bounds.max.y - bounds.min.y + guard height > 0.001 else { return } + let targetHeight: Float = 2 + entity.transform.scale = SIMD3(repeating: targetHeight / height) + updateOrbitTarget(for: entity) + } + + private func updateOrbitTarget(for entity: Entity) { + let bounds = entity.visualBounds(relativeTo: nil) + orbitTarget = (bounds.min + bounds.max) * 0.5 + let extents = bounds.max - bounds.min + let maxExtent = max(extents.x, max(extents.y, extents.z)) + orbitDistance = max(0.2, maxExtent * 1.5) + } + + private func updateCameraTransform() { + guard let cameraEntity else { return } + let position = orbitTarget + SIMD3(0, 0, -orbitDistance) + cameraEntity.look(at: orbitTarget, from: position, relativeTo: nil) + } + + private func apply(_ expression: MacExampleExpression, to vrmEntity: VRMEntity) { + resetExpressions(on: vrmEntity) + vrmEntity.setExampleExpression(expression, value: 1.0) + } + + private func resetExpressions(on vrmEntity: VRMEntity) { + for expression in MacExampleExpression.allCases { + vrmEntity.setExampleExpression(expression, value: 0.0) + } } } @@ -252,26 +392,34 @@ final class SceneKitContentViewModel { let updateTimer = Timer.publish(every: 1.0 / 60.0, on: .main, in: .common).autoconnect() func loadScene(model: MacExampleModel, expression: MacExampleExpression) async { + if currentModel == model, let vrmNode { + apply(expression, to: vrmNode) + currentExpression = expression + resumeUpdates() + return + } + + await Task.yield() + guard !Task.isCancelled else { return } + do { errorMessage = nil - scene = nil - vrmNode = nil - time = 0 let loader = try VRMSceneLoader(named: model.rawValue) let scene = try loader.loadScene() setUpCamera(in: scene) let node = scene.vrmNode - node.eulerAngles = SCNVector3(0, CGFloat(model.sceneKitInitialRotation), 0) + node.eulerAngles = SCNVector3(0, CGFloat(model.initialRotation), 0) applyPose(to: node) - node.setExampleExpression(expression, value: 1.0) + apply(expression, to: node) self.scene = scene self.vrmNode = node self.currentModel = model self.currentExpression = expression - self.lastUpdateTime = Date() + self.time = 0 + resumeUpdates() } catch { errorMessage = error.localizedDescription print("VRM Load Error: \(error)") @@ -280,9 +428,13 @@ final class SceneKitContentViewModel { func setExpression(_ expression: MacExampleExpression) { guard expression != currentExpression else { return } - vrmNode?.setExampleExpression(currentExpression, value: 0.0) currentExpression = expression - vrmNode?.setExampleExpression(expression, value: 1.0) + guard let vrmNode else { return } + apply(expression, to: vrmNode) + } + + func resumeUpdates() { + lastUpdateTime = Date() } func update() { @@ -304,7 +456,7 @@ final class SceneKitContentViewModel { angle = -0.5 + 0.5 * progress } - vrmNode.eulerAngles = SCNVector3(0, CGFloat(currentModel.sceneKitInitialRotation + angle), 0) + vrmNode.eulerAngles = SCNVector3(0, CGFloat(currentModel.initialRotation + angle), 0) vrmNode.update(at: time) } @@ -331,9 +483,32 @@ final class SceneKitContentViewModel { cameraNode.position = SCNVector3(0, 0.8, -1.6) cameraNode.rotation = SCNVector4(0, 1, 0, Float.pi) scene.rootNode.addChildNode(cameraNode) + + let lightNode = SCNNode() + lightNode.light = SCNLight() + lightNode.light?.type = .directional + lightNode.light?.intensity = 1200 + lightNode.simdPosition = -MacExampleLighting.direction + lightNode.look(at: SCNVector3Zero) + scene.rootNode.addChildNode(lightNode) + } + + private func apply(_ expression: MacExampleExpression, to vrmNode: VRMNode) { + resetExpressions(on: vrmNode) + vrmNode.setExampleExpression(expression, value: 1.0) + } + + private func resetExpressions(on vrmNode: VRMNode) { + for expression in MacExampleExpression.allCases { + vrmNode.setExampleExpression(expression, value: 0.0) + } } } +private enum MacExampleLighting { + static let direction = simd_normalize(SIMD3(0.35, 0.55, 0.75)) + static let realityKitDirection = simd_normalize(SIMD3(0, 0, -1)) +} #Preview { ContentView() diff --git a/Example/MacExample/MacExampleModel.swift b/Example/MacExample/MacExampleModel.swift index 0c3ba82..6bace4a 100644 --- a/Example/MacExample/MacExampleModel.swift +++ b/Example/MacExample/MacExampleModel.swift @@ -32,13 +32,6 @@ enum MacExampleModel: String, CaseIterable, Identifiable { } var initialRotation: Float { - switch self { - case .alicia: return .pi - case .vrm1: return 0 - } - } - - var sceneKitInitialRotation: Float { switch self { case .alicia: return 0 case .vrm1: return .pi diff --git a/Example/VisionExample/ContentView.swift b/Example/VisionExample/ContentView.swift index a6dcd1d..c20d6f3 100644 --- a/Example/VisionExample/ContentView.swift +++ b/Example/VisionExample/ContentView.swift @@ -21,6 +21,10 @@ struct MainView: View { .pickerStyle(.segmented) .disabled(appModel.immersiveSpaceState == .inTransition) + Toggle("MToon", isOn: $appModel.isMToonEnabled) + .toggleStyle(.switch) + .disabled(appModel.immersiveSpaceState == .inTransition) + Button { Task { switch appModel.immersiveSpaceState { @@ -50,17 +54,18 @@ struct ImmersiveView: View { @Environment(AppModel.self) private var appModel @State private var viewModel = ImmersiveViewModel() + private var loadConfiguration: ImmersiveLoadConfiguration { + ImmersiveLoadConfiguration(model: appModel.selectedModelName, + isMToonEnabled: appModel.isMToonEnabled) + } + var body: some View { RealityView { content in content.add(viewModel.rootEntity) } - .task { - await viewModel.loadEntity(model: appModel.selectedModelName) - } - .onChange(of: appModel.selectedModelName) { _, newValue in - Task { - await viewModel.loadEntity(model: newValue) - } + .task(id: loadConfiguration) { + await viewModel.loadEntity(model: appModel.selectedModelName, + isMToonEnabled: appModel.isMToonEnabled) } .onReceive(viewModel.updateTimer) { _ in viewModel.update() @@ -68,6 +73,11 @@ struct ImmersiveView: View { } } +private struct ImmersiveLoadConfiguration: Hashable { + let model: AppModel.ModelName + let isMToonEnabled: Bool +} + @MainActor @Observable final class ImmersiveViewModel { @@ -81,7 +91,7 @@ final class ImmersiveViewModel { let updateTimer = Timer.publish(every: 1.0 / 60.0, on: .main, in: .common).autoconnect() - func loadEntity(model: AppModel.ModelName) async { + func loadEntity(model: AppModel.ModelName, isMToonEnabled: Bool) async { let modelName = model.rawValue // Clean up previous @@ -94,7 +104,7 @@ final class ImmersiveViewModel { baseRotation = model.initialRotation do { - let loader = try VRMEntityLoader(named: modelName) + let loader = try VRMEntityLoader(named: modelName, isMToonEnabled: isMToonEnabled) let vrmEntity = try loader.loadEntity() vrmEntity.entity.transform.translation = SIMD3(0, 0, -1.5) @@ -156,6 +166,6 @@ final class ImmersiveViewModel { } vrmEntity.entity.transform.rotation = simd_quatf(angle: baseRotation + angle, axis: SIMD3(0, 1, 0)) - vrmEntity.update(at: deltaTime) + vrmEntity.update(deltaTime: deltaTime) } } diff --git a/Example/VisionExample/VisionExampleApp.swift b/Example/VisionExample/VisionExampleApp.swift index 150e5ba..e93f121 100644 --- a/Example/VisionExample/VisionExampleApp.swift +++ b/Example/VisionExample/VisionExampleApp.swift @@ -29,6 +29,7 @@ struct VisionExampleApp: App { final class AppModel { let immersiveSpaceID = "ImmersiveSpace" var immersiveSpaceState: ImmersiveSpaceState = .closed + var isMToonEnabled = true enum ImmersiveSpaceState { case closed, inTransition, open diff --git a/Package.swift b/Package.swift index 9f85092..c444949 100644 --- a/Package.swift +++ b/Package.swift @@ -21,7 +21,11 @@ let package = Package( ), .target( name: "VRMRealityKit", - dependencies: ["VRMKit", "VRMKitRuntime"] + dependencies: ["VRMKit", "VRMKitRuntime"], + // Shaders/MToon.metal is compiled offline into the per-platform + // metallibs under Resources by Scripts/build-mtoon-metallibs.sh. + exclude: ["Shaders"], + resources: [.process("Resources")] ), .testTarget( @@ -32,6 +36,14 @@ let package = Package( .testTarget( name: "VRMSceneKitTests", dependencies: ["VRMSceneKit"], + resources: [ + .copy("../VRMKitTests/Assets/AliciaSolid.vrm"), + .copy("../VRMKitTests/Assets/Seed-san.vrm") + ] + ), + .testTarget( + name: "VRMRealityKitTests", + dependencies: ["VRMRealityKit"], resources: [.copy("../VRMKitTests/Assets/AliciaSolid.vrm"), .copy("../VRMKitTests/Assets/Seed-san.vrm")] ), ] diff --git a/README.md b/README.md index 50b1750..c35f94a 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,8 @@ vrm.gltf.jsonData.nodes[0].name ## Render VRM +VRMRealityKit enables MToon by default on iOS and macOS. visionOS uses the existing Unlit / PBR fallback because RealityKit's `CustomMaterial` is unavailable there. VRMSceneKit is deprecated and also keeps its existing fallback material conversion rather than implementing the new MToon renderer. + ```swift import RealityKit import VRMKit @@ -81,6 +83,52 @@ anchor.addChild(vrmEntity.entity) arView.scene.addAnchor(anchor) ``` +### AR session integration + +Outline creation and shadow casting are independent loader options. For a conservative live-AR configuration, disable both while keeping the MToon surface shader enabled. + +On visionOS, MToon and outline creation fall back automatically, and `isShadowCastingEnabled` has no effect because the required RealityKit APIs are unavailable. + +Disable grounding shadows on the host `ARView` and call `VRMEntity.update(at:)` every frame for spring bones, constraints, skinning, and MToon UV animation: + +```swift +import ARKit +import Combine +import RealityKit +import VRMRealityKit + +let arView = ARView(frame: bounds) +arView.renderOptions.insert(.disableGroundingShadows) + +let config = ARWorldTrackingConfiguration() +config.planeDetection = [.horizontal] +arView.session.run(config) + +let loader = try VRMEntityLoader( + named: "model.vrm", + isOutlineEnabled: false, + isShadowCastingEnabled: false +) +let vrmEntity = try loader.loadEntity() + +var time: TimeInterval = 0 +let subscription = arView.scene.subscribe(to: SceneEvents.Update.self) { event in + time += event.deltaTime + vrmEntity.setMToonLightDirection(SIMD3(0, 0, -1)) + vrmEntity.update(at: time) +} + +let anchor = AnchorEntity(world: transform) +anchor.addChild(vrmEntity.entity) +arView.scene.addAnchor(anchor) +``` + +> Calling `VRMEntity.update(at:)` every frame is required for skinning, constraints, spring bones, and MToon UV animation. + +Set `isMToonEnabled: false` only when you want to disable MToon entirely and use the legacy Unlit / PBR conversion instead. + +On the package's minimum supported RealityKit versions, custom meshes expose only `TEXCOORD_0` and `CustomMaterial` has one material-level UV transform. MToon textures that request another UV set therefore use `TEXCOORD_0`. If UV-accessed texture slots specify different `KHR_texture_transform` values, VRMRealityKit applies the first transform in material-slot order to all UV-accessed MToon textures and logs a warning. Expression texture transform binds still update all UV-accessed textures together as required by VRMC_vrm. + ### Render VRM (SwiftUI) ```swift @@ -193,7 +241,7 @@ let image = try loader.loadThumbnail(from: vrm) - [x] Decoding VRM 1.0 file - [x] Render an avatar by RealityKit (as VRM 0.x) - [x] Render an avatar by RealityKit (as VRM 1.x) -- [ ] VRM shaders support (MToon) +- [x] VRM shaders support (MToon, RealityKit) - [ ] Improve rendering quality - [ ] Animation support (vrma) - [ ] VRM editing function diff --git a/Scripts/build-mtoon-metallibs.sh b/Scripts/build-mtoon-metallibs.sh new file mode 100755 index 0000000..f157356 --- /dev/null +++ b/Scripts/build-mtoon-metallibs.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Precompiles the RealityKit MToon shader into per-platform Metal libraries. +# +# CustomMaterial shaders must be compiled offline (TN3133): SwiftPM/Xcode Metal +# compilation of package targets is not reliable across build environments +# (swift build does not compile .metal, and Xcode 26+ requires a separately +# installed Metal Toolchain). The resulting .metallib files are committed to +# the repository and loaded at runtime with MTLDevice.makeLibrary(URL:). +# +# Run this script whenever Sources/VRMRealityKit/Shaders/MToon.metal changes: +# ./Scripts/build-mtoon-metallibs.sh +set -euo pipefail + +cd "$(dirname "$0")/.." + +SOURCE="Sources/VRMRealityKit/Shaders/MToon.metal" +RESOURCES="Sources/VRMRealityKit/Resources" + +mkdir -p "$RESOURCES" + +compile() { + local sdk="$1" + local min_flag="$2" + local output="$3" + echo "Compiling $SOURCE for $sdk -> $output" + xcrun -sdk "$sdk" metal "$min_flag" -o "$RESOURCES/$output" "$SOURCE" +} + +# Minimum OS versions match the CustomMaterial availability used by VRMEntityLoader. +compile macosx -mmacosx-version-min=12.0 MToon-macos.metallib +compile iphoneos -mios-version-min=15.0 MToon-ios.metallib +compile iphonesimulator -miphonesimulator-version-min=15.0 MToon-iossim.metallib + +# Record the shader source hash so tests can detect stale metallibs. +shasum -a 256 "$SOURCE" | awk '{print $1}' > "$RESOURCES/MToonShaderSource.sha256" + +echo "Done. Regenerated metallibs:" +ls -la "$RESOURCES"/MToon-*.metallib "$RESOURCES/MToonShaderSource.sha256" diff --git a/Sources/VRMKit/Extensions/Data+GLTF.swift b/Sources/VRMKit/Extensions/Data+GLTF.swift index fe6784c..2293ceb 100644 --- a/Sources/VRMKit/Extensions/Data+GLTF.swift +++ b/Sources/VRMKit/Extensions/Data+GLTF.swift @@ -31,7 +31,7 @@ package extension Data { return subdata(in: offset.. + package let emissiveFactor: SIMD3 + package let shadeColorFactor: SIMD4 + package let shadingShiftFactor: Float + package let shadingShiftTextureScale: Float + package let shadingToonyFactor: Float + package let giEqualizationFactor: Float + package let matcapFactor: SIMD3 + package let parametricRimColorFactor: SIMD4 + package let rimLightingMixFactor: Float + package let parametricRimFresnelPowerFactor: Float + package let parametricRimLiftFactor: Float + package let outlineWidthMode: OutlineWidthMode + package let outlineWidthFactor: Float + package let outlineColorFactor: SIMD4 + package let outlineLightingMixFactor: Float + package let uvAnimationScrollXSpeedFactor: Float + package let uvAnimationScrollYSpeedFactor: Float + package let uvAnimationRotationSpeedFactor: Float + package let transparentWithZWrite: Bool + package let renderQueueOffsetNumber: Int + package let alphaMode: GLTF.Material.AlphaMode + package let alphaCutoff: Float + package let cullMode: CullMode + package let normalScale: Float + package let baseColorTexture: Texture? + package let emissiveTexture: Texture? + package let shadeMultiplyTexture: Texture? + package let shadingShiftTexture: Texture? + package let normalTexture: Texture? + package let matcapTexture: Texture? + package let rimMultiplyTexture: Texture? + package let outlineWidthMultiplyTexture: Texture? + package let uvAnimationMaskTexture: Texture? + + package init?(material: GLTF.Material, materialProperty: VRM0.MaterialProperty?) { + if let mtoon = material.extensions?.materialsMToon { + self.init(vrm1: mtoon, material: material) + return + } + + guard let materialProperty, + materialProperty.vrmShader == .mToon || materialProperty.shader.lowercased().contains("mtoon") else { + return nil + } + self.init(vrm0: materialProperty, material: material) + } +} + +package extension MToonMaterialDescriptor { + var hasOutline: Bool { + switch outlineWidthMode { + case .none: + return false + case .worldCoordinates, .screenCoordinates: + return outlineWidthFactor > 0 + } + } +} + +package extension MToonMaterialDescriptor.OutlineWidthMode { + var rawValue: Float { + switch self { + case .none: + return 0 + case .worldCoordinates: + return 1 + case .screenCoordinates: + return 2 + } + } +} + +private extension MToonMaterialDescriptor { + init(vrm1 mtoon: GLTF.Material.MaterialExtensions.MaterialsMToon, material: GLTF.Material) { + let pbr = material.pbrMetallicRoughness + let baseColor = (pbr?.baseColorFactor).map(SIMD4.init) ?? SIMD4(1, 1, 1, 1) + let shadeColor = SIMD4(mtoon.shadeColorFactor, default: SIMD4(0, 0, 0, 1)) + let matcapFactor = SIMD3(mtoon.matcapFactor, default: SIMD3(1, 1, 1)) + let rimColor = SIMD4(mtoon.parametricRimColorFactor, default: SIMD4(0, 0, 0, 1)) + let outlineColor = SIMD4(mtoon.outlineColorFactor, default: SIMD4(0, 0, 0, 1)) + + self.baseColorFactor = baseColor + self.emissiveFactor = SIMD3(material.emissiveFactor) + self.shadeColorFactor = shadeColor + self.shadingShiftFactor = Float(mtoon.shadingShiftFactor ?? 0) + self.shadingShiftTextureScale = Float(mtoon.shadingShiftTexture?.scale ?? 1) + self.shadingToonyFactor = Float(mtoon.shadingToonyFactor ?? 0.9) + self.giEqualizationFactor = Float(mtoon.giEqualizationFactor ?? 0.9) + self.matcapFactor = matcapFactor + self.parametricRimColorFactor = rimColor + self.rimLightingMixFactor = Float(mtoon.rimLightingMixFactor ?? 1) + self.parametricRimFresnelPowerFactor = Float(mtoon.parametricRimFresnelPowerFactor ?? 5) + self.parametricRimLiftFactor = Float(mtoon.parametricRimLiftFactor ?? 0) + self.outlineWidthMode = .init(vrm1: mtoon.outlineWidthMode) + self.outlineWidthFactor = Float(mtoon.outlineWidthFactor ?? 0) + self.outlineColorFactor = outlineColor + self.outlineLightingMixFactor = Float(mtoon.outlineLightingMixFactor ?? 1) + self.uvAnimationScrollXSpeedFactor = Float(mtoon.uvAnimationScrollXSpeedFactor ?? 0) + self.uvAnimationScrollYSpeedFactor = Float(mtoon.uvAnimationScrollYSpeedFactor ?? 0) + self.uvAnimationRotationSpeedFactor = Float(mtoon.uvAnimationRotationSpeedFactor ?? 0) + self.transparentWithZWrite = mtoon.transparentWithZWrite ?? false + self.renderQueueOffsetNumber = mtoon.renderQueueOffsetNumber ?? 0 + self.alphaMode = material.alphaMode + self.alphaCutoff = material.alphaCutoff + self.cullMode = material.doubleSided ? .none : .back + self.normalScale = Float(material.normalTexture?.scale ?? 1) + self.baseColorTexture = pbr?.baseColorTexture.map(MToonMaterialDescriptor.Texture.init) + self.emissiveTexture = material.emissiveTexture.map(MToonMaterialDescriptor.Texture.init) + self.shadeMultiplyTexture = mtoon.shadeMultiplyTexture.map(MToonMaterialDescriptor.Texture.init) + self.shadingShiftTexture = mtoon.shadingShiftTexture.map(MToonMaterialDescriptor.Texture.init) + self.normalTexture = material.normalTexture.map(MToonMaterialDescriptor.Texture.init) + self.matcapTexture = mtoon.matcapTexture.map(MToonMaterialDescriptor.Texture.init) + self.rimMultiplyTexture = mtoon.rimMultiplyTexture.map(MToonMaterialDescriptor.Texture.init) + self.outlineWidthMultiplyTexture = mtoon.outlineWidthMultiplyTexture.map(MToonMaterialDescriptor.Texture.init) + self.uvAnimationMaskTexture = mtoon.uvAnimationMaskTexture.map(MToonMaterialDescriptor.Texture.init) + } + + init(vrm0 property: VRM0.MaterialProperty, material: GLTF.Material) { + let floats = property.floatProperties.dictionaryValue + let textures = property.textureProperties + let vectors = property.vectorProperties.dictionaryValue + let pbr = material.pbrMetallicRoughness + let baseColor = vectors.simd4("_Color") ?? (pbr?.baseColorFactor).map(SIMD4.init) ?? SIMD4(1, 1, 1, 1) + let emissiveColor = vectors.simd3("_EmissionColor") ?? SIMD3(0, 0, 0) + let shadeColor = vectors.simd4("_ShadeColor") ?? SIMD4(0.97, 0.81, 0.86, 1) + let rimColor = vectors.simd4("_RimColor") ?? SIMD4(0, 0, 0, 1) + let outlineColor = vectors.simd4("_OutlineColor") ?? SIMD4(0, 0, 0, 1) + let alphaMode = GLTF.Material.AlphaMode(vrm0: property, fallback: material.alphaMode) + let transparentWithZWrite = property.keywordMap["_ZWRITE_ON"] ?? false + let cullMode: CullMode + switch floats.float("_CullMode") { + case .some(0): + cullMode = .none + case .some(1): + cullMode = .front + case .some(2): + cullMode = .back + default: + cullMode = material.doubleSided ? .none : .back + } + let hasMToonNormalTexture = textures["_BumpMap"] != nil + let shadeShift0 = floats.float("_ShadeShift") ?? 0 + let shadeToony0 = floats.float("_ShadeToony") ?? 0.9 + let rangeMin = shadeShift0 + let rangeMax = simd_mix(Float(1), shadeShift0, shadeToony0) + + self.baseColorFactor = baseColor + self.emissiveFactor = emissiveColor + self.shadeColorFactor = shadeColor + self.shadingShiftFactor = (-(rangeMax + rangeMin) / 2).clamped(to: -1 ... 1) + self.shadingShiftTextureScale = 1 + self.shadingToonyFactor = ((2 - (rangeMax - rangeMin)) / 2).clamped(to: 0 ... 1) + self.giEqualizationFactor = (1 - (floats.float("_IndirectLightIntensity") ?? 0.1)).clamped(to: 0 ... 1) + self.matcapFactor = SIMD3(1, 1, 1) + self.parametricRimColorFactor = rimColor + self.rimLightingMixFactor = floats.float("_RimLightingMix") ?? 0 + self.parametricRimFresnelPowerFactor = floats.float("_RimFresnelPower") ?? 1 + self.parametricRimLiftFactor = floats.float("_RimLift") ?? 0 + self.outlineWidthMode = .init(vrm0: floats.float("_OutlineWidthMode") ?? 0) + self.outlineWidthFactor = (floats.float("_OutlineWidth") ?? 0) * 0.01 + self.outlineColorFactor = outlineColor + self.outlineLightingMixFactor = floats.float("_OutlineLightingMix") ?? 1 + self.uvAnimationScrollXSpeedFactor = floats.float("_UvAnimScrollX") ?? 0 + self.uvAnimationScrollYSpeedFactor = floats.float("_UvAnimScrollY") ?? 0 + self.uvAnimationRotationSpeedFactor = (floats.float("_UvAnimRotation") ?? 0) * 2 * Float.pi + self.transparentWithZWrite = transparentWithZWrite + self.renderQueueOffsetNumber = Self.vrm0RenderQueueOffset(renderQueue: property.renderQueue, + alphaMode: alphaMode, + transparentWithZWrite: transparentWithZWrite) + self.alphaMode = alphaMode + self.alphaCutoff = floats.float("_Cutoff") ?? material.alphaCutoff + self.cullMode = cullMode + self.normalScale = hasMToonNormalTexture + ? (floats.float("_BumpScale") ?? 1) + : Float(material.normalTexture?.scale ?? 1) + self.baseColorTexture = textures["_MainTex"].map(MToonMaterialDescriptor.Texture.init) + self.emissiveTexture = textures["_EmissionMap"].map(MToonMaterialDescriptor.Texture.init) + self.shadeMultiplyTexture = (textures["_ShadeTexture"] ?? textures["_MainTex"]) + .map(MToonMaterialDescriptor.Texture.init) + self.shadingShiftTexture = nil + self.normalTexture = textures["_BumpMap"].map(MToonMaterialDescriptor.Texture.init) ?? material.normalTexture.map(MToonMaterialDescriptor.Texture.init) + self.matcapTexture = textures["_SphereAdd"].map(MToonMaterialDescriptor.Texture.init) + self.rimMultiplyTexture = textures["_RimTexture"].map(MToonMaterialDescriptor.Texture.init) + self.outlineWidthMultiplyTexture = textures["_OutlineWidthTexture"].map(MToonMaterialDescriptor.Texture.init) + self.uvAnimationMaskTexture = textures["_UvAnimMaskTexture"].map(MToonMaterialDescriptor.Texture.init) + } + + static func vrm0RenderQueueOffset(renderQueue: Int, + alphaMode: GLTF.Material.AlphaMode, + transparentWithZWrite: Bool) -> Int { + guard renderQueue > 0, alphaMode == .BLEND else { return 0 } + if transparentWithZWrite { + return (renderQueue - 2501).clamped(to: 0 ... 9) + } + return (renderQueue - 3000).clamped(to: -9 ... 0) + } +} + +private extension MToonMaterialDescriptor.OutlineWidthMode { + init(vrm1 mode: GLTF.Material.MaterialExtensions.MaterialsMToon.MaterialsMToonOutlineWidthMode?) { + switch mode { + case .some(.worldCoordinates): + self = .worldCoordinates + case .some(.screenCoordinates): + self = .screenCoordinates + case .some(.none), nil: + self = .none + } + } + + init(vrm0 mode: Float) { + switch Int(mode) { + case 1: + self = .worldCoordinates + case 2: + self = .screenCoordinates + default: + self = .none + } + } +} + +private extension MToonMaterialDescriptor.Texture { + init(_ textureInfo: GLTF.TextureInfo) { + self.init(index: textureInfo.index, texCoord: textureInfo.texCoord) + } + + init(_ textureInfo: GLTF.Material.NormalTextureInfo) { + self.init(index: textureInfo.index, texCoord: textureInfo.texCoord) + } + + init(_ textureInfo: GLTF.Material.MaterialExtensions.MaterialsMToon.MaterialsMToonTextureInfo) { + self.init(index: textureInfo.index, texCoord: textureInfo.texCoord ?? 0) + } + + init(_ textureInfo: GLTF.Material.MaterialExtensions.MaterialsMToon.MaterialsMToonShadingShiftTexture) { + self.init(index: textureInfo.index, texCoord: textureInfo.texCoord ?? 0) + } +} + +private extension GLTF.Material.AlphaMode { + init(vrm0 property: VRM0.MaterialProperty, fallback: GLTF.Material.AlphaMode) { + if let renderType = property.tagMap["RenderType"]?.lowercased() { + switch renderType { + case "opaque": + self = .OPAQUE + return + case "transparentcutout", "cutout": + self = .MASK + return + case "transparent": + self = .BLEND + return + default: + break + } + } + + if property.keywordMap["_ALPHAPREMULTIPLY_ON"] == true || property.keywordMap["_ALPHABLEND_ON"] == true { + self = .BLEND + } else if property.keywordMap["_ALPHATEST_ON"] == true { + self = .MASK + } else { + self = fallback + } + } +} + +private extension CodableAny { + var dictionaryValue: [String: Any] { + return value as? [String: Any] ?? [:] + } +} + +private extension Dictionary where Key == String, Value == Any { + func float(_ key: String) -> Float? { + switch self[key] { + case let value as Float: + return value + case let value as Double: + return Float(value) + case let value as Int: + return Float(value) + case let value as NSNumber: + return value.floatValue + default: + return nil + } + } + + func simd4(_ key: String) -> SIMD4? { + return (self[key] as? [Any]).map(SIMD4.init) + } + + func simd3(_ key: String) -> SIMD3? { + return (self[key] as? [Any]).map(SIMD3.init) + } +} + +private extension Comparable { + func clamped(to range: ClosedRange) -> Self { + return min(max(self, range.lowerBound), range.upperBound) + } +} + +private extension SIMD3 where Scalar == Float { + init(_ color: Color3) { + self.init(color.r, color.g, color.b) + } + + init(_ values: [Any]) { + self.init(values.float(at: 0, default: 0), + values.float(at: 1, default: 0), + values.float(at: 2, default: 0)) + } +} + +private extension SIMD4 where Scalar == Float { + init(_ values: [Double]?, default defaultValue: SIMD4) { + guard let values else { + self = defaultValue + return + } + self.init(Float(values[safe: 0] ?? Double(defaultValue.x)), + Float(values[safe: 1] ?? Double(defaultValue.y)), + Float(values[safe: 2] ?? Double(defaultValue.z)), + Float(values[safe: 3] ?? Double(defaultValue.w))) + } + + init(_ color: GLTF.Color4) { + self.init(color.r, color.g, color.b, color.a) + } + + init(_ values: [Any]) { + self.init(values.float(at: 0, default: 1), + values.float(at: 1, default: 1), + values.float(at: 2, default: 1), + values.float(at: 3, default: 1)) + } +} + +private extension Array where Element == Any { + func float(at index: Int, default defaultValue: Float) -> Float { + guard indices.contains(index) else { return defaultValue } + switch self[index] { + case let value as Float: + return value + case let value as Double: + return Float(value) + case let value as Int: + return Float(value) + case let value as NSNumber: + return value.floatValue + default: + return defaultValue + } + } +} diff --git a/Sources/VRMRealityKit/CustomType/MToonMaterialParameters.swift b/Sources/VRMRealityKit/CustomType/MToonMaterialParameters.swift new file mode 100644 index 0000000..089809f --- /dev/null +++ b/Sources/VRMRealityKit/CustomType/MToonMaterialParameters.swift @@ -0,0 +1,198 @@ +#if canImport(RealityKit) +import Foundation +import Metal +import RealityKit +import simd +import VRMKit +import VRMKitRuntime + +@available(iOS 18.0, macOS 15.0, visionOS 2.0, *) +struct MToonMaterialParametersComponent: Component { + var parameters: MToonMaterialParameters +} + +@available(iOS 18.0, macOS 15.0, visionOS 2.0, *) +struct MToonMaterialParameters { + static let defaultLightDirection = simd_normalize(SIMD3(0.35, 0.55, 0.75)) + static let baseParameterRowCount = 17 + static let samplerRowCount = MToonTextureSlot.allCases.count + static let textureRowCount = baseParameterRowCount + samplerRowCount + static let defaultSampler = SIMD4(0, 0, 0, 0) + + var baseColor: SIMD4 + var shadeColor: SIMD4 + var rimColor: SIMD4 + var matcapColor: SIMD4 + var outlineColor: SIMD4 + var shadeParams: SIMD4 + var rimParams: SIMD4 + var outlineParams: SIMD4 + var uvAnimation: SIMD4 + var featureFlags: SIMD4 + var extraFlags: SIMD4 + var emissiveFactor: SIMD4 + var lightColor = SIMD4(1, 1, 1, 1) + var ambientColor = SIMD4(0, 0, 0, 1) + var uvTransform = SIMD4(1, 1, 0, 0) + var uvTransformRotation = SIMD4(1, 0, 0, 0) + var normalParameters: SIMD4 + var samplers = Array(repeating: MToonMaterialParameters.defaultSampler, + count: MToonMaterialParameters.samplerRowCount) + var lightDirection: SIMD3 = MToonMaterialParameters.defaultLightDirection + var elapsedTime: Float = 0 + + init(_ mtoon: MToonMaterialDescriptor) { + baseColor = mtoon.baseColorFactor + shadeColor = mtoon.shadeColorFactor + rimColor = mtoon.parametricRimColorFactor + matcapColor = SIMD4(mtoon.matcapFactor.x, mtoon.matcapFactor.y, mtoon.matcapFactor.z, 1) + outlineColor = mtoon.outlineColorFactor + emissiveFactor = SIMD4(mtoon.emissiveFactor.x, mtoon.emissiveFactor.y, mtoon.emissiveFactor.z, 1) + shadeParams = SIMD4(mtoon.shadingShiftFactor, + mtoon.shadingToonyFactor, + mtoon.giEqualizationFactor, + mtoon.alphaCutoff) + rimParams = SIMD4(mtoon.parametricRimFresnelPowerFactor, + mtoon.parametricRimLiftFactor, + mtoon.rimLightingMixFactor, + 0) + outlineParams = SIMD4(mtoon.outlineWidthFactor, + mtoon.outlineWidthMode.rawValue, + mtoon.outlineLightingMixFactor, + mtoon.hasOutline ? 1 : 0) + uvAnimation = SIMD4(mtoon.uvAnimationScrollXSpeedFactor, + mtoon.uvAnimationScrollYSpeedFactor, + mtoon.uvAnimationRotationSpeedFactor, + mtoon.shadingShiftTextureScale) + featureFlags = SIMD4(mtoon.matcapTexture == nil ? 0 : 1, + mtoon.rimMultiplyTexture == nil ? 0 : 1, + mtoon.shadingShiftTexture == nil ? 0 : 1, + mtoon.uvAnimationMaskTexture == nil ? 0 : 1) + extraFlags = SIMD4(mtoon.normalTexture == nil ? 0 : 1, + mtoon.shadeMultiplyTexture == nil ? 0 : 1, + mtoon.emissiveTexture == nil ? 0 : 1, + mtoon.alphaMode.mtoonRawValue) + normalParameters = SIMD4(mtoon.normalScale, 0, 0, 0) + } + + var customValue: SIMD4 { + SIMD4(lightDirection.x, lightDirection.y, lightDirection.z, elapsedTime) + } + + mutating func setColor(_ color: SIMD4, + for type: VRM1.Expressions.Expression.MaterialColorBind.MaterialColorType) -> Bool { + switch type { + case .color: + baseColor = color + case .shadeColor: + shadeColor = color + case .matcapColor: + matcapColor = color + case .rimColor: + rimColor = color + case .outlineColor: + outlineColor = color + case .emissionColor: + emissiveFactor = SIMD4(color.x, color.y, color.z, emissiveFactor.w) + } + return true + } + + func color(for type: VRM1.Expressions.Expression.MaterialColorBind.MaterialColorType) -> SIMD4? { + switch type { + case .color: + return baseColor + case .shadeColor: + return shadeColor + case .matcapColor: + return matcapColor + case .rimColor: + return rimColor + case .outlineColor: + return outlineColor + case .emissionColor: + return emissiveFactor + } + } + + mutating func setTextureTransform(scale: SIMD2, + offset: SIMD2, + rotation: Float) { + uvTransform = SIMD4(scale.x, scale.y, offset.x, offset.y) + uvTransformRotation = SIMD4(cos(rotation), sin(rotation), 0, 0) + } + + mutating func setSampler(_ sampler: SIMD4, for slot: MToonTextureSlot) { + samplers[slot.rawValue] = sampler + } + + @MainActor + func textureResource() throws -> TextureResource { + let rows = [ + baseColor, + shadeColor, + rimColor, + matcapColor, + outlineColor, + shadeParams, + rimParams, + outlineParams, + uvAnimation, + featureFlags, + extraFlags, + emissiveFactor, + lightColor, + ambientColor, + uvTransform, + uvTransformRotation, + normalParameters + ] + samplers + precondition(rows.count == Self.textureRowCount) + let data = rows.withUnsafeBufferPointer { Data(buffer: $0) } + let mip = TextureResource.Contents.MipmapLevel.mip( + data: data, + bytesPerRow: MemoryLayout>.stride * rows.count + ) + return try TextureResource(dimensions: .dimensions(width: rows.count, height: 1), + format: .raw(pixelFormat: .rgba32Float), + contents: .init(mipmapLevels: [mip])) + } +} + +enum MToonTextureSlot: Int, CaseIterable { + case base + case shade + case shadingShift + case normal + case matcap + case emissive + case rim + case outlineWidth + case uvAnimationMask + + var semantic: TextureResource.Semantic { + switch self { + case .shadingShift, .outlineWidth, .uvAnimationMask: + return .raw + case .normal: + return .normal + case .base, .shade, .matcap, .emissive, .rim: + return .color + } + } +} + +private extension GLTF.Material.AlphaMode { + var mtoonRawValue: Float { + switch self { + case .OPAQUE: + return 0 + case .MASK: + return 1 + case .BLEND: + return 2 + } + } +} + +#endif diff --git a/Sources/VRMRealityKit/CustomType/VRMEntity.swift b/Sources/VRMRealityKit/CustomType/VRMEntity.swift index 017b33d..37ba08e 100644 --- a/Sources/VRMRealityKit/CustomType/VRMEntity.swift +++ b/Sources/VRMRealityKit/CustomType/VRMEntity.swift @@ -1,6 +1,7 @@ #if canImport(RealityKit) import CoreGraphics import Foundation +import OSLog import RealityKit import simd import VRMKit @@ -22,6 +23,8 @@ struct VRMMaterialIndexComponent: Component { @available(iOS 18.0, macOS 15.0, visionOS 2.0, *) @MainActor public final class VRMEntity { + private static let logger = Logger(subsystem: "dev.tattn.VRMKit", category: "MToon") + public let vrm: VRM public let entity: Entity public let humanoid = Humanoid() @@ -30,6 +33,7 @@ public final class VRMEntity { var blendShapeClips: [BlendShapeKey: BlendShapeClip] = [:] var expressionClips: [ExpressionKey: ExpressionClip] = [:] + private var expressionWeights: [ExpressionKey: Float] = [:] private var materialColorClips: [ExpressionKey: [MaterialColorBinding]] = [:] private var textureTransformClips: [ExpressionKey: [TextureTransformBinding]] = [:] private var firstPersonAnnotations: [FirstPersonAnnotation] = [] @@ -37,6 +41,10 @@ public final class VRMEntity { private var modelEntitiesByMaterialIndex: [Int: [ModelEntity]] = [:] private var springBones: [VRMEntitySpringBone] = [] private var nodeConstraints: [NodeConstraintBinding] = [] + private var mtoonLightDirection = MToonMaterialParameters.defaultLightDirection + private var mtoonLightColor = SIMD3(1, 1, 1) + private var mtoonAmbientColor = SIMD3(0, 0, 0) + private var mtoonElapsedTime: Float = 0 struct SkinBinding { let modelEntity: ModelEntity @@ -61,6 +69,7 @@ public final class VRMEntity { func setUpBlendShapes(nodes: [Entity?], meshes: [Entity?], loader: VRMEntityLoader) throws { blendShapeClips = [:] expressionClips = [:] + expressionWeights = [:] materialColorClips = [:] textureTransformClips = [:] @@ -101,14 +110,14 @@ public final class VRMEntity { isBinary: expressionClip.expression.isBinary ?? false) expressionClips[runtimeClip.key] = runtimeClip - let colorBindings: [MaterialColorBinding] = expressionClip.expression.materialColorBinds? + let colorBindings: [MaterialColorBinding] = try expressionClip.expression.materialColorBinds? .compactMap { bind in guard bind.targetValue.count >= 3 else { return nil } - guard let material = try? loader.material(withMaterialIndex: bind.material) else { return nil } return MaterialColorBinding(materialIndex: bind.material, type: bind.type, targetValue: SIMD4(bind.targetValue, default: 1.0), - baseValue: material.currentColor(for: bind.type)) + baseValue: try loader.currentMaterialColor(withMaterialIndex: bind.material, + type: bind.type)) } ?? [] if !colorBindings.isEmpty { materialColorClips[runtimeClip.key] = colorBindings @@ -121,6 +130,7 @@ public final class VRMEntity { return TextureTransformBinding(materialIndex: bind.material, baseScale: base.scale, baseOffset: base.offset, + baseRotation: base.rotation, targetScale: SIMD2(bind.scale, default: 1.0), targetOffset: SIMD2(bind.offset, default: 0.0)) } ?? [] @@ -261,10 +271,38 @@ public final class VRMEntity { modelEntitiesByMaterialIndex[materialIndex, default: []].append(modelEntity) } - public func update(at time: TimeInterval) { + /// Advances spring bones, node constraints, skinning, and MToon runtime state by one frame. + /// + /// Pass ``SceneEvents/Update/deltaTime`` when calling this from a RealityKit update subscription. + public func update(deltaTime: TimeInterval) { + let deltaTime = max(0, deltaTime) + updateMToonRuntime(deltaTime: Float(deltaTime)) nodeConstraints.forEach { $0.apply() } updateSkinning() - springBones.forEach { $0.update(deltaTime: time) } + springBones.forEach { $0.update(deltaTime: deltaTime) } + } + + /// Compatibility entry point. The argument is interpreted as elapsed time since the previous frame. + public func update(at deltaTime: TimeInterval) { + update(deltaTime: deltaTime) + } + + public func setMToonLightDirection(_ direction: SIMD3) { + let length = simd_length(direction) + mtoonLightDirection = length > 0.001 ? direction / length : MToonMaterialParameters.defaultLightDirection + updateMToonRuntime(deltaTime: 0) + } + + /// Sets the explicit main light color used by MToon CustomMaterial shaders. The default is white. + public func setMToonLightColor(_ color: SIMD3) { + mtoonLightColor = color + updateMToonLightingParameters() + } + + /// Sets the explicit ambient color used by the MToon GI approximation. The default is black. + public func setMToonAmbientColor(_ color: SIMD3) { + mtoonAmbientColor = color + updateMToonLightingParameters() } private func updateSkinning() { @@ -366,24 +404,20 @@ public final class VRMEntity { } public func setExpression(value: CGFloat, for key: ExpressionKey) { - guard let clip = expressionClip(for: key) else { return } + guard let key = canonicalExpressionKey(for: key), + let clip = expressionClips[key] else { return } let normalized = max(0.0, min(1.0, clip.isBinary ? round(value) : value)) - for binding in clip.values { - let weight = Float(binding.weight / 100.0) * Float(normalized) - applyBlendShapeWeight(weight, targetIndex: binding.index, on: binding.mesh) - } - for binding in materialColorClip(for: key) { - binding.apply(value: Float(normalized), on: self) - } - for binding in textureTransformClip(for: key) { - binding.apply(value: Float(normalized), on: self) + if normalized > 0 { + expressionWeights[key] = Float(normalized) + } else { + expressionWeights.removeValue(forKey: key) } + applyExpressions() } public func expression(for key: ExpressionKey) -> CGFloat { - guard let clip = expressionClip(for: key), - let binding = clip.values.first else { return 0 } - return CGFloat(readBlendShapeWeight(targetIndex: binding.index, on: binding.mesh)) + guard let key = canonicalExpressionKey(for: key) else { return 0 } + return CGFloat(expressionWeights[key] ?? 0) } public func setFirstPersonRenderMode(_ mode: FirstPersonRenderMode) { @@ -400,6 +434,10 @@ public final class VRMEntity { let vrmColor = VRMColor(simd: color) for modelEntity in models { guard var component = modelEntity.components[ModelComponent.self] else { continue } + if updateMToonColor(color, type: type, on: modelEntity, modelComponent: &component) { + modelEntity.components.set(component) + continue + } component.materials = component.materials.map { material in material.settingColor(vrmColor, for: type) } @@ -409,43 +447,189 @@ public final class VRMEntity { fileprivate func applyTextureTransform(scale: SIMD2, offset: SIMD2, + rotation: Float, materialIndex: Int) { guard let models = modelEntitiesByMaterialIndex[materialIndex] else { return } - let transform = MaterialParameterTypes.TextureCoordinateTransform(offset: offset, scale: scale) for modelEntity in models { guard var component = modelEntity.components[ModelComponent.self] else { continue } component.materials = component.materials.map { material in - material.settingTextureTransform(transform) + material.settingTextureTransform(scale: scale, offset: offset, rotation: rotation) } + updateMToonTextureTransform(scale: scale, offset: offset, rotation: rotation, on: modelEntity, modelComponent: &component) modelEntity.components.set(component) } } - private func expressionClip(for key: ExpressionKey) -> ExpressionClip? { - if let clip = expressionClips[key] { return clip } - if let legacyKey = key.legacyBlendShapeKey, - let expressionKey = legacyKey.expressionKey { - return expressionClips[expressionKey] + private func updateMToonRuntime(deltaTime: Float) { +#if !os(visionOS) + mtoonElapsedTime += deltaTime + for modelEntity in modelEntities(in: entity) { + guard var state = modelEntity.components[MToonMaterialParametersComponent.self], + var component = modelEntity.components[ModelComponent.self] else { continue } + state.parameters.lightDirection = mtoonLightDirection + state.parameters.elapsedTime = mtoonElapsedTime + applyMToonParameters(state.parameters, to: &component, updateParameterTexture: false) + modelEntity.components.set(state) + modelEntity.components.set(component) } - return nil +#endif } - private func materialColorClip(for key: ExpressionKey) -> [MaterialColorBinding] { - if let clip = materialColorClips[key] { return clip } - if let legacyKey = key.legacyBlendShapeKey, - let expressionKey = legacyKey.expressionKey { - return materialColorClips[expressionKey] ?? [] + private func updateMToonLightingParameters() { +#if !os(visionOS) + for modelEntity in modelEntities(in: entity) { + guard var state = modelEntity.components[MToonMaterialParametersComponent.self], + var component = modelEntity.components[ModelComponent.self] else { continue } + state.parameters.lightColor = SIMD4(mtoonLightColor.x, mtoonLightColor.y, mtoonLightColor.z, 1) + state.parameters.ambientColor = SIMD4(mtoonAmbientColor.x, mtoonAmbientColor.y, mtoonAmbientColor.z, 1) + state.parameters.lightDirection = mtoonLightDirection + state.parameters.elapsedTime = mtoonElapsedTime + applyMToonParameters(state.parameters, to: &component, updateParameterTexture: true) + modelEntity.components.set(state) + modelEntity.components.set(component) + } +#endif + } + + private func updateMToonColor(_ color: SIMD4, + type: VRM1.Expressions.Expression.MaterialColorBind.MaterialColorType, + on modelEntity: ModelEntity, + modelComponent: inout ModelComponent) -> Bool { +#if os(visionOS) + return false +#else + guard var state = modelEntity.components[MToonMaterialParametersComponent.self] else { return false } + guard state.parameters.setColor(color, for: type) else { return false } + state.parameters.lightColor = SIMD4(mtoonLightColor.x, mtoonLightColor.y, mtoonLightColor.z, 1) + state.parameters.ambientColor = SIMD4(mtoonAmbientColor.x, mtoonAmbientColor.y, mtoonAmbientColor.z, 1) + state.parameters.lightDirection = mtoonLightDirection + state.parameters.elapsedTime = mtoonElapsedTime + applyMToonParameters(state.parameters, to: &modelComponent, updateParameterTexture: true) + modelEntity.components.set(state) + return true +#endif + } + + private func updateMToonTextureTransform(scale: SIMD2, + offset: SIMD2, + rotation: Float, + on modelEntity: ModelEntity, + modelComponent: inout ModelComponent) { +#if !os(visionOS) + guard var state = modelEntity.components[MToonMaterialParametersComponent.self] else { return } + state.parameters.setTextureTransform(scale: scale, offset: offset, rotation: rotation) + state.parameters.lightColor = SIMD4(mtoonLightColor.x, mtoonLightColor.y, mtoonLightColor.z, 1) + state.parameters.ambientColor = SIMD4(mtoonAmbientColor.x, mtoonAmbientColor.y, mtoonAmbientColor.z, 1) + state.parameters.lightDirection = mtoonLightDirection + state.parameters.elapsedTime = mtoonElapsedTime + applyMToonParameters(state.parameters, to: &modelComponent, updateParameterTexture: true) + modelEntity.components.set(state) +#endif + } + +#if !os(visionOS) + private func applyMToonParameters(_ parameters: MToonMaterialParameters, + to component: inout ModelComponent, + updateParameterTexture: Bool) { + component.materials = component.materials.map { material in + guard var material = material as? CustomMaterial else { return material } + material.custom.value = parameters.customValue + if updateParameterTexture { + do { + material.custom.texture = CustomMaterial.Texture(try parameters.textureResource()) + } catch { + Self.logger.error("Failed to update MToon parameter texture: \(error.localizedDescription, privacy: .public)") + } + } + return material } - return [] } +#endif - private func textureTransformClip(for key: ExpressionKey) -> [TextureTransformBinding] { - if let clip = textureTransformClips[key] { return clip } + private func canonicalExpressionKey(for key: ExpressionKey) -> ExpressionKey? { + if expressionClips[key] != nil { return key } if let legacyKey = key.legacyBlendShapeKey, - let expressionKey = legacyKey.expressionKey { - return textureTransformClips[expressionKey] ?? [] + let expressionKey = legacyKey.expressionKey, + expressionClips[expressionKey] != nil { + return expressionKey + } + return nil + } + + private func applyExpressions() { + var morphBindings: [MorphBindingKey: BlendShapeBinding] = [:] + var morphWeights: [MorphBindingKey: Float] = [:] + for clip in expressionClips.values { + for binding in clip.values { + let key = MorphBindingKey(mesh: binding.mesh, targetIndex: binding.index) + morphBindings[key] = binding + morphWeights[key] = 0 + } + } + for (expressionKey, expressionWeight) in expressionWeights { + guard let clip = expressionClips[expressionKey] else { continue } + for binding in clip.values { + let key = MorphBindingKey(mesh: binding.mesh, targetIndex: binding.index) + morphWeights[key, default: 0] += Float(binding.weight / 100.0) * expressionWeight + } + } + for (key, binding) in morphBindings { + applyBlendShapeWeight(morphWeights[key] ?? 0, + targetIndex: binding.index, + on: binding.mesh) + } + if enableNormalTangentBlendShape { + let meshes = morphBindings.values.reduce(into: [ObjectIdentifier: Entity]()) { + $0[ObjectIdentifier($1.mesh)] = $1.mesh + } + meshes.values.forEach(updateBlendShapeNormalsAndTangents) + } + + var colorBindings: [MaterialColorBindingKey: MaterialColorBinding] = [:] + var colors: [MaterialColorBindingKey: SIMD4] = [:] + for bindings in materialColorClips.values { + for binding in bindings { + let key = binding.key + colorBindings[key] = binding + colors[key] = binding.baseValue + } + } + for (expressionKey, expressionWeight) in expressionWeights { + for binding in materialColorClips[expressionKey] ?? [] { + colors[binding.key, default: binding.baseValue] += + (binding.targetValue - binding.baseValue) * expressionWeight + } + } + for (key, binding) in colorBindings { + applyMaterialColor(colors[key] ?? binding.baseValue, + type: binding.type, + materialIndex: binding.materialIndex) + } + + var transformBindings: [Int: TextureTransformBinding] = [:] + var scales: [Int: SIMD2] = [:] + var offsets: [Int: SIMD2] = [:] + for bindings in textureTransformClips.values { + for binding in bindings { + transformBindings[binding.materialIndex] = binding + scales[binding.materialIndex] = binding.baseScale + offsets[binding.materialIndex] = binding.baseOffset + } + } + for (expressionKey, expressionWeight) in expressionWeights { + for binding in textureTransformClips[expressionKey] ?? [] { + scales[binding.materialIndex, default: binding.baseScale] += + (binding.targetScale - binding.baseScale) * expressionWeight + offsets[binding.materialIndex, default: binding.baseOffset] += + (binding.targetOffset - binding.baseOffset) * expressionWeight + } + } + for (materialIndex, binding) in transformBindings { + applyTextureTransform(scale: scales[materialIndex] ?? binding.baseScale, + offset: offsets[materialIndex] ?? binding.baseOffset, + rotation: binding.baseRotation, + materialIndex: materialIndex) } - return [] } private func modelEntities(in root: Entity) -> [ModelEntity] { @@ -697,6 +881,23 @@ private struct NodeConstraintBinding { } } +@available(iOS 18.0, macOS 15.0, visionOS 2.0, *) +private struct MorphBindingKey: Hashable { + let mesh: ObjectIdentifier + let targetIndex: Int + + init(mesh: Entity, targetIndex: Int) { + self.mesh = ObjectIdentifier(mesh) + self.targetIndex = targetIndex + } +} + +@available(iOS 18.0, macOS 15.0, visionOS 2.0, *) +private struct MaterialColorBindingKey: Hashable { + let materialIndex: Int + let type: String +} + @available(iOS 18.0, macOS 15.0, visionOS 2.0, *) private struct MaterialColorBinding { let materialIndex: Int @@ -704,11 +905,8 @@ private struct MaterialColorBinding { let targetValue: SIMD4 let baseValue: SIMD4 - @MainActor - func apply(value: Float, on entity: VRMEntity) { - entity.applyMaterialColor(baseValue + (targetValue - baseValue) * value, - type: type, - materialIndex: materialIndex) + var key: MaterialColorBindingKey { + MaterialColorBindingKey(materialIndex: materialIndex, type: type.rawValue) } } @@ -717,17 +915,10 @@ private struct TextureTransformBinding { let materialIndex: Int let baseScale: SIMD2 let baseOffset: SIMD2 + let baseRotation: Float let targetScale: SIMD2 let targetOffset: SIMD2 - @MainActor - func apply(value: Float, on entity: VRMEntity) { - let scale = baseScale + (targetScale - baseScale) * value - let offset = baseOffset + (targetOffset - baseOffset) * value - entity.applyTextureTransform(scale: scale, - offset: offset, - materialIndex: materialIndex) - } } @available(iOS 18.0, macOS 15.0, visionOS 2.0, *) @@ -753,11 +944,15 @@ private extension Entity { } @available(iOS 18.0, macOS 15.0, visionOS 2.0, *) -private extension Material { +extension Material { var currentTextureTransform: MaterialParameterTypes.TextureCoordinateTransform { switch self { case let material as UnlitMaterial: return material.textureCoordinateTransform +#if !os(visionOS) + case let material as CustomMaterial: + return material.textureCoordinateTransform +#endif case let material as PhysicallyBasedMaterial: return material.textureCoordinateTransform default: @@ -768,58 +963,97 @@ private extension Material { func currentColor(for type: VRM1.Expressions.Expression.MaterialColorBind.MaterialColorType) -> SIMD4 { switch self { case let material as UnlitMaterial: - return material.color.tint.simd + switch type { + case .color: + return material.color.tint.simd + case .emissionColor, .shadeColor, .matcapColor, .rimColor, .outlineColor: + return SIMD4(1, 1, 1, 1) + } +#if !os(visionOS) + case let material as CustomMaterial: + switch type { + case .color: + return material.baseColor.tint.simd + case .rimColor: + return material.emissiveColor.color.simd + case .emissionColor, .shadeColor, .matcapColor, .outlineColor: + return SIMD4(1, 1, 1, 1) + } +#endif case let material as PhysicallyBasedMaterial: switch type { case .color: return material.baseColor.tint.simd case .emissionColor: return material.emissiveColor.color.simd - case .shadeColor: - return material.baseColor.tint.simd case .matcapColor, .rimColor: return material.emissiveColor.color.simd - case .outlineColor: - return material.baseColor.tint.simd + case .shadeColor, .outlineColor: + return SIMD4(1, 1, 1, 1) } default: return SIMD4(1, 1, 1, 1) } } - func settingColor(_ color: VRMColor, - for type: VRM1.Expressions.Expression.MaterialColorBind.MaterialColorType) -> Material { + func settingTextureTransform(scale: SIMD2, offset: SIMD2, rotation: Float = 0) -> Material { + let transform = MaterialParameterTypes.TextureCoordinateTransform(offset: offset, scale: scale, rotation: rotation) switch self { case var material as UnlitMaterial: - material.color.tint = color + material.textureCoordinateTransform = transform return material +#if !os(visionOS) + case var material as CustomMaterial: + material.textureCoordinateTransform = transform + return material +#endif case var material as PhysicallyBasedMaterial: - switch type { - case .color, .shadeColor, .outlineColor: - material.baseColor.tint = color - case .emissionColor: - material.emissiveColor.color = color - case .matcapColor, .rimColor: - material.emissiveColor.color = color - } + material.textureCoordinateTransform = transform return material default: return self } } - func settingTextureTransform(_ transform: MaterialParameterTypes.TextureCoordinateTransform) -> Material { + func settingColor(_ color: VRMColor, + for type: VRM1.Expressions.Expression.MaterialColorBind.MaterialColorType) -> Material { switch self { case var material as UnlitMaterial: - material.textureCoordinateTransform = transform + switch type { + case .color: + material.color.tint = color + case .emissionColor, .shadeColor, .matcapColor, .rimColor, .outlineColor: + break + } + return material +#if !os(visionOS) + case var material as CustomMaterial: + switch type { + case .color: + material.baseColor.tint = color + case .rimColor: + material.emissiveColor.color = color + case .shadeColor, .emissionColor, .matcapColor, .outlineColor: + break + } return material +#endif case var material as PhysicallyBasedMaterial: - material.textureCoordinateTransform = transform + switch type { + case .color: + material.baseColor.tint = color + case .emissionColor: + material.emissiveColor.color = color + case .matcapColor, .rimColor: + material.emissiveColor.color = color + case .shadeColor, .outlineColor: + break + } return material default: return self } } -} +} #endif diff --git a/Sources/VRMRealityKit/Resources/MToon-ios.metallib b/Sources/VRMRealityKit/Resources/MToon-ios.metallib new file mode 100644 index 0000000..c479475 Binary files /dev/null and b/Sources/VRMRealityKit/Resources/MToon-ios.metallib differ diff --git a/Sources/VRMRealityKit/Resources/MToon-iossim.metallib b/Sources/VRMRealityKit/Resources/MToon-iossim.metallib new file mode 100644 index 0000000..35ca7ad Binary files /dev/null and b/Sources/VRMRealityKit/Resources/MToon-iossim.metallib differ diff --git a/Sources/VRMRealityKit/Resources/MToon-macos.metallib b/Sources/VRMRealityKit/Resources/MToon-macos.metallib new file mode 100644 index 0000000..9cf30d3 Binary files /dev/null and b/Sources/VRMRealityKit/Resources/MToon-macos.metallib differ diff --git a/Sources/VRMRealityKit/Resources/MToonShaderSource.sha256 b/Sources/VRMRealityKit/Resources/MToonShaderSource.sha256 new file mode 100644 index 0000000..62ceebd --- /dev/null +++ b/Sources/VRMRealityKit/Resources/MToonShaderSource.sha256 @@ -0,0 +1 @@ +cdae50164eee9841247f414c290f55057e46fa4d2fad690aa9bb1eb249e1baac diff --git a/Sources/VRMRealityKit/Shaders/MToon.metal b/Sources/VRMRealityKit/Shaders/MToon.metal new file mode 100644 index 0000000..588a4a2 --- /dev/null +++ b/Sources/VRMRealityKit/Shaders/MToon.metal @@ -0,0 +1,388 @@ +#include + +using namespace metal; + +constexpr sampler mtoonLinearClampSampler(coord::normalized, + address::clamp_to_edge, + filter::linear, + mip_filter::linear); +constexpr sampler mtoonNearestClampSampler(coord::normalized, + address::clamp_to_edge, + filter::nearest, + mip_filter::nearest); +constexpr sampler mtoonParameterSampler(coord::normalized, + address::clamp_to_edge, + filter::nearest, + mip_filter::none); + +constant float mtoonEpsilon = 0.00001; +constant float mtoonParameterTextureWidth = 26.0; +constant float mtoonSamplerParameterStart = 17.0; + +half4 mtoonParameter(realitykit::texture::textures textures, float row) +{ + return textures.custom().sample(mtoonParameterSampler, + float2((row + 0.5) / mtoonParameterTextureWidth, 0.5)); +} + +half4 mtoonSamplerParameter(realitykit::texture::textures textures, float slot) +{ + return mtoonParameter(textures, mtoonSamplerParameterStart + slot); +} + +float mtoonWrappedCoordinate(float value, half wrapMode) +{ + if (wrapMode > 1.5h) { + float mirrored = fract(value * 0.5) * 2.0; + return 1.0 - abs(mirrored - 1.0); + } + if (wrapMode > 0.5h) { + return clamp(value, 0.0, 1.0); + } + return fract(value); +} + +float2 mtoonWrappedUV(float2 uv, half4 samplerParameters) +{ + return float2(mtoonWrappedCoordinate(uv.x, samplerParameters.x), + mtoonWrappedCoordinate(uv.y, samplerParameters.y)); +} + +half4 mtoonSample(texture2d texture, float2 uv, half4 samplerParameters) +{ + float2 wrappedUV = mtoonWrappedUV(uv, samplerParameters); + if (samplerParameters.z > 0.5h) { + return texture.sample(mtoonNearestClampSampler, wrappedUV); + } + return texture.sample(mtoonLinearClampSampler, wrappedUV); +} + +float mtoonLinearstep(float a, float b, float t) +{ + return saturate((t - a) / max(b - a, mtoonEpsilon)); +} + +float2 mtoonTextureUV(float2 uv) +{ + return float2(uv.x, 1.0 - uv.y); +} + +float2 mtoonTransformedUV(float2 uv, half4 uvTransform, half4 uvTransformRotation) +{ + float2 transformed = uv * float2(uvTransform.xy); + float c = float(uvTransformRotation.x); + float s = float(uvTransformRotation.y); + transformed = float2(transformed.x * c - transformed.y * s, + transformed.x * s + transformed.y * c); + return transformed + float2(uvTransform.zw); +} + +float3 mtoonLightDirection(float4 customValue) +{ + float len = length(customValue.xyz); + if (len < 0.001) { + return normalize(float3(0.35, 0.55, 0.75)); + } + return customValue.xyz / len; +} + +float2 mtoonMatcapUV(float3 normal, float4x4 modelToView) +{ + float3 viewNormal = normalize((modelToView * float4(normal, 0.0)).xyz); + return float2(viewNormal.x * 0.5 + 0.5, 0.5 - viewNormal.y * 0.5); +} + +float3 mtoonShadingNormal(realitykit::surface_parameters params, + float2 uv, + half4 extraFlags, + half normalScale, + half4 normalSampler) +{ + float3 geometryNormal = normalize(params.geometry().normal()); + if (extraFlags.x < 0.5h) { + return geometryNormal; + } + half3 tangentNormal = realitykit::unpack_normal(mtoonSample(params.textures().normal(), uv, normalSampler).rgb, + normalScale); + float3 rawTangent = params.geometry().tangent(); + float3 rawBitangent = params.geometry().bitangent(); + if (dot(rawTangent, rawTangent) < 0.000001 || dot(rawBitangent, rawBitangent) < 0.000001) { + return geometryNormal; + } + float3 tangent = normalize(rawTangent); + float3 bitangent = normalize(rawBitangent); + return normalize(tangent * float(tangentNormal.x) + + bitangent * float(tangentNormal.y) + + geometryNormal * float(tangentNormal.z)); +} + +float mtoonAlpha(float alphaMode, float baseAlpha, float cutoff) +{ + if (alphaMode < 0.5) { + return 1.0; + } + if (alphaMode < 1.5) { + if (baseAlpha < cutoff) { + discard_fragment(); + } + return 1.0; + } + return baseAlpha; +} + +float2 mtoonAnimatedSurfaceUV(realitykit::surface_parameters params, + float2 uv, + half4 uvAnimation, + half4 featureFlags, + half4 uvAnimationMaskSampler, + half4 uvTransform, + half4 uvTransformRotation) +{ + float time = params.uniforms().custom_parameter().w; + float mask = 1.0; + if (featureFlags.w > 0.5h) { + float2 maskUV = mtoonTextureUV(mtoonTransformedUV(uv, uvTransform, uvTransformRotation)); + mask = float(mtoonSample(params.textures().ambient_occlusion(), maskUV, uvAnimationMaskSampler).b); + } + + float angle = float(uvAnimation.z) * time * mask; + float2 center = float2(0.5, 0.5); + float2 centered = uv - center; + float s = sin(angle); + float c = cos(angle); + float2 rotated = float2(centered.x * c - centered.y * s, + centered.x * s + centered.y * c) + center; + return rotated + float2(float(uvAnimation.x), float(uvAnimation.y)) * time * mask; +} + +[[visible]] +void mtoonSurface(realitykit::surface_parameters params) +{ + auto textures = params.textures(); + auto surface = params.surface(); + auto material = params.material_constants(); + + half4 baseColorFactor = mtoonParameter(textures, 0.0); + half4 shadeColorFactor = mtoonParameter(textures, 1.0); + half4 rimColorFactor = mtoonParameter(textures, 2.0); + half4 matcapFactor = mtoonParameter(textures, 3.0); + half4 shadeParams = mtoonParameter(textures, 5.0); + half4 rimParams = mtoonParameter(textures, 6.0); + half4 uvAnimation = mtoonParameter(textures, 8.0); + half4 featureFlags = mtoonParameter(textures, 9.0); + half4 extraFlags = mtoonParameter(textures, 10.0); + half4 emissiveFactor = mtoonParameter(textures, 11.0); + half4 lightColorParameter = mtoonParameter(textures, 12.0); + half4 giColorParameter = mtoonParameter(textures, 13.0); + half4 uvTransform = mtoonParameter(textures, 14.0); + half4 uvTransformRotation = mtoonParameter(textures, 15.0); + half4 normalParameters = mtoonParameter(textures, 16.0); + half4 baseSampler = mtoonSamplerParameter(textures, 0.0); + half4 shadeSampler = mtoonSamplerParameter(textures, 1.0); + half4 shadingShiftSampler = mtoonSamplerParameter(textures, 2.0); + half4 normalSampler = mtoonSamplerParameter(textures, 3.0); + half4 matcapSampler = mtoonSamplerParameter(textures, 4.0); + half4 emissiveSampler = mtoonSamplerParameter(textures, 5.0); + half4 rimSampler = mtoonSamplerParameter(textures, 6.0); + + half4 uvAnimationMaskSampler = mtoonSamplerParameter(textures, 8.0); + float2 uv = mtoonAnimatedSurfaceUV(params, + params.geometry().uv0(), + uvAnimation, + featureFlags, + uvAnimationMaskSampler, + uvTransform, + uvTransformRotation); + uv = mtoonTransformedUV(uv, uvTransform, uvTransformRotation); + uv = mtoonTextureUV(uv); + + half4 baseSample = mtoonSample(textures.base_color(), uv, baseSampler); + half4 shadeSample = extraFlags.y > 0.5h + ? mtoonSample(textures.roughness(), uv, shadeSampler) + : half4(1.0h); + + float shift = float(shadeParams.x); + if (featureFlags.z > 0.5h) { + half shadingShift = mtoonSample(textures.specular(), uv, shadingShiftSampler).r; + shift += float(shadingShift) * float(uvAnimation.w); + } + + float3 normal = mtoonShadingNormal(params, uv, extraFlags, normalParameters.x, normalSampler); + float3 lightDirection = mtoonLightDirection(params.uniforms().custom_parameter()); + float shadingToony = clamp(float(shadeParams.y), 0.0, 1.0); + float shading = mtoonLinearstep(-1.0 + shadingToony, + 1.0 - shadingToony, + dot(normal, lightDirection) + shift); + + float3 litColor = float3(baseSample.rgb * baseColorFactor.rgb); + float3 shadeColor = float3(shadeSample.rgb * shadeColorFactor.rgb); + float3 lightColor = float3(lightColorParameter.rgb); + float3 giColor = float3(giColorParameter.rgb); + float giLuma = dot(giColor, float3(0.2126, 0.7152, 0.0722)); + // RealityKit does not expose UniVRM's full GI pipeline; approximate giEqualization by neutralizing ambient hue only. + giColor = mix(giColor, float3(giLuma), clamp(float(shadeParams.z), 0.0, 1.0)); + + float3 direct = mix(shadeColor, litColor, shading) * lightColor; + float3 indirect = litColor * giColor; + float3 color = direct + indirect; + + float3 rim = float3(0.0); + if (featureFlags.x > 0.5h) { + float2 matcapUV = mtoonMatcapUV(normal, params.uniforms().model_to_view()); + rim += float3(mtoonSample(textures.metallic(), matcapUV, matcapSampler).rgb * matcapFactor.rgb); + } + + float3 viewDirection = normalize(params.geometry().view_direction()); + float rimBase = saturate(1.0 - dot(normal, viewDirection) + float(rimParams.y)); + float parametricRim = pow(rimBase, max(float(rimParams.x), mtoonEpsilon)); + rim += parametricRim * float3(rimColorFactor.rgb); + + if (featureFlags.y > 0.5h) { + rim *= float3(mtoonSample(textures.clearcoat_roughness(), uv, rimSampler).rgb); + } + rim *= mix(float3(1.0), direct + indirect, clamp(float(rimParams.z), 0.0, 1.0)); + color += rim; + + float3 emissiveTexture = extraFlags.z > 0.5h + ? float3(mtoonSample(textures.emissive_color(), uv, emissiveSampler).rgb) + : float3(1.0); + color += float3(emissiveFactor.rgb) * emissiveTexture; + + float baseAlpha = float(baseSample.a * baseColorFactor.a); + float cutoff = material.opacity_threshold() > 0.0 ? material.opacity_threshold() : float(shadeParams.w); + float opacity = mtoonAlpha(float(extraFlags.w), baseAlpha, cutoff); + + surface.set_base_color(half3(0.0h)); + surface.set_emissive_color(half3(color)); + surface.set_opacity(half(opacity)); + surface.set_roughness(1.0h); + surface.set_metallic(0.0h); +} + +[[visible]] +void mtoonOutlineSurface(realitykit::surface_parameters params) +{ + auto textures = params.textures(); + auto surface = params.surface(); + auto material = params.material_constants(); + half4 outlineColor = mtoonParameter(textures, 4.0); + half4 shadeParams = mtoonParameter(textures, 5.0); + half4 outlineParams = mtoonParameter(textures, 7.0); + half4 uvAnimation = mtoonParameter(textures, 8.0); + half4 featureFlags = mtoonParameter(textures, 9.0); + half4 extraFlags = mtoonParameter(textures, 10.0); + half4 lightColorParameter = mtoonParameter(textures, 12.0); + half4 uvTransform = mtoonParameter(textures, 14.0); + half4 uvTransformRotation = mtoonParameter(textures, 15.0); + half4 baseSampler = mtoonSamplerParameter(textures, 0.0); + half4 uvAnimationMaskSampler = mtoonSamplerParameter(textures, 8.0); + + float2 uv = mtoonAnimatedSurfaceUV(params, + params.geometry().uv0(), + uvAnimation, + featureFlags, + uvAnimationMaskSampler, + uvTransform, + uvTransformRotation); + uv = mtoonTextureUV(mtoonTransformedUV(uv, uvTransform, uvTransformRotation)); + half4 baseSample = mtoonSample(textures.base_color(), uv, baseSampler); + half4 baseColorFactor = mtoonParameter(textures, 0.0); + + float cutoff = material.opacity_threshold() > 0.0 ? material.opacity_threshold() : float(shadeParams.w); + float opacity = mtoonAlpha(float(extraFlags.w), float(baseSample.a * baseColorFactor.a), cutoff); + // RealityKit does not expose the fully evaluated lit term here; use runtime light color as the lit approximation. + float3 outlineLit = mix(float3(1.0), float3(lightColorParameter.rgb), clamp(float(outlineParams.z), 0.0, 1.0)); + float3 finalColor = float3(outlineColor.rgb) * outlineLit; + + surface.set_base_color(half3(0.0h)); + surface.set_emissive_color(half3(finalColor)); + surface.set_opacity(half(opacity)); + surface.set_roughness(1.0h); + surface.set_metallic(0.0h); +} + +float2 mtoonAnimatedUV(realitykit::geometry_parameters params, + float2 uv, + half4 uvAnimation, + half4 featureFlags, + half4 uvAnimationMaskSampler, + half4 uvTransform, + half4 uvTransformRotation) +{ + float time = params.uniforms().custom_parameter().w; + float mask = 1.0; + if (featureFlags.w > 0.5h) { + float2 maskUV = mtoonTextureUV(mtoonTransformedUV(uv, uvTransform, uvTransformRotation)); + mask = float(mtoonSample(params.textures().ambient_occlusion(), maskUV, uvAnimationMaskSampler).b); + } + + float angle = float(uvAnimation.z) * time * mask; + float2 center = float2(0.5, 0.5); + float2 centered = uv - center; + float s = sin(angle); + float c = cos(angle); + float2 rotated = float2(centered.x * c - centered.y * s, + centered.x * s + centered.y * c) + center; + return rotated + float2(float(uvAnimation.x), float(uvAnimation.y)) * time * mask; +} + +float mtoonScreenOutlineWidth(realitykit::geometry_parameters params, float width, float3 modelNormal) +{ + float4x4 modelToView = params.uniforms().model_to_view(); + float4x4 viewToProjection = params.uniforms().view_to_projection(); + float4 viewPosition = modelToView * float4(params.geometry().model_position(), 1.0); + float3 viewNormal = normalize((modelToView * float4(modelNormal, 0.0)).xyz); + float4 clipPosition = viewToProjection * viewPosition; + float4 offsetClipPosition = viewToProjection * (viewPosition + float4(viewNormal, 0.0)); + float clipW = clipPosition.w; + if (abs(clipW) < mtoonEpsilon) { + clipW = clipW < 0.0 ? -mtoonEpsilon : mtoonEpsilon; + } + float offsetClipW = offsetClipPosition.w; + if (abs(offsetClipW) < mtoonEpsilon) { + offsetClipW = offsetClipW < 0.0 ? -mtoonEpsilon : mtoonEpsilon; + } + float2 ndc = clipPosition.xy / clipW; + float2 offsetNdc = offsetClipPosition.xy / offsetClipW; + float ndcPerModelUnit = length(offsetNdc - ndc); + if (ndcPerModelUnit < mtoonEpsilon) { + return 0.0; + } + // geometry_parameters has projection matrices but no viewport height, so this treats width as a normalized screen-height fraction. + return (width * 2.0) / ndcPerModelUnit; +} + +[[visible]] +void mtoonOutlineGeometry(realitykit::geometry_parameters params) +{ + half4 uvTransform = mtoonParameter(params.textures(), 14.0); + half4 uvTransformRotation = mtoonParameter(params.textures(), 15.0); + float2 uv = params.geometry().uv0(); + half4 uvAnimation = mtoonParameter(params.textures(), 8.0); + half4 featureFlags = mtoonParameter(params.textures(), 9.0); + half4 uvAnimationMaskSampler = mtoonSamplerParameter(params.textures(), 8.0); + uv = mtoonAnimatedUV(params, + uv, + uvAnimation, + featureFlags, + uvAnimationMaskSampler, + uvTransform, + uvTransformRotation); + uv = mtoonTransformedUV(uv, uvTransform, uvTransformRotation); + params.geometry().set_uv0(uv); + + half4 outlineParams = mtoonParameter(params.textures(), 7.0); + if (outlineParams.w < 0.5h) { + return; + } + + float2 widthUV = mtoonTextureUV(uv); + half4 outlineWidthSampler = mtoonSamplerParameter(params.textures(), 7.0); + float widthMask = float(mtoonSample(params.textures().clearcoat(), widthUV, outlineWidthSampler).g); + float width = max(0.0, float(outlineParams.x)) * widthMask; + float3 modelNormal = normalize(params.geometry().normal()); + if (outlineParams.y > 1.5h) { + width = mtoonScreenOutlineWidth(params, width, modelNormal); + } + params.geometry().set_model_position_offset(modelNormal * width); +} diff --git a/Sources/VRMRealityKit/VRMEntityLoader+convenience.swift b/Sources/VRMRealityKit/VRMEntityLoader+convenience.swift index 9cf5236..2aed280 100644 --- a/Sources/VRMRealityKit/VRMEntityLoader+convenience.swift +++ b/Sources/VRMRealityKit/VRMEntityLoader+convenience.swift @@ -4,19 +4,53 @@ import VRMKit @available(iOS 18.0, macOS 15.0, visionOS 2.0, *) extension VRMEntityLoader { - public convenience init(withURL url: URL, rootDirectory: URL? = nil) throws { + /// Loads a VRM from a file URL. + /// + /// - Parameters: + /// - url: VRM file location. + /// - rootDirectory: Optional base directory for external glTF resources. + /// - isMToonEnabled: When `false`, MToon is fully disabled and Unlit / PBR fallbacks are used. + /// - isOutlineEnabled: Controls creation of MToon outline entities. + /// - isShadowCastingEnabled: Controls model participation in RealityKit shadow passes. + public convenience init(withURL url: URL, + rootDirectory: URL? = nil, + isMToonEnabled: Bool = true, + isOutlineEnabled: Bool = true, + isShadowCastingEnabled: Bool = true) throws { let vrm = try VRMLoader().load(withURL: url) - self.init(vrm: vrm, rootDirectory: rootDirectory) + self.init(vrm: vrm, + rootDirectory: rootDirectory, + isMToonEnabled: isMToonEnabled, + isOutlineEnabled: isOutlineEnabled, + isShadowCastingEnabled: isShadowCastingEnabled) } - public convenience init(named: String, rootDirectory: URL? = nil) throws { + /// Loads a bundled VRM resource. + public convenience init(named: String, + rootDirectory: URL? = nil, + isMToonEnabled: Bool = true, + isOutlineEnabled: Bool = true, + isShadowCastingEnabled: Bool = true) throws { let vrm = try VRMLoader().load(named: named) - self.init(vrm: vrm, rootDirectory: rootDirectory) + self.init(vrm: vrm, + rootDirectory: rootDirectory, + isMToonEnabled: isMToonEnabled, + isOutlineEnabled: isOutlineEnabled, + isShadowCastingEnabled: isShadowCastingEnabled) } - public convenience init(withData data: Data, rootDirectory: URL? = nil) throws { + /// Loads a VRM from in-memory data. + public convenience init(withData data: Data, + rootDirectory: URL? = nil, + isMToonEnabled: Bool = true, + isOutlineEnabled: Bool = true, + isShadowCastingEnabled: Bool = true) throws { let vrm = try VRMLoader().load(withData: data) - self.init(vrm: vrm, rootDirectory: rootDirectory) + self.init(vrm: vrm, + rootDirectory: rootDirectory, + isMToonEnabled: isMToonEnabled, + isOutlineEnabled: isOutlineEnabled, + isShadowCastingEnabled: isShadowCastingEnabled) } } #endif diff --git a/Sources/VRMRealityKit/VRMEntityLoader.swift b/Sources/VRMRealityKit/VRMEntityLoader.swift index cd362e3..c425c24 100644 --- a/Sources/VRMRealityKit/VRMEntityLoader.swift +++ b/Sources/VRMRealityKit/VRMEntityLoader.swift @@ -1,7 +1,9 @@ #if canImport(RealityKit) import CoreGraphics +import Foundation import RealityKit import Metal +import OSLog import VRMKit import VRMKitRuntime @@ -15,17 +17,48 @@ open class VRMEntityLoader { private var rootDirectory: URL? = nil private let entityName: String? private weak var currentEntity: VRMEntity? + private static let logger = Logger(subsystem: "dev.tattn.VRMKit", category: "MToon") + // VRMEntityLoader is @MainActor-isolated, so these mutable shader caches are only touched on the main actor. + private static let mtoonShaderDevice = MTLCreateSystemDefaultDevice() + private static var mtoonDefaultLibraryCache: MTLLibrary? + private static let requiredMToonFunctionNames: Set = [ + "mtoonSurface", + "mtoonOutlineSurface", + "mtoonOutlineGeometry" + ] private var textureCacheBySemantic: [TextureResource.Semantic: [Int: TextureResource]] = [:] private var metallicRoughnessCache: [Int: (metal: TextureResource, rough: TextureResource)] = [:] private var samplerCache: [Int: MaterialParameters.Texture.Sampler] = [:] + private var defaultSamplerCache: MaterialParameters.Texture.Sampler? + private var whiteTextureCache: TextureResource? + private var neutralNormalTextureCache: TextureResource? + private var mtoonParameterCache: [Int: MToonMaterialParameters] = [:] + private var mtoonOutlineMaterialCache: [Int: Material] = [:] + private var loggedMToonUVLimitations: Set = [] private var enableNormalTangentBlendShape = false // NOTE: Setting this to true currently has no effect - - public init(vrm: VRM, rootDirectory: URL? = nil) { + /// When `false`, MToon materials are not created and the loader falls back to Unlit / PBR materials. + /// visionOS always uses the fallback because `CustomMaterial` is unavailable there. + public let isMToonEnabled: Bool + /// Controls creation of MToon's inverted-hull outline entities. + /// visionOS does not create MToon outlines because `CustomMaterial` is unavailable there. + public let isOutlineEnabled: Bool + /// Controls whether loaded model entities participate in RealityKit shadow passes. + /// This option has no effect on visionOS, where the required shadow components are unavailable. + public let isShadowCastingEnabled: Bool + + public init(vrm: VRM, + rootDirectory: URL? = nil, + isMToonEnabled: Bool = true, + isOutlineEnabled: Bool = true, + isShadowCastingEnabled: Bool = true) { self.vrm = vrm self.gltf = vrm.gltf.jsonData self.rootDirectory = rootDirectory self.entityName = vrm.meta.title self.entityData = EntityData(vrm: gltf) + self.isMToonEnabled = isMToonEnabled + self.isOutlineEnabled = isOutlineEnabled + self.isShadowCastingEnabled = isShadowCastingEnabled } public func loadEntity() throws -> VRMEntity { @@ -52,6 +85,10 @@ open class VRMEntityLoader { try vrmEntity.setUpSpringBones(loader: self) // TODO: animations. + if !isShadowCastingEnabled { + disableShadowCasting(in: vrmEntity.entity) + } + entityData.entities[index] = vrmEntity return vrmEntity } @@ -158,8 +195,8 @@ open class VRMEntityLoader { let sharedTargets = targetsByPositionAccessor[positionAccessor] { resolvedPrimitive.targets = sharedTargets } - if let modelEntity = try modelEntity(withPrimitive: resolvedPrimitive, skinIndex: skinIndex) { - meshEntity.addChild(modelEntity) + if let primitiveEntity = try modelEntity(withPrimitive: resolvedPrimitive, skinIndex: skinIndex) { + meshEntity.addChild(primitiveEntity) } } @@ -176,7 +213,7 @@ open class VRMEntityLoader { return meshEntity } - private func modelEntity(withPrimitive primitive: GLTF.Mesh.Primitive, skinIndex: Int?) throws -> ModelEntity? { + private func modelEntity(withPrimitive primitive: GLTF.Mesh.Primitive, skinIndex: Int?) throws -> Entity? { guard supportsTriangles(primitive.mode) else { return nil } let attributes = primitive.attributes.rawValue @@ -336,6 +373,9 @@ open class VRMEntityLoader { let modelEntity = ModelEntity(mesh: mesh, materials: [material]) if let materialIndex = primitive.material { modelEntity.components.set(VRMMaterialIndexComponent(materialIndex: materialIndex)) + if let parameters = try mtoonParameters(withMaterialIndex: materialIndex) { + modelEntity.components.set(MToonMaterialParametersComponent(parameters: parameters)) + } } if hasBlendShapes { let mapping = BlendShapeWeightsMapping(meshResource: mesh) @@ -352,6 +392,35 @@ open class VRMEntityLoader { if let skinIndex, let boundSkeleton { try registerSkinBinding(modelEntity: modelEntity, skinIndex: skinIndex, skeleton: boundSkeleton) } + if let materialIndex = primitive.material, + let outlineMaterial = try mtoonOutlineMaterial(withMaterialIndex: materialIndex) { + let outlineEntity = ModelEntity(mesh: mesh, materials: [outlineMaterial]) + outlineEntity.name = "\(modelEntity.name)_outline" + outlineEntity.components.set(VRMMaterialIndexComponent(materialIndex: materialIndex)) + if let parameters = try mtoonParameters(withMaterialIndex: materialIndex) { + outlineEntity.components.set(MToonMaterialParametersComponent(parameters: parameters)) + } + if hasBlendShapes { + let mapping = BlendShapeWeightsMapping(meshResource: mesh) + outlineEntity.components.set(BlendShapeWeightsComponent(weightsMapping: mapping)) + } + if enableNormalTangentBlendShape, + !finalNormalOffsets.isEmpty || !finalTangentOffsets.isEmpty { + let component = BlendShapeNormalTangentComponent(baseNormals: finalNormals, + baseTangents: finalTangents, + normalOffsets: finalNormalOffsets, + tangentOffsets: finalTangentOffsets) + outlineEntity.components.set(component) + } + if let skinIndex, let boundSkeleton { + try registerSkinBinding(modelEntity: outlineEntity, skinIndex: skinIndex, skeleton: boundSkeleton) + } + let container = Entity() + container.name = "\(modelEntity.name)_container" + container.addChild(outlineEntity) + container.addChild(modelEntity) + return container + } return modelEntity } @@ -413,9 +482,10 @@ open class VRMEntityLoader { return vrm0.materialPropertyNameMap[name] }() let shaderName = materialProperty?.shader.lowercased() + let mtoon = MToonMaterialDescriptor(material: gltfMaterial, materialProperty: materialProperty) // MToon / Unlit variants are not PBR, so use UnlitMaterial for consistent rendering // This matches SceneKit's behavior which uses lightingModel = .constant - let isMToon = shaderName?.contains("mtoon") == true || gltfMaterial.extensions?.materialsMToon != nil + let isMToon = mtoon != nil || shaderName?.contains("mtoon") == true || gltfMaterial.extensions?.materialsMToon != nil let isUnlit = shaderName?.contains("unlit") == true || gltfMaterial.extensions?.materialsUnlit != nil let useUnlit = isMToon || isUnlit let hasAlphaPremultiply = materialProperty?.keywordMap["_ALPHAPREMULTIPLY_ON"] == true @@ -451,6 +521,17 @@ open class VRMEntityLoader { alpha: CGFloat(factor.a)) }() +#if !os(visionOS) + if isMToonEnabled, let mtoon, let library = mtoonShaderLibrary() { + let textureTransform = try mtoonTextureTransform(withMaterialIndex: index) + let material = try customMToonMaterial(mtoon, + textureTransform: textureTransform, + library: library) + entityData.materials[index] = material + return material + } +#endif + if useUnlit { var material = UnlitMaterial() if let pbr = gltfMaterial.pbrMetallicRoughness, @@ -522,6 +603,350 @@ open class VRMEntityLoader { return material } +#if !os(visionOS) + private func customMToonMaterial(_ mtoon: MToonMaterialDescriptor, + textureTransform: MaterialParameterTypes.TextureCoordinateTransform, + library: MTLLibrary) throws -> Material { + // RealityKit has no material-level draw-order hook, so + // MToon renderQueueOffsetNumber falls back to 0 in this renderer. + let surface = CustomMaterial.SurfaceShader(named: "mtoonSurface", in: library) + var material = try CustomMaterial(surfaceShader: surface, lightingModel: .unlit) + if let baseTexture = mtoon.baseColorTexture { + let textureParam = try customMToonTexture(withTextureIndex: baseTexture.index, slot: .base) + material.baseColor = .init(tint: .white, texture: textureParam) + } else { + material.baseColor = .init(tint: .white, texture: try whiteCustomTexture()) + } + + if let shadeTexture = mtoon.shadeMultiplyTexture { + material.roughness.texture = try customMToonTexture(withTextureIndex: shadeTexture.index, slot: .shade) + } else { + material.roughness.texture = try whiteCustomTexture() + } + if let shadingShiftTexture = mtoon.shadingShiftTexture { + material.specular.texture = try customMToonTexture(withTextureIndex: shadingShiftTexture.index, + slot: .shadingShift) + } else { + material.specular.texture = try whiteCustomTexture() + } + if let matcapTexture = mtoon.matcapTexture { + material.metallic.texture = try customMToonTexture(withTextureIndex: matcapTexture.index, slot: .matcap) + } else { + material.metallic.texture = try whiteCustomTexture() + } + + if let normalTexture = mtoon.normalTexture { + material.normal.texture = try customMToonTexture(withTextureIndex: normalTexture.index, slot: .normal) + } else { + material.normal.texture = try neutralNormalCustomTexture() + } + + if let emissiveTexture = mtoon.emissiveTexture { + let textureParam = try customMToonTexture(withTextureIndex: emissiveTexture.index, slot: .emissive) + material.emissiveColor = .init(color: .white, texture: textureParam) + } else { + material.emissiveColor = .init(color: .white, texture: try whiteCustomTexture()) + } + if let rimTexture = mtoon.rimMultiplyTexture { + material.clearcoatRoughness.texture = try customMToonTexture(withTextureIndex: rimTexture.index, + slot: .rim) + } else { + material.clearcoatRoughness.texture = try whiteCustomTexture() + } + if let outlineWidthTexture = mtoon.outlineWidthMultiplyTexture { + material.clearcoat.texture = try customMToonTexture(withTextureIndex: outlineWidthTexture.index, + slot: .outlineWidth) + } else { + material.clearcoat.texture = try whiteCustomTexture() + } + if let uvMaskTexture = mtoon.uvAnimationMaskTexture { + material.ambientOcclusion.texture = try customMToonTexture(withTextureIndex: uvMaskTexture.index, + slot: .uvAnimationMask) + } else { + material.ambientOcclusion.texture = try whiteCustomTexture() + } + + applyAlphaMode(mtoon.alphaMode, alphaCutoff: mtoon.alphaCutoff, to: &material) + switch mtoon.cullMode { + case .none: + material.faceCulling = .none + case .front: + material.faceCulling = .front + case .back: + material.faceCulling = .back + } + material.textureCoordinateTransform = textureTransform + + let parameters = try mtoonParameters(for: mtoon, textureTransform: textureTransform) + material.custom.value = parameters.customValue + material.custom.texture = CustomMaterial.Texture(try parameters.textureResource()) + return material + } + + private func customMToonOutlineMaterial(_ mtoon: MToonMaterialDescriptor, + textureTransform: MaterialParameterTypes.TextureCoordinateTransform, + library: MTLLibrary) throws -> Material { + let surface = CustomMaterial.SurfaceShader(named: "mtoonOutlineSurface", in: library) + let geometry = CustomMaterial.GeometryModifier(named: "mtoonOutlineGeometry", in: library) + var material = try CustomMaterial(surfaceShader: surface, + geometryModifier: geometry, + lightingModel: .unlit) + material.faceCulling = .front + if let baseTexture = mtoon.baseColorTexture { + material.baseColor = .init(tint: .white, + texture: try customMToonTexture(withTextureIndex: baseTexture.index, + slot: .base)) + } else { + material.baseColor = .init(tint: .white, texture: try whiteCustomTexture()) + } + if let outlineWidthTexture = mtoon.outlineWidthMultiplyTexture { + material.clearcoat.texture = try customMToonTexture(withTextureIndex: outlineWidthTexture.index, + slot: .outlineWidth) + } else { + material.clearcoat.texture = try whiteCustomTexture() + } + if let uvMaskTexture = mtoon.uvAnimationMaskTexture { + material.ambientOcclusion.texture = try customMToonTexture(withTextureIndex: uvMaskTexture.index, + slot: .uvAnimationMask) + } else { + material.ambientOcclusion.texture = try whiteCustomTexture() + } + applyAlphaMode(mtoon.alphaMode, alphaCutoff: mtoon.alphaCutoff, to: &material) + material.textureCoordinateTransform = textureTransform + let parameters = try mtoonParameters(for: mtoon, textureTransform: textureTransform) + material.custom.value = parameters.customValue + material.custom.texture = CustomMaterial.Texture(try parameters.textureResource()) + return material + } +#endif + + private func disableShadowCasting(on modelEntity: ModelEntity) { +#if !os(visionOS) + modelEntity.components.set(DynamicLightShadowComponent(castsShadow: false)) + modelEntity.components.set(GroundingShadowComponent(castsShadow: false, receivesShadow: false)) +#else + _ = modelEntity +#endif + } + + private func disableShadowCasting(in entity: Entity) { + if let modelEntity = entity as? ModelEntity { + disableShadowCasting(on: modelEntity) + } + for child in entity.children { + disableShadowCasting(in: child) + } + } + + func currentMaterialColor(withMaterialIndex index: Int, + type: VRM1.Expressions.Expression.MaterialColorBind.MaterialColorType) throws -> SIMD4 { + if let color = try mtoonParameters(withMaterialIndex: index)?.color(for: type) { + return color + } + return try material(withMaterialIndex: index).currentColor(for: type) + } + + private func mtoonParameters(withMaterialIndex index: Int) throws -> MToonMaterialParameters? { + guard isMToonEnabled else { + return nil + } + if let parameters = mtoonParameterCache[index] { + return parameters + } + guard let descriptor = try mtoonDescriptor(withMaterialIndex: index) else { + return nil + } + let parameters = try mtoonParameters(for: descriptor, + textureTransform: mtoonTextureTransform(withMaterialIndex: index)) + mtoonParameterCache[index] = parameters + return parameters + } + + private func mtoonParameters(for descriptor: MToonMaterialDescriptor, + textureTransform: MaterialParameterTypes.TextureCoordinateTransform) throws -> MToonMaterialParameters { + var parameters = MToonMaterialParameters(descriptor) + parameters.setTextureTransform(scale: textureTransform.scale, + offset: textureTransform.offset, + rotation: textureTransform.rotation) + try parameters.setSampler(mtoonSamplerParameters(for: descriptor.baseColorTexture), for: .base) + try parameters.setSampler(mtoonSamplerParameters(for: descriptor.shadeMultiplyTexture), for: .shade) + try parameters.setSampler(mtoonSamplerParameters(for: descriptor.shadingShiftTexture), for: .shadingShift) + try parameters.setSampler(mtoonSamplerParameters(for: descriptor.normalTexture), for: .normal) + try parameters.setSampler(mtoonSamplerParameters(for: descriptor.matcapTexture), for: .matcap) + try parameters.setSampler(mtoonSamplerParameters(for: descriptor.emissiveTexture), for: .emissive) + try parameters.setSampler(mtoonSamplerParameters(for: descriptor.rimMultiplyTexture), for: .rim) + try parameters.setSampler(mtoonSamplerParameters(for: descriptor.outlineWidthMultiplyTexture), for: .outlineWidth) + try parameters.setSampler(mtoonSamplerParameters(for: descriptor.uvAnimationMaskTexture), for: .uvAnimationMask) + return parameters + } + + private func mtoonTextureTransform(withMaterialIndex index: Int) throws -> MaterialParameterTypes.TextureCoordinateTransform { + let materials = try gltf.load(\.materials) + guard materials.indices.contains(index) else { + throw VRMError._dataInconsistent("Material index \(index) out of bounds") + } + let material = materials[index] + let mtoon = material.extensions?.materialsMToon + // Custom meshes on the minimum supported RealityKit versions expose only + // TEXCOORD_0 and CustomMaterial has one material-level UV transform. Use + // the first UV-accessed MToon transform deterministically. Expression + // textureTransformBinds still update every UV-accessed texture together, + // as required by VRMC_vrm. + let textureInfos: [(texCoord: Int, extensions: CodableAny?)] = [ + material.pbrMetallicRoughness?.baseColorTexture.map { ($0.texCoord, $0.extensions) }, + mtoon?.shadeMultiplyTexture.map { ($0.texCoord ?? 0, $0.extensions) }, + mtoon?.shadingShiftTexture.map { ($0.texCoord ?? 0, $0.extensions) }, + material.normalTexture.map { ($0.texCoord, $0.extensions) }, + material.emissiveTexture.map { ($0.texCoord, $0.extensions) }, + mtoon?.rimMultiplyTexture.map { ($0.texCoord ?? 0, $0.extensions) }, + mtoon?.outlineWidthMultiplyTexture.map { ($0.texCoord ?? 0, $0.extensions) }, + mtoon?.uvAnimationMaskTexture.map { ($0.texCoord ?? 0, $0.extensions) } + ].compactMap { $0 } + + let transforms = textureInfos.compactMap { textureTransform(from: $0.extensions) } + let selectedTransform = transforms.first ?? MaterialParameterTypes.TextureCoordinateTransform() + let usesUnsupportedTexCoord = textureInfos.contains { + textureCoordinate(from: $0.extensions, default: $0.texCoord) != 0 + } + let hasDifferentTransforms = textureInfos.contains { + let transform = textureTransform(from: $0.extensions) + ?? MaterialParameterTypes.TextureCoordinateTransform() + return textureTransform(transform, differsFrom: selectedTransform) + } + if (usesUnsupportedTexCoord || hasDifferentTransforms), + loggedMToonUVLimitations.insert(index).inserted { + if usesUnsupportedTexCoord { + Self.logger.warning("MToon material \(index, privacy: .public) requests a nonzero texCoord; RealityKit uses TEXCOORD_0 on supported deployment targets.") + } + if hasDifferentTransforms { + Self.logger.warning("MToon material \(index, privacy: .public) has per-texture KHR_texture_transform values; RealityKit uses the first UV-accessed transform for all MToon textures.") + } + } + return selectedTransform + } + + private func textureTransform(from extensions: CodableAny?) -> MaterialParameterTypes.TextureCoordinateTransform? { + guard let extensions = extensions?.value as? [String: Any], + let transform = extensions["KHR_texture_transform"] as? [String: Any] else { + return nil + } + let offset = transform.simd2Value(forKey: "offset", default: SIMD2(0, 0)) + let scale = transform.simd2Value(forKey: "scale", default: SIMD2(1, 1)) + let rotation = transform.floatValue(forKey: "rotation", default: 0) + return MaterialParameterTypes.TextureCoordinateTransform(offset: offset, + scale: scale, + rotation: rotation) + } + + private func textureCoordinate(from extensions: CodableAny?, default defaultValue: Int) -> Int { + guard let extensions = extensions?.value as? [String: Any], + let transform = extensions["KHR_texture_transform"] as? [String: Any], + let texCoord = transform["texCoord"] else { + return defaultValue + } + return Int(numericFloatValue(texCoord, default: Float(defaultValue))) + } + + private func textureTransform(_ lhs: MaterialParameterTypes.TextureCoordinateTransform, + differsFrom rhs: MaterialParameterTypes.TextureCoordinateTransform) -> Bool { + lhs.scale != rhs.scale || lhs.offset != rhs.offset || abs(lhs.rotation - rhs.rotation) > 0.000_001 + } + + private func mtoonOutlineMaterial(withMaterialIndex index: Int) throws -> Material? { +#if os(visionOS) + return nil +#else + guard isMToonEnabled else { + return nil + } + guard isOutlineEnabled else { + return nil + } + if let material = mtoonOutlineMaterialCache[index] { + return material + } + guard let descriptor = try mtoonDescriptor(withMaterialIndex: index), + descriptor.hasOutline, + let library = mtoonShaderLibrary() else { + return nil + } + let material = try customMToonOutlineMaterial(descriptor, + textureTransform: mtoonTextureTransform(withMaterialIndex: index), + library: library) + mtoonOutlineMaterialCache[index] = material + return material +#endif + } + + private func mtoonDescriptor(withMaterialIndex index: Int) throws -> MToonMaterialDescriptor? { + let materials = try gltf.load(\.materials) + guard materials.indices.contains(index) else { + throw VRMError._dataInconsistent("Material index \(index) out of bounds") + } + let gltfMaterial = materials[index] + let materialProperty: VRM0.MaterialProperty? = { + guard case .v0(let vrm0) = vrm, + let name = gltfMaterial.name else { return nil } + return vrm0.materialPropertyNameMap[name] + }() + return MToonMaterialDescriptor(material: gltfMaterial, materialProperty: materialProperty) + } + + private func mtoonShaderLibrary() -> MTLLibrary? { + guard let device = Self.mtoonShaderDevice else { + Self.logger.error("Failed to create Metal device for MToon shader.") + return nil + } + return Self.mtoonDefaultLibrary(device: device) + } + + // The MToon shader is precompiled offline into per-platform metallibs by + // Scripts/build-mtoon-metallibs.sh and bundled as package resources. + // This avoids depending on the consumer's build system compiling the + // package's .metal source (unsupported by `swift build`, and unreliable + // on Xcode versions where the Metal Toolchain is a separate download), + // and is safe for App Store / sandboxed distribution. + private static let bundledMToonLibraryResourceName: String? = { +#if os(macOS) && !targetEnvironment(macCatalyst) + return "MToon-macos" +#elseif os(iOS) && targetEnvironment(simulator) + return "MToon-iossim" +#elseif os(iOS) && !targetEnvironment(macCatalyst) + return "MToon-ios" +#else + // No precompiled MToon library is bundled for this platform + // (e.g. Mac Catalyst); MToon rendering falls back to UnlitMaterial. + return nil +#endif + }() + + private static func mtoonDefaultLibrary(device: MTLDevice) -> MTLLibrary? { + if let library = mtoonDefaultLibraryCache { + return library + } + guard let resourceName = bundledMToonLibraryResourceName else { + logger.error("No precompiled MToon shader library is bundled for this platform.") + return nil + } + guard let libraryURL = Bundle.module.url(forResource: resourceName, withExtension: "metallib") else { + logger.error("Missing bundled MToon shader library resource: \(resourceName, privacy: .public).metallib") + return nil + } + do { + let library = try device.makeLibrary(URL: libraryURL) + guard requiredMToonFunctionNames.isSubset(of: Set(library.functionNames)) else { + logger.error("Bundled MToon shader library is missing required functions: \(library.functionNames, privacy: .public)") + return nil + } + logger.notice("Loaded bundled MToon shader library: \(resourceName, privacy: .public).metallib") + mtoonDefaultLibraryCache = library + return library + } catch { + logger.error("Failed to load bundled MToon shader library: \(error.localizedDescription, privacy: .public)") + return nil + } + } + func texture(withTextureIndex index: Int, semantic: TextureResource.Semantic = .color) throws -> TextureResource { if semantic == .color, let cache = try entityData.load(\.textures, index: index) { return cache @@ -550,6 +975,80 @@ open class VRMEntityLoader { return MaterialParameters.Texture(texture, sampler: sampler) } +#if !os(visionOS) + private func customTexture(withTextureIndex index: Int, + semantic: TextureResource.Semantic = .color) throws -> CustomMaterial.Texture { + CustomMaterial.Texture(try texture(withTextureIndex: index, semantic: semantic)) + } + + private func customMToonTexture(withTextureIndex index: Int, + slot: MToonTextureSlot) throws -> CustomMaterial.Texture { + try customTexture(withTextureIndex: index, semantic: slot.semantic) + } +#endif + + private func whiteTextureParameter() throws -> MaterialParameters.Texture { + return MaterialParameters.Texture(try whiteTextureResource(), sampler: defaultSampler()) + } + +#if !os(visionOS) + private func whiteCustomTexture() throws -> CustomMaterial.Texture { + CustomMaterial.Texture(try whiteTextureResource()) + } + + private func neutralNormalCustomTexture() throws -> CustomMaterial.Texture { + CustomMaterial.Texture(try neutralNormalTextureResource()) + } +#endif + + private func whiteTextureResource() throws -> TextureResource { + if let whiteTextureCache { + return whiteTextureCache + } + let data = Data([255, 255, 255, 255]) + guard let provider = CGDataProvider(data: data as CFData), + let image = CGImage(width: 1, + height: 1, + bitsPerComponent: 8, + bitsPerPixel: 32, + bytesPerRow: 4, + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue), + provider: provider, + decode: nil, + shouldInterpolate: false, + intent: .defaultIntent) else { + throw VRMError._dataInconsistent("failed to create white texture") + } + let texture = try TextureResource(image: image, options: .init(semantic: .color)) + whiteTextureCache = texture + return texture + } + + private func neutralNormalTextureResource() throws -> TextureResource { + if let neutralNormalTextureCache { + return neutralNormalTextureCache + } + let data = Data([128, 128, 255, 255]) + guard let provider = CGDataProvider(data: data as CFData), + let image = CGImage(width: 1, + height: 1, + bitsPerComponent: 8, + bitsPerPixel: 32, + bytesPerRow: 4, + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue), + provider: provider, + decode: nil, + shouldInterpolate: false, + intent: .defaultIntent) else { + throw VRMError._dataInconsistent("failed to create neutral normal texture") + } + let texture = try TextureResource(image: image, options: .init(semantic: .normal)) + neutralNormalTextureCache = texture + return texture + } + private func sampler(withTextureIndex index: Int) throws -> MaterialParameters.Texture.Sampler { if let cache = samplerCache[index] { return cache @@ -567,6 +1066,17 @@ open class VRMEntityLoader { return sampler } + private func defaultSampler() -> MaterialParameters.Texture.Sampler { + if let defaultSamplerCache { + return defaultSamplerCache + } + let descriptor = MTLSamplerDescriptor() + applyDefaultSampler(to: descriptor) + let sampler = MaterialParameters.Texture.Sampler(descriptor) + defaultSamplerCache = sampler + return sampler + } + private func applySampler(_ sampler: GLTF.Sampler, to descriptor: MTLSamplerDescriptor) { let magFilter = sampler.magFilter ?? .LINEAR let minFilter = sampler.minFilter ?? .LINEAR_MIPMAP_LINEAR @@ -587,6 +1097,41 @@ open class VRMEntityLoader { descriptor.tAddressMode = metalWrap(.REPEAT) } + private func mtoonSamplerParameters(for texture: MToonMaterialDescriptor.Texture?) throws -> SIMD4 { + guard let texture else { + return MToonMaterialParameters.defaultSampler + } + let textures = try gltf.load(\.textures) + guard textures.indices.contains(texture.index) else { + throw VRMError._dataInconsistent("Texture index \(texture.index) out of bounds") + } + guard let samplerIndex = textures[texture.index].sampler else { + return MToonMaterialParameters.defaultSampler + } + let samplers = try gltf.load(\.samplers) + guard samplers.indices.contains(samplerIndex) else { + throw VRMError._dataInconsistent("Sampler index \(samplerIndex) out of bounds") + } + return mtoonSamplerParameters(samplers[samplerIndex]) + } + + private func mtoonSamplerParameters(_ sampler: GLTF.Sampler) -> SIMD4 { + let minFilter = sampler.minFilter ?? .LINEAR_MIPMAP_LINEAR + let useNearest = sampler.magFilter == .NEAREST || minFilter.usesNearestMinification + return SIMD4(mtoonWrapMode(sampler.wrapS), + mtoonWrapMode(sampler.wrapT), + useNearest ? 1 : 0, + 0) + } + + private func mtoonWrapMode(_ wrap: GLTF.Sampler.Wrap) -> Float { + switch wrap { + case .REPEAT: return 0 + case .CLAMP_TO_EDGE: return 1 + case .MIRRORED_REPEAT: return 2 + } + } + private func metalFilter(_ filter: GLTF.Sampler.MagFilter) -> MTLSamplerMinMagFilter { switch filter { case .NEAREST: return .nearest @@ -763,6 +1308,24 @@ open class VRMEntityLoader { } } +#if !os(visionOS) + private func applyAlphaMode(_ mode: GLTF.Material.AlphaMode, + alphaCutoff: Float, + to material: inout CustomMaterial) { + switch mode { + case .OPAQUE: + material.blending = .opaque + material.opacityThreshold = nil + case .MASK: + material.blending = .opaque + material.opacityThreshold = alphaCutoff + case .BLEND: + material.blending = .transparent(opacity: .init(scale: 1.0)) + material.opacityThreshold = nil + } + } +#endif + private struct AccessorSlice { let data: Data let componentsPerVector: Int @@ -1448,4 +2011,59 @@ open class VRMEntityLoader { } } +private extension GLTF.Sampler.MinFilter { + var usesNearestMinification: Bool { + switch self { + case .NEAREST, .NEAREST_MIPMAP_NEAREST, .NEAREST_MIPMAP_LINEAR: + return true + case .LINEAR, .LINEAR_MIPMAP_NEAREST, .LINEAR_MIPMAP_LINEAR: + return false + } + } +} + +private extension Dictionary where Key == String, Value == Any { + func simd2Value(forKey key: String, default defaultValue: SIMD2) -> SIMD2 { + guard let values = self[key] as? [Any] else { return defaultValue } + return SIMD2(values.float(at: 0, default: defaultValue.x), + values.float(at: 1, default: defaultValue.y)) + } + + func floatValue(forKey key: String, default defaultValue: Float) -> Float { + guard let value = self[key] else { return defaultValue } + return numericFloatValue(value, default: defaultValue) + } +} + +private extension Array where Element == Any { + func float(at index: Int, default defaultValue: Float) -> Float { + guard indices.contains(index) else { return defaultValue } + return numericFloatValue(self[index], default: defaultValue) + } +} + +private func numericFloatValue(_ value: Any, default defaultValue: Float) -> Float { + switch value { + case let value as Float: + return value + case let value as Double: + return Float(value) + case let value as Int: + return Float(value) + case let value as NSNumber: + return value.floatValue + default: + return defaultValue + } +} + +private extension VRMColor { + convenience init(simd color: SIMD4) { + self.init(red: CGFloat(color.x), + green: CGFloat(color.y), + blue: CGFloat(color.z), + alpha: CGFloat(color.w)) + } +} + #endif diff --git a/Tests/VRMKitTests/BinaryGLTFTests.swift b/Tests/VRMKitTests/BinaryGLTFTests.swift index c22f348..a049465 100644 --- a/Tests/VRMKitTests/BinaryGLTFTests.swift +++ b/Tests/VRMKitTests/BinaryGLTFTests.swift @@ -13,5 +13,11 @@ class BinaryGLTFTests: XCTestCase { XCTAssertEqual(json.asset.generator, "UniGLTF") XCTAssertEqual(json.asset.version, "2.0") } + + func testStridedSubdataCopiesBytes() { + let data = Data([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + let strided = data.subdata(offset: 1, size: 2, stride: 4, count: 2) + XCTAssertEqual(Array(strided), [1, 2, 5, 6]) + } } diff --git a/Tests/VRMRealityKitTests/ARIntegrationTests.swift b/Tests/VRMRealityKitTests/ARIntegrationTests.swift new file mode 100644 index 0000000..6e6788b --- /dev/null +++ b/Tests/VRMRealityKitTests/ARIntegrationTests.swift @@ -0,0 +1,189 @@ +#if canImport(RealityKit) +import Foundation +import RealityKit +import Testing +import VRMKit +@testable import VRMRealityKit + +#if os(iOS) +import ARKit +#endif + +@Suite("AR MToon Integration") +@MainActor +struct ARIntegrationTests { + +#if !os(visionOS) + @Test + func outlineAndShadowOptionsAreIndependent() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let url = try seedSanURL() + + let noOutlineLoader = try VRMEntityLoader(withURL: url, + isOutlineEnabled: false, + isShadowCastingEnabled: true) + let noOutlineEntity = try noOutlineLoader.loadEntity() + #expect(noOutlineLoader.isOutlineEnabled == false) + #expect(noOutlineLoader.isShadowCastingEnabled) + #expect(noOutlineEntities(in: noOutlineEntity.entity)) + #expect(hasCustomMaterial(in: noOutlineEntity.entity)) + + let noShadowLoader = try VRMEntityLoader(withURL: url, + isOutlineEnabled: true, + isShadowCastingEnabled: false) + let noShadowEntity = try noShadowLoader.loadEntity() + #expect(noShadowLoader.isOutlineEnabled) + #expect(noShadowLoader.isShadowCastingEnabled == false) + #expect(hasOutlineEntities(in: noShadowEntity.entity)) + assertShadowCastingDisabled(in: noShadowEntity.entity) + } + + @Test + func mtoonMaterialsPreserveAlphaModeWhenAROptionsAreDisabled() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let url = try seedSanURL() + let loader = try VRMEntityLoader(withURL: url, + isOutlineEnabled: false, + isShadowCastingEnabled: false) + let opaqueMaterial = try #require(loader.material(withMaterialIndex: 0) as? CustomMaterial) + let blendMaterial = try #require(loader.material(withMaterialIndex: 4) as? CustomMaterial) + + #expect(isOpaque(opaqueMaterial.blending)) + #expect(isTransparent(blendMaterial.blending)) + } + + @Test + func disabledMToonUsesFallbackMaterialWithAROptions() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let url = try seedSanURL() + let loader = try VRMEntityLoader(withURL: url, + isMToonEnabled: false, + isOutlineEnabled: false, + isShadowCastingEnabled: false) + let vrmEntity = try loader.loadEntity() + + #expect(!hasCustomMaterial(in: vrmEntity.entity)) + assertShadowCastingDisabled(in: vrmEntity.entity) + } + + @Test + func defaultOptionsUseMToonAndOutlineEntities() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let url = try seedSanURL() + let loader = try VRMEntityLoader(withURL: url) + let vrmEntity = try loader.loadEntity() + + #expect(loader.isMToonEnabled) + #expect(loader.isOutlineEnabled) + #expect(loader.isShadowCastingEnabled) + #expect(hasCustomMaterial(in: vrmEntity.entity)) + #expect(hasOutlineEntities(in: vrmEntity.entity)) + } + +#if os(iOS) && !os(visionOS) + @Test + func loadMToonEntityInARViewSurvivesRenderFrames() async throws { + guard #available(iOS 18.0, *) else { return } + let arView = ARView(frame: CGRect(x: 0, y: 0, width: 390, height: 844)) + arView.renderOptions.insert(.disableGroundingShadows) + + let config = ARWorldTrackingConfiguration() + config.planeDetection = [.horizontal] + arView.session.run(config) + + let url = try seedSanURL() + let loader = try VRMEntityLoader(withURL: url, + isOutlineEnabled: false, + isShadowCastingEnabled: false) + let vrmEntity = try loader.loadEntity() + + let subscription = arView.scene.subscribe(to: SceneEvents.Update.self) { event in + vrmEntity.setMToonLightDirection(SIMD3(0, 0, -1)) + vrmEntity.update(deltaTime: event.deltaTime) + } + + let anchor = AnchorEntity(world: .zero) + anchor.addChild(vrmEntity.entity) + arView.scene.addAnchor(anchor) + + try await Task.sleep(nanoseconds: 1_000_000_000) + + subscription.cancel() + } +#endif + + private func seedSanURL() throws -> URL { + try #require(Bundle.module.url(forResource: "Seed-san", withExtension: "vrm")) + } + + private func allModelEntities(in entity: Entity) -> [ModelEntity] { + var result: [ModelEntity] = [] + if let model = entity as? ModelEntity { + result.append(model) + } + for child in entity.children { + result.append(contentsOf: allModelEntities(in: child)) + } + return result + } + + private func hasCustomMaterial(in entity: Entity) -> Bool { + allModelEntities(in: entity) + .flatMap { $0.components[ModelComponent.self]?.materials ?? [] } + .contains { $0 is CustomMaterial } + } + + private func noOutlineEntities(in entity: Entity) -> Bool { + if entity.name.hasSuffix("_outline") { + return false + } + return entity.children.allSatisfy { noOutlineEntities(in: $0) } + } + + private func hasOutlineEntities(in entity: Entity) -> Bool { + !noOutlineEntities(in: entity) + } + + @available(iOS 18.0, macOS 15.0, visionOS 2.0, *) + private func assertShadowCastingDisabled(in entity: Entity) { + for modelEntity in allModelEntities(in: entity) { + let dynamic = modelEntity.components[DynamicLightShadowComponent.self] + #expect(dynamic?.castsShadow == false) + + let grounding = modelEntity.components[GroundingShadowComponent.self] + #expect(grounding?.castsShadow == false) + #expect(grounding?.receivesShadow == false) + } + } + + private func isOpaque(_ blending: CustomMaterial.Blending) -> Bool { + if case .opaque = blending { + return true + } + return false + } + + private func isTransparent(_ blending: CustomMaterial.Blending) -> Bool { + if case .transparent = blending { + return true + } + return false + } +#endif + +#if os(visionOS) + @Test + func visionOSUsesFallbackMaterialWhenMToonIsRequested() throws { + guard #available(visionOS 2.0, *) else { return } + let url = try #require(Bundle.module.url(forResource: "Seed-san", withExtension: "vrm")) + let loader = try VRMEntityLoader(withURL: url) + let material = try loader.material(withMaterialIndex: 0) + + #expect(loader.isMToonEnabled) + #expect(loader.isOutlineEnabled) + #expect(loader.isShadowCastingEnabled) + #expect(material is UnlitMaterial || material is PhysicallyBasedMaterial) + } +#endif +} +#endif diff --git a/Tests/VRMRealityKitTests/MToonMaterialDescriptorTests.swift b/Tests/VRMRealityKitTests/MToonMaterialDescriptorTests.swift new file mode 100644 index 0000000..27bb097 --- /dev/null +++ b/Tests/VRMRealityKitTests/MToonMaterialDescriptorTests.swift @@ -0,0 +1,272 @@ +import Foundation +import Testing +@testable import VRMKit +@testable import VRMKitRuntime + +@Suite +struct MToonMaterialDescriptorTests { + @Test + func testVRM0DefaultValuesMigrateToMToon10Domain() throws { + let descriptor = try #require(MToonMaterialDescriptor(material: material(), + materialProperty: vrm0MaterialProperty())) + + #expect(descriptor.shadingToonyFactor.isApproximatelyEqual(to: 0.95)) + #expect(descriptor.shadingShiftFactor.isApproximatelyEqual(to: -0.05)) + #expect(descriptor.giEqualizationFactor.isApproximatelyEqual(to: 0.9)) + #expect(descriptor.rimLightingMixFactor == 0) + } + + @Test + func testVRM0MigratesOutlineWidthAndUvRotationUnits() throws { + let descriptor = try #require(MToonMaterialDescriptor( + material: material(), + materialProperty: vrm0MaterialProperty(floats: #""" + { + "_OutlineWidthMode": 1, + "_OutlineWidth": 3.5, + "_UvAnimRotation": 0.25 + } + """#) + )) + + #expect(descriptor.outlineWidthMode == .worldCoordinates) + #expect(descriptor.outlineWidthFactor.isApproximatelyEqual(to: 0.035)) + #expect(descriptor.uvAnimationRotationSpeedFactor.isApproximatelyEqual(to: 0.5 * Float.pi)) + } + + @Test + func testVRM0RenderQueueOffsetMigratesTransparentQueues() throws { + let transparentNoZWrite = try descriptor(renderQueue: 2994, + keywordMap: #"{"_ALPHABLEND_ON": true}"#, + tagMap: #"{"RenderType": "Transparent"}"#) + #expect(transparentNoZWrite.renderQueueOffsetNumber == -6) + + let transparentNoZWriteClamped = try descriptor(renderQueue: 2980, + keywordMap: #"{"_ALPHABLEND_ON": true}"#, + tagMap: #"{"RenderType": "Transparent"}"#) + #expect(transparentNoZWriteClamped.renderQueueOffsetNumber == -9) + + let transparentZWrite = try descriptor(renderQueue: 2508, + keywordMap: #"{"_ALPHABLEND_ON": true, "_ZWRITE_ON": true}"#, + tagMap: #"{"RenderType": "Transparent"}"#) + #expect(transparentZWrite.renderQueueOffsetNumber == 7) + + let transparentZWriteClamped = try descriptor(renderQueue: 2520, + keywordMap: #"{"_ALPHABLEND_ON": true, "_ZWRITE_ON": true}"#, + tagMap: #"{"RenderType": "Transparent"}"#) + #expect(transparentZWriteClamped.renderQueueOffsetNumber == 9) + + let shaderDefaultQueue = try descriptor(renderQueue: 0, + keywordMap: #"{"_ALPHABLEND_ON": true}"#, + tagMap: #"{"RenderType": "Transparent"}"#) + #expect(shaderDefaultQueue.renderQueueOffsetNumber == 0) + } + + @Test + func testVRM0EmissiveFieldsUseEmissionProperties() throws { + let descriptor = try #require(MToonMaterialDescriptor( + material: material(), + materialProperty: vrm0MaterialProperty(textures: #"{"_EmissionMap": 3}"#, + vectors: #"{"_EmissionColor": [0.2, 0.3, 0.4, 1.0]}"#) + )) + + #expect(descriptor.emissiveFactor.isApproximatelyEqual(to: SIMD3(0.2, 0.3, 0.4))) + #expect(descriptor.emissiveTexture?.index == 3) + #expect(descriptor.emissiveTexture?.texCoord == 0) + } + + @Test + func testVRM0ShadeTextureTakesPriorityOverMainTexture() throws { + let descriptor = try #require(MToonMaterialDescriptor( + material: material(), + materialProperty: vrm0MaterialProperty(textures: #"{"_MainTex": 2, "_ShadeTexture": 4}"#) + )) + + #expect(descriptor.baseColorTexture?.index == 2) + #expect(descriptor.shadeMultiplyTexture?.index == 4) + #expect(descriptor.shadeMultiplyTexture?.texCoord == 0) + } + + @Test + func testVRM0ShadeTextureFallsBackToMainTexture() throws { + let descriptor = try #require(MToonMaterialDescriptor( + material: material(), + materialProperty: vrm0MaterialProperty(textures: #"{"_MainTex": 2}"#) + )) + + #expect(descriptor.baseColorTexture?.index == 2) + #expect(descriptor.shadeMultiplyTexture?.index == 2) + #expect(descriptor.shadeMultiplyTexture?.texCoord == 0) + } + + @Test + func testVRM1MissingShadeTextureRemainsNil() throws { + let descriptor = try #require(MToonMaterialDescriptor( + material: material(#""" + { + "pbrMetallicRoughness": { + "baseColorTexture": { "index": 2, "texCoord": 1 } + }, + "extensions": { + "VRMC_materials_mtoon": { + "specVersion": "1.0" + } + } + } + """#), + materialProperty: nil + )) + + #expect(descriptor.baseColorTexture?.index == 2) + #expect(descriptor.shadeMultiplyTexture == nil) + } + + @Test + func testVRM1MToon10Defaults() throws { + let descriptor = try #require(MToonMaterialDescriptor( + material: vrm1Material(), + materialProperty: nil + )) + + #expect(descriptor.shadeColorFactor == SIMD4(0, 0, 0, 1)) + #expect(descriptor.parametricRimFresnelPowerFactor == 5) + #expect(descriptor.cullMode == .back) + #expect(descriptor.normalScale == 1) + } + + @Test + func testVRM0CullModePreservesFrontBackAndDisabledValues() throws { + let expected: [(Float, MToonMaterialDescriptor.CullMode)] = [ + (0, .none), + (1, .front), + (2, .back) + ] + + for (value, cullMode) in expected { + let descriptor = try #require(MToonMaterialDescriptor( + material: material(), + materialProperty: vrm0MaterialProperty(floats: #"{"_CullMode": \#(value)}"#) + )) + #expect(descriptor.cullMode == cullMode) + } + } + + @Test + func testVRM0InvalidCullModeUsesMaterialFallback() throws { + let descriptor = try #require(MToonMaterialDescriptor( + material: material(), + materialProperty: vrm0MaterialProperty(floats: #"{"_CullMode": 1.5}"#) + )) + + #expect(descriptor.cullMode == .back) + } + + @Test + func testVRM1DoubleSidedDisablesCulling() throws { + let descriptor = try #require(MToonMaterialDescriptor( + material: material(#"{"doubleSided": true, "extensions": {"VRMC_materials_mtoon": {"specVersion": "1.0"}}}"#), + materialProperty: nil + )) + + #expect(descriptor.cullMode == .none) + } + + @Test + func testNormalScaleUsesVRM0AndVRM1MaterialValues() throws { + let vrm0 = try #require(MToonMaterialDescriptor( + material: material(), + materialProperty: vrm0MaterialProperty(floats: #"{"_BumpScale": 0.4}"#, + textures: #"{"_BumpMap": 1}"#) + )) + let vrm1 = try #require(MToonMaterialDescriptor( + material: material(#"{"normalTexture": {"index": 1, "scale": 0.65}, "extensions": {"VRMC_materials_mtoon": {"specVersion": "1.0"}}}"#), + materialProperty: nil + )) + + #expect(vrm0.normalScale.isApproximatelyEqual(to: 0.4)) + #expect(vrm1.normalScale.isApproximatelyEqual(to: 0.65)) + } + + @Test + func testVRM1EmissiveFieldsAndMatcapDefaultUseGltfMaterial() throws { + let gltfMaterial = try material(#""" + { + "emissiveFactor": [0.6, 0.7, 0.8], + "emissiveTexture": { "index": 5, "texCoord": 1 }, + "extensions": { + "VRMC_materials_mtoon": { + "specVersion": "1.0" + } + } + } + """#) + let descriptor = try #require(MToonMaterialDescriptor(material: gltfMaterial, materialProperty: nil)) + + #expect(descriptor.emissiveFactor.isApproximatelyEqual(to: SIMD3(0.6, 0.7, 0.8))) + #expect(descriptor.emissiveTexture?.index == 5) + #expect(descriptor.emissiveTexture?.texCoord == 1) + #expect(descriptor.matcapFactor.isApproximatelyEqual(to: SIMD3(1, 1, 1))) + } + + private func descriptor(renderQueue: Int, + keywordMap: String, + tagMap: String) throws -> MToonMaterialDescriptor { + return try #require(MToonMaterialDescriptor( + material: material(#"{"alphaMode": "OPAQUE"}"#), + materialProperty: vrm0MaterialProperty(renderQueue: renderQueue, + keywordMap: keywordMap, + tagMap: tagMap) + )) + } + + private func material(_ json: String = "{}") throws -> GLTF.Material { + return try JSONDecoder().decode(GLTF.Material.self, from: Data(json.utf8)) + } + + private func vrm1Material() throws -> GLTF.Material { + return try material(#""" + { + "extensions": { + "VRMC_materials_mtoon": { + "specVersion": "1.0" + } + } + } + """#) + } + + private func vrm0MaterialProperty(renderQueue: Int = 0, + floats: String = "{}", + keywordMap: String = "{}", + tagMap: String = "{}", + textures: String = "{}", + vectors: String = "{}") throws -> VRM0.MaterialProperty { + let json = #""" + { + "name": "MToon", + "shader": "VRM/MToon", + "renderQueue": \#(renderQueue), + "floatProperties": \#(floats), + "keywordMap": \#(keywordMap), + "tagMap": \#(tagMap), + "textureProperties": \#(textures), + "vectorProperties": \#(vectors) + } + """# + return try JSONDecoder().decode(VRM0.MaterialProperty.self, from: Data(json.utf8)) + } +} + +private extension Float { + func isApproximatelyEqual(to other: Float, tolerance: Float = 0.0001) -> Bool { + return abs(self - other) < tolerance + } +} + +private extension SIMD3 where Scalar == Float { + func isApproximatelyEqual(to other: SIMD3, tolerance: Float = 0.0001) -> Bool { + return abs(x - other.x) < tolerance && + abs(y - other.y) < tolerance && + abs(z - other.z) < tolerance + } +} diff --git a/Tests/VRMRealityKitTests/VRM1RealityKitTests.swift b/Tests/VRMRealityKitTests/VRM1RealityKitTests.swift new file mode 100644 index 0000000..af617ae --- /dev/null +++ b/Tests/VRMRealityKitTests/VRM1RealityKitTests.swift @@ -0,0 +1,922 @@ +#if canImport(RealityKit) +import CryptoKit +import Foundation +import Metal +import RealityKit +import Testing +import VRMKit +@testable import VRMRealityKit + +@Suite +@MainActor +struct VRM1RealityKitTests { + +#if !os(visionOS) + @Test + func testVRM1MToonCustomMaterialUsesParameterTexture() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let url = try #require(Bundle.module.url(forResource: "Seed-san", withExtension: "vrm"), "Failed to load Seed-san.vrm resource from test bundle.") + let vrmLoader = try VRMEntityLoader(withURL: url) + let material = try vrmLoader.material(withMaterialIndex: 0) + let customMaterial = try #require(material as? CustomMaterial, + "Expected default MToon rendering to load a CustomMaterial. Run Scripts/build-mtoon-metallibs.sh and verify the package resources.") + + #expect(customMaterial.custom.texture != nil) + #expect(customMaterial.normal.texture != nil) + #expect(customMaterial.roughness.texture != nil) + #expect(customMaterial.emissiveColor.texture != nil) + #expect(customMaterial.clearcoat.texture != nil) + #expect(customMaterial.clearcoatRoughness.texture != nil) + + let direction = MToonMaterialParameters.defaultLightDirection + #expect(abs(customMaterial.custom.value.x - direction.x) < 0.0001) + #expect(abs(customMaterial.custom.value.y - direction.y) < 0.0001) + #expect(abs(customMaterial.custom.value.z - direction.z) < 0.0001) + } + + @Test + func testVRM1MToonRenderingCanBeDisabled() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let url = try #require(Bundle.module.url(forResource: "Seed-san", withExtension: "vrm"), "Failed to load Seed-san.vrm resource from test bundle.") + let defaultLoader = try VRMEntityLoader(withURL: url) + let defaultMaterial = try defaultLoader.material(withMaterialIndex: 0) + _ = try #require(defaultMaterial as? CustomMaterial, + "Expected default MToon rendering to load a CustomMaterial. Run Scripts/build-mtoon-metallibs.sh and verify the package resources.") + + let disabledLoader = try VRMEntityLoader(withURL: url, isMToonEnabled: false) + let disabledMaterial = try disabledLoader.material(withMaterialIndex: 0) + #expect(!(disabledMaterial is CustomMaterial)) + #expect(disabledMaterial is UnlitMaterial) + + let disabledEntity = try disabledLoader.loadEntity() + let disabledModels = modelEntities(in: disabledEntity.entity) + let hasCustomMaterial = disabledModels.contains { modelEntity in + guard let model = modelEntity.components[ModelComponent.self] else { return false } + return model.materials.contains { $0 is CustomMaterial } + } + let hasMToonParameters = disabledModels.contains { + $0.components[MToonMaterialParametersComponent.self] != nil + } + #expect(!hasCustomMaterial) + #expect(!hasMToonParameters) + } + + @Test + func testVRM1MToonShaderUsesSingleUnlitOutput() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let shader = try mtoonShaderSource() + + #expect(shader.contains("surface.set_base_color(half3(0.0h));\n surface.set_emissive_color(half3(color));")) + #expect(shader.contains("surface.set_base_color(half3(0.0h));\n surface.set_emissive_color(half3(finalColor));")) + #expect(!shader.contains("void mtoonGeometry")) + } + + @Test + func testVRM1MToonShaderUsesPackedMaskChannels() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let shader = try mtoonShaderSource() + let packedMaskSample = SIMD4(0.125, 0.5, 0.875, 1.0) + + #expect(try sampledChannelValue(in: shader, + marker: "mtoonSample(textures.specular(), uv, shadingShiftSampler)", + sample: packedMaskSample) == packedMaskSample.x) + #expect(try sampledChannelValue(in: shader, + marker: "mtoonSample(params.textures().clearcoat(), widthUV, outlineWidthSampler)", + sample: packedMaskSample) == packedMaskSample.y) + #expect(try sampledChannelValue(in: shader, + marker: "mtoonSample(params.textures().ambient_occlusion(), maskUV, uvAnimationMaskSampler)", + sample: packedMaskSample) == packedMaskSample.z) + } + + @Test + func testMToonParameterTextureRowsMatchMetalConstant() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let url = try #require(Bundle.module.url(forResource: "Seed-san", withExtension: "vrm"), "Failed to load Seed-san.vrm resource from test bundle.") + let vrmLoader = try VRMEntityLoader(withURL: url) + let vrmEntity = try vrmLoader.loadEntity() + let parameters = try firstMToonParameters(in: vrmEntity.entity) + let texture = try parameters.textureResource() + let shader = try mtoonShaderSource() + + #expect(MToonMaterialParameters.baseParameterRowCount == 17) + #expect(MToonMaterialParameters.samplerRowCount == MToonTextureSlot.allCases.count) + #expect(MToonMaterialParameters.textureRowCount == 26) + #expect(parameters.samplers.count == MToonMaterialParameters.samplerRowCount) + #expect(texture.width == MToonMaterialParameters.textureRowCount) + #expect(texture.height == 1) + #expect(shader.contains("constant float mtoonParameterTextureWidth = 26.0;")) + #expect(shader.contains("constant float mtoonSamplerParameterStart = 17.0;")) + } + + @Test + func testMToonNormalScaleIsPassedToShaderParameters() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let url = try modifiedSeedSanURL(fileName: "normal-scale") { json in + guard var materials = json["materials"] as? [[String: Any]], + materials.indices.contains(0) else { + throw VRMError.dataInconsistent("Missing Seed-san material fixture data") + } + materials[0]["normalTexture"] = ["index": 0, "scale": 0.35] + json["materials"] = materials + } + defer { try? FileManager.default.removeItem(at: url) } + + let loader = try VRMEntityLoader(withURL: url, isOutlineEnabled: false) + let vrmEntity = try loader.loadEntity() + let parameters = try mtoonParameters(in: vrmEntity.entity, materialIndex: 0) + + #expect(abs(parameters.normalParameters.x - 0.35) < 0.0001) + #expect(try mtoonShaderSource().contains("normalParameters.x, normalSampler")) + } + + @Test + func testMToonMaskTextureSlotsUseRawSemantic() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let textureIndex = 0 + let url = try modifiedSeedSanURL(fileName: "raw-mask-textures") { json in + guard var materials = json["materials"] as? [[String: Any]], + materials.indices.contains(0), + var extensions = materials[0]["extensions"] as? [String: Any], + var mtoon = extensions["VRMC_materials_mtoon"] as? [String: Any] else { + throw VRMError.dataInconsistent("Missing Seed-san MToon fixture data") + } + mtoon["shadingShiftTexture"] = ["index": textureIndex] + mtoon["outlineWidthMultiplyTexture"] = ["index": textureIndex] + mtoon["uvAnimationMaskTexture"] = ["index": textureIndex] + extensions["VRMC_materials_mtoon"] = mtoon + materials[0]["extensions"] = extensions + json["materials"] = materials + } + defer { try? FileManager.default.removeItem(at: url) } + + let loader = try VRMEntityLoader(withURL: url, isOutlineEnabled: false) + let material = try #require(loader.material(withMaterialIndex: 0) as? CustomMaterial) + let rawTexture = try loader.texture(withTextureIndex: textureIndex, semantic: .raw) + let colorTexture = try loader.texture(withTextureIndex: textureIndex, semantic: .color) + + #expect(material.specular.texture != nil) + #expect(material.clearcoat.texture != nil) + #expect(material.ambientOcclusion.texture != nil) + #expect(MToonTextureSlot.shadingShift.semantic == .raw) + #expect(MToonTextureSlot.outlineWidth.semantic == .raw) + #expect(MToonTextureSlot.uvAnimationMask.semantic == .raw) + #expect(rawTexture !== colorTexture) + } + + @Test + func testMToonEmissiveFlagFactorAndEmissionColorBind() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let url = try #require(Bundle.module.url(forResource: "Seed-san", withExtension: "vrm"), "Failed to load Seed-san.vrm resource from test bundle.") + let vrmLoader = try VRMEntityLoader(withURL: url) + let vrmEntity = try vrmLoader.loadEntity() + var parameters = try firstMToonParameters(in: vrmEntity.entity) + let boundColor = SIMD4(0.25, 0.5, 0.75, 0.2) + + #expect(parameters.extraFlags.z == 0 || parameters.extraFlags.z == 1) + #expect(parameters.color(for: .emissionColor)?.isApproximatelyEqual(to: parameters.emissiveFactor) == true) + let didSetEmissionColor = parameters.setColor(boundColor, for: .emissionColor) + #expect(didSetEmissionColor) + #expect(parameters.emissiveFactor.isApproximatelyEqual(to: SIMD4(0.25, 0.5, 0.75, 1))) + #expect(parameters.color(for: .emissionColor)?.isApproximatelyEqual(to: parameters.emissiveFactor) == true) + } + + @Test + func testMToonShadeMultiplyTextureFallsBackToWhite() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let url = try seedSanURLWithNonDefaultEyeSampler() + defer { try? FileManager.default.removeItem(at: url) } + let vrmLoader = try VRMEntityLoader(withURL: url, isOutlineEnabled: false) + let vrmEntity = try vrmLoader.loadEntity() + let eyeTransparentParameters = try mtoonParameters(in: vrmEntity.entity, materialIndex: 4) + + #expect(eyeTransparentParameters.samplers[MToonTextureSlot.base.rawValue] != MToonMaterialParameters.defaultSampler) + #expect(eyeTransparentParameters.samplers[MToonTextureSlot.shade.rawValue] == MToonMaterialParameters.defaultSampler) + #expect(try mtoonShaderSource().contains("extraFlags.y > 0.5h")) + } + + @Test + func testMToonRespectsDoubleSidedMaterialFlag() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let sourceURL = try #require(Bundle.module.url(forResource: "Seed-san", withExtension: "vrm")) + let singleSidedLoader = try VRMEntityLoader(withURL: sourceURL, isOutlineEnabled: false) + let singleSided = try #require(singleSidedLoader.material(withMaterialIndex: 0) as? CustomMaterial) + #expect(singleSided.faceCulling == .back) + + let doubleSidedURL = try modifiedSeedSanURL(fileName: "double-sided") { json in + guard var materials = json["materials"] as? [[String: Any]], + materials.indices.contains(0) else { + throw VRMError.dataInconsistent("Missing Seed-san material fixture data") + } + materials[0]["doubleSided"] = true + json["materials"] = materials + } + defer { try? FileManager.default.removeItem(at: doubleSidedURL) } + + let doubleSidedLoader = try VRMEntityLoader(withURL: doubleSidedURL, isOutlineEnabled: false) + let doubleSided = try #require(doubleSidedLoader.material(withMaterialIndex: 0) as? CustomMaterial) + #expect(doubleSided.faceCulling == .none) + } + + @Test + func testTransparentOutlinePreservesBaseTextureAlpha() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let url = try modifiedSeedSanURL(fileName: "transparent-outline") { json in + guard var materials = json["materials"] as? [[String: Any]], + materials.indices.contains(0) else { + throw VRMError.dataInconsistent("Missing Seed-san material fixture data") + } + materials[0]["alphaMode"] = "BLEND" + var pbr = materials[0]["pbrMetallicRoughness"] as? [String: Any] ?? [:] + pbr["baseColorFactor"] = [1.0, 1.0, 1.0, 0.5] + materials[0]["pbrMetallicRoughness"] = pbr + json["materials"] = materials + } + defer { try? FileManager.default.removeItem(at: url) } + + let loader = try VRMEntityLoader(withURL: url) + let vrmEntity = try loader.loadEntity() + let outline = try customMaterial(in: vrmEntity.entity, + materialIndex: 0, + faceCulling: .front) + #expect(isTransparent(outline.blending)) + #expect(outline.baseColor.texture != nil) + + let shader = try mtoonShaderSource() + #expect(shader.contains("baseSample.a * baseColorFactor.a")) + #expect(shader.contains("mtoonAlpha(float(extraFlags.w)")) + } + + @Test + func testMToonUsesFirstUVTransformWhenTextureSlotsDiffer() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let url = try modifiedSeedSanURL(fileName: "different-uv-transforms") { json in + guard var materials = json["materials"] as? [[String: Any]], + materials.indices.contains(0), + var pbr = materials[0]["pbrMetallicRoughness"] as? [String: Any], + var baseTexture = pbr["baseColorTexture"] as? [String: Any], + var extensions = materials[0]["extensions"] as? [String: Any], + var mtoon = extensions["VRMC_materials_mtoon"] as? [String: Any], + var shadeTexture = mtoon["shadeMultiplyTexture"] as? [String: Any] else { + throw VRMError.dataInconsistent("Missing Seed-san MToon texture fixture data") + } + baseTexture["texCoord"] = 1 + baseTexture["extensions"] = [ + "KHR_texture_transform": [ + "offset": [0.25, 0.5], + "scale": [0.75, 0.5], + "rotation": 0.2, + "texCoord": 1 + ] + ] + pbr["baseColorTexture"] = baseTexture + materials[0]["pbrMetallicRoughness"] = pbr + + shadeTexture["extensions"] = [ + "KHR_texture_transform": [ + "offset": [0.9, 0.8], + "scale": [0.4, 0.3] + ] + ] + mtoon["shadeMultiplyTexture"] = shadeTexture + extensions["VRMC_materials_mtoon"] = mtoon + materials[0]["extensions"] = extensions + json["materials"] = materials + } + defer { try? FileManager.default.removeItem(at: url) } + + let loader = try VRMEntityLoader(withURL: url, isOutlineEnabled: false) + let material = try #require(loader.material(withMaterialIndex: 0) as? CustomMaterial) + #expect(material.textureCoordinateTransform.offset.isApproximatelyEqual(to: SIMD2(0.25, 0.5))) + #expect(material.textureCoordinateTransform.scale.isApproximatelyEqual(to: SIMD2(0.75, 0.5))) + #expect(abs(material.textureCoordinateTransform.rotation - 0.2) < 0.0001) + } + + @Test + func testSetMToonLightAndAmbientColorUpdateParameterRows() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let url = try #require(Bundle.module.url(forResource: "Seed-san", withExtension: "vrm"), "Failed to load Seed-san.vrm resource from test bundle.") + let vrmLoader = try VRMEntityLoader(withURL: url) + let vrmEntity = try vrmLoader.loadEntity() + let lightColor = SIMD3(0.8, 0.7, 0.6) + let ambientColor = SIMD3(0.05, 0.1, 0.15) + + vrmEntity.setMToonLightColor(lightColor) + vrmEntity.setMToonAmbientColor(ambientColor) + + let parameters = try firstMToonParameters(in: vrmEntity.entity) + #expect(parameters.lightColor.isApproximatelyEqual(to: SIMD4(0.8, 0.7, 0.6, 1))) + #expect(parameters.ambientColor.isApproximatelyEqual(to: SIMD4(0.05, 0.1, 0.15, 1))) + let material = try firstCustomMaterial(in: vrmEntity.entity) + #expect(material.custom.texture != nil) + } + + @Test + func testMToonShaderUsesMToon10LightingAndTextureSlots() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let shader = try mtoonShaderSource() + + #expect(shader.contains("mtoonLinearstep(-1.0 + shadingToony")) + #expect(shader.contains("shift += float(shadingShift) * float(uvAnimation.w);")) + #expect(shader.contains("mtoonSample(textures.clearcoat_roughness(), uv, rimSampler)")) + #expect(shader.contains("mtoonSample(textures.emissive_color(), uv, emissiveSampler)")) + #expect(shader.contains("float3 direct = mix(shadeColor, litColor, shading) * lightColor;")) + #expect(shader.contains("float3 indirect = litColor * giColor;")) + #expect(!shader.contains("* 2.0 - 1.0) * float(uvAnimation.w)")) + #expect(!shader.contains("dot(normal, lightDirection) * 0.5 + 0.5")) + } + + @Test + func testMToonShaderAppliesTextureTransformInSurfaceShader() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let shader = try mtoonShaderSource() + + let animatedUV = try #require(shader.range(of: "float2 uv = mtoonAnimatedSurfaceUV")) + let textureTransform = try #require(shader.range(of: "uv = mtoonTransformedUV(uv, uvTransform, uvTransformRotation);")) + #expect(animatedUV.lowerBound < textureTransform.lowerBound) + #expect(shader.contains("mtoonTextureUV(mtoonTransformedUV(uv, uvTransform, uvTransformRotation))")) + #expect(!shader.contains("params.uniforms().uv0_transform() * params.geometry().uv0()")) + } + + @Test + func testMToonTextureTransformBindUpdatesParameterTexture() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let url = try #require(Bundle.module.url(forResource: "Seed-san", withExtension: "vrm"), "Failed to load Seed-san.vrm resource from test bundle.") + let vrmLoader = try VRMEntityLoader(withURL: url, isOutlineEnabled: false) + let vrmEntity = try vrmLoader.loadEntity() + + vrmEntity.setExpression(value: 1, for: .preset(.happy)) + + let parameters = try mtoonParameters(in: vrmEntity.entity, materialIndex: 11) + #expect(parameters.uvTransform.isApproximatelyEqual(to: SIMD4(1, 1, 0.25, 0))) + #expect(parameters.uvTransformRotation.isApproximatelyEqual(to: SIMD4(1, 0, 0, 0))) + let material = try customMaterial(in: vrmEntity.entity, materialIndex: 11) + #expect(material.textureCoordinateTransform.offset == SIMD2(0.25, 0)) + #expect(material.textureCoordinateTransform.scale == SIMD2(1, 1)) + } + + @Test + func testExpressionTextureTransformsAccumulateAndResetIndependently() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let url = try #require(Bundle.module.url(forResource: "Seed-san", withExtension: "vrm")) + let loader = try VRMEntityLoader(withURL: url, isOutlineEnabled: false) + let vrmEntity = try loader.loadEntity() + + vrmEntity.setExpression(value: 1, for: .preset(.happy)) + vrmEntity.setExpression(value: 1, for: .preset(.angry)) + var parameters = try mtoonParameters(in: vrmEntity.entity, materialIndex: 11) + #expect(parameters.uvTransform.isApproximatelyEqual(to: SIMD4(1, 1, 0.75, 0))) + #expect(vrmEntity.expression(for: .preset(.happy)) == 1) + #expect(vrmEntity.expression(for: .preset(.angry)) == 1) + + vrmEntity.setExpression(value: 0, for: .preset(.happy)) + parameters = try mtoonParameters(in: vrmEntity.entity, materialIndex: 11) + #expect(parameters.uvTransform.isApproximatelyEqual(to: SIMD4(1, 1, 0.5, 0))) + #expect(vrmEntity.expression(for: .preset(.happy)) == 0) + #expect(vrmEntity.expression(for: .preset(.angry)) == 1) + } + + @Test + func testExpressionMaterialColorsAccumulateAndResetIndependently() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let url = try modifiedSeedSanURL(fileName: "accumulated-material-colors") { json in + guard var materials = json["materials"] as? [[String: Any]], + materials.indices.contains(0), + var pbr = materials[0]["pbrMetallicRoughness"] as? [String: Any], + var extensions = json["extensions"] as? [String: Any], + var vrm = extensions["VRMC_vrm"] as? [String: Any], + var expressions = vrm["expressions"] as? [String: Any], + var preset = expressions["preset"] as? [String: Any], + var happy = preset["happy"] as? [String: Any], + var angry = preset["angry"] as? [String: Any] else { + throw VRMError.dataInconsistent("Missing Seed-san expression fixture data") + } + pbr["baseColorFactor"] = [1.0, 1.0, 1.0, 1.0] + materials[0]["pbrMetallicRoughness"] = pbr + happy["materialColorBinds"] = [[ + "material": 0, + "type": "color", + "targetValue": [0.8, 1.0, 1.0, 1.0] + ]] + angry["materialColorBinds"] = [[ + "material": 0, + "type": "color", + "targetValue": [1.0, 0.6, 1.0, 1.0] + ]] + preset["happy"] = happy + preset["angry"] = angry + expressions["preset"] = preset + vrm["expressions"] = expressions + extensions["VRMC_vrm"] = vrm + json["extensions"] = extensions + json["materials"] = materials + } + defer { try? FileManager.default.removeItem(at: url) } + + let loader = try VRMEntityLoader(withURL: url, isOutlineEnabled: false) + let vrmEntity = try loader.loadEntity() + vrmEntity.setExpression(value: 1, for: .preset(.happy)) + vrmEntity.setExpression(value: 1, for: .preset(.angry)) + var parameters = try mtoonParameters(in: vrmEntity.entity, materialIndex: 0) + #expect(parameters.baseColor.isApproximatelyEqual(to: SIMD4(0.8, 0.6, 1, 1))) + + vrmEntity.setExpression(value: 0, for: .preset(.happy)) + parameters = try mtoonParameters(in: vrmEntity.entity, materialIndex: 0) + #expect(parameters.baseColor.isApproximatelyEqual(to: SIMD4(1, 0.6, 1, 1))) + } + + @Test + func testMToonShaderUsesPrecompiledSafeSamplerParameters() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let shader = try mtoonShaderSource() + + #expect(shader.contains("mtoonLinearClampSampler")) + #expect(shader.contains("mtoonNearestClampSampler")) + #expect(shader.contains("mtoonWrappedCoordinate")) + #expect(shader.contains("mtoonSamplerParameter(textures, 0.0)")) + #expect(!shader.contains("mtoonBaseSampler")) + #expect(!shader.contains("mtoonShadeSampler")) + } + + @Test + func testMToonLoaderUsesBundledPrecompiledLibraryOnly() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let source = try realityKitLoaderSource() + + #expect(source.contains("makeLibrary(URL: libraryURL)")) + #expect(source.contains("requiredMToonFunctionNames.isSubset")) + #expect(!source.contains("makeLibrary(source:")) + #expect(!source.contains("makeDefaultLibrary")) + #expect(!source.contains("SDKROOT")) + #expect(!source.contains("DEVELOPER_DIR")) + #expect(!source.contains("/usr/bin/xcrun")) + #expect(!source.contains("includeGeometryModifier")) + #expect(!source.contains("\"mtoonGeometry\"")) + } + + @Test + func testMToonPackageResourcesKeepShaderSourceOutOfBundleResources() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let manifest = try packageManifestSource() + + #expect(manifest.range(of: #"exclude:\s*\[[^\]]*"Shaders"[^\]]*\]"#, + options: .regularExpression) != nil) + #expect(manifest.range(of: #"resources:\s*\[[^\]]*\.process\s*\(\s*"Resources"\s*\)[^\]]*\]"#, + options: .regularExpression) != nil) + #expect(manifest.range(of: #"\.copy\s*\(\s*"Shaders"\s*\)"#, + options: .regularExpression) == nil) + } + + @Test + func testBundledMToonMetallibsExistAndMatchShaderSource() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let bundle = try #require(vrmRealityKitResourceBundle(), "Failed to locate VRMRealityKit resource bundle.") + + for resourceName in ["MToon-macos", "MToon-ios", "MToon-iossim"] { + #expect(bundle.url(forResource: resourceName, withExtension: "metallib") != nil, + "Missing bundled metallib: \(resourceName).metallib. Run Scripts/build-mtoon-metallibs.sh.") + } + + // Detect stale metallibs: the hash recorded at metallib build time must + // match the current shader source. If this fails, re-run + // Scripts/build-mtoon-metallibs.sh and commit the regenerated resources. + let hashURL = try #require(bundle.url(forResource: "MToonShaderSource", withExtension: "sha256"), + "Missing MToonShaderSource.sha256. Run Scripts/build-mtoon-metallibs.sh.") + let recordedHash = try String(contentsOf: hashURL, encoding: .utf8) + .trimmingCharacters(in: .whitespacesAndNewlines) + let shaderData = try Data(contentsOf: mtoonShaderSourceURL()) + let currentHash = SHA256.hash(data: shaderData).map { String(format: "%02x", $0) }.joined() + #expect(recordedHash == currentHash, + "Bundled MToon metallibs are stale. Run Scripts/build-mtoon-metallibs.sh and commit the regenerated resources.") + } + + @Test + func testMToonShadeColorBindDoesNotOverwriteCustomLightDirection() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let url = try #require(Bundle.module.url(forResource: "Seed-san", withExtension: "vrm"), "Failed to load Seed-san.vrm resource from test bundle.") + let vrmLoader = try VRMEntityLoader(withURL: url) + let material = try vrmLoader.material(withMaterialIndex: 0) + let customMaterial = try #require(material as? CustomMaterial, + "Expected default MToon rendering to load a CustomMaterial. Run Scripts/build-mtoon-metallibs.sh and verify the package resources.") + let initialValue = customMaterial.custom.value + + let updatedMaterial = customMaterial.settingColor(VRMColor(red: 0.2, green: 0.3, blue: 0.4, alpha: 1), + for: .shadeColor) + let updatedCustomMaterial = try #require(updatedMaterial as? CustomMaterial) + + #expect(updatedCustomMaterial.custom.value == initialValue) + } +#endif + + @Test + func testFallbackShadeAndOutlineColorBindsDoNotOverwriteBaseColor() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let baseColor = VRMColor(red: 0.1, green: 0.2, blue: 0.3, alpha: 1) + let boundColor = VRMColor(red: 0.8, green: 0.7, blue: 0.6, alpha: 1) + + var pbr = PhysicallyBasedMaterial() + pbr.baseColor.tint = baseColor + let shadeUpdatedPBR = try #require(pbr.settingColor(boundColor, for: .shadeColor) as? PhysicallyBasedMaterial) + let outlineUpdatedPBR = try #require(pbr.settingColor(boundColor, for: .outlineColor) as? PhysicallyBasedMaterial) + let colorUpdatedPBR = try #require(pbr.settingColor(boundColor, for: .color) as? PhysicallyBasedMaterial) + + #expect(shadeUpdatedPBR.baseColor.tint.isApproximatelyEqual(to: baseColor)) + #expect(outlineUpdatedPBR.baseColor.tint.isApproximatelyEqual(to: baseColor)) + #expect(colorUpdatedPBR.baseColor.tint.isApproximatelyEqual(to: boundColor)) + #expect(pbr.currentColor(for: .shadeColor).isApproximatelyEqual(to: SIMD4(1, 1, 1, 1))) + #expect(pbr.currentColor(for: .outlineColor).isApproximatelyEqual(to: SIMD4(1, 1, 1, 1))) + + var unlit = UnlitMaterial() + unlit.color.tint = baseColor + let shadeUpdatedUnlit = try #require(unlit.settingColor(boundColor, for: .shadeColor) as? UnlitMaterial) + let colorUpdatedUnlit = try #require(unlit.settingColor(boundColor, for: .color) as? UnlitMaterial) + + #expect(shadeUpdatedUnlit.color.tint.isApproximatelyEqual(to: baseColor)) + #expect(colorUpdatedUnlit.color.tint.isApproximatelyEqual(to: boundColor)) + #expect(unlit.currentColor(for: .shadeColor).isApproximatelyEqual(to: SIMD4(1, 1, 1, 1))) + } + + @Test + func testVRM1FirstPersonAutoHidesHeadDescendants() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let url = try #require(Bundle.module.url(forResource: "Seed-san", withExtension: "vrm"), "Failed to load Seed-san.vrm resource from test bundle.") + let vrmLoader = try VRMEntityLoader(withURL: url) + let vrmEntity = try vrmLoader.loadEntity() + let annotatedEntity = try vrmLoader.node(withNodeIndex: 0) + + #expect(annotatedEntity.isEnabled == true) + vrmEntity.setFirstPersonRenderMode(.firstPerson) + #expect(annotatedEntity.isEnabled == false) + vrmEntity.setFirstPersonRenderMode(.thirdPerson) + #expect(annotatedEntity.isEnabled == true) + } + +#if !os(visionOS) + @Test + func testUpdateUsesDeltaTimeForMToonRuntime() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let url = try #require(Bundle.module.url(forResource: "Seed-san", withExtension: "vrm"), "Failed to load Seed-san.vrm resource from test bundle.") + let vrmLoader = try VRMEntityLoader(withURL: url) + let vrmEntity = try vrmLoader.loadEntity() + + vrmEntity.update(deltaTime: 0.25) + let firstFrameMaterial = try firstCustomMaterial(in: vrmEntity.entity) + #expect(abs(firstFrameMaterial.custom.value.w - 0.25) < 0.0001) + + vrmEntity.update(at: 0.5) + let secondFrameMaterial = try firstCustomMaterial(in: vrmEntity.entity) + #expect(abs(secondFrameMaterial.custom.value.w - 0.75) < 0.0001) + } + + @Test + func testVRM1MToonOutlineEntityIsCreated() throws { + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { return } + let url = try #require(Bundle.module.url(forResource: "Seed-san", withExtension: "vrm"), "Failed to load Seed-san.vrm resource from test bundle.") + let vrmLoader = try VRMEntityLoader(withURL: url) + let vrmEntity = try vrmLoader.loadEntity() + let outlineEntities = modelEntities(in: vrmEntity.entity).filter { modelEntity in + guard let model = modelEntity.components[ModelComponent.self], + let material = model.materials.first as? CustomMaterial else { + return false + } + return material.faceCulling == .front + } + + #expect(!outlineEntities.isEmpty) + } +#endif + + @available(iOS 18.0, macOS 15.0, visionOS 2.0, *) + private func firstMToonParameters(in root: Entity) throws -> MToonMaterialParameters { + for modelEntity in modelEntities(in: root) { + if let component = modelEntity.components[MToonMaterialParametersComponent.self] { + return component.parameters + } + } + throw VRMError.dataInconsistent("Expected at least one MToon parameters component") + } + + @available(iOS 18.0, macOS 15.0, visionOS 2.0, *) + private func mtoonParameters(in root: Entity, materialIndex: Int) throws -> MToonMaterialParameters { + for modelEntity in modelEntities(in: root) { + guard modelEntity.components[VRMMaterialIndexComponent.self]?.materialIndex == materialIndex, + let component = modelEntity.components[MToonMaterialParametersComponent.self] else { + continue + } + return component.parameters + } + throw VRMError.dataInconsistent("Expected MToon parameters for material \(materialIndex)") + } + +#if !os(visionOS) + @available(iOS 18.0, macOS 15.0, visionOS 2.0, *) + private func customMaterial(in root: Entity, materialIndex: Int) throws -> CustomMaterial { + try customMaterial(in: root, materialIndex: materialIndex, faceCulling: nil) + } + + @available(iOS 18.0, macOS 15.0, visionOS 2.0, *) + private func customMaterial(in root: Entity, + materialIndex: Int, + faceCulling: CustomMaterial.FaceCulling?) throws -> CustomMaterial { + for modelEntity in modelEntities(in: root) { + guard modelEntity.components[VRMMaterialIndexComponent.self]?.materialIndex == materialIndex, + let model = modelEntity.components[ModelComponent.self], + let material = model.materials.first as? CustomMaterial else { + continue + } + if let faceCulling, material.faceCulling != faceCulling { + continue + } + return material + } + throw VRMError.dataInconsistent("Expected CustomMaterial for material \(materialIndex)") + } +#endif + + private func modelEntities(in root: Entity) -> [ModelEntity] { + var result: [ModelEntity] = [] + var stack: [Entity] = [root] + while let entity = stack.popLast() { + if let modelEntity = entity as? ModelEntity { + result.append(modelEntity) + } + stack.append(contentsOf: entity.children) + } + return result + } + +#if !os(visionOS) + private func vrmRealityKitResourceBundle() -> Bundle? { + let bundleName = "VRMKit_VRMRealityKit.bundle" + var baseURLs = [ + Bundle.main.bundleURL, + Bundle.main.bundleURL.deletingLastPathComponent() + ] + if let resourceURL = Bundle.main.resourceURL { + baseURLs.append(resourceURL) + } + baseURLs += Bundle.allBundles.compactMap(\.resourceURL) + baseURLs += Bundle.allFrameworks.compactMap(\.resourceURL) + + for baseURL in baseURLs { + let bundleURL = baseURL.appendingPathComponent(bundleName) + if let bundle = Bundle(url: bundleURL), + bundle.url(forResource: "MToon-macos", withExtension: "metallib") != nil { + return bundle + } + } + + return (Bundle.allBundles + Bundle.allFrameworks).first { + $0.url(forResource: "MToon-macos", withExtension: "metallib") != nil + } + } + + private func firstCustomMaterial(in root: Entity) throws -> CustomMaterial { + for modelEntity in modelEntities(in: root) { + guard let model = modelEntity.components[ModelComponent.self] else { continue } + if let material = model.materials.first(where: { $0 is CustomMaterial }) as? CustomMaterial { + return material + } + } + throw VRMError.dataInconsistent("Expected at least one CustomMaterial") + } +#endif + + private func mtoonShaderSource() throws -> String { + return try String(contentsOf: mtoonShaderSourceURL(), encoding: .utf8) + } + + private func mtoonShaderSourceURL() -> URL { + let testFile = URL(fileURLWithPath: #filePath) + let packageRoot = testFile + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + return packageRoot + .appendingPathComponent("Sources") + .appendingPathComponent("VRMRealityKit") + .appendingPathComponent("Shaders") + .appendingPathComponent("MToon.metal") + } + + private func realityKitLoaderSource() throws -> String { + let testFile = URL(fileURLWithPath: #filePath) + let packageRoot = testFile + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + let loaderURL = packageRoot + .appendingPathComponent("Sources") + .appendingPathComponent("VRMRealityKit") + .appendingPathComponent("VRMEntityLoader.swift") + return try String(contentsOf: loaderURL, encoding: .utf8) + } + + private func packageManifestSource() throws -> String { + let testFile = URL(fileURLWithPath: #filePath) + let packageRoot = testFile + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + let manifestURL = packageRoot.appendingPathComponent("Package.swift") + return try String(contentsOf: manifestURL, encoding: .utf8) + } + + private func seedSanURLWithNonDefaultEyeSampler() throws -> URL { + try modifiedSeedSanURL(fileName: "nondefault-eye-sampler") { json in + guard var samplers = json["samplers"] as? [[String: Any]], + samplers.indices.contains(7) else { + throw VRMError.dataInconsistent("Missing Seed-san sampler fixture data") + } + samplers[7]["magFilter"] = 9728 + samplers[7]["minFilter"] = 9728 + samplers[7]["wrapS"] = 33071 + samplers[7]["wrapT"] = 33071 + json["samplers"] = samplers + } + } + + private func modifiedSeedSanURL(fileName: String, + modify: (inout [String: Any]) throws -> Void) throws -> URL { + let sourceURL = try #require(Bundle.module.url(forResource: "Seed-san", withExtension: "vrm"), + "Failed to load Seed-san.vrm resource from test bundle.") + let data = try Data(contentsOf: sourceURL) + guard data.count >= 20, + Array(data.prefix(4)) == [0x67, 0x6c, 0x54, 0x46] else { + throw VRMError.dataInconsistent("Expected GLB test asset") + } + + let version = data.readUInt32LE(at: 4) + var offset = 12 + var chunks: [(type: UInt32, data: Data)] = [] + while offset + 8 <= data.count { + let length = Int(data.readUInt32LE(at: offset)) + let type = data.readUInt32LE(at: offset + 4) + offset += 8 + guard offset + length <= data.count else { + throw VRMError.dataInconsistent("Invalid GLB chunk length") + } + chunks.append((type: type, data: Data(data[offset ..< offset + length]))) + offset += length + } + + guard let jsonIndex = chunks.firstIndex(where: { $0.type == 0x4e4f534a }) else { + throw VRMError.dataInconsistent("Missing GLB JSON chunk") + } + var jsonData = chunks[jsonIndex].data + while jsonData.last == 0x20 || jsonData.last == 0x00 { + jsonData.removeLast() + } + guard var json = try JSONSerialization.jsonObject(with: jsonData) as? [String: Any] else { + throw VRMError.dataInconsistent("Invalid Seed-san JSON fixture data") + } + try modify(&json) + + chunks[jsonIndex].data = try JSONSerialization + .data(withJSONObject: json) + .paddedGLBChunk(padding: 0x20) + + var output = Data() + output.append(contentsOf: [0x67, 0x6c, 0x54, 0x46]) + output.appendUInt32LE(version) + output.appendUInt32LE(0) + for chunk in chunks { + output.appendUInt32LE(UInt32(chunk.data.count)) + output.appendUInt32LE(chunk.type) + output.append(chunk.data) + } + output.writeUInt32LE(UInt32(output.count), at: 8) + + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("Seed-san-\(fileName)-\(UUID().uuidString)") + .appendingPathExtension("vrm") + try output.write(to: url) + return url + } + +#if !os(visionOS) + @available(iOS 18.0, macOS 15.0, *) + private func isTransparent(_ blending: CustomMaterial.Blending) -> Bool { + if case .transparent = blending { + return true + } + return false + } +#endif +} + +private extension Data { + func readUInt32LE(at offset: Int) -> UInt32 { + var value = UInt32(self[offset]) + value |= UInt32(self[offset + 1]) << 8 + value |= UInt32(self[offset + 2]) << 16 + value |= UInt32(self[offset + 3]) << 24 + return value + } + + mutating func appendUInt32LE(_ value: UInt32) { + append(UInt8(value & 0xff)) + append(UInt8((value >> 8) & 0xff)) + append(UInt8((value >> 16) & 0xff)) + append(UInt8((value >> 24) & 0xff)) + } + + mutating func writeUInt32LE(_ value: UInt32, at offset: Int) { + self[offset] = UInt8(value & 0xff) + self[offset + 1] = UInt8((value >> 8) & 0xff) + self[offset + 2] = UInt8((value >> 16) & 0xff) + self[offset + 3] = UInt8((value >> 24) & 0xff) + } + + func paddedGLBChunk(padding: UInt8) -> Data { + var padded = self + while padded.count % 4 != 0 { + padded.append(padding) + } + return padded + } +} + +private extension VRMColor { + func isApproximatelyEqual(to other: VRMColor, tolerance: Float = 0.0001) -> Bool { + testSIMD.isApproximatelyEqual(to: other.testSIMD, tolerance: tolerance) + } + + var testSIMD: SIMD4 { + #if os(macOS) + let color = usingColorSpace(.deviceRGB) ?? self + return SIMD4(Float(color.redComponent), + Float(color.greenComponent), + Float(color.blueComponent), + Float(color.alphaComponent)) + #else + var red: CGFloat = 0 + var green: CGFloat = 0 + var blue: CGFloat = 0 + var alpha: CGFloat = 0 + getRed(&red, green: &green, blue: &blue, alpha: &alpha) + return SIMD4(Float(red), Float(green), Float(blue), Float(alpha)) + #endif + } +} + +private extension SIMD3 where Scalar == Float { + func isApproximatelyEqual(to other: SIMD3, tolerance: Float = 0.0001) -> Bool { + abs(x - other.x) < tolerance && + abs(y - other.y) < tolerance && + abs(z - other.z) < tolerance + } +} + +private extension SIMD2 where Scalar == Float { + func isApproximatelyEqual(to other: SIMD2, tolerance: Float = 0.0001) -> Bool { + abs(x - other.x) < tolerance && abs(y - other.y) < tolerance + } +} + +private extension SIMD4 where Scalar == Float { + func isApproximatelyEqual(to other: SIMD4, tolerance: Float = 0.0001) -> Bool { + abs(x - other.x) < tolerance && + abs(y - other.y) < tolerance && + abs(z - other.z) < tolerance && + abs(w - other.w) < tolerance + } +} + +private enum ShaderChannel: String { + case r + case g + case b + case a + + func value(in color: SIMD4) -> Float { + switch self { + case .r: return color.x + case .g: return color.y + case .b: return color.z + case .a: return color.w + } + } +} + +private func sampledChannelValue(in source: String, + marker: String, + sample: SIMD4) throws -> Float { + let channel = try sampledChannel(in: source, marker: marker) + return channel.value(in: sample) +} + +private func sampledChannel(in source: String, marker: String) throws -> ShaderChannel { + guard let markerRange = source.range(of: marker) else { + throw VRMError.dataInconsistent("Expected shader sample marker: \(marker)") + } + guard let dotIndex = source[markerRange.upperBound...].firstIndex(of: ".") else { + throw VRMError.dataInconsistent("Expected channel access after shader sample marker: \(marker)") + } + let channelIndex = source.index(after: dotIndex) + guard channelIndex < source.endIndex, + let channel = ShaderChannel(rawValue: String(source[channelIndex])) else { + throw VRMError.dataInconsistent("Expected r/g/b/a channel after shader sample marker: \(marker)") + } + return channel +} +#endif