Logitech Muse for Apple Vision Pro Tracking, Pressure Sensitive Inputs, and Haptics in RealityKit

AppModel.swift

import SwiftUI

@MainActor
@Observable
class AppModel {
    enum ImmersiveSpaceState {
        case closed
        case inTransition
        case open
    }
    var immersiveSpaceState = ImmersiveSpaceState.closed
    var immersiveSpaceType: ImmersiveSpaceType = .simpleExample
}

enum ImmersiveSpaceType: String, CaseIterable {
    case simpleExample
    
    var title: String {
        return "Simple Example"
    }
}

ImmersiveStylusSimpleExampleView.swift

import CoreHaptics
import GameController
import RealityKit
import SwiftUI

struct ImmersiveStylusSimpleExampleView: View {
    @State private var stylusManager = StylusManager()

    var body: some View {
        RealityView { content in
            let root = Entity()
            content.add(root)
            stylusManager.rootEntity = root
            await stylusManager.handleControllerSetup()
        }
        .task {
            // Don't forget to add the Accessory Tracking capability
            let configuration = SpatialTrackingSession.Configuration(tracking: [.accessory])
            let session = SpatialTrackingSession()
            await session.run(configuration)
        }
    }
}

@MainActor
final class StylusManager {
    var rootEntity: Entity?
    private var hapticEngines: [ObjectIdentifier: CHHapticEngine] = [:]
    private var hapticPlayers: [ObjectIdentifier: CHHapticPatternPlayer] = [:]
    private var inflationHapticPlayers: [ObjectIdentifier: CHHapticPatternPlayer] = [:]
    private var activePressureSphere: ModelEntity?
    private var activeSphereColor: UIColor = .systemRed
    private var maxScale: Float = 0.5
    private let baseRadius: Float = 0.01
    private var tipIndicators: [AnchorEntity: ModelEntity] = [:]

    func handleControllerSetup() async {
        // Existing connections
        let styluses = GCStylus.styli

        for stylus in styluses where stylus.productCategory == GCProductCategorySpatialStylus {
            try? await setupAccessory(stylus: stylus)
        }

        NotificationCenter.default.addObserver(
            forName: NSNotification.Name.GCStylusDidConnect, object: nil, queue: .main
        ) { [weak self] note in
            guard let self,
                  let stylus = note.object as? GCStylus,
                  stylus.productCategory == GCProductCategorySpatialStylus else { return }
            Task { @MainActor in
                try? await self.setupAccessory(stylus: stylus)
            }
        }
    }

    private func setupAccessory(stylus: GCStylus) async throws {
        guard let root = rootEntity else { return }
        let source = try await AnchoringComponent.AccessoryAnchoringSource(device: stylus)
        
        // List available locations (aim and origin)
        print("📍 Available locations: \(source.accessoryLocations)")

        guard let location = source.locationName(named: "aim") else { return }

        let anchor = AnchorEntity(
            .accessory(from: source, location: location),
            trackingMode: .predicted,
            physicsSimulation: .none
        )
        root.addChild(anchor)

        let key = ObjectIdentifier(stylus)

        // Setup haptics if available
        setupHaptics(for: stylus, key: key)
        setupStylusInputs(stylus: stylus, anchor: anchor, key: key)
        addStylusTipIndicator(to: anchor)
    }

    private func setupHaptics(for stylus: GCStylus, key: ObjectIdentifier) {
        guard let deviceHaptics = stylus.haptics else { return }
        
        // Create haptic engine
        let engine = deviceHaptics.createEngine(withLocality: .default)
        do {
            try engine?.start()
            hapticEngines[key] = engine
            
            // Create a simple "tap" pattern for main button press
            let tapPattern = try CHHapticPattern(events: [
                CHHapticEvent(eventType: .hapticTransient, parameters: [
                    CHHapticEventParameter(parameterID: .hapticIntensity, value: 0.85),
                    CHHapticEventParameter(parameterID: .hapticSharpness, value: 0.5)
                ], relativeTime: 0.0)
            ], parameters: [])
            
            let player = try engine?.makePlayer(with: tapPattern)
            hapticPlayers[key] = player
            
            // Create an "inflation" pattern for tip and secondary pressure increase
            let inflationPattern = try CHHapticPattern(events: [
                CHHapticEvent(eventType: .hapticContinuous, parameters: [
                    CHHapticEventParameter(parameterID: .hapticIntensity, value: 0.55),
                    CHHapticEventParameter(parameterID: .hapticSharpness, value: 1.0)
                ], relativeTime: 0.0, duration: 0.1)
            ], parameters: [])
            
            let inflationPlayer = try engine?.makePlayer(with: inflationPattern)
            inflationHapticPlayers[key] = inflationPlayer
        } catch {
            print("❌ Failed to setup haptics: \(error)")
        }
    }
    
    private func playHaptic(for key: ObjectIdentifier) {
        guard let player = hapticPlayers[key] else { return }
        
        do {
            try player.start(atTime: CHHapticTimeImmediate)
        } catch {
            print("❌ Failed to play haptic: \(error)")
        }
    }
    
    private func playInflationHaptic(for key: ObjectIdentifier) {
        guard let player = inflationHapticPlayers[key] else { return }
        
        do {
            try player.start(atTime: CHHapticTimeImmediate)
        } catch {
            print("❌ Failed to play inflation haptic: \(error)")
        }
    }

    private func addStylusTipIndicator(to anchor: AnchorEntity) {
        let tipSphere = ModelEntity(
            mesh: .generateSphere(radius: 0.003),
            materials: [SimpleMaterial(color: .white, isMetallic: false)]
        )
        anchor.addChild(tipSphere)
        tipIndicators[anchor] = tipSphere
    }

    private func setupStylusInputs(stylus: GCStylus, anchor: AnchorEntity, key: ObjectIdentifier) {
        guard let input = stylus.input else { return }
        
        // Handle main button (simple button pressed)
        input.buttons[.stylusPrimaryButton]?.pressedInput.pressedDidChangeHandler = { [weak self] _, _, pressed in
            guard pressed, let self else { return }
            Task { @MainActor in
                self.playHaptic(for: key)
                self.spawnSphere(at: anchor, color: .systemBlue, radius: 0.02)
            }
        }

        // Handle secondary button pressure changes - create and increase scale of red sphere
        input.buttons[.stylusSecondaryButton]?.pressedInput.valueDidChangeHandler = { [weak self] _, _, pressure in
            guard let self else { return }
            Task { @MainActor in
                if pressure > 0.05 {
                    self.updatePressureSphere(at: anchor, pressure: pressure, color: .systemRed, key: key)
                } else if pressure <= 0 {
                    self.dropPressureSphere(at: anchor)
                }
            }
        }

        // Handle tip pressure changes - create and increase scale of green sphere
        input.buttons[.stylusTip]?.pressedInput.valueDidChangeHandler = { [weak self] _, _, pressure in
            guard let self else { return }
            Task { @MainActor in
                if pressure > 0.05 {
                    self.updatePressureSphere(at: anchor, pressure: pressure, color: .systemGreen, key: key)
                } else if pressure <= 0 {
                    self.dropPressureSphere(at: anchor)
                }
            }
        }
    }

    private func spawnSphere(at anchor: AnchorEntity, color: UIColor, radius: Float) {
        guard let root = rootEntity else { return }
        
        let worldTransform = anchor.transformMatrix(relativeTo: nil)
        let worldPosition = SIMD3<Float>(worldTransform.columns.3.x,
                                          worldTransform.columns.3.y,
                                          worldTransform.columns.3.z)
        
        let sphere = ModelEntity(
            mesh: .generateSphere(radius: radius),
            materials: [SimpleMaterial(color: color, isMetallic: false)]
        )
        sphere.position = worldPosition
        
        root.addChild(sphere)
    }
    
    private func updatePressureSphere(at anchor: AnchorEntity, pressure: Float, color: UIColor, key: ObjectIdentifier) {
        if activePressureSphere == nil {
            // Hide the tip indicator while inflating
            tipIndicators[anchor]?.isEnabled = false
            
            // Create new sphere
            let sphere = ModelEntity(
                mesh: .generateSphere(radius: baseRadius),
                materials: [SimpleMaterial(color: color, isMetallic: false)]
            )
            anchor.addChild(sphere)
            activePressureSphere = sphere
            activeSphereColor = color
            maxScale = 0.5  // Reset max scale for new sphere
        }
        
        // Scale the sphere based on pressure (0.0 to 1.0)
        let scale = 0.5 + (pressure * 5.5)
        
        // Only scale UP, never down
        if scale > maxScale {
            maxScale = scale
            activePressureSphere?.scale = SIMD3<Float>(repeating: maxScale)
            
            // Offset the sphere by its scaled radius in the -Z direction (forward from stylus tip)
            // This creates the "balloon blowing" effect
            let offset = baseRadius * maxScale
            activePressureSphere?.position = SIMD3<Float>(0, 0, -offset)
            
            // Play subtle haptic feedback while inflating
            playInflationHaptic(for: key)
            
            let colorEmoji = color == .systemRed ? "🔴" : "🟢"
            print("\(colorEmoji) Pressure: \(String(format: "%.2f", pressure)), Scale: \(String(format: "%.2f", maxScale))")
        }
    }
    
    private func dropPressureSphere(at anchor: AnchorEntity) {
        guard let sphere = activePressureSphere, let root = rootEntity else { return }
        
        // Get the current world position before detaching
        let worldTransform = sphere.transformMatrix(relativeTo: nil)
        let worldPosition = SIMD3<Float>(worldTransform.columns.3.x,
                                          worldTransform.columns.3.y,
                                          worldTransform.columns.3.z)
        let worldScale = sphere.scale
        
        // Remove from anchor
        sphere.removeFromParent()
        
        // Create a new sphere at the world position (dropped)
        let droppedSphere = ModelEntity(
            mesh: .generateSphere(radius: 0.01),
            materials: [SimpleMaterial(color: activeSphereColor, isMetallic: false)]
        )
        droppedSphere.position = worldPosition
        droppedSphere.scale = worldScale
        root.addChild(droppedSphere)
        
        print("💧 Dropped sphere at scale \(worldScale)")
        
        // Show the tip indicator again
        tipIndicators[anchor]?.isEnabled = true
        
        // Clear the reference
        activePressureSphere = nil
    }
}

LogitechMusePlaygroundApp.swift

import SwiftUI

@main
struct LogitechMusePlaygroundApp: App {
    @State private var appModel = AppModel()
    
    var body: some Scene {
        WindowGroup {
            MainWindowView()
                .environment(appModel)
        }
        .windowResizability(.contentSize)

        ImmersiveSpace(id: ImmersiveSpaceType.simpleExample.rawValue) {
            ImmersiveStylusSimpleExampleView()
                .onAppear { appModel.immersiveSpaceState = .open }
                .onDisappear { appModel.immersiveSpaceState = .closed }
        }
        .immersionStyle(selection: .constant(.mixed), in: .mixed)
     }
}

MainWindowView.swift

import RealityKit
import SwiftUI

struct MainWindowView: View {
    @Environment(AppModel.self) private var appModel
    @Environment(\.dismissImmersiveSpace) private var dismissImmersiveSpace
    @Environment(\.openImmersiveSpace) private var openImmersiveSpace
    
    var body: some View {
        @Bindable var appModel = appModel
        
        VStack(spacing: 24) {
            if appModel.immersiveSpaceState != .open {
                Text("Select experience and press start")
                Picker("Experience", selection: $appModel.immersiveSpaceType) {
                    ForEach(ImmersiveSpaceType.allCases, id: \.self) { type in
                        Text(type.title).tag(type)
                    }
                }
            } else {
                SimpleExampleWindowView()
            }
            
            Button(action: {
                toggleImmersiveSpace(type: appModel.immersiveSpaceType)
            },
                   label: {
                Text(appModel.immersiveSpaceState == .open ? "Exit" : "Start")
            })
            .animation(.none, value: 0)
            .fontWeight(.semibold)
            .disabled(appModel.immersiveSpaceState == .inTransition)
        }
        .padding(36)
        .animation(.default, value: appModel.immersiveSpaceState)
    }
    
    func toggleImmersiveSpace(type: ImmersiveSpaceType) {
        Task { @MainActor in
            switch appModel.immersiveSpaceState {
                case .open:
                    appModel.immersiveSpaceState = .inTransition
                    await dismissImmersiveSpace()
                case .closed:
                    appModel.immersiveSpaceState = .inTransition
                    switch await openImmersiveSpace(id: type.rawValue) {
                        case .opened:
                            break

                        case .userCancelled, .error:
                            fallthrough
                        @unknown default:
                            appModel.immersiveSpaceState = .closed
                    }

                case .inTransition:
                    break
            }
        }
    }
}

#Preview(windowStyle: .automatic) {
    MainWindowView()
        .environment(AppModel())
}

SimpleExampleWindowView.swift

import SwiftUI

struct SimpleExampleWindowView: View {
    var body: some View {
        VStack(alignment: .leading, spacing: 12) {
            HStack {
                Circle().frame(width: 12).foregroundStyle(Color.white)
                Text("Ready when white sphere appears")
            }
            
            HStack {
                Circle().frame(width: 12).foregroundStyle(Color.green)
                ButtonTypeIcon(kind: .pressure)
                Text("Tip pressure increase grows green spheres")
            }
            
            HStack {
                Circle().frame(width: 12).foregroundStyle(Color.blue)
                ButtonTypeIcon(kind: .push)
                Text("Main button creates blue spheres")
            }
            
            HStack {
                Circle().frame(width: 12).foregroundStyle(Color.red)
                ButtonTypeIcon(kind: .pressure)
                Text("Secondary button pressure increase grows red spheres")
            }
        }
        .frame(width: 500, height: 170)
    }
    
    struct ButtonTypeIcon: View {
        enum Kind {
            case push
            case pressure
        }

        let kind: Kind

        var body: some View {
            switch kind {
            case .push:
                Image(systemName: "cursorarrow.rays")
                    .font(.system(size: 24, weight: .regular))
            case .pressure:
                Image(systemName: "dial.medium")
                    .font(.system(size: 24, weight: .regular))
            }
        }
    }
}

#Preview(windowStyle: .automatic) {
    SimpleExampleWindowView()
}

Hey! So the code works perfectly in a native visionOS app — tip position, buttons, pressure, everything.
I’m now trying to bridge that Swift plugin into Unity.

Using your code, I set up the bridge and was able to get the primary button, secondary button, secondary pressure, tip pressure, and haptics all working inside the Unity app.

But the tip position is the only thing that isn’t coming through. In Unity, it always returns a Vector3.zero every frame, even though the native build detects it correctly.

Could you help me figure out how to get the tip position working in Unity?
Or is this a limitation from Apple’s side when running through a Unity plugin?