supervisor: x2apic/ioapic and infrastructure

applications/peek.lisp

@@ -122,7 +122,7 @@
     "DCA"
     "SSE4.1"
     "SSE4.2"
-    nil
+    "x2APIC"
     nil
     "POPCNT"))
 

drivers/sound.lisp

@@ -132,7 +132,7 @@
     (fill buffer 0.0 :start start :end end))
   (prog1
       ;; Try-lock the sink mutex. If the music player holds it (inside
-      ;; output-sound's transcode path), return T immediately — the
+      ;; output-sound's transcode path), return T immediately: the
       ;; buffer is already zeroed, so the HDA plays silence for this
       ;; period. Next period will retry.
       (if (mezzano.supervisor:acquire-mutex *sink-lock* nil)
@@ -311,7 +311,7 @@
       (loop
           (when (buffer-empty sink)
             ;; Buffer is currently empty, sink is not live.
-            ;; Only add to the sink list if not already present — the
+            ;; Only add to the sink list if not already present. the
             ;; callback may have left it if it was re-filled concurrently.
             (unless (member sink *sinks*)
               (setf (sup:event-state *sinks-present-event*) t)

supervisor/acpi.lisp

@@ -380,6 +380,12 @@
              (:area :wired))
   address)
 
+(defstruct (acpi-madt-processor-x2apic
+             (:area :wired))
+  x2apic-id
+  flags
+  acpi-processor-uid)
+
 (defun read-acpi-madt-table (address)
   (let ((table (make-acpi-madt-table :address address))
         (n-controller-entries 0))
@@ -393,9 +399,10 @@
       (loop
          (when (>= offset total-length)
            (return))
-         (when (<= (physical-memref-unsigned-byte-8 (+ address offset)) 5)
-           ;; Ignore entries with an unknown type.
-           (incf n-controller-entries))
+         (let ((type (physical-memref-unsigned-byte-8 (+ address offset))))
+           (when (or (<= type 5) (eql type 9))
+             ;; Ignore entries with an unknown type.
+             (incf n-controller-entries)))
          (incf offset (physical-memref-unsigned-byte-8 (+ address offset 1)))))
     (setf (acpi-madt-table-controllers table) (sys.int::make-simple-vector n-controller-entries :wired))
     (let ((offset 44)
@@ -440,7 +447,13 @@
              (5 ;; Local APIC address override.
               (setf (svref (acpi-madt-table-controllers table) current)
                     (make-acpi-madt-lapic-address-override
-                     :address (physical-memref-unsigned-byte-64 (+ address offset 4))))))
+                     :address (physical-memref-unsigned-byte-64 (+ address offset 4)))))
+             (9 ;; Processor local x2APIC.
+              (setf (svref (acpi-madt-table-controllers table) current)
+                    (make-acpi-madt-processor-x2apic
+                     :x2apic-id (physical-memref-unsigned-byte-32 (+ address offset 4))
+                     :flags (physical-memref-unsigned-byte-32 (+ address offset 8))
+                     :acpi-processor-uid (physical-memref-unsigned-byte-32 (+ address offset 12))))))
            (incf current)
            (incf offset len))))
     table))

supervisor/arm64/cpu.lisp

@@ -21,6 +21,7 @@
   page-fault-hook)
 
 (defun initialize-boot-cpu ()
+  (setf *tlb-shootdown-in-progress* nil)
   (setf (arm64-cpu-self *bsp-cpu*) *bsp-cpu*)
   (setf (arm64-cpu-state *bsp-cpu*) :online)
   (setf (arm64-cpu-idle-thread *bsp-cpu*)
@@ -77,8 +78,18 @@
 (defun local-cpu ()
   (local-cpu-info))
 
+(defun cpu-memory-barrier ()
+  "Full inner-shareable memory barrier for ordering lock data accesses."
+  (%dsb.ish))
+
 (defun initialize-cpu ()
   (setf (arm64-cpu-cpu-id *bsp-cpu*) (fdt-boot-cpuid))
+  (setf (cpu-cpu-index *bsp-cpu*) 0)
+  (setf (cpu-inhibit-scheduling *bsp-cpu*) 0)
+  (setf (cpu-tlb-generation *bsp-cpu*) 0)
+  ;; Allocate MCS node for BSP if not already allocated by cold-generator.
+  (when (null (cpu-mcs-node *bsp-cpu*))
+    (setf (cpu-mcs-node *bsp-cpu*) (%make-mcs-node)))
   (push-wired *bsp-cpu* *cpus*))
 
 (sys.int::define-lap-function %el0-common ()
@@ -224,52 +235,58 @@
   (mezzano.lap.arm64:hlt 4))
 
 (defun broadcast-panic-ipi ()
-  (broadcast-ipi +panic-sgi-id+))
+  (send-ipi-to-all +panic-sgi-id+))
 
 (defun panic-ipi-handler (interrupt-frame)
   (declare (ignore interrupt-frame))
   (loop (%arch-panic-stop)))
 
 (defun broadcast-wakeup-ipi ()
-  (broadcast-ipi +wakeup-sgi-id+))
+  (dolist (cpu *cpus*)
+    (when (eql (arm64-cpu-state cpu) :online)
+      (wake-cpu cpu))))
+
+(defun wake-cpu (cpu)
+  (when (cpu-idle-p cpu)
+    (send-ipi-to-cpu cpu +wakeup-sgi-id+)))
 
 (sys.int::defglobal *non-quiescent-cpus-remaining*)
 
-;; FIXME: quiesce-cpus-for-world-stop needs to prevent migration across CPUs.
 (defun quiesce-cpus-for-world-stop ()
   "Bring all CPUs to a consistent state to stop the world.
 Protected by the world stop lock."
+  ;; Prevent migration during the broadcast and busy-wait.
+  (setf (cpu-inhibit-scheduling (local-cpu))
+        (1+ (cpu-inhibit-scheduling (local-cpu))))
   (setf *non-quiescent-cpus-remaining* (1- *n-up-cpus*))
-  (broadcast-ipi +quiesce-sgi-id+)
+  (send-ipi-to-all +quiesce-sgi-id+)
   ;; FIXME: Use WFE/SEV instead of this spin-loop.
   (loop
      (when (eql *non-quiescent-cpus-remaining* 0)
        (return))
-     (sys.int::cpu-relax)))
+     (sys.int::cpu-relax))
+  (setf (cpu-inhibit-scheduling (local-cpu))
+        (max 0 (1- (cpu-inhibit-scheduling (local-cpu))))))
 
 ;; Save the current thread's state and switch to the CPU's idle thread.
 (defun quiesce-ipi-handler (interrupt-frame)
-  (let* ((current (current-thread))
-         (idle (local-cpu-idle-thread))
-         (was-active (not (eql current idle))))
-    (when was-active
-      (acquire-global-thread-lock)
-      ;; Return this thread to the run queue.
-      (setf (thread-state current) :runnable)
-      (push-run-queue current)
-      (preemption-timer-reset nil)
-      ;; Save thread state.
-      (save-fpu-state current)
-      (save-interrupted-state current interrupt-frame)
-      ;; Partially switch to the idle thread.
-      (setf (thread-state idle) :active))
-    ;; Have now reached a quiescent state.
-    (sys.int::%atomic-fixnum-add-symbol '*non-quiescent-cpus-remaining*
-                                        -1)
-    (when was-active
-      ;; Finally, return to the idle thread.
-      (%%switch-to-thread-common idle
-                                 idle))))
+  (cond (*debug-magic-button-hold-variable*
+         (magic-button-ipi-handler interrupt-frame))
+        (t
+         (let* ((current (current-thread))
+                (idle (local-cpu-idle-thread))
+                (was-active (not (eql current idle))))
+           (when was-active
+             (acquire-global-thread-lock)
+             (setf (thread-state current) :runnable)
+             (push-run-queue current)
+             (preemption-timer-reset nil)
+             (save-fpu-state current)
+             (save-interrupted-state current interrupt-frame)
+             (setf (thread-state idle) :active))
+           (sys.int::%atomic-fixnum-add-symbol '*non-quiescent-cpus-remaining* -1)
+           (when was-active
+             (%%switch-to-thread-common idle idle))))))
 
 ;; TODO: This needs to be fixed up to prevent multiple CPUs hitting it at
 ;; once. It can't currently happen because it is only used from IRQ handlers
@@ -280,7 +297,7 @@ Protected by the world stop lock."
 (defun stop-other-cpus-for-debug-magic-button ()
   (setf *debug-magic-button-ready-variable* (1- *n-up-cpus*)
         *debug-magic-button-hold-variable* t)
-  (broadcast-ipi +magic-button-sgi-id+)
+  (send-ipi-to-all +quiesce-sgi-id+)
   ;; Wait for other CPUs to arrive, this ensures the thread state is actually
   ;; consistent.
   (loop until (eql *debug-magic-button-ready-variable* 0)))
@@ -303,26 +320,29 @@ Protected by the world stop lock."
   (sys.int::%atomic-fixnum-add-symbol
    '*debug-magic-button-ready-variable* -1))
 
-;; TLB shootdown isn't required as ARM has cross-core TLB invalidation instructions
-
-(defun begin-tlb-shootdown ()
-  nil)
-
-(defun tlb-shootdown-single (address)
-  (declare (ignore address))
-  nil)
+;; ARM64 has hardware-broadcast TLB invalidation (TLBI IS instructions).
+;; No IPIs needed; the initiating CPU issues TLBI VAE1IS which broadcasts
+;; to all CPUs in the inner shareable domain. We still bracket the operation
+;; with inhibit-scheduling to prevent CPU migration.
 
-(defun tlb-shootdown-range (base length)
-  (declare (ignore base length))
-  nil)
+(sys.int::defglobal *tlb-shootdown-in-progress* nil)
 
-(defun tlb-shootdown-all ()
-  nil)
+(defun begin-tlb-shootdown ()
+  "Prepare for TLB shootdown on ARM64.
+TLB shootdown must be protected by the VM lock."
+  (ensure (rw-lock-write-held-p *vm-lock*) "VM lock not held when doing TLB shootdown!")
+  (ensure (not *tlb-shootdown-in-progress*) "TLB shootdown already in progress!")
+  (setf *tlb-shootdown-in-progress* t)
+  (setf (cpu-inhibit-scheduling (local-cpu))
+        (1+ (cpu-inhibit-scheduling (local-cpu)))))
 
 (defun finish-tlb-shootdown ()
-  nil)
+  (ensure *tlb-shootdown-in-progress*)
+  (setf *tlb-shootdown-in-progress* nil)
+  (setf (cpu-inhibit-scheduling (local-cpu))
+        (max 0 (1- (cpu-inhibit-scheduling (local-cpu))))))
 
-(defun check-tlb-shootdown-not-in-progress ()
+(defun check-tlb-generation-consistency ()
   nil)
 
 (defun local-cpu-idle-thread ()
@@ -356,6 +376,10 @@ Protected by the world stop lock."
                               :sp-el1 (+ (stack-base wired-stack)
                                          (stack-size wired-stack)
                                          -16))))
+    (setf (cpu-cpu-index cpu) (length *cpus*))
+    (setf (cpu-inhibit-scheduling cpu) 0)
+    (setf (cpu-tlb-generation cpu) 0)
+    (setf (cpu-mcs-node cpu) (%make-mcs-node))
     (setf (arm64-cpu-self cpu) cpu)
     (setf (sys.int::memref-unsigned-byte-64 (arm64-cpu-sp-el1 cpu))
           (sys.int::lisp-object-address cpu))

supervisor/arm64/gic.lisp

@@ -46,7 +46,6 @@
 (defconstant +panic-sgi-id+ 1)
 (defconstant +wakeup-sgi-id+ 2)
 (defconstant +quiesce-sgi-id+ 3)
-(defconstant +magic-button-sgi-id+ 4)
 
 (defun gic-dist-reg (index)
   (physical-memref-unsigned-byte-32 (+ *gic-distributor-base* index)))
@@ -102,8 +101,7 @@
   ;; Unmask our various IPIs
   (gic-unmask-interrupt +panic-sgi-id+)
   (gic-unmask-interrupt +wakeup-sgi-id+)
-  (gic-unmask-interrupt +quiesce-sgi-id+)
-  (gic-unmask-interrupt +magic-button-sgi-id+))
+  (gic-unmask-interrupt +quiesce-sgi-id+))
 
 (defun configure-gic-cpu ()
   ;; Enable the local CPU.
@@ -141,8 +139,6 @@
              (#.+quiesce-sgi-id+
               ;; nothing, handled later
               nil)
-             (#.+magic-button-sgi-id+
-              (magic-button-ipi-handler interrupt-frame))
              (t (debug-print-line "Received unknown SGI " vector))))
           (t
            ;; Normal external IRQ
@@ -174,9 +170,15 @@
 (defun platform-unmask-irq (vector)
   (gic-unmask-interrupt vector))
 
-(defun broadcast-ipi (vector)
+(defun send-ipi-to-all (vector)
   (%dsb.ishst)
   (setf (gic-dist-reg +gicd-sgir+)
         (logior (ash 1 24) ; all-but-self
                 vector))
   nil)
+
+(defun send-ipi-to-cpu (cpu vector)
+  (%dsb.ishst)
+  (setf (gic-dist-reg +gicd-sgir+)
+        (logior (ash 1 (+ 16 (cpu-cpu-index cpu)))
+                vector)))

supervisor/arm64/thread.lisp

@@ -86,13 +86,7 @@
   (mezzano.lap.arm64:ret))
 
 (sys.int::define-lap-function %%restore-full-save-thread ((thread))
-  ;; Drop the global thread lock.
-  ;; This must be done here, not in %%switch-to-thread-common to prevent
-  ;; another CPU from switching on to the old thread's stack while it is
-  ;; still in use.
-  (mezzano.lap.arm64:ldr :x1 (:symbol-global-cell *global-thread-lock*))
-  (mezzano.lap.arm64:ldr :x2 (:constant :unlocked))
-  (mezzano.lap.arm64:str :x2 (:object :x1 #.sys.int::+symbol-value-cell-value+))
+  ;; The global thread lock is released by Lisp code before calling this function.
   ;; Switch back to SP_EL0, restoring the original value of SP_EL1.
   (mezzano.lap.arm64:add :sp :x27 0)
   (mezzano.lap.arm64:msr :spsel 0)
@@ -130,13 +124,7 @@
   (mezzano.lap.arm64:eret))
 
 (sys.int::define-lap-function %%restore-partial-save-thread ((thread))
-  ;; Drop the global thread lock.
-  ;; This must be done here, not in %%switch-to-thread-common to prevent
-  ;; another CPU from switching on to the old thread's stack while it is
-  ;; still in use.
-  (mezzano.lap.arm64:ldr :x1 (:symbol-global-cell *global-thread-lock*))
-  (mezzano.lap.arm64:ldr :x2 (:constant :unlocked))
-  (mezzano.lap.arm64:str :x2 (:object :x1 #.sys.int::+symbol-value-cell-value+))
+  ;; The global thread lock is released by Lisp code before calling this function.
   ;; Switch back to SP_EL0, restoring the original value of SP_EL1.
   (mezzano.lap.arm64:add :sp :x27 0)
   (mezzano.lap.arm64:msr :spsel 0)

supervisor/debug.lisp

@@ -315,12 +315,14 @@
 (defun dump-threads ()
   (dump-thread (current-thread) (sys.int::read-frame-pointer))
   (when (boundp '*all-threads*)
-    (do ((thread *all-threads*
-                 (thread-global-next thread)))
-        ((null thread))
-      (when (not (eql thread (current-thread)))
-        (debug-print-line "----------")
-        (dump-thread thread (thread-frame-pointer thread))))))
+    (with-rcu-read-lock
+      (do ((thread *all-threads*
+                   (thread-global-next thread)))
+          ((null thread))
+        (when (and (not (eql thread (current-thread)))
+                   (not (eql (thread-state thread) :dead)))
+          (debug-print-line "----------")
+          (dump-thread thread (thread-frame-pointer thread)))))))
 
 (defun debug-dump ()
   (debug-print-line "Local CPU is " (local-cpu))

supervisor/disk.lisp

@@ -47,7 +47,7 @@
   lba
   n-sectors
   buffer
-  (lock (place-spinlock-initializer))
+  (lock :unlocked)
   (latch (make-event :name "Disk request notifier"))
   next)
 
@@ -84,7 +84,7 @@
     (setf *log-disk-requests* nil)
     (setf *disk-request-current* nil
           *disk-request-queue-head* nil
-          *disk-request-queue-lock* (place-spinlock-initializer)
+          *disk-request-queue-lock* :unlocked
           *disk-request-queue-latch* (make-event :name "Disk request queue notifier")
           *disks* '()))
   ;; Abort any queued or in-progress requests.

supervisor/entry.lisp

@@ -154,7 +154,7 @@
       (setf (sys.int::symbol-global-value 'mezzano.runtime::*active-catch-handlers*) 'nil
             (sys.int::symbol-global-value '*pseudo-atomic*) nil
             sys.int::*known-finalizers* nil
-            *big-wait-for-objects-lock* (place-spinlock-initializer)))
+            *big-wait-for-objects-lock* :unlocked))
     (initialize-early-platform)
     (when (boundp '*boot-id*)
       (setf (event-state *boot-id*) t))