drm/i915: CTB TLB invalidation fix on VM

drivers/gpu/drm/i915/gt/intel_ggtt.c

@@ -214,7 +214,7 @@ static void guc_ggtt_ct_invalidate(struct intel_gt *gt)
 	with_intel_runtime_pm_if_active(uncore->rpm, wakeref) {
 		struct intel_guc *guc = &gt->uc.guc;
 
-		intel_guc_invalidate_tlb(guc);
+		intel_guc_invalidate_tlb_guc(guc);
 	}
 }
 
@@ -252,7 +252,7 @@ static void gen12vf_ggtt_invalidate(struct i915_ggtt *ggtt)
 			continue;
 
 		with_intel_runtime_pm(gt->uncore->rpm, wakeref)
-			intel_guc_invalidate_tlb(guc);
+			intel_guc_invalidate_tlb_guc(guc);
 	}
 }
 
@@ -1049,8 +1049,7 @@ static int gen8_gmch_probe(struct i915_ggtt *ggtt)
 			I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
 	}
 
-	if (intel_uc_wants_guc(&ggtt->vm.gt->uc) &&
-	    intel_uc_wants_guc_submission(&ggtt->vm.gt->uc))
+	if (intel_uc_wants_guc_submission(&ggtt->vm.gt->uc))
 		ggtt->invalidate = guc_ggtt_invalidate;
 	else
 		ggtt->invalidate = gen8_ggtt_invalidate;

drivers/gpu/drm/i915/gt/intel_gt.h

@@ -93,14 +93,6 @@ static inline bool intel_gt_is_wedged(const struct intel_gt *gt)
 	return unlikely(test_bit(I915_WEDGED, &gt->reset.flags));
 }
 
-static inline bool intel_gt_is_enabled(const struct intel_gt *gt)
-{
-	/* Check if GT is wedged or suspended */
-	if (intel_gt_is_wedged(gt) || !intel_irqs_enabled(gt->i915))
-		return false;
-	return true;
-}
-
 int intel_gt_probe_all(struct drm_i915_private *i915);
 int intel_gt_tiles_init(struct drm_i915_private *i915);
 void intel_gt_release_all(struct drm_i915_private *i915);

drivers/gpu/drm/i915/gt/intel_gt_tlb.c

@@ -151,16 +151,20 @@ void intel_gt_tlb_invalidate(struct intel_gt *gt, u32 seqno)
 		if (tlb_seqno_passed(gt, seqno))
 			goto unlock;
 
-		if (intel_guc_submission_is_used(guc)) {
-			if (intel_guc_is_ready(guc))
-				intel_guc_invalidate_tlb_full(guc);
-		} else {
-			/*
-			 * Fall back to old path if GuC is disabled.
-			 * This is safe because GuC is not enabled and not writing to MMIO.
-			 */
-			mmio_invalidate_full(gt);
-		}
+		if (HAS_GUC_TLB_INVALIDATION(gt->i915)) {
+ 			/*
+ 			 * Fall back to old path if GuC is disabled.
+ 			 * This is safe because GuC is not enabled and not writing to MMIO.
+ 			 * Only perform GuC TLB invalidation if GuC is ready.
+ 			 * The only time GuC could not be ready is on GT reset,
+ 			 * which would clobber all the TLBs anyways, making
+ 			 * any TLB invalidation path here unnecessary.
+ 			 */
+ 			if (intel_guc_is_ready(guc))
+ 				intel_guc_invalidate_tlb_engines(guc);
+ 		} else {
+ 			mmio_invalidate_full(gt);
+ 		}
 
 		write_seqcount_invalidate(&gt->tlb.seqno);
 unlock:

drivers/gpu/drm/i915/gt/selftest_tlb.c

@@ -137,11 +137,14 @@ pte_tlbinv(struct intel_context *ce,
 	i915_request_add(rq);
 
 	/*
-	 * Short sleep to sanitycheck the batch is spinning before we begin
-	 * FIXME: needs updating to wait until the request has started
-	 * rather than waiting for a fixed amount of time.
+	 * Short sleep to sanitycheck the batch is spinning before we begin.
+	 * FIXME: Why is GSC so slow?
 	 */
-	msleep(200);
+	if (ce->engine->class == OTHER_CLASS)
+		msleep(200);
+	else
+		msleep(10);
+
 	if (va == vb) {
 		if (!i915_request_completed(rq)) {
 			pr_err("%s(%s): Semaphore sanitycheck failed %llx, with alignment %llx, using PTE size %x (phys %x, sg %x)\n",

drivers/gpu/drm/i915/gt/uc/abi/guc_actions_abi.h

@@ -188,7 +188,7 @@ enum intel_guc_state_capture_event_status {
 #define INTEL_GUC_TLB_INVAL_FLUSH_CACHE REG_BIT(31)
 
 enum intel_guc_tlb_invalidation_type {
-	INTEL_GUC_TLB_INVAL_FULL = 0x0,
+	INTEL_GUC_TLB_INVAL_ENGINES = 0x0,
 	INTEL_GUC_TLB_INVAL_GUC = 0x3,
 };
 

drivers/gpu/drm/i915/gt/uc/intel_guc.h

@@ -295,6 +295,11 @@ struct intel_guc {
 #endif
 };
 
+struct intel_guc_tlb_wait {
+	struct wait_queue_head wq;
+	bool busy;
+};
+
 /*
  * GuC version number components are only 8-bit, so converting to a 32bit 8.8.8
  * integer works.
@@ -303,11 +308,6 @@ struct intel_guc {
 #define MAKE_GUC_VER_STRUCT(ver)	MAKE_GUC_VER((ver).major, (ver).minor, (ver).patch)
 #define GUC_SUBMIT_VER(guc)		MAKE_GUC_VER_STRUCT((guc)->submission_version)
 
-struct intel_guc_tlb_wait {
-	struct wait_queue_head wq;
-	u8 status;
-} __aligned(4);
-
 static inline struct intel_guc *log_to_guc(struct intel_guc_log *log)
 {
 	return container_of(log, struct intel_guc, log);
@@ -438,8 +438,6 @@ int intel_guc_allocate_and_map_vma(struct intel_guc *guc, u32 size,
 int intel_guc_self_cfg32(struct intel_guc *guc, u16 key, u32 value);
 int intel_guc_self_cfg64(struct intel_guc *guc, u16 key, u64 value);
 
-int intel_guc_invalidate_tlb_full(struct intel_guc *guc);
-int intel_guc_invalidate_tlb(struct intel_guc *guc);
 int intel_guc_tlb_invalidation_done(struct intel_guc *guc, const u32 *hxg,
 				    u32 size);
 
@@ -543,6 +541,11 @@ void intel_guc_write_barrier(struct intel_guc *guc);
 void intel_guc_dump_time_info(struct intel_guc *guc, struct drm_printer *p);
 
 int intel_guc_sched_disable_gucid_threshold_max(struct intel_guc *guc);
+bool intel_guc_tlb_invalidation_is_available(struct intel_guc *guc);
+int intel_guc_invalidate_tlb_engines(struct intel_guc *guc);
+int intel_guc_invalidate_tlb_guc(struct intel_guc *guc);
+int intel_guc_tlb_invalidation_done(struct intel_guc *guc,
+				    const u32 *payload, u32 len);
 
 void wake_up_all_tlb_invalidate(struct intel_guc *guc);
 #endif

drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c

@@ -116,6 +116,33 @@ struct ct_incoming_msg {
 enum { CTB_SEND = 0, CTB_RECV = 1 };
 
 enum { CTB_OWNER_HOST = 0 };
+/*
+ * Some H2G commands involve a synchronous response that the driver needs
+ * to wait for. In such cases, a timeout is required to prevent the driver
+ * from waiting forever in the case of an error (either no error response
+ * is defined in the protocol or something has died and requires a reset).
+ * The specific command may be defined as having a time bound response but
+ * the CT is a queue and that time guarantee only starts from the point
+ * when the command reaches the head of the queue and is processed by GuC.
+ *
+ * Ideally there would be a helper to report the progress of a given
+ * command through the CT. However, that would require a significant
+ * amount of work in the CT layer. In the meantime, provide a reasonable
+ * estimation of the worst case latency it should take for the entire
+ * queue to drain. And therefore, how long a caller should wait before
+ * giving up on their request. The current estimate is based on empirical
+ * measurement of a test that fills the buffer with context creation and
+ * destruction requests as they seem to be the slowest operation.
+ */
+long intel_guc_ct_max_queue_time_jiffies(void)
+{
+	/*
+	 * A 4KB buffer full of context destroy commands takes a little
+	 * over a second to process so bump that to 2s to be super safe.
+	 */
+	return (CTB_H2G_BUFFER_SIZE * HZ) / SZ_2K;
+}
+
 
 /* FIXME: MTL cache coherency issue - HSD 22016122933 */
 static noinline void mtl_workaround_worker_func(struct work_struct *wrk);
@@ -1199,6 +1226,9 @@ static int ct_process_request(struct intel_guc_ct *ct, struct ct_incoming_msg *r
 		CT_ERROR(ct, "Received GuC exception notification!\n");
 		ret = 0;
 		break;
+	case INTEL_GUC_ACTION_TLB_INVALIDATION_DONE:
+ 		ret = intel_guc_tlb_invalidation_done(guc, payload, len);
+ 		break;
 	default:
 		ret = -EOPNOTSUPP;
 		break;
@@ -1273,13 +1303,12 @@ static int ct_handle_event(struct intel_guc_ct *ct, struct ct_incoming_msg *requ
 	case INTEL_GUC_ACTION_TLB_INVALIDATION_DONE:
 		g2h_release_space(ct, request->size);
 	}
-
-	/* Handle tlb invalidation response in interrupt context */
-	if (action == INTEL_GUC_ACTION_TLB_INVALIDATION_DONE) {
-		int ret = intel_guc_tlb_invalidation_done(ct_to_guc(ct), hxg, request->size);
-		ct_free_msg(request);
-		return ret;
-	}
+	/*
+	 * TLB invalidation responses must be handled immediately as processing
+	 * of other G2H notifications may be blocked by an invalidation request.
+	 */
+	if (action == INTEL_GUC_ACTION_TLB_INVALIDATION_DONE)
+		return ct_process_request(ct, request);
 
 	spin_lock_irqsave(&ct->requests.lock, flags);
 	list_add_tail(&request->link, &ct->requests.incoming);

drivers/gpu/drm/i915/gt/uc/intel_guc_ct.h

@@ -117,6 +117,8 @@ struct intel_guc_ct {
 #endif
 };
 
+long intel_guc_ct_max_queue_time_jiffies(void);
+
 void intel_guc_ct_init_early(struct intel_guc_ct *ct);
 int intel_guc_ct_init(struct intel_guc_ct *ct);
 void intel_guc_ct_fini(struct intel_guc_ct *ct);