From d1101de6837aae350174bfcd29b1e1b08b70562e Mon Sep 17 00:00:00 2001 From: Kui Wen Date: Tue, 25 Mar 2025 07:52:41 +0000 Subject: [PATCH] drm/i915: CTB TLB invalidation fix on VM Port and refine the patch https://github.com/intel/linux-intel-lts/commit/c75552e4fc1b556c4d33ec20ec1e5c99f666068e for fixing TLB invalidation timeout issue Signed-off-by: Kui Wen --- drivers/gpu/drm/i915/gt/intel_ggtt.c | 7 +- drivers/gpu/drm/i915/gt/intel_gt.h | 8 - drivers/gpu/drm/i915/gt/intel_gt_tlb.c | 24 +-- drivers/gpu/drm/i915/gt/selftest_tlb.c | 11 +- .../gpu/drm/i915/gt/uc/abi/guc_actions_abi.h | 2 +- drivers/gpu/drm/i915/gt/uc/intel_guc.h | 17 +- drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c | 43 ++++- drivers/gpu/drm/i915/gt/uc/intel_guc_ct.h | 2 + .../gpu/drm/i915/gt/uc/intel_guc_submission.c | 161 +++++++++--------- drivers/gpu/drm/i915/gt/uc/intel_uc.c | 7 + drivers/gpu/drm/i915/i915_driver.c | 5 - drivers/gpu/drm/i915/i915_drv.h | 2 + drivers/gpu/drm/i915/i915_pci.c | 2 + drivers/gpu/drm/i915/intel_device_info.h | 1 + 14 files changed, 161 insertions(+), 131 deletions(-) diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt.c b/drivers/gpu/drm/i915/gt/intel_ggtt.c index 4bd82d7100066..ff4752ee0118c 100644 --- a/drivers/gpu/drm/i915/gt/intel_ggtt.c +++ b/drivers/gpu/drm/i915/gt/intel_ggtt.c @@ -214,7 +214,7 @@ static void guc_ggtt_ct_invalidate(struct intel_gt *gt) with_intel_runtime_pm_if_active(uncore->rpm, wakeref) { struct intel_guc *guc = >->uc.guc; - intel_guc_invalidate_tlb(guc); + intel_guc_invalidate_tlb_guc(guc); } } @@ -252,7 +252,7 @@ static void gen12vf_ggtt_invalidate(struct i915_ggtt *ggtt) continue; with_intel_runtime_pm(gt->uncore->rpm, wakeref) - intel_guc_invalidate_tlb(guc); + intel_guc_invalidate_tlb_guc(guc); } } @@ -1049,8 +1049,7 @@ static int gen8_gmch_probe(struct i915_ggtt *ggtt) I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND; } - if (intel_uc_wants_guc(&ggtt->vm.gt->uc) && - intel_uc_wants_guc_submission(&ggtt->vm.gt->uc)) + if (intel_uc_wants_guc_submission(&ggtt->vm.gt->uc)) ggtt->invalidate = guc_ggtt_invalidate; else ggtt->invalidate = gen8_ggtt_invalidate; diff --git a/drivers/gpu/drm/i915/gt/intel_gt.h b/drivers/gpu/drm/i915/gt/intel_gt.h index 086a02d8b17b3..ac598c2ed06cc 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt.h +++ b/drivers/gpu/drm/i915/gt/intel_gt.h @@ -93,14 +93,6 @@ static inline bool intel_gt_is_wedged(const struct intel_gt *gt) return unlikely(test_bit(I915_WEDGED, >->reset.flags)); } -static inline bool intel_gt_is_enabled(const struct intel_gt *gt) -{ - /* Check if GT is wedged or suspended */ - if (intel_gt_is_wedged(gt) || !intel_irqs_enabled(gt->i915)) - return false; - return true; -} - int intel_gt_probe_all(struct drm_i915_private *i915); int intel_gt_tiles_init(struct drm_i915_private *i915); void intel_gt_release_all(struct drm_i915_private *i915); diff --git a/drivers/gpu/drm/i915/gt/intel_gt_tlb.c b/drivers/gpu/drm/i915/gt/intel_gt_tlb.c index 3b6ff0c9f862a..cbf40e6806481 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_tlb.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_tlb.c @@ -151,16 +151,20 @@ void intel_gt_tlb_invalidate(struct intel_gt *gt, u32 seqno) if (tlb_seqno_passed(gt, seqno)) goto unlock; - if (intel_guc_submission_is_used(guc)) { - if (intel_guc_is_ready(guc)) - intel_guc_invalidate_tlb_full(guc); - } else { - /* - * Fall back to old path if GuC is disabled. - * This is safe because GuC is not enabled and not writing to MMIO. - */ - mmio_invalidate_full(gt); - } + if (HAS_GUC_TLB_INVALIDATION(gt->i915)) { + /* + * Fall back to old path if GuC is disabled. + * This is safe because GuC is not enabled and not writing to MMIO. + * Only perform GuC TLB invalidation if GuC is ready. + * The only time GuC could not be ready is on GT reset, + * which would clobber all the TLBs anyways, making + * any TLB invalidation path here unnecessary. + */ + if (intel_guc_is_ready(guc)) + intel_guc_invalidate_tlb_engines(guc); + } else { + mmio_invalidate_full(gt); + } write_seqcount_invalidate(>->tlb.seqno); unlock: diff --git a/drivers/gpu/drm/i915/gt/selftest_tlb.c b/drivers/gpu/drm/i915/gt/selftest_tlb.c index a927d9f8c12cf..f7eba292a8f8d 100644 --- a/drivers/gpu/drm/i915/gt/selftest_tlb.c +++ b/drivers/gpu/drm/i915/gt/selftest_tlb.c @@ -137,11 +137,14 @@ pte_tlbinv(struct intel_context *ce, i915_request_add(rq); /* - * Short sleep to sanitycheck the batch is spinning before we begin - * FIXME: needs updating to wait until the request has started - * rather than waiting for a fixed amount of time. + * Short sleep to sanitycheck the batch is spinning before we begin. + * FIXME: Why is GSC so slow? */ - msleep(200); + if (ce->engine->class == OTHER_CLASS) + msleep(200); + else + msleep(10); + if (va == vb) { if (!i915_request_completed(rq)) { pr_err("%s(%s): Semaphore sanitycheck failed %llx, with alignment %llx, using PTE size %x (phys %x, sg %x)\n", diff --git a/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_abi.h b/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_abi.h index 9dff8012d5e76..33f253410d0c8 100644 --- a/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_abi.h +++ b/drivers/gpu/drm/i915/gt/uc/abi/guc_actions_abi.h @@ -188,7 +188,7 @@ enum intel_guc_state_capture_event_status { #define INTEL_GUC_TLB_INVAL_FLUSH_CACHE REG_BIT(31) enum intel_guc_tlb_invalidation_type { - INTEL_GUC_TLB_INVAL_FULL = 0x0, + INTEL_GUC_TLB_INVAL_ENGINES = 0x0, INTEL_GUC_TLB_INVAL_GUC = 0x3, }; diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.h b/drivers/gpu/drm/i915/gt/uc/intel_guc.h index 33ec219c4fb99..6544f4e583658 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.h @@ -295,6 +295,11 @@ struct intel_guc { #endif }; +struct intel_guc_tlb_wait { + struct wait_queue_head wq; + bool busy; +}; + /* * GuC version number components are only 8-bit, so converting to a 32bit 8.8.8 * integer works. @@ -303,11 +308,6 @@ struct intel_guc { #define MAKE_GUC_VER_STRUCT(ver) MAKE_GUC_VER((ver).major, (ver).minor, (ver).patch) #define GUC_SUBMIT_VER(guc) MAKE_GUC_VER_STRUCT((guc)->submission_version) -struct intel_guc_tlb_wait { - struct wait_queue_head wq; - u8 status; -} __aligned(4); - static inline struct intel_guc *log_to_guc(struct intel_guc_log *log) { return container_of(log, struct intel_guc, log); @@ -438,8 +438,6 @@ int intel_guc_allocate_and_map_vma(struct intel_guc *guc, u32 size, int intel_guc_self_cfg32(struct intel_guc *guc, u16 key, u32 value); int intel_guc_self_cfg64(struct intel_guc *guc, u16 key, u64 value); -int intel_guc_invalidate_tlb_full(struct intel_guc *guc); -int intel_guc_invalidate_tlb(struct intel_guc *guc); int intel_guc_tlb_invalidation_done(struct intel_guc *guc, const u32 *hxg, u32 size); @@ -543,6 +541,11 @@ void intel_guc_write_barrier(struct intel_guc *guc); void intel_guc_dump_time_info(struct intel_guc *guc, struct drm_printer *p); int intel_guc_sched_disable_gucid_threshold_max(struct intel_guc *guc); +bool intel_guc_tlb_invalidation_is_available(struct intel_guc *guc); +int intel_guc_invalidate_tlb_engines(struct intel_guc *guc); +int intel_guc_invalidate_tlb_guc(struct intel_guc *guc); +int intel_guc_tlb_invalidation_done(struct intel_guc *guc, + const u32 *payload, u32 len); void wake_up_all_tlb_invalidate(struct intel_guc *guc); #endif diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c index 0ccac0b856b4b..d7d9dbd54c79e 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c @@ -116,6 +116,33 @@ struct ct_incoming_msg { enum { CTB_SEND = 0, CTB_RECV = 1 }; enum { CTB_OWNER_HOST = 0 }; +/* + * Some H2G commands involve a synchronous response that the driver needs + * to wait for. In such cases, a timeout is required to prevent the driver + * from waiting forever in the case of an error (either no error response + * is defined in the protocol or something has died and requires a reset). + * The specific command may be defined as having a time bound response but + * the CT is a queue and that time guarantee only starts from the point + * when the command reaches the head of the queue and is processed by GuC. + * + * Ideally there would be a helper to report the progress of a given + * command through the CT. However, that would require a significant + * amount of work in the CT layer. In the meantime, provide a reasonable + * estimation of the worst case latency it should take for the entire + * queue to drain. And therefore, how long a caller should wait before + * giving up on their request. The current estimate is based on empirical + * measurement of a test that fills the buffer with context creation and + * destruction requests as they seem to be the slowest operation. + */ +long intel_guc_ct_max_queue_time_jiffies(void) +{ + /* + * A 4KB buffer full of context destroy commands takes a little + * over a second to process so bump that to 2s to be super safe. + */ + return (CTB_H2G_BUFFER_SIZE * HZ) / SZ_2K; +} + /* FIXME: MTL cache coherency issue - HSD 22016122933 */ static noinline void mtl_workaround_worker_func(struct work_struct *wrk); @@ -1199,6 +1226,9 @@ static int ct_process_request(struct intel_guc_ct *ct, struct ct_incoming_msg *r CT_ERROR(ct, "Received GuC exception notification!\n"); ret = 0; break; + case INTEL_GUC_ACTION_TLB_INVALIDATION_DONE: + ret = intel_guc_tlb_invalidation_done(guc, payload, len); + break; default: ret = -EOPNOTSUPP; break; @@ -1273,13 +1303,12 @@ static int ct_handle_event(struct intel_guc_ct *ct, struct ct_incoming_msg *requ case INTEL_GUC_ACTION_TLB_INVALIDATION_DONE: g2h_release_space(ct, request->size); } - - /* Handle tlb invalidation response in interrupt context */ - if (action == INTEL_GUC_ACTION_TLB_INVALIDATION_DONE) { - int ret = intel_guc_tlb_invalidation_done(ct_to_guc(ct), hxg, request->size); - ct_free_msg(request); - return ret; - } + /* + * TLB invalidation responses must be handled immediately as processing + * of other G2H notifications may be blocked by an invalidation request. + */ + if (action == INTEL_GUC_ACTION_TLB_INVALIDATION_DONE) + return ct_process_request(ct, request); spin_lock_irqsave(&ct->requests.lock, flags); list_add_tail(&request->link, &ct->requests.incoming); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.h index 6439f8c102a35..0073c672ae18b 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.h @@ -117,6 +117,8 @@ struct intel_guc_ct { #endif }; +long intel_guc_ct_max_queue_time_jiffies(void); + void intel_guc_ct_init_early(struct intel_guc_ct *ct); int intel_guc_ct_init(struct intel_guc_ct *ct); void intel_guc_ct_fini(struct intel_guc_ct *ct); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c index 8eaf327b323a8..9019acd7dc989 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c @@ -1765,22 +1765,21 @@ static void __guc_reset_context(struct intel_context *ce, intel_engine_mask_t st intel_context_put(parent); } -static void wake_up_tlb_invalidate(struct intel_guc_tlb_wait *wait) -{ - /* Barrier to ensure the store is observed by the woken thread */ - smp_store_mb(wait->status, 0); - wake_up(&wait->wq); -} - void wake_up_all_tlb_invalidate(struct intel_guc *guc) { struct intel_guc_tlb_wait *wait; unsigned long i; + if (!intel_guc_tlb_invalidation_is_available(guc)) + return; + + xa_lock_irq(&guc->tlb_lookup); xa_for_each(&guc->tlb_lookup, i, wait) - wake_up_tlb_invalidate(wait); + wake_up(&wait->wq); + xa_unlock_irq(&guc->tlb_lookup); } + void intel_guc_submission_reset(struct intel_guc *guc, intel_engine_mask_t stalled) { struct intel_context *ce; @@ -1922,6 +1921,9 @@ void intel_guc_submission_cancel_requests(struct intel_guc *guc) void intel_guc_submission_reset_finish(struct intel_guc *guc) { + struct intel_guc_tlb_wait *wait; + unsigned long i; + /* Reset called during driver load or during wedge? */ if (unlikely(!guc_submission_initialized(guc) || intel_gt_is_wedged(guc_to_gt(guc)))) { @@ -1940,17 +1942,38 @@ void intel_guc_submission_reset_finish(struct intel_guc *guc) intel_guc_global_policies_update(guc); enable_submission(guc); intel_gt_unpark_heartbeats(guc_to_gt(guc)); + + /* + * The full GT reset will have cleared the TLB caches and flushed the + * G2H message queue; we can release all the blocked waiters. + */ + if (intel_guc_tlb_invalidation_is_available(guc)) { + xa_lock_irq(&guc->tlb_lookup); + xa_for_each(&guc->tlb_lookup, i, wait) + wake_up(&wait->wq); + xa_unlock_irq(&guc->tlb_lookup); + } + } static void destroyed_worker_func(struct work_struct *w); static void reset_fail_worker_func(struct work_struct *w); static int number_mlrc_guc_id(struct intel_guc *guc); +bool intel_guc_tlb_invalidation_is_available(struct intel_guc *guc) +{ + return HAS_GUC_TLB_INVALIDATION(guc_to_gt(guc)->i915) && + intel_guc_is_ready(guc); +} + static int init_tlb_lookup(struct intel_guc *guc) { struct intel_guc_tlb_wait *wait; int err; + if (!HAS_GUC_TLB_INVALIDATION(guc_to_gt(guc)->i915)) + return 0; + xa_init_flags(&guc->tlb_lookup, XA_FLAGS_ALLOC); wait = kzalloc(sizeof(*wait), GFP_KERNEL); @@ -1960,7 +1983,7 @@ static int init_tlb_lookup(struct intel_guc *guc) init_waitqueue_head(&wait->wq); err = xa_alloc_cyclic_irq(&guc->tlb_lookup, &guc->serial_slot, wait, xa_limit_32b, &guc->next_seqno, GFP_KERNEL); - if (err == -ENOMEM) { + if (err < 0) { kfree(wait); return err; } @@ -1971,12 +1994,13 @@ static int init_tlb_lookup(struct intel_guc *guc) static void fini_tlb_lookup(struct intel_guc *guc) { struct intel_guc_tlb_wait *wait; + if (!HAS_GUC_TLB_INVALIDATION(guc_to_gt(guc)->i915)) + return; wait = xa_load(&guc->tlb_lookup, guc->serial_slot); - if (wait) { - GEM_BUG_ON(wait->status); - kfree(wait); - } + if (wait && wait->busy) + guc_err(guc, "Unexpected busy item in tlb_lookup on fini\n"); + kfree(wait); xa_destroy(&guc->tlb_lookup); } @@ -4742,33 +4766,22 @@ static void wait_wake_outstanding_tlb_g2h(struct intel_guc *guc, u32 seqno) xa_lock_irqsave(&guc->tlb_lookup, flags); wait = xa_load(&guc->tlb_lookup, seqno); - /* We received a response after the waiting task did exit with a timeout */ - if (unlikely(!wait)) - drm_dbg(&guc_to_gt(guc)->i915->drm, - "Stale TLB invalidation response with seqno %d\n", seqno); - if (wait) - wake_up_tlb_invalidate(wait); + wake_up(&wait->wq); + else + guc_dbg(guc, + "Stale TLB invalidation response with seqno %d\n", seqno); xa_unlock_irqrestore(&guc->tlb_lookup, flags); } -int intel_guc_tlb_invalidation_done(struct intel_guc *guc, const u32 *hxg, u32 size) +int intel_guc_tlb_invalidation_done(struct intel_guc *guc, const u32 *payload, u32 len) { - u32 seqno, hxg_len, len; - - /* - * FIXME: these calculations would be better done signed. That - * way underflow can be detected as well. - */ - hxg_len = size - GUC_CTB_MSG_MIN_LEN; - len = hxg_len - GUC_HXG_MSG_MIN_LEN; - if (unlikely(len < 1)) + if (len < 1) return -EPROTO; - seqno = hxg[GUC_HXG_MSG_MIN_LEN]; - wait_wake_outstanding_tlb_g2h(guc, seqno); + wait_wake_outstanding_tlb_g2h(guc, payload[0]); return 0; } @@ -4780,11 +4793,6 @@ static long must_wait_woken(struct wait_queue_entry *wq_entry, long timeout) * As we are called from page reclaim in any task context, * we may be invoked from stopped kthreads, but we *must* * complete the wait from the HW . - * - * A second problem is that since we are called under reclaim - * and wait_woken() inspected the thread state, it makes an invalid - * assumption that all PF_KTHREAD tasks have set_kthread_struct() - * called upon them, and will trigger a GPF in is_kthread_should_stop(). */ do { set_current_state(TASK_UNINTERRUPTIBLE); @@ -4801,6 +4809,15 @@ static long must_wait_woken(struct wait_queue_entry *wq_entry, long timeout) return timeout; } +static bool intel_gt_is_enabled(const struct intel_gt *gt) +{ + /* Check if GT is wedged or suspended */ + if (intel_gt_is_wedged(gt) || !intel_irqs_enabled(gt->i915)) + return false; + return true; +} + + static int guc_send_invalidate_tlb(struct intel_guc *guc, u32 type) { struct intel_guc_tlb_wait _wq, *wq = &_wq; @@ -4818,11 +4835,15 @@ static int guc_send_invalidate_tlb(struct intel_guc *guc, u32 type) INTEL_GUC_TLB_INVAL_FLUSH_CACHE, }; u32 size = ARRAY_SIZE(action); - - if (!intel_guc_ct_enabled(&guc->ct) || - !intel_gt_is_enabled(gt)) + + /* + * Early guard against GT enablement. TLB invalidation should not be + * attempted if the GT is disabled due to suspend/wedge. + */ + if (!intel_gt_is_enabled(gt)) return -EINVAL; + init_waitqueue_head(&_wq.wq); if (xa_alloc_cyclic_irq(&guc->tlb_lookup, &seqno, wq, @@ -4832,15 +4853,15 @@ static int guc_send_invalidate_tlb(struct intel_guc *guc, u32 type) xa_lock_irq(&guc->tlb_lookup); wq = xa_load(&guc->tlb_lookup, guc->serial_slot); wait_event_lock_irq(wq->wq, - !READ_ONCE(wq->status), + !READ_ONCE(wq->busy), guc->tlb_lookup.xa_lock); /* - * Update wq->status under lock to ensure only one waiter can + * Update wq->busy under lock to ensure only one waiter can * issue the TLB invalidation command using the serial slot at a - * time. The condition is set to false before releasing the lock + * time. The condition is set to true before releasing the lock * so that other caller continue to wait until woken up again. */ - wq->status = 1; + wq->busy = true; xa_unlock_irq(&guc->tlb_lookup); seqno = guc->serial_slot; @@ -4855,47 +4876,17 @@ static int guc_send_invalidate_tlb(struct intel_guc *guc, u32 type) goto out; } /* - * GuC has a timeout of 1ms for a TLB invalidation response from GAM. On a - * timeout GuC drops the request and has no mechanism to notify the host about - * the timeout. So keep a larger timeout that accounts for this individual - * timeout and max number of outstanding invalidation requests that can be - * queued in CT buffer. - * - * Although the invalidation request itself should complete within 1ms, the - * request might be in a long queue of other, slower, CTB requests. If the - * CTB buffer is fully backed up, a multi-second delay is possible. Make a - * while loop of two 1-second-waits with debug prints to catch attention for - * potential issues. - * - * FIXME: Add a check for the CTB buffer processing to have passed the TLB - * invalidation request before starting the timeout. Blindly waiting for 2 - * seconds doesn't really ensure the failure is not just a consequence of - * slow GuC processing. + * Late guard against GT enablement. It is not an error for the TLB + * invalidation to time out if the GT is disabled during the process + * due to suspend/wedge. In fact, the TLB invalidation is cancelled + * in this case. */ -#define OUTSTANDING_GUC_TIMEOUT_PERIOD (HZ) - while (!timeout) { - timeout = must_wait_woken(&wait, OUTSTANDING_GUC_TIMEOUT_PERIOD); - if (timeout) - break; - gt_dbg(gt, "TLB invalidation (seqno=%u) pending for %us\n", seqno, ++elapsed); - if (elapsed >= 2) { - /* - * FIXME: Real TLB invalidation timeout is critical and warrants a GT - * reset. However, it's possible that after this long wait the GT could - * just come out from a reset thus it appears to be enabled, then this - * code here wedges the GT again. - */ - if (intel_gt_is_enabled(gt)) { - gt_err(gt, - "TLB invalidation response timed out for seqno %u\n", - seqno); - intel_gt_set_wedged(gt); - err = -ETIME; - } - break; - } + if (!must_wait_woken(&wait, intel_guc_ct_max_queue_time_jiffies()) && + intel_gt_is_enabled(gt)) { + guc_err(guc, + "TLB invalidation response timed out for seqno %u\n", seqno); + err = -ETIME; } - out: remove_wait_queue(&wq->wq, &wait); if (seqno != guc->serial_slot) @@ -4905,13 +4896,13 @@ static int guc_send_invalidate_tlb(struct intel_guc *guc, u32 type) } /* Full TLB invalidation */ -int intel_guc_invalidate_tlb_full(struct intel_guc *guc) +int intel_guc_invalidate_tlb_engines(struct intel_guc *guc) { - return guc_send_invalidate_tlb(guc, INTEL_GUC_TLB_INVAL_FULL); + return guc_send_invalidate_tlb(guc, INTEL_GUC_TLB_INVAL_ENGINES); } /* GuC TLB Invalidation: Invalidate the TLB's of GuC itself. */ -int intel_guc_invalidate_tlb(struct intel_guc *guc) +int intel_guc_invalidate_tlb_guc(struct intel_guc *guc) { return guc_send_invalidate_tlb(guc, INTEL_GUC_TLB_INVAL_GUC); } diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc.c b/drivers/gpu/drm/i915/gt/uc/intel_uc.c index f4c7c4deafce7..ca5828831b168 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc.c @@ -806,6 +806,8 @@ void intel_uc_suspend(struct intel_uc *uc) /* flush the GSC worker */ intel_gsc_uc_flush_work(&uc->gsc); + wake_up_all_tlb_invalidate(guc); + if (!intel_guc_is_ready(guc)) { guc->interrupts.enabled = false; return; @@ -857,6 +859,11 @@ static int __uc_resume(struct intel_uc *uc, bool enable_communication) intel_gsc_uc_resume(&uc->gsc); + if (intel_guc_tlb_invalidation_is_available(guc)) { + intel_guc_invalidate_tlb_engines(guc); + intel_guc_invalidate_tlb_guc(guc); + } + return 0; } diff --git a/drivers/gpu/drm/i915/i915_driver.c b/drivers/gpu/drm/i915/i915_driver.c index 81af5a0a1427a..f18fc79f015b2 100644 --- a/drivers/gpu/drm/i915/i915_driver.c +++ b/drivers/gpu/drm/i915/i915_driver.c @@ -1312,11 +1312,6 @@ static int i915_drm_resume(struct drm_device *dev) intel_gvt_resume(dev_priv); - for_each_gt(gt, dev_priv, i) { - intel_guc_invalidate_tlb_full(>->uc.guc); - intel_guc_invalidate_tlb(>->uc.guc); - } - enable_rpm_wakeref_asserts(&dev_priv->runtime_pm); return 0; diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 6a31275792a6a..41c56714decba 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -921,6 +921,8 @@ int i915_gem_get_obj_info(struct drm_i915_error_state_buf *m, #define HAS_GUC_DEPRIVILEGE(i915) \ (INTEL_INFO(i915)->has_guc_deprivilege) +#define HAS_GUC_TLB_INVALIDATION(i915) (INTEL_INFO(i915)->has_guc_tlb_invalidation) + #define HAS_3D_PIPELINE(i915) (INTEL_INFO(i915)->has_3d_pipeline) #define HAS_ONE_EU_PER_FUSE_BIT(i915) (INTEL_INFO(i915)->has_one_eu_per_fuse_bit) diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c index c1139451d5302..5729d5c8bf132 100644 --- a/drivers/gpu/drm/i915/i915_pci.c +++ b/drivers/gpu/drm/i915/i915_pci.c @@ -702,6 +702,7 @@ static const struct intel_device_info adl_p_info = { .__runtime.ppgtt_size = 47, .dma_mask_size = 39, .has_sriov = 1, + .has_guc_tlb_invalidation = 1, }; #undef GEN @@ -837,6 +838,7 @@ static const struct intel_device_info mtl_info = { .has_flat_ccs = 0, .has_gmd_id = 1, .has_guc_deprivilege = 1, + .has_guc_tlb_invalidation = 1, .has_llc = 0, .has_memirq = 1, .has_mslice_steering = 0, diff --git a/drivers/gpu/drm/i915/intel_device_info.h b/drivers/gpu/drm/i915/intel_device_info.h index e4a101d74880d..9f74440c7c17e 100644 --- a/drivers/gpu/drm/i915/intel_device_info.h +++ b/drivers/gpu/drm/i915/intel_device_info.h @@ -161,6 +161,7 @@ enum intel_ppgtt_type { func(has_heci_pxp); \ func(has_heci_gscfi); \ func(has_guc_deprivilege); \ + func(has_guc_tlb_invalidation); \ func(has_l3_ccs_read); \ func(has_l3_dpf); \ func(has_llc); \