mirrors/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2025-11-04 02:30:34 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

drm/xe: add XE_BO_FLAG_PINNED_LATE_RESTORE

Browse source 
With the idea of having more pinned objects using the blitter engine
where possible, during suspend/resume, mark the pinned objects which
can be done during the late phase once submission/migration has been
setup. Start out simple with lrc and page-tables from userspace.

v2:
 - s/early_restore/late_restore; early restore was way too bold with too
   many places being impacted at once.
v3:
 - Split late vs early into separate lists, to align with newly added
   apply-to-pinned infra.
v4:
 - Rebase.
v5:
 - Make sure we restore the late phase kernel_bo_present in igpu.

Signed-off-by: Matthew Auld <matthew.auld@intel.com>
Cc: Satyanarayana K V P <satyanarayana.k.v.p@intel.com>
Cc: Thomas Hellström <thomas.hellstrom@linux.intel.com>
Cc: Matthew Brost <matthew.brost@intel.com>
Reviewed-by: Satyanarayana K V P <satyanarayana.k.v.p@intel.com>
Link: https://lore.kernel.org/r/20250403102440.266113-13-matthew.auld@intel.com
This commit is contained in:
Matthew Auld 2025-04-03 11:24:45 +01:00
parent 58fa61ce4a
commit 7f387e6012
9 changed files with 93 additions and 52 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

4
drivers/gpu/drm/xe/tests/xe_bo.c
View file

@ -252,7 +252,7 @@ static int evict_test_run_tile(struct xe_device *xe, struct xe_tile *tile, struc
for_each_gt(__gt, xe, id) for_each_gt(__gt, xe, id)
xe_gt_sanitize(__gt); xe_gt_sanitize(__gt);
err = xe_bo_restore_kernel(xe); err = xe_bo_restore_early(xe);
/* /*
* Snapshotting the CTB and copying back a potentially old * Snapshotting the CTB and copying back a potentially old
* version seems risky, depending on what might have been * version seems risky, depending on what might have been
@ -273,7 +273,7 @@ static int evict_test_run_tile(struct xe_device *xe, struct xe_tile *tile, struc
goto cleanup_all; goto cleanup_all;
} }
err = xe_bo_restore_user(xe); err = xe_bo_restore_late(xe);
if (err) { if (err) {
KUNIT_FAIL(test, "restore user err=%pe\n", ERR_PTR(err)); KUNIT_FAIL(test, "restore user err=%pe\n", ERR_PTR(err));
goto cleanup_all; goto cleanup_all;

11
drivers/gpu/drm/xe/xe_bo.c
View file

@ -1121,7 +1121,7 @@ int xe_bo_evict_pinned(struct xe_bo *bo)
goto out_unlock_bo; goto out_unlock_bo;
} }
if (xe_bo_is_user(bo)) { if (xe_bo_is_user(bo) || (bo->flags & XE_BO_FLAG_PINNED_LATE_RESTORE)) {
struct xe_migrate *migrate; struct xe_migrate *migrate;
struct dma_fence *fence; struct dma_fence *fence;
@ -1216,7 +1216,7 @@ int xe_bo_restore_pinned(struct xe_bo *bo)
goto out_backup; goto out_backup;
} }
if (xe_bo_is_user(bo)) { if (xe_bo_is_user(bo) || (bo->flags & XE_BO_FLAG_PINNED_LATE_RESTORE)) {
struct xe_migrate *migrate; struct xe_migrate *migrate;
struct dma_fence *fence; struct dma_fence *fence;
@ -2187,7 +2187,7 @@ int xe_bo_pin_external(struct xe_bo *bo)
return err; return err;
spin_lock(&xe->pinned.lock); spin_lock(&xe->pinned.lock);
list_add_tail(&bo->pinned_link, &xe->pinned.external); list_add_tail(&bo->pinned_link, &xe->pinned.late.external);
spin_unlock(&xe->pinned.lock); spin_unlock(&xe->pinned.lock);
} }
@ -2232,7 +2232,10 @@ int xe_bo_pin(struct xe_bo *bo)
if (mem_type_is_vram(place->mem_type) || bo->flags & XE_BO_FLAG_GGTT) { if (mem_type_is_vram(place->mem_type) || bo->flags & XE_BO_FLAG_GGTT) {
spin_lock(&xe->pinned.lock); spin_lock(&xe->pinned.lock);
list_add_tail(&bo->pinned_link, &xe->pinned.kernel_bo_present); if (bo->flags & XE_BO_FLAG_PINNED_LATE_RESTORE)
list_add_tail(&bo->pinned_link, &xe->pinned.late.kernel_bo_present);
else
list_add_tail(&bo->pinned_link, &xe->pinned.early.kernel_bo_present);
spin_unlock(&xe->pinned.lock); spin_unlock(&xe->pinned.lock);
} }

9
drivers/gpu/drm/xe/xe_bo.h
View file

@ -40,10 +40,11 @@
#define XE_BO_FLAG_NEEDS_2M BIT(16) #define XE_BO_FLAG_NEEDS_2M BIT(16)
#define XE_BO_FLAG_GGTT_INVALIDATE BIT(17) #define XE_BO_FLAG_GGTT_INVALIDATE BIT(17)
#define XE_BO_FLAG_PINNED_NORESTORE BIT(18) #define XE_BO_FLAG_PINNED_NORESTORE BIT(18)
#define XE_BO_FLAG_GGTT0 BIT(19) #define XE_BO_FLAG_PINNED_LATE_RESTORE BIT(19)
#define XE_BO_FLAG_GGTT1 BIT(20) #define XE_BO_FLAG_GGTT0 BIT(20)
#define XE_BO_FLAG_GGTT2 BIT(21) #define XE_BO_FLAG_GGTT1 BIT(21)
#define XE_BO_FLAG_GGTT3 BIT(22) #define XE_BO_FLAG_GGTT2 BIT(22)
#define XE_BO_FLAG_GGTT3 BIT(23)
#define XE_BO_FLAG_GGTT_ALL (XE_BO_FLAG_GGTT0 | \ #define XE_BO_FLAG_GGTT_ALL (XE_BO_FLAG_GGTT0 | \
XE_BO_FLAG_GGTT1 | \ XE_BO_FLAG_GGTT1 | \
XE_BO_FLAG_GGTT2 | \ XE_BO_FLAG_GGTT2 | \

64
drivers/gpu/drm/xe/xe_bo_evict.c
View file

@ -91,10 +91,14 @@ int xe_bo_evict_all(struct xe_device *xe)
} }
} }
ret = xe_bo_apply_to_pinned(xe, &xe->pinned.external, ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.external,
&xe->pinned.external, &xe->pinned.late.external,
xe_bo_evict_pinned); xe_bo_evict_pinned);
if (!ret)
ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present,
&xe->pinned.late.evicted, xe_bo_evict_pinned);
/* /*
* Wait for all user BO to be evicted as those evictions depend on the * Wait for all user BO to be evicted as those evictions depend on the
* memory moved below. * memory moved below.
@ -105,8 +109,8 @@ int xe_bo_evict_all(struct xe_device *xe)
if (ret) if (ret)
return ret; return ret;
return xe_bo_apply_to_pinned(xe, &xe->pinned.kernel_bo_present, return xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present,
&xe->pinned.evicted, &xe->pinned.early.evicted,
xe_bo_evict_pinned); xe_bo_evict_pinned);
} }
@ -137,13 +141,14 @@ static int xe_bo_restore_and_map_ggtt(struct xe_bo *bo)
* We expect validate to trigger a move VRAM and our move code * We expect validate to trigger a move VRAM and our move code
* should setup the iosys map. * should setup the iosys map.
*/ */
xe_assert(xe, !iosys_map_is_null(&bo->vmap)); xe_assert(xe, !(bo->flags & XE_BO_FLAG_PINNED_LATE_RESTORE) ||
!iosys_map_is_null(&bo->vmap));
return 0; return 0;
} }
/** /**
* xe_bo_restore_kernel - restore kernel BOs to VRAM * xe_bo_restore_early - restore early phase kernel BOs to VRAM
* *
* @xe: xe device * @xe: xe device
* *
@ -153,34 +158,44 @@ static int xe_bo_restore_and_map_ggtt(struct xe_bo *bo)
* This function should be called early, before trying to init the GT, on device * This function should be called early, before trying to init the GT, on device
* resume. * resume.
*/ */
int xe_bo_restore_kernel(struct xe_device *xe) int xe_bo_restore_early(struct xe_device *xe)
{ {
return xe_bo_apply_to_pinned(xe, &xe->pinned.evicted, return xe_bo_apply_to_pinned(xe, &xe->pinned.early.evicted,
&xe->pinned.kernel_bo_present, &xe->pinned.early.kernel_bo_present,
xe_bo_restore_and_map_ggtt); xe_bo_restore_and_map_ggtt);
} }
/** /**
* xe_bo_restore_user - restore pinned user BOs to VRAM * xe_bo_restore_late - restore pinned late phase BOs
* *
* @xe: xe device * @xe: xe device
* *
* Move pinned user BOs from temporary (typically system) memory to VRAM via * Move pinned user and kernel BOs which can use blitter from temporary
* CPU. All moves done via TTM calls. * (typically system) memory to VRAM. All moves done via TTM calls.
* *
* This function should be called late, after GT init, on device resume. * This function should be called late, after GT init, on device resume.
*/ */
int xe_bo_restore_user(struct xe_device *xe) int xe_bo_restore_late(struct xe_device *xe)
{ {
struct xe_tile *tile; struct xe_tile *tile;
int ret, id; int ret, id;
ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.evicted,
&xe->pinned.late.kernel_bo_present,
xe_bo_restore_and_map_ggtt);
for_each_tile(tile, xe, id)
xe_tile_migrate_wait(tile);
if (ret)
return ret;
if (!IS_DGFX(xe)) if (!IS_DGFX(xe))
return 0; return 0;
/* Pinned user memory in VRAM should be validated on resume */ /* Pinned user memory in VRAM should be validated on resume */
ret = xe_bo_apply_to_pinned(xe, &xe->pinned.external, ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.external,
&xe->pinned.external, &xe->pinned.late.external,
xe_bo_restore_pinned); xe_bo_restore_pinned);
/* Wait for restore to complete */ /* Wait for restore to complete */
@ -195,8 +210,8 @@ static void xe_bo_pci_dev_remove_pinned(struct xe_device *xe)
struct xe_tile *tile; struct xe_tile *tile;
unsigned int id; unsigned int id;
(void)xe_bo_apply_to_pinned(xe, &xe->pinned.external, (void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.external,
&xe->pinned.external, &xe->pinned.late.external,
xe_bo_dma_unmap_pinned); xe_bo_dma_unmap_pinned);
for_each_tile(tile, xe, id) for_each_tile(tile, xe, id)
xe_tile_migrate_wait(tile); xe_tile_migrate_wait(tile);
@ -241,8 +256,11 @@ static void xe_bo_pinned_fini(void *arg)
{ {
struct xe_device *xe = arg; struct xe_device *xe = arg;
(void)xe_bo_apply_to_pinned(xe, &xe->pinned.kernel_bo_present, (void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present,
&xe->pinned.kernel_bo_present, &xe->pinned.late.kernel_bo_present,
xe_bo_dma_unmap_pinned);
(void)xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present,
&xe->pinned.early.kernel_bo_present,
xe_bo_dma_unmap_pinned); xe_bo_dma_unmap_pinned);
} }
@ -259,9 +277,11 @@ static void xe_bo_pinned_fini(void *arg)
int xe_bo_pinned_init(struct xe_device *xe) int xe_bo_pinned_init(struct xe_device *xe)
{ {
spin_lock_init(&xe->pinned.lock); spin_lock_init(&xe->pinned.lock);
INIT_LIST_HEAD(&xe->pinned.kernel_bo_present); INIT_LIST_HEAD(&xe->pinned.early.kernel_bo_present);
INIT_LIST_HEAD(&xe->pinned.external); INIT_LIST_HEAD(&xe->pinned.early.evicted);
INIT_LIST_HEAD(&xe->pinned.evicted); INIT_LIST_HEAD(&xe->pinned.late.kernel_bo_present);
INIT_LIST_HEAD(&xe->pinned.late.evicted);
INIT_LIST_HEAD(&xe->pinned.late.external);
return devm_add_action_or_reset(xe->drm.dev, xe_bo_pinned_fini, xe); return devm_add_action_or_reset(xe->drm.dev, xe_bo_pinned_fini, xe);
} }

4
drivers/gpu/drm/xe/xe_bo_evict.h
View file

@ -9,8 +9,8 @@
struct xe_device; struct xe_device;
int xe_bo_evict_all(struct xe_device *xe); int xe_bo_evict_all(struct xe_device *xe);
int xe_bo_restore_kernel(struct xe_device *xe); int xe_bo_restore_early(struct xe_device *xe);
int xe_bo_restore_user(struct xe_device *xe); int xe_bo_restore_late(struct xe_device *xe);
void xe_bo_pci_dev_remove_all(struct xe_device *xe); void xe_bo_pci_dev_remove_all(struct xe_device *xe);

22
drivers/gpu/drm/xe/xe_device_types.h
View file

@ -422,12 +422,22 @@ struct xe_device {
struct { struct {
/** @pinned.lock: protected pinned BO list state */ /** @pinned.lock: protected pinned BO list state */
spinlock_t lock; spinlock_t lock;
/** @pinned.kernel_bo_present: pinned kernel BO that are present */ /** @pinned.early: early pinned lists */
struct list_head kernel_bo_present; struct {
/** @pinned.evicted: pinned BO that have been evicted */ /** @pinned.early.kernel_bo_present: pinned kernel BO that are present */
struct list_head evicted; struct list_head kernel_bo_present;
/** @pinned.external: pinned external and dma-buf. */ /** @pinned.early.evicted: pinned BO that have been evicted */
struct list_head external; struct list_head evicted;
} early;
/** @pinned.late: late pinned lists */
struct {
/** @pinned.late.kernel_bo_present: pinned kernel BO that are present */
struct list_head kernel_bo_present;
/** @pinned.late.evicted: pinned BO that have been evicted */
struct list_head evicted;
/** @pinned.external: pinned external and dma-buf. */
struct list_head external;
} late;
} pinned; } pinned;
/** @ufence_wq: user fence wait queue */ /** @ufence_wq: user fence wait queue */

10
drivers/gpu/drm/xe/xe_lrc.c
View file

@ -896,6 +896,7 @@ static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe,
void *init_data = NULL; void *init_data = NULL;
u32 arb_enable; u32 arb_enable;
u32 lrc_size; u32 lrc_size;
u32 bo_flags;
int err; int err;
kref_init(&lrc->refcount); kref_init(&lrc->refcount);
@ -904,15 +905,18 @@ static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe,
if (xe_gt_has_indirect_ring_state(gt)) if (xe_gt_has_indirect_ring_state(gt))
lrc->flags |= XE_LRC_FLAG_INDIRECT_RING_STATE; lrc->flags |= XE_LRC_FLAG_INDIRECT_RING_STATE;
bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile) | XE_BO_FLAG_GGTT |
XE_BO_FLAG_GGTT_INVALIDATE;
if (vm && vm->xef) /* userspace */
bo_flags |= XE_BO_FLAG_PINNED_LATE_RESTORE;
/* /*
* FIXME: Perma-pinning LRC as we don't yet support moving GGTT address * FIXME: Perma-pinning LRC as we don't yet support moving GGTT address
* via VM bind calls. * via VM bind calls.
*/ */
lrc->bo = xe_bo_create_pin_map(xe, tile, vm, lrc_size, lrc->bo = xe_bo_create_pin_map(xe, tile, vm, lrc_size,
ttm_bo_type_kernel, ttm_bo_type_kernel,
XE_BO_FLAG_VRAM_IF_DGFX(tile) | bo_flags);
XE_BO_FLAG_GGTT |
XE_BO_FLAG_GGTT_INVALIDATE);
if (IS_ERR(lrc->bo)) if (IS_ERR(lrc->bo))
return PTR_ERR(lrc->bo); return PTR_ERR(lrc->bo);

8
drivers/gpu/drm/xe/xe_pm.c
View file

@ -188,7 +188,7 @@ int xe_pm_resume(struct xe_device *xe)
* This only restores pinned memory which is the memory required for the * This only restores pinned memory which is the memory required for the
* GT(s) to resume. * GT(s) to resume.
*/ */
err = xe_bo_restore_kernel(xe); err = xe_bo_restore_early(xe);
if (err) if (err)
goto err; goto err;
@ -199,7 +199,7 @@ int xe_pm_resume(struct xe_device *xe)
xe_display_pm_resume(xe); xe_display_pm_resume(xe);
err = xe_bo_restore_user(xe); err = xe_bo_restore_late(xe);
if (err) if (err)
goto err; goto err;
@ -484,7 +484,7 @@ int xe_pm_runtime_resume(struct xe_device *xe)
* This only restores pinned memory which is the memory * This only restores pinned memory which is the memory
* required for the GT(s) to resume. * required for the GT(s) to resume.
*/ */
err = xe_bo_restore_kernel(xe); err = xe_bo_restore_early(xe);
if (err) if (err)
goto out; goto out;
} }
@ -497,7 +497,7 @@ int xe_pm_runtime_resume(struct xe_device *xe)
xe_display_pm_runtime_resume(xe); xe_display_pm_runtime_resume(xe);
if (xe->d3cold.allowed) { if (xe->d3cold.allowed) {
err = xe_bo_restore_user(xe); err = xe_bo_restore_late(xe);
if (err) if (err)
goto out; goto out;
} }

13
drivers/gpu/drm/xe/xe_pt.c
View file

@ -103,6 +103,7 @@ struct xe_pt *xe_pt_create(struct xe_vm *vm, struct xe_tile *tile,
{ {
struct xe_pt *pt; struct xe_pt *pt;
struct xe_bo *bo; struct xe_bo *bo;
u32 bo_flags;
int err; int err;
if (level) { if (level) {
@ -115,14 +116,16 @@ struct xe_pt *xe_pt_create(struct xe_vm *vm, struct xe_tile *tile,
if (!pt) if (!pt)
return ERR_PTR(-ENOMEM); return ERR_PTR(-ENOMEM);
bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile) |
XE_BO_FLAG_IGNORE_MIN_PAGE_SIZE | XE_BO_FLAG_PINNED |
XE_BO_FLAG_NO_RESV_EVICT | XE_BO_FLAG_PAGETABLE;
if (vm->xef) /* userspace */
bo_flags |= XE_BO_FLAG_PINNED_LATE_RESTORE;
pt->level = level; pt->level = level;
bo = xe_bo_create_pin_map(vm->xe, tile, vm, SZ_4K, bo = xe_bo_create_pin_map(vm->xe, tile, vm, SZ_4K,
ttm_bo_type_kernel, ttm_bo_type_kernel,
XE_BO_FLAG_VRAM_IF_DGFX(tile) | bo_flags);
XE_BO_FLAG_IGNORE_MIN_PAGE_SIZE |
XE_BO_FLAG_PINNED |
XE_BO_FLAG_NO_RESV_EVICT |
XE_BO_FLAG_PAGETABLE);
if (IS_ERR(bo)) { if (IS_ERR(bo)) {
err = PTR_ERR(bo); err = PTR_ERR(bo);
goto err_kfree; goto err_kfree;