drm/amdkfd: implement restore_mqd callbacks for GFX12/12.1

kfd_mqd_manager_v12.c (GFX 12.0) and kfd_mqd_manager_v12_1.c (GFX 12.1)
do not implement restore_mqd callbacks, leaving the function pointers
NULL and causing CRIU restore to return -EOPNOTSUPP on GFX12.

Implement restore_mqd for both compute and SDMA queues in
kfd_mqd_manager_v12.c and kfd_mqd_manager_v12_1.c, modeled after the
GFX 11 implementation with the following improvements:
- update cp_mqd_base_addr_lo/hi to the newly allocated MQD address,
  fixing a pre-existing gap shared with v11 where the in-MQD copy
  still pointed at the old checkpoint-time address after restore
- memset the full allocation before memcpy for compute queues to avoid
  stale data in the GTT sub-allocator tail; SDMA MQDs use sizeof(*m)
  since they are packed at mqd_size stride in a shared BO

checkpoint_mqd registration is deferred to a follow-up patch that also
implements get_checkpoint_info, so that checkpoint and restore are
enabled together as a complete and testable unit.

Note: GFX12.1 restore handles XCC0 only. Multi-XCC CRIU restore is
currently unreachable due to a separate validation issue in
kfd_criu_restore_queue(). A pr_warn_once() is emitted if a multi-XCC
device is encountered.

Signed-off-by: Vladimir Marioukhine <Vladimir.Marioukhine@amd.com>
Reviewed-by: Alex Deucher <alexander.deucher@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
(cherry picked from commit b1f9601237d050f5df478464cf51bf1fff29a256)
Cc: stable@vger.kernel.org

3 files changed, 132 insertions, 2 deletions

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index 9811e4e10291..2f78395a0c31 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -770,10 +770,12 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
 	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
 			q->properties.type)];
 	if (qd && !mqd_mgr->restore_mqd) {
-		pr_debug("restore_mqd not implemented for this GPU\n");
+		pr_debug("restore_mqd not implemented for queue type %d\n",
+			 q->properties.type);
 		retval = -EOPNOTSUPP;
 		goto deallocate_vmid;
 	}
+
 	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
 		retval = allocate_hqd(dqm, q);
 		if (retval)
@@ -2250,7 +2252,8 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 	mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
 			q->properties.type)];
 	if (qd && !mqd_mgr->restore_mqd) {
-		pr_debug("restore_mqd not implemented for this GPU\n");
+		pr_debug("restore_mqd not implemented for queue type %d\n",
+			 q->properties.type);
 		retval = -EOPNOTSUPP;
 		goto out_deallocate_doorbell;
 	}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c
index 7c387fa90076..63f25a60baa5 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c
@@ -380,6 +380,63 @@ static int debugfs_show_mqd_sdma(struct seq_file *m, void *data)
 
 #endif
 
+static void restore_mqd(struct mqd_manager *mm, void **mqd,
+			 struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
+			 struct queue_properties *qp, const void *mqd_src,
+			 const void *ctl_stack_src, const u32 ctl_stack_size)
+{
+	u64 addr;
+	struct v12_compute_mqd *m;
+
+	m = (struct v12_compute_mqd *)mqd_mem_obj->cpu_ptr;
+	addr = mqd_mem_obj->gpu_addr;
+
+	memset(m, 0, AMDGPU_MQD_SIZE_ALIGN(mm->mqd_size));
+	memcpy(m, mqd_src, sizeof(*m));
+
+	/* Update MQD base address to the newly allocated location */
+	m->cp_mqd_base_addr_lo = lower_32_bits(addr);
+	m->cp_mqd_base_addr_hi = upper_32_bits(addr);
+
+	m->cp_hqd_pq_doorbell_control &=
+		~CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK;
+	m->cp_hqd_pq_doorbell_control |=
+		qp->doorbell_off << CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
+	pr_debug("cp_hqd_pq_doorbell_control 0x%x\n", m->cp_hqd_pq_doorbell_control);
+
+	*mqd = m;
+	if (gart_addr)
+		*gart_addr = addr;
+
+	qp->is_active = 0;
+}
+
+static void restore_mqd_sdma(struct mqd_manager *mm, void **mqd,
+			      struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
+			      struct queue_properties *qp,
+			      const void *mqd_src,
+			      const void *ctl_stack_src,
+			      const u32 ctl_stack_size)
+{
+	u64 addr;
+	struct v12_sdma_mqd *m;
+
+	m = (struct v12_sdma_mqd *)mqd_mem_obj->cpu_ptr;
+	addr = mqd_mem_obj->gpu_addr;
+
+	memset(m, 0, AMDGPU_MQD_SIZE_ALIGN(mm->mqd_size));
+	memcpy(m, mqd_src, sizeof(*m));
+
+	m->sdmax_rlcx_doorbell_offset =
+		qp->doorbell_off << SDMA0_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT;
+
+	*mqd = m;
+	if (gart_addr)
+		*gart_addr = addr;
+
+	qp->is_active = 0;
+}
+
 struct mqd_manager *mqd_manager_init_v12(enum KFD_MQD_TYPE type,
 		struct kfd_node *dev)
 {
@@ -407,6 +464,7 @@ struct mqd_manager *mqd_manager_init_v12(enum KFD_MQD_TYPE type,
 		mqd->mqd_size = sizeof(struct v12_compute_mqd);
 		mqd->get_wave_state = get_wave_state;
 		mqd->mqd_stride = kfd_mqd_stride;
+		mqd->restore_mqd = restore_mqd;
 #if defined(CONFIG_DEBUG_FS)
 		mqd->debugfs_show_mqd = debugfs_show_mqd;
 #endif
@@ -453,6 +511,7 @@ struct mqd_manager *mqd_manager_init_v12(enum KFD_MQD_TYPE type,
 		mqd->is_occupied = kfd_is_occupied_sdma;
 		mqd->mqd_size = sizeof(struct v12_sdma_mqd);
 		mqd->mqd_stride = kfd_mqd_stride;
+		mqd->restore_mqd = restore_mqd_sdma;
 #if defined(CONFIG_DEBUG_FS)
 		mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;
 #endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
index 431a940f91f3..708bbb08c089 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
@@ -641,6 +641,72 @@ static int debugfs_show_mqd_sdma(struct seq_file *m, void *data)
 
 #endif
 
+static void restore_mqd_v12_1(struct mqd_manager *mm, void **mqd,
+			       struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
+			       struct queue_properties *qp, const void *mqd_src,
+			       const void *ctl_stack_src, const u32 ctl_stack_size)
+{
+	u64 addr;
+	struct v12_1_compute_mqd *m;
+
+	/*
+	 * GFX12.1 is multi-XCC capable but this restore handles XCC0 only.
+	 * Multi-XCC CRIU restore is currently unreachable because
+	 * kfd_criu_restore_queue() validates against unscaled mqd_size.
+	 */
+	if (NUM_XCC(mm->dev->xcc_mask) > 1)
+		pr_warn_once("GFX12.1 multi-XCC CRIU restore not fully supported\n");
+
+	m = (struct v12_1_compute_mqd *)mqd_mem_obj->cpu_ptr;
+	addr = mqd_mem_obj->gpu_addr;
+
+	memset(m, 0, AMDGPU_MQD_SIZE_ALIGN(mm->mqd_size) *
+		     NUM_XCC(mm->dev->xcc_mask));
+	memcpy(m, mqd_src, sizeof(*m));
+
+	/* Update MQD base address to the newly allocated location */
+	m->cp_mqd_base_addr_lo = lower_32_bits(addr);
+	m->cp_mqd_base_addr_hi = upper_32_bits(addr);
+
+	m->cp_hqd_pq_doorbell_control &=
+		~CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK;
+	m->cp_hqd_pq_doorbell_control |=
+		qp->doorbell_off << CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
+	pr_debug("cp_hqd_pq_doorbell_control 0x%x\n", m->cp_hqd_pq_doorbell_control);
+
+	*mqd = m;
+	if (gart_addr)
+		*gart_addr = addr;
+
+	qp->is_active = 0;
+}
+
+static void restore_mqd_sdma_v12_1(struct mqd_manager *mm, void **mqd,
+				    struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
+				    struct queue_properties *qp,
+				    const void *mqd_src,
+				    const void *ctl_stack_src,
+				    const u32 ctl_stack_size)
+{
+	u64 addr;
+	struct v12_sdma_mqd *m;
+
+	m = (struct v12_sdma_mqd *)mqd_mem_obj->cpu_ptr;
+	addr = mqd_mem_obj->gpu_addr;
+
+	memset(m, 0, AMDGPU_MQD_SIZE_ALIGN(mm->mqd_size));
+	memcpy(m, mqd_src, sizeof(*m));
+
+	m->sdmax_rlcx_doorbell_offset =
+		qp->doorbell_off << SDMA0_SDMA_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT;
+
+	*mqd = m;
+	if (gart_addr)
+		*gart_addr = addr;
+
+	qp->is_active = 0;
+}
+
 struct mqd_manager *mqd_manager_init_v12_1(enum KFD_MQD_TYPE type,
 		struct kfd_node *dev)
 {
@@ -668,6 +734,7 @@ struct mqd_manager *mqd_manager_init_v12_1(enum KFD_MQD_TYPE type,
 		mqd->mqd_size = sizeof(struct v12_1_compute_mqd);
 		mqd->get_wave_state = get_wave_state_v12_1;
 		mqd->mqd_stride = kfd_mqd_stride;
+		mqd->restore_mqd = restore_mqd_v12_1;
 #if defined(CONFIG_DEBUG_FS)
 		mqd->debugfs_show_mqd = debugfs_show_mqd;
 #endif
@@ -714,6 +781,7 @@ struct mqd_manager *mqd_manager_init_v12_1(enum KFD_MQD_TYPE type,
 		mqd->is_occupied = kfd_is_occupied_sdma;
 		mqd->mqd_size = sizeof(struct v12_sdma_mqd);
 		mqd->mqd_stride = kfd_mqd_stride;
+		mqd->restore_mqd = restore_mqd_sdma_v12_1;
 #if defined(CONFIG_DEBUG_FS)
 		mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;
 #endif