drm/i915: Extract GT render sleep (rc6) management

Continuing the theme of breaking intel_pm.c up in a reasonable chunk of
powermanagement utilities, pull out the rc6 setup into its GT handler.

Based on a patch by Chris Wilson.
Signed-off-by: Andi Shyti <andi.shyti@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Link: https://patchwork.freedesktop.org/patch/msgid/20190919143840.20384-1-andi.shyti@intel.com
Link: https://patchwork.freedesktop.org/patch/msgid/20190927110849.28734-1-chris@chris-wilson.co.uk

drivers/gpu/drm/i915/Makefile

@@ -85,6 +85,7 @@ gt-y += \
 	gt/intel_gt_pm_irq.o \
 	gt/intel_hangcheck.o \
 	gt/intel_lrc.o \
+	gt/intel_rc6.o \
 	gt/intel_renderstate.o \
 	gt/intel_reset.o \
 	gt/intel_ringbuffer.o \

drivers/gpu/drm/i915/gem/i915_gem_pm.c

@@ -137,7 +137,6 @@ static bool switch_to_kernel_context_sync(struct intel_gt *gt)
 
 bool i915_gem_load_power_context(struct drm_i915_private *i915)
 {
-	intel_gt_pm_enable(&i915->gt);
 	return switch_to_kernel_context_sync(&i915->gt);
 }
 
@@ -188,6 +187,7 @@ void i915_gem_suspend(struct drm_i915_private *i915)
 	i915_gem_drain_freed_objects(i915);
 
 	intel_uc_suspend(&i915->gt.uc);
+	intel_gt_suspend(&i915->gt);
 }
 
 static struct drm_i915_gem_object *first_mm_object(struct list_head *list)

drivers/gpu/drm/i915/gt/intel_engine_pm.c

@@ -11,6 +11,7 @@
 #include "intel_engine_pool.h"
 #include "intel_gt.h"
 #include "intel_gt_pm.h"
+#include "intel_rc6.h"
 
 static int __engine_unpark(struct intel_wakeref *wf)
 {

drivers/gpu/drm/i915/gt/intel_gt.c

@@ -7,6 +7,7 @@
 #include "intel_gt.h"
 #include "intel_gt_pm.h"
 #include "intel_mocs.h"
+#include "intel_rc6.h"
 #include "intel_uncore.h"
 #include "intel_pm.h"
 
@@ -369,6 +370,8 @@ int intel_gt_init(struct intel_gt *gt)
 	if (err)
 		return err;
 
+	intel_gt_pm_init(gt);
+
 	return 0;
 }
 
@@ -387,8 +390,8 @@ void intel_gt_driver_release(struct intel_gt *gt)
 {
 	/* Paranoia: make sure we have disabled everything before we exit. */
 	intel_gt_pm_disable(gt);
+	intel_gt_pm_fini(gt);
 
-	intel_cleanup_gt_powersave(gt->i915);
 	intel_gt_fini_scratch(gt);
 }
 

drivers/gpu/drm/i915/gt/intel_gt_pm.c

@@ -11,6 +11,7 @@
 #include "intel_gt.h"
 #include "intel_gt_pm.h"
 #include "intel_pm.h"
+#include "intel_rc6.h"
 #include "intel_wakeref.h"
 
 static void pm_notify(struct intel_gt *gt, int state)
@@ -90,6 +91,16 @@ void intel_gt_pm_init_early(struct intel_gt *gt)
 	BLOCKING_INIT_NOTIFIER_HEAD(&gt->pm_notifications);
 }
 
+void intel_gt_pm_init(struct intel_gt *gt)
+{
+	/*
+	 * Enabling power-management should be "self-healing". If we cannot
+	 * enable a feature, simply leave it disabled with a notice to the
+	 * user.
+	 */
+	intel_rc6_init(&gt->rc6);
+}
+
 static bool reset_engines(struct intel_gt *gt)
 {
 	if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
@@ -124,40 +135,14 @@ void intel_gt_sanitize(struct intel_gt *gt, bool force)
 		__intel_engine_reset(engine, false);
 }
 
-static bool is_mock_device(const struct intel_gt *gt)
-{
-	return I915_SELFTEST_ONLY(gt->awake == -1);
-}
-
-void intel_gt_pm_enable(struct intel_gt *gt)
+void intel_gt_pm_disable(struct intel_gt *gt)
 {
-	struct intel_engine_cs *engine;
-	enum intel_engine_id id;
-
-	/* Powersaving is controlled by the host when inside a VM */
-	if (intel_vgpu_active(gt->i915))
-		return;
-
-	if (is_mock_device(gt))
-		return;
-
-	intel_gt_pm_get(gt);
-
-	for_each_engine(engine, gt->i915, id) {
-		intel_engine_pm_get(engine);
-		engine->serial++; /* force kernel context reload */
-		intel_engine_pm_put(engine);
-	}
-
-	intel_gt_pm_put(gt);
+	intel_sanitize_gt_powersave(gt->i915);
 }
 
-void intel_gt_pm_disable(struct intel_gt *gt)
+void intel_gt_pm_fini(struct intel_gt *gt)
 {
-	if (is_mock_device(gt))
-		return;
-
-	intel_sanitize_gt_powersave(gt->i915);
+	intel_rc6_fini(&gt->rc6);
 }
 
 int intel_gt_resume(struct intel_gt *gt)
@@ -173,6 +158,9 @@ int intel_gt_resume(struct intel_gt *gt)
 	 * allowing us to fixup the user contexts on their first pin.
 	 */
 	intel_gt_pm_get(gt);
+	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
+	intel_rc6_sanitize(&gt->rc6);
+
 	for_each_engine(engine, gt->i915, id) {
 		struct intel_context *ce;
 
@@ -197,11 +185,51 @@ int intel_gt_resume(struct intel_gt *gt)
 			break;
 		}
 	}
+
+	intel_rc6_enable(&gt->rc6);
+	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
 	intel_gt_pm_put(gt);
 
 	return err;
 }
 
+static void wait_for_idle(struct intel_gt *gt)
+{
+	mutex_lock(&gt->i915->drm.struct_mutex); /* XXX */
+	do {
+		if (i915_gem_wait_for_idle(gt->i915,
+					   I915_WAIT_LOCKED,
+					   I915_GEM_IDLE_TIMEOUT) == -ETIME) {
+			/* XXX hide warning from gem_eio */
+			if (i915_modparams.reset) {
+				dev_err(gt->i915->drm.dev,
+					"Failed to idle engines, declaring wedged!\n");
+				GEM_TRACE_DUMP();
+			}
+
+			/*
+			 * Forcibly cancel outstanding work and leave
+			 * the gpu quiet.
+			 */
+			intel_gt_set_wedged(gt);
+		}
+	} while (i915_retire_requests(gt->i915));
+	mutex_unlock(&gt->i915->drm.struct_mutex);
+
+	intel_gt_pm_wait_for_idle(gt);
+}
+
+void intel_gt_suspend(struct intel_gt *gt)
+{
+	intel_wakeref_t wakeref;
+
+	/* We expect to be idle already; but also want to be independent */
+	wait_for_idle(gt);
+
+	with_intel_runtime_pm(&gt->i915->runtime_pm, wakeref)
+		intel_rc6_disable(&gt->rc6);
+}
+
 void intel_gt_runtime_suspend(struct intel_gt *gt)
 {
 	intel_uc_runtime_suspend(&gt->uc);
@@ -213,3 +241,7 @@ int intel_gt_runtime_resume(struct intel_gt *gt)
 
 	return intel_uc_runtime_resume(&gt->uc);
 }
+
+#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
+#include "selftest_gt_pm.c"
+#endif

drivers/gpu/drm/i915/gt/intel_gt_pm.h

@@ -43,12 +43,21 @@ static inline int intel_gt_pm_wait_for_idle(struct intel_gt *gt)
 }
 
 void intel_gt_pm_init_early(struct intel_gt *gt);
-void intel_gt_pm_enable(struct intel_gt *gt);
+void intel_gt_pm_init(struct intel_gt *gt);
 void intel_gt_pm_disable(struct intel_gt *gt);
+void intel_gt_pm_fini(struct intel_gt *gt);
 
 void intel_gt_sanitize(struct intel_gt *gt, bool force);
+
 int intel_gt_resume(struct intel_gt *gt);
+void intel_gt_suspend(struct intel_gt *gt);
+
 void intel_gt_runtime_suspend(struct intel_gt *gt);
 int intel_gt_runtime_resume(struct intel_gt *gt);
 
+static inline bool is_mock_gt(const struct intel_gt *gt)
+{
+	return I915_SELFTEST_ONLY(gt->awake == -1);
+}
+
 #endif /* INTEL_GT_PM_H */

drivers/gpu/drm/i915/gt/intel_gt_types.h

@@ -18,6 +18,7 @@
 #include "i915_vma.h"
 #include "intel_engine_types.h"
 #include "intel_reset_types.h"
+#include "intel_rc6_types.h"
 #include "intel_wakeref.h"
 
 struct drm_i915_private;
@@ -67,6 +68,8 @@ struct intel_gt {
 	 */
 	intel_wakeref_t awake;
 
+	struct intel_rc6 rc6;
+
 	struct blocking_notifier_head pm_notifications;
 
 	ktime_t last_init_time;

drivers/gpu/drm/i915/gt/intel_rc6.h

+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef INTEL_RC6_H
+#define INTEL_RC6_H
+
+#include "i915_reg.h"
+
+struct intel_engine_cs;
+struct intel_rc6;
+
+void intel_rc6_init(struct intel_rc6 *rc6);
+void intel_rc6_fini(struct intel_rc6 *rc6);
+
+void intel_rc6_sanitize(struct intel_rc6 *rc6);
+void intel_rc6_enable(struct intel_rc6 *rc6);
+void intel_rc6_disable(struct intel_rc6 *rc6);
+
+u64 intel_rc6_residency_ns(struct intel_rc6 *rc6, i915_reg_t reg);
+u64 intel_rc6_residency_us(struct intel_rc6 *rc6, i915_reg_t reg);
+
+#endif /* INTEL_RC6_H */

drivers/gpu/drm/i915/gt/intel_rc6_types.h

+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef INTEL_RC6_TYPES_H
+#define INTEL_RC6_TYPES_H
+
+#include <linux/spinlock.h>
+#include <linux/types.h>
+
+#include "intel_engine_types.h"
+
+struct drm_i915_gem_object;
+
+struct intel_rc6 {
+	u64 prev_hw_residency[4];
+	u64 cur_residency[4];
+
+	struct drm_i915_gem_object *pctx;
+
+	bool supported : 1;
+	bool enabled : 1;
+	bool wakeref : 1;
+};
+
+#endif /* INTEL_RC6_TYPES_H */

drivers/gpu/drm/i915/gt/selftest_gt_pm.c

+
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2019 Intel Corporation
+ */
+
+static int live_gt_resume(void *arg)
+{
+	struct intel_gt *gt = arg;
+	IGT_TIMEOUT(end_time);
+	int err;
+
+	/* Do several suspend/resume cycles to check we don't explode! */
+	do {
+		intel_gt_suspend(gt);
+
+		if (gt->rc6.enabled) {
+			pr_err("rc6 still enabled after suspend!\n");
+			intel_gt_set_wedged_on_init(gt);
+			err = -EINVAL;
+			break;
+		}
+
+		err = intel_gt_resume(gt);
+		if (err)
+			break;
+
+		if (gt->rc6.supported && !gt->rc6.enabled) {
+			pr_err("rc6 not enabled upon resume!\n");
+			intel_gt_set_wedged_on_init(gt);
+			err = -EINVAL;
+			break;
+		}
+	} while (!__igt_timeout(end_time, NULL));
+
+	return err;
+}
+
+int intel_gt_pm_live_selftests(struct drm_i915_private *i915)
+{
+	static const struct i915_subtest tests[] = {
+		SUBTEST(live_gt_resume),
+	};
+
+	if (intel_gt_is_wedged(&i915->gt))
+		return 0;
+
+	return intel_gt_live_subtests(tests, &i915->gt);
+}

drivers/gpu/drm/i915/i915_debugfs.c

@@ -42,6 +42,7 @@
 #include "gem/i915_gem_context.h"
 #include "gt/intel_gt_pm.h"
 #include "gt/intel_reset.h"
+#include "gt/intel_rc6.h"
 #include "gt/uc/intel_guc_submission.h"
 
 #include "i915_debugfs.h"
@@ -1168,11 +1169,13 @@ static void print_rc6_res(struct seq_file *m,
 			  const char *title,
 			  const i915_reg_t reg)
 {
-	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct drm_i915_private *i915 = node_to_i915(m->private);
+	intel_wakeref_t wakeref;
 
-	seq_printf(m, "%s %u (%llu us)\n",
-		   title, I915_READ(reg),
-		   intel_rc6_residency_us(dev_priv, reg));
+	with_intel_runtime_pm(&i915->runtime_pm, wakeref)
+		seq_printf(m, "%s %u (%llu us)\n", title,
+			   intel_uncore_read(&i915->uncore, reg),
+			   intel_rc6_residency_us(&i915->gt.rc6, reg));
 }
 
 static int vlv_drpc_info(struct seq_file *m)

drivers/gpu/drm/i915/i915_drv.h

@@ -599,19 +599,12 @@ struct intel_rps {
 	struct intel_rps_ei ei;
 };
 
-struct intel_rc6 {
-	bool enabled;
-	u64 prev_hw_residency[4];
-	u64 cur_residency[4];
-};
-
 struct intel_llc_pstate {
 	bool enabled;
 };
 
 struct intel_gen6_power_mgmt {
 	struct intel_rps rps;
-	struct intel_rc6 rc6;
 	struct intel_llc_pstate llc_pstate;
 };
 

drivers/gpu/drm/i915/i915_pmu.c

@@ -11,6 +11,7 @@
 #include "gt/intel_engine_pm.h"
 #include "gt/intel_engine_user.h"
 #include "gt/intel_gt_pm.h"
+#include "gt/intel_rc6.h"
 
 #include "i915_drv.h"
 #include "i915_pmu.h"
@@ -116,21 +117,21 @@ static bool pmu_needs_timer(struct i915_pmu *pmu, bool gpu_active)
 	return enable;
 }
 
-static u64 __get_rc6(const struct intel_gt *gt)
+static u64 __get_rc6(struct intel_gt *gt)
 {
 	struct drm_i915_private *i915 = gt->i915;
 	u64 val;
 
-	val = intel_rc6_residency_ns(i915,
+	val = intel_rc6_residency_ns(&gt->rc6,
 				     IS_VALLEYVIEW(i915) ?
 				     VLV_GT_RENDER_RC6 :
 				     GEN6_GT_GFX_RC6);
 
 	if (HAS_RC6p(i915))
-		val += intel_rc6_residency_ns(i915, GEN6_GT_GFX_RC6p);
+		val += intel_rc6_residency_ns(&gt->rc6, GEN6_GT_GFX_RC6p);
 
 	if (HAS_RC6pp(i915))
-		val += intel_rc6_residency_ns(i915, GEN6_GT_GFX_RC6pp);
+		val += intel_rc6_residency_ns(&gt->rc6, GEN6_GT_GFX_RC6pp);
 
 	return val;
 }

drivers/gpu/drm/i915/i915_sysfs.c

@@ -30,6 +30,8 @@
 #include <linux/stat.h>
 #include <linux/sysfs.h>
 
+#include "gt/intel_rc6.h"
+
 #include "i915_drv.h"
 #include "i915_sysfs.h"
 #include "intel_pm.h"
@@ -49,7 +51,7 @@ static u32 calc_residency(struct drm_i915_private *dev_priv,
 	u64 res = 0;
 
 	with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref)
-		res = intel_rc6_residency_us(dev_priv, reg);
+		res = intel_rc6_residency_us(&dev_priv->gt.rc6, reg);
 
 	return DIV_ROUND_CLOSEST_ULL(res, 1000);
 }

drivers/gpu/drm/i915/intel_pm.h

@@ -32,7 +32,6 @@ void intel_pm_setup(struct drm_i915_private *dev_priv);
 void intel_gpu_ips_init(struct drm_i915_private *dev_priv);
 void intel_gpu_ips_teardown(void);
 void intel_init_gt_powersave(struct drm_i915_private *dev_priv);
-void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv);
 void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv);
 void intel_enable_gt_powersave(struct drm_i915_private *dev_priv);
 void intel_disable_gt_powersave(struct drm_i915_private *dev_priv);
@@ -72,8 +71,6 @@ void intel_enable_ipc(struct drm_i915_private *dev_priv);
 
 int intel_gpu_freq(struct drm_i915_private *dev_priv, int val);
 int intel_freq_opcode(struct drm_i915_private *dev_priv, int val);
-u64 intel_rc6_residency_ns(struct drm_i915_private *dev_priv, i915_reg_t reg);
-u64 intel_rc6_residency_us(struct drm_i915_private *dev_priv, i915_reg_t reg);
 
 u32 intel_get_cagf(struct drm_i915_private *dev_priv, u32 rpstat1);
 

drivers/gpu/drm/i915/selftests/i915_live_selftests.h

@@ -16,6 +16,7 @@ selftest(gt_engines, intel_engine_live_selftests)
 selftest(gt_timelines, intel_timeline_live_selftests)
 selftest(gt_contexts, intel_context_live_selftests)
 selftest(gt_lrc, intel_lrc_live_selftests)
+selftest(gt_pm, intel_gt_pm_live_selftests)
 selftest(requests, i915_request_live_selftests)
 selftest(active, i915_active_live_selftests)
 selftest(objects, i915_gem_object_live_selftests)