Commit 100d46bd authored by Linus Torvalds's avatar Linus Torvalds

Merge Intel Gen8/Gen9 graphics fixes from Jon Bloomfield.

This fixes two different classes of bugs in the Intel graphics hardware:

MMIO register read hang:
 "On Intels Gen8 and Gen9 Graphics hardware, a read of specific graphics
  MMIO registers when the product is in certain low power states causes
  a system hang.

  There are two potential triggers for DoS:
    a) H/W corruption of the RC6 save/restore vector
    b) Hard hang within the MIPI hardware

  This prevents the DoS in two areas of the hardware:
    1) Detect corruption of RC6 address on exit from low-power state,
       and if we find it corrupted, disable RC6 and RPM
    2) Permanently lower the MIPI MMIO timeout"

Blitter command streamer unrestricted memory accesses:
 "On Intels Gen9 Graphics hardware the Blitter Command Streamer (BCS)
  allows writing to Memory Mapped Input Output (MMIO) that should be
  blocked. With modifications of page tables, this can lead to privilege
  escalation. This exposure is limited to the Guest Physical Address
  space and does not allow for access outside of the graphics virtual
  machine.

  This series establishes a software parser into the Blitter command
  stream to scan for, and prevent, reads or writes to MMIO's that should
  not be accessible to non-privileged contexts.

  Much of the command parser infrastructure has existed for some time,
  and is used on Ivybridge/Haswell/Valleyview derived products to allow
  the use of features normally blocked by hardware. In this legacy
  context, the command parser is employed to allow normally unprivileged
  submissions to be run with elevated privileges in order to grant
  access to a limited set of extra capabilities. In this mode the parser
  is optional; In the event that the parser finds any construct that it
  cannot properly validate (e.g. nested command buffers), it simply
  aborts the scan and submits the buffer in non-privileged mode.

  For Gen9 Graphics, this series makes the parser mandatory for all
  Blitter submissions. The incoming user buffer is first copied to a
  kernel owned buffer, and parsed. If all checks are successful the
  kernel owned buffer is mapped READ-ONLY and submitted on behalf of the
  user. If any checks fail, or the parser is unable to complete the scan
  (nested buffers), it is forcibly rejected. The successfully scanned
  buffer is executed with NORMAL user privileges (key difference from
  legacy usage).

  Modern usermode does not use the Blitter on later hardware, having
  switched over to using the 3D engine instead for performance reasons.
  There are however some legacy usermode apps that rely on Blitter,
  notably the SNA X-Server. There are no known usermode applications
  that require nested command buffers on the Blitter, so the forcible
  rejection of such buffers in this patch series is considered an
  acceptable limitation"

* Intel graphics fixes in emailed bundle from Jon Bloomfield <jon.bloomfield@intel.com>:
  drm/i915/cmdparser: Fix jump whitelist clearing
  drm/i915/gen8+: Add RC6 CTX corruption WA
  drm/i915: Lower RM timeout to avoid DSI hard hangs
  drm/i915/cmdparser: Ignore Length operands during command matching
  drm/i915/cmdparser: Add support for backward jumps
  drm/i915/cmdparser: Use explicit goto for error paths
  drm/i915: Add gen9 BCS cmdparsing
  drm/i915: Allow parsing of unsized batches
  drm/i915: Support ro ppgtt mapped cmdparser shadow buffers
  drm/i915: Add support for mandatory cmdparsing
  drm/i915: Remove Master tables from cmdparser
  drm/i915: Disable Secure Batches for gen6+
  drm/i915: Rename gen7 cmdparser tables
parents de620fb9 ea0b163b
...@@ -319,6 +319,8 @@ static void i915_gem_context_free(struct i915_gem_context *ctx) ...@@ -319,6 +319,8 @@ static void i915_gem_context_free(struct i915_gem_context *ctx)
free_engines(rcu_access_pointer(ctx->engines)); free_engines(rcu_access_pointer(ctx->engines));
mutex_destroy(&ctx->engines_mutex); mutex_destroy(&ctx->engines_mutex);
kfree(ctx->jump_whitelist);
if (ctx->timeline) if (ctx->timeline)
intel_timeline_put(ctx->timeline); intel_timeline_put(ctx->timeline);
...@@ -441,6 +443,9 @@ __create_context(struct drm_i915_private *i915) ...@@ -441,6 +443,9 @@ __create_context(struct drm_i915_private *i915)
for (i = 0; i < ARRAY_SIZE(ctx->hang_timestamp); i++) for (i = 0; i < ARRAY_SIZE(ctx->hang_timestamp); i++)
ctx->hang_timestamp[i] = jiffies - CONTEXT_FAST_HANG_JIFFIES; ctx->hang_timestamp[i] = jiffies - CONTEXT_FAST_HANG_JIFFIES;
ctx->jump_whitelist = NULL;
ctx->jump_whitelist_cmds = 0;
return ctx; return ctx;
err_free: err_free:
......
...@@ -192,6 +192,13 @@ struct i915_gem_context { ...@@ -192,6 +192,13 @@ struct i915_gem_context {
* per vm, which may be one per context or shared with the global GTT) * per vm, which may be one per context or shared with the global GTT)
*/ */
struct radix_tree_root handles_vma; struct radix_tree_root handles_vma;
/** jump_whitelist: Bit array for tracking cmds during cmdparsing
* Guarded by struct_mutex
*/
unsigned long *jump_whitelist;
/** jump_whitelist_cmds: No of cmd slots available */
u32 jump_whitelist_cmds;
}; };
#endif /* __I915_GEM_CONTEXT_TYPES_H__ */ #endif /* __I915_GEM_CONTEXT_TYPES_H__ */
...@@ -296,7 +296,9 @@ static inline u64 gen8_noncanonical_addr(u64 address) ...@@ -296,7 +296,9 @@ static inline u64 gen8_noncanonical_addr(u64 address)
static inline bool eb_use_cmdparser(const struct i915_execbuffer *eb) static inline bool eb_use_cmdparser(const struct i915_execbuffer *eb)
{ {
return intel_engine_needs_cmd_parser(eb->engine) && eb->batch_len; return intel_engine_requires_cmd_parser(eb->engine) ||
(intel_engine_using_cmd_parser(eb->engine) &&
eb->args->batch_len);
} }
static int eb_create(struct i915_execbuffer *eb) static int eb_create(struct i915_execbuffer *eb)
...@@ -1955,40 +1957,94 @@ static int i915_reset_gen7_sol_offsets(struct i915_request *rq) ...@@ -1955,40 +1957,94 @@ static int i915_reset_gen7_sol_offsets(struct i915_request *rq)
return 0; return 0;
} }
static struct i915_vma *eb_parse(struct i915_execbuffer *eb, bool is_master) static struct i915_vma *
shadow_batch_pin(struct i915_execbuffer *eb, struct drm_i915_gem_object *obj)
{
struct drm_i915_private *dev_priv = eb->i915;
struct i915_vma * const vma = *eb->vma;
struct i915_address_space *vm;
u64 flags;
/*
* PPGTT backed shadow buffers must be mapped RO, to prevent
* post-scan tampering
*/
if (CMDPARSER_USES_GGTT(dev_priv)) {
flags = PIN_GLOBAL;
vm = &dev_priv->ggtt.vm;
} else if (vma->vm->has_read_only) {
flags = PIN_USER;
vm = vma->vm;
i915_gem_object_set_readonly(obj);
} else {
DRM_DEBUG("Cannot prevent post-scan tampering without RO capable vm\n");
return ERR_PTR(-EINVAL);
}
return i915_gem_object_pin(obj, vm, NULL, 0, 0, flags);
}
static struct i915_vma *eb_parse(struct i915_execbuffer *eb)
{ {
struct intel_engine_pool_node *pool; struct intel_engine_pool_node *pool;
struct i915_vma *vma; struct i915_vma *vma;
u64 batch_start;
u64 shadow_batch_start;
int err; int err;
pool = intel_engine_pool_get(&eb->engine->pool, eb->batch_len); pool = intel_engine_pool_get(&eb->engine->pool, eb->batch_len);
if (IS_ERR(pool)) if (IS_ERR(pool))
return ERR_CAST(pool); return ERR_CAST(pool);
err = intel_engine_cmd_parser(eb->engine, vma = shadow_batch_pin(eb, pool->obj);
if (IS_ERR(vma))
goto err;
batch_start = gen8_canonical_addr(eb->batch->node.start) +
eb->batch_start_offset;
shadow_batch_start = gen8_canonical_addr(vma->node.start);
err = intel_engine_cmd_parser(eb->gem_context,
eb->engine,
eb->batch->obj, eb->batch->obj,
pool->obj, batch_start,
eb->batch_start_offset, eb->batch_start_offset,
eb->batch_len, eb->batch_len,
is_master); pool->obj,
shadow_batch_start);
if (err) { if (err) {
if (err == -EACCES) /* unhandled chained batch */ i915_vma_unpin(vma);
/*
* Unsafe GGTT-backed buffers can still be submitted safely
* as non-secure.
* For PPGTT backing however, we have no choice but to forcibly
* reject unsafe buffers
*/
if (CMDPARSER_USES_GGTT(eb->i915) && (err == -EACCES))
/* Execute original buffer non-secure */
vma = NULL; vma = NULL;
else else
vma = ERR_PTR(err); vma = ERR_PTR(err);
goto err; goto err;
} }
vma = i915_gem_object_ggtt_pin(pool->obj, NULL, 0, 0, 0);
if (IS_ERR(vma))
goto err;
eb->vma[eb->buffer_count] = i915_vma_get(vma); eb->vma[eb->buffer_count] = i915_vma_get(vma);
eb->flags[eb->buffer_count] = eb->flags[eb->buffer_count] =
__EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_REF; __EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_REF;
vma->exec_flags = &eb->flags[eb->buffer_count]; vma->exec_flags = &eb->flags[eb->buffer_count];
eb->buffer_count++; eb->buffer_count++;
eb->batch_start_offset = 0;
eb->batch = vma;
if (CMDPARSER_USES_GGTT(eb->i915))
eb->batch_flags |= I915_DISPATCH_SECURE;
/* eb->batch_len unchanged */
vma->private = pool; vma->private = pool;
return vma; return vma;
...@@ -2421,6 +2477,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, ...@@ -2421,6 +2477,7 @@ i915_gem_do_execbuffer(struct drm_device *dev,
struct drm_i915_gem_exec_object2 *exec, struct drm_i915_gem_exec_object2 *exec,
struct drm_syncobj **fences) struct drm_syncobj **fences)
{ {
struct drm_i915_private *i915 = to_i915(dev);
struct i915_execbuffer eb; struct i915_execbuffer eb;
struct dma_fence *in_fence = NULL; struct dma_fence *in_fence = NULL;
struct dma_fence *exec_fence = NULL; struct dma_fence *exec_fence = NULL;
...@@ -2432,7 +2489,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, ...@@ -2432,7 +2489,7 @@ i915_gem_do_execbuffer(struct drm_device *dev,
BUILD_BUG_ON(__EXEC_OBJECT_INTERNAL_FLAGS & BUILD_BUG_ON(__EXEC_OBJECT_INTERNAL_FLAGS &
~__EXEC_OBJECT_UNKNOWN_FLAGS); ~__EXEC_OBJECT_UNKNOWN_FLAGS);
eb.i915 = to_i915(dev); eb.i915 = i915;
eb.file = file; eb.file = file;
eb.args = args; eb.args = args;
if (DBG_FORCE_RELOC || !(args->flags & I915_EXEC_NO_RELOC)) if (DBG_FORCE_RELOC || !(args->flags & I915_EXEC_NO_RELOC))
...@@ -2452,8 +2509,15 @@ i915_gem_do_execbuffer(struct drm_device *dev, ...@@ -2452,8 +2509,15 @@ i915_gem_do_execbuffer(struct drm_device *dev,
eb.batch_flags = 0; eb.batch_flags = 0;
if (args->flags & I915_EXEC_SECURE) { if (args->flags & I915_EXEC_SECURE) {
if (INTEL_GEN(i915) >= 11)
return -ENODEV;
/* Return -EPERM to trigger fallback code on old binaries. */
if (!HAS_SECURE_BATCHES(i915))
return -EPERM;
if (!drm_is_current_master(file) || !capable(CAP_SYS_ADMIN)) if (!drm_is_current_master(file) || !capable(CAP_SYS_ADMIN))
return -EPERM; return -EPERM;
eb.batch_flags |= I915_DISPATCH_SECURE; eb.batch_flags |= I915_DISPATCH_SECURE;
} }
...@@ -2530,34 +2594,19 @@ i915_gem_do_execbuffer(struct drm_device *dev, ...@@ -2530,34 +2594,19 @@ i915_gem_do_execbuffer(struct drm_device *dev,
goto err_vma; goto err_vma;
} }
if (eb.batch_len == 0)
eb.batch_len = eb.batch->size - eb.batch_start_offset;
if (eb_use_cmdparser(&eb)) { if (eb_use_cmdparser(&eb)) {
struct i915_vma *vma; struct i915_vma *vma;
vma = eb_parse(&eb, drm_is_current_master(file)); vma = eb_parse(&eb);
if (IS_ERR(vma)) { if (IS_ERR(vma)) {
err = PTR_ERR(vma); err = PTR_ERR(vma);
goto err_vma; goto err_vma;
} }
if (vma) {
/*
* Batch parsed and accepted:
*
* Set the DISPATCH_SECURE bit to remove the NON_SECURE
* bit from MI_BATCH_BUFFER_START commands issued in
* the dispatch_execbuffer implementations. We
* specifically don't want that set on batches the
* command parser has accepted.
*/
eb.batch_flags |= I915_DISPATCH_SECURE;
eb.batch_start_offset = 0;
eb.batch = vma;
}
} }
if (eb.batch_len == 0)
eb.batch_len = eb.batch->size - eb.batch_start_offset;
/* /*
* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure * snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
* batch" bit. Hence we need to pin secure batches into the global gtt. * batch" bit. Hence we need to pin secure batches into the global gtt.
......
...@@ -475,12 +475,13 @@ struct intel_engine_cs { ...@@ -475,12 +475,13 @@ struct intel_engine_cs {
struct intel_engine_hangcheck hangcheck; struct intel_engine_hangcheck hangcheck;
#define I915_ENGINE_NEEDS_CMD_PARSER BIT(0) #define I915_ENGINE_USING_CMD_PARSER BIT(0)
#define I915_ENGINE_SUPPORTS_STATS BIT(1) #define I915_ENGINE_SUPPORTS_STATS BIT(1)
#define I915_ENGINE_HAS_PREEMPTION BIT(2) #define I915_ENGINE_HAS_PREEMPTION BIT(2)
#define I915_ENGINE_HAS_SEMAPHORES BIT(3) #define I915_ENGINE_HAS_SEMAPHORES BIT(3)
#define I915_ENGINE_NEEDS_BREADCRUMB_TASKLET BIT(4) #define I915_ENGINE_NEEDS_BREADCRUMB_TASKLET BIT(4)
#define I915_ENGINE_IS_VIRTUAL BIT(5) #define I915_ENGINE_IS_VIRTUAL BIT(5)
#define I915_ENGINE_REQUIRES_CMD_PARSER BIT(7)
unsigned int flags; unsigned int flags;
/* /*
...@@ -541,9 +542,15 @@ struct intel_engine_cs { ...@@ -541,9 +542,15 @@ struct intel_engine_cs {
}; };
static inline bool static inline bool
intel_engine_needs_cmd_parser(const struct intel_engine_cs *engine) intel_engine_using_cmd_parser(const struct intel_engine_cs *engine)
{ {
return engine->flags & I915_ENGINE_NEEDS_CMD_PARSER; return engine->flags & I915_ENGINE_USING_CMD_PARSER;
}
static inline bool
intel_engine_requires_cmd_parser(const struct intel_engine_cs *engine)
{
return engine->flags & I915_ENGINE_REQUIRES_CMD_PARSER;
} }
static inline bool static inline bool
......
...@@ -38,6 +38,9 @@ static int __gt_unpark(struct intel_wakeref *wf) ...@@ -38,6 +38,9 @@ static int __gt_unpark(struct intel_wakeref *wf)
gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ); gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
GEM_BUG_ON(!gt->awake); GEM_BUG_ON(!gt->awake);
if (NEEDS_RC6_CTX_CORRUPTION_WA(i915))
intel_uncore_forcewake_get(&i915->uncore, FORCEWAKE_ALL);
intel_enable_gt_powersave(i915); intel_enable_gt_powersave(i915);
i915_update_gfx_val(i915); i915_update_gfx_val(i915);
...@@ -67,6 +70,11 @@ static int __gt_park(struct intel_wakeref *wf) ...@@ -67,6 +70,11 @@ static int __gt_park(struct intel_wakeref *wf)
if (INTEL_GEN(i915) >= 6) if (INTEL_GEN(i915) >= 6)
gen6_rps_idle(i915); gen6_rps_idle(i915);
if (NEEDS_RC6_CTX_CORRUPTION_WA(i915)) {
i915_rc6_ctx_wa_check(i915);
intel_uncore_forcewake_put(&i915->uncore, FORCEWAKE_ALL);
}
/* Everything switched off, flush any residual interrupt just in case */ /* Everything switched off, flush any residual interrupt just in case */
intel_synchronize_irq(i915); intel_synchronize_irq(i915);
......
This diff is collapsed.
...@@ -1850,6 +1850,8 @@ static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation) ...@@ -1850,6 +1850,8 @@ static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation)
i915_gem_suspend_late(dev_priv); i915_gem_suspend_late(dev_priv);
i915_rc6_ctx_wa_suspend(dev_priv);
intel_uncore_suspend(&dev_priv->uncore); intel_uncore_suspend(&dev_priv->uncore);
intel_power_domains_suspend(dev_priv, intel_power_domains_suspend(dev_priv,
...@@ -2053,6 +2055,8 @@ static int i915_drm_resume_early(struct drm_device *dev) ...@@ -2053,6 +2055,8 @@ static int i915_drm_resume_early(struct drm_device *dev)
intel_power_domains_resume(dev_priv); intel_power_domains_resume(dev_priv);
i915_rc6_ctx_wa_resume(dev_priv);
intel_gt_sanitize(&dev_priv->gt, true); intel_gt_sanitize(&dev_priv->gt, true);
enable_rpm_wakeref_asserts(&dev_priv->runtime_pm); enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
......
...@@ -593,6 +593,8 @@ struct intel_rps { ...@@ -593,6 +593,8 @@ struct intel_rps {
struct intel_rc6 { struct intel_rc6 {
bool enabled; bool enabled;
bool ctx_corrupted;
intel_wakeref_t ctx_corrupted_wakeref;
u64 prev_hw_residency[4]; u64 prev_hw_residency[4];
u64 cur_residency[4]; u64 cur_residency[4];
}; };
...@@ -2075,9 +2077,16 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, ...@@ -2075,9 +2077,16 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915,
#define VEBOX_MASK(dev_priv) \ #define VEBOX_MASK(dev_priv) \
ENGINE_INSTANCES_MASK(dev_priv, VECS0, I915_MAX_VECS) ENGINE_INSTANCES_MASK(dev_priv, VECS0, I915_MAX_VECS)
/*
* The Gen7 cmdparser copies the scanned buffer to the ggtt for execution
* All later gens can run the final buffer from the ppgtt
*/
#define CMDPARSER_USES_GGTT(dev_priv) IS_GEN(dev_priv, 7)
#define HAS_LLC(dev_priv) (INTEL_INFO(dev_priv)->has_llc) #define HAS_LLC(dev_priv) (INTEL_INFO(dev_priv)->has_llc)
#define HAS_SNOOP(dev_priv) (INTEL_INFO(dev_priv)->has_snoop) #define HAS_SNOOP(dev_priv) (INTEL_INFO(dev_priv)->has_snoop)
#define HAS_EDRAM(dev_priv) ((dev_priv)->edram_size_mb) #define HAS_EDRAM(dev_priv) ((dev_priv)->edram_size_mb)
#define HAS_SECURE_BATCHES(dev_priv) (INTEL_GEN(dev_priv) < 6)
#define HAS_WT(dev_priv) ((IS_HASWELL(dev_priv) || \ #define HAS_WT(dev_priv) ((IS_HASWELL(dev_priv) || \
IS_BROADWELL(dev_priv)) && HAS_EDRAM(dev_priv)) IS_BROADWELL(dev_priv)) && HAS_EDRAM(dev_priv))
...@@ -2110,10 +2119,12 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, ...@@ -2110,10 +2119,12 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915,
/* Early gen2 have a totally busted CS tlb and require pinned batches. */ /* Early gen2 have a totally busted CS tlb and require pinned batches. */
#define HAS_BROKEN_CS_TLB(dev_priv) (IS_I830(dev_priv) || IS_I845G(dev_priv)) #define HAS_BROKEN_CS_TLB(dev_priv) (IS_I830(dev_priv) || IS_I845G(dev_priv))
#define NEEDS_RC6_CTX_CORRUPTION_WA(dev_priv) \
(IS_BROADWELL(dev_priv) || IS_GEN(dev_priv, 9))
/* WaRsDisableCoarsePowerGating:skl,cnl */ /* WaRsDisableCoarsePowerGating:skl,cnl */
#define NEEDS_WaRsDisableCoarsePowerGating(dev_priv) \ #define NEEDS_WaRsDisableCoarsePowerGating(dev_priv) \
(IS_CANNONLAKE(dev_priv) || \ (IS_CANNONLAKE(dev_priv) || IS_GEN(dev_priv, 9))
IS_SKL_GT3(dev_priv) || IS_SKL_GT4(dev_priv))
#define HAS_GMBUS_IRQ(dev_priv) (INTEL_GEN(dev_priv) >= 4) #define HAS_GMBUS_IRQ(dev_priv) (INTEL_GEN(dev_priv) >= 4)
#define HAS_GMBUS_BURST_READ(dev_priv) (INTEL_GEN(dev_priv) >= 10 || \ #define HAS_GMBUS_BURST_READ(dev_priv) (INTEL_GEN(dev_priv) >= 10 || \
...@@ -2284,6 +2295,14 @@ int i915_gem_object_unbind(struct drm_i915_gem_object *obj, ...@@ -2284,6 +2295,14 @@ int i915_gem_object_unbind(struct drm_i915_gem_object *obj,
unsigned long flags); unsigned long flags);
#define I915_GEM_OBJECT_UNBIND_ACTIVE BIT(0) #define I915_GEM_OBJECT_UNBIND_ACTIVE BIT(0)
struct i915_vma * __must_check
i915_gem_object_pin(struct drm_i915_gem_object *obj,
struct i915_address_space *vm,
const struct i915_ggtt_view *view,
u64 size,
u64 alignment,
u64 flags);
void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv); void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv);
static inline int __must_check static inline int __must_check
...@@ -2393,12 +2412,14 @@ const char *i915_cache_level_str(struct drm_i915_private *i915, int type); ...@@ -2393,12 +2412,14 @@ const char *i915_cache_level_str(struct drm_i915_private *i915, int type);
int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv); int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv);
void intel_engine_init_cmd_parser(struct intel_engine_cs *engine); void intel_engine_init_cmd_parser(struct intel_engine_cs *engine);
void intel_engine_cleanup_cmd_parser(struct intel_engine_cs *engine); void intel_engine_cleanup_cmd_parser(struct intel_engine_cs *engine);
int intel_engine_cmd_parser(struct intel_engine_cs *engine, int intel_engine_cmd_parser(struct i915_gem_context *cxt,
struct intel_engine_cs *engine,
struct drm_i915_gem_object *batch_obj, struct drm_i915_gem_object *batch_obj,
struct drm_i915_gem_object *shadow_batch_obj, u64 user_batch_start,
u32 batch_start_offset, u32 batch_start_offset,
u32 batch_len, u32 batch_len,
bool is_master); struct drm_i915_gem_object *shadow_batch_obj,
u64 shadow_batch_start);
/* intel_device_info.c */ /* intel_device_info.c */
static inline struct intel_device_info * static inline struct intel_device_info *
......
...@@ -964,6 +964,20 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj, ...@@ -964,6 +964,20 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
{ {
struct drm_i915_private *dev_priv = to_i915(obj->base.dev); struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
struct i915_address_space *vm = &dev_priv->ggtt.vm; struct i915_address_space *vm = &dev_priv->ggtt.vm;
return i915_gem_object_pin(obj, vm, view, size, alignment,
flags | PIN_GLOBAL);
}
struct i915_vma *
i915_gem_object_pin(struct drm_i915_gem_object *obj,
struct i915_address_space *vm,
const struct i915_ggtt_view *view,
u64 size,
u64 alignment,
u64 flags)
{
struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
struct i915_vma *vma; struct i915_vma *vma;
int ret; int ret;
...@@ -1038,7 +1052,7 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj, ...@@ -1038,7 +1052,7 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
return ERR_PTR(ret); return ERR_PTR(ret);
} }
ret = i915_vma_pin(vma, size, alignment, flags | PIN_GLOBAL); ret = i915_vma_pin(vma, size, alignment, flags);
if (ret) if (ret)
return ERR_PTR(ret); return ERR_PTR(ret);
......
...@@ -62,7 +62,7 @@ int i915_getparam_ioctl(struct drm_device *dev, void *data, ...@@ -62,7 +62,7 @@ int i915_getparam_ioctl(struct drm_device *dev, void *data,
value = !!(i915->caps.scheduler & I915_SCHEDULER_CAP_SEMAPHORES); value = !!(i915->caps.scheduler & I915_SCHEDULER_CAP_SEMAPHORES);
break; break;
case I915_PARAM_HAS_SECURE_BATCHES: case I915_PARAM_HAS_SECURE_BATCHES:
value = capable(CAP_SYS_ADMIN); value = HAS_SECURE_BATCHES(i915) && capable(CAP_SYS_ADMIN);
break; break;
case I915_PARAM_CMD_PARSER_VERSION: case I915_PARAM_CMD_PARSER_VERSION:
value = i915_cmd_parser_get_version(i915); value = i915_cmd_parser_get_version(i915);
......
...@@ -471,6 +471,8 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) ...@@ -471,6 +471,8 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
#define ECOCHK_PPGTT_WT_HSW (0x2 << 3) #define ECOCHK_PPGTT_WT_HSW (0x2 << 3)
#define ECOCHK_PPGTT_WB_HSW (0x3 << 3) #define ECOCHK_PPGTT_WB_HSW (0x3 << 3)
#define GEN8_RC6_CTX_INFO _MMIO(0x8504)
#define GAC_ECO_BITS _MMIO(0x14090) #define GAC_ECO_BITS _MMIO(0x14090)
#define ECOBITS_SNB_BIT (1 << 13) #define ECOBITS_SNB_BIT (1 << 13)
#define ECOBITS_PPGTT_CACHE64B (3 << 8) #define ECOBITS_PPGTT_CACHE64B (3 << 8)
...@@ -555,6 +557,10 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) ...@@ -555,6 +557,10 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
*/ */
#define BCS_SWCTRL _MMIO(0x22200) #define BCS_SWCTRL _MMIO(0x22200)
/* There are 16 GPR registers */
#define BCS_GPR(n) _MMIO(0x22600 + (n) * 8)
#define BCS_GPR_UDW(n) _MMIO(0x22600 + (n) * 8 + 4)
#define GPGPU_THREADS_DISPATCHED _MMIO(0x2290) #define GPGPU_THREADS_DISPATCHED _MMIO(0x2290)
#define GPGPU_THREADS_DISPATCHED_UDW _MMIO(0x2290 + 4) #define GPGPU_THREADS_DISPATCHED_UDW _MMIO(0x2290 + 4)
#define HS_INVOCATION_COUNT _MMIO(0x2300) #define HS_INVOCATION_COUNT _MMIO(0x2300)
...@@ -7211,6 +7217,10 @@ enum { ...@@ -7211,6 +7217,10 @@ enum {
#define TGL_DMC_DEBUG_DC5_COUNT _MMIO(0x101084) #define TGL_DMC_DEBUG_DC5_COUNT _MMIO(0x101084)
#define TGL_DMC_DEBUG_DC6_COUNT _MMIO(0x101088) #define TGL_DMC_DEBUG_DC6_COUNT _MMIO(0x101088)
/* Display Internal Timeout Register */
#define RM_TIMEOUT _MMIO(0x42060)
#define MMIO_TIMEOUT_US(us) ((us) << 0)
/* interrupts */ /* interrupts */
#define DE_MASTER_IRQ_CONTROL (1 << 31) #define DE_MASTER_IRQ_CONTROL (1 << 31)
#define DE_SPRITEB_FLIP_DONE (1 << 29) #define DE_SPRITEB_FLIP_DONE (1 << 29)
......
...@@ -126,6 +126,14 @@ static void bxt_init_clock_gating(struct drm_i915_private *dev_priv) ...@@ -126,6 +126,14 @@ static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
*/ */
I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) | I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
PWM1_GATING_DIS | PWM2_GATING_DIS); PWM1_GATING_DIS | PWM2_GATING_DIS);
/*
* Lower the display internal timeout.
* This is needed to avoid any hard hangs when DSI port PLL
* is off and a MMIO access is attempted by any privilege
* application, using batch buffers or any other means.
*/
I915_WRITE(RM_TIMEOUT, MMIO_TIMEOUT_US(950));
} }
static void glk_init_clock_gating(struct drm_i915_private *dev_priv) static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
...@@ -8544,6 +8552,100 @@ static void intel_init_emon(struct drm_i915_private *dev_priv) ...@@ -8544,6 +8552,100 @@ static void intel_init_emon(struct drm_i915_private *dev_priv)
dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK); dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
} }
static bool i915_rc6_ctx_corrupted(struct drm_i915_private *dev_priv)
{
return !I915_READ(GEN8_RC6_CTX_INFO);
}
static void i915_rc6_ctx_wa_init(struct drm_i915_private *i915)
{
if (!NEEDS_RC6_CTX_CORRUPTION_WA(i915))
return;
if (i915_rc6_ctx_corrupted(i915)) {
DRM_INFO("RC6 context corrupted, disabling runtime power management\n");
i915->gt_pm.rc6.ctx_corrupted = true;
i915->gt_pm.rc6.ctx_corrupted_wakeref =
intel_runtime_pm_get(&i915->runtime_pm);
}
}
static void i915_rc6_ctx_wa_cleanup(struct drm_i915_private *i915)
{
if (i915->gt_pm.rc6.ctx_corrupted) {
intel_runtime_pm_put(&i915->runtime_pm,
i915->gt_pm.rc6.ctx_corrupted_wakeref);
i915->gt_pm.rc6.ctx_corrupted = false;
}
}
/**
* i915_rc6_ctx_wa_suspend - system suspend sequence for the RC6 CTX WA
* @i915: i915 device
*
* Perform any steps needed to clean up the RC6 CTX WA before system suspend.
*/
void i915_rc6_ctx_wa_suspend(struct drm_i915_private *i915)
{
if (i915->gt_pm.rc6.ctx_corrupted)
intel_runtime_pm_put(&i915->runtime_pm,
i915->gt_pm.rc6.ctx_corrupted_wakeref);
}
/**
* i915_rc6_ctx_wa_resume - system resume sequence for the RC6 CTX WA
* @i915: i915 device
*
* Perform any steps needed to re-init the RC6 CTX WA after system resume.
*/
void i915_rc6_ctx_wa_resume(struct drm_i915_private *i915)
{
if (!i915->gt_pm.rc6.ctx_corrupted)
return;
if (i915_rc6_ctx_corrupted(i915)) {
i915->gt_pm.rc6.ctx_corrupted_wakeref =
intel_runtime_pm_get(&i915->runtime_pm);
return;
}
DRM_INFO("RC6 context restored, re-enabling runtime power management\n");
i915->gt_pm.rc6.ctx_corrupted = false;
}
static void intel_disable_rc6(struct drm_i915_private *dev_priv);
/**
* i915_rc6_ctx_wa_check - check for a new RC6 CTX corruption
* @i915: i915 device
*
* Check if an RC6 CTX corruption has happened since the last check and if so
* disable RC6 and runtime power management.
*
* Return false if no context corruption has happened since the last call of
* this function, true otherwise.
*/
bool i915_rc6_ctx_wa_check(struct drm_i915_private *i915)
{
if (!NEEDS_RC6_CTX_CORRUPTION_WA(i915))
return false;
if (i915->gt_pm.rc6.ctx_corrupted)
return false;
if (!i915_rc6_ctx_corrupted(i915))
return false;
DRM_NOTE("RC6 context corruption, disabling runtime power management\n");
intel_disable_rc6(i915);
i915->gt_pm.rc6.ctx_corrupted = true;
i915->gt_pm.rc6.ctx_corrupted_wakeref =
intel_runtime_pm_get_noresume(&i915->runtime_pm);
return true;
}
void intel_init_gt_powersave(struct drm_i915_private *dev_priv) void intel_init_gt_powersave(struct drm_i915_private *dev_priv)
{ {
struct intel_rps *rps = &dev_priv->gt_pm.rps; struct intel_rps *rps = &dev_priv->gt_pm.rps;
...@@ -8557,6 +8659,8 @@ void intel_init_gt_powersave(struct drm_i915_private *dev_priv) ...@@ -8557,6 +8659,8 @@ void intel_init_gt_powersave(struct drm_i915_private *dev_priv)
pm_runtime_get(&dev_priv->drm.pdev->dev); pm_runtime_get(&dev_priv->drm.pdev->dev);
} }
i915_rc6_ctx_wa_init(dev_priv);
/* Initialize RPS limits (for userspace) */ /* Initialize RPS limits (for userspace) */
if (IS_CHERRYVIEW(dev_priv)) if (IS_CHERRYVIEW(dev_priv))
cherryview_init_gt_powersave(dev_priv); cherryview_init_gt_powersave(dev_priv);
...@@ -8595,6 +8699,8 @@ void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv) ...@@ -8595,6 +8699,8 @@ void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
if (IS_VALLEYVIEW(dev_priv)) if (IS_VALLEYVIEW(dev_priv))
valleyview_cleanup_gt_powersave(dev_priv); valleyview_cleanup_gt_powersave(dev_priv);
i915_rc6_ctx_wa_cleanup(dev_priv);
if (!HAS_RC6(dev_priv)) if (!HAS_RC6(dev_priv))
pm_runtime_put(&dev_priv->drm.pdev->dev); pm_runtime_put(&dev_priv->drm.pdev->dev);
} }
...@@ -8623,7 +8729,7 @@ static inline void intel_disable_llc_pstate(struct drm_i915_private *i915) ...@@ -8623,7 +8729,7 @@ static inline void intel_disable_llc_pstate(struct drm_i915_private *i915)
i915->gt_pm.llc_pstate.enabled = false; i915->gt_pm.llc_pstate.enabled = false;
} }
static void intel_disable_rc6(struct drm_i915_private *dev_priv) static void __intel_disable_rc6(struct drm_i915_private *dev_priv)
{ {
lockdep_assert_held(&dev_priv->gt_pm.rps.lock); lockdep_assert_held(&dev_priv->gt_pm.rps.lock);
...@@ -8642,6 +8748,15 @@ static void intel_disable_rc6(struct drm_i915_private *dev_priv) ...@@ -8642,6 +8748,15 @@ static void intel_disable_rc6(struct drm_i915_private *dev_priv)
dev_priv->gt_pm.rc6.enabled = false; dev_priv->gt_pm.rc6.enabled = false;
} }
static void intel_disable_rc6(struct drm_i915_private *dev_priv)
{
struct intel_rps *rps = &dev_priv->gt_pm.rps;
mutex_lock(&rps->lock);
__intel_disable_rc6(dev_priv);
mutex_unlock(&rps->lock);
}
static void intel_disable_rps(struct drm_i915_private *dev_priv) static void intel_disable_rps(struct drm_i915_private *dev_priv)
{ {
lockdep_assert_held(&dev_priv->gt_pm.rps.lock); lockdep_assert_held(&dev_priv->gt_pm.rps.lock);
...@@ -8667,7 +8782,7 @@ void intel_disable_gt_powersave(struct drm_i915_private *dev_priv) ...@@ -8667,7 +8782,7 @@ void intel_disable_gt_powersave(struct drm_i915_private *dev_priv)
{ {
mutex_lock(&dev_priv->gt_pm.rps.lock); mutex_lock(&dev_priv->gt_pm.rps.lock);
intel_disable_rc6(dev_priv); __intel_disable_rc6(dev_priv);
intel_disable_rps(dev_priv); intel_disable_rps(dev_priv);
if (HAS_LLC(dev_priv)) if (HAS_LLC(dev_priv))
intel_disable_llc_pstate(dev_priv); intel_disable_llc_pstate(dev_priv);
...@@ -8694,6 +8809,9 @@ static void intel_enable_rc6(struct drm_i915_private *dev_priv) ...@@ -8694,6 +8809,9 @@ static void intel_enable_rc6(struct drm_i915_private *dev_priv)
if (dev_priv->gt_pm.rc6.enabled) if (dev_priv->gt_pm.rc6.enabled)
return; return;
if (dev_priv->gt_pm.rc6.ctx_corrupted)
return;
if (IS_CHERRYVIEW(dev_priv)) if (IS_CHERRYVIEW(dev_priv))
cherryview_enable_rc6(dev_priv); cherryview_enable_rc6(dev_priv);
else if (IS_VALLEYVIEW(dev_priv)) else if (IS_VALLEYVIEW(dev_priv))
......
...@@ -36,6 +36,9 @@ void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv); ...@@ -36,6 +36,9 @@ void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv);
void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv); void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv);
void intel_enable_gt_powersave(struct drm_i915_private *dev_priv); void intel_enable_gt_powersave(struct drm_i915_private *dev_priv);
void intel_disable_gt_powersave(struct drm_i915_private *dev_priv); void intel_disable_gt_powersave(struct drm_i915_private *dev_priv);
bool i915_rc6_ctx_wa_check(struct drm_i915_private *i915);
void i915_rc6_ctx_wa_suspend(struct drm_i915_private *i915);
void i915_rc6_ctx_wa_resume(struct drm_i915_private *i915);
void gen6_rps_busy(struct drm_i915_private *dev_priv); void gen6_rps_busy(struct drm_i915_private *dev_priv);
void gen6_rps_idle(struct drm_i915_private *dev_priv); void gen6_rps_idle(struct drm_i915_private *dev_priv);
void gen6_rps_boost(struct i915_request *rq); void gen6_rps_boost(struct i915_request *rq);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment