intel_lrc.c 106 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
/*
 * Copyright © 2014 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Ben Widawsky <ben@bwidawsk.net>
 *    Michel Thierry <michel.thierry@intel.com>
 *    Thomas Daniel <thomas.daniel@intel.com>
 *    Oscar Mateo <oscar.mateo@intel.com>
 *
 */

31 32 33 34
/**
 * DOC: Logical Rings, Logical Ring Contexts and Execlists
 *
 * Motivation:
35 36 37 38
 * GEN8 brings an expansion of the HW contexts: "Logical Ring Contexts".
 * These expanded contexts enable a number of new abilities, especially
 * "Execlists" (also implemented in this file).
 *
39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89
 * One of the main differences with the legacy HW contexts is that logical
 * ring contexts incorporate many more things to the context's state, like
 * PDPs or ringbuffer control registers:
 *
 * The reason why PDPs are included in the context is straightforward: as
 * PPGTTs (per-process GTTs) are actually per-context, having the PDPs
 * contained there mean you don't need to do a ppgtt->switch_mm yourself,
 * instead, the GPU will do it for you on the context switch.
 *
 * But, what about the ringbuffer control registers (head, tail, etc..)?
 * shouldn't we just need a set of those per engine command streamer? This is
 * where the name "Logical Rings" starts to make sense: by virtualizing the
 * rings, the engine cs shifts to a new "ring buffer" with every context
 * switch. When you want to submit a workload to the GPU you: A) choose your
 * context, B) find its appropriate virtualized ring, C) write commands to it
 * and then, finally, D) tell the GPU to switch to that context.
 *
 * Instead of the legacy MI_SET_CONTEXT, the way you tell the GPU to switch
 * to a contexts is via a context execution list, ergo "Execlists".
 *
 * LRC implementation:
 * Regarding the creation of contexts, we have:
 *
 * - One global default context.
 * - One local default context for each opened fd.
 * - One local extra context for each context create ioctl call.
 *
 * Now that ringbuffers belong per-context (and not per-engine, like before)
 * and that contexts are uniquely tied to a given engine (and not reusable,
 * like before) we need:
 *
 * - One ringbuffer per-engine inside each context.
 * - One backing object per-engine inside each context.
 *
 * The global default context starts its life with these new objects fully
 * allocated and populated. The local default context for each opened fd is
 * more complex, because we don't know at creation time which engine is going
 * to use them. To handle this, we have implemented a deferred creation of LR
 * contexts:
 *
 * The local context starts its life as a hollow or blank holder, that only
 * gets populated for a given engine once we receive an execbuffer. If later
 * on we receive another execbuffer ioctl for the same context but a different
 * engine, we allocate/populate a new ringbuffer and context backing object and
 * so on.
 *
 * Finally, regarding local contexts created using the ioctl call: as they are
 * only allowed with the render ring, we can allocate & populate them right
 * away (no need to defer anything, at least for now).
 *
 * Execlists implementation:
90 91
 * Execlists are the new method by which, on gen8+ hardware, workloads are
 * submitted for execution (as opposed to the legacy, ringbuffer-based, method).
92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132
 * This method works as follows:
 *
 * When a request is committed, its commands (the BB start and any leading or
 * trailing commands, like the seqno breadcrumbs) are placed in the ringbuffer
 * for the appropriate context. The tail pointer in the hardware context is not
 * updated at this time, but instead, kept by the driver in the ringbuffer
 * structure. A structure representing this request is added to a request queue
 * for the appropriate engine: this structure contains a copy of the context's
 * tail after the request was written to the ring buffer and a pointer to the
 * context itself.
 *
 * If the engine's request queue was empty before the request was added, the
 * queue is processed immediately. Otherwise the queue will be processed during
 * a context switch interrupt. In any case, elements on the queue will get sent
 * (in pairs) to the GPU's ExecLists Submit Port (ELSP, for short) with a
 * globally unique 20-bits submission ID.
 *
 * When execution of a request completes, the GPU updates the context status
 * buffer with a context complete event and generates a context switch interrupt.
 * During the interrupt handling, the driver examines the events in the buffer:
 * for each context complete event, if the announced ID matches that on the head
 * of the request queue, then that request is retired and removed from the queue.
 *
 * After processing, if any requests were retired and the queue is not empty
 * then a new execution list can be submitted. The two requests at the front of
 * the queue are next to be submitted but since a context may not occur twice in
 * an execution list, if subsequent requests have the same ID as the first then
 * the two requests must be combined. This is done simply by discarding requests
 * at the head of the queue until either only one requests is left (in which case
 * we use a NULL second context) or the first two requests have unique IDs.
 *
 * By always executing the first two requests in the queue the driver ensures
 * that the GPU is kept as busy as possible. In the case where a single context
 * completes but a second context is still executing, the request for this second
 * context will be at the head of the queue when we remove the first one. This
 * request will then be resubmitted along with a new request for a different context,
 * which will cause the hardware to continue executing the second request and queue
 * the new request (the GPU detects the condition of a context getting preempted
 * with the same context and optimizes the context switch flow by not doing
 * preemption, but just sampling the new tail pointer).
 *
133
 */
134
#include <linux/interrupt.h>
135

136 137
#include "gem/i915_gem_context.h"

138
#include "i915_drv.h"
139
#include "i915_gem_render_state.h"
140
#include "i915_vgpu.h"
141
#include "intel_engine_pm.h"
142
#include "intel_lrc_reg.h"
143
#include "intel_mocs.h"
144
#include "intel_reset.h"
145
#include "intel_workarounds.h"
146

147 148 149 150 151 152 153 154 155 156 157 158 159
#define RING_EXECLIST_QFULL		(1 << 0x2)
#define RING_EXECLIST1_VALID		(1 << 0x3)
#define RING_EXECLIST0_VALID		(1 << 0x4)
#define RING_EXECLIST_ACTIVE_STATUS	(3 << 0xE)
#define RING_EXECLIST1_ACTIVE		(1 << 0x11)
#define RING_EXECLIST0_ACTIVE		(1 << 0x12)

#define GEN8_CTX_STATUS_IDLE_ACTIVE	(1 << 0)
#define GEN8_CTX_STATUS_PREEMPTED	(1 << 1)
#define GEN8_CTX_STATUS_ELEMENT_SWITCH	(1 << 2)
#define GEN8_CTX_STATUS_ACTIVE_IDLE	(1 << 3)
#define GEN8_CTX_STATUS_COMPLETE	(1 << 4)
#define GEN8_CTX_STATUS_LITE_RESTORE	(1 << 15)
160

161
#define GEN8_CTX_STATUS_COMPLETED_MASK \
162
	 (GEN8_CTX_STATUS_COMPLETE | GEN8_CTX_STATUS_PREEMPTED)
163

164 165
#define CTX_DESC_FORCE_RESTORE BIT_ULL(2)

166 167
/* Typical size of the average request (2 pipecontrols and a MI_BB) */
#define EXECLISTS_REQUEST_SIZE 64 /* bytes */
168
#define WA_TAIL_DWORDS 2
169
#define WA_TAIL_BYTES (sizeof(u32) * WA_TAIL_DWORDS)
170

171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195
struct virtual_engine {
	struct intel_engine_cs base;
	struct intel_context context;

	/*
	 * We allow only a single request through the virtual engine at a time
	 * (each request in the timeline waits for the completion fence of
	 * the previous before being submitted). By restricting ourselves to
	 * only submitting a single request, each request is placed on to a
	 * physical to maximise load spreading (by virtue of the late greedy
	 * scheduling -- each real engine takes the next available request
	 * upon idling).
	 */
	struct i915_request *request;

	/*
	 * We keep a rbtree of available virtual engines inside each physical
	 * engine, sorted by priority. Here we preallocate the nodes we need
	 * for the virtual engine, indexed by physical_engine->id.
	 */
	struct ve_node {
		struct rb_node rb;
		int prio;
	} nodes[I915_NUM_ENGINES];

196 197 198 199 200 201 202 203 204 205 206 207
	/*
	 * Keep track of bonded pairs -- restrictions upon on our selection
	 * of physical engines any particular request may be submitted to.
	 * If we receive a submit-fence from a master engine, we will only
	 * use one of sibling_mask physical engines.
	 */
	struct ve_bond {
		const struct intel_engine_cs *master;
		intel_engine_mask_t sibling_mask;
	} *bonds;
	unsigned int num_bonds;

208 209 210 211 212 213 214 215 216 217 218
	/* And finally, which physical engines this virtual engine maps onto. */
	unsigned int num_siblings;
	struct intel_engine_cs *siblings[0];
};

static struct virtual_engine *to_virtual_engine(struct intel_engine_cs *engine)
{
	GEM_BUG_ON(!intel_engine_is_virtual(engine));
	return container_of(engine, struct virtual_engine, base);
}

219 220
static int execlists_context_deferred_alloc(struct intel_context *ce,
					    struct intel_engine_cs *engine);
221
static void execlists_init_reg_state(u32 *reg_state,
222
				     struct intel_context *ce,
223 224
				     struct intel_engine_cs *engine,
				     struct intel_ring *ring);
225

226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244
static inline u32 intel_hws_preempt_address(struct intel_engine_cs *engine)
{
	return (i915_ggtt_offset(engine->status_page.vma) +
		I915_GEM_HWS_PREEMPT_ADDR);
}

static inline void
ring_set_paused(const struct intel_engine_cs *engine, int state)
{
	/*
	 * We inspect HWS_PREEMPT with a semaphore inside
	 * engine->emit_fini_breadcrumb. If the dword is true,
	 * the ring is paused as the semaphore will busywait
	 * until the dword is false.
	 */
	engine->status_page.addr[I915_GEM_HWS_PREEMPT] = state;
	wmb();
}

245 246 247 248 249 250 251
static inline struct i915_priolist *to_priolist(struct rb_node *rb)
{
	return rb_entry(rb, struct i915_priolist, node);
}

static inline int rq_prio(const struct i915_request *rq)
{
252
	return rq->sched.attr.priority;
253 254
}

255 256
static int effective_prio(const struct i915_request *rq)
{
257 258 259 260
	int prio = rq_prio(rq);

	/*
	 * On unwinding the active request, we give it a priority bump
261 262 263
	 * if it has completed waiting on any semaphore. If we know that
	 * the request has already started, we can prevent an unwanted
	 * preempt-to-idle cycle by taking that into account now.
264
	 */
265 266
	if (__i915_request_has_started(rq))
		prio |= I915_PRIORITY_NOSEMAPHORE;
267

268
	/* Restrict mere WAIT boosts from triggering preemption */
269
	BUILD_BUG_ON(__NO_PREEMPTION & ~I915_PRIORITY_MASK); /* only internal */
270
	return prio | __NO_PREEMPTION;
271 272
}

273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289
static int queue_prio(const struct intel_engine_execlists *execlists)
{
	struct i915_priolist *p;
	struct rb_node *rb;

	rb = rb_first_cached(&execlists->queue);
	if (!rb)
		return INT_MIN;

	/*
	 * As the priolist[] are inverted, with the highest priority in [0],
	 * we have to flip the index value to become priority.
	 */
	p = to_priolist(rb);
	return ((p->priority + 1) << I915_USER_PRIORITY_SHIFT) - ffs(p->used);
}

290
static inline bool need_preempt(const struct intel_engine_cs *engine,
291 292
				const struct i915_request *rq,
				struct rb_node *rb)
293
{
294
	int last_prio;
295 296 297 298 299 300 301 302 303 304 305 306 307

	/*
	 * Check if the current priority hint merits a preemption attempt.
	 *
	 * We record the highest value priority we saw during rescheduling
	 * prior to this dequeue, therefore we know that if it is strictly
	 * less than the current tail of ESLP[0], we do not need to force
	 * a preempt-to-idle cycle.
	 *
	 * However, the priority hint is a mere hint that we may need to
	 * preempt. If that hint is stale or we may be trying to preempt
	 * ourselves, ignore the request.
	 */
308
	last_prio = effective_prio(rq);
309 310
	if (!i915_scheduler_need_preempt(engine->execlists.queue_priority_hint,
					 last_prio))
311 312 313 314 315 316
		return false;

	/*
	 * Check against the first request in ELSP[1], it will, thanks to the
	 * power of PI, be the highest priority of that context.
	 */
317 318
	if (!list_is_last(&rq->sched.link, &engine->active.requests) &&
	    rq_prio(list_next_entry(rq, sched.link)) > last_prio)
319 320
		return true;

321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339
	if (rb) {
		struct virtual_engine *ve =
			rb_entry(rb, typeof(*ve), nodes[engine->id].rb);
		bool preempt = false;

		if (engine == ve->siblings[0]) { /* only preempt one sibling */
			struct i915_request *next;

			rcu_read_lock();
			next = READ_ONCE(ve->request);
			if (next)
				preempt = rq_prio(next) > last_prio;
			rcu_read_unlock();
		}

		if (preempt)
			return preempt;
	}

340 341 342 343 344 345 346 347 348 349 350 351 352 353
	/*
	 * If the inflight context did not trigger the preemption, then maybe
	 * it was the set of queued requests? Pick the highest priority in
	 * the queue (the first active priolist) and see if it deserves to be
	 * running instead of ELSP[0].
	 *
	 * The highest priority request in the queue can not be either
	 * ELSP[0] or ELSP[1] as, thanks again to PI, if it was the same
	 * context, it's priority would not exceed ELSP[0] aka last_prio.
	 */
	return queue_prio(&engine->execlists) > last_prio;
}

__maybe_unused static inline bool
354
assert_priority_queue(const struct i915_request *prev,
355
		      const struct i915_request *next)
356
{
357 358 359 360 361 362 363
	/*
	 * Without preemption, the prev may refer to the still active element
	 * which we refuse to let go.
	 *
	 * Even with preemption, there are times when we think it is better not
	 * to preempt and leave an ostensibly lower priority request in flight.
	 */
364
	if (i915_request_is_active(prev))
365 366 367
		return true;

	return rq_prio(prev) >= rq_prio(next);
368 369
}

370
/*
371 372 373 374 375
 * The context descriptor encodes various attributes of a context,
 * including its GTT address and some flags. Because it's fairly
 * expensive to calculate, we'll just do it once and cache the result,
 * which remains valid until the context is unpinned.
 *
376 377
 * This is what a descriptor looks like, from LSB to MSB::
 *
378
 *      bits  0-11:    flags, GEN8_CTX_* (cached in ctx->desc_template)
379
 *      bits 12-31:    LRCA, GTT address of (the HWSP of) this context
380
 *      bits 32-52:    ctx ID, a globally unique tag (highest bit used by GuC)
381 382
 *      bits 53-54:    mbz, reserved for use by hardware
 *      bits 55-63:    group ID, currently unused and set to 0
383 384 385 386 387 388 389 390 391 392 393 394
 *
 * Starting from Gen11, the upper dword of the descriptor has a new format:
 *
 *      bits 32-36:    reserved
 *      bits 37-47:    SW context ID
 *      bits 48:53:    engine instance
 *      bit 54:        mbz, reserved for use by hardware
 *      bits 55-60:    SW counter
 *      bits 61-63:    engine class
 *
 * engine info, SW context ID and SW counter need to form a unique number
 * (Context ID) per lrc.
395
 */
396 397
static u64
lrc_descriptor(struct intel_context *ce, struct intel_engine_cs *engine)
398
{
399
	struct i915_gem_context *ctx = ce->gem_context;
400
	u64 desc;
401

402 403
	BUILD_BUG_ON(MAX_CONTEXT_HW_ID > (BIT(GEN8_CTX_ID_WIDTH)));
	BUILD_BUG_ON(GEN11_MAX_CONTEXT_HW_ID > (BIT(GEN11_SW_CTX_ID_WIDTH)));
404

405
	desc = ctx->desc_template;				/* bits  0-11 */
406 407
	GEM_BUG_ON(desc & GENMASK_ULL(63, 12));

408
	desc |= i915_ggtt_offset(ce->state) + LRC_HEADER_PAGES * PAGE_SIZE;
409
								/* bits 12-31 */
410 411
	GEM_BUG_ON(desc & GENMASK_ULL(63, 32));

412 413 414 415 416
	/*
	 * The following 32bits are copied into the OA reports (dword 2).
	 * Consider updating oa_get_render_ctx_id in i915_perf.c when changing
	 * anything below.
	 */
417
	if (INTEL_GEN(engine->i915) >= 11) {
418 419 420 421 422 423 424 425 426 427 428 429 430 431 432
		GEM_BUG_ON(ctx->hw_id >= BIT(GEN11_SW_CTX_ID_WIDTH));
		desc |= (u64)ctx->hw_id << GEN11_SW_CTX_ID_SHIFT;
								/* bits 37-47 */

		desc |= (u64)engine->instance << GEN11_ENGINE_INSTANCE_SHIFT;
								/* bits 48-53 */

		/* TODO: decide what to do with SW counter (bits 55-60) */

		desc |= (u64)engine->class << GEN11_ENGINE_CLASS_SHIFT;
								/* bits 61-63 */
	} else {
		GEM_BUG_ON(ctx->hw_id >= BIT(GEN8_CTX_ID_WIDTH));
		desc |= (u64)ctx->hw_id << GEN8_CTX_ID_SHIFT;	/* bits 32-52 */
	}
433

434
	return desc;
435 436
}

437
static void unwind_wa_tail(struct i915_request *rq)
438 439 440 441 442
{
	rq->tail = intel_ring_wrap(rq->ring, rq->wa_tail - WA_TAIL_BYTES);
	assert_ring_tail_valid(rq->ring, rq->tail);
}

443
static struct i915_request *
444
__unwind_incomplete_requests(struct intel_engine_cs *engine)
445
{
446
	struct i915_request *rq, *rn, *active = NULL;
447
	struct list_head *uninitialized_var(pl);
448
	int prio = I915_PRIORITY_INVALID;
449

450
	lockdep_assert_held(&engine->active.lock);
451 452

	list_for_each_entry_safe_reverse(rq, rn,
453 454
					 &engine->active.requests,
					 sched.link) {
455 456
		struct intel_engine_cs *owner;

457
		if (i915_request_completed(rq))
458
			continue; /* XXX */
459

460
		__i915_request_unsubmit(rq);
461 462
		unwind_wa_tail(rq);

463 464 465 466 467 468 469 470 471 472 473 474 475 476 477
		/*
		 * Push the request back into the queue for later resubmission.
		 * If this request is not native to this physical engine (i.e.
		 * it came from a virtual source), push it back onto the virtual
		 * engine so that it can be moved across onto another physical
		 * engine as load dictates.
		 */
		owner = rq->hw_context->engine;
		if (likely(owner == engine)) {
			GEM_BUG_ON(rq_prio(rq) == I915_PRIORITY_INVALID);
			if (rq_prio(rq) != prio) {
				prio = rq_prio(rq);
				pl = i915_sched_lookup_priolist(engine, prio);
			}
			GEM_BUG_ON(RB_EMPTY_ROOT(&engine->execlists.queue.rb_root));
478

479
			list_move(&rq->sched.link, pl);
480 481 482 483 484 485
			active = rq;
		} else {
			rq->engine = owner;
			owner->submit_request(rq);
			active = NULL;
		}
486 487
	}

488
	return active;
489 490
}

491
struct i915_request *
492 493 494 495 496
execlists_unwind_incomplete_requests(struct intel_engine_execlists *execlists)
{
	struct intel_engine_cs *engine =
		container_of(execlists, typeof(*engine), execlists);

497
	return __unwind_incomplete_requests(engine);
498 499
}

500
static inline void
501
execlists_context_status_change(struct i915_request *rq, unsigned long status)
502
{
503 504 505 506 507 508
	/*
	 * Only used when GVT-g is enabled now. When GVT-g is disabled,
	 * The compiler should eliminate this function as dead-code.
	 */
	if (!IS_ENABLED(CONFIG_DRM_I915_GVT))
		return;
509

510 511
	atomic_notifier_call_chain(&rq->engine->context_status_notifier,
				   status, rq);
512 513
}

514 515
static inline struct i915_request *
execlists_schedule_in(struct i915_request *rq, int idx)
516
{
517 518
	struct intel_context *ce = rq->hw_context;
	int count;
519

520
	trace_i915_request_in(rq, idx);
521

522 523 524 525 526 527 528 529 530 531 532
	count = intel_context_inflight_count(ce);
	if (!count) {
		intel_context_get(ce);
		ce->inflight = rq->engine;

		execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_IN);
		intel_engine_context_in(ce->inflight);
	}

	intel_context_inflight_inc(ce);
	GEM_BUG_ON(intel_context_inflight(ce) != rq->engine);
533

534
	return i915_request_get(rq);
535 536
}

537
static void kick_siblings(struct i915_request *rq, struct intel_context *ce)
538
{
539
	struct virtual_engine *ve = container_of(ce, typeof(*ve), context);
540 541 542 543 544 545
	struct i915_request *next = READ_ONCE(ve->request);

	if (next && next->execution_mask & ~rq->execution_mask)
		tasklet_schedule(&ve->base.execlists.tasklet);
}

546
static inline void
547
execlists_schedule_out(struct i915_request *rq)
548
{
549 550 551 552
	struct intel_context *ce = rq->hw_context;

	GEM_BUG_ON(!intel_context_inflight_count(ce));

553
	trace_i915_request_out(rq);
554

555 556 557 558 559 560 561 562 563 564 565 566 567 568
	intel_context_inflight_dec(ce);
	if (!intel_context_inflight_count(ce)) {
		intel_engine_context_out(ce->inflight);
		execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_OUT);

		/*
		 * If this is part of a virtual engine, its next request may
		 * have been blocked waiting for access to the active context.
		 * We have to kick all the siblings again in case we need to
		 * switch (e.g. the next request is not runnable on this
		 * engine). Hopefully, we will already have submitted the next
		 * request before the tasklet runs and do not need to rebuild
		 * each virtual tree and kick everyone again.
		 */
569
		ce->inflight = NULL;
570 571
		if (rq->engine != ce->engine)
			kick_siblings(rq, ce);
572 573

		intel_context_put(ce);
574 575 576
	}

	i915_request_put(rq);
577 578
}

579
static u64 execlists_update_context(const struct i915_request *rq)
580
{
581
	struct intel_context *ce = rq->hw_context;
582
	u64 desc;
583

584 585
	ce->lrc_reg_state[CTX_RING_TAIL + 1] =
		intel_ring_set_tail(rq->ring, rq->tail);
586

587 588 589 590 591 592 593 594 595
	/*
	 * Make sure the context image is complete before we submit it to HW.
	 *
	 * Ostensibly, writes (including the WCB) should be flushed prior to
	 * an uncached write such as our mmio register access, the empirical
	 * evidence (esp. on Braswell) suggests that the WC write into memory
	 * may not be visible to the HW prior to the completion of the UC
	 * register write and that we may begin execution from the context
	 * before its image is complete leading to invalid PD chasing.
596 597 598 599 600
	 *
	 * Furthermore, Braswell, at least, wants a full mb to be sure that
	 * the writes are coherent in memory (visible to the GPU) prior to
	 * execution, and not just visible to other CPUs (as is the result of
	 * wmb).
601
	 */
602
	mb();
603 604 605 606 607

	desc = ce->lrc_desc;
	ce->lrc_desc &= ~CTX_DESC_FORCE_RESTORE;

	return desc;
608 609
}

610
static inline void write_desc(struct intel_engine_execlists *execlists, u64 desc, u32 port)
611
{
612 613 614 615 616 617 618
	if (execlists->ctrl_reg) {
		writel(lower_32_bits(desc), execlists->submit_reg + port * 2);
		writel(upper_32_bits(desc), execlists->submit_reg + port * 2 + 1);
	} else {
		writel(upper_32_bits(desc), execlists->submit_reg);
		writel(lower_32_bits(desc), execlists->submit_reg);
	}
619 620
}

621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669
static __maybe_unused void
trace_ports(const struct intel_engine_execlists *execlists,
	    const char *msg,
	    struct i915_request * const *ports)
{
	const struct intel_engine_cs *engine =
		container_of(execlists, typeof(*engine), execlists);

	GEM_TRACE("%s: %s { %llx:%lld%s, %llx:%lld }\n",
		  engine->name, msg,
		  ports[0]->fence.context,
		  ports[0]->fence.seqno,
		  i915_request_completed(ports[0]) ? "!" :
		  i915_request_started(ports[0]) ? "*" :
		  "",
		  ports[1] ? ports[1]->fence.context : 0,
		  ports[1] ? ports[1]->fence.seqno : 0);
}

static __maybe_unused bool
assert_pending_valid(const struct intel_engine_execlists *execlists,
		     const char *msg)
{
	struct i915_request * const *port, *rq;
	struct intel_context *ce = NULL;

	trace_ports(execlists, msg, execlists->pending);

	if (execlists->pending[execlists_num_ports(execlists)])
		return false;

	for (port = execlists->pending; (rq = *port); port++) {
		if (ce == rq->hw_context)
			return false;

		ce = rq->hw_context;
		if (i915_request_completed(rq))
			continue;

		if (i915_active_is_idle(&ce->active))
			return false;

		if (!i915_vma_is_pinned(ce->state))
			return false;
	}

	return ce;
}

670
static void execlists_submit_ports(struct intel_engine_cs *engine)
671
{
672
	struct intel_engine_execlists *execlists = &engine->execlists;
673
	unsigned int n;
674

675 676
	GEM_BUG_ON(!assert_pending_valid(execlists, "submit"));

677 678 679 680 681 682 683 684
	/*
	 * We can skip acquiring intel_runtime_pm_get() here as it was taken
	 * on our behalf by the request (see i915_gem_mark_busy()) and it will
	 * not be relinquished until the device is idle (see
	 * i915_gem_idle_work_handler()). As a precaution, we make sure
	 * that all ELSP are drained i.e. we have processed the CSB,
	 * before allowing ourselves to idle and calling intel_runtime_pm_put().
	 */
685
	GEM_BUG_ON(!intel_wakeref_active(&engine->wakeref));
686

687 688 689 690 691 692 693
	/*
	 * ELSQ note: the submit queue is not cleared after being submitted
	 * to the HW so we need to make sure we always clean it up. This is
	 * currently ensured by the fact that we always write the same number
	 * of elsq entries, keep this in mind before changing the loop below.
	 */
	for (n = execlists_num_ports(execlists); n--; ) {
694
		struct i915_request *rq = execlists->pending[n];
695

696 697 698
		write_desc(execlists,
			   rq ? execlists_update_context(rq) : 0,
			   n);
699
	}
700 701 702 703

	/* we need to manually load the submit queue */
	if (execlists->ctrl_reg)
		writel(EL_CTRL_LOAD, execlists->ctrl_reg);
704 705
}

706
static bool ctx_single_port_submission(const struct intel_context *ce)
707
{
708
	return (IS_ENABLED(CONFIG_DRM_I915_GVT) &&
709
		i915_gem_context_force_single_submission(ce->gem_context));
710
}
711

712 713
static bool can_merge_ctx(const struct intel_context *prev,
			  const struct intel_context *next)
714 715 716
{
	if (prev != next)
		return false;
717

718 719
	if (ctx_single_port_submission(prev))
		return false;
720

721
	return true;
722 723
}

724 725 726
static bool can_merge_rq(const struct i915_request *prev,
			 const struct i915_request *next)
{
727
	GEM_BUG_ON(prev == next);
728 729 730 731 732 733 734 735
	GEM_BUG_ON(!assert_priority_queue(prev, next));

	if (!can_merge_ctx(prev->hw_context, next->hw_context))
		return false;

	return true;
}

736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785
static void virtual_update_register_offsets(u32 *regs,
					    struct intel_engine_cs *engine)
{
	u32 base = engine->mmio_base;

	/* Must match execlists_init_reg_state()! */

	regs[CTX_CONTEXT_CONTROL] =
		i915_mmio_reg_offset(RING_CONTEXT_CONTROL(base));
	regs[CTX_RING_HEAD] = i915_mmio_reg_offset(RING_HEAD(base));
	regs[CTX_RING_TAIL] = i915_mmio_reg_offset(RING_TAIL(base));
	regs[CTX_RING_BUFFER_START] = i915_mmio_reg_offset(RING_START(base));
	regs[CTX_RING_BUFFER_CONTROL] = i915_mmio_reg_offset(RING_CTL(base));

	regs[CTX_BB_HEAD_U] = i915_mmio_reg_offset(RING_BBADDR_UDW(base));
	regs[CTX_BB_HEAD_L] = i915_mmio_reg_offset(RING_BBADDR(base));
	regs[CTX_BB_STATE] = i915_mmio_reg_offset(RING_BBSTATE(base));
	regs[CTX_SECOND_BB_HEAD_U] =
		i915_mmio_reg_offset(RING_SBBADDR_UDW(base));
	regs[CTX_SECOND_BB_HEAD_L] = i915_mmio_reg_offset(RING_SBBADDR(base));
	regs[CTX_SECOND_BB_STATE] = i915_mmio_reg_offset(RING_SBBSTATE(base));

	regs[CTX_CTX_TIMESTAMP] =
		i915_mmio_reg_offset(RING_CTX_TIMESTAMP(base));
	regs[CTX_PDP3_UDW] = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, 3));
	regs[CTX_PDP3_LDW] = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, 3));
	regs[CTX_PDP2_UDW] = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, 2));
	regs[CTX_PDP2_LDW] = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, 2));
	regs[CTX_PDP1_UDW] = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, 1));
	regs[CTX_PDP1_LDW] = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, 1));
	regs[CTX_PDP0_UDW] = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, 0));
	regs[CTX_PDP0_LDW] = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, 0));

	if (engine->class == RENDER_CLASS) {
		regs[CTX_RCS_INDIRECT_CTX] =
			i915_mmio_reg_offset(RING_INDIRECT_CTX(base));
		regs[CTX_RCS_INDIRECT_CTX_OFFSET] =
			i915_mmio_reg_offset(RING_INDIRECT_CTX_OFFSET(base));
		regs[CTX_BB_PER_CTX_PTR] =
			i915_mmio_reg_offset(RING_BB_PER_CTX_PTR(base));

		regs[CTX_R_PWR_CLK_STATE] =
			i915_mmio_reg_offset(GEN8_R_PWR_CLK_STATE);
	}
}

static bool virtual_matches(const struct virtual_engine *ve,
			    const struct i915_request *rq,
			    const struct intel_engine_cs *engine)
{
786
	const struct intel_engine_cs *inflight;
787

788 789 790
	if (!(rq->execution_mask & engine->mask)) /* We peeked too soon! */
		return false;

791 792 793 794 795 796 797 798 799
	/*
	 * We track when the HW has completed saving the context image
	 * (i.e. when we have seen the final CS event switching out of
	 * the context) and must not overwrite the context image before
	 * then. This restricts us to only using the active engine
	 * while the previous virtualized request is inflight (so
	 * we reuse the register offsets). This is a very small
	 * hystersis on the greedy seelction algorithm.
	 */
800
	inflight = intel_context_inflight(&ve->context);
801
	if (inflight && inflight != engine)
802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822
		return false;

	return true;
}

static void virtual_xfer_breadcrumbs(struct virtual_engine *ve,
				     struct intel_engine_cs *engine)
{
	struct intel_engine_cs *old = ve->siblings[0];

	/* All unattached (rq->engine == old) must already be completed */

	spin_lock(&old->breadcrumbs.irq_lock);
	if (!list_empty(&ve->context.signal_link)) {
		list_move_tail(&ve->context.signal_link,
			       &engine->breadcrumbs.signalers);
		intel_engine_queue_breadcrumbs(engine);
	}
	spin_unlock(&old->breadcrumbs.irq_lock);
}

823 824 825 826 827 828 829 830 831 832 833
static struct i915_request *
last_active(const struct intel_engine_execlists *execlists)
{
	struct i915_request * const *last = execlists->active;

	while (*last && i915_request_completed(*last))
		last++;

	return *last;
}

834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908
static void
defer_request(struct i915_request * const rq, struct list_head * const pl)
{
	struct i915_dependency *p;

	/*
	 * We want to move the interrupted request to the back of
	 * the round-robin list (i.e. its priority level), but
	 * in doing so, we must then move all requests that were in
	 * flight and were waiting for the interrupted request to
	 * be run after it again.
	 */
	list_move_tail(&rq->sched.link, pl);

	list_for_each_entry(p, &rq->sched.waiters_list, wait_link) {
		struct i915_request *w =
			container_of(p->waiter, typeof(*w), sched);

		/* Leave semaphores spinning on the other engines */
		if (w->engine != rq->engine)
			continue;

		/* No waiter should start before the active request completed */
		GEM_BUG_ON(i915_request_started(w));

		GEM_BUG_ON(rq_prio(w) > rq_prio(rq));
		if (rq_prio(w) < rq_prio(rq))
			continue;

		if (list_empty(&w->sched.link))
			continue; /* Not yet submitted; unready */

		/*
		 * This should be very shallow as it is limited by the
		 * number of requests that can fit in a ring (<64) and
		 * the number of contexts that can be in flight on this
		 * engine.
		 */
		defer_request(w, pl);
	}
}

static void defer_active(struct intel_engine_cs *engine)
{
	struct i915_request *rq;

	rq = __unwind_incomplete_requests(engine);
	if (!rq)
		return;

	defer_request(rq, i915_sched_lookup_priolist(engine, rq_prio(rq)));
}

static bool
need_timeslice(struct intel_engine_cs *engine, const struct i915_request *rq)
{
	int hint;

	if (list_is_last(&rq->sched.link, &engine->active.requests))
		return false;

	hint = max(rq_prio(list_next_entry(rq, sched.link)),
		   engine->execlists.queue_priority_hint);

	return hint >= rq_prio(rq);
}

static bool
enable_timeslice(struct intel_engine_cs *engine)
{
	struct i915_request *last = last_active(&engine->execlists);

	return last && need_timeslice(engine, last);
}

909
static void execlists_dequeue(struct intel_engine_cs *engine)
910
{
911
	struct intel_engine_execlists * const execlists = &engine->execlists;
912 913 914
	struct i915_request **port = execlists->pending;
	struct i915_request ** const last_port = port + execlists->port_mask;
	struct i915_request *last;
915
	struct rb_node *rb;
916 917
	bool submit = false;

918 919
	/*
	 * Hardware submission is through 2 ports. Conceptually each port
920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937
	 * has a (RING_START, RING_HEAD, RING_TAIL) tuple. RING_START is
	 * static for a context, and unique to each, so we only execute
	 * requests belonging to a single context from each ring. RING_HEAD
	 * is maintained by the CS in the context image, it marks the place
	 * where it got up to last time, and through RING_TAIL we tell the CS
	 * where we want to execute up to this time.
	 *
	 * In this list the requests are in order of execution. Consecutive
	 * requests from the same context are adjacent in the ringbuffer. We
	 * can combine these requests into a single RING_TAIL update:
	 *
	 *              RING_HEAD...req1...req2
	 *                                    ^- RING_TAIL
	 * since to execute req2 the CS must first execute req1.
	 *
	 * Our goal then is to point each port to the end of a consecutive
	 * sequence of requests as being the most optimal (fewest wake ups
	 * and context switches) submission.
938
	 */
939

940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959
	for (rb = rb_first_cached(&execlists->virtual); rb; ) {
		struct virtual_engine *ve =
			rb_entry(rb, typeof(*ve), nodes[engine->id].rb);
		struct i915_request *rq = READ_ONCE(ve->request);

		if (!rq) { /* lazily cleanup after another engine handled rq */
			rb_erase_cached(rb, &execlists->virtual);
			RB_CLEAR_NODE(rb);
			rb = rb_first_cached(&execlists->virtual);
			continue;
		}

		if (!virtual_matches(ve, rq, engine)) {
			rb = rb_next(rb);
			continue;
		}

		break;
	}

960 961 962 963 964 965 966 967 968
	/*
	 * If the queue is higher priority than the last
	 * request in the currently active context, submit afresh.
	 * We will resubmit again afterwards in case we need to split
	 * the active context to interject the preemption request,
	 * i.e. we will retrigger preemption following the ack in case
	 * of trouble.
	 */
	last = last_active(execlists);
969
	if (last) {
970
		if (need_preempt(engine, last, rb)) {
971 972 973 974 975 976 977 978 979 980 981 982
			GEM_TRACE("%s: preempting last=%llx:%lld, prio=%d, hint=%d\n",
				  engine->name,
				  last->fence.context,
				  last->fence.seqno,
				  last->sched.attr.priority,
				  execlists->queue_priority_hint);
			/*
			 * Don't let the RING_HEAD advance past the breadcrumb
			 * as we unwind (and until we resubmit) so that we do
			 * not accidentally tell it to go backwards.
			 */
			ring_set_paused(engine, 1);
983

984 985 986 987 988 989 990 991
			/*
			 * Note that we have not stopped the GPU at this point,
			 * so we are unwinding the incomplete requests as they
			 * remain inflight and so by the time we do complete
			 * the preemption, some of the unwound requests may
			 * complete!
			 */
			__unwind_incomplete_requests(engine);
992

993 994 995 996 997 998 999 1000 1001
			/*
			 * If we need to return to the preempted context, we
			 * need to skip the lite-restore and force it to
			 * reload the RING_TAIL. Otherwise, the HW has a
			 * tendency to ignore us rewinding the TAIL to the
			 * end of an earlier request.
			 */
			last->hw_context->lrc_desc |= CTX_DESC_FORCE_RESTORE;
			last = NULL;
1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027
		} else if (need_timeslice(engine, last) &&
			   !timer_pending(&engine->execlists.timer)) {
			GEM_TRACE("%s: expired last=%llx:%lld, prio=%d, hint=%d\n",
				  engine->name,
				  last->fence.context,
				  last->fence.seqno,
				  last->sched.attr.priority,
				  execlists->queue_priority_hint);

			ring_set_paused(engine, 1);
			defer_active(engine);

			/*
			 * Unlike for preemption, if we rewind and continue
			 * executing the same context as previously active,
			 * the order of execution will remain the same and
			 * the tail will only advance. We do not need to
			 * force a full context restore, as a lite-restore
			 * is sufficient to resample the monotonic TAIL.
			 *
			 * If we switch to any other context, similarly we
			 * will not rewind TAIL of current context, and
			 * normal save/restore will preserve state and allow
			 * us to later continue executing the same request.
			 */
			last = NULL;
1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051
		} else {
			/*
			 * Otherwise if we already have a request pending
			 * for execution after the current one, we can
			 * just wait until the next CS event before
			 * queuing more. In either case we will force a
			 * lite-restore preemption event, but if we wait
			 * we hopefully coalesce several updates into a single
			 * submission.
			 */
			if (!list_is_last(&last->sched.link,
					  &engine->active.requests))
				return;

			/*
			 * WaIdleLiteRestore:bdw,skl
			 * Apply the wa NOOPs to prevent
			 * ring:HEAD == rq:TAIL as we resubmit the
			 * request. See gen8_emit_fini_breadcrumb() for
			 * where we prepare the padding after the
			 * end of the request.
			 */
			last->tail = last->wa_tail;
		}
1052 1053
	}

1054 1055 1056 1057 1058
	while (rb) { /* XXX virtual is always taking precedence */
		struct virtual_engine *ve =
			rb_entry(rb, typeof(*ve), nodes[engine->id].rb);
		struct i915_request *rq;

1059
		spin_lock(&ve->base.active.lock);
1060 1061 1062

		rq = ve->request;
		if (unlikely(!rq)) { /* lost the race to a sibling */
1063
			spin_unlock(&ve->base.active.lock);
1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075
			rb_erase_cached(rb, &execlists->virtual);
			RB_CLEAR_NODE(rb);
			rb = rb_first_cached(&execlists->virtual);
			continue;
		}

		GEM_BUG_ON(rq != ve->request);
		GEM_BUG_ON(rq->engine != &ve->base);
		GEM_BUG_ON(rq->hw_context != &ve->context);

		if (rq_prio(rq) >= queue_prio(execlists)) {
			if (!virtual_matches(ve, rq, engine)) {
1076
				spin_unlock(&ve->base.active.lock);
1077 1078 1079 1080
				rb = rb_next(rb);
				continue;
			}

1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095
			if (i915_request_completed(rq)) {
				ve->request = NULL;
				ve->base.execlists.queue_priority_hint = INT_MIN;
				rb_erase_cached(rb, &execlists->virtual);
				RB_CLEAR_NODE(rb);

				rq->engine = engine;
				__i915_request_submit(rq);

				spin_unlock(&ve->base.active.lock);

				rb = rb_first_cached(&execlists->virtual);
				continue;
			}

1096
			if (last && !can_merge_rq(last, rq)) {
1097
				spin_unlock(&ve->base.active.lock);
1098
				return; /* leave this for another */
1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114
			}

			GEM_TRACE("%s: virtual rq=%llx:%lld%s, new engine? %s\n",
				  engine->name,
				  rq->fence.context,
				  rq->fence.seqno,
				  i915_request_completed(rq) ? "!" :
				  i915_request_started(rq) ? "*" :
				  "",
				  yesno(engine != ve->siblings[0]));

			ve->request = NULL;
			ve->base.execlists.queue_priority_hint = INT_MIN;
			rb_erase_cached(rb, &execlists->virtual);
			RB_CLEAR_NODE(rb);

1115
			GEM_BUG_ON(!(rq->execution_mask & engine->mask));
1116 1117 1118 1119 1120 1121
			rq->engine = engine;

			if (engine != ve->siblings[0]) {
				u32 *regs = ve->context.lrc_reg_state;
				unsigned int n;

1122
				GEM_BUG_ON(READ_ONCE(ve->context.inflight));
1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146
				virtual_update_register_offsets(regs, engine);

				if (!list_empty(&ve->context.signals))
					virtual_xfer_breadcrumbs(ve, engine);

				/*
				 * Move the bound engine to the top of the list
				 * for future execution. We then kick this
				 * tasklet first before checking others, so that
				 * we preferentially reuse this set of bound
				 * registers.
				 */
				for (n = 1; n < ve->num_siblings; n++) {
					if (ve->siblings[n] == engine) {
						swap(ve->siblings[n],
						     ve->siblings[0]);
						break;
					}
				}

				GEM_BUG_ON(ve->siblings[0] != engine);
			}

			__i915_request_submit(rq);
1147 1148 1149 1150
			if (!i915_request_completed(rq)) {
				submit = true;
				last = rq;
			}
1151 1152
		}

1153
		spin_unlock(&ve->base.active.lock);
1154 1155 1156
		break;
	}

1157
	while ((rb = rb_first_cached(&execlists->queue))) {
1158
		struct i915_priolist *p = to_priolist(rb);
1159
		struct i915_request *rq, *rn;
1160
		int i;
1161

1162
		priolist_for_each_request_consume(rq, rn, p, i) {
1163 1164 1165
			if (i915_request_completed(rq))
				goto skip;

1166 1167 1168 1169 1170 1171 1172 1173 1174 1175
			/*
			 * Can we combine this request with the current port?
			 * It has to be the same context/ringbuffer and not
			 * have any exceptions (e.g. GVT saying never to
			 * combine contexts).
			 *
			 * If we can combine the requests, we can execute both
			 * by updating the RING_TAIL to point to the end of the
			 * second request, and so we never need to tell the
			 * hardware about the first.
1176
			 */
1177
			if (last && !can_merge_rq(last, rq)) {
1178 1179 1180 1181 1182
				/*
				 * If we are on the second port and cannot
				 * combine this request with the last, then we
				 * are done.
				 */
1183
				if (port == last_port)
1184 1185
					goto done;

1186 1187 1188 1189 1190 1191 1192 1193
				/*
				 * We must not populate both ELSP[] with the
				 * same LRCA, i.e. we must submit 2 different
				 * contexts if we submit 2 ELSP.
				 */
				if (last->hw_context == rq->hw_context)
					goto done;

1194 1195 1196 1197 1198 1199 1200
				/*
				 * If GVT overrides us we only ever submit
				 * port[0], leaving port[1] empty. Note that we
				 * also have to be careful that we don't queue
				 * the same context (even though a different
				 * request) to the second port.
				 */
1201
				if (ctx_single_port_submission(last->hw_context) ||
1202
				    ctx_single_port_submission(rq->hw_context))
1203 1204
					goto done;

1205
				*port = execlists_schedule_in(last, port - execlists->pending);
1206 1207
				port++;
			}
1208

1209 1210
			last = rq;
			submit = true;
1211 1212
skip:
			__i915_request_submit(rq);
1213
		}
1214

1215
		rb_erase_cached(&p->node, &execlists->queue);
1216
		i915_priolist_free(p);
1217
	}
1218

1219
done:
1220 1221 1222
	/*
	 * Here be a bit of magic! Or sleight-of-hand, whichever you prefer.
	 *
1223
	 * We choose the priority hint such that if we add a request of greater
1224 1225 1226
	 * priority than this, we kick the submission tasklet to decide on
	 * the right order of submitting the requests to hardware. We must
	 * also be prepared to reorder requests as they are in-flight on the
1227
	 * HW. We derive the priority hint then as the first "hole" in
1228 1229 1230 1231
	 * the HW submission ports and if there are no available slots,
	 * the priority of the lowest executing request, i.e. last.
	 *
	 * When we do receive a higher priority request ready to run from the
1232
	 * user, see queue_request(), the priority hint is bumped to that
1233 1234 1235
	 * request triggering preemption on the next dequeue (or subsequent
	 * interrupt for secondary ports).
	 */
1236
	execlists->queue_priority_hint = queue_prio(execlists);
1237 1238 1239
	GEM_TRACE("%s: queue_priority_hint:%d, submit:%s\n",
		  engine->name, execlists->queue_priority_hint,
		  yesno(submit));
1240

1241
	if (submit) {
1242 1243
		*port = execlists_schedule_in(last, port - execlists->pending);
		memset(port + 1, 0, (last_port - port) * sizeof(*port));
1244 1245
		execlists_submit_ports(engine);
	}
1246 1247
}

1248
void
1249
execlists_cancel_port_requests(struct intel_engine_execlists * const execlists)
1250
{
1251
	struct i915_request * const *port, *rq;
1252

1253 1254 1255
	for (port = execlists->pending; (rq = *port); port++)
		execlists_schedule_out(rq);
	memset(execlists->pending, 0, sizeof(execlists->pending));
1256

1257 1258 1259 1260
	for (port = execlists->active; (rq = *port); port++)
		execlists_schedule_out(rq);
	execlists->active =
		memset(execlists->inflight, 0, sizeof(execlists->inflight));
1261 1262
}

1263 1264 1265 1266 1267 1268 1269
static inline void
invalidate_csb_entries(const u32 *first, const u32 *last)
{
	clflush((void *)first);
	clflush((void *)last);
}

1270 1271 1272 1273 1274 1275
static inline bool
reset_in_progress(const struct intel_engine_execlists *execlists)
{
	return unlikely(!__tasklet_is_enabled(&execlists->tasklet));
}

1276
static void process_csb(struct intel_engine_cs *engine)
1277
{
1278
	struct intel_engine_execlists * const execlists = &engine->execlists;
1279
	const u32 * const buf = execlists->csb_status;
1280
	const u8 num_entries = execlists->csb_size;
1281
	u8 head, tail;
1282

1283
	lockdep_assert_held(&engine->active.lock);
1284
	GEM_BUG_ON(USES_GUC_SUBMISSION(engine->i915));
1285

1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300
	/*
	 * Note that csb_write, csb_status may be either in HWSP or mmio.
	 * When reading from the csb_write mmio register, we have to be
	 * careful to only use the GEN8_CSB_WRITE_PTR portion, which is
	 * the low 4bits. As it happens we know the next 4bits are always
	 * zero and so we can simply masked off the low u8 of the register
	 * and treat it identically to reading from the HWSP (without having
	 * to use explicit shifting and masking, and probably bifurcating
	 * the code to handle the legacy mmio read).
	 */
	head = execlists->csb_head;
	tail = READ_ONCE(*execlists->csb_write);
	GEM_TRACE("%s cs-irq head=%d, tail=%d\n", engine->name, head, tail);
	if (unlikely(head == tail))
		return;
1301

1302 1303 1304 1305 1306 1307 1308 1309 1310
	/*
	 * Hopefully paired with a wmb() in HW!
	 *
	 * We must complete the read of the write pointer before any reads
	 * from the CSB, so that we do not see stale values. Without an rmb
	 * (lfence) the HW may speculatively perform the CSB[] reads *before*
	 * we perform the READ_ONCE(*csb_write).
	 */
	rmb();
1311

1312
	do {
1313 1314
		unsigned int status;

1315
		if (++head == num_entries)
1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335
			head = 0;

		/*
		 * We are flying near dragons again.
		 *
		 * We hold a reference to the request in execlist_port[]
		 * but no more than that. We are operating in softirq
		 * context and so cannot hold any mutex or sleep. That
		 * prevents us stopping the requests we are processing
		 * in port[] from being retired simultaneously (the
		 * breadcrumb will be complete before we see the
		 * context-switch). As we only hold the reference to the
		 * request, any pointer chasing underneath the request
		 * is subject to a potential use-after-free. Thus we
		 * store all of the bookkeeping within port[] as
		 * required, and avoid using unguarded pointers beneath
		 * request itself. The same applies to the atomic
		 * status notifier.
		 */

1336
		GEM_TRACE("%s csb[%d]: status=0x%08x:0x%08x\n",
1337
			  engine->name, head,
1338
			  buf[2 * head + 0], buf[2 * head + 1]);
1339

1340
		status = buf[2 * head];
1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351
		if (status & GEN8_CTX_STATUS_IDLE_ACTIVE) {
			GEM_BUG_ON(*execlists->active);
promote:
			GEM_BUG_ON(!assert_pending_valid(execlists, "promote"));
			execlists->active =
				memcpy(execlists->inflight,
				       execlists->pending,
				       execlists_num_ports(execlists) *
				       sizeof(*execlists->pending));
			execlists->pending[0] = NULL;

1352 1353 1354
			if (enable_timeslice(engine))
				mod_timer(&execlists->timer, jiffies + 1);

1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370
			if (!inject_preempt_hang(execlists))
				ring_set_paused(engine, 0);
		} else if (status & GEN8_CTX_STATUS_PREEMPTED) {
			struct i915_request * const *port = execlists->active;

			trace_ports(execlists, "preempted", execlists->active);

			while (*port)
				execlists_schedule_out(*port++);

			goto promote;
		} else if (*execlists->active) {
			struct i915_request *rq = *execlists->active++;

			trace_ports(execlists, "completed",
				    execlists->active - 1);
1371

1372 1373 1374 1375 1376 1377 1378
			/*
			 * We rely on the hardware being strongly
			 * ordered, that the breadcrumb write is
			 * coherent (visible from the CPU) before the
			 * user interrupt and CSB is processed.
			 */
			GEM_BUG_ON(!i915_request_completed(rq));
1379
			execlists_schedule_out(rq);
1380

1381 1382
			GEM_BUG_ON(execlists->active - execlists->inflight >
				   execlists_num_ports(execlists));
1383
		}
1384
	} while (head != tail);
1385

1386
	execlists->csb_head = head;
1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398

	/*
	 * Gen11 has proven to fail wrt global observation point between
	 * entry and tail update, failing on the ordering and thus
	 * we see an old entry in the context status buffer.
	 *
	 * Forcibly evict out entries for the next gpu csb update,
	 * to increase the odds that we get a fresh entries with non
	 * working hardware. The cost for doing so comes out mostly with
	 * the wash as hardware, working or not, will need to do the
	 * invalidation before.
	 */
1399
	invalidate_csb_entries(&buf[0], &buf[num_entries - 1]);
1400
}
1401

1402
static void __execlists_submission_tasklet(struct intel_engine_cs *const engine)
1403
{
1404
	lockdep_assert_held(&engine->active.lock);
1405

1406
	process_csb(engine);
1407
	if (!engine->execlists.pending[0])
1408
		execlists_dequeue(engine);
1409 1410
}

1411 1412 1413 1414 1415 1416 1417 1418 1419
/*
 * Check the unread Context Status Buffers and manage the submission of new
 * contexts to the ELSP accordingly.
 */
static void execlists_submission_tasklet(unsigned long data)
{
	struct intel_engine_cs * const engine = (struct intel_engine_cs *)data;
	unsigned long flags;

1420
	spin_lock_irqsave(&engine->active.lock, flags);
1421
	__execlists_submission_tasklet(engine);
1422
	spin_unlock_irqrestore(&engine->active.lock, flags);
1423 1424
}

1425 1426 1427 1428 1429 1430 1431 1432 1433
static void execlists_submission_timer(struct timer_list *timer)
{
	struct intel_engine_cs *engine =
		from_timer(engine, timer, execlists.timer);

	/* Kick the tasklet for some interrupt coalescing and reset handling */
	tasklet_hi_schedule(&engine->execlists.tasklet);
}

1434
static void queue_request(struct intel_engine_cs *engine,
1435
			  struct i915_sched_node *node,
1436
			  int prio)
1437
{
1438
	GEM_BUG_ON(!list_empty(&node->link));
1439
	list_add_tail(&node->link, i915_sched_lookup_priolist(engine, prio));
1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452
}

static void __submit_queue_imm(struct intel_engine_cs *engine)
{
	struct intel_engine_execlists * const execlists = &engine->execlists;

	if (reset_in_progress(execlists))
		return; /* defer until we restart the engine following reset */

	if (execlists->tasklet.func == execlists_submission_tasklet)
		__execlists_submission_tasklet(engine);
	else
		tasklet_hi_schedule(&execlists->tasklet);
1453 1454
}

1455 1456
static void submit_queue(struct intel_engine_cs *engine,
			 const struct i915_request *rq)
1457
{
1458 1459 1460 1461 1462 1463 1464
	struct intel_engine_execlists *execlists = &engine->execlists;

	if (rq_prio(rq) <= execlists->queue_priority_hint)
		return;

	execlists->queue_priority_hint = rq_prio(rq);
	__submit_queue_imm(engine);
1465 1466
}

1467
static void execlists_submit_request(struct i915_request *request)
1468
{
1469
	struct intel_engine_cs *engine = request->engine;
1470
	unsigned long flags;
1471

1472
	/* Will be called from irq-context when using foreign fences. */
1473
	spin_lock_irqsave(&engine->active.lock, flags);
1474

1475
	queue_request(engine, &request->sched, rq_prio(request));
1476

1477
	GEM_BUG_ON(RB_EMPTY_ROOT(&engine->execlists.queue.rb_root));
1478
	GEM_BUG_ON(list_empty(&request->sched.link));
1479

1480
	submit_queue(engine, request);
1481

1482
	spin_unlock_irqrestore(&engine->active.lock, flags);
1483 1484
}

1485
static void __execlists_context_fini(struct intel_context *ce)
1486
{
1487
	intel_ring_put(ce->ring);
1488 1489 1490

	GEM_BUG_ON(i915_gem_object_is_active(ce->state->obj));
	i915_gem_object_put(ce->state->obj);
1491 1492
}

1493
static void execlists_context_destroy(struct kref *kref)
1494
{
1495 1496
	struct intel_context *ce = container_of(kref, typeof(*ce), ref);

1497
	GEM_BUG_ON(!i915_active_is_idle(&ce->active));
1498
	GEM_BUG_ON(intel_context_is_pinned(ce));
1499 1500 1501 1502 1503 1504 1505

	if (ce->state)
		__execlists_context_fini(ce);

	intel_context_free(ce);
}

1506
static void execlists_context_unpin(struct intel_context *ce)
1507
{
1508
	i915_gem_context_unpin_hw_id(ce->gem_context);
1509
	i915_gem_object_unpin_map(ce->state->obj);
1510 1511
}

1512
static void
1513 1514
__execlists_update_reg_state(struct intel_context *ce,
			     struct intel_engine_cs *engine)
1515 1516
{
	struct intel_ring *ring = ce->ring;
1517 1518 1519 1520
	u32 *regs = ce->lrc_reg_state;

	GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->head));
	GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->tail));
1521 1522 1523 1524 1525 1526 1527

	regs[CTX_RING_BUFFER_START + 1] = i915_ggtt_offset(ring->vma);
	regs[CTX_RING_HEAD + 1] = ring->head;
	regs[CTX_RING_TAIL + 1] = ring->tail;

	/* RPCS */
	if (engine->class == RENDER_CLASS)
1528
		regs[CTX_R_PWR_CLK_STATE + 1] =
1529
			intel_sseu_make_rpcs(engine->i915, &ce->sseu);
1530 1531
}

1532 1533 1534
static int
__execlists_context_pin(struct intel_context *ce,
			struct intel_engine_cs *engine)
1535
{
1536
	void *vaddr;
1537
	int ret;
1538

1539
	GEM_BUG_ON(!ce->gem_context->vm);
1540 1541

	ret = execlists_context_deferred_alloc(ce, engine);
1542 1543
	if (ret)
		goto err;
1544
	GEM_BUG_ON(!ce->state);
1545

1546 1547 1548 1549
	ret = intel_context_active_acquire(ce,
					   engine->i915->ggtt.pin_bias |
					   PIN_OFFSET_BIAS |
					   PIN_HIGH);
1550
	if (ret)
1551
		goto err;
1552

1553
	vaddr = i915_gem_object_pin_map(ce->state->obj,
1554
					i915_coherent_map_type(engine->i915) |
1555
					I915_MAP_OVERRIDE);
1556 1557
	if (IS_ERR(vaddr)) {
		ret = PTR_ERR(vaddr);
1558
		goto unpin_active;
1559 1560
	}

1561
	ret = i915_gem_context_pin_hw_id(ce->gem_context);
1562
	if (ret)
1563
		goto unpin_map;
1564

1565
	ce->lrc_desc = lrc_descriptor(ce, engine);
1566
	ce->lrc_reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE;
1567
	__execlists_update_reg_state(ce, engine);
1568

1569
	return 0;
1570

1571
unpin_map:
1572
	i915_gem_object_unpin_map(ce->state->obj);
1573 1574
unpin_active:
	intel_context_active_release(ce);
1575
err:
1576
	return ret;
1577 1578
}

1579
static int execlists_context_pin(struct intel_context *ce)
1580
{
1581
	return __execlists_context_pin(ce, ce->engine);
1582 1583
}

1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605
static void execlists_context_reset(struct intel_context *ce)
{
	/*
	 * Because we emit WA_TAIL_DWORDS there may be a disparity
	 * between our bookkeeping in ce->ring->head and ce->ring->tail and
	 * that stored in context. As we only write new commands from
	 * ce->ring->tail onwards, everything before that is junk. If the GPU
	 * starts reading from its RING_HEAD from the context, it may try to
	 * execute that junk and die.
	 *
	 * The contexts that are stilled pinned on resume belong to the
	 * kernel, and are local to each engine. All other contexts will
	 * have their head/tail sanitized upon pinning before use, so they
	 * will never see garbage,
	 *
	 * So to avoid that we reset the context images upon resume. For
	 * simplicity, we just zero everything out.
	 */
	intel_ring_reset(ce->ring, 0);
	__execlists_update_reg_state(ce, ce->engine);
}

1606
static const struct intel_context_ops execlists_context_ops = {
1607
	.pin = execlists_context_pin,
1608
	.unpin = execlists_context_unpin,
1609

1610 1611 1612
	.enter = intel_context_enter_engine,
	.exit = intel_context_exit_engine,

1613
	.reset = execlists_context_reset,
1614 1615 1616
	.destroy = execlists_context_destroy,
};

1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641
static int gen8_emit_init_breadcrumb(struct i915_request *rq)
{
	u32 *cs;

	GEM_BUG_ON(!rq->timeline->has_initial_breadcrumb);

	cs = intel_ring_begin(rq, 6);
	if (IS_ERR(cs))
		return PTR_ERR(cs);

	/*
	 * Check if we have been preempted before we even get started.
	 *
	 * After this point i915_request_started() reports true, even if
	 * we get preempted and so are no longer running.
	 */
	*cs++ = MI_ARB_CHECK;
	*cs++ = MI_NOOP;

	*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
	*cs++ = rq->timeline->hwsp_offset;
	*cs++ = 0;
	*cs++ = rq->fence.seqno - 1;

	intel_ring_advance(rq, cs);
1642 1643 1644 1645

	/* Record the updated position of the request's payload */
	rq->infix = intel_ring_offset(rq, cs);

1646 1647 1648
	return 0;
}

1649 1650 1651
static int emit_pdps(struct i915_request *rq)
{
	const struct intel_engine_cs * const engine = rq->engine;
1652
	struct i915_ppgtt * const ppgtt =
1653
		i915_vm_to_ppgtt(rq->gem_context->vm);
1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683
	int err, i;
	u32 *cs;

	GEM_BUG_ON(intel_vgpu_active(rq->i915));

	/*
	 * Beware ye of the dragons, this sequence is magic!
	 *
	 * Small changes to this sequence can cause anything from
	 * GPU hangs to forcewake errors and machine lockups!
	 */

	/* Flush any residual operations from the context load */
	err = engine->emit_flush(rq, EMIT_FLUSH);
	if (err)
		return err;

	/* Magic required to prevent forcewake errors! */
	err = engine->emit_flush(rq, EMIT_INVALIDATE);
	if (err)
		return err;

	cs = intel_ring_begin(rq, 4 * GEN8_3LVL_PDPES + 2);
	if (IS_ERR(cs))
		return PTR_ERR(cs);

	/* Ensure the LRI have landed before we invalidate & continue */
	*cs++ = MI_LOAD_REGISTER_IMM(2 * GEN8_3LVL_PDPES) | MI_LRI_FORCE_POSTED;
	for (i = GEN8_3LVL_PDPES; i--; ) {
		const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);
1684
		u32 base = engine->mmio_base;
1685

1686
		*cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, i));
1687
		*cs++ = upper_32_bits(pd_daddr);
1688
		*cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, i));
1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703
		*cs++ = lower_32_bits(pd_daddr);
	}
	*cs++ = MI_NOOP;

	intel_ring_advance(rq, cs);

	/* Be doubly sure the LRI have landed before proceeding */
	err = engine->emit_flush(rq, EMIT_FLUSH);
	if (err)
		return err;

	/* Re-invalidate the TLB for luck */
	return engine->emit_flush(rq, EMIT_INVALIDATE);
}

1704
static int execlists_request_alloc(struct i915_request *request)
1705
{
1706
	int ret;
1707

1708
	GEM_BUG_ON(!intel_context_is_pinned(request->hw_context));
1709

1710 1711
	/*
	 * Flush enough space to reduce the likelihood of waiting after
1712 1713 1714 1715 1716
	 * we start building the request - in which case we will just
	 * have to repeat work.
	 */
	request->reserved_space += EXECLISTS_REQUEST_SIZE;

1717 1718
	/*
	 * Note that after this point, we have committed to using
1719 1720 1721 1722 1723 1724
	 * this request as it is being used to both track the
	 * state of engine initialisation and liveness of the
	 * golden renderstate above. Think twice before you try
	 * to cancel/unwind this request now.
	 */

1725
	/* Unconditionally invalidate GPU caches and TLBs. */
1726
	if (i915_vm_is_4lvl(request->gem_context->vm))
1727 1728 1729 1730 1731 1732
		ret = request->engine->emit_flush(request, EMIT_INVALIDATE);
	else
		ret = emit_pdps(request);
	if (ret)
		return ret;

1733 1734 1735 1736
	request->reserved_space -= EXECLISTS_REQUEST_SIZE;
	return 0;
}

1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752
/*
 * In this WA we need to set GEN8_L3SQCREG4[21:21] and reset it after
 * PIPE_CONTROL instruction. This is required for the flush to happen correctly
 * but there is a slight complication as this is applied in WA batch where the
 * values are only initialized once so we cannot take register value at the
 * beginning and reuse it further; hence we save its value to memory, upload a
 * constant value with bit21 set and then we restore it back with the saved value.
 * To simplify the WA, a constant value is formed by using the default value
 * of this register. This shouldn't be a problem because we are only modifying
 * it for a short period and this batch in non-premptible. We can ofcourse
 * use additional instructions that read the actual value of the register
 * at that time and set our bit of interest but it makes the WA complicated.
 *
 * This WA is also required for Gen9 so extracting as a function avoids
 * code duplication.
 */
1753 1754
static u32 *
gen8_emit_flush_coherentl3_wa(struct intel_engine_cs *engine, u32 *batch)
1755
{
1756
	/* NB no one else is allowed to scribble over scratch + 256! */
1757 1758
	*batch++ = MI_STORE_REGISTER_MEM_GEN8 | MI_SRM_LRM_GLOBAL_GTT;
	*batch++ = i915_mmio_reg_offset(GEN8_L3SQCREG4);
1759
	*batch++ = i915_scratch_offset(engine->i915) + 256;
1760 1761 1762 1763 1764 1765
	*batch++ = 0;

	*batch++ = MI_LOAD_REGISTER_IMM(1);
	*batch++ = i915_mmio_reg_offset(GEN8_L3SQCREG4);
	*batch++ = 0x40400000 | GEN8_LQSC_FLUSH_COHERENT_LINES;

1766 1767 1768 1769
	batch = gen8_emit_pipe_control(batch,
				       PIPE_CONTROL_CS_STALL |
				       PIPE_CONTROL_DC_FLUSH_ENABLE,
				       0);
1770 1771 1772

	*batch++ = MI_LOAD_REGISTER_MEM_GEN8 | MI_SRM_LRM_GLOBAL_GTT;
	*batch++ = i915_mmio_reg_offset(GEN8_L3SQCREG4);
1773
	*batch++ = i915_scratch_offset(engine->i915) + 256;
1774 1775 1776
	*batch++ = 0;

	return batch;
1777 1778
}

1779 1780 1781 1782 1783 1784
/*
 * Typically we only have one indirect_ctx and per_ctx batch buffer which are
 * initialized at the beginning and shared across all contexts but this field
 * helps us to have multiple batches at different offsets and select them based
 * on a criteria. At the moment this batch always start at the beginning of the page
 * and at this point we don't have multiple wa_ctx batch buffers.
1785
 *
1786 1787
 * The number of WA applied are not known at the beginning; we use this field
 * to return the no of DWORDS written.
1788
 *
1789 1790 1791 1792
 * It is to be noted that this batch does not contain MI_BATCH_BUFFER_END
 * so it adds NOOPs as padding to make it cacheline aligned.
 * MI_BATCH_BUFFER_END will be added to perctx batch and both of them together
 * makes a complete batch buffer.
1793
 */
1794
static u32 *gen8_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch)
1795
{
1796
	/* WaDisableCtxRestoreArbitration:bdw,chv */
1797
	*batch++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1798

1799
	/* WaFlushCoherentL3CacheLinesAtContextSwitch:bdw */
1800 1801
	if (IS_BROADWELL(engine->i915))
		batch = gen8_emit_flush_coherentl3_wa(engine, batch);
1802

1803 1804
	/* WaClearSlmSpaceAtContextSwitch:bdw,chv */
	/* Actual scratch location is at 128 bytes offset */
1805 1806 1807 1808 1809
	batch = gen8_emit_pipe_control(batch,
				       PIPE_CONTROL_FLUSH_L3 |
				       PIPE_CONTROL_GLOBAL_GTT_IVB |
				       PIPE_CONTROL_CS_STALL |
				       PIPE_CONTROL_QW_WRITE,
1810
				       i915_scratch_offset(engine->i915) +
1811
				       2 * CACHELINE_BYTES);
1812

1813 1814
	*batch++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;

1815
	/* Pad to end of cacheline */
1816 1817
	while ((unsigned long)batch % CACHELINE_BYTES)
		*batch++ = MI_NOOP;
1818 1819 1820 1821 1822 1823 1824

	/*
	 * MI_BATCH_BUFFER_END is not required in Indirect ctx BB because
	 * execution depends on the length specified in terms of cache lines
	 * in the register CTX_RCS_INDIRECT_CTX
	 */

1825
	return batch;
1826 1827
}

1828 1829 1830 1831 1832 1833
struct lri {
	i915_reg_t reg;
	u32 value;
};

static u32 *emit_lri(u32 *batch, const struct lri *lri, unsigned int count)
1834
{
1835
	GEM_BUG_ON(!count || count > 63);
1836

1837 1838 1839 1840 1841 1842
	*batch++ = MI_LOAD_REGISTER_IMM(count);
	do {
		*batch++ = i915_mmio_reg_offset(lri->reg);
		*batch++ = lri->value;
	} while (lri++, --count);
	*batch++ = MI_NOOP;
1843

1844 1845
	return batch;
}
1846

1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870
static u32 *gen9_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch)
{
	static const struct lri lri[] = {
		/* WaDisableGatherAtSetShaderCommonSlice:skl,bxt,kbl,glk */
		{
			COMMON_SLICE_CHICKEN2,
			__MASKED_FIELD(GEN9_DISABLE_GATHER_AT_SET_SHADER_COMMON_SLICE,
				       0),
		},

		/* BSpec: 11391 */
		{
			FF_SLICE_CHICKEN,
			__MASKED_FIELD(FF_SLICE_CHICKEN_CL_PROVOKING_VERTEX_FIX,
				       FF_SLICE_CHICKEN_CL_PROVOKING_VERTEX_FIX),
		},

		/* BSpec: 11299 */
		{
			_3D_CHICKEN3,
			__MASKED_FIELD(_3D_CHICKEN_SF_PROVOKING_VERTEX_FIX,
				       _3D_CHICKEN_SF_PROVOKING_VERTEX_FIX),
		}
	};
1871

1872
	*batch++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1873

1874 1875
	/* WaFlushCoherentL3CacheLinesAtContextSwitch:skl,bxt,glk */
	batch = gen8_emit_flush_coherentl3_wa(engine, batch);
1876

1877
	batch = emit_lri(batch, lri, ARRAY_SIZE(lri));
1878

1879
	/* WaMediaPoolStateCmdInWABB:bxt,glk */
1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893
	if (HAS_POOLED_EU(engine->i915)) {
		/*
		 * EU pool configuration is setup along with golden context
		 * during context initialization. This value depends on
		 * device type (2x6 or 3x6) and needs to be updated based
		 * on which subslice is disabled especially for 2x6
		 * devices, however it is safe to load default
		 * configuration of 3x6 device instead of masking off
		 * corresponding bits because HW ignores bits of a disabled
		 * subslice and drops down to appropriate config. Please
		 * see render_state_setup() in i915_gem_render_state.c for
		 * possible configurations, to avoid duplication they are
		 * not shown here again.
		 */
1894 1895 1896 1897 1898 1899
		*batch++ = GEN9_MEDIA_POOL_STATE;
		*batch++ = GEN9_MEDIA_POOL_ENABLE;
		*batch++ = 0x00777000;
		*batch++ = 0;
		*batch++ = 0;
		*batch++ = 0;
1900 1901
	}

1902 1903
	*batch++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;

1904
	/* Pad to end of cacheline */
1905 1906
	while ((unsigned long)batch % CACHELINE_BYTES)
		*batch++ = MI_NOOP;
1907

1908
	return batch;
1909 1910
}

1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944
static u32 *
gen10_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch)
{
	int i;

	/*
	 * WaPipeControlBefore3DStateSamplePattern: cnl
	 *
	 * Ensure the engine is idle prior to programming a
	 * 3DSTATE_SAMPLE_PATTERN during a context restore.
	 */
	batch = gen8_emit_pipe_control(batch,
				       PIPE_CONTROL_CS_STALL,
				       0);
	/*
	 * WaPipeControlBefore3DStateSamplePattern says we need 4 dwords for
	 * the PIPE_CONTROL followed by 12 dwords of 0x0, so 16 dwords in
	 * total. However, a PIPE_CONTROL is 6 dwords long, not 4, which is
	 * confusing. Since gen8_emit_pipe_control() already advances the
	 * batch by 6 dwords, we advance the other 10 here, completing a
	 * cacheline. It's not clear if the workaround requires this padding
	 * before other commands, or if it's just the regular padding we would
	 * already have for the workaround bb, so leave it here for now.
	 */
	for (i = 0; i < 10; i++)
		*batch++ = MI_NOOP;

	/* Pad to end of cacheline */
	while ((unsigned long)batch % CACHELINE_BYTES)
		*batch++ = MI_NOOP;

	return batch;
}

1945 1946 1947
#define CTX_WA_BB_OBJ_SIZE (PAGE_SIZE)

static int lrc_setup_wa_ctx(struct intel_engine_cs *engine)
1948
{
1949 1950 1951
	struct drm_i915_gem_object *obj;
	struct i915_vma *vma;
	int err;
1952

1953
	obj = i915_gem_object_create_shmem(engine->i915, CTX_WA_BB_OBJ_SIZE);
1954 1955
	if (IS_ERR(obj))
		return PTR_ERR(obj);
1956

1957
	vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL);
1958 1959 1960
	if (IS_ERR(vma)) {
		err = PTR_ERR(vma);
		goto err;
1961 1962
	}

1963
	err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
1964 1965 1966 1967
	if (err)
		goto err;

	engine->wa_ctx.vma = vma;
1968
	return 0;
1969 1970 1971 1972

err:
	i915_gem_object_put(obj);
	return err;
1973 1974
}

1975
static void lrc_destroy_wa_ctx(struct intel_engine_cs *engine)
1976
{
1977
	i915_vma_unpin_and_release(&engine->wa_ctx.vma, 0);
1978 1979
}

1980 1981
typedef u32 *(*wa_bb_func_t)(struct intel_engine_cs *engine, u32 *batch);

1982
static int intel_init_workaround_bb(struct intel_engine_cs *engine)
1983
{
1984
	struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx;
1985 1986 1987
	struct i915_wa_ctx_bb *wa_bb[2] = { &wa_ctx->indirect_ctx,
					    &wa_ctx->per_ctx };
	wa_bb_func_t wa_bb_fn[2];
1988
	struct page *page;
1989 1990
	void *batch, *batch_ptr;
	unsigned int i;
1991
	int ret;
1992

1993 1994
	if (engine->class != RENDER_CLASS)
		return 0;
1995

1996
	switch (INTEL_GEN(engine->i915)) {
1997 1998
	case 11:
		return 0;
1999
	case 10:
2000 2001 2002
		wa_bb_fn[0] = gen10_init_indirectctx_bb;
		wa_bb_fn[1] = NULL;
		break;
2003 2004
	case 9:
		wa_bb_fn[0] = gen9_init_indirectctx_bb;
2005
		wa_bb_fn[1] = NULL;
2006 2007 2008
		break;
	case 8:
		wa_bb_fn[0] = gen8_init_indirectctx_bb;
2009
		wa_bb_fn[1] = NULL;
2010 2011 2012
		break;
	default:
		MISSING_CASE(INTEL_GEN(engine->i915));
2013
		return 0;
2014
	}
2015

2016
	ret = lrc_setup_wa_ctx(engine);
2017 2018 2019 2020 2021
	if (ret) {
		DRM_DEBUG_DRIVER("Failed to setup context WA page: %d\n", ret);
		return ret;
	}

2022
	page = i915_gem_object_get_dirty_page(wa_ctx->vma->obj, 0);
2023
	batch = batch_ptr = kmap_atomic(page);
2024

2025 2026 2027 2028 2029 2030 2031
	/*
	 * Emit the two workaround batch buffers, recording the offset from the
	 * start of the workaround batch buffer object for each and their
	 * respective sizes.
	 */
	for (i = 0; i < ARRAY_SIZE(wa_bb_fn); i++) {
		wa_bb[i]->offset = batch_ptr - batch;
2032 2033
		if (GEM_DEBUG_WARN_ON(!IS_ALIGNED(wa_bb[i]->offset,
						  CACHELINE_BYTES))) {
2034 2035 2036
			ret = -EINVAL;
			break;
		}
2037 2038
		if (wa_bb_fn[i])
			batch_ptr = wa_bb_fn[i](engine, batch_ptr);
2039
		wa_bb[i]->size = batch_ptr - (batch + wa_bb[i]->offset);
2040 2041
	}

2042 2043
	BUG_ON(batch_ptr - batch > CTX_WA_BB_OBJ_SIZE);

2044 2045
	kunmap_atomic(batch);
	if (ret)
2046
		lrc_destroy_wa_ctx(engine);
2047 2048 2049 2050

	return ret;
}

2051
static void enable_execlists(struct intel_engine_cs *engine)
2052
{
2053
	intel_engine_set_hwsp_writemask(engine, ~0u); /* HWSTAM */
2054

2055 2056 2057 2058
	if (INTEL_GEN(engine->i915) >= 11)
		ENGINE_WRITE(engine,
			     RING_MODE_GEN7,
			     _MASKED_BIT_ENABLE(GEN11_GFX_DISABLE_LEGACY_MODE));
2059
	else
2060 2061 2062
		ENGINE_WRITE(engine,
			     RING_MODE_GEN7,
			     _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE));
2063

2064
	ENGINE_WRITE(engine, RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING));
2065

2066 2067 2068 2069
	ENGINE_WRITE(engine,
		     RING_HWS_PGA,
		     i915_ggtt_offset(engine->status_page.vma));
	ENGINE_POSTING_READ(engine, RING_HWS_PGA);
2070 2071
}

2072 2073 2074 2075
static bool unexpected_starting_state(struct intel_engine_cs *engine)
{
	bool unexpected = false;

2076
	if (ENGINE_READ(engine, RING_MI_MODE) & STOP_RING) {
2077 2078 2079 2080 2081 2082 2083
		DRM_DEBUG_DRIVER("STOP_RING still set in RING_MI_MODE\n");
		unexpected = true;
	}

	return unexpected;
}

2084
static int execlists_resume(struct intel_engine_cs *engine)
2085
{
2086
	intel_engine_apply_workarounds(engine);
2087
	intel_engine_apply_whitelist(engine);
2088

2089
	intel_mocs_init_engine(engine);
2090

2091
	intel_engine_reset_breadcrumbs(engine);
2092

2093 2094 2095 2096 2097 2098
	if (GEM_SHOW_DEBUG() && unexpected_starting_state(engine)) {
		struct drm_printer p = drm_debug_printer(__func__);

		intel_engine_dump(engine, &p, NULL);
	}

2099
	enable_execlists(engine);
2100

2101
	return 0;
2102 2103
}

2104
static void execlists_reset_prepare(struct intel_engine_cs *engine)
2105 2106
{
	struct intel_engine_execlists * const execlists = &engine->execlists;
2107
	unsigned long flags;
2108

2109 2110
	GEM_TRACE("%s: depth<-%d\n", engine->name,
		  atomic_read(&execlists->tasklet.count));
2111 2112 2113 2114 2115 2116

	/*
	 * Prevent request submission to the hardware until we have
	 * completed the reset in i915_gem_reset_finish(). If a request
	 * is completed by one engine, it may then queue a request
	 * to a second via its execlists->tasklet *just* as we are
2117
	 * calling engine->resume() and also writing the ELSP.
2118 2119 2120 2121
	 * Turning off the execlists->tasklet until the reset is over
	 * prevents the race.
	 */
	__tasklet_disable_sync_once(&execlists->tasklet);
2122
	GEM_BUG_ON(!reset_in_progress(execlists));
2123

2124 2125
	intel_engine_stop_cs(engine);

2126
	/* And flush any current direct submission. */
2127 2128
	spin_lock_irqsave(&engine->active.lock, flags);
	spin_unlock_irqrestore(&engine->active.lock, flags);
2129 2130
}

2131
static void reset_csb_pointers(struct intel_engine_cs *engine)
2132
{
2133
	struct intel_engine_execlists * const execlists = &engine->execlists;
2134 2135
	const unsigned int reset_value = execlists->csb_size - 1;

2136 2137
	ring_set_paused(engine, 0);

2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148
	/*
	 * After a reset, the HW starts writing into CSB entry [0]. We
	 * therefore have to set our HEAD pointer back one entry so that
	 * the *first* entry we check is entry 0. To complicate this further,
	 * as we don't wait for the first interrupt after reset, we have to
	 * fake the HW write to point back to the last entry so that our
	 * inline comparison of our cached head position against the last HW
	 * write works even before the first interrupt.
	 */
	execlists->csb_head = reset_value;
	WRITE_ONCE(*execlists->csb_write, reset_value);
2149
	wmb(); /* Make sure this is visible to HW (paranoia?) */
2150 2151 2152 2153 2154

	invalidate_csb_entries(&execlists->csb_status[0],
			       &execlists->csb_status[reset_value]);
}

2155 2156
static struct i915_request *active_request(struct i915_request *rq)
{
2157
	const struct list_head * const list = &rq->engine->active.requests;
2158 2159 2160
	const struct intel_context * const context = rq->hw_context;
	struct i915_request *active = NULL;

2161
	list_for_each_entry_from_reverse(rq, list, sched.link) {
2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173
		if (i915_request_completed(rq))
			break;

		if (rq->hw_context != context)
			break;

		active = rq;
	}

	return active;
}

2174
static void __execlists_reset(struct intel_engine_cs *engine, bool stalled)
2175
{
2176
	struct intel_engine_execlists * const execlists = &engine->execlists;
2177
	struct intel_context *ce;
2178
	struct i915_request *rq;
2179
	u32 *regs;
2180

2181 2182 2183
	process_csb(engine); /* drain preemption events */

	/* Following the reset, we need to reload the CSB read/write pointers */
2184
	reset_csb_pointers(engine);
2185 2186 2187 2188 2189 2190

	/*
	 * Save the currently executing context, even if we completed
	 * its request, it was still running at the time of the
	 * reset and will have been clobbered.
	 */
2191 2192 2193
	rq = execlists_active(execlists);
	if (!rq)
		return;
2194

2195
	ce = rq->hw_context;
2196 2197 2198
	GEM_BUG_ON(i915_active_is_idle(&ce->active));
	GEM_BUG_ON(!i915_vma_is_pinned(ce->state));
	rq = active_request(rq);
2199

2200 2201 2202 2203 2204 2205 2206 2207 2208
	/*
	 * Catch up with any missed context-switch interrupts.
	 *
	 * Ideally we would just read the remaining CSB entries now that we
	 * know the gpu is idle. However, the CSB registers are sometimes^W
	 * often trashed across a GPU reset! Instead we have to rely on
	 * guessing the missed context-switch events by looking at what
	 * requests were completed.
	 */
2209
	execlists_cancel_port_requests(execlists);
2210

2211 2212
	if (!rq) {
		ce->ring->head = ce->ring->tail;
2213
		goto out_replay;
2214 2215 2216
	}

	ce->ring->head = intel_ring_wrap(ce->ring, rq->head);
2217

2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229
	/*
	 * If this request hasn't started yet, e.g. it is waiting on a
	 * semaphore, we need to avoid skipping the request or else we
	 * break the signaling chain. However, if the context is corrupt
	 * the request will not restart and we will be stuck with a wedged
	 * device. It is quite often the case that if we issue a reset
	 * while the GPU is loading the context image, that the context
	 * image becomes corrupt.
	 *
	 * Otherwise, if we have not started yet, the request should replay
	 * perfectly and we do not need to flag the result as being erroneous.
	 */
2230
	if (!i915_request_started(rq))
2231
		goto out_replay;
2232

2233 2234
	/*
	 * If the request was innocent, we leave the request in the ELSP
2235 2236 2237 2238 2239 2240 2241 2242 2243
	 * and will try to replay it on restarting. The context image may
	 * have been corrupted by the reset, in which case we may have
	 * to service a new GPU hang, but more likely we can continue on
	 * without impact.
	 *
	 * If the request was guilty, we presume the context is corrupt
	 * and have to at least restore the RING register in the context
	 * image back to the expected values to skip over the guilty request.
	 */
2244
	i915_reset_request(rq, stalled);
2245
	if (!stalled)
2246
		goto out_replay;
2247

2248 2249
	/*
	 * We want a simple context + ring to execute the breadcrumb update.
2250 2251 2252 2253 2254 2255
	 * We cannot rely on the context being intact across the GPU hang,
	 * so clear it and rebuild just what we need for the breadcrumb.
	 * All pending requests for this context will be zapped, and any
	 * future request will be after userspace has had the opportunity
	 * to recreate its own state.
	 */
2256
	regs = ce->lrc_reg_state;
2257 2258 2259 2260
	if (engine->pinned_default_state) {
		memcpy(regs, /* skip restoring the vanilla PPHWSP */
		       engine->pinned_default_state + LRC_STATE_PN * PAGE_SIZE,
		       engine->context_size - PAGE_SIZE);
2261
	}
2262
	execlists_init_reg_state(regs, ce, engine, ce->ring);
2263

2264
out_replay:
2265 2266
	GEM_TRACE("%s replay {head:%04x, tail:%04x\n",
		  engine->name, ce->ring->head, ce->ring->tail);
2267 2268 2269
	intel_ring_update_space(ce->ring);
	__execlists_update_reg_state(ce, engine);

2270 2271
	/* Push back any incomplete requests for replay after the reset. */
	__unwind_incomplete_requests(engine);
2272
}
2273

2274 2275 2276 2277 2278 2279
static void execlists_reset(struct intel_engine_cs *engine, bool stalled)
{
	unsigned long flags;

	GEM_TRACE("%s\n", engine->name);

2280
	spin_lock_irqsave(&engine->active.lock, flags);
2281 2282 2283

	__execlists_reset(engine, stalled);

2284
	spin_unlock_irqrestore(&engine->active.lock, flags);
2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314
}

static void nop_submission_tasklet(unsigned long data)
{
	/* The driver is wedged; don't process any more events. */
}

static void execlists_cancel_requests(struct intel_engine_cs *engine)
{
	struct intel_engine_execlists * const execlists = &engine->execlists;
	struct i915_request *rq, *rn;
	struct rb_node *rb;
	unsigned long flags;

	GEM_TRACE("%s\n", engine->name);

	/*
	 * Before we call engine->cancel_requests(), we should have exclusive
	 * access to the submission state. This is arranged for us by the
	 * caller disabling the interrupt generation, the tasklet and other
	 * threads that may then access the same state, giving us a free hand
	 * to reset state. However, we still need to let lockdep be aware that
	 * we know this state may be accessed in hardirq context, so we
	 * disable the irq around this manipulation and we want to keep
	 * the spinlock focused on its duties and not accidentally conflate
	 * coverage to the submission's irq state. (Similarly, although we
	 * shouldn't need to disable irq around the manipulation of the
	 * submission's irq state, we also wish to remind ourselves that
	 * it is irq state.)
	 */
2315
	spin_lock_irqsave(&engine->active.lock, flags);
2316 2317 2318 2319

	__execlists_reset(engine, true);

	/* Mark all executing requests as skipped. */
2320
	list_for_each_entry(rq, &engine->active.requests, sched.link) {
2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342
		if (!i915_request_signaled(rq))
			dma_fence_set_error(&rq->fence, -EIO);

		i915_request_mark_complete(rq);
	}

	/* Flush the queued requests to the timeline list (for retiring). */
	while ((rb = rb_first_cached(&execlists->queue))) {
		struct i915_priolist *p = to_priolist(rb);
		int i;

		priolist_for_each_request_consume(rq, rn, p, i) {
			list_del_init(&rq->sched.link);
			__i915_request_submit(rq);
			dma_fence_set_error(&rq->fence, -EIO);
			i915_request_mark_complete(rq);
		}

		rb_erase_cached(&p->node, &execlists->queue);
		i915_priolist_free(p);
	}

2343 2344 2345 2346 2347 2348 2349 2350
	/* Cancel all attached virtual engines */
	while ((rb = rb_first_cached(&execlists->virtual))) {
		struct virtual_engine *ve =
			rb_entry(rb, typeof(*ve), nodes[engine->id].rb);

		rb_erase_cached(rb, &execlists->virtual);
		RB_CLEAR_NODE(rb);

2351
		spin_lock(&ve->base.active.lock);
2352 2353 2354 2355 2356 2357 2358 2359
		if (ve->request) {
			ve->request->engine = engine;
			__i915_request_submit(ve->request);
			dma_fence_set_error(&ve->request->fence, -EIO);
			i915_request_mark_complete(ve->request);
			ve->base.execlists.queue_priority_hint = INT_MIN;
			ve->request = NULL;
		}
2360
		spin_unlock(&ve->base.active.lock);
2361 2362
	}

2363 2364 2365 2366 2367 2368 2369
	/* Remaining _unready_ requests will be nop'ed when submitted */

	execlists->queue_priority_hint = INT_MIN;
	execlists->queue = RB_ROOT_CACHED;

	GEM_BUG_ON(__tasklet_is_enabled(&execlists->tasklet));
	execlists->tasklet.func = nop_submission_tasklet;
2370

2371
	spin_unlock_irqrestore(&engine->active.lock, flags);
2372 2373
}

2374 2375
static void execlists_reset_finish(struct intel_engine_cs *engine)
{
2376 2377
	struct intel_engine_execlists * const execlists = &engine->execlists;

2378
	/*
2379 2380 2381
	 * After a GPU reset, we may have requests to replay. Do so now while
	 * we still have the forcewake to be sure that the GPU is not allowed
	 * to sleep before we restart and reload a context.
2382
	 */
2383
	GEM_BUG_ON(!reset_in_progress(execlists));
2384 2385
	if (!RB_EMPTY_ROOT(&execlists->queue.rb_root))
		execlists->tasklet.func(execlists->tasklet.data);
2386

2387 2388 2389
	if (__tasklet_enable(&execlists->tasklet))
		/* And kick in case we missed a new request submission. */
		tasklet_hi_schedule(&execlists->tasklet);
2390 2391
	GEM_TRACE("%s: depth->%d\n", engine->name,
		  atomic_read(&execlists->tasklet.count));
2392 2393
}

2394
static int gen8_emit_bb_start(struct i915_request *rq,
2395
			      u64 offset, u32 len,
2396
			      const unsigned int flags)
2397
{
2398
	u32 *cs;
2399

2400
	cs = intel_ring_begin(rq, 4);
2401 2402
	if (IS_ERR(cs))
		return PTR_ERR(cs);
2403

2404 2405 2406 2407 2408 2409 2410
	/*
	 * WaDisableCtxRestoreArbitration:bdw,chv
	 *
	 * We don't need to perform MI_ARB_ENABLE as often as we do (in
	 * particular all the gen that do not need the w/a at all!), if we
	 * took care to make sure that on every switch into this context
	 * (both ordinary and for preemption) that arbitrartion was enabled
2411 2412 2413 2414 2415
	 * we would be fine.  However, for gen8 there is another w/a that
	 * requires us to not preempt inside GPGPU execution, so we keep
	 * arbitration disabled for gen8 batches. Arbitration will be
	 * re-enabled before we close the request
	 * (engine->emit_fini_breadcrumb).
2416
	 */
2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439
	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;

	/* FIXME(BDW+): Address space and security selectors. */
	*cs++ = MI_BATCH_BUFFER_START_GEN8 |
		(flags & I915_DISPATCH_SECURE ? 0 : BIT(8));
	*cs++ = lower_32_bits(offset);
	*cs++ = upper_32_bits(offset);

	intel_ring_advance(rq, cs);

	return 0;
}

static int gen9_emit_bb_start(struct i915_request *rq,
			      u64 offset, u32 len,
			      const unsigned int flags)
{
	u32 *cs;

	cs = intel_ring_begin(rq, 6);
	if (IS_ERR(cs))
		return PTR_ERR(cs);

2440 2441
	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;

2442
	*cs++ = MI_BATCH_BUFFER_START_GEN8 |
2443
		(flags & I915_DISPATCH_SECURE ? 0 : BIT(8));
2444 2445
	*cs++ = lower_32_bits(offset);
	*cs++ = upper_32_bits(offset);
2446 2447 2448

	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
	*cs++ = MI_NOOP;
2449

2450
	intel_ring_advance(rq, cs);
2451 2452 2453 2454

	return 0;
}

2455
static void gen8_logical_ring_enable_irq(struct intel_engine_cs *engine)
2456
{
2457 2458 2459
	ENGINE_WRITE(engine, RING_IMR,
		     ~(engine->irq_enable_mask | engine->irq_keep_mask));
	ENGINE_POSTING_READ(engine, RING_IMR);
2460 2461
}

2462
static void gen8_logical_ring_disable_irq(struct intel_engine_cs *engine)
2463
{
2464
	ENGINE_WRITE(engine, RING_IMR, ~engine->irq_keep_mask);
2465 2466
}

2467
static int gen8_emit_flush(struct i915_request *request, u32 mode)
2468
{
2469
	u32 cmd, *cs;
2470

2471 2472 2473
	cs = intel_ring_begin(request, 4);
	if (IS_ERR(cs))
		return PTR_ERR(cs);
2474 2475 2476

	cmd = MI_FLUSH_DW + 1;

2477 2478 2479 2480 2481 2482 2483
	/* We always require a command barrier so that subsequent
	 * commands, such as breadcrumb interrupts, are strictly ordered
	 * wrt the contents of the write cache being flushed to memory
	 * (and thus being coherent from the CPU).
	 */
	cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;

2484
	if (mode & EMIT_INVALIDATE) {
2485
		cmd |= MI_INVALIDATE_TLB;
2486
		if (request->engine->class == VIDEO_DECODE_CLASS)
2487
			cmd |= MI_INVALIDATE_BSD;
2488 2489
	}

2490 2491 2492 2493 2494
	*cs++ = cmd;
	*cs++ = I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT;
	*cs++ = 0; /* upper addr */
	*cs++ = 0; /* value */
	intel_ring_advance(request, cs);
2495 2496 2497 2498

	return 0;
}

2499
static int gen8_emit_flush_render(struct i915_request *request,
2500
				  u32 mode)
2501
{
2502
	struct intel_engine_cs *engine = request->engine;
2503
	u32 scratch_addr =
2504
		i915_scratch_offset(engine->i915) + 2 * CACHELINE_BYTES;
2505
	bool vf_flush_wa = false, dc_flush_wa = false;
2506
	u32 *cs, flags = 0;
2507
	int len;
2508 2509 2510

	flags |= PIPE_CONTROL_CS_STALL;

2511
	if (mode & EMIT_FLUSH) {
2512 2513
		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
2514
		flags |= PIPE_CONTROL_DC_FLUSH_ENABLE;
2515
		flags |= PIPE_CONTROL_FLUSH_ENABLE;
2516 2517
	}

2518
	if (mode & EMIT_INVALIDATE) {
2519 2520 2521 2522 2523 2524 2525 2526 2527
		flags |= PIPE_CONTROL_TLB_INVALIDATE;
		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_QW_WRITE;
		flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;

2528 2529 2530 2531
		/*
		 * On GEN9: before VF_CACHE_INVALIDATE we need to emit a NULL
		 * pipe control.
		 */
2532
		if (IS_GEN(request->i915, 9))
2533
			vf_flush_wa = true;
2534 2535 2536 2537

		/* WaForGAMHang:kbl */
		if (IS_KBL_REVID(request->i915, 0, KBL_REVID_B0))
			dc_flush_wa = true;
2538
	}
2539

2540 2541 2542 2543 2544 2545 2546 2547
	len = 6;

	if (vf_flush_wa)
		len += 6;

	if (dc_flush_wa)
		len += 12;

2548 2549 2550
	cs = intel_ring_begin(request, len);
	if (IS_ERR(cs))
		return PTR_ERR(cs);
2551

2552 2553
	if (vf_flush_wa)
		cs = gen8_emit_pipe_control(cs, 0, 0);
2554

2555 2556 2557
	if (dc_flush_wa)
		cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_DC_FLUSH_ENABLE,
					    0);
2558

2559
	cs = gen8_emit_pipe_control(cs, flags, scratch_addr);
2560

2561 2562
	if (dc_flush_wa)
		cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_CS_STALL, 0);
2563

2564
	intel_ring_advance(request, cs);
2565 2566 2567 2568

	return 0;
}

2569 2570 2571 2572 2573
/*
 * Reserve space for 2 NOOPs at the end of each request to be
 * used as a workaround for not being allowed to do lite
 * restore with HEAD==TAIL (WaIdleLiteRestore).
 */
2574
static u32 *gen8_emit_wa_tail(struct i915_request *request, u32 *cs)
2575
{
2576 2577
	/* Ensure there's always at least one preemption point per-request. */
	*cs++ = MI_ARB_CHECK;
2578 2579
	*cs++ = MI_NOOP;
	request->wa_tail = intel_ring_offset(request, cs);
2580 2581

	return cs;
2582
}
2583

2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596
static u32 *emit_preempt_busywait(struct i915_request *request, u32 *cs)
{
	*cs++ = MI_SEMAPHORE_WAIT |
		MI_SEMAPHORE_GLOBAL_GTT |
		MI_SEMAPHORE_POLL |
		MI_SEMAPHORE_SAD_EQ_SDD;
	*cs++ = 0;
	*cs++ = intel_hws_preempt_address(request->engine);
	*cs++ = 0;

	return cs;
}

2597
static u32 *gen8_emit_fini_breadcrumb(struct i915_request *request, u32 *cs)
2598
{
2599 2600
	cs = gen8_emit_ggtt_write(cs,
				  request->fence.seqno,
2601 2602
				  request->timeline->hwsp_offset,
				  0);
2603
	*cs++ = MI_USER_INTERRUPT;
2604

2605
	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
2606
	cs = emit_preempt_busywait(request, cs);
2607

2608
	request->tail = intel_ring_offset(request, cs);
2609
	assert_ring_tail_valid(request->ring, request->tail);
2610

2611
	return gen8_emit_wa_tail(request, cs);
2612
}
2613

2614
static u32 *gen8_emit_fini_breadcrumb_rcs(struct i915_request *request, u32 *cs)
2615
{
2616
	/* XXX flush+write+CS_STALL all in one upsets gem_concurrent_blt:kbl */
2617
	cs = gen8_emit_ggtt_write_rcs(cs,
2618 2619
				      request->fence.seqno,
				      request->timeline->hwsp_offset,
2620 2621
				      PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH |
				      PIPE_CONTROL_DEPTH_CACHE_FLUSH |
2622 2623 2624 2625 2626
				      PIPE_CONTROL_DC_FLUSH_ENABLE);
	cs = gen8_emit_pipe_control(cs,
				    PIPE_CONTROL_FLUSH_ENABLE |
				    PIPE_CONTROL_CS_STALL,
				    0);
2627
	*cs++ = MI_USER_INTERRUPT;
2628

2629
	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
2630
	cs = emit_preempt_busywait(request, cs);
2631

2632
	request->tail = intel_ring_offset(request, cs);
2633
	assert_ring_tail_valid(request->ring, request->tail);
2634

2635
	return gen8_emit_wa_tail(request, cs);
2636
}
2637

2638
static int gen8_init_rcs_context(struct i915_request *rq)
2639 2640 2641
{
	int ret;

2642
	ret = intel_engine_emit_ctx_wa(rq);
2643 2644 2645
	if (ret)
		return ret;

2646
	ret = intel_rcs_context_init_mocs(rq);
2647 2648 2649 2650 2651 2652 2653
	/*
	 * Failing to program the MOCS is non-fatal.The system will not
	 * run at peak performance. So generate an error and carry on.
	 */
	if (ret)
		DRM_ERROR("MOCS failed to program: expect performance issues.\n");

2654
	return i915_gem_render_state_emit(rq);
2655 2656
}

2657 2658
static void execlists_park(struct intel_engine_cs *engine)
{
2659
	del_timer_sync(&engine->execlists.timer);
2660 2661 2662
	intel_engine_park(engine);
}

2663
void intel_execlists_set_default_submission(struct intel_engine_cs *engine)
2664
{
2665
	engine->submit_request = execlists_submit_request;
2666
	engine->cancel_requests = execlists_cancel_requests;
2667
	engine->schedule = i915_schedule;
2668
	engine->execlists.tasklet.func = execlists_submission_tasklet;
2669

2670
	engine->reset.prepare = execlists_reset_prepare;
2671 2672
	engine->reset.reset = execlists_reset;
	engine->reset.finish = execlists_reset_finish;
2673

2674
	engine->park = execlists_park;
2675
	engine->unpark = NULL;
2676 2677

	engine->flags |= I915_ENGINE_SUPPORTS_STATS;
2678 2679
	if (!intel_vgpu_active(engine->i915))
		engine->flags |= I915_ENGINE_HAS_SEMAPHORES;
2680
	if (HAS_LOGICAL_RING_PREEMPTION(engine->i915))
2681
		engine->flags |= I915_ENGINE_HAS_PREEMPTION;
2682 2683
}

2684 2685 2686 2687 2688 2689 2690
static void execlists_destroy(struct intel_engine_cs *engine)
{
	intel_engine_cleanup_common(engine);
	lrc_destroy_wa_ctx(engine);
	kfree(engine);
}

2691
static void
2692
logical_ring_default_vfuncs(struct intel_engine_cs *engine)
2693 2694
{
	/* Default vfuncs which can be overriden by each engine. */
2695 2696

	engine->destroy = execlists_destroy;
2697
	engine->resume = execlists_resume;
2698 2699 2700 2701

	engine->reset.prepare = execlists_reset_prepare;
	engine->reset.reset = execlists_reset;
	engine->reset.finish = execlists_reset_finish;
2702

2703
	engine->cops = &execlists_context_ops;
2704 2705
	engine->request_alloc = execlists_request_alloc;

2706
	engine->emit_flush = gen8_emit_flush;
2707 2708
	engine->emit_init_breadcrumb = gen8_emit_init_breadcrumb;
	engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb;
2709

2710
	engine->set_default_submission = intel_execlists_set_default_submission;
2711

2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722
	if (INTEL_GEN(engine->i915) < 11) {
		engine->irq_enable = gen8_logical_ring_enable_irq;
		engine->irq_disable = gen8_logical_ring_disable_irq;
	} else {
		/*
		 * TODO: On Gen11 interrupt masks need to be clear
		 * to allow C6 entry. Keep interrupts enabled at
		 * and take the hit of generating extra interrupts
		 * until a more refined solution exists.
		 */
	}
2723 2724 2725 2726
	if (IS_GEN(engine->i915, 8))
		engine->emit_bb_start = gen8_emit_bb_start;
	else
		engine->emit_bb_start = gen9_emit_bb_start;
2727 2728
}

2729
static inline void
2730
logical_ring_default_irqs(struct intel_engine_cs *engine)
2731
{
2732 2733 2734 2735
	unsigned int shift = 0;

	if (INTEL_GEN(engine->i915) < 11) {
		const u8 irq_shifts[] = {
2736 2737 2738 2739 2740
			[RCS0]  = GEN8_RCS_IRQ_SHIFT,
			[BCS0]  = GEN8_BCS_IRQ_SHIFT,
			[VCS0]  = GEN8_VCS0_IRQ_SHIFT,
			[VCS1]  = GEN8_VCS1_IRQ_SHIFT,
			[VECS0] = GEN8_VECS_IRQ_SHIFT,
2741 2742 2743 2744 2745
		};

		shift = irq_shifts[engine->id];
	}

2746 2747
	engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << shift;
	engine->irq_keep_mask = GT_CONTEXT_SWITCH_INTERRUPT << shift;
2748 2749
}

2750
int intel_execlists_submission_setup(struct intel_engine_cs *engine)
2751 2752 2753 2754
{
	/* Intentionally left blank. */
	engine->buffer = NULL;

2755 2756
	tasklet_init(&engine->execlists.tasklet,
		     execlists_submission_tasklet, (unsigned long)engine);
2757
	timer_setup(&engine->execlists.timer, execlists_submission_timer, 0);
2758 2759 2760

	logical_ring_default_vfuncs(engine);
	logical_ring_default_irqs(engine);
2761

2762 2763 2764 2765 2766 2767
	if (engine->class == RENDER_CLASS) {
		engine->init_context = gen8_init_rcs_context;
		engine->emit_flush = gen8_emit_flush_render;
		engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb_rcs;
	}

2768
	return 0;
2769 2770
}

2771
int intel_execlists_submission_init(struct intel_engine_cs *engine)
2772
{
2773
	struct intel_engine_execlists * const execlists = &engine->execlists;
2774 2775
	struct drm_i915_private *i915 = engine->i915;
	struct intel_uncore *uncore = engine->uncore;
2776
	u32 base = engine->mmio_base;
2777 2778
	int ret;

2779
	ret = intel_engine_init_common(engine);
2780
	if (ret)
2781
		return ret;
2782

2783
	intel_engine_init_workarounds(engine);
2784 2785 2786 2787 2788 2789 2790 2791 2792
	intel_engine_init_whitelist(engine);

	if (intel_init_workaround_bb(engine))
		/*
		 * We continue even if we fail to initialize WA batch
		 * because we only expect rare glitches but nothing
		 * critical to prevent us from using GPU
		 */
		DRM_ERROR("WA batch buffer initialization failed\n");
2793

2794
	if (HAS_LOGICAL_RING_ELSQ(i915)) {
2795
		execlists->submit_reg = uncore->regs +
2796
			i915_mmio_reg_offset(RING_EXECLIST_SQ_CONTENTS(base));
2797
		execlists->ctrl_reg = uncore->regs +
2798
			i915_mmio_reg_offset(RING_EXECLIST_CONTROL(base));
2799
	} else {
2800
		execlists->submit_reg = uncore->regs +
2801
			i915_mmio_reg_offset(RING_ELSP(base));
2802
	}
2803

2804
	execlists->csb_status =
2805
		&engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX];
2806

2807
	execlists->csb_write =
2808
		&engine->status_page.addr[intel_hws_csb_write_index(i915)];
2809

2810
	if (INTEL_GEN(i915) < 11)
2811 2812 2813
		execlists->csb_size = GEN8_CSB_ENTRIES;
	else
		execlists->csb_size = GEN11_CSB_ENTRIES;
2814

2815
	reset_csb_pointers(engine);
2816

2817 2818 2819
	return 0;
}

2820
static u32 intel_lr_indirect_ctx_offset(struct intel_engine_cs *engine)
2821 2822 2823
{
	u32 indirect_ctx_offset;

2824
	switch (INTEL_GEN(engine->i915)) {
2825
	default:
2826
		MISSING_CASE(INTEL_GEN(engine->i915));
2827
		/* fall through */
2828 2829 2830 2831
	case 11:
		indirect_ctx_offset =
			GEN11_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT;
		break;
2832 2833 2834 2835
	case 10:
		indirect_ctx_offset =
			GEN10_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT;
		break;
2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848
	case 9:
		indirect_ctx_offset =
			GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT;
		break;
	case 8:
		indirect_ctx_offset =
			GEN8_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT;
		break;
	}

	return indirect_ctx_offset;
}

2849
static void execlists_init_reg_state(u32 *regs,
2850
				     struct intel_context *ce,
2851 2852
				     struct intel_engine_cs *engine,
				     struct intel_ring *ring)
2853
{
2854
	struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(ce->gem_context->vm);
2855
	bool rcs = engine->class == RENDER_CLASS;
2856
	u32 base = engine->mmio_base;
2857

2858 2859
	/*
	 * A context is actually a big batch buffer with several
2860 2861 2862 2863 2864
	 * MI_LOAD_REGISTER_IMM commands followed by (reg, value) pairs. The
	 * values we are setting here are only for the first context restore:
	 * on a subsequent save, the GPU will recreate this batchbuffer with new
	 * values (including all the missing MI_LOAD_REGISTER_IMM commands that
	 * we are not initializing here).
2865 2866
	 *
	 * Must keep consistent with virtual_update_register_offsets().
2867 2868 2869 2870
	 */
	regs[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(rcs ? 14 : 11) |
				 MI_LRI_FORCE_POSTED;

2871
	CTX_REG(regs, CTX_CONTEXT_CONTROL, RING_CONTEXT_CONTROL(base),
2872
		_MASKED_BIT_DISABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT) |
2873
		_MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH));
2874
	if (INTEL_GEN(engine->i915) < 11) {
2875 2876 2877 2878
		regs[CTX_CONTEXT_CONTROL + 1] |=
			_MASKED_BIT_DISABLE(CTX_CTRL_ENGINE_CTX_SAVE_INHIBIT |
					    CTX_CTRL_RS_CTX_ENABLE);
	}
2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890
	CTX_REG(regs, CTX_RING_HEAD, RING_HEAD(base), 0);
	CTX_REG(regs, CTX_RING_TAIL, RING_TAIL(base), 0);
	CTX_REG(regs, CTX_RING_BUFFER_START, RING_START(base), 0);
	CTX_REG(regs, CTX_RING_BUFFER_CONTROL, RING_CTL(base),
		RING_CTL_SIZE(ring->size) | RING_VALID);
	CTX_REG(regs, CTX_BB_HEAD_U, RING_BBADDR_UDW(base), 0);
	CTX_REG(regs, CTX_BB_HEAD_L, RING_BBADDR(base), 0);
	CTX_REG(regs, CTX_BB_STATE, RING_BBSTATE(base), RING_BB_PPGTT);
	CTX_REG(regs, CTX_SECOND_BB_HEAD_U, RING_SBBADDR_UDW(base), 0);
	CTX_REG(regs, CTX_SECOND_BB_HEAD_L, RING_SBBADDR(base), 0);
	CTX_REG(regs, CTX_SECOND_BB_STATE, RING_SBBSTATE(base), 0);
	if (rcs) {
2891 2892
		struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx;

2893 2894 2895
		CTX_REG(regs, CTX_RCS_INDIRECT_CTX, RING_INDIRECT_CTX(base), 0);
		CTX_REG(regs, CTX_RCS_INDIRECT_CTX_OFFSET,
			RING_INDIRECT_CTX_OFFSET(base), 0);
2896
		if (wa_ctx->indirect_ctx.size) {
2897
			u32 ggtt_offset = i915_ggtt_offset(wa_ctx->vma);
2898

2899
			regs[CTX_RCS_INDIRECT_CTX + 1] =
2900 2901
				(ggtt_offset + wa_ctx->indirect_ctx.offset) |
				(wa_ctx->indirect_ctx.size / CACHELINE_BYTES);
2902

2903
			regs[CTX_RCS_INDIRECT_CTX_OFFSET + 1] =
2904
				intel_lr_indirect_ctx_offset(engine) << 6;
2905 2906 2907 2908 2909
		}

		CTX_REG(regs, CTX_BB_PER_CTX_PTR, RING_BB_PER_CTX_PTR(base), 0);
		if (wa_ctx->per_ctx.size) {
			u32 ggtt_offset = i915_ggtt_offset(wa_ctx->vma);
2910

2911
			regs[CTX_BB_PER_CTX_PTR + 1] =
2912
				(ggtt_offset + wa_ctx->per_ctx.offset) | 0x01;
2913
		}
2914
	}
2915 2916 2917 2918

	regs[CTX_LRI_HEADER_1] = MI_LOAD_REGISTER_IMM(9) | MI_LRI_FORCE_POSTED;

	CTX_REG(regs, CTX_CTX_TIMESTAMP, RING_CTX_TIMESTAMP(base), 0);
2919
	/* PDP values well be assigned later if needed */
2920 2921 2922 2923 2924 2925 2926 2927
	CTX_REG(regs, CTX_PDP3_UDW, GEN8_RING_PDP_UDW(base, 3), 0);
	CTX_REG(regs, CTX_PDP3_LDW, GEN8_RING_PDP_LDW(base, 3), 0);
	CTX_REG(regs, CTX_PDP2_UDW, GEN8_RING_PDP_UDW(base, 2), 0);
	CTX_REG(regs, CTX_PDP2_LDW, GEN8_RING_PDP_LDW(base, 2), 0);
	CTX_REG(regs, CTX_PDP1_UDW, GEN8_RING_PDP_UDW(base, 1), 0);
	CTX_REG(regs, CTX_PDP1_LDW, GEN8_RING_PDP_LDW(base, 1), 0);
	CTX_REG(regs, CTX_PDP0_UDW, GEN8_RING_PDP_UDW(base, 0), 0);
	CTX_REG(regs, CTX_PDP0_LDW, GEN8_RING_PDP_LDW(base, 0), 0);
2928

2929
	if (i915_vm_is_4lvl(&ppgtt->vm)) {
2930 2931 2932 2933
		/* 64b PPGTT (48bit canonical)
		 * PDP0_DESCRIPTOR contains the base address to PML4 and
		 * other PDP Descriptors are ignored.
		 */
2934
		ASSIGN_CTX_PML4(ppgtt, regs);
2935
	} else {
2936 2937 2938 2939
		ASSIGN_CTX_PDP(ppgtt, regs, 3);
		ASSIGN_CTX_PDP(ppgtt, regs, 2);
		ASSIGN_CTX_PDP(ppgtt, regs, 1);
		ASSIGN_CTX_PDP(ppgtt, regs, 0);
2940 2941
	}

2942 2943
	if (rcs) {
		regs[CTX_LRI_HEADER_2] = MI_LOAD_REGISTER_IMM(1);
2944
		CTX_REG(regs, CTX_R_PWR_CLK_STATE, GEN8_R_PWR_CLK_STATE, 0);
2945

2946
		i915_oa_init_reg_state(engine, ce, regs);
2947
	}
2948 2949

	regs[CTX_END] = MI_BATCH_BUFFER_END;
2950
	if (INTEL_GEN(engine->i915) >= 10)
2951
		regs[CTX_END] |= BIT(0);
2952 2953 2954
}

static int
2955
populate_lr_context(struct intel_context *ce,
2956 2957 2958 2959 2960
		    struct drm_i915_gem_object *ctx_obj,
		    struct intel_engine_cs *engine,
		    struct intel_ring *ring)
{
	void *vaddr;
2961
	u32 *regs;
2962 2963 2964 2965 2966 2967 2968 2969 2970
	int ret;

	vaddr = i915_gem_object_pin_map(ctx_obj, I915_MAP_WB);
	if (IS_ERR(vaddr)) {
		ret = PTR_ERR(vaddr);
		DRM_DEBUG_DRIVER("Could not map object pages! (%d)\n", ret);
		return ret;
	}

2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981
	if (engine->default_state) {
		/*
		 * We only want to copy over the template context state;
		 * skipping over the headers reserved for GuC communication,
		 * leaving those as zero.
		 */
		const unsigned long start = LRC_HEADER_PAGES * PAGE_SIZE;
		void *defaults;

		defaults = i915_gem_object_pin_map(engine->default_state,
						   I915_MAP_WB);
2982 2983 2984 2985
		if (IS_ERR(defaults)) {
			ret = PTR_ERR(defaults);
			goto err_unpin_ctx;
		}
2986 2987 2988 2989 2990

		memcpy(vaddr + start, defaults + start, engine->context_size);
		i915_gem_object_unpin_map(engine->default_state);
	}

2991 2992
	/* The second page of the context object contains some fields which must
	 * be set up prior to the first execution. */
2993
	regs = vaddr + LRC_STATE_PN * PAGE_SIZE;
2994
	execlists_init_reg_state(regs, ce, engine, ring);
2995 2996 2997
	if (!engine->default_state)
		regs[CTX_CONTEXT_CONTROL + 1] |=
			_MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT);
2998

2999
	ret = 0;
3000
err_unpin_ctx:
3001 3002 3003
	__i915_gem_object_flush_map(ctx_obj,
				    LRC_HEADER_PAGES * PAGE_SIZE,
				    engine->context_size);
3004
	i915_gem_object_unpin_map(ctx_obj);
3005
	return ret;
3006 3007
}

3008 3009
static struct i915_timeline *get_timeline(struct i915_gem_context *ctx)
{
3010 3011 3012 3013
	if (ctx->timeline)
		return i915_timeline_get(ctx->timeline);
	else
		return i915_timeline_create(ctx->i915, NULL);
3014 3015 3016 3017
}

static int execlists_context_deferred_alloc(struct intel_context *ce,
					    struct intel_engine_cs *engine)
3018
{
3019
	struct drm_i915_gem_object *ctx_obj;
3020
	struct i915_vma *vma;
3021
	u32 context_size;
3022
	struct intel_ring *ring;
3023
	struct i915_timeline *timeline;
3024 3025
	int ret;

3026 3027
	if (ce->state)
		return 0;
3028

3029
	context_size = round_up(engine->context_size, I915_GTT_PAGE_SIZE);
3030

3031 3032 3033 3034 3035
	/*
	 * Before the actual start of the context image, we insert a few pages
	 * for our own use and for sharing with the GuC.
	 */
	context_size += LRC_HEADER_PAGES * PAGE_SIZE;
3036

3037
	ctx_obj = i915_gem_object_create_shmem(engine->i915, context_size);
3038 3039
	if (IS_ERR(ctx_obj))
		return PTR_ERR(ctx_obj);
3040

3041
	vma = i915_vma_instance(ctx_obj, &engine->gt->ggtt->vm, NULL);
3042 3043 3044 3045 3046
	if (IS_ERR(vma)) {
		ret = PTR_ERR(vma);
		goto error_deref_obj;
	}

3047
	timeline = get_timeline(ce->gem_context);
3048 3049 3050 3051 3052
	if (IS_ERR(timeline)) {
		ret = PTR_ERR(timeline);
		goto error_deref_obj;
	}

3053 3054 3055
	ring = intel_engine_create_ring(engine,
					timeline,
					ce->gem_context->ring_size);
3056
	i915_timeline_put(timeline);
3057 3058
	if (IS_ERR(ring)) {
		ret = PTR_ERR(ring);
3059
		goto error_deref_obj;
3060 3061
	}

3062
	ret = populate_lr_context(ce, ctx_obj, engine, ring);
3063 3064
	if (ret) {
		DRM_DEBUG_DRIVER("Failed to populate LRC: %d\n", ret);
3065
		goto error_ring_free;
3066 3067
	}

3068
	ce->ring = ring;
3069
	ce->state = vma;
3070 3071

	return 0;
3072

3073
error_ring_free:
3074
	intel_ring_put(ring);
3075
error_deref_obj:
3076
	i915_gem_object_put(ctx_obj);
3077
	return ret;
3078
}
3079

3080 3081 3082 3083 3084
static struct list_head *virtual_queue(struct virtual_engine *ve)
{
	return &ve->base.execlists.default_priolist.requests[0];
}

3085 3086 3087 3088 3089 3090
static void virtual_context_destroy(struct kref *kref)
{
	struct virtual_engine *ve =
		container_of(kref, typeof(*ve), context.ref);
	unsigned int n;

3091
	GEM_BUG_ON(!list_empty(virtual_queue(ve)));
3092
	GEM_BUG_ON(ve->request);
3093
	GEM_BUG_ON(ve->context.inflight);
3094 3095 3096 3097 3098 3099 3100 3101

	for (n = 0; n < ve->num_siblings; n++) {
		struct intel_engine_cs *sibling = ve->siblings[n];
		struct rb_node *node = &ve->nodes[sibling->id].rb;

		if (RB_EMPTY_NODE(node))
			continue;

3102
		spin_lock_irq(&sibling->active.lock);
3103 3104 3105 3106 3107

		/* Detachment is lazily performed in the execlists tasklet */
		if (!RB_EMPTY_NODE(node))
			rb_erase_cached(node, &sibling->execlists.virtual);

3108
		spin_unlock_irq(&sibling->active.lock);
3109 3110 3111 3112 3113 3114
	}
	GEM_BUG_ON(__tasklet_is_scheduled(&ve->base.execlists.tasklet));

	if (ve->context.state)
		__execlists_context_fini(&ve->context);

3115
	kfree(ve->bonds);
3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186
	kfree(ve);
}

static void virtual_engine_initial_hint(struct virtual_engine *ve)
{
	int swp;

	/*
	 * Pick a random sibling on starting to help spread the load around.
	 *
	 * New contexts are typically created with exactly the same order
	 * of siblings, and often started in batches. Due to the way we iterate
	 * the array of sibling when submitting requests, sibling[0] is
	 * prioritised for dequeuing. If we make sure that sibling[0] is fairly
	 * randomised across the system, we also help spread the load by the
	 * first engine we inspect being different each time.
	 *
	 * NB This does not force us to execute on this engine, it will just
	 * typically be the first we inspect for submission.
	 */
	swp = prandom_u32_max(ve->num_siblings);
	if (!swp)
		return;

	swap(ve->siblings[swp], ve->siblings[0]);
	virtual_update_register_offsets(ve->context.lrc_reg_state,
					ve->siblings[0]);
}

static int virtual_context_pin(struct intel_context *ce)
{
	struct virtual_engine *ve = container_of(ce, typeof(*ve), context);
	int err;

	/* Note: we must use a real engine class for setting up reg state */
	err = __execlists_context_pin(ce, ve->siblings[0]);
	if (err)
		return err;

	virtual_engine_initial_hint(ve);
	return 0;
}

static void virtual_context_enter(struct intel_context *ce)
{
	struct virtual_engine *ve = container_of(ce, typeof(*ve), context);
	unsigned int n;

	for (n = 0; n < ve->num_siblings; n++)
		intel_engine_pm_get(ve->siblings[n]);
}

static void virtual_context_exit(struct intel_context *ce)
{
	struct virtual_engine *ve = container_of(ce, typeof(*ve), context);
	unsigned int n;

	for (n = 0; n < ve->num_siblings; n++)
		intel_engine_pm_put(ve->siblings[n]);
}

static const struct intel_context_ops virtual_context_ops = {
	.pin = virtual_context_pin,
	.unpin = execlists_context_unpin,

	.enter = virtual_context_enter,
	.exit = virtual_context_exit,

	.destroy = virtual_context_destroy,
};

3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211
static intel_engine_mask_t virtual_submission_mask(struct virtual_engine *ve)
{
	struct i915_request *rq;
	intel_engine_mask_t mask;

	rq = READ_ONCE(ve->request);
	if (!rq)
		return 0;

	/* The rq is ready for submission; rq->execution_mask is now stable. */
	mask = rq->execution_mask;
	if (unlikely(!mask)) {
		/* Invalid selection, submit to a random engine in error */
		i915_request_skip(rq, -ENODEV);
		mask = ve->siblings[0]->mask;
	}

	GEM_TRACE("%s: rq=%llx:%lld, mask=%x, prio=%d\n",
		  ve->base.name,
		  rq->fence.context, rq->fence.seqno,
		  mask, ve->base.execlists.queue_priority_hint);

	return mask;
}

3212 3213 3214 3215
static void virtual_submission_tasklet(unsigned long data)
{
	struct virtual_engine * const ve = (struct virtual_engine *)data;
	const int prio = ve->base.execlists.queue_priority_hint;
3216
	intel_engine_mask_t mask;
3217 3218
	unsigned int n;

3219 3220 3221 3222 3223 3224
	rcu_read_lock();
	mask = virtual_submission_mask(ve);
	rcu_read_unlock();
	if (unlikely(!mask))
		return;

3225 3226 3227 3228 3229 3230 3231
	local_irq_disable();
	for (n = 0; READ_ONCE(ve->request) && n < ve->num_siblings; n++) {
		struct intel_engine_cs *sibling = ve->siblings[n];
		struct ve_node * const node = &ve->nodes[sibling->id];
		struct rb_node **parent, *rb;
		bool first;

3232 3233
		if (unlikely(!(mask & sibling->mask))) {
			if (!RB_EMPTY_NODE(&node->rb)) {
3234
				spin_lock(&sibling->active.lock);
3235 3236 3237
				rb_erase_cached(&node->rb,
						&sibling->execlists.virtual);
				RB_CLEAR_NODE(&node->rb);
3238
				spin_unlock(&sibling->active.lock);
3239 3240 3241 3242
			}
			continue;
		}

3243
		spin_lock(&sibling->active.lock);
3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286

		if (!RB_EMPTY_NODE(&node->rb)) {
			/*
			 * Cheat and avoid rebalancing the tree if we can
			 * reuse this node in situ.
			 */
			first = rb_first_cached(&sibling->execlists.virtual) ==
				&node->rb;
			if (prio == node->prio || (prio > node->prio && first))
				goto submit_engine;

			rb_erase_cached(&node->rb, &sibling->execlists.virtual);
		}

		rb = NULL;
		first = true;
		parent = &sibling->execlists.virtual.rb_root.rb_node;
		while (*parent) {
			struct ve_node *other;

			rb = *parent;
			other = rb_entry(rb, typeof(*other), rb);
			if (prio > other->prio) {
				parent = &rb->rb_left;
			} else {
				parent = &rb->rb_right;
				first = false;
			}
		}

		rb_link_node(&node->rb, rb, parent);
		rb_insert_color_cached(&node->rb,
				       &sibling->execlists.virtual,
				       first);

submit_engine:
		GEM_BUG_ON(RB_EMPTY_NODE(&node->rb));
		node->prio = prio;
		if (first && prio > sibling->execlists.queue_priority_hint) {
			sibling->execlists.queue_priority_hint = prio;
			tasklet_hi_schedule(&sibling->execlists.tasklet);
		}

3287
		spin_unlock(&sibling->active.lock);
3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303
	}
	local_irq_enable();
}

static void virtual_submit_request(struct i915_request *rq)
{
	struct virtual_engine *ve = to_virtual_engine(rq->engine);

	GEM_TRACE("%s: rq=%llx:%lld\n",
		  ve->base.name,
		  rq->fence.context,
		  rq->fence.seqno);

	GEM_BUG_ON(ve->base.submit_request != virtual_submit_request);

	GEM_BUG_ON(ve->request);
3304 3305
	GEM_BUG_ON(!list_empty(virtual_queue(ve)));

3306 3307 3308
	ve->base.execlists.queue_priority_hint = rq_prio(rq);
	WRITE_ONCE(ve->request, rq);

3309 3310
	list_move_tail(&rq->sched.link, virtual_queue(ve));

3311 3312 3313
	tasklet_schedule(&ve->base.execlists.tasklet);
}

3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345
static struct ve_bond *
virtual_find_bond(struct virtual_engine *ve,
		  const struct intel_engine_cs *master)
{
	int i;

	for (i = 0; i < ve->num_bonds; i++) {
		if (ve->bonds[i].master == master)
			return &ve->bonds[i];
	}

	return NULL;
}

static void
virtual_bond_execute(struct i915_request *rq, struct dma_fence *signal)
{
	struct virtual_engine *ve = to_virtual_engine(rq->engine);
	struct ve_bond *bond;

	bond = virtual_find_bond(ve, to_request(signal)->engine);
	if (bond) {
		intel_engine_mask_t old, new, cmp;

		cmp = READ_ONCE(rq->execution_mask);
		do {
			old = cmp;
			new = cmp & bond->sibling_mask;
		} while ((cmp = cmpxchg(&rq->execution_mask, old, new)) != old);
	}
}

3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365
struct intel_context *
intel_execlists_create_virtual(struct i915_gem_context *ctx,
			       struct intel_engine_cs **siblings,
			       unsigned int count)
{
	struct virtual_engine *ve;
	unsigned int n;
	int err;

	if (count == 0)
		return ERR_PTR(-EINVAL);

	if (count == 1)
		return intel_context_create(ctx, siblings[0]);

	ve = kzalloc(struct_size(ve, siblings, count), GFP_KERNEL);
	if (!ve)
		return ERR_PTR(-ENOMEM);

	ve->base.i915 = ctx->i915;
3366
	ve->base.gt = siblings[0]->gt;
3367 3368 3369 3370 3371 3372
	ve->base.id = -1;
	ve->base.class = OTHER_CLASS;
	ve->base.uabi_class = I915_ENGINE_CLASS_INVALID;
	ve->base.instance = I915_ENGINE_CLASS_INVALID_VIRTUAL;
	ve->base.flags = I915_ENGINE_IS_VIRTUAL;

3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387
	/*
	 * The decision on whether to submit a request using semaphores
	 * depends on the saturated state of the engine. We only compute
	 * this during HW submission of the request, and we need for this
	 * state to be globally applied to all requests being submitted
	 * to this engine. Virtual engines encompass more than one physical
	 * engine and so we cannot accurately tell in advance if one of those
	 * engines is already saturated and so cannot afford to use a semaphore
	 * and be pessimized in priority for doing so -- if we are the only
	 * context using semaphores after all other clients have stopped, we
	 * will be starved on the saturated system. Such a global switch for
	 * semaphores is less than ideal, but alas is the current compromise.
	 */
	ve->base.saturated = ALL_ENGINES;

3388 3389
	snprintf(ve->base.name, sizeof(ve->base.name), "virtual");

3390
	intel_engine_init_active(&ve->base, ENGINE_VIRTUAL);
3391 3392 3393 3394 3395 3396 3397 3398

	intel_engine_init_execlists(&ve->base);

	ve->base.cops = &virtual_context_ops;
	ve->base.request_alloc = execlists_request_alloc;

	ve->base.schedule = i915_schedule;
	ve->base.submit_request = virtual_submit_request;
3399
	ve->base.bond_execute = virtual_bond_execute;
3400

3401
	INIT_LIST_HEAD(virtual_queue(ve));
3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489
	ve->base.execlists.queue_priority_hint = INT_MIN;
	tasklet_init(&ve->base.execlists.tasklet,
		     virtual_submission_tasklet,
		     (unsigned long)ve);

	intel_context_init(&ve->context, ctx, &ve->base);

	for (n = 0; n < count; n++) {
		struct intel_engine_cs *sibling = siblings[n];

		GEM_BUG_ON(!is_power_of_2(sibling->mask));
		if (sibling->mask & ve->base.mask) {
			DRM_DEBUG("duplicate %s entry in load balancer\n",
				  sibling->name);
			err = -EINVAL;
			goto err_put;
		}

		/*
		 * The virtual engine implementation is tightly coupled to
		 * the execlists backend -- we push out request directly
		 * into a tree inside each physical engine. We could support
		 * layering if we handle cloning of the requests and
		 * submitting a copy into each backend.
		 */
		if (sibling->execlists.tasklet.func !=
		    execlists_submission_tasklet) {
			err = -ENODEV;
			goto err_put;
		}

		GEM_BUG_ON(RB_EMPTY_NODE(&ve->nodes[sibling->id].rb));
		RB_CLEAR_NODE(&ve->nodes[sibling->id].rb);

		ve->siblings[ve->num_siblings++] = sibling;
		ve->base.mask |= sibling->mask;

		/*
		 * All physical engines must be compatible for their emission
		 * functions (as we build the instructions during request
		 * construction and do not alter them before submission
		 * on the physical engine). We use the engine class as a guide
		 * here, although that could be refined.
		 */
		if (ve->base.class != OTHER_CLASS) {
			if (ve->base.class != sibling->class) {
				DRM_DEBUG("invalid mixing of engine class, sibling %d, already %d\n",
					  sibling->class, ve->base.class);
				err = -EINVAL;
				goto err_put;
			}
			continue;
		}

		ve->base.class = sibling->class;
		ve->base.uabi_class = sibling->uabi_class;
		snprintf(ve->base.name, sizeof(ve->base.name),
			 "v%dx%d", ve->base.class, count);
		ve->base.context_size = sibling->context_size;

		ve->base.emit_bb_start = sibling->emit_bb_start;
		ve->base.emit_flush = sibling->emit_flush;
		ve->base.emit_init_breadcrumb = sibling->emit_init_breadcrumb;
		ve->base.emit_fini_breadcrumb = sibling->emit_fini_breadcrumb;
		ve->base.emit_fini_breadcrumb_dw =
			sibling->emit_fini_breadcrumb_dw;
	}

	return &ve->context;

err_put:
	intel_context_put(&ve->context);
	return ERR_PTR(err);
}

struct intel_context *
intel_execlists_clone_virtual(struct i915_gem_context *ctx,
			      struct intel_engine_cs *src)
{
	struct virtual_engine *se = to_virtual_engine(src);
	struct intel_context *dst;

	dst = intel_execlists_create_virtual(ctx,
					     se->siblings,
					     se->num_siblings);
	if (IS_ERR(dst))
		return dst;

3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503
	if (se->num_bonds) {
		struct virtual_engine *de = to_virtual_engine(dst->engine);

		de->bonds = kmemdup(se->bonds,
				    sizeof(*se->bonds) * se->num_bonds,
				    GFP_KERNEL);
		if (!de->bonds) {
			intel_context_put(dst);
			return ERR_PTR(-ENOMEM);
		}

		de->num_bonds = se->num_bonds;
	}

3504 3505 3506
	return dst;
}

3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542
int intel_virtual_engine_attach_bond(struct intel_engine_cs *engine,
				     const struct intel_engine_cs *master,
				     const struct intel_engine_cs *sibling)
{
	struct virtual_engine *ve = to_virtual_engine(engine);
	struct ve_bond *bond;
	int n;

	/* Sanity check the sibling is part of the virtual engine */
	for (n = 0; n < ve->num_siblings; n++)
		if (sibling == ve->siblings[n])
			break;
	if (n == ve->num_siblings)
		return -EINVAL;

	bond = virtual_find_bond(ve, master);
	if (bond) {
		bond->sibling_mask |= sibling->mask;
		return 0;
	}

	bond = krealloc(ve->bonds,
			sizeof(*bond) * (ve->num_bonds + 1),
			GFP_KERNEL);
	if (!bond)
		return -ENOMEM;

	bond[ve->num_bonds].master = master;
	bond[ve->num_bonds].sibling_mask = sibling->mask;

	ve->bonds = bond;
	ve->num_bonds++;

	return 0;
}

3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555
void intel_execlists_show_requests(struct intel_engine_cs *engine,
				   struct drm_printer *m,
				   void (*show_request)(struct drm_printer *m,
							struct i915_request *rq,
							const char *prefix),
				   unsigned int max)
{
	const struct intel_engine_execlists *execlists = &engine->execlists;
	struct i915_request *rq, *last;
	unsigned long flags;
	unsigned int count;
	struct rb_node *rb;

3556
	spin_lock_irqsave(&engine->active.lock, flags);
3557 3558 3559

	last = NULL;
	count = 0;
3560
	list_for_each_entry(rq, &engine->active.requests, sched.link) {
3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576
		if (count++ < max - 1)
			show_request(m, rq, "\t\tE ");
		else
			last = rq;
	}
	if (last) {
		if (count > max) {
			drm_printf(m,
				   "\t\t...skipping %d executing requests...\n",
				   count - max);
		}
		show_request(m, last, "\t\tE ");
	}

	last = NULL;
	count = 0;
3577 3578 3579
	if (execlists->queue_priority_hint != INT_MIN)
		drm_printf(m, "\t\tQueue priority hint: %d\n",
			   execlists->queue_priority_hint);
3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599
	for (rb = rb_first_cached(&execlists->queue); rb; rb = rb_next(rb)) {
		struct i915_priolist *p = rb_entry(rb, typeof(*p), node);
		int i;

		priolist_for_each_request(rq, p, i) {
			if (count++ < max - 1)
				show_request(m, rq, "\t\tQ ");
			else
				last = rq;
		}
	}
	if (last) {
		if (count > max) {
			drm_printf(m,
				   "\t\t...skipping %d queued requests...\n",
				   count - max);
		}
		show_request(m, last, "\t\tQ ");
	}

3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622
	last = NULL;
	count = 0;
	for (rb = rb_first_cached(&execlists->virtual); rb; rb = rb_next(rb)) {
		struct virtual_engine *ve =
			rb_entry(rb, typeof(*ve), nodes[engine->id].rb);
		struct i915_request *rq = READ_ONCE(ve->request);

		if (rq) {
			if (count++ < max - 1)
				show_request(m, rq, "\t\tV ");
			else
				last = rq;
		}
	}
	if (last) {
		if (count > max) {
			drm_printf(m,
				   "\t\t...skipping %d virtual requests...\n",
				   count - max);
		}
		show_request(m, last, "\t\tV ");
	}

3623
	spin_unlock_irqrestore(&engine->active.lock, flags);
3624 3625
}

3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656
void intel_lr_context_reset(struct intel_engine_cs *engine,
			    struct intel_context *ce,
			    u32 head,
			    bool scrub)
{
	/*
	 * We want a simple context + ring to execute the breadcrumb update.
	 * We cannot rely on the context being intact across the GPU hang,
	 * so clear it and rebuild just what we need for the breadcrumb.
	 * All pending requests for this context will be zapped, and any
	 * future request will be after userspace has had the opportunity
	 * to recreate its own state.
	 */
	if (scrub) {
		u32 *regs = ce->lrc_reg_state;

		if (engine->pinned_default_state) {
			memcpy(regs, /* skip restoring the vanilla PPHWSP */
			       engine->pinned_default_state + LRC_STATE_PN * PAGE_SIZE,
			       engine->context_size - PAGE_SIZE);
		}
		execlists_init_reg_state(regs, ce, engine, ce->ring);
	}

	/* Rerun the request; its payload has been neutered (if guilty). */
	ce->ring->head = head;
	intel_ring_update_space(ce->ring);

	__execlists_update_reg_state(ce, engine);
}

3657
#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
3658
#include "selftest_lrc.c"
3659
#endif