cpufreq_ondemand.c 15.3 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7 8 9 10 11 12
/*
 *  drivers/cpufreq/cpufreq_ondemand.c
 *
 *  Copyright (C)  2001 Russell King
 *            (C)  2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
 *                      Jun Nakajima <jun.nakajima@intel.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

13 14
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

Linus Torvalds's avatar
Linus Torvalds committed
15
#include <linux/cpufreq.h>
16 17
#include <linux/init.h>
#include <linux/kernel.h>
Linus Torvalds's avatar
Linus Torvalds committed
18
#include <linux/kernel_stat.h>
19 20
#include <linux/kobject.h>
#include <linux/module.h>
21
#include <linux/mutex.h>
22 23
#include <linux/percpu-defs.h>
#include <linux/sysfs.h>
24
#include <linux/tick.h>
25
#include <linux/types.h>
Linus Torvalds's avatar
Linus Torvalds committed
26

27
#include "cpufreq_governor.h"
Linus Torvalds's avatar
Linus Torvalds committed
28

29
/* On-demand governor macors */
30
#define DEF_FREQUENCY_DOWN_DIFFERENTIAL		(10)
Linus Torvalds's avatar
Linus Torvalds committed
31
#define DEF_FREQUENCY_UP_THRESHOLD		(80)
32 33
#define DEF_SAMPLING_DOWN_FACTOR		(1)
#define MAX_SAMPLING_DOWN_FACTOR		(100000)
34 35
#define MICRO_FREQUENCY_DOWN_DIFFERENTIAL	(3)
#define MICRO_FREQUENCY_UP_THRESHOLD		(95)
36
#define MICRO_FREQUENCY_MIN_SAMPLE_RATE		(10000)
37
#define MIN_FREQUENCY_UP_THRESHOLD		(11)
Linus Torvalds's avatar
Linus Torvalds committed
38 39
#define MAX_FREQUENCY_UP_THRESHOLD		(100)

40 41
static struct dbs_data od_dbs_data;
static DEFINE_PER_CPU(struct od_cpu_dbs_info_s, od_cpu_dbs_info);
Linus Torvalds's avatar
Linus Torvalds committed
42

43 44 45 46
#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND
static struct cpufreq_governor cpufreq_gov_ondemand;
#endif

47
static struct od_dbs_tuners od_tuners = {
48
	.up_threshold = DEF_FREQUENCY_UP_THRESHOLD,
49
	.sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR,
50
	.down_differential = DEF_FREQUENCY_DOWN_DIFFERENTIAL,
51
	.ignore_nice = 0,
52
	.powersave_bias = 0,
Linus Torvalds's avatar
Linus Torvalds committed
53 54
};

55
static void ondemand_powersave_bias_init_cpu(int cpu)
56
{
57
	struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, cpu);
58

59 60 61
	dbs_info->freq_table = cpufreq_frequency_get_table(cpu);
	dbs_info->freq_lo = 0;
}
62

63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83
/*
 * Not all CPUs want IO time to be accounted as busy; this depends on how
 * efficient idling at a higher frequency/voltage is.
 * Pavel Machek says this is not so for various generations of AMD and old
 * Intel systems.
 * Mike Chan (androidlcom) calis this is also not true for ARM.
 * Because of this, whitelist specific known (series) of CPUs by default, and
 * leave all others up to the user.
 */
static int should_io_be_busy(void)
{
#if defined(CONFIG_X86)
	/*
	 * For Intel, Core 2 (model 15) andl later have an efficient idle.
	 */
	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
			boot_cpu_data.x86 == 6 &&
			boot_cpu_data.x86_model >= 15)
		return 1;
#endif
	return 0;
84 85
}

86 87 88 89 90
/*
 * Find right freq to be set now with powersave_bias on.
 * Returns the freq_hi to be used right now and will set freq_hi_jiffies,
 * freq_lo, and freq_lo_jiffies in percpu area for averaging freqs.
 */
91
static unsigned int powersave_bias_target(struct cpufreq_policy *policy,
92
		unsigned int freq_next, unsigned int relation)
93 94 95 96 97
{
	unsigned int freq_req, freq_reduc, freq_avg;
	unsigned int freq_hi, freq_lo;
	unsigned int index = 0;
	unsigned int jiffies_total, jiffies_hi, jiffies_lo;
98
	struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info,
99
						   policy->cpu);
100 101 102 103 104 105 106 107 108 109

	if (!dbs_info->freq_table) {
		dbs_info->freq_lo = 0;
		dbs_info->freq_lo_jiffies = 0;
		return freq_next;
	}

	cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_next,
			relation, &index);
	freq_req = dbs_info->freq_table[index].frequency;
110
	freq_reduc = freq_req * od_tuners.powersave_bias / 1000;
111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128
	freq_avg = freq_req - freq_reduc;

	/* Find freq bounds for freq_avg in freq_table */
	index = 0;
	cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_avg,
			CPUFREQ_RELATION_H, &index);
	freq_lo = dbs_info->freq_table[index].frequency;
	index = 0;
	cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_avg,
			CPUFREQ_RELATION_L, &index);
	freq_hi = dbs_info->freq_table[index].frequency;

	/* Find out how long we have to be in hi and lo freqs */
	if (freq_hi == freq_lo) {
		dbs_info->freq_lo = 0;
		dbs_info->freq_lo_jiffies = 0;
		return freq_lo;
	}
129
	jiffies_total = usecs_to_jiffies(od_tuners.sampling_rate);
130 131 132 133 134 135 136 137 138 139 140 141 142 143
	jiffies_hi = (freq_avg - freq_lo) * jiffies_total;
	jiffies_hi += ((freq_hi - freq_lo) / 2);
	jiffies_hi /= (freq_hi - freq_lo);
	jiffies_lo = jiffies_total - jiffies_hi;
	dbs_info->freq_lo = freq_lo;
	dbs_info->freq_lo_jiffies = jiffies_lo;
	dbs_info->freq_hi_jiffies = jiffies_hi;
	return freq_hi;
}

static void ondemand_powersave_bias_init(void)
{
	int i;
	for_each_online_cpu(i) {
144
		ondemand_powersave_bias_init_cpu(i);
145 146 147
	}
}

148 149 150 151 152 153
static void dbs_freq_increase(struct cpufreq_policy *p, unsigned int freq)
{
	if (od_tuners.powersave_bias)
		freq = powersave_bias_target(p, freq, CPUFREQ_RELATION_H);
	else if (p->cur == p->max)
		return;
154

155 156 157 158 159 160 161 162 163 164 165 166 167 168
	__cpufreq_driver_target(p, freq, od_tuners.powersave_bias ?
			CPUFREQ_RELATION_L : CPUFREQ_RELATION_H);
}

/*
 * Every sampling_rate, we check, if current idle time is less than 20%
 * (default), then we try to increase frequency Every sampling_rate, we look for
 * a the lowest frequency which can sustain the load while keeping idle time
 * over 30%. If such a frequency exist, we try to decrease to this frequency.
 *
 * Any frequency increase takes it to the maximum frequency. Frequency reduction
 * happens at minimum steps of 5% (default) of current frequency
 */
static void od_check_cpu(int cpu, unsigned int load_freq)
Linus Torvalds's avatar
Linus Torvalds committed
169
{
170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216
	struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, cpu);
	struct cpufreq_policy *policy = dbs_info->cdbs.cur_policy;

	dbs_info->freq_lo = 0;

	/* Check for frequency increase */
	if (load_freq > od_tuners.up_threshold * policy->cur) {
		/* If switching to max speed, apply sampling_down_factor */
		if (policy->cur < policy->max)
			dbs_info->rate_mult =
				od_tuners.sampling_down_factor;
		dbs_freq_increase(policy, policy->max);
		return;
	}

	/* Check for frequency decrease */
	/* if we cannot reduce the frequency anymore, break out early */
	if (policy->cur == policy->min)
		return;

	/*
	 * The optimal frequency is the frequency that is the lowest that can
	 * support the current CPU usage without triggering the up policy. To be
	 * safe, we focus 10 points under the threshold.
	 */
	if (load_freq < (od_tuners.up_threshold - od_tuners.down_differential) *
			policy->cur) {
		unsigned int freq_next;
		freq_next = load_freq / (od_tuners.up_threshold -
				od_tuners.down_differential);

		/* No longer fully busy, reset rate_mult */
		dbs_info->rate_mult = 1;

		if (freq_next < policy->min)
			freq_next = policy->min;

		if (!od_tuners.powersave_bias) {
			__cpufreq_driver_target(policy, freq_next,
					CPUFREQ_RELATION_L);
		} else {
			int freq = powersave_bias_target(policy, freq_next,
					CPUFREQ_RELATION_L);
			__cpufreq_driver_target(policy, freq,
					CPUFREQ_RELATION_L);
		}
	}
Linus Torvalds's avatar
Linus Torvalds committed
217 218
}

219 220 221 222 223 224
static void od_dbs_timer(struct work_struct *work)
{
	struct od_cpu_dbs_info_s *dbs_info =
		container_of(work, struct od_cpu_dbs_info_s, cdbs.work.work);
	unsigned int cpu = dbs_info->cdbs.cpu;
	int delay, sample_type = dbs_info->sample_type;
Linus Torvalds's avatar
Linus Torvalds committed
225

226 227 228 229 230 231 232 233 234 235 236 237 238 239 240
	mutex_lock(&dbs_info->cdbs.timer_mutex);

	/* Common NORMAL_SAMPLE setup */
	dbs_info->sample_type = OD_NORMAL_SAMPLE;
	if (sample_type == OD_SUB_SAMPLE) {
		delay = dbs_info->freq_lo_jiffies;
		__cpufreq_driver_target(dbs_info->cdbs.cur_policy,
			dbs_info->freq_lo, CPUFREQ_RELATION_H);
	} else {
		dbs_check_cpu(&od_dbs_data, cpu);
		if (dbs_info->freq_lo) {
			/* Setup timer for SUB_SAMPLE */
			dbs_info->sample_type = OD_SUB_SAMPLE;
			delay = dbs_info->freq_hi_jiffies;
		} else {
241 242
			delay = delay_for_sampling_rate(od_tuners.sampling_rate
						* dbs_info->rate_mult);
243 244 245
		}
	}

246 247
	schedule_delayed_work_on(smp_processor_id(), &dbs_info->cdbs.work,
			delay);
248 249 250 251 252 253 254 255 256
	mutex_unlock(&dbs_info->cdbs.timer_mutex);
}

/************************** sysfs interface ************************/

static ssize_t show_sampling_rate_min(struct kobject *kobj,
				      struct attribute *attr, char *buf)
{
	return sprintf(buf, "%u\n", od_dbs_data.min_sampling_rate);
Linus Torvalds's avatar
Linus Torvalds committed
257 258
}

259 260 261 262 263
/**
 * update_sampling_rate - update sampling rate effective immediately if needed.
 * @new_rate: new sampling rate
 *
 * If new rate is smaller than the old, simply updaing
264 265 266 267 268 269 270
 * dbs_tuners_int.sampling_rate might not be appropriate. For example, if the
 * original sampling_rate was 1 second and the requested new sampling rate is 10
 * ms because the user needs immediate reaction from ondemand governor, but not
 * sure if higher frequency will be required or not, then, the governor may
 * change the sampling rate too late; up to 1 second later. Thus, if we are
 * reducing the sampling rate, we need to make the new value effective
 * immediately.
271 272 273 274 275
 */
static void update_sampling_rate(unsigned int new_rate)
{
	int cpu;

276 277
	od_tuners.sampling_rate = new_rate = max(new_rate,
			od_dbs_data.min_sampling_rate);
278 279 280

	for_each_online_cpu(cpu) {
		struct cpufreq_policy *policy;
281
		struct od_cpu_dbs_info_s *dbs_info;
282 283 284 285 286
		unsigned long next_sampling, appointed_at;

		policy = cpufreq_cpu_get(cpu);
		if (!policy)
			continue;
287 288 289 290
		if (policy->governor != &cpufreq_gov_ondemand) {
			cpufreq_cpu_put(policy);
			continue;
		}
291 292 293
		dbs_info = &per_cpu(od_cpu_dbs_info, policy->cpu);
		cpufreq_cpu_put(policy);

294
		mutex_lock(&dbs_info->cdbs.timer_mutex);
295

296 297
		if (!delayed_work_pending(&dbs_info->cdbs.work)) {
			mutex_unlock(&dbs_info->cdbs.timer_mutex);
298 299 300
			continue;
		}

301 302
		next_sampling = jiffies + usecs_to_jiffies(new_rate);
		appointed_at = dbs_info->cdbs.work.timer.expires;
303 304 305

		if (time_before(next_sampling, appointed_at)) {

306 307 308
			mutex_unlock(&dbs_info->cdbs.timer_mutex);
			cancel_delayed_work_sync(&dbs_info->cdbs.work);
			mutex_lock(&dbs_info->cdbs.timer_mutex);
309

310 311 312
			schedule_delayed_work_on(dbs_info->cdbs.cpu,
					&dbs_info->cdbs.work,
					usecs_to_jiffies(new_rate));
313 314

		}
315
		mutex_unlock(&dbs_info->cdbs.timer_mutex);
316 317 318
	}
}

319 320
static ssize_t store_sampling_rate(struct kobject *a, struct attribute *b,
				   const char *buf, size_t count)
Linus Torvalds's avatar
Linus Torvalds committed
321 322 323
{
	unsigned int input;
	int ret;
324
	ret = sscanf(buf, "%u", &input);
325 326
	if (ret != 1)
		return -EINVAL;
327
	update_sampling_rate(input);
Linus Torvalds's avatar
Linus Torvalds committed
328 329 330
	return count;
}

331 332 333 334 335 336 337 338 339
static ssize_t store_io_is_busy(struct kobject *a, struct attribute *b,
				   const char *buf, size_t count)
{
	unsigned int input;
	int ret;

	ret = sscanf(buf, "%u", &input);
	if (ret != 1)
		return -EINVAL;
340
	od_tuners.io_is_busy = !!input;
341 342 343
	return count;
}

344 345
static ssize_t store_up_threshold(struct kobject *a, struct attribute *b,
				  const char *buf, size_t count)
Linus Torvalds's avatar
Linus Torvalds committed
346 347 348
{
	unsigned int input;
	int ret;
349
	ret = sscanf(buf, "%u", &input);
Linus Torvalds's avatar
Linus Torvalds committed
350

351
	if (ret != 1 || input > MAX_FREQUENCY_UP_THRESHOLD ||
352
			input < MIN_FREQUENCY_UP_THRESHOLD) {
Linus Torvalds's avatar
Linus Torvalds committed
353 354
		return -EINVAL;
	}
355
	od_tuners.up_threshold = input;
Linus Torvalds's avatar
Linus Torvalds committed
356 357 358
	return count;
}

359 360 361 362 363 364 365 366 367
static ssize_t store_sampling_down_factor(struct kobject *a,
			struct attribute *b, const char *buf, size_t count)
{
	unsigned int input, j;
	int ret;
	ret = sscanf(buf, "%u", &input);

	if (ret != 1 || input > MAX_SAMPLING_DOWN_FACTOR || input < 1)
		return -EINVAL;
368
	od_tuners.sampling_down_factor = input;
369 370 371

	/* Reset down sampling multiplier in case it was active */
	for_each_online_cpu(j) {
372 373
		struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info,
				j);
374 375 376 377 378
		dbs_info->rate_mult = 1;
	}
	return count;
}

379 380
static ssize_t store_ignore_nice_load(struct kobject *a, struct attribute *b,
				      const char *buf, size_t count)
381 382 383 384 385
{
	unsigned int input;
	int ret;

	unsigned int j;
386

387
	ret = sscanf(buf, "%u", &input);
388
	if (ret != 1)
389 390
		return -EINVAL;

391
	if (input > 1)
392
		input = 1;
393

394
	if (input == od_tuners.ignore_nice) { /* nothing to do */
395 396
		return count;
	}
397
	od_tuners.ignore_nice = input;
398

399
	/* we need to re-evaluate prev_cpu_idle */
400
	for_each_online_cpu(j) {
401
		struct od_cpu_dbs_info_s *dbs_info;
402
		dbs_info = &per_cpu(od_cpu_dbs_info, j);
403 404 405 406 407
		dbs_info->cdbs.prev_cpu_idle = get_cpu_idle_time(j,
						&dbs_info->cdbs.prev_cpu_wall);
		if (od_tuners.ignore_nice)
			dbs_info->cdbs.prev_cpu_nice =
				kcpustat_cpu(j).cpustat[CPUTIME_NICE];
408

409 410 411 412
	}
	return count;
}

413 414
static ssize_t store_powersave_bias(struct kobject *a, struct attribute *b,
				    const char *buf, size_t count)
415 416 417 418 419 420 421 422 423 424 425
{
	unsigned int input;
	int ret;
	ret = sscanf(buf, "%u", &input);

	if (ret != 1)
		return -EINVAL;

	if (input > 1000)
		input = 1000;

426
	od_tuners.powersave_bias = input;
427 428 429 430
	ondemand_powersave_bias_init();
	return count;
}

431 432 433 434 435 436 437
show_one(od, sampling_rate, sampling_rate);
show_one(od, io_is_busy, io_is_busy);
show_one(od, up_threshold, up_threshold);
show_one(od, sampling_down_factor, sampling_down_factor);
show_one(od, ignore_nice_load, ignore_nice);
show_one(od, powersave_bias, powersave_bias);

438
define_one_global_rw(sampling_rate);
439
define_one_global_rw(io_is_busy);
440
define_one_global_rw(up_threshold);
441
define_one_global_rw(sampling_down_factor);
442 443
define_one_global_rw(ignore_nice_load);
define_one_global_rw(powersave_bias);
444
define_one_global_ro(sampling_rate_min);
Linus Torvalds's avatar
Linus Torvalds committed
445

446
static struct attribute *dbs_attributes[] = {
Linus Torvalds's avatar
Linus Torvalds committed
447 448 449
	&sampling_rate_min.attr,
	&sampling_rate.attr,
	&up_threshold.attr,
450
	&sampling_down_factor.attr,
451
	&ignore_nice_load.attr,
452
	&powersave_bias.attr,
453
	&io_is_busy.attr,
Linus Torvalds's avatar
Linus Torvalds committed
454 455 456
	NULL
};

457
static struct attribute_group od_attr_group = {
Linus Torvalds's avatar
Linus Torvalds committed
458 459 460 461 462 463
	.attrs = dbs_attributes,
	.name = "ondemand",
};

/************************** sysfs end ************************/

464
define_get_cpu_dbs_routines(od_cpu_dbs_info);
465

466 467 468 469 470 471
static struct od_ops od_ops = {
	.io_busy = should_io_be_busy,
	.powersave_bias_init_cpu = ondemand_powersave_bias_init_cpu,
	.powersave_bias_target = powersave_bias_target,
	.freq_increase = dbs_freq_increase,
};
472

473 474 475 476 477 478 479 480 481 482
static struct dbs_data od_dbs_data = {
	.governor = GOV_ONDEMAND,
	.attr_group = &od_attr_group,
	.tuners = &od_tuners,
	.get_cpu_cdbs = get_cpu_cdbs,
	.get_cpu_dbs_info_s = get_cpu_dbs_info_s,
	.gov_dbs_timer = od_dbs_timer,
	.gov_check_cpu = od_check_cpu,
	.gov_ops = &od_ops,
};
Linus Torvalds's avatar
Linus Torvalds committed
483

484 485
static int od_cpufreq_governor_dbs(struct cpufreq_policy *policy,
		unsigned int event)
Linus Torvalds's avatar
Linus Torvalds committed
486
{
487
	return cpufreq_governor_dbs(&od_dbs_data, policy, event);
Linus Torvalds's avatar
Linus Torvalds committed
488 489
}

490 491
#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND
static
492
#endif
493 494 495 496 497 498
struct cpufreq_governor cpufreq_gov_ondemand = {
	.name			= "ondemand",
	.governor		= od_cpufreq_governor_dbs,
	.max_transition_latency	= TRANSITION_LATENCY_LIMIT,
	.owner			= THIS_MODULE,
};
Linus Torvalds's avatar
Linus Torvalds committed
499 500 501

static int __init cpufreq_gov_dbs_init(void)
{
502 503
	u64 idle_time;
	int cpu = get_cpu();
504

505
	mutex_init(&od_dbs_data.mutex);
506
	idle_time = get_cpu_idle_time_us(cpu, NULL);
507
	put_cpu();
508 509
	if (idle_time != -1ULL) {
		/* Idle micro accounting is supported. Use finer thresholds */
510 511
		od_tuners.up_threshold = MICRO_FREQUENCY_UP_THRESHOLD;
		od_tuners.down_differential = MICRO_FREQUENCY_DOWN_DIFFERENTIAL;
512
		/*
513
		 * In nohz/micro accounting case we set the minimum frequency
514 515 516
		 * not depending on HZ, but fixed (very low). The deferred
		 * timer might skip some samples if idle/sleeping as needed.
		*/
517
		od_dbs_data.min_sampling_rate = MICRO_FREQUENCY_MIN_SAMPLE_RATE;
518 519
	} else {
		/* For correct statistics, we need 10 ticks for each measure */
520 521
		od_dbs_data.min_sampling_rate = MIN_SAMPLING_RATE_RATIO *
			jiffies_to_usecs(10);
522
	}
523

524
	return cpufreq_register_governor(&cpufreq_gov_ondemand);
Linus Torvalds's avatar
Linus Torvalds committed
525 526 527 528
}

static void __exit cpufreq_gov_dbs_exit(void)
{
529
	cpufreq_unregister_governor(&cpufreq_gov_ondemand);
Linus Torvalds's avatar
Linus Torvalds committed
530 531
}

532 533 534
MODULE_AUTHOR("Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>");
MODULE_AUTHOR("Alexey Starikovskiy <alexey.y.starikovskiy@intel.com>");
MODULE_DESCRIPTION("'cpufreq_ondemand' - A dynamic cpufreq governor for "
535
	"Low Latency Frequency Transition capable processors");
536
MODULE_LICENSE("GPL");
Linus Torvalds's avatar
Linus Torvalds committed
537

538 539 540
#ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND
fs_initcall(cpufreq_gov_dbs_init);
#else
Linus Torvalds's avatar
Linus Torvalds committed
541
module_init(cpufreq_gov_dbs_init);
542
#endif
Linus Torvalds's avatar
Linus Torvalds committed
543
module_exit(cpufreq_gov_dbs_exit);