diff options
Diffstat (limited to 'drivers/devfreq')
| -rw-r--r-- | drivers/devfreq/Kconfig | 43 | ||||
| -rw-r--r-- | drivers/devfreq/Makefile | 4 | ||||
| -rw-r--r-- | drivers/devfreq/devfreq-event.c | 4 | ||||
| -rw-r--r-- | drivers/devfreq/devfreq.c | 488 | ||||
| -rw-r--r-- | drivers/devfreq/event/Kconfig | 6 | ||||
| -rw-r--r-- | drivers/devfreq/event/exynos-nocp.c | 2 | ||||
| -rw-r--r-- | drivers/devfreq/event/exynos-nocp.h | 2 | ||||
| -rw-r--r-- | drivers/devfreq/event/exynos-ppmu.c | 114 | ||||
| -rw-r--r-- | drivers/devfreq/event/exynos-ppmu.h | 2 | ||||
| -rw-r--r-- | drivers/devfreq/event/rockchip-dfi.c | 5 | ||||
| -rw-r--r-- | drivers/devfreq/exynos-bus.c | 284 | ||||
| -rw-r--r-- | drivers/devfreq/governor.h | 3 | ||||
| -rw-r--r-- | drivers/devfreq/governor_passive.c | 7 | ||||
| -rw-r--r-- | drivers/devfreq/imx8m-ddrc.c | 471 | ||||
| -rw-r--r-- | drivers/devfreq/rk3399_dmc.c | 21 | ||||
| -rw-r--r-- | drivers/devfreq/tegra20-devfreq.c | 212 | ||||
| -rw-r--r-- | drivers/devfreq/tegra30-devfreq.c (renamed from drivers/devfreq/tegra-devfreq.c) | 590 |
17 files changed, 1701 insertions, 557 deletions
diff --git a/drivers/devfreq/Kconfig b/drivers/devfreq/Kconfig index ba98a4e3ad33..0b1df12e0f21 100644 --- a/drivers/devfreq/Kconfig +++ b/drivers/devfreq/Kconfig @@ -77,13 +77,12 @@ config DEVFREQ_GOV_PASSIVE comment "DEVFREQ Drivers" config ARM_EXYNOS_BUS_DEVFREQ - tristate "ARM EXYNOS Generic Memory Bus DEVFREQ Driver" + tristate "ARM Exynos Generic Memory Bus DEVFREQ Driver" depends on ARCH_EXYNOS || COMPILE_TEST select DEVFREQ_GOV_SIMPLE_ONDEMAND select DEVFREQ_GOV_PASSIVE select DEVFREQ_EVENT_EXYNOS_PPMU select PM_DEVFREQ_EVENT - select PM_OPP help This adds the common DEVFREQ driver for Exynos Memory bus. Exynos Memory bus has one more group of memory bus (e.g, MIF and INT block). @@ -92,27 +91,49 @@ config ARM_EXYNOS_BUS_DEVFREQ and adjusts the operating frequencies and voltages with OPP support. This does not yet operate with optimal voltages. -config ARM_TEGRA_DEVFREQ - tristate "Tegra DEVFREQ Driver" - depends on ARCH_TEGRA_124_SOC +config ARM_IMX8M_DDRC_DEVFREQ + tristate "i.MX8M DDRC DEVFREQ Driver" + depends on (ARCH_MXC && HAVE_ARM_SMCCC) || \ + (COMPILE_TEST && HAVE_ARM_SMCCC) select DEVFREQ_GOV_SIMPLE_ONDEMAND - select PM_OPP + select DEVFREQ_GOV_USERSPACE + help + This adds the DEVFREQ driver for the i.MX8M DDR Controller. It allows + adjusting DRAM frequency. + +config ARM_TEGRA_DEVFREQ + tristate "NVIDIA Tegra30/114/124/210 DEVFREQ Driver" + depends on ARCH_TEGRA_3x_SOC || ARCH_TEGRA_114_SOC || \ + ARCH_TEGRA_132_SOC || ARCH_TEGRA_124_SOC || \ + ARCH_TEGRA_210_SOC || \ + COMPILE_TEST + depends on COMMON_CLK help This adds the DEVFREQ driver for the Tegra family of SoCs. It reads ACTMON counters of memory controllers and adjusts the operating frequencies and voltages with OPP support. +config ARM_TEGRA20_DEVFREQ + tristate "NVIDIA Tegra20 DEVFREQ Driver" + depends on (TEGRA_MC && TEGRA20_EMC) || COMPILE_TEST + depends on COMMON_CLK + select DEVFREQ_GOV_SIMPLE_ONDEMAND + help + This adds the DEVFREQ driver for the Tegra20 family of SoCs. + It reads Memory Controller counters and adjusts the operating + frequencies and voltages with OPP support. + config ARM_RK3399_DMC_DEVFREQ tristate "ARM RK3399 DMC DEVFREQ Driver" - depends on ARCH_ROCKCHIP + depends on (ARCH_ROCKCHIP && HAVE_ARM_SMCCC) || \ + (COMPILE_TEST && HAVE_ARM_SMCCC) select DEVFREQ_EVENT_ROCKCHIP_DFI select DEVFREQ_GOV_SIMPLE_ONDEMAND select PM_DEVFREQ_EVENT - select PM_OPP help - This adds the DEVFREQ driver for the RK3399 DMC(Dynamic Memory Controller). - It sets the frequency for the memory controller and reads the usage counts - from hardware. + This adds the DEVFREQ driver for the RK3399 DMC(Dynamic Memory Controller). + It sets the frequency for the memory controller and reads the usage counts + from hardware. source "drivers/devfreq/event/Kconfig" diff --git a/drivers/devfreq/Makefile b/drivers/devfreq/Makefile index 32b8d4d3f12c..3eb4d5e6635c 100644 --- a/drivers/devfreq/Makefile +++ b/drivers/devfreq/Makefile @@ -9,8 +9,10 @@ obj-$(CONFIG_DEVFREQ_GOV_PASSIVE) += governor_passive.o # DEVFREQ Drivers obj-$(CONFIG_ARM_EXYNOS_BUS_DEVFREQ) += exynos-bus.o +obj-$(CONFIG_ARM_IMX8M_DDRC_DEVFREQ) += imx8m-ddrc.o obj-$(CONFIG_ARM_RK3399_DMC_DEVFREQ) += rk3399_dmc.o -obj-$(CONFIG_ARM_TEGRA_DEVFREQ) += tegra-devfreq.o +obj-$(CONFIG_ARM_TEGRA_DEVFREQ) += tegra30-devfreq.o +obj-$(CONFIG_ARM_TEGRA20_DEVFREQ) += tegra20-devfreq.o # DEVFREQ Event Drivers obj-$(CONFIG_PM_DEVFREQ_EVENT) += event/ diff --git a/drivers/devfreq/devfreq-event.c b/drivers/devfreq/devfreq-event.c index 3dc5fd6065a3..8c31b0f2e28f 100644 --- a/drivers/devfreq/devfreq-event.c +++ b/drivers/devfreq/devfreq-event.c @@ -346,9 +346,9 @@ EXPORT_SYMBOL_GPL(devfreq_event_add_edev); /** * devfreq_event_remove_edev() - Remove the devfreq-event device registered. - * @dev : the devfreq-event device + * @edev : the devfreq-event device * - * Note that this function remove the registered devfreq-event device. + * Note that this function removes the registered devfreq-event device. */ int devfreq_event_remove_edev(struct devfreq_event_dev *edev) { diff --git a/drivers/devfreq/devfreq.c b/drivers/devfreq/devfreq.c index ab22bf8a12d6..cceee8bc3c2f 100644 --- a/drivers/devfreq/devfreq.c +++ b/drivers/devfreq/devfreq.c @@ -10,6 +10,7 @@ #include <linux/kernel.h> #include <linux/kmod.h> #include <linux/sched.h> +#include <linux/debugfs.h> #include <linux/errno.h> #include <linux/err.h> #include <linux/init.h> @@ -24,12 +25,16 @@ #include <linux/printk.h> #include <linux/hrtimer.h> #include <linux/of.h> +#include <linux/pm_qos.h> #include "governor.h" #define CREATE_TRACE_POINTS #include <trace/events/devfreq.h> +#define HZ_PER_KHZ 1000 + static struct class *devfreq_class; +static struct dentry *devfreq_debugfs; /* * devfreq core provides delayed work based load monitoring helper @@ -99,6 +104,54 @@ static unsigned long find_available_max_freq(struct devfreq *devfreq) } /** + * get_freq_range() - Get the current freq range + * @devfreq: the devfreq instance + * @min_freq: the min frequency + * @max_freq: the max frequency + * + * This takes into consideration all constraints. + */ +static void get_freq_range(struct devfreq *devfreq, + unsigned long *min_freq, + unsigned long *max_freq) +{ + unsigned long *freq_table = devfreq->profile->freq_table; + s32 qos_min_freq, qos_max_freq; + + lockdep_assert_held(&devfreq->lock); + + /* + * Initialize minimum/maximum frequency from freq table. + * The devfreq drivers can initialize this in either ascending or + * descending order and devfreq core supports both. + */ + if (freq_table[0] < freq_table[devfreq->profile->max_state - 1]) { + *min_freq = freq_table[0]; + *max_freq = freq_table[devfreq->profile->max_state - 1]; + } else { + *min_freq = freq_table[devfreq->profile->max_state - 1]; + *max_freq = freq_table[0]; + } + + /* Apply constraints from PM QoS */ + qos_min_freq = dev_pm_qos_read_value(devfreq->dev.parent, + DEV_PM_QOS_MIN_FREQUENCY); + qos_max_freq = dev_pm_qos_read_value(devfreq->dev.parent, + DEV_PM_QOS_MAX_FREQUENCY); + *min_freq = max(*min_freq, (unsigned long)HZ_PER_KHZ * qos_min_freq); + if (qos_max_freq != PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE) + *max_freq = min(*max_freq, + (unsigned long)HZ_PER_KHZ * qos_max_freq); + + /* Apply constraints from OPP interface */ + *min_freq = max(*min_freq, devfreq->scaling_min_freq); + *max_freq = min(*max_freq, devfreq->scaling_max_freq); + + if (*min_freq > *max_freq) + *min_freq = *max_freq; +} + +/** * devfreq_get_freq_level() - Lookup freq_table for the frequency * @devfreq: the devfreq instance * @freq: the target frequency @@ -158,9 +211,10 @@ static int set_freq_table(struct devfreq *devfreq) int devfreq_update_status(struct devfreq *devfreq, unsigned long freq) { int lev, prev_lev, ret = 0; - unsigned long cur_time; + u64 cur_time; - cur_time = jiffies; + lockdep_assert_held(&devfreq->lock); + cur_time = get_jiffies_64(); /* Immediately exit if previous_freq is not initialized yet. */ if (!devfreq->previous_freq) @@ -172,8 +226,8 @@ int devfreq_update_status(struct devfreq *devfreq, unsigned long freq) goto out; } - devfreq->time_in_state[prev_lev] += - cur_time - devfreq->last_stat_updated; + devfreq->stats.time_in_state[prev_lev] += + cur_time - devfreq->stats.last_update; lev = devfreq_get_freq_level(devfreq, freq); if (lev < 0) { @@ -182,13 +236,13 @@ int devfreq_update_status(struct devfreq *devfreq, unsigned long freq) } if (lev != prev_lev) { - devfreq->trans_table[(prev_lev * - devfreq->profile->max_state) + lev]++; - devfreq->total_trans++; + devfreq->stats.trans_table[ + (prev_lev * devfreq->profile->max_state) + lev]++; + devfreq->stats.total_trans++; } out: - devfreq->last_stat_updated = cur_time; + devfreq->stats.last_update = cur_time; return ret; } EXPORT_SYMBOL(devfreq_update_status); @@ -254,7 +308,7 @@ static struct devfreq_governor *try_then_request_governor(const char *name) /* Restore previous state before return */ mutex_lock(&devfreq_list_lock); if (err) - return ERR_PTR(err); + return (err < 0) ? ERR_PTR(err) : ERR_PTR(-EINVAL); governor = find_devfreq_governor(name); } @@ -350,16 +404,7 @@ int update_devfreq(struct devfreq *devfreq) err = devfreq->governor->get_target_freq(devfreq, &freq); if (err) return err; - - /* - * Adjust the frequency with user freq, QoS and available freq. - * - * List from the highest priority - * max_freq - * min_freq - */ - max_freq = min(devfreq->scaling_max_freq, devfreq->max_freq); - min_freq = max(devfreq->scaling_min_freq, devfreq->min_freq); + get_freq_range(devfreq, &min_freq, &max_freq); if (freq < min_freq) { freq = min_freq; @@ -402,13 +447,16 @@ static void devfreq_monitor(struct work_struct *work) * devfreq_monitor_start() - Start load monitoring of devfreq instance * @devfreq: the devfreq instance. * - * Helper function for starting devfreq device load monitoing. By + * Helper function for starting devfreq device load monitoring. By * default delayed work based monitoring is supported. Function * to be called from governor in response to DEVFREQ_GOV_START * event when device is added to devfreq framework. */ void devfreq_monitor_start(struct devfreq *devfreq) { + if (devfreq->governor->interrupt_driven) + return; + INIT_DEFERRABLE_WORK(&devfreq->work, devfreq_monitor); if (devfreq->profile->polling_ms) queue_delayed_work(devfreq_wq, &devfreq->work, @@ -420,12 +468,15 @@ EXPORT_SYMBOL(devfreq_monitor_start); * devfreq_monitor_stop() - Stop load monitoring of a devfreq instance * @devfreq: the devfreq instance. * - * Helper function to stop devfreq device load monitoing. Function + * Helper function to stop devfreq device load monitoring. Function * to be called from governor in response to DEVFREQ_GOV_STOP * event when device is removed from devfreq framework. */ void devfreq_monitor_stop(struct devfreq *devfreq) { + if (devfreq->governor->interrupt_driven) + return; + cancel_delayed_work_sync(&devfreq->work); } EXPORT_SYMBOL(devfreq_monitor_stop); @@ -434,7 +485,7 @@ EXPORT_SYMBOL(devfreq_monitor_stop); * devfreq_monitor_suspend() - Suspend load monitoring of a devfreq instance * @devfreq: the devfreq instance. * - * Helper function to suspend devfreq device load monitoing. Function + * Helper function to suspend devfreq device load monitoring. Function * to be called from governor in response to DEVFREQ_GOV_SUSPEND * event or when polling interval is set to zero. * @@ -453,6 +504,10 @@ void devfreq_monitor_suspend(struct devfreq *devfreq) devfreq_update_status(devfreq, devfreq->previous_freq); devfreq->stop_polling = true; mutex_unlock(&devfreq->lock); + + if (devfreq->governor->interrupt_driven) + return; + cancel_delayed_work_sync(&devfreq->work); } EXPORT_SYMBOL(devfreq_monitor_suspend); @@ -461,7 +516,7 @@ EXPORT_SYMBOL(devfreq_monitor_suspend); * devfreq_monitor_resume() - Resume load monitoring of a devfreq instance * @devfreq: the devfreq instance. * - * Helper function to resume devfreq device load monitoing. Function + * Helper function to resume devfreq device load monitoring. Function * to be called from governor in response to DEVFREQ_GOV_RESUME * event or when polling interval is set to non-zero. */ @@ -473,12 +528,16 @@ void devfreq_monitor_resume(struct devfreq *devfreq) if (!devfreq->stop_polling) goto out; + if (devfreq->governor->interrupt_driven) + goto out_update; + if (!delayed_work_pending(&devfreq->work) && devfreq->profile->polling_ms) queue_delayed_work(devfreq_wq, &devfreq->work, msecs_to_jiffies(devfreq->profile->polling_ms)); - devfreq->last_stat_updated = jiffies; +out_update: + devfreq->stats.last_update = get_jiffies_64(); devfreq->stop_polling = false; if (devfreq->profile->get_cur_freq && @@ -509,6 +568,9 @@ void devfreq_interval_update(struct devfreq *devfreq, unsigned int *delay) if (devfreq->stop_polling) goto out; + if (devfreq->governor->interrupt_driven) + goto out; + /* if new delay is zero, stop polling */ if (!new_delay) { mutex_unlock(&devfreq->lock); @@ -550,26 +612,69 @@ static int devfreq_notifier_call(struct notifier_block *nb, unsigned long type, void *devp) { struct devfreq *devfreq = container_of(nb, struct devfreq, nb); - int ret; + int err = -EINVAL; mutex_lock(&devfreq->lock); devfreq->scaling_min_freq = find_available_min_freq(devfreq); - if (!devfreq->scaling_min_freq) { - mutex_unlock(&devfreq->lock); - return -EINVAL; - } + if (!devfreq->scaling_min_freq) + goto out; devfreq->scaling_max_freq = find_available_max_freq(devfreq); if (!devfreq->scaling_max_freq) { - mutex_unlock(&devfreq->lock); - return -EINVAL; + devfreq->scaling_max_freq = ULONG_MAX; + goto out; } - ret = update_devfreq(devfreq); + err = update_devfreq(devfreq); + +out: mutex_unlock(&devfreq->lock); + if (err) + dev_err(devfreq->dev.parent, + "failed to update frequency from OPP notifier (%d)\n", + err); - return ret; + return NOTIFY_OK; +} + +/** + * qos_notifier_call() - Common handler for QoS constraints. + * @devfreq: the devfreq instance. + */ +static int qos_notifier_call(struct devfreq *devfreq) +{ + int err; + + mutex_lock(&devfreq->lock); + err = update_devfreq(devfreq); + mutex_unlock(&devfreq->lock); + if (err) + dev_err(devfreq->dev.parent, + "failed to update frequency from PM QoS (%d)\n", + err); + + return NOTIFY_OK; +} + +/** + * qos_min_notifier_call() - Callback for QoS min_freq changes. + * @nb: Should be devfreq->nb_min + */ +static int qos_min_notifier_call(struct notifier_block *nb, + unsigned long val, void *ptr) +{ + return qos_notifier_call(container_of(nb, struct devfreq, nb_min)); +} + +/** + * qos_max_notifier_call() - Callback for QoS max_freq changes. + * @nb: Should be devfreq->nb_max + */ +static int qos_max_notifier_call(struct notifier_block *nb, + unsigned long val, void *ptr) +{ + return qos_notifier_call(container_of(nb, struct devfreq, nb_max)); } /** @@ -581,16 +686,36 @@ static int devfreq_notifier_call(struct notifier_block *nb, unsigned long type, static void devfreq_dev_release(struct device *dev) { struct devfreq *devfreq = to_devfreq(dev); + int err; mutex_lock(&devfreq_list_lock); - if (IS_ERR(find_device_devfreq(devfreq->dev.parent))) { - mutex_unlock(&devfreq_list_lock); - dev_warn(&devfreq->dev, "releasing devfreq which doesn't exist\n"); - return; - } list_del(&devfreq->node); mutex_unlock(&devfreq_list_lock); + err = dev_pm_qos_remove_notifier(devfreq->dev.parent, &devfreq->nb_max, + DEV_PM_QOS_MAX_FREQUENCY); + if (err && err != -ENOENT) + dev_warn(dev->parent, + "Failed to remove max_freq notifier: %d\n", err); + err = dev_pm_qos_remove_notifier(devfreq->dev.parent, &devfreq->nb_min, + DEV_PM_QOS_MIN_FREQUENCY); + if (err && err != -ENOENT) + dev_warn(dev->parent, + "Failed to remove min_freq notifier: %d\n", err); + + if (dev_pm_qos_request_active(&devfreq->user_max_freq_req)) { + err = dev_pm_qos_remove_request(&devfreq->user_max_freq_req); + if (err) + dev_warn(dev->parent, + "Failed to remove max_freq request: %d\n", err); + } + if (dev_pm_qos_request_active(&devfreq->user_min_freq_req)) { + err = dev_pm_qos_remove_request(&devfreq->user_min_freq_req); + if (err) + dev_warn(dev->parent, + "Failed to remove min_freq request: %d\n", err); + } + if (devfreq->profile->exit) devfreq->profile->exit(devfreq->dev.parent); @@ -625,7 +750,7 @@ struct devfreq *devfreq_add_device(struct device *dev, devfreq = find_device_devfreq(dev); mutex_unlock(&devfreq_list_lock); if (!IS_ERR(devfreq)) { - dev_err(dev, "%s: Unable to create devfreq for the device.\n", + dev_err(dev, "%s: devfreq device already exists!\n", __func__); err = -EINVAL; goto err_out; @@ -642,6 +767,7 @@ struct devfreq *devfreq_add_device(struct device *dev, devfreq->dev.parent = dev; devfreq->dev.class = devfreq_class; devfreq->dev.release = devfreq_dev_release; + INIT_LIST_HEAD(&devfreq->node); devfreq->profile = profile; strncpy(devfreq->governor_name, governor_name, DEVFREQ_NAME_LEN); devfreq->previous_freq = profile->initial_freq; @@ -663,7 +789,6 @@ struct devfreq *devfreq_add_device(struct device *dev, err = -EINVAL; goto err_dev; } - devfreq->min_freq = devfreq->scaling_min_freq; devfreq->scaling_max_freq = find_available_max_freq(devfreq); if (!devfreq->scaling_max_freq) { @@ -671,7 +796,6 @@ struct devfreq *devfreq_add_device(struct device *dev, err = -EINVAL; goto err_dev; } - devfreq->max_freq = devfreq->scaling_max_freq; devfreq->suspend_freq = dev_pm_opp_get_suspend_opp_freq(dev); atomic_set(&devfreq->suspend_count, 0); @@ -685,33 +809,56 @@ struct devfreq *devfreq_add_device(struct device *dev, goto err_out; } - devfreq->trans_table = devm_kzalloc(&devfreq->dev, + devfreq->stats.trans_table = devm_kzalloc(&devfreq->dev, array3_size(sizeof(unsigned int), devfreq->profile->max_state, devfreq->profile->max_state), GFP_KERNEL); - if (!devfreq->trans_table) { + if (!devfreq->stats.trans_table) { mutex_unlock(&devfreq->lock); err = -ENOMEM; goto err_devfreq; } - devfreq->time_in_state = devm_kcalloc(&devfreq->dev, + devfreq->stats.time_in_state = devm_kcalloc(&devfreq->dev, devfreq->profile->max_state, - sizeof(unsigned long), + sizeof(*devfreq->stats.time_in_state), GFP_KERNEL); - if (!devfreq->time_in_state) { + if (!devfreq->stats.time_in_state) { mutex_unlock(&devfreq->lock); err = -ENOMEM; goto err_devfreq; } - devfreq->last_stat_updated = jiffies; + devfreq->stats.total_trans = 0; + devfreq->stats.last_update = get_jiffies_64(); srcu_init_notifier_head(&devfreq->transition_notifier_list); mutex_unlock(&devfreq->lock); + err = dev_pm_qos_add_request(dev, &devfreq->user_min_freq_req, + DEV_PM_QOS_MIN_FREQUENCY, 0); + if (err < 0) + goto err_devfreq; + err = dev_pm_qos_add_request(dev, &devfreq->user_max_freq_req, + DEV_PM_QOS_MAX_FREQUENCY, + PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE); + if (err < 0) + goto err_devfreq; + + devfreq->nb_min.notifier_call = qos_min_notifier_call; + err = dev_pm_qos_add_notifier(devfreq->dev.parent, &devfreq->nb_min, + DEV_PM_QOS_MIN_FREQUENCY); + if (err) + goto err_devfreq; + + devfreq->nb_max.notifier_call = qos_max_notifier_call; + err = dev_pm_qos_add_notifier(devfreq->dev.parent, &devfreq->nb_max, + DEV_PM_QOS_MAX_FREQUENCY); + if (err) + goto err_devfreq; + mutex_lock(&devfreq_list_lock); governor = try_then_request_governor(devfreq->governor_name); @@ -867,7 +1014,7 @@ EXPORT_SYMBOL_GPL(devfreq_get_devfreq_by_phandle); /** * devm_devfreq_remove_device() - Resource-managed devfreq_remove_device() - * @dev: the device to add devfreq feature. + * @dev: the device from which to remove devfreq feature. * @devfreq: the devfreq instance to be removed */ void devm_devfreq_remove_device(struct device *dev, struct devfreq *devfreq) @@ -903,7 +1050,9 @@ int devfreq_suspend_device(struct devfreq *devfreq) } if (devfreq->suspend_freq) { + mutex_lock(&devfreq->lock); ret = devfreq_set_target(devfreq, devfreq->suspend_freq, 0); + mutex_unlock(&devfreq->lock); if (ret) return ret; } @@ -931,7 +1080,9 @@ int devfreq_resume_device(struct devfreq *devfreq) return 0; if (devfreq->resume_freq) { + mutex_lock(&devfreq->lock); ret = devfreq_set_target(devfreq, devfreq->resume_freq, 0); + mutex_unlock(&devfreq->lock); if (ret) return ret; } @@ -1111,6 +1262,14 @@ err_out: } EXPORT_SYMBOL(devfreq_remove_governor); +static ssize_t name_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct devfreq *devfreq = to_devfreq(dev); + return sprintf(buf, "%s\n", dev_name(devfreq->dev.parent)); +} +static DEVICE_ATTR_RO(name); + static ssize_t governor_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -1195,7 +1354,7 @@ static ssize_t available_governors_show(struct device *d, * The devfreq with immutable governor (e.g., passive) shows * only own governor. */ - if (df->governor->immutable) { + if (df->governor && df->governor->immutable) { count = scnprintf(&buf[count], DEVFREQ_NAME_LEN, "%s ", df->governor_name); /* @@ -1281,42 +1440,39 @@ static ssize_t min_freq_store(struct device *dev, struct device_attribute *attr, unsigned long value; int ret; + /* + * Protect against theoretical sysfs writes between + * device_add and dev_pm_qos_add_request + */ + if (!dev_pm_qos_request_active(&df->user_min_freq_req)) + return -EAGAIN; + ret = sscanf(buf, "%lu", &value); if (ret != 1) return -EINVAL; - mutex_lock(&df->lock); - - if (value) { - if (value > df->max_freq) { - ret = -EINVAL; - goto unlock; - } - } else { - unsigned long *freq_table = df->profile->freq_table; - - /* Get minimum frequency according to sorting order */ - if (freq_table[0] < freq_table[df->profile->max_state - 1]) - value = freq_table[0]; - else - value = freq_table[df->profile->max_state - 1]; - } + /* Round down to kHz for PM QoS */ + ret = dev_pm_qos_update_request(&df->user_min_freq_req, + value / HZ_PER_KHZ); + if (ret < 0) + return ret; - df->min_freq = value; - update_devfreq(df); - ret = count; -unlock: - mutex_unlock(&df->lock); - return ret; + return count; } static ssize_t min_freq_show(struct device *dev, struct device_attribute *attr, char *buf) { struct devfreq *df = to_devfreq(dev); + unsigned long min_freq, max_freq; - return sprintf(buf, "%lu\n", max(df->scaling_min_freq, df->min_freq)); + mutex_lock(&df->lock); + get_freq_range(df, &min_freq, &max_freq); + mutex_unlock(&df->lock); + + return sprintf(buf, "%lu\n", min_freq); } +static DEVICE_ATTR_RW(min_freq); static ssize_t max_freq_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) @@ -1325,42 +1481,50 @@ static ssize_t max_freq_store(struct device *dev, struct device_attribute *attr, unsigned long value; int ret; + /* + * Protect against theoretical sysfs writes between + * device_add and dev_pm_qos_add_request + */ + if (!dev_pm_qos_request_active(&df->user_max_freq_req)) + return -EINVAL; + ret = sscanf(buf, "%lu", &value); if (ret != 1) return -EINVAL; - mutex_lock(&df->lock); - - if (value) { - if (value < df->min_freq) { - ret = -EINVAL; - goto unlock; - } - } else { - unsigned long *freq_table = df->profile->freq_table; + /* + * PM QoS frequencies are in kHz so we need to convert. Convert by + * rounding upwards so that the acceptable interval never shrinks. + * + * For example if the user writes "666666666" to sysfs this value will + * be converted to 666667 kHz and back to 666667000 Hz before an OPP + * lookup, this ensures that an OPP of 666666666Hz is still accepted. + * + * A value of zero means "no limit". + */ + if (value) + value = DIV_ROUND_UP(value, HZ_PER_KHZ); + else + value = PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE; - /* Get maximum frequency according to sorting order */ - if (freq_table[0] < freq_table[df->profile->max_state - 1]) - value = freq_table[df->profile->max_state - 1]; - else - value = freq_table[0]; - } + ret = dev_pm_qos_update_request(&df->user_max_freq_req, value); + if (ret < 0) + return ret; - df->max_freq = value; - update_devfreq(df); - ret = count; -unlock: - mutex_unlock(&df->lock); - return ret; + return count; } -static DEVICE_ATTR_RW(min_freq); static ssize_t max_freq_show(struct device *dev, struct device_attribute *attr, char *buf) { struct devfreq *df = to_devfreq(dev); + unsigned long min_freq, max_freq; - return sprintf(buf, "%lu\n", min(df->scaling_max_freq, df->max_freq)); + mutex_lock(&df->lock); + get_freq_range(df, &min_freq, &max_freq); + mutex_unlock(&df->lock); + + return sprintf(buf, "%lu\n", max_freq); } static DEVICE_ATTR_RW(max_freq); @@ -1397,12 +1561,17 @@ static ssize_t trans_stat_show(struct device *dev, int i, j; unsigned int max_state = devfreq->profile->max_state; - if (!devfreq->stop_polling && - devfreq_update_status(devfreq, devfreq->previous_freq)) - return 0; if (max_state == 0) return sprintf(buf, "Not Supported.\n"); + mutex_lock(&devfreq->lock); + if (!devfreq->stop_polling && + devfreq_update_status(devfreq, devfreq->previous_freq)) { + mutex_unlock(&devfreq->lock); + return 0; + } + mutex_unlock(&devfreq->lock); + len = sprintf(buf, " From : To\n"); len += sprintf(buf + len, " :"); for (i = 0; i < max_state; i++) @@ -1422,18 +1591,47 @@ static ssize_t trans_stat_show(struct device *dev, devfreq->profile->freq_table[i]); for (j = 0; j < max_state; j++) len += sprintf(buf + len, "%10u", - devfreq->trans_table[(i * max_state) + j]); - len += sprintf(buf + len, "%10u\n", - jiffies_to_msecs(devfreq->time_in_state[i])); + devfreq->stats.trans_table[(i * max_state) + j]); + + len += sprintf(buf + len, "%10llu\n", (u64) + jiffies64_to_msecs(devfreq->stats.time_in_state[i])); } len += sprintf(buf + len, "Total transition : %u\n", - devfreq->total_trans); + devfreq->stats.total_trans); return len; } -static DEVICE_ATTR_RO(trans_stat); + +static ssize_t trans_stat_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct devfreq *df = to_devfreq(dev); + int err, value; + + if (df->profile->max_state == 0) + return count; + + err = kstrtoint(buf, 10, &value); + if (err || value != 0) + return -EINVAL; + + mutex_lock(&df->lock); + memset(df->stats.time_in_state, 0, (df->profile->max_state * + sizeof(*df->stats.time_in_state))); + memset(df->stats.trans_table, 0, array3_size(sizeof(unsigned int), + df->profile->max_state, + df->profile->max_state)); + df->stats.total_trans = 0; + df->stats.last_update = get_jiffies_64(); + mutex_unlock(&df->lock); + + return count; +} +static DEVICE_ATTR_RW(trans_stat); static struct attribute *devfreq_attrs[] = { + &dev_attr_name.attr, &dev_attr_governor.attr, &dev_attr_available_governors.attr, &dev_attr_cur_freq.attr, @@ -1447,6 +1645,81 @@ static struct attribute *devfreq_attrs[] = { }; ATTRIBUTE_GROUPS(devfreq); +/** + * devfreq_summary_show() - Show the summary of the devfreq devices + * @s: seq_file instance to show the summary of devfreq devices + * @data: not used + * + * Show the summary of the devfreq devices via 'devfreq_summary' debugfs file. + * It helps that user can know the detailed information of the devfreq devices. + * + * Return 0 always because it shows the information without any data change. + */ +static int devfreq_summary_show(struct seq_file *s, void *data) +{ + struct devfreq *devfreq; + struct devfreq *p_devfreq = NULL; + unsigned long cur_freq, min_freq, max_freq; + unsigned int polling_ms; + + seq_printf(s, "%-30s %-10s %-10s %-15s %10s %12s %12s %12s\n", + "dev_name", + "dev", + "parent_dev", + "governor", + "polling_ms", + "cur_freq_Hz", + "min_freq_Hz", + "max_freq_Hz"); + seq_printf(s, "%30s %10s %10s %15s %10s %12s %12s %12s\n", + "------------------------------", + "----------", + "----------", + "---------------", + "----------", + "------------", + "------------", + "------------"); + + mutex_lock(&devfreq_list_lock); + + list_for_each_entry_reverse(devfreq, &devfreq_list, node) { +#if IS_ENABLED(CONFIG_DEVFREQ_GOV_PASSIVE) + if (!strncmp(devfreq->governor_name, DEVFREQ_GOV_PASSIVE, + DEVFREQ_NAME_LEN)) { + struct devfreq_passive_data *data = devfreq->data; + + if (data) + p_devfreq = data->parent; + } else { + p_devfreq = NULL; + } +#endif + + mutex_lock(&devfreq->lock); + cur_freq = devfreq->previous_freq, + get_freq_range(devfreq, &min_freq, &max_freq); + polling_ms = devfreq->profile->polling_ms, + mutex_unlock(&devfreq->lock); + + seq_printf(s, + "%-30s %-10s %-10s %-15s %10d %12ld %12ld %12ld\n", + dev_name(devfreq->dev.parent), + dev_name(&devfreq->dev), + p_devfreq ? dev_name(&p_devfreq->dev) : "null", + devfreq->governor_name, + polling_ms, + cur_freq, + min_freq, + max_freq); + } + + mutex_unlock(&devfreq_list_lock); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(devfreq_summary); + static int __init devfreq_init(void) { devfreq_class = class_create(THIS_MODULE, "devfreq"); @@ -1463,6 +1736,11 @@ static int __init devfreq_init(void) } devfreq_class->dev_groups = devfreq_groups; + devfreq_debugfs = debugfs_create_dir("devfreq", NULL); + debugfs_create_file("devfreq_summary", 0444, + devfreq_debugfs, NULL, + &devfreq_summary_fops); + return 0; } subsys_initcall(devfreq_init); @@ -1656,7 +1934,7 @@ static void devm_devfreq_notifier_release(struct device *dev, void *res) /** * devm_devfreq_register_notifier() - - Resource-managed devfreq_register_notifier() + * - Resource-managed devfreq_register_notifier() * @dev: The devfreq user device. (parent of devfreq) * @devfreq: The devfreq object. * @nb: The notifier block to be unregistered. @@ -1692,7 +1970,7 @@ EXPORT_SYMBOL(devm_devfreq_register_notifier); /** * devm_devfreq_unregister_notifier() - - Resource-managed devfreq_unregister_notifier() + * - Resource-managed devfreq_unregister_notifier() * @dev: The devfreq user device. (parent of devfreq) * @devfreq: The devfreq object. * @nb: The notifier block to be unregistered. diff --git a/drivers/devfreq/event/Kconfig b/drivers/devfreq/event/Kconfig index cef2cf5347ca..878825372f6f 100644 --- a/drivers/devfreq/event/Kconfig +++ b/drivers/devfreq/event/Kconfig @@ -15,7 +15,7 @@ menuconfig PM_DEVFREQ_EVENT if PM_DEVFREQ_EVENT config DEVFREQ_EVENT_EXYNOS_NOCP - tristate "EXYNOS NoC (Network On Chip) Probe DEVFREQ event Driver" + tristate "Exynos NoC (Network On Chip) Probe DEVFREQ event Driver" depends on ARCH_EXYNOS || COMPILE_TEST select PM_OPP select REGMAP_MMIO @@ -24,7 +24,7 @@ config DEVFREQ_EVENT_EXYNOS_NOCP (Network on Chip) Probe counters to measure the bandwidth of AXI bus. config DEVFREQ_EVENT_EXYNOS_PPMU - tristate "EXYNOS PPMU (Platform Performance Monitoring Unit) DEVFREQ event Driver" + tristate "Exynos PPMU (Platform Performance Monitoring Unit) DEVFREQ event Driver" depends on ARCH_EXYNOS || COMPILE_TEST select PM_OPP help @@ -34,7 +34,7 @@ config DEVFREQ_EVENT_EXYNOS_PPMU config DEVFREQ_EVENT_ROCKCHIP_DFI tristate "ROCKCHIP DFI DEVFREQ event Driver" - depends on ARCH_ROCKCHIP + depends on ARCH_ROCKCHIP || COMPILE_TEST help This add the devfreq-event driver for Rockchip SoC. It provides DFI (DDR Monitor Module) driver to count ddr load. diff --git a/drivers/devfreq/event/exynos-nocp.c b/drivers/devfreq/event/exynos-nocp.c index 1c565926db9f..ccc531ee6938 100644 --- a/drivers/devfreq/event/exynos-nocp.c +++ b/drivers/devfreq/event/exynos-nocp.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * exynos-nocp.c - EXYNOS NoC (Network On Chip) Probe support + * exynos-nocp.c - Exynos NoC (Network On Chip) Probe support * * Copyright (c) 2016 Samsung Electronics Co., Ltd. * Author : Chanwoo Choi <cw00.choi@samsung.com> diff --git a/drivers/devfreq/event/exynos-nocp.h b/drivers/devfreq/event/exynos-nocp.h index 55cc96284a36..2d6f08cfd0c5 100644 --- a/drivers/devfreq/event/exynos-nocp.h +++ b/drivers/devfreq/event/exynos-nocp.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * exynos-nocp.h - EXYNOS NoC (Network on Chip) Probe header file + * exynos-nocp.h - Exynos NoC (Network on Chip) Probe header file * * Copyright (c) 2016 Samsung Electronics Co., Ltd. * Author : Chanwoo Choi <cw00.choi@samsung.com> diff --git a/drivers/devfreq/event/exynos-ppmu.c b/drivers/devfreq/event/exynos-ppmu.c index 3ee3dd5653aa..17ed980d9099 100644 --- a/drivers/devfreq/event/exynos-ppmu.c +++ b/drivers/devfreq/event/exynos-ppmu.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * exynos_ppmu.c - EXYNOS PPMU (Platform Performance Monitoring Unit) support + * exynos_ppmu.c - Exynos PPMU (Platform Performance Monitoring Unit) support * * Copyright (c) 2014-2015 Samsung Electronics Co., Ltd. * Author : Chanwoo Choi <cw00.choi@samsung.com> @@ -13,6 +13,7 @@ #include <linux/kernel.h> #include <linux/module.h> #include <linux/of_address.h> +#include <linux/of_device.h> #include <linux/platform_device.h> #include <linux/regmap.h> #include <linux/suspend.h> @@ -20,6 +21,11 @@ #include "exynos-ppmu.h" +enum exynos_ppmu_type { + EXYNOS_TYPE_PPMU, + EXYNOS_TYPE_PPMU_V2, +}; + struct exynos_ppmu_data { struct clk *clk; }; @@ -33,6 +39,7 @@ struct exynos_ppmu { struct regmap *regmap; struct exynos_ppmu_data ppmu; + enum exynos_ppmu_type ppmu_type; }; #define PPMU_EVENT(name) \ @@ -86,19 +93,30 @@ static struct __exynos_ppmu_events { PPMU_EVENT(d1-cpu), PPMU_EVENT(d1-general), PPMU_EVENT(d1-rt), + + /* For Exynos5422 SoC */ + PPMU_EVENT(dmc0_0), + PPMU_EVENT(dmc0_1), + PPMU_EVENT(dmc1_0), + PPMU_EVENT(dmc1_1), }; -static int exynos_ppmu_find_ppmu_id(struct devfreq_event_dev *edev) +static int __exynos_ppmu_find_ppmu_id(const char *edev_name) { int i; for (i = 0; i < ARRAY_SIZE(ppmu_events); i++) - if (!strcmp(edev->desc->name, ppmu_events[i].name)) + if (!strcmp(edev_name, ppmu_events[i].name)) return ppmu_events[i].id; return -EINVAL; } +static int exynos_ppmu_find_ppmu_id(struct devfreq_event_dev *edev) +{ + return __exynos_ppmu_find_ppmu_id(edev->desc->name); +} + /* * The devfreq-event ops structure for PPMU v1.1 */ @@ -151,9 +169,9 @@ static int exynos_ppmu_set_event(struct devfreq_event_dev *edev) if (ret < 0) return ret; - /* Set the event of Read/Write data count */ + /* Set the event of proper data type monitoring */ ret = regmap_write(info->regmap, PPMU_BEVTxSEL(id), - PPMU_RO_DATA_CNT | PPMU_WO_DATA_CNT); + edev->desc->event_type); if (ret < 0) return ret; @@ -365,23 +383,11 @@ static int exynos_ppmu_v2_set_event(struct devfreq_event_dev *edev) if (ret < 0) return ret; - /* Set the event of Read/Write data count */ - switch (id) { - case PPMU_PMNCNT0: - case PPMU_PMNCNT1: - case PPMU_PMNCNT2: - ret = regmap_write(info->regmap, PPMU_V2_CH_EVx_TYPE(id), - PPMU_V2_RO_DATA_CNT | PPMU_V2_WO_DATA_CNT); - if (ret < 0) - return ret; - break; - case PPMU_PMNCNT3: - ret = regmap_write(info->regmap, PPMU_V2_CH_EVx_TYPE(id), - PPMU_V2_EVT3_RW_DATA_CNT); - if (ret < 0) - return ret; - break; - } + /* Set the event of proper data type monitoring */ + ret = regmap_write(info->regmap, PPMU_V2_CH_EVx_TYPE(id), + edev->desc->event_type); + if (ret < 0) + return ret; /* Reset cycle counter/performance counter and enable PPMU */ ret = regmap_read(info->regmap, PPMU_V2_PMNC, &pmnc); @@ -480,31 +486,24 @@ static const struct devfreq_event_ops exynos_ppmu_v2_ops = { static const struct of_device_id exynos_ppmu_id_match[] = { { .compatible = "samsung,exynos-ppmu", - .data = (void *)&exynos_ppmu_ops, + .data = (void *)EXYNOS_TYPE_PPMU, }, { .compatible = "samsung,exynos-ppmu-v2", - .data = (void *)&exynos_ppmu_v2_ops, + .data = (void *)EXYNOS_TYPE_PPMU_V2, }, { /* sentinel */ }, }; MODULE_DEVICE_TABLE(of, exynos_ppmu_id_match); -static struct devfreq_event_ops *exynos_bus_get_ops(struct device_node *np) -{ - const struct of_device_id *match; - - match = of_match_node(exynos_ppmu_id_match, np); - return (struct devfreq_event_ops *)match->data; -} - static int of_get_devfreq_events(struct device_node *np, struct exynos_ppmu *info) { struct devfreq_event_desc *desc; - struct devfreq_event_ops *event_ops; struct device *dev = info->dev; struct device_node *events_np, *node; int i, j, count; + const struct of_device_id *of_id; + int ret; events_np = of_get_child_by_name(np, "events"); if (!events_np) { @@ -512,7 +511,6 @@ static int of_get_devfreq_events(struct device_node *np, "failed to get child node of devfreq-event devices\n"); return -EINVAL; } - event_ops = exynos_bus_get_ops(np); count = of_get_child_count(events_np); desc = devm_kcalloc(dev, count, sizeof(*desc), GFP_KERNEL); @@ -520,6 +518,12 @@ static int of_get_devfreq_events(struct device_node *np, return -ENOMEM; info->num_events = count; + of_id = of_match_device(exynos_ppmu_id_match, dev); + if (of_id) + info->ppmu_type = (enum exynos_ppmu_type)of_id->data; + else + return -EINVAL; + j = 0; for_each_child_of_node(events_np, node) { for (i = 0; i < ARRAY_SIZE(ppmu_events); i++) { @@ -537,10 +541,49 @@ static int of_get_devfreq_events(struct device_node *np, continue; } - desc[j].ops = event_ops; + switch (info->ppmu_type) { + case EXYNOS_TYPE_PPMU: + desc[j].ops = &exynos_ppmu_ops; + break; + case EXYNOS_TYPE_PPMU_V2: + desc[j].ops = &exynos_ppmu_v2_ops; + break; + } + desc[j].driver_data = info; of_property_read_string(node, "event-name", &desc[j].name); + ret = of_property_read_u32(node, "event-data-type", + &desc[j].event_type); + if (ret) { + /* Set the event of proper data type counting. + * Check if the data type has been defined in DT, + * use default if not. + */ + if (info->ppmu_type == EXYNOS_TYPE_PPMU_V2) { + int id; + /* Not all registers take the same value for + * read+write data count. + */ + id = __exynos_ppmu_find_ppmu_id(desc[j].name); + + switch (id) { + case PPMU_PMNCNT0: + case PPMU_PMNCNT1: + case PPMU_PMNCNT2: + desc[j].event_type = PPMU_V2_RO_DATA_CNT + | PPMU_V2_WO_DATA_CNT; + break; + case PPMU_PMNCNT3: + desc[j].event_type = + PPMU_V2_EVT3_RW_DATA_CNT; + break; + } + } else { + desc[j].event_type = PPMU_RO_DATA_CNT | + PPMU_WO_DATA_CNT; + } + } j++; } @@ -633,7 +676,6 @@ static int exynos_ppmu_probe(struct platform_device *pdev) for (i = 0; i < info->num_events; i++) { edev[i] = devm_devfreq_event_add_edev(&pdev->dev, &desc[i]); if (IS_ERR(edev[i])) { - ret = PTR_ERR(edev[i]); dev_err(&pdev->dev, "failed to add devfreq-event device\n"); return PTR_ERR(edev[i]); diff --git a/drivers/devfreq/event/exynos-ppmu.h b/drivers/devfreq/event/exynos-ppmu.h index 284420047455..97f667d0cbdd 100644 --- a/drivers/devfreq/event/exynos-ppmu.h +++ b/drivers/devfreq/event/exynos-ppmu.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * exynos_ppmu.h - EXYNOS PPMU header file + * exynos_ppmu.h - Exynos PPMU header file * * Copyright (c) 2015 Samsung Electronics Co., Ltd. * Author : Chanwoo Choi <cw00.choi@samsung.com> diff --git a/drivers/devfreq/event/rockchip-dfi.c b/drivers/devfreq/event/rockchip-dfi.c index 5d1042188727..9a88faaf8b27 100644 --- a/drivers/devfreq/event/rockchip-dfi.c +++ b/drivers/devfreq/event/rockchip-dfi.c @@ -177,7 +177,6 @@ static int rockchip_dfi_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct rockchip_dfi *data; - struct resource *res; struct devfreq_event_desc *desc; struct device_node *np = pdev->dev.of_node, *node; @@ -185,8 +184,7 @@ static int rockchip_dfi_probe(struct platform_device *pdev) if (!data) return -ENOMEM; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - data->regs = devm_ioremap_resource(&pdev->dev, res); + data->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(data->regs)) return PTR_ERR(data->regs); @@ -200,6 +198,7 @@ static int rockchip_dfi_probe(struct platform_device *pdev) node = of_parse_phandle(np, "rockchip,pmu", 0); if (node) { data->regmap_pmu = syscon_node_to_regmap(node); + of_node_put(node); if (IS_ERR(data->regmap_pmu)) return PTR_ERR(data->regmap_pmu); } diff --git a/drivers/devfreq/exynos-bus.c b/drivers/devfreq/exynos-bus.c index d9f377912c10..8fa8eb541373 100644 --- a/drivers/devfreq/exynos-bus.c +++ b/drivers/devfreq/exynos-bus.c @@ -15,14 +15,12 @@ #include <linux/device.h> #include <linux/export.h> #include <linux/module.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/pm_opp.h> #include <linux/platform_device.h> #include <linux/regulator/consumer.h> -#include <linux/slab.h> #define DEFAULT_SATURATION_RATIO 40 -#define DEFAULT_VOLTAGE_TOLERANCE 2 struct exynos_bus { struct device *dev; @@ -34,9 +32,8 @@ struct exynos_bus { unsigned long curr_freq; - struct regulator *regulator; + struct opp_table *opp_table; struct clk *clk; - unsigned int voltage_tolerance; unsigned int ratio; }; @@ -90,62 +87,29 @@ static int exynos_bus_get_event(struct exynos_bus *bus, } /* - * Must necessary function for devfreq simple-ondemand governor + * devfreq function for both simple-ondemand and passive governor */ static int exynos_bus_target(struct device *dev, unsigned long *freq, u32 flags) { struct exynos_bus *bus = dev_get_drvdata(dev); struct dev_pm_opp *new_opp; - unsigned long old_freq, new_freq, new_volt, tol; int ret = 0; - /* Get new opp-bus instance according to new bus clock */ + /* Get correct frequency for bus. */ new_opp = devfreq_recommended_opp(dev, freq, flags); if (IS_ERR(new_opp)) { dev_err(dev, "failed to get recommended opp instance\n"); return PTR_ERR(new_opp); } - new_freq = dev_pm_opp_get_freq(new_opp); - new_volt = dev_pm_opp_get_voltage(new_opp); dev_pm_opp_put(new_opp); - old_freq = bus->curr_freq; - - if (old_freq == new_freq) - return 0; - tol = new_volt * bus->voltage_tolerance / 100; - /* Change voltage and frequency according to new OPP level */ mutex_lock(&bus->lock); + ret = dev_pm_opp_set_rate(dev, *freq); + if (!ret) + bus->curr_freq = *freq; - if (old_freq < new_freq) { - ret = regulator_set_voltage_tol(bus->regulator, new_volt, tol); - if (ret < 0) { - dev_err(bus->dev, "failed to set voltage\n"); - goto out; - } - } - - ret = clk_set_rate(bus->clk, new_freq); - if (ret < 0) { - dev_err(dev, "failed to change clock of bus\n"); - clk_set_rate(bus->clk, old_freq); - goto out; - } - - if (old_freq > new_freq) { - ret = regulator_set_voltage_tol(bus->regulator, new_volt, tol); - if (ret < 0) { - dev_err(bus->dev, "failed to set voltage\n"); - goto out; - } - } - bus->curr_freq = new_freq; - - dev_dbg(dev, "Set the frequency of bus (%luHz -> %luHz, %luHz)\n", - old_freq, new_freq, clk_get_rate(bus->clk)); -out: mutex_unlock(&bus->lock); return ret; @@ -162,6 +126,7 @@ static int exynos_bus_get_dev_status(struct device *dev, ret = exynos_bus_get_event(bus, &edata); if (ret < 0) { + dev_err(dev, "failed to get event from devfreq-event devices\n"); stat->total_time = stat->busy_time = 0; goto err; } @@ -191,57 +156,12 @@ static void exynos_bus_exit(struct device *dev) if (ret < 0) dev_warn(dev, "failed to disable the devfreq-event devices\n"); - if (bus->regulator) - regulator_disable(bus->regulator); - dev_pm_opp_of_remove_table(dev); clk_disable_unprepare(bus->clk); -} - -/* - * Must necessary function for devfreq passive governor - */ -static int exynos_bus_passive_target(struct device *dev, unsigned long *freq, - u32 flags) -{ - struct exynos_bus *bus = dev_get_drvdata(dev); - struct dev_pm_opp *new_opp; - unsigned long old_freq, new_freq; - int ret = 0; - - /* Get new opp-bus instance according to new bus clock */ - new_opp = devfreq_recommended_opp(dev, freq, flags); - if (IS_ERR(new_opp)) { - dev_err(dev, "failed to get recommended opp instance\n"); - return PTR_ERR(new_opp); - } - - new_freq = dev_pm_opp_get_freq(new_opp); - dev_pm_opp_put(new_opp); - - old_freq = bus->curr_freq; - - if (old_freq == new_freq) - return 0; - - /* Change the frequency according to new OPP level */ - mutex_lock(&bus->lock); - - ret = clk_set_rate(bus->clk, new_freq); - if (ret < 0) { - dev_err(dev, "failed to set the clock of bus\n"); - goto out; + if (bus->opp_table) { + dev_pm_opp_put_regulators(bus->opp_table); + bus->opp_table = NULL; } - - *freq = new_freq; - bus->curr_freq = new_freq; - - dev_dbg(dev, "Set the frequency of bus (%luHz -> %luHz, %luHz)\n", - old_freq, new_freq, clk_get_rate(bus->clk)); -out: - mutex_unlock(&bus->lock); - - return ret; } static void exynos_bus_passive_exit(struct device *dev) @@ -256,21 +176,19 @@ static int exynos_bus_parent_parse_of(struct device_node *np, struct exynos_bus *bus) { struct device *dev = bus->dev; + struct opp_table *opp_table; + const char *vdd = "vdd"; int i, ret, count, size; - /* Get the regulator to provide each bus with the power */ - bus->regulator = devm_regulator_get(dev, "vdd"); - if (IS_ERR(bus->regulator)) { - dev_err(dev, "failed to get VDD regulator\n"); - return PTR_ERR(bus->regulator); - } - - ret = regulator_enable(bus->regulator); - if (ret < 0) { - dev_err(dev, "failed to enable VDD regulator\n"); + opp_table = dev_pm_opp_set_regulators(dev, &vdd, 1); + if (IS_ERR(opp_table)) { + ret = PTR_ERR(opp_table); + dev_err(dev, "failed to set regulators %d\n", ret); return ret; } + bus->opp_table = opp_table; + /* * Get the devfreq-event devices to get the current utilization of * buses. This raw data will be used in devfreq ondemand governor. @@ -311,14 +229,11 @@ static int exynos_bus_parent_parse_of(struct device_node *np, if (of_property_read_u32(np, "exynos,saturation-ratio", &bus->ratio)) bus->ratio = DEFAULT_SATURATION_RATIO; - if (of_property_read_u32(np, "exynos,voltage-tolerance", - &bus->voltage_tolerance)) - bus->voltage_tolerance = DEFAULT_VOLTAGE_TOLERANCE; - return 0; err_regulator: - regulator_disable(bus->regulator); + dev_pm_opp_put_regulators(bus->opp_table); + bus->opp_table = NULL; return ret; } @@ -372,51 +287,12 @@ err_clk: return ret; } -static int exynos_bus_probe(struct platform_device *pdev) +static int exynos_bus_profile_init(struct exynos_bus *bus, + struct devfreq_dev_profile *profile) { - struct device *dev = &pdev->dev; - struct device_node *np = dev->of_node, *node; - struct devfreq_dev_profile *profile; + struct device *dev = bus->dev; struct devfreq_simple_ondemand_data *ondemand_data; - struct devfreq_passive_data *passive_data; - struct devfreq *parent_devfreq; - struct exynos_bus *bus; - int ret, max_state; - unsigned long min_freq, max_freq; - - if (!np) { - dev_err(dev, "failed to find devicetree node\n"); - return -EINVAL; - } - - bus = devm_kzalloc(&pdev->dev, sizeof(*bus), GFP_KERNEL); - if (!bus) - return -ENOMEM; - mutex_init(&bus->lock); - bus->dev = &pdev->dev; - platform_set_drvdata(pdev, bus); - - /* Parse the device-tree to get the resource information */ - ret = exynos_bus_parse_of(np, bus); - if (ret < 0) - return ret; - - profile = devm_kzalloc(dev, sizeof(*profile), GFP_KERNEL); - if (!profile) { - ret = -ENOMEM; - goto err; - } - - node = of_parse_phandle(dev->of_node, "devfreq", 0); - if (node) { - of_node_put(node); - goto passive; - } else { - ret = exynos_bus_parent_parse_of(np, bus); - } - - if (ret < 0) - goto err; + int ret; /* Initialize the struct profile and governor data for parent device */ profile->polling_ms = 50; @@ -425,10 +301,9 @@ static int exynos_bus_probe(struct platform_device *pdev) profile->exit = exynos_bus_exit; ondemand_data = devm_kzalloc(dev, sizeof(*ondemand_data), GFP_KERNEL); - if (!ondemand_data) { - ret = -ENOMEM; - goto err; - } + if (!ondemand_data) + return -ENOMEM; + ondemand_data->upthreshold = 40; ondemand_data->downdifferential = 5; @@ -438,15 +313,14 @@ static int exynos_bus_probe(struct platform_device *pdev) ondemand_data); if (IS_ERR(bus->devfreq)) { dev_err(dev, "failed to add devfreq device\n"); - ret = PTR_ERR(bus->devfreq); - goto err; + return PTR_ERR(bus->devfreq); } /* Register opp_notifier to catch the change of OPP */ ret = devm_devfreq_register_opp_notifier(dev, bus->devfreq); if (ret < 0) { dev_err(dev, "failed to register opp notifier\n"); - goto err; + return ret; } /* @@ -456,33 +330,44 @@ static int exynos_bus_probe(struct platform_device *pdev) ret = exynos_bus_enable_edev(bus); if (ret < 0) { dev_err(dev, "failed to enable devfreq-event devices\n"); - goto err; + return ret; } ret = exynos_bus_set_event(bus); if (ret < 0) { dev_err(dev, "failed to set event to devfreq-event devices\n"); - goto err; + goto err_edev; } - goto out; -passive: + return 0; + +err_edev: + if (exynos_bus_disable_edev(bus)) + dev_warn(dev, "failed to disable the devfreq-event devices\n"); + + return ret; +} + +static int exynos_bus_profile_init_passive(struct exynos_bus *bus, + struct devfreq_dev_profile *profile) +{ + struct device *dev = bus->dev; + struct devfreq_passive_data *passive_data; + struct devfreq *parent_devfreq; + /* Initialize the struct profile and governor data for passive device */ - profile->target = exynos_bus_passive_target; + profile->target = exynos_bus_target; profile->exit = exynos_bus_passive_exit; /* Get the instance of parent devfreq device */ parent_devfreq = devfreq_get_devfreq_by_phandle(dev, 0); - if (IS_ERR(parent_devfreq)) { - ret = -EPROBE_DEFER; - goto err; - } + if (IS_ERR(parent_devfreq)) + return -EPROBE_DEFER; passive_data = devm_kzalloc(dev, sizeof(*passive_data), GFP_KERNEL); - if (!passive_data) { - ret = -ENOMEM; - goto err; - } + if (!passive_data) + return -ENOMEM; + passive_data->parent = parent_devfreq; /* Add devfreq device for exynos bus with passive governor */ @@ -491,11 +376,61 @@ passive: if (IS_ERR(bus->devfreq)) { dev_err(dev, "failed to add devfreq dev with passive governor\n"); - ret = PTR_ERR(bus->devfreq); - goto err; + return PTR_ERR(bus->devfreq); + } + + return 0; +} + +static int exynos_bus_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node, *node; + struct devfreq_dev_profile *profile; + struct exynos_bus *bus; + int ret, max_state; + unsigned long min_freq, max_freq; + bool passive = false; + + if (!np) { + dev_err(dev, "failed to find devicetree node\n"); + return -EINVAL; } -out: + bus = devm_kzalloc(&pdev->dev, sizeof(*bus), GFP_KERNEL); + if (!bus) + return -ENOMEM; + mutex_init(&bus->lock); + bus->dev = &pdev->dev; + platform_set_drvdata(pdev, bus); + + profile = devm_kzalloc(dev, sizeof(*profile), GFP_KERNEL); + if (!profile) + return -ENOMEM; + + node = of_parse_phandle(dev->of_node, "devfreq", 0); + if (node) { + of_node_put(node); + passive = true; + } else { + ret = exynos_bus_parent_parse_of(np, bus); + if (ret < 0) + return ret; + } + + /* Parse the device-tree to get the resource information */ + ret = exynos_bus_parse_of(np, bus); + if (ret < 0) + goto err_reg; + + if (passive) + ret = exynos_bus_profile_init_passive(bus, profile); + else + ret = exynos_bus_profile_init(bus, profile); + + if (ret < 0) + goto err; + max_state = bus->devfreq->profile->max_state; min_freq = (bus->devfreq->profile->freq_table[0] / 1000); max_freq = (bus->devfreq->profile->freq_table[max_state - 1] / 1000); @@ -507,6 +442,11 @@ out: err: dev_pm_opp_of_remove_table(dev); clk_disable_unprepare(bus->clk); +err_reg: + if (!passive) { + dev_pm_opp_put_regulators(bus->opp_table); + bus->opp_table = NULL; + } return ret; } diff --git a/drivers/devfreq/governor.h b/drivers/devfreq/governor.h index bbe5ff9fcecf..dc7533ccc3db 100644 --- a/drivers/devfreq/governor.h +++ b/drivers/devfreq/governor.h @@ -31,6 +31,8 @@ * @name: Governor's name * @immutable: Immutable flag for governor. If the value is 1, * this govenror is never changeable to other governor. + * @interrupt_driven: Devfreq core won't schedule polling work for this + * governor if value is set to 1. * @get_target_freq: Returns desired operating frequency for the device. * Basically, get_target_freq will run * devfreq_dev_profile.get_dev_status() to get the @@ -49,6 +51,7 @@ struct devfreq_governor { const char name[DEVFREQ_NAME_LEN]; const unsigned int immutable; + const unsigned int interrupt_driven; int (*get_target_freq)(struct devfreq *this, unsigned long *freq); int (*event_handler)(struct devfreq *devfreq, unsigned int event, void *data); diff --git a/drivers/devfreq/governor_passive.c b/drivers/devfreq/governor_passive.c index 58308948b863..be6eeab9c814 100644 --- a/drivers/devfreq/governor_passive.c +++ b/drivers/devfreq/governor_passive.c @@ -149,7 +149,6 @@ static int devfreq_passive_notifier_call(struct notifier_block *nb, static int devfreq_passive_event_handler(struct devfreq *devfreq, unsigned int event, void *data) { - struct device *dev = devfreq->dev.parent; struct devfreq_passive_data *p_data = (struct devfreq_passive_data *)devfreq->data; struct devfreq *parent = (struct devfreq *)p_data->parent; @@ -165,12 +164,12 @@ static int devfreq_passive_event_handler(struct devfreq *devfreq, p_data->this = devfreq; nb->notifier_call = devfreq_passive_notifier_call; - ret = devm_devfreq_register_notifier(dev, parent, nb, + ret = devfreq_register_notifier(parent, nb, DEVFREQ_TRANSITION_NOTIFIER); break; case DEVFREQ_GOV_STOP: - devm_devfreq_unregister_notifier(dev, parent, nb, - DEVFREQ_TRANSITION_NOTIFIER); + WARN_ON(devfreq_unregister_notifier(parent, nb, + DEVFREQ_TRANSITION_NOTIFIER)); break; default: break; diff --git a/drivers/devfreq/imx8m-ddrc.c b/drivers/devfreq/imx8m-ddrc.c new file mode 100644 index 000000000000..bc82d3653bff --- /dev/null +++ b/drivers/devfreq/imx8m-ddrc.c @@ -0,0 +1,471 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2019 NXP + */ + +#include <linux/module.h> +#include <linux/device.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> +#include <linux/devfreq.h> +#include <linux/pm_opp.h> +#include <linux/clk.h> +#include <linux/clk-provider.h> +#include <linux/arm-smccc.h> + +#define IMX_SIP_DDR_DVFS 0xc2000004 + +/* Query available frequencies. */ +#define IMX_SIP_DDR_DVFS_GET_FREQ_COUNT 0x10 +#define IMX_SIP_DDR_DVFS_GET_FREQ_INFO 0x11 + +/* + * This should be in a 1:1 mapping with devicetree OPPs but + * firmware provides additional info. + */ +struct imx8m_ddrc_freq { + unsigned long rate; + unsigned long smcarg; + int dram_core_parent_index; + int dram_alt_parent_index; + int dram_apb_parent_index; +}; + +/* Hardware limitation */ +#define IMX8M_DDRC_MAX_FREQ_COUNT 4 + +/* + * i.MX8M DRAM Controller clocks have the following structure (abridged): + * + * +----------+ |\ +------+ + * | dram_pll |-------|M| dram_core | | + * +----------+ |U|---------->| D | + * /--|X| | D | + * dram_alt_root | |/ | R | + * | | C | + * +---------+ | | + * |FIX DIV/4| | | + * +---------+ | | + * composite: | | | + * +----------+ | | | + * | dram_alt |----/ | | + * +----------+ | | + * | dram_apb |-------------------->| | + * +----------+ +------+ + * + * The dram_pll is used for higher rates and dram_alt is used for lower rates. + * + * Frequency switching is implemented in TF-A (via SMC call) and can change the + * configuration of the clocks, including mux parents. The dram_alt and + * dram_apb clocks are "imx composite" and their parent can change too. + * + * We need to prepare/enable the new mux parents head of switching and update + * their information afterwards. + */ +struct imx8m_ddrc { + struct devfreq_dev_profile profile; + struct devfreq *devfreq; + + /* For frequency switching: */ + struct clk *dram_core; + struct clk *dram_pll; + struct clk *dram_alt; + struct clk *dram_apb; + + int freq_count; + struct imx8m_ddrc_freq freq_table[IMX8M_DDRC_MAX_FREQ_COUNT]; +}; + +static struct imx8m_ddrc_freq *imx8m_ddrc_find_freq(struct imx8m_ddrc *priv, + unsigned long rate) +{ + struct imx8m_ddrc_freq *freq; + int i; + + /* + * Firmware reports values in MT/s, so we round-down from Hz + * Rounding is extra generous to ensure a match. + */ + rate = DIV_ROUND_CLOSEST(rate, 250000); + for (i = 0; i < priv->freq_count; ++i) { + freq = &priv->freq_table[i]; + if (freq->rate == rate || + freq->rate + 1 == rate || + freq->rate - 1 == rate) + return freq; + } + + return NULL; +} + +static void imx8m_ddrc_smc_set_freq(int target_freq) +{ + struct arm_smccc_res res; + u32 online_cpus = 0; + int cpu; + + local_irq_disable(); + + for_each_online_cpu(cpu) + online_cpus |= (1 << (cpu * 8)); + + /* change the ddr freqency */ + arm_smccc_smc(IMX_SIP_DDR_DVFS, target_freq, online_cpus, + 0, 0, 0, 0, 0, &res); + + local_irq_enable(); +} + +static struct clk *clk_get_parent_by_index(struct clk *clk, int index) +{ + struct clk_hw *hw; + + hw = clk_hw_get_parent_by_index(__clk_get_hw(clk), index); + + return hw ? hw->clk : NULL; +} + +static int imx8m_ddrc_set_freq(struct device *dev, struct imx8m_ddrc_freq *freq) +{ + struct imx8m_ddrc *priv = dev_get_drvdata(dev); + struct clk *new_dram_core_parent; + struct clk *new_dram_alt_parent; + struct clk *new_dram_apb_parent; + int ret; + + /* + * Fetch new parents + * + * new_dram_alt_parent and new_dram_apb_parent are optional but + * new_dram_core_parent is not. + */ + new_dram_core_parent = clk_get_parent_by_index( + priv->dram_core, freq->dram_core_parent_index - 1); + if (!new_dram_core_parent) { + dev_err(dev, "failed to fetch new dram_core parent\n"); + return -EINVAL; + } + if (freq->dram_alt_parent_index) { + new_dram_alt_parent = clk_get_parent_by_index( + priv->dram_alt, + freq->dram_alt_parent_index - 1); + if (!new_dram_alt_parent) { + dev_err(dev, "failed to fetch new dram_alt parent\n"); + return -EINVAL; + } + } else + new_dram_alt_parent = NULL; + + if (freq->dram_apb_parent_index) { + new_dram_apb_parent = clk_get_parent_by_index( + priv->dram_apb, + freq->dram_apb_parent_index - 1); + if (!new_dram_apb_parent) { + dev_err(dev, "failed to fetch new dram_apb parent\n"); + return -EINVAL; + } + } else + new_dram_apb_parent = NULL; + + /* increase reference counts and ensure clks are ON before switch */ + ret = clk_prepare_enable(new_dram_core_parent); + if (ret) { + dev_err(dev, "failed to enable new dram_core parent: %d\n", + ret); + goto out; + } + ret = clk_prepare_enable(new_dram_alt_parent); + if (ret) { + dev_err(dev, "failed to enable new dram_alt parent: %d\n", + ret); + goto out_disable_core_parent; + } + ret = clk_prepare_enable(new_dram_apb_parent); + if (ret) { + dev_err(dev, "failed to enable new dram_apb parent: %d\n", + ret); + goto out_disable_alt_parent; + } + + imx8m_ddrc_smc_set_freq(freq->smcarg); + + /* update parents in clk tree after switch. */ + ret = clk_set_parent(priv->dram_core, new_dram_core_parent); + if (ret) + dev_warn(dev, "failed to set dram_core parent: %d\n", ret); + if (new_dram_alt_parent) { + ret = clk_set_parent(priv->dram_alt, new_dram_alt_parent); + if (ret) + dev_warn(dev, "failed to set dram_alt parent: %d\n", + ret); + } + if (new_dram_apb_parent) { + ret = clk_set_parent(priv->dram_apb, new_dram_apb_parent); + if (ret) + dev_warn(dev, "failed to set dram_apb parent: %d\n", + ret); + } + + /* + * Explicitly refresh dram PLL rate. + * + * Even if it's marked with CLK_GET_RATE_NOCACHE the rate will not be + * automatically refreshed when clk_get_rate is called on children. + */ + clk_get_rate(priv->dram_pll); + + /* + * clk_set_parent transfer the reference count from old parent. + * now we drop extra reference counts used during the switch + */ + clk_disable_unprepare(new_dram_apb_parent); +out_disable_alt_parent: + clk_disable_unprepare(new_dram_alt_parent); +out_disable_core_parent: + clk_disable_unprepare(new_dram_core_parent); +out: + return ret; +} + +static int imx8m_ddrc_target(struct device *dev, unsigned long *freq, u32 flags) +{ + struct imx8m_ddrc *priv = dev_get_drvdata(dev); + struct imx8m_ddrc_freq *freq_info; + struct dev_pm_opp *new_opp; + unsigned long old_freq, new_freq; + int ret; + + new_opp = devfreq_recommended_opp(dev, freq, flags); + if (IS_ERR(new_opp)) { + ret = PTR_ERR(new_opp); + dev_err(dev, "failed to get recommended opp: %d\n", ret); + return ret; + } + dev_pm_opp_put(new_opp); + + old_freq = clk_get_rate(priv->dram_core); + if (*freq == old_freq) + return 0; + + freq_info = imx8m_ddrc_find_freq(priv, *freq); + if (!freq_info) + return -EINVAL; + + /* + * Read back the clk rate to verify switch was correct and so that + * we can report it on all error paths. + */ + ret = imx8m_ddrc_set_freq(dev, freq_info); + + new_freq = clk_get_rate(priv->dram_core); + if (ret) + dev_err(dev, "ddrc failed freq switch to %lu from %lu: error %d. now at %lu\n", + *freq, old_freq, ret, new_freq); + else if (*freq != new_freq) + dev_err(dev, "ddrc failed freq update to %lu from %lu, now at %lu\n", + *freq, old_freq, new_freq); + else + dev_dbg(dev, "ddrc freq set to %lu (was %lu)\n", + *freq, old_freq); + + return ret; +} + +static int imx8m_ddrc_get_cur_freq(struct device *dev, unsigned long *freq) +{ + struct imx8m_ddrc *priv = dev_get_drvdata(dev); + + *freq = clk_get_rate(priv->dram_core); + + return 0; +} + +static int imx8m_ddrc_get_dev_status(struct device *dev, + struct devfreq_dev_status *stat) +{ + struct imx8m_ddrc *priv = dev_get_drvdata(dev); + + stat->busy_time = 0; + stat->total_time = 0; + stat->current_frequency = clk_get_rate(priv->dram_core); + + return 0; +} + +static int imx8m_ddrc_init_freq_info(struct device *dev) +{ + struct imx8m_ddrc *priv = dev_get_drvdata(dev); + struct arm_smccc_res res; + int index; + + /* An error here means DDR DVFS API not supported by firmware */ + arm_smccc_smc(IMX_SIP_DDR_DVFS, IMX_SIP_DDR_DVFS_GET_FREQ_COUNT, + 0, 0, 0, 0, 0, 0, &res); + priv->freq_count = res.a0; + if (priv->freq_count <= 0 || + priv->freq_count > IMX8M_DDRC_MAX_FREQ_COUNT) + return -ENODEV; + + for (index = 0; index < priv->freq_count; ++index) { + struct imx8m_ddrc_freq *freq = &priv->freq_table[index]; + + arm_smccc_smc(IMX_SIP_DDR_DVFS, IMX_SIP_DDR_DVFS_GET_FREQ_INFO, + index, 0, 0, 0, 0, 0, &res); + /* Result should be strictly positive */ + if ((long)res.a0 <= 0) + return -ENODEV; + + freq->rate = res.a0; + freq->smcarg = index; + freq->dram_core_parent_index = res.a1; + freq->dram_alt_parent_index = res.a2; + freq->dram_apb_parent_index = res.a3; + + /* dram_core has 2 options: dram_pll or dram_alt_root */ + if (freq->dram_core_parent_index != 1 && + freq->dram_core_parent_index != 2) + return -ENODEV; + /* dram_apb and dram_alt have exactly 8 possible parents */ + if (freq->dram_alt_parent_index > 8 || + freq->dram_apb_parent_index > 8) + return -ENODEV; + /* dram_core from alt requires explicit dram_alt parent */ + if (freq->dram_core_parent_index == 2 && + freq->dram_alt_parent_index == 0) + return -ENODEV; + } + + return 0; +} + +static int imx8m_ddrc_check_opps(struct device *dev) +{ + struct imx8m_ddrc *priv = dev_get_drvdata(dev); + struct imx8m_ddrc_freq *freq_info; + struct dev_pm_opp *opp; + unsigned long freq; + int i, opp_count; + + /* Enumerate DT OPPs and disable those not supported by firmware */ + opp_count = dev_pm_opp_get_opp_count(dev); + if (opp_count < 0) + return opp_count; + for (i = 0, freq = 0; i < opp_count; ++i, ++freq) { + opp = dev_pm_opp_find_freq_ceil(dev, &freq); + if (IS_ERR(opp)) { + dev_err(dev, "Failed enumerating OPPs: %ld\n", + PTR_ERR(opp)); + return PTR_ERR(opp); + } + dev_pm_opp_put(opp); + + freq_info = imx8m_ddrc_find_freq(priv, freq); + if (!freq_info) { + dev_info(dev, "Disable unsupported OPP %luHz %luMT/s\n", + freq, DIV_ROUND_CLOSEST(freq, 250000)); + dev_pm_opp_disable(dev, freq); + } + } + + return 0; +} + +static void imx8m_ddrc_exit(struct device *dev) +{ + dev_pm_opp_of_remove_table(dev); +} + +static int imx8m_ddrc_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct imx8m_ddrc *priv; + const char *gov = DEVFREQ_GOV_USERSPACE; + int ret; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + platform_set_drvdata(pdev, priv); + + ret = imx8m_ddrc_init_freq_info(dev); + if (ret) { + dev_err(dev, "failed to init firmware freq info: %d\n", ret); + return ret; + } + + priv->dram_core = devm_clk_get(dev, "core"); + if (IS_ERR(priv->dram_core)) { + ret = PTR_ERR(priv->dram_core); + dev_err(dev, "failed to fetch core clock: %d\n", ret); + return ret; + } + priv->dram_pll = devm_clk_get(dev, "pll"); + if (IS_ERR(priv->dram_pll)) { + ret = PTR_ERR(priv->dram_pll); + dev_err(dev, "failed to fetch pll clock: %d\n", ret); + return ret; + } + priv->dram_alt = devm_clk_get(dev, "alt"); + if (IS_ERR(priv->dram_alt)) { + ret = PTR_ERR(priv->dram_alt); + dev_err(dev, "failed to fetch alt clock: %d\n", ret); + return ret; + } + priv->dram_apb = devm_clk_get(dev, "apb"); + if (IS_ERR(priv->dram_apb)) { + ret = PTR_ERR(priv->dram_apb); + dev_err(dev, "failed to fetch apb clock: %d\n", ret); + return ret; + } + + ret = dev_pm_opp_of_add_table(dev); + if (ret < 0) { + dev_err(dev, "failed to get OPP table\n"); + return ret; + } + + ret = imx8m_ddrc_check_opps(dev); + if (ret < 0) + goto err; + + priv->profile.polling_ms = 1000; + priv->profile.target = imx8m_ddrc_target; + priv->profile.get_dev_status = imx8m_ddrc_get_dev_status; + priv->profile.exit = imx8m_ddrc_exit; + priv->profile.get_cur_freq = imx8m_ddrc_get_cur_freq; + priv->profile.initial_freq = clk_get_rate(priv->dram_core); + + priv->devfreq = devm_devfreq_add_device(dev, &priv->profile, + gov, NULL); + if (IS_ERR(priv->devfreq)) { + ret = PTR_ERR(priv->devfreq); + dev_err(dev, "failed to add devfreq device: %d\n", ret); + goto err; + } + + return 0; + +err: + dev_pm_opp_of_remove_table(dev); + return ret; +} + +static const struct of_device_id imx8m_ddrc_of_match[] = { + { .compatible = "fsl,imx8m-ddrc", }, + { /* sentinel */ }, +}; +MODULE_DEVICE_TABLE(of, imx8m_ddrc_of_match); + +static struct platform_driver imx8m_ddrc_platdrv = { + .probe = imx8m_ddrc_probe, + .driver = { + .name = "imx8m-ddrc-devfreq", + .of_match_table = of_match_ptr(imx8m_ddrc_of_match), + }, +}; +module_platform_driver(imx8m_ddrc_platdrv); + +MODULE_DESCRIPTION("i.MX8M DDR Controller frequency driver"); +MODULE_AUTHOR("Leonard Crestez <leonard.crestez@nxp.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/devfreq/rk3399_dmc.c b/drivers/devfreq/rk3399_dmc.c index 682465fa57e1..24f04f78285b 100644 --- a/drivers/devfreq/rk3399_dmc.c +++ b/drivers/devfreq/rk3399_dmc.c @@ -351,7 +351,7 @@ static int rk3399_dmcfreq_probe(struct platform_device *pdev) /* * Get dram timing and pass it to arm trust firmware, - * the dram drvier in arm trust firmware will get these + * the dram driver in arm trust firmware will get these * timing and to do dram initial. */ if (!of_get_ddr_timings(&data->timing, np)) { @@ -364,7 +364,8 @@ static int rk3399_dmcfreq_probe(struct platform_device *pdev) if (res.a0) { dev_err(dev, "Failed to set dram param: %ld\n", res.a0); - return -EINVAL; + ret = -EINVAL; + goto err_edev; } } } @@ -372,8 +373,11 @@ static int rk3399_dmcfreq_probe(struct platform_device *pdev) node = of_parse_phandle(np, "rockchip,pmu", 0); if (node) { data->regmap_pmu = syscon_node_to_regmap(node); - if (IS_ERR(data->regmap_pmu)) - return PTR_ERR(data->regmap_pmu); + of_node_put(node); + if (IS_ERR(data->regmap_pmu)) { + ret = PTR_ERR(data->regmap_pmu); + goto err_edev; + } } regmap_read(data->regmap_pmu, RK3399_PMUGRF_OS_REG2, &val); @@ -391,7 +395,8 @@ static int rk3399_dmcfreq_probe(struct platform_device *pdev) data->odt_dis_freq = data->timing.lpddr4_odt_dis_freq; break; default: - return -EINVAL; + ret = -EINVAL; + goto err_edev; }; arm_smccc_smc(ROCKCHIP_SIP_DRAM_FREQ, 0, 0, @@ -425,7 +430,8 @@ static int rk3399_dmcfreq_probe(struct platform_device *pdev) */ if (dev_pm_opp_of_add_table(dev)) { dev_err(dev, "Invalid operating-points in device tree.\n"); - return -EINVAL; + ret = -EINVAL; + goto err_edev; } of_property_read_u32(np, "upthreshold", @@ -465,6 +471,9 @@ static int rk3399_dmcfreq_probe(struct platform_device *pdev) err_free_opp: dev_pm_opp_of_remove_table(&pdev->dev); +err_edev: + devfreq_event_disable_edev(data->edev); + return ret; } diff --git a/drivers/devfreq/tegra20-devfreq.c b/drivers/devfreq/tegra20-devfreq.c new file mode 100644 index 000000000000..ff82bac9ee4e --- /dev/null +++ b/drivers/devfreq/tegra20-devfreq.c @@ -0,0 +1,212 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * NVIDIA Tegra20 devfreq driver + * + * Copyright (C) 2019 GRATE-DRIVER project + */ + +#include <linux/clk.h> +#include <linux/devfreq.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> +#include <linux/pm_opp.h> +#include <linux/slab.h> + +#include <soc/tegra/mc.h> + +#include "governor.h" + +#define MC_STAT_CONTROL 0x90 +#define MC_STAT_EMC_CLOCK_LIMIT 0xa0 +#define MC_STAT_EMC_CLOCKS 0xa4 +#define MC_STAT_EMC_CONTROL 0xa8 +#define MC_STAT_EMC_COUNT 0xb8 + +#define EMC_GATHER_CLEAR (1 << 8) +#define EMC_GATHER_ENABLE (3 << 8) + +struct tegra_devfreq { + struct devfreq *devfreq; + struct clk *emc_clock; + void __iomem *regs; +}; + +static int tegra_devfreq_target(struct device *dev, unsigned long *freq, + u32 flags) +{ + struct tegra_devfreq *tegra = dev_get_drvdata(dev); + struct devfreq *devfreq = tegra->devfreq; + struct dev_pm_opp *opp; + unsigned long rate; + int err; + + opp = devfreq_recommended_opp(dev, freq, flags); + if (IS_ERR(opp)) + return PTR_ERR(opp); + + rate = dev_pm_opp_get_freq(opp); + dev_pm_opp_put(opp); + + err = clk_set_min_rate(tegra->emc_clock, rate); + if (err) + return err; + + err = clk_set_rate(tegra->emc_clock, 0); + if (err) + goto restore_min_rate; + + return 0; + +restore_min_rate: + clk_set_min_rate(tegra->emc_clock, devfreq->previous_freq); + + return err; +} + +static int tegra_devfreq_get_dev_status(struct device *dev, + struct devfreq_dev_status *stat) +{ + struct tegra_devfreq *tegra = dev_get_drvdata(dev); + + /* + * EMC_COUNT returns number of memory events, that number is lower + * than the number of clocks. Conversion ratio of 1/8 results in a + * bit higher bandwidth than actually needed, it is good enough for + * the time being because drivers don't support requesting minimum + * needed memory bandwidth yet. + * + * TODO: adjust the ratio value once relevant drivers will support + * memory bandwidth management. + */ + stat->busy_time = readl_relaxed(tegra->regs + MC_STAT_EMC_COUNT); + stat->total_time = readl_relaxed(tegra->regs + MC_STAT_EMC_CLOCKS) / 8; + stat->current_frequency = clk_get_rate(tegra->emc_clock); + + writel_relaxed(EMC_GATHER_CLEAR, tegra->regs + MC_STAT_CONTROL); + writel_relaxed(EMC_GATHER_ENABLE, tegra->regs + MC_STAT_CONTROL); + + return 0; +} + +static struct devfreq_dev_profile tegra_devfreq_profile = { + .polling_ms = 500, + .target = tegra_devfreq_target, + .get_dev_status = tegra_devfreq_get_dev_status, +}; + +static struct tegra_mc *tegra_get_memory_controller(void) +{ + struct platform_device *pdev; + struct device_node *np; + struct tegra_mc *mc; + + np = of_find_compatible_node(NULL, NULL, "nvidia,tegra20-mc-gart"); + if (!np) + return ERR_PTR(-ENOENT); + + pdev = of_find_device_by_node(np); + of_node_put(np); + if (!pdev) + return ERR_PTR(-ENODEV); + + mc = platform_get_drvdata(pdev); + if (!mc) + return ERR_PTR(-EPROBE_DEFER); + + return mc; +} + +static int tegra_devfreq_probe(struct platform_device *pdev) +{ + struct tegra_devfreq *tegra; + struct tegra_mc *mc; + unsigned long max_rate; + unsigned long rate; + int err; + + mc = tegra_get_memory_controller(); + if (IS_ERR(mc)) { + err = PTR_ERR(mc); + dev_err(&pdev->dev, "failed to get memory controller: %d\n", + err); + return err; + } + + tegra = devm_kzalloc(&pdev->dev, sizeof(*tegra), GFP_KERNEL); + if (!tegra) + return -ENOMEM; + + /* EMC is a system-critical clock that is always enabled */ + tegra->emc_clock = devm_clk_get(&pdev->dev, "emc"); + if (IS_ERR(tegra->emc_clock)) { + err = PTR_ERR(tegra->emc_clock); + dev_err(&pdev->dev, "failed to get emc clock: %d\n", err); + return err; + } + + tegra->regs = mc->regs; + + max_rate = clk_round_rate(tegra->emc_clock, ULONG_MAX); + + for (rate = 0; rate <= max_rate; rate++) { + rate = clk_round_rate(tegra->emc_clock, rate); + + err = dev_pm_opp_add(&pdev->dev, rate, 0); + if (err) { + dev_err(&pdev->dev, "failed to add opp: %d\n", err); + goto remove_opps; + } + } + + /* + * Reset statistic gathers state, select global bandwidth for the + * statistics collection mode and set clocks counter saturation + * limit to maximum. + */ + writel_relaxed(0x00000000, tegra->regs + MC_STAT_CONTROL); + writel_relaxed(0x00000000, tegra->regs + MC_STAT_EMC_CONTROL); + writel_relaxed(0xffffffff, tegra->regs + MC_STAT_EMC_CLOCK_LIMIT); + + platform_set_drvdata(pdev, tegra); + + tegra->devfreq = devfreq_add_device(&pdev->dev, &tegra_devfreq_profile, + DEVFREQ_GOV_SIMPLE_ONDEMAND, NULL); + if (IS_ERR(tegra->devfreq)) { + err = PTR_ERR(tegra->devfreq); + goto remove_opps; + } + + return 0; + +remove_opps: + dev_pm_opp_remove_all_dynamic(&pdev->dev); + + return err; +} + +static int tegra_devfreq_remove(struct platform_device *pdev) +{ + struct tegra_devfreq *tegra = platform_get_drvdata(pdev); + + devfreq_remove_device(tegra->devfreq); + dev_pm_opp_remove_all_dynamic(&pdev->dev); + + return 0; +} + +static struct platform_driver tegra_devfreq_driver = { + .probe = tegra_devfreq_probe, + .remove = tegra_devfreq_remove, + .driver = { + .name = "tegra20-devfreq", + }, +}; +module_platform_driver(tegra_devfreq_driver); + +MODULE_ALIAS("platform:tegra20-devfreq"); +MODULE_AUTHOR("Dmitry Osipenko <digetx@gmail.com>"); +MODULE_DESCRIPTION("NVIDIA Tegra20 devfreq driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/devfreq/tegra-devfreq.c b/drivers/devfreq/tegra30-devfreq.c index 35c38aad8b4f..0b65f89d74d5 100644 --- a/drivers/devfreq/tegra-devfreq.c +++ b/drivers/devfreq/tegra30-devfreq.c @@ -11,11 +11,13 @@ #include <linux/devfreq.h> #include <linux/interrupt.h> #include <linux/io.h> +#include <linux/irq.h> #include <linux/module.h> -#include <linux/mod_devicetable.h> +#include <linux/of_device.h> #include <linux/platform_device.h> #include <linux/pm_opp.h> #include <linux/reset.h> +#include <linux/workqueue.h> #include "governor.h" @@ -33,6 +35,8 @@ #define ACTMON_DEV_CTRL_CONSECUTIVE_ABOVE_WMARK_EN BIT(30) #define ACTMON_DEV_CTRL_ENB BIT(31) +#define ACTMON_DEV_CTRL_STOP 0x00000000 + #define ACTMON_DEV_UPPER_WMARK 0x4 #define ACTMON_DEV_LOWER_WMARK 0x8 #define ACTMON_DEV_INIT_AVG 0xc @@ -68,6 +72,8 @@ #define KHZ 1000 +#define KHZ_MAX (ULONG_MAX / KHZ) + /* Assume that the bus is saturated if the utilization is 25% */ #define BUS_SATURATION_RATIO 25 @@ -90,9 +96,10 @@ struct tegra_devfreq_device_config { unsigned int boost_down_threshold; /* - * Threshold of activity (cycles) below which the CPU frequency isn't - * to be taken into account. This is to avoid increasing the EMC - * frequency when the CPU is very busy but not accessing the bus often. + * Threshold of activity (cycles translated to kHz) below which the + * CPU frequency isn't to be taken into account. This is to avoid + * increasing the EMC frequency when the CPU is very busy but not + * accessing the bus often. */ u32 avg_dependency_threshold; }; @@ -102,7 +109,7 @@ enum tegra_actmon_device { MCCPU, }; -static struct tegra_devfreq_device_config actmon_device_configs[] = { +static const struct tegra_devfreq_device_config actmon_device_configs[] = { { /* MCALL: All memory accesses (including from the CPUs) */ .offset = 0x1c0, @@ -117,10 +124,10 @@ static struct tegra_devfreq_device_config actmon_device_configs[] = { .offset = 0x200, .irq_mask = 1 << 25, .boost_up_coeff = 800, - .boost_down_coeff = 90, + .boost_down_coeff = 40, .boost_up_threshold = 27, .boost_down_threshold = 10, - .avg_dependency_threshold = 50000, + .avg_dependency_threshold = 16000, /* 16MHz in kHz units */ }, }; @@ -132,7 +139,6 @@ static struct tegra_devfreq_device_config actmon_device_configs[] = { struct tegra_devfreq_device { const struct tegra_devfreq_device_config *config; void __iomem *regs; - spinlock_t lock; /* Average event count sampled in the last interrupt */ u32 avg_count; @@ -157,9 +163,16 @@ struct tegra_devfreq { struct clk *emc_clock; unsigned long max_freq; unsigned long cur_freq; - struct notifier_block rate_change_nb; + struct notifier_block clk_rate_change_nb; + + struct delayed_work cpufreq_update_work; + struct notifier_block cpu_rate_change_nb; struct tegra_devfreq_device devices[ARRAY_SIZE(actmon_device_configs)]; + + unsigned int irq; + + bool started; }; struct tegra_actmon_emc_ratio { @@ -167,8 +180,8 @@ struct tegra_actmon_emc_ratio { unsigned long emc_freq; }; -static struct tegra_actmon_emc_ratio actmon_emc_ratios[] = { - { 1400000, ULONG_MAX }, +static const struct tegra_actmon_emc_ratio actmon_emc_ratios[] = { + { 1400000, KHZ_MAX }, { 1200000, 750000 }, { 1100000, 600000 }, { 1000000, 500000 }, @@ -179,37 +192,45 @@ static struct tegra_actmon_emc_ratio actmon_emc_ratios[] = { static u32 actmon_readl(struct tegra_devfreq *tegra, u32 offset) { - return readl(tegra->regs + offset); + return readl_relaxed(tegra->regs + offset); } static void actmon_writel(struct tegra_devfreq *tegra, u32 val, u32 offset) { - writel(val, tegra->regs + offset); + writel_relaxed(val, tegra->regs + offset); } static u32 device_readl(struct tegra_devfreq_device *dev, u32 offset) { - return readl(dev->regs + offset); + return readl_relaxed(dev->regs + offset); } static void device_writel(struct tegra_devfreq_device *dev, u32 val, u32 offset) { - writel(val, dev->regs + offset); + writel_relaxed(val, dev->regs + offset); } -static unsigned long do_percent(unsigned long val, unsigned int pct) +static unsigned long do_percent(unsigned long long val, unsigned int pct) { - return val * pct / 100; + val = val * pct; + do_div(val, 100); + + /* + * High freq + high boosting percent + large polling interval are + * resulting in integer overflow when watermarks are calculated. + */ + return min_t(u64, val, U32_MAX); } static void tegra_devfreq_update_avg_wmark(struct tegra_devfreq *tegra, struct tegra_devfreq_device *dev) { - u32 avg = dev->avg_count; u32 avg_band_freq = tegra->max_freq * ACTMON_DEFAULT_AVG_BAND / KHZ; - u32 band = avg_band_freq * ACTMON_SAMPLING_PERIOD; + u32 band = avg_band_freq * tegra->devfreq->profile->polling_ms; + u32 avg; + avg = min(dev->avg_count, U32_MAX - band); device_writel(dev, avg + band, ACTMON_DEV_AVG_UPPER_WMARK); avg = max(dev->avg_count, band); @@ -219,7 +240,7 @@ static void tegra_devfreq_update_avg_wmark(struct tegra_devfreq *tegra, static void tegra_devfreq_update_wmark(struct tegra_devfreq *tegra, struct tegra_devfreq_device *dev) { - u32 val = tegra->cur_freq * ACTMON_SAMPLING_PERIOD; + u32 val = tegra->cur_freq * tegra->devfreq->profile->polling_ms; device_writel(dev, do_percent(val, dev->config->boost_up_threshold), ACTMON_DEV_UPPER_WMARK); @@ -228,21 +249,11 @@ static void tegra_devfreq_update_wmark(struct tegra_devfreq *tegra, ACTMON_DEV_LOWER_WMARK); } -static void actmon_write_barrier(struct tegra_devfreq *tegra) -{ - /* ensure the update has reached the ACTMON */ - wmb(); - actmon_readl(tegra, ACTMON_GLB_STATUS); -} - static void actmon_isr_device(struct tegra_devfreq *tegra, struct tegra_devfreq_device *dev) { - unsigned long flags; u32 intr_status, dev_ctrl; - spin_lock_irqsave(&dev->lock, flags); - dev->avg_count = device_readl(dev, ACTMON_DEV_AVG_COUNT); tegra_devfreq_update_avg_wmark(tegra, dev); @@ -259,10 +270,10 @@ static void actmon_isr_device(struct tegra_devfreq *tegra, dev_ctrl |= ACTMON_DEV_CTRL_CONSECUTIVE_BELOW_WMARK_EN; - if (dev->boost_freq >= tegra->max_freq) + if (dev->boost_freq >= tegra->max_freq) { + dev_ctrl &= ~ACTMON_DEV_CTRL_CONSECUTIVE_ABOVE_WMARK_EN; dev->boost_freq = tegra->max_freq; - else - dev_ctrl |= ACTMON_DEV_CTRL_CONSECUTIVE_ABOVE_WMARK_EN; + } } else if (intr_status & ACTMON_DEV_INTR_CONSECUTIVE_LOWER) { /* * new_boost = old_boost * down_coef @@ -273,51 +284,22 @@ static void actmon_isr_device(struct tegra_devfreq *tegra, dev_ctrl |= ACTMON_DEV_CTRL_CONSECUTIVE_ABOVE_WMARK_EN; - if (dev->boost_freq < (ACTMON_BOOST_FREQ_STEP >> 1)) - dev->boost_freq = 0; - else - dev_ctrl |= ACTMON_DEV_CTRL_CONSECUTIVE_BELOW_WMARK_EN; - } - - if (dev->config->avg_dependency_threshold) { - if (dev->avg_count >= dev->config->avg_dependency_threshold) - dev_ctrl |= ACTMON_DEV_CTRL_CONSECUTIVE_BELOW_WMARK_EN; - else if (dev->boost_freq == 0) + if (dev->boost_freq < (ACTMON_BOOST_FREQ_STEP >> 1)) { dev_ctrl &= ~ACTMON_DEV_CTRL_CONSECUTIVE_BELOW_WMARK_EN; + dev->boost_freq = 0; + } } device_writel(dev, dev_ctrl, ACTMON_DEV_CTRL); device_writel(dev, ACTMON_INTR_STATUS_CLEAR, ACTMON_DEV_INTR_STATUS); - - actmon_write_barrier(tegra); - - spin_unlock_irqrestore(&dev->lock, flags); -} - -static irqreturn_t actmon_isr(int irq, void *data) -{ - struct tegra_devfreq *tegra = data; - bool handled = false; - unsigned int i; - u32 val; - - val = actmon_readl(tegra, ACTMON_GLB_STATUS); - for (i = 0; i < ARRAY_SIZE(tegra->devices); i++) { - if (val & tegra->devices[i].config->irq_mask) { - actmon_isr_device(tegra, tegra->devices + i); - handled = true; - } - } - - return handled ? IRQ_WAKE_THREAD : IRQ_NONE; } static unsigned long actmon_cpu_to_emc_rate(struct tegra_devfreq *tegra, unsigned long cpu_freq) { unsigned int i; - struct tegra_actmon_emc_ratio *ratio = actmon_emc_ratios; + const struct tegra_actmon_emc_ratio *ratio = actmon_emc_ratios; for (i = 0; i < ARRAY_SIZE(actmon_emc_ratios); i++, ratio++) { if (cpu_freq >= ratio->cpu_freq) { @@ -331,112 +313,160 @@ static unsigned long actmon_cpu_to_emc_rate(struct tegra_devfreq *tegra, return 0; } +static unsigned long actmon_device_target_freq(struct tegra_devfreq *tegra, + struct tegra_devfreq_device *dev) +{ + unsigned int avg_sustain_coef; + unsigned long target_freq; + + target_freq = dev->avg_count / tegra->devfreq->profile->polling_ms; + avg_sustain_coef = 100 * 100 / dev->config->boost_up_threshold; + target_freq = do_percent(target_freq, avg_sustain_coef); + + return target_freq; +} + static void actmon_update_target(struct tegra_devfreq *tegra, struct tegra_devfreq_device *dev) { unsigned long cpu_freq = 0; unsigned long static_cpu_emc_freq = 0; - unsigned int avg_sustain_coef; - unsigned long flags; - if (dev->config->avg_dependency_threshold) { - cpu_freq = cpufreq_get(0); - static_cpu_emc_freq = actmon_cpu_to_emc_rate(tegra, cpu_freq); - } - - spin_lock_irqsave(&dev->lock, flags); + dev->target_freq = actmon_device_target_freq(tegra, dev); - dev->target_freq = dev->avg_count / ACTMON_SAMPLING_PERIOD; - avg_sustain_coef = 100 * 100 / dev->config->boost_up_threshold; - dev->target_freq = do_percent(dev->target_freq, avg_sustain_coef); - dev->target_freq += dev->boost_freq; + if (dev->config->avg_dependency_threshold && + dev->config->avg_dependency_threshold <= dev->target_freq) { + cpu_freq = cpufreq_quick_get(0); + static_cpu_emc_freq = actmon_cpu_to_emc_rate(tegra, cpu_freq); - if (dev->avg_count >= dev->config->avg_dependency_threshold) + dev->target_freq += dev->boost_freq; dev->target_freq = max(dev->target_freq, static_cpu_emc_freq); - - spin_unlock_irqrestore(&dev->lock, flags); + } else { + dev->target_freq += dev->boost_freq; + } } static irqreturn_t actmon_thread_isr(int irq, void *data) { struct tegra_devfreq *tegra = data; + bool handled = false; + unsigned int i; + u32 val; mutex_lock(&tegra->devfreq->lock); - update_devfreq(tegra->devfreq); + + val = actmon_readl(tegra, ACTMON_GLB_STATUS); + for (i = 0; i < ARRAY_SIZE(tegra->devices); i++) { + if (val & tegra->devices[i].config->irq_mask) { + actmon_isr_device(tegra, tegra->devices + i); + handled = true; + } + } + + if (handled) + update_devfreq(tegra->devfreq); + mutex_unlock(&tegra->devfreq->lock); - return IRQ_HANDLED; + return handled ? IRQ_HANDLED : IRQ_NONE; } -static int tegra_actmon_rate_notify_cb(struct notifier_block *nb, - unsigned long action, void *ptr) +static int tegra_actmon_clk_notify_cb(struct notifier_block *nb, + unsigned long action, void *ptr) { struct clk_notifier_data *data = ptr; struct tegra_devfreq *tegra; struct tegra_devfreq_device *dev; unsigned int i; - unsigned long flags; if (action != POST_RATE_CHANGE) return NOTIFY_OK; - tegra = container_of(nb, struct tegra_devfreq, rate_change_nb); + tegra = container_of(nb, struct tegra_devfreq, clk_rate_change_nb); tegra->cur_freq = data->new_rate / KHZ; for (i = 0; i < ARRAY_SIZE(tegra->devices); i++) { dev = &tegra->devices[i]; - spin_lock_irqsave(&dev->lock, flags); tegra_devfreq_update_wmark(tegra, dev); - spin_unlock_irqrestore(&dev->lock, flags); } - actmon_write_barrier(tegra); - return NOTIFY_OK; } -static void tegra_actmon_enable_interrupts(struct tegra_devfreq *tegra) +static void tegra_actmon_delayed_update(struct work_struct *work) { - struct tegra_devfreq_device *dev; - u32 val; - unsigned int i; + struct tegra_devfreq *tegra = container_of(work, struct tegra_devfreq, + cpufreq_update_work.work); - for (i = 0; i < ARRAY_SIZE(tegra->devices); i++) { - dev = &tegra->devices[i]; + mutex_lock(&tegra->devfreq->lock); + update_devfreq(tegra->devfreq); + mutex_unlock(&tegra->devfreq->lock); +} + +static unsigned long +tegra_actmon_cpufreq_contribution(struct tegra_devfreq *tegra, + unsigned int cpu_freq) +{ + struct tegra_devfreq_device *actmon_dev = &tegra->devices[MCCPU]; + unsigned long static_cpu_emc_freq, dev_freq; - val = device_readl(dev, ACTMON_DEV_CTRL); - val |= ACTMON_DEV_CTRL_AVG_ABOVE_WMARK_EN; - val |= ACTMON_DEV_CTRL_AVG_BELOW_WMARK_EN; - val |= ACTMON_DEV_CTRL_CONSECUTIVE_BELOW_WMARK_EN; - val |= ACTMON_DEV_CTRL_CONSECUTIVE_ABOVE_WMARK_EN; + dev_freq = actmon_device_target_freq(tegra, actmon_dev); - device_writel(dev, val, ACTMON_DEV_CTRL); - } + /* check whether CPU's freq is taken into account at all */ + if (dev_freq < actmon_dev->config->avg_dependency_threshold) + return 0; + + static_cpu_emc_freq = actmon_cpu_to_emc_rate(tegra, cpu_freq); + + if (dev_freq >= static_cpu_emc_freq) + return 0; - actmon_write_barrier(tegra); + return static_cpu_emc_freq; } -static void tegra_actmon_disable_interrupts(struct tegra_devfreq *tegra) +static int tegra_actmon_cpu_notify_cb(struct notifier_block *nb, + unsigned long action, void *ptr) { - struct tegra_devfreq_device *dev; - u32 val; - unsigned int i; + struct cpufreq_freqs *freqs = ptr; + struct tegra_devfreq *tegra; + unsigned long old, new, delay; - for (i = 0; i < ARRAY_SIZE(tegra->devices); i++) { - dev = &tegra->devices[i]; + if (action != CPUFREQ_POSTCHANGE) + return NOTIFY_OK; - val = device_readl(dev, ACTMON_DEV_CTRL); - val &= ~ACTMON_DEV_CTRL_AVG_ABOVE_WMARK_EN; - val &= ~ACTMON_DEV_CTRL_AVG_BELOW_WMARK_EN; - val &= ~ACTMON_DEV_CTRL_CONSECUTIVE_BELOW_WMARK_EN; - val &= ~ACTMON_DEV_CTRL_CONSECUTIVE_ABOVE_WMARK_EN; + tegra = container_of(nb, struct tegra_devfreq, cpu_rate_change_nb); + + /* + * Quickly check whether CPU frequency should be taken into account + * at all, without blocking CPUFreq's core. + */ + if (mutex_trylock(&tegra->devfreq->lock)) { + old = tegra_actmon_cpufreq_contribution(tegra, freqs->old); + new = tegra_actmon_cpufreq_contribution(tegra, freqs->new); + mutex_unlock(&tegra->devfreq->lock); - device_writel(dev, val, ACTMON_DEV_CTRL); + /* + * If CPU's frequency shouldn't be taken into account at + * the moment, then there is no need to update the devfreq's + * state because ISR will re-check CPU's frequency on the + * next interrupt. + */ + if (old == new) + return NOTIFY_OK; } - actmon_write_barrier(tegra); + /* + * CPUFreq driver should support CPUFREQ_ASYNC_NOTIFICATION in order + * to allow asynchronous notifications. This means we can't block + * here for too long, otherwise CPUFreq's core will complain with a + * warning splat. + */ + delay = msecs_to_jiffies(ACTMON_SAMPLING_PERIOD); + schedule_delayed_work(&tegra->cpufreq_update_work, delay); + + return NOTIFY_OK; } static void tegra_actmon_configure_device(struct tegra_devfreq *tegra, @@ -444,9 +474,12 @@ static void tegra_actmon_configure_device(struct tegra_devfreq *tegra, { u32 val = 0; + /* reset boosting on governor's restart */ + dev->boost_freq = 0; + dev->target_freq = tegra->cur_freq; - dev->avg_count = tegra->cur_freq * ACTMON_SAMPLING_PERIOD; + dev->avg_count = tegra->cur_freq * tegra->devfreq->profile->polling_ms; device_writel(dev, dev->avg_count, ACTMON_DEV_INIT_AVG); tegra_devfreq_update_avg_wmark(tegra, dev); @@ -462,34 +495,151 @@ static void tegra_actmon_configure_device(struct tegra_devfreq *tegra, << ACTMON_DEV_CTRL_CONSECUTIVE_BELOW_WMARK_NUM_SHIFT; val |= (ACTMON_ABOVE_WMARK_WINDOW - 1) << ACTMON_DEV_CTRL_CONSECUTIVE_ABOVE_WMARK_NUM_SHIFT; + val |= ACTMON_DEV_CTRL_AVG_ABOVE_WMARK_EN; + val |= ACTMON_DEV_CTRL_AVG_BELOW_WMARK_EN; + val |= ACTMON_DEV_CTRL_CONSECUTIVE_ABOVE_WMARK_EN; val |= ACTMON_DEV_CTRL_ENB; device_writel(dev, val, ACTMON_DEV_CTRL); +} + +static void tegra_actmon_stop_devices(struct tegra_devfreq *tegra) +{ + struct tegra_devfreq_device *dev = tegra->devices; + unsigned int i; - actmon_write_barrier(tegra); + for (i = 0; i < ARRAY_SIZE(tegra->devices); i++, dev++) { + device_writel(dev, ACTMON_DEV_CTRL_STOP, ACTMON_DEV_CTRL); + device_writel(dev, ACTMON_INTR_STATUS_CLEAR, + ACTMON_DEV_INTR_STATUS); + } +} + +static int tegra_actmon_resume(struct tegra_devfreq *tegra) +{ + unsigned int i; + int err; + + if (!tegra->devfreq->profile->polling_ms || !tegra->started) + return 0; + + actmon_writel(tegra, tegra->devfreq->profile->polling_ms - 1, + ACTMON_GLB_PERIOD_CTRL); + + /* + * CLK notifications are needed in order to reconfigure the upper + * consecutive watermark in accordance to the actual clock rate + * to avoid unnecessary upper interrupts. + */ + err = clk_notifier_register(tegra->emc_clock, + &tegra->clk_rate_change_nb); + if (err) { + dev_err(tegra->devfreq->dev.parent, + "Failed to register rate change notifier\n"); + return err; + } + + tegra->cur_freq = clk_get_rate(tegra->emc_clock) / KHZ; + + for (i = 0; i < ARRAY_SIZE(tegra->devices); i++) + tegra_actmon_configure_device(tegra, &tegra->devices[i]); + + /* + * We are estimating CPU's memory bandwidth requirement based on + * amount of memory accesses and system's load, judging by CPU's + * frequency. We also don't want to receive events about CPU's + * frequency transaction when governor is stopped, hence notifier + * is registered dynamically. + */ + err = cpufreq_register_notifier(&tegra->cpu_rate_change_nb, + CPUFREQ_TRANSITION_NOTIFIER); + if (err) { + dev_err(tegra->devfreq->dev.parent, + "Failed to register rate change notifier: %d\n", err); + goto err_stop; + } + + enable_irq(tegra->irq); + + return 0; + +err_stop: + tegra_actmon_stop_devices(tegra); + + clk_notifier_unregister(tegra->emc_clock, &tegra->clk_rate_change_nb); + + return err; +} + +static int tegra_actmon_start(struct tegra_devfreq *tegra) +{ + int ret = 0; + + if (!tegra->started) { + tegra->started = true; + + ret = tegra_actmon_resume(tegra); + if (ret) + tegra->started = false; + } + + return ret; +} + +static void tegra_actmon_pause(struct tegra_devfreq *tegra) +{ + if (!tegra->devfreq->profile->polling_ms || !tegra->started) + return; + + disable_irq(tegra->irq); + + cpufreq_unregister_notifier(&tegra->cpu_rate_change_nb, + CPUFREQ_TRANSITION_NOTIFIER); + + cancel_delayed_work_sync(&tegra->cpufreq_update_work); + + tegra_actmon_stop_devices(tegra); + + clk_notifier_unregister(tegra->emc_clock, &tegra->clk_rate_change_nb); +} + +static void tegra_actmon_stop(struct tegra_devfreq *tegra) +{ + tegra_actmon_pause(tegra); + tegra->started = false; } static int tegra_devfreq_target(struct device *dev, unsigned long *freq, u32 flags) { struct tegra_devfreq *tegra = dev_get_drvdata(dev); + struct devfreq *devfreq = tegra->devfreq; struct dev_pm_opp *opp; - unsigned long rate = *freq * KHZ; + unsigned long rate; + int err; - opp = devfreq_recommended_opp(dev, &rate, flags); + opp = devfreq_recommended_opp(dev, freq, flags); if (IS_ERR(opp)) { - dev_err(dev, "Failed to find opp for %lu KHz\n", *freq); + dev_err(dev, "Failed to find opp for %lu Hz\n", *freq); return PTR_ERR(opp); } rate = dev_pm_opp_get_freq(opp); dev_pm_opp_put(opp); - clk_set_min_rate(tegra->emc_clock, rate); - clk_set_rate(tegra->emc_clock, 0); + err = clk_set_min_rate(tegra->emc_clock, rate * KHZ); + if (err) + return err; - *freq = rate; + err = clk_set_rate(tegra->emc_clock, 0); + if (err) + goto restore_min_rate; return 0; + +restore_min_rate: + clk_set_min_rate(tegra->emc_clock, devfreq->previous_freq); + + return err; } static int tegra_devfreq_get_dev_status(struct device *dev, @@ -497,13 +647,15 @@ static int tegra_devfreq_get_dev_status(struct device *dev, { struct tegra_devfreq *tegra = dev_get_drvdata(dev); struct tegra_devfreq_device *actmon_dev; + unsigned long cur_freq; - stat->current_frequency = tegra->cur_freq; + cur_freq = READ_ONCE(tegra->cur_freq); /* To be used by the tegra governor */ stat->private_data = tegra; /* The below are to be used by the other governors */ + stat->current_frequency = cur_freq; actmon_dev = &tegra->devices[MCALL]; @@ -514,7 +666,7 @@ static int tegra_devfreq_get_dev_status(struct device *dev, stat->busy_time *= 100 / BUS_SATURATION_RATIO; /* Number of cycles in a sampling period */ - stat->total_time = ACTMON_SAMPLING_PERIOD * tegra->cur_freq; + stat->total_time = tegra->devfreq->profile->polling_ms * cur_freq; stat->busy_time = min(stat->busy_time, stat->total_time); @@ -522,7 +674,7 @@ static int tegra_devfreq_get_dev_status(struct device *dev, } static struct devfreq_dev_profile tegra_devfreq_profile = { - .polling_ms = 0, + .polling_ms = ACTMON_SAMPLING_PERIOD, .target = tegra_devfreq_target, .get_dev_status = tegra_devfreq_get_dev_status, }; @@ -562,55 +714,77 @@ static int tegra_governor_event_handler(struct devfreq *devfreq, unsigned int event, void *data) { struct tegra_devfreq *tegra = dev_get_drvdata(devfreq->dev.parent); + unsigned int *new_delay = data; + int ret = 0; + + /* + * Couple devfreq-device with the governor early because it is + * needed at the moment of governor's start (used by ISR). + */ + tegra->devfreq = devfreq; switch (event) { case DEVFREQ_GOV_START: devfreq_monitor_start(devfreq); - tegra_actmon_enable_interrupts(tegra); + ret = tegra_actmon_start(tegra); break; case DEVFREQ_GOV_STOP: - tegra_actmon_disable_interrupts(tegra); + tegra_actmon_stop(tegra); devfreq_monitor_stop(devfreq); break; + case DEVFREQ_GOV_INTERVAL: + /* + * ACTMON hardware supports up to 256 milliseconds for the + * sampling period. + */ + if (*new_delay > 256) { + ret = -EINVAL; + break; + } + + tegra_actmon_pause(tegra); + devfreq_interval_update(devfreq, new_delay); + ret = tegra_actmon_resume(tegra); + break; + case DEVFREQ_GOV_SUSPEND: - tegra_actmon_disable_interrupts(tegra); + tegra_actmon_stop(tegra); devfreq_monitor_suspend(devfreq); break; case DEVFREQ_GOV_RESUME: devfreq_monitor_resume(devfreq); - tegra_actmon_enable_interrupts(tegra); + ret = tegra_actmon_start(tegra); break; } - return 0; + return ret; } static struct devfreq_governor tegra_devfreq_governor = { .name = "tegra_actmon", .get_target_freq = tegra_governor_get_target, .event_handler = tegra_governor_event_handler, + .immutable = true, + .interrupt_driven = true, }; static int tegra_devfreq_probe(struct platform_device *pdev) { - struct tegra_devfreq *tegra; struct tegra_devfreq_device *dev; - struct resource *res; + struct tegra_devfreq *tegra; + struct devfreq *devfreq; unsigned int i; - unsigned long rate; - int irq; + long rate; int err; tegra = devm_kzalloc(&pdev->dev, sizeof(*tegra), GFP_KERNEL); if (!tegra) return -ENOMEM; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - - tegra->regs = devm_ioremap_resource(&pdev->dev, res); + tegra->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(tegra->regs)) return PTR_ERR(tegra->regs); @@ -632,13 +806,20 @@ static int tegra_devfreq_probe(struct platform_device *pdev) return PTR_ERR(tegra->emc_clock); } - clk_set_rate(tegra->emc_clock, ULONG_MAX); + err = platform_get_irq(pdev, 0); + if (err < 0) { + dev_err(&pdev->dev, "Failed to get IRQ: %d\n", err); + return err; + } + tegra->irq = err; - tegra->rate_change_nb.notifier_call = tegra_actmon_rate_notify_cb; - err = clk_notifier_register(tegra->emc_clock, &tegra->rate_change_nb); + irq_set_status_flags(tegra->irq, IRQ_NOAUTOEN); + + err = devm_request_threaded_irq(&pdev->dev, tegra->irq, NULL, + actmon_thread_isr, IRQF_ONESHOT, + "tegra-devfreq", tegra); if (err) { - dev_err(&pdev->dev, - "Failed to register rate change notifier\n"); + dev_err(&pdev->dev, "Interrupt request failed: %d\n", err); return err; } @@ -653,76 +834,92 @@ static int tegra_devfreq_probe(struct platform_device *pdev) reset_control_deassert(tegra->reset); - tegra->max_freq = clk_round_rate(tegra->emc_clock, ULONG_MAX) / KHZ; - tegra->cur_freq = clk_get_rate(tegra->emc_clock) / KHZ; + rate = clk_round_rate(tegra->emc_clock, ULONG_MAX); + if (rate < 0) { + dev_err(&pdev->dev, "Failed to round clock rate: %ld\n", rate); + return rate; + } - actmon_writel(tegra, ACTMON_SAMPLING_PERIOD - 1, - ACTMON_GLB_PERIOD_CTRL); + tegra->max_freq = rate / KHZ; for (i = 0; i < ARRAY_SIZE(actmon_device_configs); i++) { dev = tegra->devices + i; dev->config = actmon_device_configs + i; dev->regs = tegra->regs + dev->config->offset; - spin_lock_init(&dev->lock); - - tegra_actmon_configure_device(tegra, dev); } for (rate = 0; rate <= tegra->max_freq * KHZ; rate++) { rate = clk_round_rate(tegra->emc_clock, rate); - dev_pm_opp_add(&pdev->dev, rate, 0); - } - irq = platform_get_irq(pdev, 0); - if (irq < 0) { - dev_err(&pdev->dev, "Failed to get IRQ: %d\n", irq); - return irq; + if (rate < 0) { + dev_err(&pdev->dev, + "Failed to round clock rate: %ld\n", rate); + err = rate; + goto remove_opps; + } + + err = dev_pm_opp_add(&pdev->dev, rate / KHZ, 0); + if (err) { + dev_err(&pdev->dev, "Failed to add OPP: %d\n", err); + goto remove_opps; + } } platform_set_drvdata(pdev, tegra); - err = devm_request_threaded_irq(&pdev->dev, irq, actmon_isr, - actmon_thread_isr, IRQF_SHARED, - "tegra-devfreq", tegra); + tegra->clk_rate_change_nb.notifier_call = tegra_actmon_clk_notify_cb; + tegra->cpu_rate_change_nb.notifier_call = tegra_actmon_cpu_notify_cb; + + INIT_DELAYED_WORK(&tegra->cpufreq_update_work, + tegra_actmon_delayed_update); + + err = devfreq_add_governor(&tegra_devfreq_governor); if (err) { - dev_err(&pdev->dev, "Interrupt request failed\n"); - return err; + dev_err(&pdev->dev, "Failed to add governor: %d\n", err); + goto remove_opps; } tegra_devfreq_profile.initial_freq = clk_get_rate(tegra->emc_clock); - tegra->devfreq = devm_devfreq_add_device(&pdev->dev, - &tegra_devfreq_profile, - "tegra_actmon", - NULL); + tegra_devfreq_profile.initial_freq /= KHZ; + + devfreq = devfreq_add_device(&pdev->dev, &tegra_devfreq_profile, + "tegra_actmon", NULL); + if (IS_ERR(devfreq)) { + err = PTR_ERR(devfreq); + goto remove_governor; + } return 0; + +remove_governor: + devfreq_remove_governor(&tegra_devfreq_governor); + +remove_opps: + dev_pm_opp_remove_all_dynamic(&pdev->dev); + + reset_control_reset(tegra->reset); + clk_disable_unprepare(tegra->clock); + + return err; } static int tegra_devfreq_remove(struct platform_device *pdev) { struct tegra_devfreq *tegra = platform_get_drvdata(pdev); - int irq = platform_get_irq(pdev, 0); - u32 val; - unsigned int i; - - for (i = 0; i < ARRAY_SIZE(actmon_device_configs); i++) { - val = device_readl(&tegra->devices[i], ACTMON_DEV_CTRL); - val &= ~ACTMON_DEV_CTRL_ENB; - device_writel(&tegra->devices[i], val, ACTMON_DEV_CTRL); - } - actmon_write_barrier(tegra); + devfreq_remove_device(tegra->devfreq); + devfreq_remove_governor(&tegra_devfreq_governor); - devm_free_irq(&pdev->dev, irq, tegra); - - clk_notifier_unregister(tegra->emc_clock, &tegra->rate_change_nb); + dev_pm_opp_remove_all_dynamic(&pdev->dev); + reset_control_reset(tegra->reset); clk_disable_unprepare(tegra->clock); return 0; } static const struct of_device_id tegra_devfreq_of_match[] = { + { .compatible = "nvidia,tegra30-actmon" }, { .compatible = "nvidia,tegra124-actmon" }, { }, }; @@ -737,36 +934,7 @@ static struct platform_driver tegra_devfreq_driver = { .of_match_table = tegra_devfreq_of_match, }, }; - -static int __init tegra_devfreq_init(void) -{ - int ret = 0; - - ret = devfreq_add_governor(&tegra_devfreq_governor); - if (ret) { - pr_err("%s: failed to add governor: %d\n", __func__, ret); - return ret; - } - - ret = platform_driver_register(&tegra_devfreq_driver); - if (ret) - devfreq_remove_governor(&tegra_devfreq_governor); - - return ret; -} -module_init(tegra_devfreq_init) - -static void __exit tegra_devfreq_exit(void) -{ - int ret = 0; - - platform_driver_unregister(&tegra_devfreq_driver); - - ret = devfreq_remove_governor(&tegra_devfreq_governor); - if (ret) - pr_err("%s: failed to remove governor: %d\n", __func__, ret); -} -module_exit(tegra_devfreq_exit) +module_platform_driver(tegra_devfreq_driver); MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("Tegra devfreq driver"); |
