diff options
| -rw-r--r-- | block/blk-cgroup.c | 52 | ||||
| -rw-r--r-- | block/blk-cgroup.h | 60 | ||||
| -rw-r--r-- | block/blk-throttle.c | 37 | ||||
| -rw-r--r-- | block/cfq-iosched.c | 46 |
4 files changed, 112 insertions, 83 deletions
diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index 997570329517..3d495528a765 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -58,11 +58,6 @@ static bool blkcg_policy_enabled(struct request_queue *q, return pol && test_bit(pol->plid, q->blkcg_pols); } -static size_t blkg_pd_size(const struct blkcg_policy *pol) -{ - return sizeof(struct blkg_policy_data) + pol->pdata_size; -} - /** * blkg_free - free a blkg * @blkg: blkg to free @@ -122,7 +117,7 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q) continue; /* alloc per-policy data and attach it to blkg */ - pd = kzalloc_node(blkg_pd_size(pol), GFP_ATOMIC, q->node); + pd = kzalloc_node(pol->pd_size, GFP_ATOMIC, q->node); if (!pd) { blkg_free(blkg); return NULL; @@ -346,7 +341,8 @@ static const char *blkg_dev_name(struct blkcg_gq *blkg) * cftype->read_seq_string method. */ void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg, - u64 (*prfill)(struct seq_file *, void *, int), + u64 (*prfill)(struct seq_file *, + struct blkg_policy_data *, int), const struct blkcg_policy *pol, int data, bool show_total) { @@ -357,7 +353,7 @@ void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg, spin_lock_irq(&blkcg->lock); hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) if (blkcg_policy_enabled(blkg->q, pol)) - total += prfill(sf, blkg->pd[pol->plid]->pdata, data); + total += prfill(sf, blkg->pd[pol->plid], data); spin_unlock_irq(&blkcg->lock); if (show_total) @@ -368,14 +364,14 @@ EXPORT_SYMBOL_GPL(blkcg_print_blkgs); /** * __blkg_prfill_u64 - prfill helper for a single u64 value * @sf: seq_file to print to - * @pdata: policy private data of interest + * @pd: policy private data of interest * @v: value to print * - * Print @v to @sf for the device assocaited with @pdata. + * Print @v to @sf for the device assocaited with @pd. */ -u64 __blkg_prfill_u64(struct seq_file *sf, void *pdata, u64 v) +u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v) { - const char *dname = blkg_dev_name(pdata_to_blkg(pdata)); + const char *dname = blkg_dev_name(pd->blkg); if (!dname) return 0; @@ -388,12 +384,12 @@ EXPORT_SYMBOL_GPL(__blkg_prfill_u64); /** * __blkg_prfill_rwstat - prfill helper for a blkg_rwstat * @sf: seq_file to print to - * @pdata: policy private data of interest + * @pd: policy private data of interest * @rwstat: rwstat to print * - * Print @rwstat to @sf for the device assocaited with @pdata. + * Print @rwstat to @sf for the device assocaited with @pd. */ -u64 __blkg_prfill_rwstat(struct seq_file *sf, void *pdata, +u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd, const struct blkg_rwstat *rwstat) { static const char *rwstr[] = { @@ -402,7 +398,7 @@ u64 __blkg_prfill_rwstat(struct seq_file *sf, void *pdata, [BLKG_RWSTAT_SYNC] = "Sync", [BLKG_RWSTAT_ASYNC] = "Async", }; - const char *dname = blkg_dev_name(pdata_to_blkg(pdata)); + const char *dname = blkg_dev_name(pd->blkg); u64 v; int i; @@ -421,30 +417,31 @@ u64 __blkg_prfill_rwstat(struct seq_file *sf, void *pdata, /** * blkg_prfill_stat - prfill callback for blkg_stat * @sf: seq_file to print to - * @pdata: policy private data of interest - * @off: offset to the blkg_stat in @pdata + * @pd: policy private data of interest + * @off: offset to the blkg_stat in @pd * * prfill callback for printing a blkg_stat. */ -u64 blkg_prfill_stat(struct seq_file *sf, void *pdata, int off) +u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd, int off) { - return __blkg_prfill_u64(sf, pdata, blkg_stat_read(pdata + off)); + return __blkg_prfill_u64(sf, pd, blkg_stat_read((void *)pd + off)); } EXPORT_SYMBOL_GPL(blkg_prfill_stat); /** * blkg_prfill_rwstat - prfill callback for blkg_rwstat * @sf: seq_file to print to - * @pdata: policy private data of interest - * @off: offset to the blkg_rwstat in @pdata + * @pd: policy private data of interest + * @off: offset to the blkg_rwstat in @pd * * prfill callback for printing a blkg_rwstat. */ -u64 blkg_prfill_rwstat(struct seq_file *sf, void *pdata, int off) +u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd, + int off) { - struct blkg_rwstat rwstat = blkg_rwstat_read(pdata + off); + struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd + off); - return __blkg_prfill_rwstat(sf, pdata, &rwstat); + return __blkg_prfill_rwstat(sf, pd, &rwstat); } EXPORT_SYMBOL_GPL(blkg_prfill_rwstat); @@ -733,7 +730,7 @@ int blkcg_activate_policy(struct request_queue *q, /* allocate policy_data for all existing blkgs */ while (cnt--) { - pd = kzalloc_node(blkg_pd_size(pol), GFP_KERNEL, q->node); + pd = kzalloc_node(pol->pd_size, GFP_KERNEL, q->node); if (!pd) { ret = -ENOMEM; goto out_free; @@ -832,6 +829,9 @@ int blkcg_policy_register(struct blkcg_policy *pol) { int i, ret; + if (WARN_ON(pol->pd_size < sizeof(struct blkg_policy_data))) + return -EINVAL; + mutex_lock(&blkcg_pol_mutex); /* find an empty slot */ diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h index a443b84d2c16..18b021e1c05f 100644 --- a/block/blk-cgroup.h +++ b/block/blk-cgroup.h @@ -59,16 +59,25 @@ struct blkg_rwstat { uint64_t cnt[BLKG_RWSTAT_NR]; }; -/* per-blkg per-policy data */ +/* + * A blkcg_gq (blkg) is association between a block cgroup (blkcg) and a + * request_queue (q). This is used by blkcg policies which need to track + * information per blkcg - q pair. + * + * There can be multiple active blkcg policies and each has its private + * data on each blkg, the size of which is determined by + * blkcg_policy->pd_size. blkcg core allocates and frees such areas + * together with blkg and invokes pd_init/exit_fn() methods. + * + * Such private data must embed struct blkg_policy_data (pd) at the + * beginning and pd_size can't be smaller than pd. + */ struct blkg_policy_data { /* the blkg this per-policy data belongs to */ struct blkcg_gq *blkg; /* used during policy activation */ struct list_head alloc_node; - - /* pol->pdata_size bytes of private data used by policy impl */ - char pdata[] __aligned(__alignof__(unsigned long long)); }; /* association between a blk cgroup and a request queue */ @@ -100,7 +109,7 @@ struct blkcg_policy { struct blkcg_policy_ops ops; int plid; /* policy specific private data size */ - size_t pdata_size; + size_t pd_size; /* cgroup files for the policy */ struct cftype *cftypes; }; @@ -125,14 +134,16 @@ void blkcg_deactivate_policy(struct request_queue *q, const struct blkcg_policy *pol); void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg, - u64 (*prfill)(struct seq_file *, void *, int), + u64 (*prfill)(struct seq_file *, + struct blkg_policy_data *, int), const struct blkcg_policy *pol, int data, bool show_total); -u64 __blkg_prfill_u64(struct seq_file *sf, void *pdata, u64 v); -u64 __blkg_prfill_rwstat(struct seq_file *sf, void *pdata, +u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v); +u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd, const struct blkg_rwstat *rwstat); -u64 blkg_prfill_stat(struct seq_file *sf, void *pdata, int off); -u64 blkg_prfill_rwstat(struct seq_file *sf, void *pdata, int off); +u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd, int off); +u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd, + int off); struct blkg_conf_ctx { struct gendisk *disk; @@ -152,26 +163,21 @@ void blkg_conf_finish(struct blkg_conf_ctx *ctx); * * Return pointer to private data associated with the @blkg-@pol pair. */ -static inline void *blkg_to_pdata(struct blkcg_gq *blkg, - struct blkcg_policy *pol) +static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg, + struct blkcg_policy *pol) { - return blkg ? blkg->pd[pol->plid]->pdata : NULL; + return blkg ? blkg->pd[pol->plid] : NULL; } /** * pdata_to_blkg - get blkg associated with policy private data - * @pdata: policy private data of interest + * @pd: policy private data of interest * - * @pdata is policy private data. Determine the blkg it's associated with. + * @pd is policy private data. Determine the blkg it's associated with. */ -static inline struct blkcg_gq *pdata_to_blkg(void *pdata) +static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd) { - if (pdata) { - struct blkg_policy_data *pd = - container_of(pdata, struct blkg_policy_data, pdata); - return pd->blkg; - } - return NULL; + return pd ? pd->blkg : NULL; } /** @@ -342,6 +348,9 @@ static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat) struct cgroup; +struct blkg_policy_data { +}; + struct blkcg_gq { }; @@ -361,10 +370,9 @@ static inline int blkcg_activate_policy(struct request_queue *q, static inline void blkcg_deactivate_policy(struct request_queue *q, const struct blkcg_policy *pol) { } -static inline void *blkg_to_pdata(struct blkcg_gq *blkg, - struct blkcg_policy *pol) { return NULL; } -static inline struct blkcg_gq *pdata_to_blkg(void *pdata, - struct blkcg_policy *pol) { return NULL; } +static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg, + struct blkcg_policy *pol) { return NULL; } +static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd) { return NULL; } static inline char *blkg_path(struct blkcg_gq *blkg) { return NULL; } static inline void blkg_get(struct blkcg_gq *blkg) { } static inline void blkg_put(struct blkcg_gq *blkg) { } diff --git a/block/blk-throttle.c b/block/blk-throttle.c index 00c7eff66ecf..6a0a17a83862 100644 --- a/block/blk-throttle.c +++ b/block/blk-throttle.c @@ -49,6 +49,9 @@ struct tg_stats_cpu { }; struct throtl_grp { + /* must be the first member */ + struct blkg_policy_data pd; + /* active throtl group service_tree member */ struct rb_node rb_node; @@ -120,14 +123,19 @@ static LIST_HEAD(tg_stats_alloc_list); static void tg_stats_alloc_fn(struct work_struct *); static DECLARE_DELAYED_WORK(tg_stats_alloc_work, tg_stats_alloc_fn); +static inline struct throtl_grp *pd_to_tg(struct blkg_policy_data *pd) +{ + return pd ? container_of(pd, struct throtl_grp, pd) : NULL; +} + static inline struct throtl_grp *blkg_to_tg(struct blkcg_gq *blkg) { - return blkg_to_pdata(blkg, &blkcg_policy_throtl); + return pd_to_tg(blkg_to_pd(blkg, &blkcg_policy_throtl)); } static inline struct blkcg_gq *tg_to_blkg(struct throtl_grp *tg) { - return pdata_to_blkg(tg); + return pd_to_blkg(&tg->pd); } static inline struct throtl_grp *td_root_tg(struct throtl_data *td) @@ -931,9 +939,10 @@ throtl_schedule_delayed_work(struct throtl_data *td, unsigned long delay) } } -static u64 tg_prfill_cpu_rwstat(struct seq_file *sf, void *pdata, int off) +static u64 tg_prfill_cpu_rwstat(struct seq_file *sf, + struct blkg_policy_data *pd, int off) { - struct throtl_grp *tg = pdata; + struct throtl_grp *tg = pd_to_tg(pd); struct blkg_rwstat rwstat = { }, tmp; int i, cpu; @@ -945,7 +954,7 @@ static u64 tg_prfill_cpu_rwstat(struct seq_file *sf, void *pdata, int off) rwstat.cnt[i] += tmp.cnt[i]; } - return __blkg_prfill_rwstat(sf, pdata, &rwstat); + return __blkg_prfill_rwstat(sf, pd, &rwstat); } static int tg_print_cpu_rwstat(struct cgroup *cgrp, struct cftype *cft, @@ -958,22 +967,26 @@ static int tg_print_cpu_rwstat(struct cgroup *cgrp, struct cftype *cft, return 0; } -static u64 tg_prfill_conf_u64(struct seq_file *sf, void *pdata, int off) +static u64 tg_prfill_conf_u64(struct seq_file *sf, struct blkg_policy_data *pd, + int off) { - u64 v = *(u64 *)(pdata + off); + struct throtl_grp *tg = pd_to_tg(pd); + u64 v = *(u64 *)((void *)tg + off); if (v == -1) return 0; - return __blkg_prfill_u64(sf, pdata, v); + return __blkg_prfill_u64(sf, pd, v); } -static u64 tg_prfill_conf_uint(struct seq_file *sf, void *pdata, int off) +static u64 tg_prfill_conf_uint(struct seq_file *sf, struct blkg_policy_data *pd, + int off) { - unsigned int v = *(unsigned int *)(pdata + off); + struct throtl_grp *tg = pd_to_tg(pd); + unsigned int v = *(unsigned int *)((void *)tg + off); if (v == -1) return 0; - return __blkg_prfill_u64(sf, pdata, v); + return __blkg_prfill_u64(sf, pd, v); } static int tg_print_conf_u64(struct cgroup *cgrp, struct cftype *cft, @@ -1092,7 +1105,7 @@ static struct blkcg_policy blkcg_policy_throtl = { .pd_exit_fn = throtl_pd_exit, .pd_reset_stats_fn = throtl_pd_reset_stats, }, - .pdata_size = sizeof(struct throtl_grp), + .pd_size = sizeof(struct throtl_grp), .cftypes = throtl_files, }; diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index 792218281d91..7865cc38ea77 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -217,6 +217,9 @@ struct cfqg_stats { /* This is per cgroup per device grouping structure */ struct cfq_group { + /* must be the first member */ + struct blkg_policy_data pd; + /* group service_tree member */ struct rb_node rb_node; @@ -409,6 +412,21 @@ CFQ_CFQQ_FNS(deep); CFQ_CFQQ_FNS(wait_busy); #undef CFQ_CFQQ_FNS +static inline struct cfq_group *pd_to_cfqg(struct blkg_policy_data *pd) +{ + return pd ? container_of(pd, struct cfq_group, pd) : NULL; +} + +static inline struct cfq_group *blkg_to_cfqg(struct blkcg_gq *blkg) +{ + return pd_to_cfqg(blkg_to_pd(blkg, &blkcg_policy_cfq)); +} + +static inline struct blkcg_gq *cfqg_to_blkg(struct cfq_group *cfqg) +{ + return pd_to_blkg(&cfqg->pd); +} + #if defined(CONFIG_CFQ_GROUP_IOSCHED) && defined(CONFIG_DEBUG_BLK_CGROUP) /* cfqg stats flags */ @@ -553,16 +571,6 @@ static inline void cfqg_stats_update_avg_queue_size(struct cfq_group *cfqg) { } #ifdef CONFIG_CFQ_GROUP_IOSCHED -static inline struct cfq_group *blkg_to_cfqg(struct blkcg_gq *blkg) -{ - return blkg_to_pdata(blkg, &blkcg_policy_cfq); -} - -static inline struct blkcg_gq *cfqg_to_blkg(struct cfq_group *cfqg) -{ - return pdata_to_blkg(cfqg); -} - static inline void cfqg_get(struct cfq_group *cfqg) { return blkg_get(cfqg_to_blkg(cfqg)); @@ -662,8 +670,6 @@ static void cfq_pd_reset_stats(struct blkcg_gq *blkg) #else /* CONFIG_CFQ_GROUP_IOSCHED */ -static inline struct cfq_group *blkg_to_cfqg(struct blkcg_gq *blkg) { return NULL; } -static inline struct blkcg_gq *cfqg_to_blkg(struct cfq_group *cfqg) { return NULL; } static inline void cfqg_get(struct cfq_group *cfqg) { } static inline void cfqg_put(struct cfq_group *cfqg) { } @@ -1374,13 +1380,14 @@ static void cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg) cfqg_get(cfqg); } -static u64 cfqg_prfill_weight_device(struct seq_file *sf, void *pdata, int off) +static u64 cfqg_prfill_weight_device(struct seq_file *sf, + struct blkg_policy_data *pd, int off) { - struct cfq_group *cfqg = pdata; + struct cfq_group *cfqg = pd_to_cfqg(pd); if (!cfqg->dev_weight) return 0; - return __blkg_prfill_u64(sf, pdata, cfqg->dev_weight); + return __blkg_prfill_u64(sf, pd, cfqg->dev_weight); } static int cfqg_print_weight_device(struct cgroup *cgrp, struct cftype *cft, @@ -1467,9 +1474,10 @@ static int cfqg_print_rwstat(struct cgroup *cgrp, struct cftype *cft, } #ifdef CONFIG_DEBUG_BLK_CGROUP -static u64 cfqg_prfill_avg_queue_size(struct seq_file *sf, void *pdata, int off) +static u64 cfqg_prfill_avg_queue_size(struct seq_file *sf, + struct blkg_policy_data *pd, int off) { - struct cfq_group *cfqg = pdata; + struct cfq_group *cfqg = pd_to_cfqg(pd); u64 samples = blkg_stat_read(&cfqg->stats.avg_queue_size_samples); u64 v = 0; @@ -1477,7 +1485,7 @@ static u64 cfqg_prfill_avg_queue_size(struct seq_file *sf, void *pdata, int off) v = blkg_stat_read(&cfqg->stats.avg_queue_size_sum); do_div(v, samples); } - __blkg_prfill_u64(sf, pdata, v); + __blkg_prfill_u64(sf, pd, v); return 0; } @@ -4161,7 +4169,7 @@ static struct blkcg_policy blkcg_policy_cfq = { .pd_init_fn = cfq_pd_init, .pd_reset_stats_fn = cfq_pd_reset_stats, }, - .pdata_size = sizeof(struct cfq_group), + .pd_size = sizeof(struct cfq_group), .cftypes = cfq_blkcg_files, }; #endif |
