diff options
| -rw-r--r-- | arch/s390/include/asm/perf_event.h | 4 | ||||
| -rw-r--r-- | arch/s390/kernel/perf_cpum_sf.c | 527 |
2 files changed, 411 insertions, 120 deletions
diff --git a/arch/s390/include/asm/perf_event.h b/arch/s390/include/asm/perf_event.h index 23d2dfa8201d..99d7f4e333c2 100644 --- a/arch/s390/include/asm/perf_event.h +++ b/arch/s390/include/asm/perf_event.h @@ -49,6 +49,10 @@ extern unsigned long perf_misc_flags(struct pt_regs *regs); #define PERF_CPUM_SF_MAX_CTR 1 #define PERF_EVENT_CPUM_SF 0xB0000UL /* Raw event ID */ +#define REG_NONE 0 +#define REG_OVERFLOW 1 +#define OVERFLOW_REG(hwc) ((hwc)->extra_reg.config) +#define SFB_ALLOC_REG(hwc) ((hwc)->extra_reg.alloc) #define TEAR_REG(hwc) ((hwc)->last_tag) #define SAMPL_RATE(hwc) ((hwc)->event_base) diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c index ae5e0192160d..ea1656073dac 100644 --- a/arch/s390/kernel/perf_cpum_sf.c +++ b/arch/s390/kernel/perf_cpum_sf.c @@ -17,6 +17,8 @@ #include <linux/percpu.h> #include <linux/notifier.h> #include <linux/export.h> +#include <linux/mm.h> +#include <linux/moduleparam.h> #include <asm/cpu_mf.h> #include <asm/irq.h> #include <asm/debug.h> @@ -26,34 +28,54 @@ * At least one table is required for the sampling buffer structure. * A single table contains up to 511 pointers to sample-data-blocks. */ -#define CPUM_SF_MIN_SDBT 1 +#define CPUM_SF_MIN_SDBT 1 -/* Minimum number of sample-data-blocks: - * The minimum designates a single page for sample-data-block, i.e., - * up to 126 sample-data-blocks with a size of 32 bytes (bsdes). +/* Number of sample-data-blocks per sample-data-block-table (SDBT): + * The table contains SDB origin (8 bytes) and one SDBT origin that + * points to the next table. */ -#define CPUM_SF_MIN_SDB 126 +#define CPUM_SF_SDB_PER_TABLE ((PAGE_SIZE - 8) / 8) -/* Maximum number of sample-data-blocks: - * The maximum number designates approx. 256K per CPU including - * the given number of sample-data-blocks and taking the number - * of sample-data-block tables into account. +/* Maximum page offset for an SDBT table-link entry: + * If this page offset is reached, a table-link entry to the next SDBT + * must be added. + */ +#define CPUM_SF_SDBT_TL_OFFSET (CPUM_SF_SDB_PER_TABLE * 8) +static inline int require_table_link(const void *sdbt) +{ + return ((unsigned long) sdbt & ~PAGE_MASK) == CPUM_SF_SDBT_TL_OFFSET; +} + +/* Minimum and maximum sampling buffer sizes: + * + * This number represents the maximum size of the sampling buffer + * taking the number of sample-data-block-tables into account. * - * Later, this number can be increased for extending the sampling - * buffer, for example, by factor 2 (512K) or 4 (1M). + * Sampling buffer size Buffer characteristics + * --------------------------------------------------- + * 64KB == 16 pages (4KB per page) + * 1 page for SDB-tables + * 15 pages for SDBs + * + * 32MB == 8192 pages (4KB per page) + * 16 pages for SDB-tables + * 8176 pages for SDBs */ -#define CPUM_SF_MAX_SDB 6471 +static unsigned long __read_mostly CPUM_SF_MIN_SDB = 15; +static unsigned long __read_mostly CPUM_SF_MAX_SDB = 8176; struct sf_buffer { - unsigned long sdbt; /* Sample-data-block-table origin */ + unsigned long *sdbt; /* Sample-data-block-table origin */ /* buffer characteristics (required for buffer increments) */ - unsigned long num_sdb; /* Number of sample-data-blocks */ - unsigned long tail; /* last sample-data-block-table */ + unsigned long num_sdb; /* Number of sample-data-blocks */ + unsigned long num_sdbt; /* Number of sample-data-block-tables */ + unsigned long *tail; /* last sample-data-block-table */ }; struct cpu_hw_sf { /* CPU-measurement sampling information block */ struct hws_qsi_info_block qsi; + /* CPU-measurement sampling control block */ struct hws_lsctl_request_block lsctl; struct sf_buffer sfb; /* Sampling buffer */ unsigned int flags; /* Status flags */ @@ -65,11 +87,22 @@ static DEFINE_PER_CPU(struct cpu_hw_sf, cpu_hw_sf); static debug_info_t *sfdbg; /* + * sf_disable() - Switch off sampling facility + */ +static int sf_disable(void) +{ + struct hws_lsctl_request_block sreq; + + memset(&sreq, 0, sizeof(sreq)); + return lsctl(&sreq); +} + +/* * sf_buffer_available() - Check for an allocated sampling buffer */ static int sf_buffer_available(struct cpu_hw_sf *cpuhw) { - return (cpuhw->sfb.sdbt) ? 1 : 0; + return !!cpuhw->sfb.sdbt; } /* @@ -77,32 +110,32 @@ static int sf_buffer_available(struct cpu_hw_sf *cpuhw) */ static void free_sampling_buffer(struct sf_buffer *sfb) { - unsigned long sdbt, *curr; + unsigned long *sdbt, *curr; if (!sfb->sdbt) return; sdbt = sfb->sdbt; - curr = (unsigned long *) sdbt; + curr = sdbt; - /* we'll free the SDBT after all SDBs are processed... */ + /* Free the SDBT after all SDBs are processed... */ while (1) { if (!*curr || !sdbt) break; - /* watch for link entry reset if found */ + /* Process table-link entries */ if (is_link_entry(curr)) { curr = get_next_sdbt(curr); if (sdbt) - free_page(sdbt); + free_page((unsigned long) sdbt); - /* we are done if we reach the origin */ - if ((unsigned long) curr == sfb->sdbt) + /* If the origin is reached, sampling buffer is freed */ + if (curr == sfb->sdbt) break; else - sdbt = (unsigned long) curr; + sdbt = curr; } else { - /* process SDB pointer */ + /* Process SDB pointer */ if (*curr) { free_page(*curr); curr++; @@ -111,10 +144,106 @@ static void free_sampling_buffer(struct sf_buffer *sfb) } debug_sprintf_event(sfdbg, 5, - "free_sampling_buffer: freed sdbt=%0lx\n", sfb->sdbt); + "free_sampling_buffer: freed sdbt=%p\n", sfb->sdbt); memset(sfb, 0, sizeof(*sfb)); } +static int alloc_sample_data_block(unsigned long *sdbt, gfp_t gfp_flags) +{ + unsigned long sdb, *trailer; + + /* Allocate and initialize sample-data-block */ + sdb = get_zeroed_page(gfp_flags); + if (!sdb) + return -ENOMEM; + trailer = trailer_entry_ptr(sdb); + *trailer = SDB_TE_ALERT_REQ_MASK; + + /* Link SDB into the sample-data-block-table */ + *sdbt = sdb; + + return 0; +} + +/* + * realloc_sampling_buffer() - extend sampler memory + * + * Allocates new sample-data-blocks and adds them to the specified sampling + * buffer memory. + * + * Important: This modifies the sampling buffer and must be called when the + * sampling facility is disabled. + * + * Returns zero on success, non-zero otherwise. + */ +static int realloc_sampling_buffer(struct sf_buffer *sfb, + unsigned long num_sdb, gfp_t gfp_flags) +{ + int i, rc; + unsigned long *new, *tail; + + if (!sfb->sdbt || !sfb->tail) + return -EINVAL; + + if (!is_link_entry(sfb->tail)) + return -EINVAL; + + /* Append to the existing sampling buffer, overwriting the table-link + * register. + * The tail variables always points to the "tail" (last and table-link) + * entry in an SDB-table. + */ + tail = sfb->tail; + + /* Do a sanity check whether the table-link entry points to + * the sampling buffer origin. + */ + if (sfb->sdbt != get_next_sdbt(tail)) { + debug_sprintf_event(sfdbg, 3, "realloc_sampling_buffer: " + "sampling buffer is not linked: origin=%p" + "tail=%p\n", + (void *) sfb->sdbt, (void *) tail); + return -EINVAL; + } + + /* Allocate remaining SDBs */ + rc = 0; + for (i = 0; i < num_sdb; i++) { + /* Allocate a new SDB-table if it is full. */ + if (require_table_link(tail)) { + new = (unsigned long *) get_zeroed_page(gfp_flags); + if (!new) { + rc = -ENOMEM; + break; + } + sfb->num_sdbt++; + /* Link current page to tail of chain */ + *tail = (unsigned long)(void *) new + 1; + tail = new; + } + + /* Allocate a new sample-data-block. + * If there is not enough memory, stop the realloc process + * and simply use what was allocated. If this is a temporary + * issue, a new realloc call (if required) might succeed. + */ + rc = alloc_sample_data_block(tail, gfp_flags); + if (rc) + break; + sfb->num_sdb++; + tail++; + } + + /* Link sampling buffer to its origin */ + *tail = (unsigned long) sfb->sdbt + 1; + sfb->tail = tail; + + debug_sprintf_event(sfdbg, 4, "realloc_sampling_buffer: new buffer" + " settings: sdbt=%lu sdb=%lu\n", + sfb->num_sdbt, sfb->num_sdb); + return rc; +} + /* * allocate_sampling_buffer() - allocate sampler memory * @@ -128,75 +257,74 @@ static void free_sampling_buffer(struct sf_buffer *sfb) */ static int alloc_sampling_buffer(struct sf_buffer *sfb, unsigned long num_sdb) { - int j, k, rc; - unsigned long *sdbt, *tail, *trailer; - unsigned long sdb; - unsigned long num_sdbt, sdb_per_table; + int rc; if (sfb->sdbt) return -EINVAL; + + /* Allocate the sample-data-block-table origin */ + sfb->sdbt = (unsigned long *) get_zeroed_page(GFP_KERNEL); + if (!sfb->sdbt) + return -ENOMEM; sfb->num_sdb = 0; + sfb->num_sdbt = 1; - /* Compute the number of required sample-data-block-tables (SDBT) */ - num_sdbt = num_sdb / ((PAGE_SIZE - 8) / 8); - if (num_sdbt < CPUM_SF_MIN_SDBT) - num_sdbt = CPUM_SF_MIN_SDBT; - sdb_per_table = (PAGE_SIZE - 8) / 8; - - debug_sprintf_event(sfdbg, 4, "alloc_sampling_buffer: num_sdbt=%lu " - "num_sdb=%lu sdb_per_table=%lu\n", - num_sdbt, num_sdb, sdb_per_table); - sdbt = NULL; - tail = sdbt; - - for (j = 0; j < num_sdbt; j++) { - sdbt = (unsigned long *) get_zeroed_page(GFP_KERNEL); - if (!sdbt) { - rc = -ENOMEM; - goto allocate_sdbt_error; - } + /* Link the table origin to point to itself to prepare for + * realloc_sampling_buffer() invocation. + */ + sfb->tail = sfb->sdbt; + *sfb->tail = (unsigned long)(void *) sfb->sdbt + 1; - /* save origin of sample-data-block-table */ - if (!sfb->sdbt) - sfb->sdbt = (unsigned long) sdbt; + /* Allocate requested number of sample-data-blocks */ + rc = realloc_sampling_buffer(sfb, num_sdb, GFP_KERNEL); + if (rc) { + free_sampling_buffer(sfb); + debug_sprintf_event(sfdbg, 4, "alloc_sampling_buffer: " + "realloc_sampling_buffer failed with rc=%i\n", rc); + } else + debug_sprintf_event(sfdbg, 4, + "alloc_sampling_buffer: tear=%p dear=%p\n", + sfb->sdbt, (void *) *sfb->sdbt); + return rc; +} - /* link current page to tail of chain */ - if (tail) - *tail = (unsigned long)(void *) sdbt + 1; +static void sfb_set_limits(unsigned long min, unsigned long max) +{ + CPUM_SF_MIN_SDB = min; + CPUM_SF_MAX_SDB = max; +} - for (k = 0; k < num_sdb && k < sdb_per_table; k++) { - /* get and set SDB page */ - sdb = get_zeroed_page(GFP_KERNEL); - if (!sdb) { - rc = -ENOMEM; - goto allocate_sdbt_error; - } - *sdbt = sdb; - trailer = trailer_entry_ptr(*sdbt); - *trailer = SDB_TE_ALERT_REQ_MASK; - sdbt++; - } - num_sdb -= k; - sfb->num_sdb += k; /* count allocated sdb's */ - tail = sdbt; - } +static unsigned long sfb_pending_allocs(struct sf_buffer *sfb, + struct hw_perf_event *hwc) +{ + if (!sfb->sdbt) + return SFB_ALLOC_REG(hwc); + if (SFB_ALLOC_REG(hwc) > sfb->num_sdb) + return SFB_ALLOC_REG(hwc) - sfb->num_sdb; + return 0; +} - rc = 0; - if (tail) - *tail = sfb->sdbt + 1; - sfb->tail = (unsigned long) (void *)tail; +static int sfb_has_pending_allocs(struct sf_buffer *sfb, + struct hw_perf_event *hwc) +{ + return sfb_pending_allocs(sfb, hwc) > 0; +} -allocate_sdbt_error: - if (rc) - free_sampling_buffer(sfb); - else - debug_sprintf_event(sfdbg, 4, - "alloc_sampling_buffer: tear=%0lx dear=%0lx\n", - sfb->sdbt, *(unsigned long *) sfb->sdbt); - return rc; +static void sfb_account_allocs(unsigned long num, struct hw_perf_event *hwc) +{ + /* Limit the number SDBs to not exceed the maximum */ + num = min_t(unsigned long, num, CPUM_SF_MAX_SDB - SFB_ALLOC_REG(hwc)); + if (num) + SFB_ALLOC_REG(hwc) += num; } -static int allocate_sdbt(struct cpu_hw_sf *cpuhw, const struct hw_perf_event *hwc) +static void sfb_init_allocs(unsigned long num, struct hw_perf_event *hwc) +{ + SFB_ALLOC_REG(hwc) = 0; + sfb_account_allocs(num, hwc); +} + +static int allocate_sdbt(struct cpu_hw_sf *cpuhw, struct hw_perf_event *hwc) { unsigned long n_sdb, freq; unsigned long factor; @@ -225,39 +353,133 @@ static int allocate_sdbt(struct cpu_hw_sf *cpuhw, const struct hw_perf_event *hw if (n_sdb < CPUM_SF_MIN_SDB) n_sdb = CPUM_SF_MIN_SDB; - /* Return if there is already a sampling buffer allocated. - * XXX Remove this later and check number of available and - * required sdb's and, if necessary, increase the sampling buffer. + /* If there is already a sampling buffer allocated, it is very likely + * that the sampling facility is enabled too. If the event to be + * initialized requires a greater sampling buffer, the allocation must + * be postponed. Changing the sampling buffer requires the sampling + * facility to be in the disabled state. So, account the number of + * required SDBs and let cpumsf_pmu_enable() resize the buffer just + * before the event is started. */ + sfb_init_allocs(n_sdb, hwc); if (sf_buffer_available(cpuhw)) return 0; debug_sprintf_event(sfdbg, 3, - "allocate_sdbt: rate=%lu f=%lu sdb=%lu/%i cpuhw=%p\n", + "allocate_sdbt: rate=%lu f=%lu sdb=%lu/%lu cpuhw=%p\n", SAMPL_RATE(hwc), freq, n_sdb, CPUM_SF_MAX_SDB, cpuhw); return alloc_sampling_buffer(&cpuhw->sfb, - min_t(unsigned long, n_sdb, CPUM_SF_MAX_SDB)); + sfb_pending_allocs(&cpuhw->sfb, hwc)); } +static unsigned long min_percent(unsigned int percent, unsigned long base, + unsigned long min) +{ + return min_t(unsigned long, min, DIV_ROUND_UP(percent * base, 100)); +} -/* Number of perf events counting hardware events */ -static atomic_t num_events; -/* Used to avoid races in calling reserve/release_cpumf_hardware */ -static DEFINE_MUTEX(pmc_reserve_mutex); +static unsigned long compute_sfb_extent(unsigned long ratio, unsigned long base) +{ + /* Use a percentage-based approach to extend the sampling facility + * buffer. Accept up to 5% sample data loss. + * Vary the extents between 1% to 5% of the current number of + * sample-data-blocks. + */ + if (ratio <= 5) + return 0; + if (ratio <= 25) + return min_percent(1, base, 1); + if (ratio <= 50) + return min_percent(1, base, 1); + if (ratio <= 75) + return min_percent(2, base, 2); + if (ratio <= 100) + return min_percent(3, base, 3); + if (ratio <= 250) + return min_percent(4, base, 4); + + return min_percent(5, base, 8); +} -/* - * sf_disable() - Switch off sampling facility +static void sfb_account_overflows(struct cpu_hw_sf *cpuhw, + struct hw_perf_event *hwc) +{ + unsigned long ratio, num; + + if (!OVERFLOW_REG(hwc)) + return; + + /* The sample_overflow contains the average number of sample data + * that has been lost because sample-data-blocks were full. + * + * Calculate the total number of sample data entries that has been + * discarded. Then calculate the ratio of lost samples to total samples + * per second in percent. + */ + ratio = DIV_ROUND_UP(100 * OVERFLOW_REG(hwc) * cpuhw->sfb.num_sdb, + sample_rate_to_freq(&cpuhw->qsi, SAMPL_RATE(hwc))); + + /* Compute number of sample-data-blocks */ + num = compute_sfb_extent(ratio, cpuhw->sfb.num_sdb); + if (num) + sfb_account_allocs(num, hwc); + + debug_sprintf_event(sfdbg, 5, "sfb: overflow: overflow=%llu ratio=%lu" + " num=%lu\n", OVERFLOW_REG(hwc), ratio, num); + OVERFLOW_REG(hwc) = 0; +} + +/* extend_sampling_buffer() - Extend sampling buffer + * @sfb: Sampling buffer structure (for local CPU) + * @hwc: Perf event hardware structure + * + * Use this function to extend the sampling buffer based on the overflow counter + * and postponed allocation extents stored in the specified Perf event hardware. + * + * Important: This function disables the sampling facility in order to safely + * change the sampling buffer structure. Do not call this function + * when the PMU is active. */ -static int sf_disable(void) +static void extend_sampling_buffer(struct sf_buffer *sfb, + struct hw_perf_event *hwc) { - struct hws_lsctl_request_block sreq; + unsigned long num, num_old; + int rc; - memset(&sreq, 0, sizeof(sreq)); - return lsctl(&sreq); + num = sfb_pending_allocs(sfb, hwc); + if (!num) + return; + num_old = sfb->num_sdb; + + /* Disable the sampling facility to reset any states and also + * clear pending measurement alerts. + */ + sf_disable(); + + /* Extend the sampling buffer. + * This memory allocation typically happens in an atomic context when + * called by perf. Because this is a reallocation, it is fine if the + * new SDB-request cannot be satisfied immediately. + */ + rc = realloc_sampling_buffer(sfb, num, GFP_ATOMIC); + if (rc) + debug_sprintf_event(sfdbg, 5, "sfb: extend: realloc " + "failed with rc=%i\n", rc); + + if (sfb_has_pending_allocs(sfb, hwc)) + debug_sprintf_event(sfdbg, 5, "sfb: extend: " + "req=%lu alloc=%lu remaining=%lu\n", + num, sfb->num_sdb - num_old, + sfb_pending_allocs(sfb, hwc)); } +/* Number of perf events counting hardware events */ +static atomic_t num_events; +/* Used to avoid races in calling reserve/release_cpumf_hardware */ +static DEFINE_MUTEX(pmc_reserve_mutex); + #define PMC_INIT 0 #define PMC_RELEASE 1 #define PMC_FAILURE 2 @@ -345,19 +567,17 @@ static void hw_init_period(struct hw_perf_event *hwc, u64 period) } static void hw_reset_registers(struct hw_perf_event *hwc, - unsigned long sdbt_origin) + unsigned long *sdbt_origin) { - TEAR_REG(hwc) = sdbt_origin; /* (re)set to first sdb table */ + /* (Re)set to first sample-data-block-table */ + TEAR_REG(hwc) = (unsigned long) sdbt_origin; } static unsigned long hw_limit_rate(const struct hws_qsi_info_block *si, unsigned long rate) { - if (rate < si->min_sampl_rate) - return si->min_sampl_rate; - if (rate > si->max_sampl_rate) - return si->max_sampl_rate; - return rate; + return clamp_t(unsigned long, rate, + si->min_sampl_rate, si->max_sampl_rate); } static int __hw_perf_event_init(struct perf_event *event) @@ -448,6 +668,10 @@ static int __hw_perf_event_init(struct perf_event *event) SAMPL_RATE(hwc) = rate; hw_init_period(hwc, SAMPL_RATE(hwc)); + /* Initialize sample data overflow accounting */ + hwc->extra_reg.reg = REG_OVERFLOW; + OVERFLOW_REG(hwc) = 0; + /* Allocate the per-CPU sampling buffer using the CPU information * from the event. If the event is not pinned to a particular * CPU (event->cpu == -1; or cpuhw == NULL), allocate sampling @@ -513,6 +737,7 @@ static int cpumsf_pmu_event_init(struct perf_event *event) static void cpumsf_pmu_enable(struct pmu *pmu) { struct cpu_hw_sf *cpuhw = &__get_cpu_var(cpu_hw_sf); + struct hw_perf_event *hwc; int err; if (cpuhw->flags & PMU_F_ENABLED) @@ -521,6 +746,26 @@ static void cpumsf_pmu_enable(struct pmu *pmu) if (cpuhw->flags & PMU_F_ERR_MASK) return; + /* Check whether to extent the sampling buffer. + * + * Two conditions trigger an increase of the sampling buffer for a + * perf event: + * 1. Postponed buffer allocations from the event initialization. + * 2. Sampling overflows that contribute to pending allocations. + * + * Note that the extend_sampling_buffer() function disables the sampling + * facility, but it can be fully re-enabled using sampling controls that + * have been saved in cpumsf_pmu_disable(). + */ + if (cpuhw->event) { + hwc = &cpuhw->event->hw; + /* Account number of overflow-designated buffer extents */ + sfb_account_overflows(cpuhw, hwc); + if (sfb_has_pending_allocs(&cpuhw->sfb, hwc)) + extend_sampling_buffer(&cpuhw->sfb, hwc); + } + + /* (Re)enable the PMU and sampling facility */ cpuhw->flags |= PMU_F_ENABLED; barrier(); @@ -632,8 +877,6 @@ static int perf_push_sample(struct perf_event *event, if (perf_event_overflow(event, &data, ®s)) { overflow = 1; event->pmu->stop(event, 0); - debug_sprintf_event(sfdbg, 4, "perf_push_sample: PMU stopped" - " because of an event overflow\n"); } perf_event_update_userpage(event); @@ -710,11 +953,11 @@ static void hw_perf_event_update(struct perf_event *event, int flush_all) struct hw_perf_event *hwc = &event->hw; struct hws_trailer_entry *te; unsigned long *sdbt; - unsigned long long event_overflow, sampl_overflow; + unsigned long long event_overflow, sampl_overflow, num_sdb; int done; sdbt = (unsigned long *) TEAR_REG(hwc); - done = event_overflow = sampl_overflow = 0; + done = event_overflow = sampl_overflow = num_sdb = 0; while (!done) { /* Get the trailer entry of the sample-data-block */ te = (struct hws_trailer_entry *) trailer_entry_ptr(*sdbt); @@ -726,17 +969,13 @@ static void hw_perf_event_update(struct perf_event *event, int flush_all) break; } - /* Check sample overflow count */ - if (te->overflow) { - /* Increment sample overflow counter */ - sampl_overflow += te->overflow; - - /* XXX: If an sample overflow occurs, increase the - * sampling buffer. Set a "realloc" flag because - * the sampler must be re-enabled for changing - * the sample-data-block-table content. + /* Check the sample overflow count */ + if (te->overflow) + /* Account sample overflows and, if a particular limit + * is reached, extend the sampling buffer. + * For details, see sfb_account_overflows(). */ - } + sampl_overflow += te->overflow; /* Timestamps are valid for full sample-data-blocks only */ debug_sprintf_event(sfdbg, 6, "hw_perf_event_update: sdbt=%p " @@ -749,6 +988,7 @@ static void hw_perf_event_update(struct perf_event *event, int flush_all) * is stopped and remaining samples will be discarded. */ hw_collect_samples(event, sdbt, &event_overflow); + num_sdb++; /* Reset trailer */ xchg(&te->overflow, 0); @@ -775,6 +1015,10 @@ static void hw_perf_event_update(struct perf_event *event, int flush_all) flush_all = 1; } + /* Account sample overflows in the event hardware structure */ + if (sampl_overflow) + OVERFLOW_REG(hwc) = DIV_ROUND_UP(OVERFLOW_REG(hwc) + + sampl_overflow, 1 + num_sdb); if (sampl_overflow || event_overflow) debug_sprintf_event(sfdbg, 4, "hw_perf_event_update: " "overflow stats: sample=%llu event=%llu\n", @@ -849,7 +1093,7 @@ static int cpumsf_pmu_add(struct perf_event *event, int flags) */ cpuhw->lsctl.s = 0; cpuhw->lsctl.h = 1; - cpuhw->lsctl.tear = cpuhw->sfb.sdbt; + cpuhw->lsctl.tear = (unsigned long) cpuhw->sfb.sdbt; cpuhw->lsctl.dear = *(unsigned long *) cpuhw->sfb.sdbt; cpuhw->lsctl.interval = SAMPL_RATE(&event->hw); hw_reset_registers(&event->hw, cpuhw->sfb.sdbt); @@ -1018,6 +1262,48 @@ static int __cpuinit cpumf_pmu_notifier(struct notifier_block *self, return NOTIFY_OK; } +static int param_get_sfb_size(char *buffer, const struct kernel_param *kp) +{ + if (!cpum_sf_avail()) + return -ENODEV; + return sprintf(buffer, "%lu,%lu", CPUM_SF_MIN_SDB, CPUM_SF_MAX_SDB); +} + +static int param_set_sfb_size(const char *val, const struct kernel_param *kp) +{ + int rc; + unsigned long min, max; + + if (!cpum_sf_avail()) + return -ENODEV; + if (!val || !strlen(val)) + return -EINVAL; + + /* Valid parameter values: "min,max" or "max" */ + min = CPUM_SF_MIN_SDB; + max = CPUM_SF_MAX_SDB; + if (strchr(val, ',')) + rc = (sscanf(val, "%lu,%lu", &min, &max) == 2) ? 0 : -EINVAL; + else + rc = kstrtoul(val, 10, &max); + + if (min < 2 || min >= max || max > get_num_physpages()) + rc = -EINVAL; + if (rc) + return rc; + + sfb_set_limits(min, max); + pr_info("Changed sampling buffer settings: min=%lu max=%lu\n", + CPUM_SF_MIN_SDB, CPUM_SF_MAX_SDB); + return 0; +} + +#define param_check_sfb_size(name, p) __param_check(name, p, void) +static struct kernel_param_ops param_ops_sfb_size = { + .set = param_set_sfb_size, + .get = param_get_sfb_size, +}; + static int __init init_cpum_sampling_pmu(void) { int err; @@ -1047,3 +1333,4 @@ out: return err; } arch_initcall(init_cpum_sampling_pmu); +core_param(cpum_sfb_size, CPUM_SF_MAX_SDB, sfb_size, 0640); |
