From 1812924bb1823950c1dc95c478b71b037057356e Mon Sep 17 00:00:00 2001 From: Cliff Wickman Date: Mon, 2 Jun 2008 08:56:14 -0500 Subject: x86, SGI UV: TLB shootdown using broadcast assist unit TLB shootdown for SGI UV. Depends on patch (in tip/x86/irq): x86-update-macros-used-by-uv-platform.patch Jack Steiner May 29 This patch provides the ability to flush TLB's in cpu's that are not on the local node. The hardware mechanism for distributing the flush messages is the UV's "broadcast assist unit". The hook to intercept TLB shootdown requests is a 2-line change to native_flush_tlb_others() (arch/x86/kernel/tlb_64.c). This code has been tested on a hardware simulator. The real hardware is not yet available. The shootdown statistics are provided through /proc/sgi_uv/ptc_statistics. The use of /sys was considered, but would have required the use of many /sys files. The debugfs was also considered, but these statistics should be available on an ongoing basis, not just for debugging. Issues to be fixed later: - The IRQ for the messaging interrupt is currently hardcoded as 200 (see UV_BAU_MESSAGE). It should be dynamically assigned in the future. - The use of appropriate udelay()'s is untested, as they are a problem in the simulator. Signed-off-by: Cliff Wickman Signed-off-by: Ingo Molnar --- arch/x86/kernel/Makefile | 2 +- arch/x86/kernel/entry_64.S | 4 + arch/x86/kernel/tlb_64.c | 5 + arch/x86/kernel/tlb_uv.c | 736 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 746 insertions(+), 1 deletion(-) create mode 100644 arch/x86/kernel/tlb_uv.c (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 605995e11ea7..4078e98f0125 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -94,7 +94,7 @@ obj-$(CONFIG_OLPC) += olpc.o ### # 64 bit specific files ifeq ($(CONFIG_X86_64),y) - obj-y += genapic_64.o genapic_flat_64.o genx2apic_uv_x.o + obj-y += genapic_64.o genapic_flat_64.o genx2apic_uv_x.o tlb_uv.o obj-$(CONFIG_X86_PM_TIMER) += pmtimer_64.o obj-$(CONFIG_AUDIT) += audit_64.o diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index 556a8df522a7..6fd1987466eb 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S @@ -720,6 +720,10 @@ ENTRY(apic_timer_interrupt) apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt END(apic_timer_interrupt) +ENTRY(uv_bau_message_intr1) + apicinterrupt 220,uv_bau_message_interrupt +END(uv_bau_message_intr1) + ENTRY(error_interrupt) apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt END(error_interrupt) diff --git a/arch/x86/kernel/tlb_64.c b/arch/x86/kernel/tlb_64.c index a1f07d793202..fc132113bdab 100644 --- a/arch/x86/kernel/tlb_64.c +++ b/arch/x86/kernel/tlb_64.c @@ -15,6 +15,8 @@ #include #include #include +#include +#include #include /* @@ -162,6 +164,9 @@ void native_flush_tlb_others(const cpumask_t *cpumaskp, struct mm_struct *mm, union smp_flush_state *f; cpumask_t cpumask = *cpumaskp; + if (is_uv_system() && uv_flush_tlb_others(&cpumask, mm, va)) + return; + /* Caller has disabled preemption */ sender = smp_processor_id() % NUM_INVALIDATE_TLB_VECTORS; f = &per_cpu(flush_state, sender); diff --git a/arch/x86/kernel/tlb_uv.c b/arch/x86/kernel/tlb_uv.c new file mode 100644 index 000000000000..28e7c68d9d78 --- /dev/null +++ b/arch/x86/kernel/tlb_uv.c @@ -0,0 +1,736 @@ +/* + * SGI UltraViolet TLB flush routines. + * + * (c) 2008 Cliff Wickman , SGI. + * + * This code is released under the GNU General Public License version 2 or + * later. + */ +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +struct bau_control **uv_bau_table_bases; +static int uv_bau_retry_limit; +static int uv_nshift; /* position of pnode (which is nasid>>1) */ +static unsigned long uv_mmask; + +char *status_table[] = { + "IDLE", + "ACTIVE", + "DESTINATION TIMEOUT", + "SOURCE TIMEOUT" +}; + +DEFINE_PER_CPU(struct ptc_stats, ptcstats); +DEFINE_PER_CPU(struct bau_control, bau_control); + +/* + * Free a software acknowledge hardware resource by clearing its Pending + * bit. This will return a reply to the sender. + * If the message has timed out, a reply has already been sent by the + * hardware but the resource has not been released. In that case our + * clear of the Timeout bit (as well) will free the resource. No reply will + * be sent (the hardware will only do one reply per message). + */ +static void +uv_reply_to_message(int resource, + struct bau_payload_queue_entry *msg, + struct bau_msg_status *msp) +{ + int fw; + + fw = (1 << (resource + UV_SW_ACK_NPENDING)) | (1 << resource); + msg->replied_to = 1; + msg->sw_ack_vector = 0; + if (msp) + msp->seen_by.bits = 0; + uv_write_local_mmr(UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS, fw); + return; +} + +/* + * Do all the things a cpu should do for a TLB shootdown message. + * Other cpu's may come here at the same time for this message. + */ +static void +uv_bau_process_message(struct bau_payload_queue_entry *msg, + int msg_slot, int sw_ack_slot) +{ + int cpu; + unsigned long this_cpu_mask; + struct bau_msg_status *msp; + + msp = __get_cpu_var(bau_control).msg_statuses + msg_slot; + cpu = uv_blade_processor_id(); + msg->number_of_cpus = + uv_blade_nr_online_cpus(uv_node_to_blade_id(numa_node_id())); + this_cpu_mask = (unsigned long)1 << cpu; + if (msp->seen_by.bits & this_cpu_mask) + return; + atomic_or_long(&msp->seen_by.bits, this_cpu_mask); + + if (msg->replied_to == 1) + return; + + if (msg->address == TLB_FLUSH_ALL) { + local_flush_tlb(); + __get_cpu_var(ptcstats).alltlb++; + } else { + __flush_tlb_one(msg->address); + __get_cpu_var(ptcstats).onetlb++; + } + + __get_cpu_var(ptcstats).requestee++; + + atomic_inc_short(&msg->acknowledge_count); + if (msg->number_of_cpus == msg->acknowledge_count) + uv_reply_to_message(sw_ack_slot, msg, msp); + return; +} + +/* + * Examine the payload queue on all the distribution nodes to see + * which messages have not been seen, and which cpu(s) have not seen them. + * + * Returns the number of cpu's that have not responded. + */ +static int +uv_examine_destinations(struct bau_target_nodemask *distribution) +{ + int sender; + int i; + int j; + int k; + int count = 0; + struct bau_control *bau_tablesp; + struct bau_payload_queue_entry *msg; + struct bau_msg_status *msp; + + sender = smp_processor_id(); + for (i = 0; i < (sizeof(struct bau_target_nodemask) * BITSPERBYTE); + i++) { + if (bau_node_isset(i, distribution)) { + bau_tablesp = uv_bau_table_bases[i]; + for (msg = bau_tablesp->va_queue_first, j = 0; + j < DESTINATION_PAYLOAD_QUEUE_SIZE; msg++, j++) { + if ((msg->sending_cpu == sender) && + (!msg->replied_to)) { + msp = bau_tablesp->msg_statuses + j; + printk(KERN_DEBUG + "blade %d: address:%#lx %d of %d, not cpu(s): ", + i, msg->address, + msg->acknowledge_count, + msg->number_of_cpus); + for (k = 0; k < msg->number_of_cpus; + k++) { + if (!((long)1 << k & msp-> + seen_by.bits)) { + count++; + printk("%d ", k); + } + } + printk("\n"); + } + } + } + } + return count; +} + +/** + * uv_flush_tlb_others - globally purge translation cache of a virtual + * address or all TLB's + * @cpumaskp: mask of all cpu's in which the address is to be removed + * @mm: mm_struct containing virtual address range + * @va: virtual address to be removed (or TLB_FLUSH_ALL for all TLB's on cpu) + * + * This is the entry point for initiating any UV global TLB shootdown. + * + * Purges the translation caches of all specified processors of the given + * virtual address, or purges all TLB's on specified processors. + * + * The caller has derived the cpumaskp from the mm_struct and has subtracted + * the local cpu from the mask. This function is called only if there + * are bits set in the mask. (e.g. flush_tlb_page()) + * + * The cpumaskp is converted into a nodemask of the nodes containing + * the cpus. + */ +int +uv_flush_tlb_others(cpumask_t *cpumaskp, struct mm_struct *mm, unsigned long va) +{ + int i; + int blade; + int cpu; + int bit; + int right_shift; + int this_blade; + int exams = 0; + int tries = 0; + long source_timeouts = 0; + long destination_timeouts = 0; + unsigned long index; + unsigned long mmr_offset; + unsigned long descriptor_status; + struct bau_activation_descriptor *bau_desc; + ktime_t time1, time2; + + cpu = uv_blade_processor_id(); + this_blade = uv_numa_blade_id(); + bau_desc = __get_cpu_var(bau_control).descriptor_base; + bau_desc += (UV_ITEMS_PER_DESCRIPTOR * cpu); + + bau_nodes_clear(&bau_desc->distribution, UV_DISTRIBUTION_SIZE); + + i = 0; + for_each_cpu_mask(bit, *cpumaskp) { + blade = uv_cpu_to_blade_id(bit); + if (blade > (UV_DISTRIBUTION_SIZE - 1)) + BUG(); + if (blade == this_blade) + continue; + bau_node_set(blade, &bau_desc->distribution); + /* leave the bits for the remote cpu's in the mask until + success; on failure we fall back to the IPI method */ + i++; + } + if (i == 0) + goto none_to_flush; + __get_cpu_var(ptcstats).requestor++; + __get_cpu_var(ptcstats).ntargeted += i; + + bau_desc->payload.address = va; + bau_desc->payload.sending_cpu = smp_processor_id(); + + if (cpu < UV_CPUS_PER_ACT_STATUS) { + mmr_offset = UVH_LB_BAU_SB_ACTIVATION_STATUS_0; + right_shift = cpu * UV_ACT_STATUS_SIZE; + } else { + mmr_offset = UVH_LB_BAU_SB_ACTIVATION_STATUS_1; + right_shift = + ((cpu - UV_CPUS_PER_ACT_STATUS) * UV_ACT_STATUS_SIZE); + } + time1 = ktime_get(); + +retry: + tries++; + index = ((unsigned long) + 1 << UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_SHFT) | cpu; + uv_write_local_mmr(UVH_LB_BAU_SB_ACTIVATION_CONTROL, index); + + while ((descriptor_status = (((unsigned long) + uv_read_local_mmr(mmr_offset) >> + right_shift) & UV_ACT_STATUS_MASK)) != + DESC_STATUS_IDLE) { + if (descriptor_status == DESC_STATUS_SOURCE_TIMEOUT) { + source_timeouts++; + if (source_timeouts > SOURCE_TIMEOUT_LIMIT) + source_timeouts = 0; + __get_cpu_var(ptcstats).s_retry++; + goto retry; + } + /* spin here looking for progress at the destinations */ + if (descriptor_status == DESC_STATUS_DESTINATION_TIMEOUT) { + destination_timeouts++; + if (destination_timeouts > DESTINATION_TIMEOUT_LIMIT) { + /* returns # of cpus not responding */ + if (uv_examine_destinations + (&bau_desc->distribution) == 0) { + __get_cpu_var(ptcstats).d_retry++; + goto retry; + } + exams++; + if (exams >= uv_bau_retry_limit) { + printk(KERN_DEBUG + "uv_flush_tlb_others"); + printk("giving up on cpu %d\n", + smp_processor_id()); + goto unsuccessful; + } + /* delays can hang up the simulator + udelay(1000); + */ + destination_timeouts = 0; + } + } + } + if (tries > 1) + __get_cpu_var(ptcstats).retriesok++; + /* on success, clear the remote cpu's from the mask so we don't + use the IPI method of shootdown on them */ + for_each_cpu_mask(bit, *cpumaskp) { + blade = uv_cpu_to_blade_id(bit); + if (blade == this_blade) + continue; + cpu_clear(bit, *cpumaskp); + } + +unsuccessful: + time2 = ktime_get(); + __get_cpu_var(ptcstats).sflush_ns += (time2.tv64 - time1.tv64); + +none_to_flush: + if (cpus_empty(*cpumaskp)) + return 1; + + /* Cause the caller to do an IPI-style TLB shootdown on + the cpu's still in the mask */ + __get_cpu_var(ptcstats).ptc_i++; + return 0; +} + +/* + * The BAU message interrupt comes here. (registered by set_intr_gate) + * See entry_64.S + * + * We received a broadcast assist message. + * + * Interrupts may have been disabled; this interrupt could represent + * the receipt of several messages. + * + * All cores/threads on this node get this interrupt. + * The last one to see it does the s/w ack. + * (the resource will not be freed until noninterruptable cpus see this + * interrupt; hardware will timeout the s/w ack and reply ERROR) + */ +void +uv_bau_message_interrupt(struct pt_regs *regs) +{ + struct bau_payload_queue_entry *pqp; + struct bau_payload_queue_entry *msg; + struct pt_regs *old_regs = set_irq_regs(regs); + ktime_t time1, time2; + int msg_slot; + int sw_ack_slot; + int fw; + int count = 0; + unsigned long local_pnode; + + ack_APIC_irq(); + exit_idle(); + irq_enter(); + + time1 = ktime_get(); + + local_pnode = uv_blade_to_pnode(uv_numa_blade_id()); + + pqp = __get_cpu_var(bau_control).va_queue_first; + msg = __get_cpu_var(bau_control).bau_msg_head; + while (msg->sw_ack_vector) { + count++; + fw = msg->sw_ack_vector; + msg_slot = msg - pqp; + sw_ack_slot = ffs(fw) - 1; + + uv_bau_process_message(msg, msg_slot, sw_ack_slot); + + msg++; + if (msg > __get_cpu_var(bau_control).va_queue_last) + msg = __get_cpu_var(bau_control).va_queue_first; + __get_cpu_var(bau_control).bau_msg_head = msg; + } + if (!count) + __get_cpu_var(ptcstats).nomsg++; + else if (count > 1) + __get_cpu_var(ptcstats).multmsg++; + + time2 = ktime_get(); + __get_cpu_var(ptcstats).dflush_ns += (time2.tv64 - time1.tv64); + + irq_exit(); + set_irq_regs(old_regs); + return; +} + +static void +uv_enable_timeouts(void) +{ + int i; + int blade; + int last_blade; + int pnode; + int cur_cpu = 0; + unsigned long apicid; + + /* better if we had each_online_blade */ + last_blade = -1; + for_each_online_node(i) { + blade = uv_node_to_blade_id(i); + if (blade == last_blade) + continue; + last_blade = blade; + apicid = per_cpu(x86_cpu_to_apicid, cur_cpu); + pnode = uv_blade_to_pnode(blade); + cur_cpu += uv_blade_nr_possible_cpus(i); + } + return; +} + +static void * +uv_ptc_seq_start(struct seq_file *file, loff_t *offset) +{ + if (*offset < num_possible_cpus()) + return offset; + return NULL; +} + +static void * +uv_ptc_seq_next(struct seq_file *file, void *data, loff_t *offset) +{ + (*offset)++; + if (*offset < num_possible_cpus()) + return offset; + return NULL; +} + +static void +uv_ptc_seq_stop(struct seq_file *file, void *data) +{ +} + +/* + * Display the statistics thru /proc + * data points to the cpu number + */ +static int +uv_ptc_seq_show(struct seq_file *file, void *data) +{ + struct ptc_stats *stat; + int cpu; + + cpu = *(loff_t *)data; + + if (!cpu) { + seq_printf(file, + "# cpu requestor requestee one all sretry dretry ptc_i "); + seq_printf(file, + "sw_ack sflush_us dflush_us sok dnomsg dmult starget\n"); + } + if (cpu < num_possible_cpus() && cpu_online(cpu)) { + stat = &per_cpu(ptcstats, cpu); + seq_printf(file, "cpu %d %ld %ld %ld %ld %ld %ld %ld ", + cpu, stat->requestor, + stat->requestee, stat->onetlb, stat->alltlb, + stat->s_retry, stat->d_retry, stat->ptc_i); + seq_printf(file, "%lx %ld %ld %ld %ld %ld %ld\n", + uv_read_global_mmr64(uv_blade_to_pnode + (uv_cpu_to_blade_id(cpu)), + UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE), + stat->sflush_ns / 1000, stat->dflush_ns / 1000, + stat->retriesok, stat->nomsg, + stat->multmsg, stat->ntargeted); + } + + return 0; +} + +/* + * 0: display meaning of the statistics + * >0: retry limit + */ +static ssize_t +uv_ptc_proc_write(struct file *file, const char __user *user, + size_t count, loff_t *data) +{ + long newmode; + char optstr[64]; + + if (copy_from_user(optstr, user, count)) + return -EFAULT; + optstr[count - 1] = '\0'; + if (strict_strtoul(optstr, 10, &newmode) < 0) { + printk(KERN_DEBUG "%s is invalid\n", optstr); + return -EINVAL; + } + + if (newmode == 0) { + printk(KERN_DEBUG "# cpu: cpu number\n"); + printk(KERN_DEBUG + "requestor: times this cpu was the flush requestor\n"); + printk(KERN_DEBUG + "requestee: times this cpu was requested to flush its TLBs\n"); + printk(KERN_DEBUG + "one: times requested to flush a single address\n"); + printk(KERN_DEBUG + "all: times requested to flush all TLB's\n"); + printk(KERN_DEBUG + "sretry: number of retries of source-side timeouts\n"); + printk(KERN_DEBUG + "dretry: number of retries of destination-side timeouts\n"); + printk(KERN_DEBUG + "ptc_i: times UV fell through to IPI-style flushes\n"); + printk(KERN_DEBUG + "sw_ack: image of UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE\n"); + printk(KERN_DEBUG + "sflush_us: microseconds spent in uv_flush_tlb_others()\n"); + printk(KERN_DEBUG + "dflush_us: microseconds spent in handling flush requests\n"); + printk(KERN_DEBUG "sok: successes on retry\n"); + printk(KERN_DEBUG "dnomsg: interrupts with no message\n"); + printk(KERN_DEBUG + "dmult: interrupts with multiple messages\n"); + printk(KERN_DEBUG "starget: nodes targeted\n"); + } else { + uv_bau_retry_limit = newmode; + printk(KERN_DEBUG "timeout retry limit:%d\n", + uv_bau_retry_limit); + } + + return count; +} + +static const struct seq_operations uv_ptc_seq_ops = { + .start = uv_ptc_seq_start, + .next = uv_ptc_seq_next, + .stop = uv_ptc_seq_stop, + .show = uv_ptc_seq_show +}; + +static int +uv_ptc_proc_open(struct inode *inode, struct file *file) +{ + return seq_open(file, &uv_ptc_seq_ops); +} + +static const struct file_operations proc_uv_ptc_operations = { + .open = uv_ptc_proc_open, + .read = seq_read, + .write = uv_ptc_proc_write, + .llseek = seq_lseek, + .release = seq_release, +}; + +static struct proc_dir_entry *proc_uv_ptc; + +static int __init +uv_ptc_init(void) +{ + static struct proc_dir_entry *sgi_proc_dir; + + sgi_proc_dir = NULL; + + if (!is_uv_system()) + return 0; + + sgi_proc_dir = proc_mkdir("sgi_uv", NULL); + if (!sgi_proc_dir) + return -EINVAL; + + proc_uv_ptc = create_proc_entry(UV_PTC_BASENAME, 0444, NULL); + if (!proc_uv_ptc) { + printk(KERN_ERR "unable to create %s proc entry\n", + UV_PTC_BASENAME); + return -EINVAL; + } + proc_uv_ptc->proc_fops = &proc_uv_ptc_operations; + return 0; +} + +static void __exit +uv_ptc_exit(void) +{ + remove_proc_entry(UV_PTC_BASENAME, NULL); +} + +module_init(uv_ptc_init); +module_exit(uv_ptc_exit); + +/* + * Initialization of BAU-related structures + */ +int __init +uv_bau_init(void) +{ + int i; + int j; + int blade; + int nblades; + int *ip; + int pnode; + int last_blade; + int cur_cpu = 0; + unsigned long pa; + unsigned long n; + unsigned long m; + unsigned long mmr_image; + unsigned long apicid; + char *cp; + struct bau_control *bau_tablesp; + struct bau_activation_descriptor *adp, *ad2; + struct bau_payload_queue_entry *pqp; + struct bau_msg_status *msp; + struct bau_control *bcp; + + if (!is_uv_system()) + return 0; + + uv_bau_retry_limit = 1; + + if ((sizeof(struct bau_local_cpumask) * BITSPERBYTE) < + MAX_CPUS_PER_NODE) { + printk(KERN_ERR + "uv_bau_init: bau_local_cpumask.bits too small\n"); + BUG(); + } + + uv_nshift = uv_hub_info->n_val; + uv_mmask = ((unsigned long)1 << uv_hub_info->n_val) - 1; + nblades = 0; + last_blade = -1; + for_each_online_node(i) { + blade = uv_node_to_blade_id(i); + if (blade == last_blade) + continue; + last_blade = blade; + nblades++; + } + + uv_bau_table_bases = (struct bau_control **) + kmalloc(nblades * sizeof(struct bau_control *), GFP_KERNEL); + if (!uv_bau_table_bases) + BUG(); + + /* better if we had each_online_blade */ + last_blade = -1; + for_each_online_node(i) { + blade = uv_node_to_blade_id(i); + if (blade == last_blade) + continue; + last_blade = blade; + + bau_tablesp = + kmalloc_node(sizeof(struct bau_control), GFP_KERNEL, i); + if (!bau_tablesp) + BUG(); + + bau_tablesp->msg_statuses = + kmalloc_node(sizeof(struct bau_msg_status) * + DESTINATION_PAYLOAD_QUEUE_SIZE, GFP_KERNEL, i); + if (!bau_tablesp->msg_statuses) + BUG(); + for (j = 0, msp = bau_tablesp->msg_statuses; + j < DESTINATION_PAYLOAD_QUEUE_SIZE; j++, msp++) { + bau_cpubits_clear(&msp->seen_by, (int) + uv_blade_nr_possible_cpus(blade)); + } + + bau_tablesp->watching = + kmalloc_node(sizeof(int) * DESTINATION_NUM_RESOURCES, + GFP_KERNEL, i); + if (!bau_tablesp->watching) + BUG(); + for (j = 0, ip = bau_tablesp->watching; + j < DESTINATION_PAYLOAD_QUEUE_SIZE; j++, ip++) { + *ip = 0; + } + + uv_bau_table_bases[i] = bau_tablesp; + + pnode = uv_blade_to_pnode(blade); + + if (sizeof(struct bau_activation_descriptor) != 64) + BUG(); + + adp = (struct bau_activation_descriptor *) + kmalloc_node(16384, GFP_KERNEL, i); + if (!adp) + BUG(); + if ((unsigned long)adp & 0xfff) + BUG(); + pa = __pa((unsigned long)adp); + n = pa >> uv_nshift; + m = pa & uv_mmask; + + mmr_image = uv_read_global_mmr64(pnode, + UVH_LB_BAU_SB_DESCRIPTOR_BASE); + if (mmr_image) + uv_write_global_mmr64(pnode, (unsigned long) + UVH_LB_BAU_SB_DESCRIPTOR_BASE, + (n << UV_DESC_BASE_PNODE_SHIFT | + m)); + for (j = 0, ad2 = adp; j < UV_ACTIVATION_DESCRIPTOR_SIZE; + j++, ad2++) { + memset(ad2, 0, + sizeof(struct bau_activation_descriptor)); + ad2->header.sw_ack_flag = 1; + ad2->header.base_dest_nodeid = + uv_blade_to_pnode(uv_cpu_to_blade_id(0)); + ad2->header.command = UV_NET_ENDPOINT_INTD; + ad2->header.int_both = 1; + /* all others need to be set to zero: + fairness chaining multilevel count replied_to */ + } + + pqp = (struct bau_payload_queue_entry *) + kmalloc_node((DESTINATION_PAYLOAD_QUEUE_SIZE + 1) * + sizeof(struct bau_payload_queue_entry), + GFP_KERNEL, i); + if (!pqp) + BUG(); + if (sizeof(struct bau_payload_queue_entry) != 32) + BUG(); + if ((unsigned long)(&((struct bau_payload_queue_entry *)0)-> + sw_ack_vector) != 15) + BUG(); + + cp = (char *)pqp + 31; + pqp = (struct bau_payload_queue_entry *) + (((unsigned long)cp >> 5) << 5); + bau_tablesp->va_queue_first = pqp; + uv_write_global_mmr64(pnode, + UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST, + ((unsigned long)pnode << + UV_PAYLOADQ_PNODE_SHIFT) | + uv_physnodeaddr(pqp)); + uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL, + uv_physnodeaddr(pqp)); + bau_tablesp->va_queue_last = + pqp + (DESTINATION_PAYLOAD_QUEUE_SIZE - 1); + uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST, + (unsigned long) + uv_physnodeaddr(bau_tablesp-> + va_queue_last)); + memset(pqp, 0, sizeof(struct bau_payload_queue_entry) * + DESTINATION_PAYLOAD_QUEUE_SIZE); + + /* this initialization can't be in firmware because the + messaging IRQ will be determined by the OS */ + apicid = per_cpu(x86_cpu_to_apicid, cur_cpu); + pa = uv_read_global_mmr64(pnode, UVH_BAU_DATA_CONFIG); + if ((pa & 0xff) != UV_BAU_MESSAGE) { + uv_write_global_mmr64(pnode, UVH_BAU_DATA_CONFIG, + ((apicid << 32) | + UV_BAU_MESSAGE)); + } + + for (j = cur_cpu; j < (cur_cpu + uv_blade_nr_possible_cpus(i)); + j++) { + bcp = (struct bau_control *)&per_cpu(bau_control, j); + bcp->bau_msg_head = bau_tablesp->va_queue_first; + bcp->va_queue_first = bau_tablesp->va_queue_first; + + bcp->va_queue_last = bau_tablesp->va_queue_last; + bcp->watching = bau_tablesp->watching; + bcp->msg_statuses = bau_tablesp->msg_statuses; + bcp->descriptor_base = adp; + } + cur_cpu += uv_blade_nr_possible_cpus(i); + } + + set_intr_gate(UV_BAU_MESSAGE, uv_bau_message_intr1); + + uv_enable_timeouts(); + + return 0; +} + +__initcall(uv_bau_init); -- cgit v1.2.1 From b194b120507276b4f09e2e14f941884e777fc7c8 Mon Sep 17 00:00:00 2001 From: Cliff Wickman Date: Thu, 12 Jun 2008 08:23:48 -0500 Subject: SGI UV: TLB shootdown using broadcast assist unit, cleanups TLB shootdown for SGI UV. v1: 6/2 original v2: 6/3 corrections/improvements per Ingo's review v3: 6/4 split atomic operations off to a separate patch (Jeremy's review) v4: 6/12 include rather than (fixes a !SMP build problem that Ingo found) fix the index on uv_table_bases[blade] Signed-off-by: Cliff Wickman Signed-off-by: Ingo Molnar --- arch/x86/kernel/tlb_64.c | 2 +- arch/x86/kernel/tlb_uv.c | 704 +++++++++++++++++++++++++---------------------- 2 files changed, 378 insertions(+), 328 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/tlb_64.c b/arch/x86/kernel/tlb_64.c index fc132113bdab..5039d0f097a2 100644 --- a/arch/x86/kernel/tlb_64.c +++ b/arch/x86/kernel/tlb_64.c @@ -165,7 +165,7 @@ void native_flush_tlb_others(const cpumask_t *cpumaskp, struct mm_struct *mm, cpumask_t cpumask = *cpumaskp; if (is_uv_system() && uv_flush_tlb_others(&cpumask, mm, va)) - return; + return; /* Caller has disabled preemption */ sender = smp_processor_id() % NUM_INVALIDATE_TLB_VECTORS; diff --git a/arch/x86/kernel/tlb_uv.c b/arch/x86/kernel/tlb_uv.c index 28e7c68d9d78..f7bc6a6fbe49 100644 --- a/arch/x86/kernel/tlb_uv.c +++ b/arch/x86/kernel/tlb_uv.c @@ -10,18 +10,20 @@ #include #include -#include #include #include #include #include #include #include +#include -struct bau_control **uv_bau_table_bases; -static int uv_bau_retry_limit; -static int uv_nshift; /* position of pnode (which is nasid>>1) */ -static unsigned long uv_mmask; +#include + +static struct bau_control **uv_bau_table_bases __read_mostly; +static int uv_bau_retry_limit __read_mostly; +static int uv_nshift __read_mostly; /* position of pnode (which is nasid>>1) */ +static unsigned long uv_mmask __read_mostly; char *status_table[] = { "IDLE", @@ -41,19 +43,18 @@ DEFINE_PER_CPU(struct bau_control, bau_control); * clear of the Timeout bit (as well) will free the resource. No reply will * be sent (the hardware will only do one reply per message). */ -static void -uv_reply_to_message(int resource, +static void uv_reply_to_message(int resource, struct bau_payload_queue_entry *msg, struct bau_msg_status *msp) { - int fw; + unsigned long dw; - fw = (1 << (resource + UV_SW_ACK_NPENDING)) | (1 << resource); + dw = (1 << (resource + UV_SW_ACK_NPENDING)) | (1 << resource); msg->replied_to = 1; msg->sw_ack_vector = 0; if (msp) msp->seen_by.bits = 0; - uv_write_local_mmr(UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS, fw); + uv_write_local_mmr(UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS, dw); return; } @@ -61,8 +62,7 @@ uv_reply_to_message(int resource, * Do all the things a cpu should do for a TLB shootdown message. * Other cpu's may come here at the same time for this message. */ -static void -uv_bau_process_message(struct bau_payload_queue_entry *msg, +static void uv_bau_process_message(struct bau_payload_queue_entry *msg, int msg_slot, int sw_ack_slot) { int cpu; @@ -103,8 +103,7 @@ uv_bau_process_message(struct bau_payload_queue_entry *msg, * * Returns the number of cpu's that have not responded. */ -static int -uv_examine_destinations(struct bau_target_nodemask *distribution) +static int uv_examine_destinations(struct bau_target_nodemask *distribution) { int sender; int i; @@ -118,34 +117,161 @@ uv_examine_destinations(struct bau_target_nodemask *distribution) sender = smp_processor_id(); for (i = 0; i < (sizeof(struct bau_target_nodemask) * BITSPERBYTE); i++) { - if (bau_node_isset(i, distribution)) { - bau_tablesp = uv_bau_table_bases[i]; - for (msg = bau_tablesp->va_queue_first, j = 0; - j < DESTINATION_PAYLOAD_QUEUE_SIZE; msg++, j++) { - if ((msg->sending_cpu == sender) && - (!msg->replied_to)) { - msp = bau_tablesp->msg_statuses + j; - printk(KERN_DEBUG + if (!bau_node_isset(i, distribution)) + continue; + bau_tablesp = uv_bau_table_bases[i]; + for (msg = bau_tablesp->va_queue_first, j = 0; + j < DESTINATION_PAYLOAD_QUEUE_SIZE; msg++, j++) { + if ((msg->sending_cpu == sender) && + (!msg->replied_to)) { + msp = bau_tablesp->msg_statuses + j; + printk(KERN_DEBUG "blade %d: address:%#lx %d of %d, not cpu(s): ", - i, msg->address, - msg->acknowledge_count, - msg->number_of_cpus); - for (k = 0; k < msg->number_of_cpus; - k++) { - if (!((long)1 << k & msp-> - seen_by.bits)) { - count++; - printk("%d ", k); - } + i, msg->address, + msg->acknowledge_count, + msg->number_of_cpus); + for (k = 0; k < msg->number_of_cpus; + k++) { + if (!((long)1 << k & msp-> + seen_by.bits)) { + count++; + printk("%d ", k); } - printk("\n"); } + printk("\n"); } } } return count; } +/* + * wait for completion of a broadcast message + * + * return COMPLETE, RETRY or GIVEUP + */ +static int uv_wait_completion(struct bau_activation_descriptor *bau_desc, + unsigned long mmr_offset, int right_shift) +{ + int exams = 0; + long destination_timeouts = 0; + long source_timeouts = 0; + unsigned long descriptor_status; + + while ((descriptor_status = (((unsigned long) + uv_read_local_mmr(mmr_offset) >> + right_shift) & UV_ACT_STATUS_MASK)) != + DESC_STATUS_IDLE) { + if (descriptor_status == DESC_STATUS_SOURCE_TIMEOUT) { + source_timeouts++; + if (source_timeouts > SOURCE_TIMEOUT_LIMIT) + source_timeouts = 0; + __get_cpu_var(ptcstats).s_retry++; + return FLUSH_RETRY; + } + /* + * spin here looking for progress at the destinations + */ + if (descriptor_status == DESC_STATUS_DESTINATION_TIMEOUT) { + destination_timeouts++; + if (destination_timeouts > DESTINATION_TIMEOUT_LIMIT) { + /* + * returns number of cpus not responding + */ + if (uv_examine_destinations + (&bau_desc->distribution) == 0) { + __get_cpu_var(ptcstats).d_retry++; + return FLUSH_RETRY; + } + exams++; + if (exams >= uv_bau_retry_limit) { + printk(KERN_DEBUG + "uv_flush_tlb_others"); + printk("giving up on cpu %d\n", + smp_processor_id()); + return FLUSH_GIVEUP; + } + /* + * delays can hang the simulator + udelay(1000); + */ + destination_timeouts = 0; + } + } + } + return FLUSH_COMPLETE; +} + +/** + * uv_flush_send_and_wait + * + * Send a broadcast and wait for a broadcast message to complete. + * + * The cpumaskp mask contains the cpus the broadcast was sent to. + * + * Returns 1 if all remote flushing was done. The mask is zeroed. + * Returns 0 if some remote flushing remains to be done. The mask is left + * unchanged. + */ +int uv_flush_send_and_wait(int cpu, int this_blade, + struct bau_activation_descriptor *bau_desc, cpumask_t *cpumaskp) +{ + int completion_status = 0; + int right_shift; + int bit; + int blade; + int tries = 0; + unsigned long index; + unsigned long mmr_offset; + cycles_t time1; + cycles_t time2; + + if (cpu < UV_CPUS_PER_ACT_STATUS) { + mmr_offset = UVH_LB_BAU_SB_ACTIVATION_STATUS_0; + right_shift = cpu * UV_ACT_STATUS_SIZE; + } else { + mmr_offset = UVH_LB_BAU_SB_ACTIVATION_STATUS_1; + right_shift = + ((cpu - UV_CPUS_PER_ACT_STATUS) * UV_ACT_STATUS_SIZE); + } + time1 = get_cycles(); + do { + tries++; + index = ((unsigned long) + 1 << UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_SHFT) | cpu; + uv_write_local_mmr(UVH_LB_BAU_SB_ACTIVATION_CONTROL, index); + completion_status = uv_wait_completion(bau_desc, mmr_offset, + right_shift); + } while (completion_status == FLUSH_RETRY); + time2 = get_cycles(); + __get_cpu_var(ptcstats).sflush += (time2 - time1); + if (tries > 1) + __get_cpu_var(ptcstats).retriesok++; + + if (completion_status == FLUSH_GIVEUP) { + /* + * Cause the caller to do an IPI-style TLB shootdown on + * the cpu's, all of which are still in the mask. + */ + __get_cpu_var(ptcstats).ptc_i++; + return 0; + } + + /* + * Success, so clear the remote cpu's from the mask so we don't + * use the IPI method of shootdown on them. + */ + for_each_cpu_mask(bit, *cpumaskp) { + blade = uv_cpu_to_blade_id(bit); + if (blade == this_blade) + continue; + cpu_clear(bit, *cpumaskp); + } + if (!cpus_empty(*cpumaskp)) + return 0; + return 1; +} + /** * uv_flush_tlb_others - globally purge translation cache of a virtual * address or all TLB's @@ -164,30 +290,25 @@ uv_examine_destinations(struct bau_target_nodemask *distribution) * * The cpumaskp is converted into a nodemask of the nodes containing * the cpus. + * + * Returns 1 if all remote flushing was done. + * Returns 0 if some remote flushing remains to be done. */ -int -uv_flush_tlb_others(cpumask_t *cpumaskp, struct mm_struct *mm, unsigned long va) +int uv_flush_tlb_others(cpumask_t *cpumaskp, struct mm_struct *mm, + unsigned long va) { int i; + int bit; int blade; int cpu; - int bit; - int right_shift; int this_blade; - int exams = 0; - int tries = 0; - long source_timeouts = 0; - long destination_timeouts = 0; - unsigned long index; - unsigned long mmr_offset; - unsigned long descriptor_status; + int locals = 0; struct bau_activation_descriptor *bau_desc; - ktime_t time1, time2; cpu = uv_blade_processor_id(); this_blade = uv_numa_blade_id(); bau_desc = __get_cpu_var(bau_control).descriptor_base; - bau_desc += (UV_ITEMS_PER_DESCRIPTOR * cpu); + bau_desc += UV_ITEMS_PER_DESCRIPTOR * cpu; bau_nodes_clear(&bau_desc->distribution, UV_DISTRIBUTION_SIZE); @@ -196,96 +317,29 @@ uv_flush_tlb_others(cpumask_t *cpumaskp, struct mm_struct *mm, unsigned long va) blade = uv_cpu_to_blade_id(bit); if (blade > (UV_DISTRIBUTION_SIZE - 1)) BUG(); - if (blade == this_blade) + if (blade == this_blade) { + locals++; continue; + } bau_node_set(blade, &bau_desc->distribution); - /* leave the bits for the remote cpu's in the mask until - success; on failure we fall back to the IPI method */ i++; } - if (i == 0) - goto none_to_flush; + if (i == 0) { + /* + * no off_node flushing; return status for local node + */ + if (locals) + return 0; + else + return 1; + } __get_cpu_var(ptcstats).requestor++; __get_cpu_var(ptcstats).ntargeted += i; bau_desc->payload.address = va; bau_desc->payload.sending_cpu = smp_processor_id(); - if (cpu < UV_CPUS_PER_ACT_STATUS) { - mmr_offset = UVH_LB_BAU_SB_ACTIVATION_STATUS_0; - right_shift = cpu * UV_ACT_STATUS_SIZE; - } else { - mmr_offset = UVH_LB_BAU_SB_ACTIVATION_STATUS_1; - right_shift = - ((cpu - UV_CPUS_PER_ACT_STATUS) * UV_ACT_STATUS_SIZE); - } - time1 = ktime_get(); - -retry: - tries++; - index = ((unsigned long) - 1 << UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_SHFT) | cpu; - uv_write_local_mmr(UVH_LB_BAU_SB_ACTIVATION_CONTROL, index); - - while ((descriptor_status = (((unsigned long) - uv_read_local_mmr(mmr_offset) >> - right_shift) & UV_ACT_STATUS_MASK)) != - DESC_STATUS_IDLE) { - if (descriptor_status == DESC_STATUS_SOURCE_TIMEOUT) { - source_timeouts++; - if (source_timeouts > SOURCE_TIMEOUT_LIMIT) - source_timeouts = 0; - __get_cpu_var(ptcstats).s_retry++; - goto retry; - } - /* spin here looking for progress at the destinations */ - if (descriptor_status == DESC_STATUS_DESTINATION_TIMEOUT) { - destination_timeouts++; - if (destination_timeouts > DESTINATION_TIMEOUT_LIMIT) { - /* returns # of cpus not responding */ - if (uv_examine_destinations - (&bau_desc->distribution) == 0) { - __get_cpu_var(ptcstats).d_retry++; - goto retry; - } - exams++; - if (exams >= uv_bau_retry_limit) { - printk(KERN_DEBUG - "uv_flush_tlb_others"); - printk("giving up on cpu %d\n", - smp_processor_id()); - goto unsuccessful; - } - /* delays can hang up the simulator - udelay(1000); - */ - destination_timeouts = 0; - } - } - } - if (tries > 1) - __get_cpu_var(ptcstats).retriesok++; - /* on success, clear the remote cpu's from the mask so we don't - use the IPI method of shootdown on them */ - for_each_cpu_mask(bit, *cpumaskp) { - blade = uv_cpu_to_blade_id(bit); - if (blade == this_blade) - continue; - cpu_clear(bit, *cpumaskp); - } - -unsuccessful: - time2 = ktime_get(); - __get_cpu_var(ptcstats).sflush_ns += (time2.tv64 - time1.tv64); - -none_to_flush: - if (cpus_empty(*cpumaskp)) - return 1; - - /* Cause the caller to do an IPI-style TLB shootdown on - the cpu's still in the mask */ - __get_cpu_var(ptcstats).ptc_i++; - return 0; + return uv_flush_send_and_wait(cpu, this_blade, bau_desc, cpumaskp); } /* @@ -302,13 +356,12 @@ none_to_flush: * (the resource will not be freed until noninterruptable cpus see this * interrupt; hardware will timeout the s/w ack and reply ERROR) */ -void -uv_bau_message_interrupt(struct pt_regs *regs) +void uv_bau_message_interrupt(struct pt_regs *regs) { struct bau_payload_queue_entry *pqp; struct bau_payload_queue_entry *msg; struct pt_regs *old_regs = set_irq_regs(regs); - ktime_t time1, time2; + cycles_t time1, time2; int msg_slot; int sw_ack_slot; int fw; @@ -319,7 +372,7 @@ uv_bau_message_interrupt(struct pt_regs *regs) exit_idle(); irq_enter(); - time1 = ktime_get(); + time1 = get_cycles(); local_pnode = uv_blade_to_pnode(uv_numa_blade_id()); @@ -343,16 +396,15 @@ uv_bau_message_interrupt(struct pt_regs *regs) else if (count > 1) __get_cpu_var(ptcstats).multmsg++; - time2 = ktime_get(); - __get_cpu_var(ptcstats).dflush_ns += (time2.tv64 - time1.tv64); + time2 = get_cycles(); + __get_cpu_var(ptcstats).dflush += (time2 - time1); irq_exit(); set_irq_regs(old_regs); return; } -static void -uv_enable_timeouts(void) +static void uv_enable_timeouts(void) { int i; int blade; @@ -361,7 +413,6 @@ uv_enable_timeouts(void) int cur_cpu = 0; unsigned long apicid; - /* better if we had each_online_blade */ last_blade = -1; for_each_online_node(i) { blade = uv_node_to_blade_id(i); @@ -375,16 +426,14 @@ uv_enable_timeouts(void) return; } -static void * -uv_ptc_seq_start(struct seq_file *file, loff_t *offset) +static void *uv_ptc_seq_start(struct seq_file *file, loff_t *offset) { if (*offset < num_possible_cpus()) return offset; return NULL; } -static void * -uv_ptc_seq_next(struct seq_file *file, void *data, loff_t *offset) +static void *uv_ptc_seq_next(struct seq_file *file, void *data, loff_t *offset) { (*offset)++; if (*offset < num_possible_cpus()) @@ -392,8 +441,7 @@ uv_ptc_seq_next(struct seq_file *file, void *data, loff_t *offset) return NULL; } -static void -uv_ptc_seq_stop(struct seq_file *file, void *data) +static void uv_ptc_seq_stop(struct seq_file *file, void *data) { } @@ -401,8 +449,7 @@ uv_ptc_seq_stop(struct seq_file *file, void *data) * Display the statistics thru /proc * data points to the cpu number */ -static int -uv_ptc_seq_show(struct seq_file *file, void *data) +static int uv_ptc_seq_show(struct seq_file *file, void *data) { struct ptc_stats *stat; int cpu; @@ -413,7 +460,7 @@ uv_ptc_seq_show(struct seq_file *file, void *data) seq_printf(file, "# cpu requestor requestee one all sretry dretry ptc_i "); seq_printf(file, - "sw_ack sflush_us dflush_us sok dnomsg dmult starget\n"); + "sw_ack sflush dflush sok dnomsg dmult starget\n"); } if (cpu < num_possible_cpus() && cpu_online(cpu)) { stat = &per_cpu(ptcstats, cpu); @@ -425,7 +472,7 @@ uv_ptc_seq_show(struct seq_file *file, void *data) uv_read_global_mmr64(uv_blade_to_pnode (uv_cpu_to_blade_id(cpu)), UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE), - stat->sflush_ns / 1000, stat->dflush_ns / 1000, + stat->sflush, stat->dflush, stat->retriesok, stat->nomsg, stat->multmsg, stat->ntargeted); } @@ -437,8 +484,7 @@ uv_ptc_seq_show(struct seq_file *file, void *data) * 0: display meaning of the statistics * >0: retry limit */ -static ssize_t -uv_ptc_proc_write(struct file *file, const char __user *user, +static ssize_t uv_ptc_proc_write(struct file *file, const char __user *user, size_t count, loff_t *data) { long newmode; @@ -471,9 +517,9 @@ uv_ptc_proc_write(struct file *file, const char __user *user, printk(KERN_DEBUG "sw_ack: image of UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE\n"); printk(KERN_DEBUG - "sflush_us: microseconds spent in uv_flush_tlb_others()\n"); + "sflush_us: cycles spent in uv_flush_tlb_others()\n"); printk(KERN_DEBUG - "dflush_us: microseconds spent in handling flush requests\n"); + "dflush_us: cycles spent in handling flush requests\n"); printk(KERN_DEBUG "sok: successes on retry\n"); printk(KERN_DEBUG "dnomsg: interrupts with no message\n"); printk(KERN_DEBUG @@ -489,40 +535,33 @@ uv_ptc_proc_write(struct file *file, const char __user *user, } static const struct seq_operations uv_ptc_seq_ops = { - .start = uv_ptc_seq_start, - .next = uv_ptc_seq_next, - .stop = uv_ptc_seq_stop, - .show = uv_ptc_seq_show + .start = uv_ptc_seq_start, + .next = uv_ptc_seq_next, + .stop = uv_ptc_seq_stop, + .show = uv_ptc_seq_show }; -static int -uv_ptc_proc_open(struct inode *inode, struct file *file) +static int uv_ptc_proc_open(struct inode *inode, struct file *file) { return seq_open(file, &uv_ptc_seq_ops); } static const struct file_operations proc_uv_ptc_operations = { - .open = uv_ptc_proc_open, - .read = seq_read, - .write = uv_ptc_proc_write, - .llseek = seq_lseek, - .release = seq_release, + .open = uv_ptc_proc_open, + .read = seq_read, + .write = uv_ptc_proc_write, + .llseek = seq_lseek, + .release = seq_release, }; -static struct proc_dir_entry *proc_uv_ptc; - -static int __init -uv_ptc_init(void) +static int __init uv_ptc_init(void) { - static struct proc_dir_entry *sgi_proc_dir; - - sgi_proc_dir = NULL; + struct proc_dir_entry *proc_uv_ptc; if (!is_uv_system()) return 0; - sgi_proc_dir = proc_mkdir("sgi_uv", NULL); - if (!sgi_proc_dir) + if (!proc_mkdir("sgi_uv", NULL)) return -EINVAL; proc_uv_ptc = create_proc_entry(UV_PTC_BASENAME, 0444, NULL); @@ -535,202 +574,213 @@ uv_ptc_init(void) return 0; } -static void __exit -uv_ptc_exit(void) +/* + * begin the initialization of the per-blade control structures + */ +static struct bau_control * __init uv_table_bases_init(int blade, int node) { - remove_proc_entry(UV_PTC_BASENAME, NULL); + int i; + int *ip; + struct bau_msg_status *msp; + struct bau_control *bau_tablesp; + + bau_tablesp = + kmalloc_node(sizeof(struct bau_control), GFP_KERNEL, node); + if (!bau_tablesp) + BUG(); + bau_tablesp->msg_statuses = + kmalloc_node(sizeof(struct bau_msg_status) * + DESTINATION_PAYLOAD_QUEUE_SIZE, GFP_KERNEL, node); + if (!bau_tablesp->msg_statuses) + BUG(); + for (i = 0, msp = bau_tablesp->msg_statuses; + i < DESTINATION_PAYLOAD_QUEUE_SIZE; i++, msp++) { + bau_cpubits_clear(&msp->seen_by, (int) + uv_blade_nr_possible_cpus(blade)); + } + bau_tablesp->watching = + kmalloc_node(sizeof(int) * DESTINATION_NUM_RESOURCES, + GFP_KERNEL, node); + if (!bau_tablesp->watching) + BUG(); + for (i = 0, ip = bau_tablesp->watching; + i < DESTINATION_PAYLOAD_QUEUE_SIZE; i++, ip++) { + *ip = 0; + } + uv_bau_table_bases[blade] = bau_tablesp; + return bau_tablesp; } -module_init(uv_ptc_init); -module_exit(uv_ptc_exit); +/* + * finish the initialization of the per-blade control structures + */ +static void __init uv_table_bases_finish(int blade, int node, int cur_cpu, + struct bau_control *bau_tablesp, + struct bau_activation_descriptor *adp) +{ + int i; + struct bau_control *bcp; + + for (i = cur_cpu; i < (cur_cpu + uv_blade_nr_possible_cpus(blade)); + i++) { + bcp = (struct bau_control *)&per_cpu(bau_control, i); + bcp->bau_msg_head = bau_tablesp->va_queue_first; + bcp->va_queue_first = bau_tablesp->va_queue_first; + bcp->va_queue_last = bau_tablesp->va_queue_last; + bcp->watching = bau_tablesp->watching; + bcp->msg_statuses = bau_tablesp->msg_statuses; + bcp->descriptor_base = adp; + } +} /* - * Initialization of BAU-related structures + * initialize the sending side's sending buffers */ -int __init -uv_bau_init(void) +static struct bau_activation_descriptor * __init +uv_activation_descriptor_init(int node, int pnode) { int i; - int j; - int blade; - int nblades; - int *ip; - int pnode; - int last_blade; - int cur_cpu = 0; unsigned long pa; - unsigned long n; unsigned long m; + unsigned long n; unsigned long mmr_image; - unsigned long apicid; + struct bau_activation_descriptor *adp; + struct bau_activation_descriptor *ad2; + + adp = (struct bau_activation_descriptor *) + kmalloc_node(16384, GFP_KERNEL, node); + if (!adp) + BUG(); + pa = __pa((unsigned long)adp); + n = pa >> uv_nshift; + m = pa & uv_mmask; + mmr_image = uv_read_global_mmr64(pnode, UVH_LB_BAU_SB_DESCRIPTOR_BASE); + if (mmr_image) + uv_write_global_mmr64(pnode, (unsigned long) + UVH_LB_BAU_SB_DESCRIPTOR_BASE, + (n << UV_DESC_BASE_PNODE_SHIFT | m)); + for (i = 0, ad2 = adp; i < UV_ACTIVATION_DESCRIPTOR_SIZE; i++, ad2++) { + memset(ad2, 0, sizeof(struct bau_activation_descriptor)); + ad2->header.sw_ack_flag = 1; + ad2->header.base_dest_nodeid = + uv_blade_to_pnode(uv_cpu_to_blade_id(0)); + ad2->header.command = UV_NET_ENDPOINT_INTD; + ad2->header.int_both = 1; + /* + * all others need to be set to zero: + * fairness chaining multilevel count replied_to + */ + } + return adp; +} + +/* + * initialize the destination side's receiving buffers + */ +static struct bau_payload_queue_entry * __init uv_payload_queue_init(int node, + int pnode, struct bau_control *bau_tablesp) +{ char *cp; - struct bau_control *bau_tablesp; - struct bau_activation_descriptor *adp, *ad2; struct bau_payload_queue_entry *pqp; - struct bau_msg_status *msp; - struct bau_control *bcp; - if (!is_uv_system()) - return 0; + pqp = (struct bau_payload_queue_entry *) + kmalloc_node((DESTINATION_PAYLOAD_QUEUE_SIZE + 1) * + sizeof(struct bau_payload_queue_entry), + GFP_KERNEL, node); + if (!pqp) + BUG(); + cp = (char *)pqp + 31; + pqp = (struct bau_payload_queue_entry *)(((unsigned long)cp >> 5) << 5); + bau_tablesp->va_queue_first = pqp; + uv_write_global_mmr64(pnode, + UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST, + ((unsigned long)pnode << + UV_PAYLOADQ_PNODE_SHIFT) | + uv_physnodeaddr(pqp)); + uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL, + uv_physnodeaddr(pqp)); + bau_tablesp->va_queue_last = + pqp + (DESTINATION_PAYLOAD_QUEUE_SIZE - 1); + uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST, + (unsigned long) + uv_physnodeaddr(bau_tablesp->va_queue_last)); + memset(pqp, 0, sizeof(struct bau_payload_queue_entry) * + DESTINATION_PAYLOAD_QUEUE_SIZE); + return pqp; +} - uv_bau_retry_limit = 1; +/* + * Initialization of each UV blade's structures + */ +static int __init uv_init_blade(int blade, int node, int cur_cpu) +{ + int pnode; + unsigned long pa; + unsigned long apicid; + struct bau_activation_descriptor *adp; + struct bau_payload_queue_entry *pqp; + struct bau_control *bau_tablesp; - if ((sizeof(struct bau_local_cpumask) * BITSPERBYTE) < - MAX_CPUS_PER_NODE) { - printk(KERN_ERR - "uv_bau_init: bau_local_cpumask.bits too small\n"); - BUG(); + bau_tablesp = uv_table_bases_init(blade, node); + pnode = uv_blade_to_pnode(blade); + adp = uv_activation_descriptor_init(node, pnode); + pqp = uv_payload_queue_init(node, pnode, bau_tablesp); + uv_table_bases_finish(blade, node, cur_cpu, bau_tablesp, adp); + /* + * the below initialization can't be in firmware because the + * messaging IRQ will be determined by the OS + */ + apicid = per_cpu(x86_cpu_to_apicid, cur_cpu); + pa = uv_read_global_mmr64(pnode, UVH_BAU_DATA_CONFIG); + if ((pa & 0xff) != UV_BAU_MESSAGE) { + uv_write_global_mmr64(pnode, UVH_BAU_DATA_CONFIG, + ((apicid << 32) | UV_BAU_MESSAGE)); } + return 0; +} + +/* + * Initialization of BAU-related structures + */ +static int __init uv_bau_init(void) +{ + int blade; + int node; + int nblades; + int last_blade; + int cur_cpu = 0; + + if (!is_uv_system()) + return 0; + uv_bau_retry_limit = 1; uv_nshift = uv_hub_info->n_val; uv_mmask = ((unsigned long)1 << uv_hub_info->n_val) - 1; nblades = 0; last_blade = -1; - for_each_online_node(i) { - blade = uv_node_to_blade_id(i); + for_each_online_node(node) { + blade = uv_node_to_blade_id(node); if (blade == last_blade) continue; last_blade = blade; nblades++; } - uv_bau_table_bases = (struct bau_control **) kmalloc(nblades * sizeof(struct bau_control *), GFP_KERNEL); if (!uv_bau_table_bases) BUG(); - - /* better if we had each_online_blade */ last_blade = -1; - for_each_online_node(i) { - blade = uv_node_to_blade_id(i); + for_each_online_node(node) { + blade = uv_node_to_blade_id(node); if (blade == last_blade) continue; last_blade = blade; - - bau_tablesp = - kmalloc_node(sizeof(struct bau_control), GFP_KERNEL, i); - if (!bau_tablesp) - BUG(); - - bau_tablesp->msg_statuses = - kmalloc_node(sizeof(struct bau_msg_status) * - DESTINATION_PAYLOAD_QUEUE_SIZE, GFP_KERNEL, i); - if (!bau_tablesp->msg_statuses) - BUG(); - for (j = 0, msp = bau_tablesp->msg_statuses; - j < DESTINATION_PAYLOAD_QUEUE_SIZE; j++, msp++) { - bau_cpubits_clear(&msp->seen_by, (int) - uv_blade_nr_possible_cpus(blade)); - } - - bau_tablesp->watching = - kmalloc_node(sizeof(int) * DESTINATION_NUM_RESOURCES, - GFP_KERNEL, i); - if (!bau_tablesp->watching) - BUG(); - for (j = 0, ip = bau_tablesp->watching; - j < DESTINATION_PAYLOAD_QUEUE_SIZE; j++, ip++) { - *ip = 0; - } - - uv_bau_table_bases[i] = bau_tablesp; - - pnode = uv_blade_to_pnode(blade); - - if (sizeof(struct bau_activation_descriptor) != 64) - BUG(); - - adp = (struct bau_activation_descriptor *) - kmalloc_node(16384, GFP_KERNEL, i); - if (!adp) - BUG(); - if ((unsigned long)adp & 0xfff) - BUG(); - pa = __pa((unsigned long)adp); - n = pa >> uv_nshift; - m = pa & uv_mmask; - - mmr_image = uv_read_global_mmr64(pnode, - UVH_LB_BAU_SB_DESCRIPTOR_BASE); - if (mmr_image) - uv_write_global_mmr64(pnode, (unsigned long) - UVH_LB_BAU_SB_DESCRIPTOR_BASE, - (n << UV_DESC_BASE_PNODE_SHIFT | - m)); - for (j = 0, ad2 = adp; j < UV_ACTIVATION_DESCRIPTOR_SIZE; - j++, ad2++) { - memset(ad2, 0, - sizeof(struct bau_activation_descriptor)); - ad2->header.sw_ack_flag = 1; - ad2->header.base_dest_nodeid = - uv_blade_to_pnode(uv_cpu_to_blade_id(0)); - ad2->header.command = UV_NET_ENDPOINT_INTD; - ad2->header.int_both = 1; - /* all others need to be set to zero: - fairness chaining multilevel count replied_to */ - } - - pqp = (struct bau_payload_queue_entry *) - kmalloc_node((DESTINATION_PAYLOAD_QUEUE_SIZE + 1) * - sizeof(struct bau_payload_queue_entry), - GFP_KERNEL, i); - if (!pqp) - BUG(); - if (sizeof(struct bau_payload_queue_entry) != 32) - BUG(); - if ((unsigned long)(&((struct bau_payload_queue_entry *)0)-> - sw_ack_vector) != 15) - BUG(); - - cp = (char *)pqp + 31; - pqp = (struct bau_payload_queue_entry *) - (((unsigned long)cp >> 5) << 5); - bau_tablesp->va_queue_first = pqp; - uv_write_global_mmr64(pnode, - UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST, - ((unsigned long)pnode << - UV_PAYLOADQ_PNODE_SHIFT) | - uv_physnodeaddr(pqp)); - uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL, - uv_physnodeaddr(pqp)); - bau_tablesp->va_queue_last = - pqp + (DESTINATION_PAYLOAD_QUEUE_SIZE - 1); - uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST, - (unsigned long) - uv_physnodeaddr(bau_tablesp-> - va_queue_last)); - memset(pqp, 0, sizeof(struct bau_payload_queue_entry) * - DESTINATION_PAYLOAD_QUEUE_SIZE); - - /* this initialization can't be in firmware because the - messaging IRQ will be determined by the OS */ - apicid = per_cpu(x86_cpu_to_apicid, cur_cpu); - pa = uv_read_global_mmr64(pnode, UVH_BAU_DATA_CONFIG); - if ((pa & 0xff) != UV_BAU_MESSAGE) { - uv_write_global_mmr64(pnode, UVH_BAU_DATA_CONFIG, - ((apicid << 32) | - UV_BAU_MESSAGE)); - } - - for (j = cur_cpu; j < (cur_cpu + uv_blade_nr_possible_cpus(i)); - j++) { - bcp = (struct bau_control *)&per_cpu(bau_control, j); - bcp->bau_msg_head = bau_tablesp->va_queue_first; - bcp->va_queue_first = bau_tablesp->va_queue_first; - - bcp->va_queue_last = bau_tablesp->va_queue_last; - bcp->watching = bau_tablesp->watching; - bcp->msg_statuses = bau_tablesp->msg_statuses; - bcp->descriptor_base = adp; - } - cur_cpu += uv_blade_nr_possible_cpus(i); + uv_init_blade(blade, node, cur_cpu); + cur_cpu += uv_blade_nr_possible_cpus(blade); } - set_intr_gate(UV_BAU_MESSAGE, uv_bau_message_intr1); - uv_enable_timeouts(); - return 0; } - __initcall(uv_bau_init); +__initcall(uv_ptc_init); -- cgit v1.2.1 From dc163a41ffba22a6ef70b51e7ddf68aa13b4b414 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 18 Jun 2008 14:15:43 +0200 Subject: SGI UV: TLB shootdown using broadcast assist unit TLB shootdown for SGI UV. v5: 6/12 corrections/improvements per Ingo's second review Signed-off-by: Cliff Wickman Signed-off-by: Ingo Molnar --- arch/x86/kernel/tlb_uv.c | 183 ++++++++++++++++++++++------------------------- 1 file changed, 85 insertions(+), 98 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/tlb_uv.c b/arch/x86/kernel/tlb_uv.c index f7bc6a6fbe49..d8705e97e8d0 100644 --- a/arch/x86/kernel/tlb_uv.c +++ b/arch/x86/kernel/tlb_uv.c @@ -25,15 +25,8 @@ static int uv_bau_retry_limit __read_mostly; static int uv_nshift __read_mostly; /* position of pnode (which is nasid>>1) */ static unsigned long uv_mmask __read_mostly; -char *status_table[] = { - "IDLE", - "ACTIVE", - "DESTINATION TIMEOUT", - "SOURCE TIMEOUT" -}; - -DEFINE_PER_CPU(struct ptc_stats, ptcstats); -DEFINE_PER_CPU(struct bau_control, bau_control); +static DEFINE_PER_CPU(struct ptc_stats, ptcstats); +static DEFINE_PER_CPU(struct bau_control, bau_control); /* * Free a software acknowledge hardware resource by clearing its Pending @@ -55,7 +48,6 @@ static void uv_reply_to_message(int resource, if (msp) msp->seen_by.bits = 0; uv_write_local_mmr(UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS, dw); - return; } /* @@ -73,7 +65,7 @@ static void uv_bau_process_message(struct bau_payload_queue_entry *msg, cpu = uv_blade_processor_id(); msg->number_of_cpus = uv_blade_nr_online_cpus(uv_node_to_blade_id(numa_node_id())); - this_cpu_mask = (unsigned long)1 << cpu; + this_cpu_mask = 1UL << cpu; if (msp->seen_by.bits & this_cpu_mask) return; atomic_or_long(&msp->seen_by.bits, this_cpu_mask); @@ -94,53 +86,60 @@ static void uv_bau_process_message(struct bau_payload_queue_entry *msg, atomic_inc_short(&msg->acknowledge_count); if (msg->number_of_cpus == msg->acknowledge_count) uv_reply_to_message(sw_ack_slot, msg, msp); - return; } /* - * Examine the payload queue on all the distribution nodes to see + * Examine the payload queue on one distribution node to see * which messages have not been seen, and which cpu(s) have not seen them. * * Returns the number of cpu's that have not responded. */ -static int uv_examine_destinations(struct bau_target_nodemask *distribution) +static int uv_examine_destination(struct bau_control *bau_tablesp, int sender) { - int sender; int i; int j; - int k; int count = 0; - struct bau_control *bau_tablesp; struct bau_payload_queue_entry *msg; struct bau_msg_status *msp; + for (msg = bau_tablesp->va_queue_first, i = 0; i < DEST_Q_SIZE; + msg++, i++) { + if ((msg->sending_cpu == sender) && (!msg->replied_to)) { + msp = bau_tablesp->msg_statuses + i; + printk(KERN_DEBUG + "blade %d: address:%#lx %d of %d, not cpu(s): ", + i, msg->address, msg->acknowledge_count, + msg->number_of_cpus); + for (j = 0; j < msg->number_of_cpus; j++) { + if (!((long)1 << j & msp-> seen_by.bits)) { + count++; + printk("%d ", j); + } + } + printk("\n"); + } + } + return count; +} + +/* + * Examine the payload queue on all the distribution nodes to see + * which messages have not been seen, and which cpu(s) have not seen them. + * + * Returns the number of cpu's that have not responded. + */ +static int uv_examine_destinations(struct bau_target_nodemask *distribution) +{ + int sender; + int i; + int count = 0; + sender = smp_processor_id(); for (i = 0; i < (sizeof(struct bau_target_nodemask) * BITSPERBYTE); i++) { if (!bau_node_isset(i, distribution)) continue; - bau_tablesp = uv_bau_table_bases[i]; - for (msg = bau_tablesp->va_queue_first, j = 0; - j < DESTINATION_PAYLOAD_QUEUE_SIZE; msg++, j++) { - if ((msg->sending_cpu == sender) && - (!msg->replied_to)) { - msp = bau_tablesp->msg_statuses + j; - printk(KERN_DEBUG - "blade %d: address:%#lx %d of %d, not cpu(s): ", - i, msg->address, - msg->acknowledge_count, - msg->number_of_cpus); - for (k = 0; k < msg->number_of_cpus; - k++) { - if (!((long)1 << k & msp-> - seen_by.bits)) { - count++; - printk("%d ", k); - } - } - printk("\n"); - } - } + count += uv_examine_destination(uv_bau_table_bases[i], sender); } return count; } @@ -150,7 +149,7 @@ static int uv_examine_destinations(struct bau_target_nodemask *distribution) * * return COMPLETE, RETRY or GIVEUP */ -static int uv_wait_completion(struct bau_activation_descriptor *bau_desc, +static int uv_wait_completion(struct bau_desc *bau_desc, unsigned long mmr_offset, int right_shift) { int exams = 0; @@ -213,8 +212,8 @@ static int uv_wait_completion(struct bau_activation_descriptor *bau_desc, * Returns 0 if some remote flushing remains to be done. The mask is left * unchanged. */ -int uv_flush_send_and_wait(int cpu, int this_blade, - struct bau_activation_descriptor *bau_desc, cpumask_t *cpumaskp) +int uv_flush_send_and_wait(int cpu, int this_blade, struct bau_desc *bau_desc, + cpumask_t *cpumaskp) { int completion_status = 0; int right_shift; @@ -237,8 +236,8 @@ int uv_flush_send_and_wait(int cpu, int this_blade, time1 = get_cycles(); do { tries++; - index = ((unsigned long) - 1 << UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_SHFT) | cpu; + index = (1UL << UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_SHFT) | + cpu; uv_write_local_mmr(UVH_LB_BAU_SB_ACTIVATION_CONTROL, index); completion_status = uv_wait_completion(bau_desc, mmr_offset, right_shift); @@ -303,7 +302,7 @@ int uv_flush_tlb_others(cpumask_t *cpumaskp, struct mm_struct *mm, int cpu; int this_blade; int locals = 0; - struct bau_activation_descriptor *bau_desc; + struct bau_desc *bau_desc; cpu = uv_blade_processor_id(); this_blade = uv_numa_blade_id(); @@ -315,8 +314,7 @@ int uv_flush_tlb_others(cpumask_t *cpumaskp, struct mm_struct *mm, i = 0; for_each_cpu_mask(bit, *cpumaskp) { blade = uv_cpu_to_blade_id(bit); - if (blade > (UV_DISTRIBUTION_SIZE - 1)) - BUG(); + BUG_ON(blade > (UV_DISTRIBUTION_SIZE - 1)); if (blade == this_blade) { locals++; continue; @@ -360,6 +358,8 @@ void uv_bau_message_interrupt(struct pt_regs *regs) { struct bau_payload_queue_entry *pqp; struct bau_payload_queue_entry *msg; + struct bau_payload_queue_entry *va_queue_first; + struct bau_payload_queue_entry *va_queue_last; struct pt_regs *old_regs = set_irq_regs(regs); cycles_t time1, time2; int msg_slot; @@ -376,7 +376,8 @@ void uv_bau_message_interrupt(struct pt_regs *regs) local_pnode = uv_blade_to_pnode(uv_numa_blade_id()); - pqp = __get_cpu_var(bau_control).va_queue_first; + pqp = va_queue_first = __get_cpu_var(bau_control).va_queue_first; + va_queue_last = __get_cpu_var(bau_control).va_queue_last; msg = __get_cpu_var(bau_control).bau_msg_head; while (msg->sw_ack_vector) { count++; @@ -387,8 +388,8 @@ void uv_bau_message_interrupt(struct pt_regs *regs) uv_bau_process_message(msg, msg_slot, sw_ack_slot); msg++; - if (msg > __get_cpu_var(bau_control).va_queue_last) - msg = __get_cpu_var(bau_control).va_queue_first; + if (msg > va_queue_last) + msg = va_queue_first; __get_cpu_var(bau_control).bau_msg_head = msg; } if (!count) @@ -401,7 +402,6 @@ void uv_bau_message_interrupt(struct pt_regs *regs) irq_exit(); set_irq_regs(old_regs); - return; } static void uv_enable_timeouts(void) @@ -423,7 +423,6 @@ static void uv_enable_timeouts(void) pnode = uv_blade_to_pnode(blade); cur_cpu += uv_blade_nr_possible_cpus(i); } - return; } static void *uv_ptc_seq_start(struct seq_file *file, loff_t *offset) @@ -535,10 +534,10 @@ static ssize_t uv_ptc_proc_write(struct file *file, const char __user *user, } static const struct seq_operations uv_ptc_seq_ops = { - .start = uv_ptc_seq_start, - .next = uv_ptc_seq_next, - .stop = uv_ptc_seq_stop, - .show = uv_ptc_seq_show + .start = uv_ptc_seq_start, + .next = uv_ptc_seq_next, + .stop = uv_ptc_seq_stop, + .show = uv_ptc_seq_show }; static int uv_ptc_proc_open(struct inode *inode, struct file *file) @@ -568,6 +567,7 @@ static int __init uv_ptc_init(void) if (!proc_uv_ptc) { printk(KERN_ERR "unable to create %s proc entry\n", UV_PTC_BASENAME); + remove_proc_entry("sgi_uv", NULL); return -EINVAL; } proc_uv_ptc->proc_fops = &proc_uv_ptc_operations; @@ -582,33 +582,26 @@ static struct bau_control * __init uv_table_bases_init(int blade, int node) int i; int *ip; struct bau_msg_status *msp; - struct bau_control *bau_tablesp; + struct bau_control *bau_tabp; - bau_tablesp = + bau_tabp = kmalloc_node(sizeof(struct bau_control), GFP_KERNEL, node); - if (!bau_tablesp) - BUG(); - bau_tablesp->msg_statuses = + BUG_ON(!bau_tabp); + bau_tabp->msg_statuses = kmalloc_node(sizeof(struct bau_msg_status) * - DESTINATION_PAYLOAD_QUEUE_SIZE, GFP_KERNEL, node); - if (!bau_tablesp->msg_statuses) - BUG(); - for (i = 0, msp = bau_tablesp->msg_statuses; - i < DESTINATION_PAYLOAD_QUEUE_SIZE; i++, msp++) { + DEST_Q_SIZE, GFP_KERNEL, node); + BUG_ON(!bau_tabp->msg_statuses); + for (i = 0, msp = bau_tabp->msg_statuses; i < DEST_Q_SIZE; i++, msp++) bau_cpubits_clear(&msp->seen_by, (int) uv_blade_nr_possible_cpus(blade)); - } - bau_tablesp->watching = - kmalloc_node(sizeof(int) * DESTINATION_NUM_RESOURCES, - GFP_KERNEL, node); - if (!bau_tablesp->watching) - BUG(); - for (i = 0, ip = bau_tablesp->watching; - i < DESTINATION_PAYLOAD_QUEUE_SIZE; i++, ip++) { + bau_tabp->watching = + kmalloc_node(sizeof(int) * DEST_NUM_RESOURCES, GFP_KERNEL, node); + BUG_ON(!bau_tabp->watching); + for (i = 0, ip = bau_tabp->watching; i < DEST_Q_SIZE; i++, ip++) { *ip = 0; } - uv_bau_table_bases[blade] = bau_tablesp; - return bau_tablesp; + uv_bau_table_bases[blade] = bau_tabp; + return bau_tabsp; } /* @@ -616,7 +609,7 @@ static struct bau_control * __init uv_table_bases_init(int blade, int node) */ static void __init uv_table_bases_finish(int blade, int node, int cur_cpu, struct bau_control *bau_tablesp, - struct bau_activation_descriptor *adp) + struct bau_desc *adp) { int i; struct bau_control *bcp; @@ -636,7 +629,7 @@ static void __init uv_table_bases_finish(int blade, int node, int cur_cpu, /* * initialize the sending side's sending buffers */ -static struct bau_activation_descriptor * __init +static struct bau_desc * __init uv_activation_descriptor_init(int node, int pnode) { int i; @@ -644,13 +637,12 @@ uv_activation_descriptor_init(int node, int pnode) unsigned long m; unsigned long n; unsigned long mmr_image; - struct bau_activation_descriptor *adp; - struct bau_activation_descriptor *ad2; + struct bau_desc *adp; + struct bau_desc *ad2; - adp = (struct bau_activation_descriptor *) + adp = (struct bau_desc *) kmalloc_node(16384, GFP_KERNEL, node); - if (!adp) - BUG(); + BUG_ON(!adp); pa = __pa((unsigned long)adp); n = pa >> uv_nshift; m = pa & uv_mmask; @@ -660,7 +652,7 @@ uv_activation_descriptor_init(int node, int pnode) UVH_LB_BAU_SB_DESCRIPTOR_BASE, (n << UV_DESC_BASE_PNODE_SHIFT | m)); for (i = 0, ad2 = adp; i < UV_ACTIVATION_DESCRIPTOR_SIZE; i++, ad2++) { - memset(ad2, 0, sizeof(struct bau_activation_descriptor)); + memset(ad2, 0, sizeof(struct bau_desc)); ad2->header.sw_ack_flag = 1; ad2->header.base_dest_nodeid = uv_blade_to_pnode(uv_cpu_to_blade_id(0)); @@ -683,12 +675,10 @@ static struct bau_payload_queue_entry * __init uv_payload_queue_init(int node, char *cp; struct bau_payload_queue_entry *pqp; - pqp = (struct bau_payload_queue_entry *) - kmalloc_node((DESTINATION_PAYLOAD_QUEUE_SIZE + 1) * - sizeof(struct bau_payload_queue_entry), - GFP_KERNEL, node); - if (!pqp) - BUG(); + pqp = (struct bau_payload_queue_entry *) kmalloc_node( + (DEST_Q_SIZE + 1) * sizeof(struct bau_payload_queue_entry), + GFP_KERNEL, node); + BUG_ON(!pqp); cp = (char *)pqp + 31; pqp = (struct bau_payload_queue_entry *)(((unsigned long)cp >> 5) << 5); bau_tablesp->va_queue_first = pqp; @@ -699,13 +689,11 @@ static struct bau_payload_queue_entry * __init uv_payload_queue_init(int node, uv_physnodeaddr(pqp)); uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL, uv_physnodeaddr(pqp)); - bau_tablesp->va_queue_last = - pqp + (DESTINATION_PAYLOAD_QUEUE_SIZE - 1); + bau_tablesp->va_queue_last = pqp + (DEST_Q_SIZE - 1); uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST, (unsigned long) uv_physnodeaddr(bau_tablesp->va_queue_last)); - memset(pqp, 0, sizeof(struct bau_payload_queue_entry) * - DESTINATION_PAYLOAD_QUEUE_SIZE); + memset(pqp, 0, sizeof(struct bau_payload_queue_entry) * DEST_Q_SIZE); return pqp; } @@ -717,7 +705,7 @@ static int __init uv_init_blade(int blade, int node, int cur_cpu) int pnode; unsigned long pa; unsigned long apicid; - struct bau_activation_descriptor *adp; + struct bau_desc *adp; struct bau_payload_queue_entry *pqp; struct bau_control *bau_tablesp; @@ -755,7 +743,7 @@ static int __init uv_bau_init(void) uv_bau_retry_limit = 1; uv_nshift = uv_hub_info->n_val; - uv_mmask = ((unsigned long)1 << uv_hub_info->n_val) - 1; + uv_mmask = (1UL << uv_hub_info->n_val) - 1; nblades = 0; last_blade = -1; for_each_online_node(node) { @@ -767,8 +755,7 @@ static int __init uv_bau_init(void) } uv_bau_table_bases = (struct bau_control **) kmalloc(nblades * sizeof(struct bau_control *), GFP_KERNEL); - if (!uv_bau_table_bases) - BUG(); + BUG_ON(!uv_bau_table_bases); last_blade = -1; for_each_online_node(node) { blade = uv_node_to_blade_id(node); -- cgit v1.2.1 From b4c286e6af24a116228c8c9f58b9a9eb5b7c000a Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 18 Jun 2008 14:28:19 +0200 Subject: SGI UV: clean up arch/x86/kernel/tlb_uv.c Signed-off-by: Ingo Molnar --- arch/x86/kernel/tlb_uv.c | 107 +++++++++++++++++++++++++++-------------------- 1 file changed, 62 insertions(+), 45 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/tlb_uv.c b/arch/x86/kernel/tlb_uv.c index d8705e97e8d0..7bdbf67a2d79 100644 --- a/arch/x86/kernel/tlb_uv.c +++ b/arch/x86/kernel/tlb_uv.c @@ -11,19 +11,22 @@ #include #include -#include -#include -#include #include +#include #include +#include +#include #include #include -static struct bau_control **uv_bau_table_bases __read_mostly; -static int uv_bau_retry_limit __read_mostly; -static int uv_nshift __read_mostly; /* position of pnode (which is nasid>>1) */ -static unsigned long uv_mmask __read_mostly; +static struct bau_control **uv_bau_table_bases __read_mostly; +static int uv_bau_retry_limit __read_mostly; + +/* position of pnode (which is nasid>>1): */ +static int uv_nshift __read_mostly; + +static unsigned long uv_mmask __read_mostly; static DEFINE_PER_CPU(struct ptc_stats, ptcstats); static DEFINE_PER_CPU(struct bau_control, bau_control); @@ -37,8 +40,8 @@ static DEFINE_PER_CPU(struct bau_control, bau_control); * be sent (the hardware will only do one reply per message). */ static void uv_reply_to_message(int resource, - struct bau_payload_queue_entry *msg, - struct bau_msg_status *msp) + struct bau_payload_queue_entry *msg, + struct bau_msg_status *msp) { unsigned long dw; @@ -55,11 +58,11 @@ static void uv_reply_to_message(int resource, * Other cpu's may come here at the same time for this message. */ static void uv_bau_process_message(struct bau_payload_queue_entry *msg, - int msg_slot, int sw_ack_slot) + int msg_slot, int sw_ack_slot) { - int cpu; unsigned long this_cpu_mask; struct bau_msg_status *msp; + int cpu; msp = __get_cpu_var(bau_control).msg_statuses + msg_slot; cpu = uv_blade_processor_id(); @@ -96,11 +99,11 @@ static void uv_bau_process_message(struct bau_payload_queue_entry *msg, */ static int uv_examine_destination(struct bau_control *bau_tablesp, int sender) { - int i; - int j; - int count = 0; struct bau_payload_queue_entry *msg; struct bau_msg_status *msp; + int count = 0; + int i; + int j; for (msg = bau_tablesp->va_queue_first, i = 0; i < DEST_Q_SIZE; msg++, i++) { @@ -111,7 +114,7 @@ static int uv_examine_destination(struct bau_control *bau_tablesp, int sender) i, msg->address, msg->acknowledge_count, msg->number_of_cpus); for (j = 0; j < msg->number_of_cpus; j++) { - if (!((long)1 << j & msp-> seen_by.bits)) { + if (!((1L << j) & msp->seen_by.bits)) { count++; printk("%d ", j); } @@ -135,8 +138,7 @@ static int uv_examine_destinations(struct bau_target_nodemask *distribution) int count = 0; sender = smp_processor_id(); - for (i = 0; i < (sizeof(struct bau_target_nodemask) * BITSPERBYTE); - i++) { + for (i = 0; i < sizeof(struct bau_target_nodemask) * BITSPERBYTE; i++) { if (!bau_node_isset(i, distribution)) continue; count += uv_examine_destination(uv_bau_table_bases[i], sender); @@ -217,11 +219,11 @@ int uv_flush_send_and_wait(int cpu, int this_blade, struct bau_desc *bau_desc, { int completion_status = 0; int right_shift; - int bit; - int blade; int tries = 0; - unsigned long index; + int blade; + int bit; unsigned long mmr_offset; + unsigned long index; cycles_t time1; cycles_t time2; @@ -294,7 +296,7 @@ int uv_flush_send_and_wait(int cpu, int this_blade, struct bau_desc *bau_desc, * Returns 0 if some remote flushing remains to be done. */ int uv_flush_tlb_others(cpumask_t *cpumaskp, struct mm_struct *mm, - unsigned long va) + unsigned long va) { int i; int bit; @@ -356,12 +358,12 @@ int uv_flush_tlb_others(cpumask_t *cpumaskp, struct mm_struct *mm, */ void uv_bau_message_interrupt(struct pt_regs *regs) { - struct bau_payload_queue_entry *pqp; - struct bau_payload_queue_entry *msg; struct bau_payload_queue_entry *va_queue_first; struct bau_payload_queue_entry *va_queue_last; + struct bau_payload_queue_entry *msg; struct pt_regs *old_regs = set_irq_regs(regs); - cycles_t time1, time2; + cycles_t time1; + cycles_t time2; int msg_slot; int sw_ack_slot; int fw; @@ -376,13 +378,14 @@ void uv_bau_message_interrupt(struct pt_regs *regs) local_pnode = uv_blade_to_pnode(uv_numa_blade_id()); - pqp = va_queue_first = __get_cpu_var(bau_control).va_queue_first; + va_queue_first = __get_cpu_var(bau_control).va_queue_first; va_queue_last = __get_cpu_var(bau_control).va_queue_last; + msg = __get_cpu_var(bau_control).bau_msg_head; while (msg->sw_ack_vector) { count++; fw = msg->sw_ack_vector; - msg_slot = msg - pqp; + msg_slot = msg - va_queue_first; sw_ack_slot = ffs(fw) - 1; uv_bau_process_message(msg, msg_slot, sw_ack_slot); @@ -484,7 +487,7 @@ static int uv_ptc_seq_show(struct seq_file *file, void *data) * >0: retry limit */ static ssize_t uv_ptc_proc_write(struct file *file, const char __user *user, - size_t count, loff_t *data) + size_t count, loff_t *data) { long newmode; char optstr[64]; @@ -587,42 +590,48 @@ static struct bau_control * __init uv_table_bases_init(int blade, int node) bau_tabp = kmalloc_node(sizeof(struct bau_control), GFP_KERNEL, node); BUG_ON(!bau_tabp); + bau_tabp->msg_statuses = kmalloc_node(sizeof(struct bau_msg_status) * DEST_Q_SIZE, GFP_KERNEL, node); BUG_ON(!bau_tabp->msg_statuses); + for (i = 0, msp = bau_tabp->msg_statuses; i < DEST_Q_SIZE; i++, msp++) bau_cpubits_clear(&msp->seen_by, (int) uv_blade_nr_possible_cpus(blade)); + bau_tabp->watching = kmalloc_node(sizeof(int) * DEST_NUM_RESOURCES, GFP_KERNEL, node); BUG_ON(!bau_tabp->watching); - for (i = 0, ip = bau_tabp->watching; i < DEST_Q_SIZE; i++, ip++) { + + for (i = 0, ip = bau_tabp->watching; i < DEST_Q_SIZE; i++, ip++) *ip = 0; - } + uv_bau_table_bases[blade] = bau_tabp; + return bau_tabsp; } /* * finish the initialization of the per-blade control structures */ -static void __init uv_table_bases_finish(int blade, int node, int cur_cpu, - struct bau_control *bau_tablesp, - struct bau_desc *adp) +static void __init +uv_table_bases_finish(int blade, int node, int cur_cpu, + struct bau_control *bau_tablesp, + struct bau_desc *adp) { - int i; struct bau_control *bcp; + int i; - for (i = cur_cpu; i < (cur_cpu + uv_blade_nr_possible_cpus(blade)); - i++) { + for (i = cur_cpu; i < cur_cpu + uv_blade_nr_possible_cpus(blade); i++) { bcp = (struct bau_control *)&per_cpu(bau_control, i); - bcp->bau_msg_head = bau_tablesp->va_queue_first; - bcp->va_queue_first = bau_tablesp->va_queue_first; - bcp->va_queue_last = bau_tablesp->va_queue_last; - bcp->watching = bau_tablesp->watching; - bcp->msg_statuses = bau_tablesp->msg_statuses; - bcp->descriptor_base = adp; + + bcp->bau_msg_head = bau_tablesp->va_queue_first; + bcp->va_queue_first = bau_tablesp->va_queue_first; + bcp->va_queue_last = bau_tablesp->va_queue_last; + bcp->watching = bau_tablesp->watching; + bcp->msg_statuses = bau_tablesp->msg_statuses; + bcp->descriptor_base = adp; } } @@ -643,14 +652,18 @@ uv_activation_descriptor_init(int node, int pnode) adp = (struct bau_desc *) kmalloc_node(16384, GFP_KERNEL, node); BUG_ON(!adp); + pa = __pa((unsigned long)adp); n = pa >> uv_nshift; m = pa & uv_mmask; + mmr_image = uv_read_global_mmr64(pnode, UVH_LB_BAU_SB_DESCRIPTOR_BASE); - if (mmr_image) + if (mmr_image) { uv_write_global_mmr64(pnode, (unsigned long) UVH_LB_BAU_SB_DESCRIPTOR_BASE, (n << UV_DESC_BASE_PNODE_SHIFT | m)); + } + for (i = 0, ad2 = adp; i < UV_ACTIVATION_DESCRIPTOR_SIZE; i++, ad2++) { memset(ad2, 0, sizeof(struct bau_desc)); ad2->header.sw_ack_flag = 1; @@ -669,16 +682,17 @@ uv_activation_descriptor_init(int node, int pnode) /* * initialize the destination side's receiving buffers */ -static struct bau_payload_queue_entry * __init uv_payload_queue_init(int node, - int pnode, struct bau_control *bau_tablesp) +static struct bau_payload_queue_entry * __init +uv_payload_queue_init(int node, int pnode, struct bau_control *bau_tablesp) { - char *cp; struct bau_payload_queue_entry *pqp; + char *cp; pqp = (struct bau_payload_queue_entry *) kmalloc_node( (DEST_Q_SIZE + 1) * sizeof(struct bau_payload_queue_entry), GFP_KERNEL, node); BUG_ON(!pqp); + cp = (char *)pqp + 31; pqp = (struct bau_payload_queue_entry *)(((unsigned long)cp >> 5) << 5); bau_tablesp->va_queue_first = pqp; @@ -694,6 +708,7 @@ static struct bau_payload_queue_entry * __init uv_payload_queue_init(int node, (unsigned long) uv_physnodeaddr(bau_tablesp->va_queue_last)); memset(pqp, 0, sizeof(struct bau_payload_queue_entry) * DEST_Q_SIZE); + return pqp; } @@ -756,6 +771,7 @@ static int __init uv_bau_init(void) uv_bau_table_bases = (struct bau_control **) kmalloc(nblades * sizeof(struct bau_control *), GFP_KERNEL); BUG_ON(!uv_bau_table_bases); + last_blade = -1; for_each_online_node(node) { blade = uv_node_to_blade_id(node); @@ -767,6 +783,7 @@ static int __init uv_bau_init(void) } set_intr_gate(UV_BAU_MESSAGE, uv_bau_message_intr1); uv_enable_timeouts(); + return 0; } __initcall(uv_bau_init); -- cgit v1.2.1 From d400524affeb84bdfc2b00cd561fbfb8c09dadd7 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 18 Jun 2008 14:51:57 +0200 Subject: SGI UV: TLB shootdown using broadcast assist unit, fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix: arch/x86/kernel/tlb_uv.c: In function ‘uv_table_bases_init': arch/x86/kernel/tlb_uv.c:612: error: ‘bau_tabsp' undeclared (first use in this function) arch/x86/kernel/tlb_uv.c:612: error: (Each undeclared identifier is reported only once arch/x86/kernel/tlb_uv.c:612: error: for each function it appears in.) Signed-off-by: Ingo Molnar --- arch/x86/kernel/tlb_uv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/tlb_uv.c b/arch/x86/kernel/tlb_uv.c index 7bdbf67a2d79..b362913f0199 100644 --- a/arch/x86/kernel/tlb_uv.c +++ b/arch/x86/kernel/tlb_uv.c @@ -609,7 +609,7 @@ static struct bau_control * __init uv_table_bases_init(int blade, int node) uv_bau_table_bases[blade] = bau_tabp; - return bau_tabsp; + return bau_tabp; } /* -- cgit v1.2.1 From b6df1b8bc1250191cfee15627697111c1cbda53f Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Thu, 19 Jun 2008 21:51:05 -0500 Subject: x86: fix stack overflow for large values of MAX_APICS physid_mask_of_physid() causes a huge stack (12k) to be created if the number of APICS is large. Replace physid_mask_of_physid() with a new function that does not create large stacks. This is a problem only on large x86_64 systems. this paves the way to increase MAX_APICS. Signed-off-by: Jack Steiner Cc: linux-mm@kvack.org Cc: mingo@elte.hu Cc: tglx@linutronix.de Signed-off-by: Ingo Molnar --- arch/x86/kernel/apic_32.c | 2 +- arch/x86/kernel/apic_64.c | 2 +- arch/x86/kernel/smpboot.c | 5 ++--- 3 files changed, 4 insertions(+), 5 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/apic_32.c b/arch/x86/kernel/apic_32.c index d5767cb19d56..3f13a1aa07cd 100644 --- a/arch/x86/kernel/apic_32.c +++ b/arch/x86/kernel/apic_32.c @@ -1269,7 +1269,7 @@ int __init APIC_init_uniprocessor(void) #ifdef CONFIG_CRASH_DUMP boot_cpu_physical_apicid = GET_APIC_ID(read_apic_id()); #endif - phys_cpu_present_map = physid_mask_of_physid(boot_cpu_physical_apicid); + physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map); setup_local_APIC(); diff --git a/arch/x86/kernel/apic_64.c b/arch/x86/kernel/apic_64.c index 0633cfd0dc29..5e22f8d2d507 100644 --- a/arch/x86/kernel/apic_64.c +++ b/arch/x86/kernel/apic_64.c @@ -942,7 +942,7 @@ int __init APIC_init_uniprocessor(void) verify_local_APIC(); - phys_cpu_present_map = physid_mask_of_physid(boot_cpu_physical_apicid); + physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map); apic_write(APIC_ID, SET_APIC_ID(boot_cpu_physical_apicid)); setup_local_APIC(); diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index 3e1cecedde42..664a5db36d40 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -1091,10 +1091,9 @@ static __init void disable_smp(void) smpboot_clear_io_apic_irqs(); #endif if (smp_found_config) - phys_cpu_present_map = - physid_mask_of_physid(boot_cpu_physical_apicid); + physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map); else - phys_cpu_present_map = physid_mask_of_physid(0); + physid_set_mask_of_physid(0, &phys_cpu_present_map); map_cpu_to_logical_apicid(); cpu_set(0, per_cpu(cpu_sibling_map, 0)); cpu_set(0, per_cpu(cpu_core_map, 0)); -- cgit v1.2.1 From cef53278682eb2604cbd99de64cdb59a8b35235a Mon Sep 17 00:00:00 2001 From: Cliff Wickman Date: Thu, 19 Jun 2008 11:16:24 -0500 Subject: x86, SGI UV: TLB shootdown using broadcast assist unit, v6 v6: 6/19 close the security hole in uv_ptc_proc_write()) > Found a potential security hole while doing that: > static ssize_t uv_ptc_proc_write(struct file *file, const char __user *user, > size_t count, loff_t *data) > if (copy_from_user(optstr, user, count)) > return -EFAULT; > > is count guaranteed to never be larger than 64? is fixed below. It adds tlb_uv.o to the Makefile. Signed-off-by: Cliff Wickman Cc: mingo@elte.hu Signed-off-by: Ingo Molnar --- arch/x86/kernel/tlb_uv.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/tlb_uv.c b/arch/x86/kernel/tlb_uv.c index b362913f0199..c503b7f04481 100644 --- a/arch/x86/kernel/tlb_uv.c +++ b/arch/x86/kernel/tlb_uv.c @@ -492,6 +492,8 @@ static ssize_t uv_ptc_proc_write(struct file *file, const char __user *user, long newmode; char optstr[64]; + if (count > 64) + return -EINVAL; if (copy_from_user(optstr, user, count)) return -EFAULT; optstr[count - 1] = '\0'; -- cgit v1.2.1 From e7eb8726d0e144f0925972c4ecee945e91a42753 Mon Sep 17 00:00:00 2001 From: Cliff Wickman Date: Mon, 23 Jun 2008 08:32:25 -0500 Subject: x86, SGI UV: uv_ptc_proc_write fix Someone could write 0 bytes to /proc/sgi_uv/ptc_statistics, causing optstr[count - 1] = '\0'; to write to who-knows-where. (Andi Kleen noticed this need from a patch I sent for similar code in the ia64 world (sn2_ptc_proc_write()).) (count less than zero is not possible here, as count is unsigned) Signed-off-by: Cliff Wickman Signed-off-by: Ingo Molnar --- arch/x86/kernel/tlb_uv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/tlb_uv.c b/arch/x86/kernel/tlb_uv.c index c503b7f04481..d0fbb7712ab0 100644 --- a/arch/x86/kernel/tlb_uv.c +++ b/arch/x86/kernel/tlb_uv.c @@ -492,7 +492,7 @@ static ssize_t uv_ptc_proc_write(struct file *file, const char __user *user, long newmode; char optstr[64]; - if (count > 64) + if (count == 0 || count > sizeof(optstr)) return -EINVAL; if (copy_from_user(optstr, user, count)) return -EFAULT; -- cgit v1.2.1