From a650d3839e7a68321e5b76264398a63019b0928b Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Fri, 12 Oct 2007 02:59:40 -0700 Subject: [SPARC64]: Make IVEC pointers 64-bit. Currently we chain IVEC entries using 32-bit "pointers" because we know that the ivector_table is in the main kernel image, thus below 4GB. This uses proper 64-bit pointers instead. Whilst this bloats up the kernel image size, this sets the infrastructure necessary to significantly shrink the kernel size by using physical addresses and dynamically allocating the ivector table. Signed-off-by: David S. Miller --- arch/sparc64/kernel/entry.S | 8 ++++---- arch/sparc64/kernel/irq.c | 41 +++++++++++++++++++++++----------------- arch/sparc64/kernel/sun4v_ivec.S | 8 ++++---- include/asm-sparc64/cpudata.h | 11 +++++------ include/asm-sparc64/irq.h | 2 +- 5 files changed, 38 insertions(+), 32 deletions(-) diff --git a/arch/sparc64/kernel/entry.S b/arch/sparc64/kernel/entry.S index 8059531bf0ac..9a785293203f 100644 --- a/arch/sparc64/kernel/entry.S +++ b/arch/sparc64/kernel/entry.S @@ -430,15 +430,15 @@ do_ivec: membar #Sync sethi %hi(ivector_table), %g2 - sllx %g3, 3, %g3 + sllx %g3, 4, %g3 or %g2, %lo(ivector_table), %g2 add %g2, %g3, %g3 TRAP_LOAD_IRQ_WORK(%g6, %g1) - lduw [%g6], %g5 /* g5 = irq_work(cpu) */ - stw %g5, [%g3 + 0x00] /* bucket->irq_chain = g5 */ - stw %g3, [%g6] /* irq_work(cpu) = bucket */ + ldx [%g6], %g5 /* g5 = irq_work(cpu) */ + stx %g5, [%g3 + 0x00] /* bucket->irq_chain = g5 */ + stx %g3, [%g6] /* irq_work(cpu) = bucket */ wr %g0, 1 << PIL_DEVICE_IRQ, %set_softint retry do_ivec_xcall: diff --git a/arch/sparc64/kernel/irq.c b/arch/sparc64/kernel/irq.c index 045ab27d4271..4db4dd576210 100644 --- a/arch/sparc64/kernel/irq.c +++ b/arch/sparc64/kernel/irq.c @@ -59,23 +59,20 @@ * of the vectored interrupt trap handler(s) in entry.S and sun4v_ivec.S */ struct ino_bucket { - /* Next handler in per-CPU IRQ worklist. We know that - * bucket pointers have the high 32-bits clear, so to - * save space we only store the bits we need. - */ -/*0x00*/unsigned int irq_chain; +/*0x00*/unsigned long irq_chain; /* Virtual interrupt number assigned to this INO. */ -/*0x04*/unsigned int virt_irq; +/*0x08*/unsigned int virt_irq; +/*0x0c*/unsigned int __pad; }; #define NUM_IVECS (IMAP_INR + 1) struct ino_bucket ivector_table[NUM_IVECS] __attribute__ ((aligned (SMP_CACHE_BYTES))); #define __irq_ino(irq) \ - (((struct ino_bucket *)(unsigned long)(irq)) - &ivector_table[0]) -#define __bucket(irq) ((struct ino_bucket *)(unsigned long)(irq)) -#define __irq(bucket) ((unsigned int)(unsigned long)(bucket)) + (((struct ino_bucket *)(irq)) - &ivector_table[0]) +#define __bucket(irq) ((struct ino_bucket *)(irq)) +#define __irq(bucket) ((unsigned long)(bucket)) /* This has to be in the main kernel image, it cannot be * turned into per-cpu data. The reason is that the main @@ -87,13 +84,13 @@ struct ino_bucket ivector_table[NUM_IVECS] __attribute__ ((aligned (SMP_CACHE_BY #define irq_work(__cpu) &(trap_block[(__cpu)].irq_worklist) static struct { - unsigned int irq; + unsigned long irq; unsigned int dev_handle; unsigned int dev_ino; } virt_to_real_irq_table[NR_IRQS]; static DEFINE_SPINLOCK(virt_irq_alloc_lock); -unsigned char virt_irq_alloc(unsigned int real_irq) +unsigned char virt_irq_alloc(unsigned long real_irq) { unsigned long flags; unsigned char ent; @@ -134,7 +131,7 @@ void virt_irq_free(unsigned int virt_irq) } #endif -static unsigned int virt_to_real_irq(unsigned char virt_irq) +static unsigned long virt_to_real_irq(unsigned char virt_irq) { return virt_to_real_irq_table[virt_irq].irq; } @@ -227,7 +224,7 @@ struct irq_handler_data { static inline struct ino_bucket *virt_irq_to_bucket(unsigned int virt_irq) { - unsigned int real_irq = virt_to_real_irq(virt_irq); + unsigned long real_irq = virt_to_real_irq(virt_irq); struct ino_bucket *bucket = NULL; if (likely(real_irq)) @@ -694,18 +691,28 @@ void handler_irq(int irq, struct pt_regs *regs) { struct ino_bucket *bucket; struct pt_regs *old_regs; + unsigned long pstate; clear_softint(1 << irq); old_regs = set_irq_regs(regs); irq_enter(); - /* Sliiiick... */ - bucket = __bucket(xchg32(irq_work(smp_processor_id()), 0)); + /* Grab an atomic snapshot of the pending IVECs. */ + __asm__ __volatile__("rdpr %%pstate, %0\n\t" + "wrpr %0, %3, %%pstate\n\t" + "ldx [%2], %1\n\t" + "stx %%g0, [%2]\n\t" + "wrpr %0, 0x0, %%pstate\n\t" + : "=&r" (pstate), "=&r" (bucket) + : "r" (irq_work(smp_processor_id())), + "i" (PSTATE_IE) + : "memory"); + while (bucket) { struct ino_bucket *next = __bucket(bucket->irq_chain); - bucket->irq_chain = 0; + bucket->irq_chain = 0UL; __do_IRQ(bucket->virt_irq); bucket = next; @@ -808,7 +815,7 @@ void init_irqwork_curcpu(void) { int cpu = hard_smp_processor_id(); - trap_block[cpu].irq_worklist = 0; + trap_block[cpu].irq_worklist = 0UL; } /* Please be very careful with register_one_mondo() and diff --git a/arch/sparc64/kernel/sun4v_ivec.S b/arch/sparc64/kernel/sun4v_ivec.S index 574bc248bca6..e3e9d4c1574b 100644 --- a/arch/sparc64/kernel/sun4v_ivec.S +++ b/arch/sparc64/kernel/sun4v_ivec.S @@ -101,14 +101,14 @@ sun4v_dev_mondo: /* Get &ivector_table[IVEC] into %g4. */ sethi %hi(ivector_table), %g4 - sllx %g3, 3, %g3 + sllx %g3, 4, %g3 or %g4, %lo(ivector_table), %g4 add %g4, %g3, %g4 /* Insert ivector_table[] entry into __irq_work[] queue. */ - lduw [%g1], %g2 /* g2 = irq_work(cpu) */ - stw %g2, [%g4 + 0x00] /* bucket->irq_chain = g2 */ - stw %g4, [%g1] /* irq_work(cpu) = bucket */ + ldx [%g1], %g2 /* g2 = irq_work(cpu) */ + stx %g2, [%g4 + 0x00] /* bucket->irq_chain = g2 */ + stx %g4, [%g1] /* irq_work(cpu) = bucket */ /* Signal the interrupt by setting (1 << pil) in %softint. */ wr %g0, 1 << PIL_DEVICE_IRQ, %set_softint diff --git a/include/asm-sparc64/cpudata.h b/include/asm-sparc64/cpudata.h index 98a6e609163e..379c219c3b7a 100644 --- a/include/asm-sparc64/cpudata.h +++ b/include/asm-sparc64/cpudata.h @@ -75,12 +75,11 @@ struct trap_per_cpu { unsigned long tsb_huge_temp; /* Dcache line 8: IRQ work list, and keep trap_block a power-of-2 in size. */ - unsigned int irq_worklist; + unsigned long irq_worklist; unsigned int cpu_mondo_qmask; unsigned int dev_mondo_qmask; unsigned int resum_qmask; unsigned int nonresum_qmask; - unsigned int __pad2[1]; void *hdesc; } __attribute__((aligned(64))); extern struct trap_per_cpu trap_block[NR_CPUS]; @@ -129,10 +128,10 @@ extern struct sun4v_2insn_patch_entry __sun4v_2insn_patch, #define TRAP_PER_CPU_TSB_HUGE 0xd0 #define TRAP_PER_CPU_TSB_HUGE_TEMP 0xd8 #define TRAP_PER_CPU_IRQ_WORKLIST 0xe0 -#define TRAP_PER_CPU_CPU_MONDO_QMASK 0xe4 -#define TRAP_PER_CPU_DEV_MONDO_QMASK 0xe8 -#define TRAP_PER_CPU_RESUM_QMASK 0xec -#define TRAP_PER_CPU_NONRESUM_QMASK 0xf0 +#define TRAP_PER_CPU_CPU_MONDO_QMASK 0xe8 +#define TRAP_PER_CPU_DEV_MONDO_QMASK 0xec +#define TRAP_PER_CPU_RESUM_QMASK 0xf0 +#define TRAP_PER_CPU_NONRESUM_QMASK 0xf4 #define TRAP_BLOCK_SZ_SHIFT 8 diff --git a/include/asm-sparc64/irq.h b/include/asm-sparc64/irq.h index bad3c28cdee8..24841c22e81b 100644 --- a/include/asm-sparc64/irq.h +++ b/include/asm-sparc64/irq.h @@ -59,7 +59,7 @@ extern unsigned int sun4u_build_msi(u32 portid, unsigned int *virt_irq_p, extern void sun4u_destroy_msi(unsigned int virt_irq); extern unsigned int sbus_build_irq(void *sbus, unsigned int ino); -extern unsigned char virt_irq_alloc(unsigned int real_irq); +extern unsigned char virt_irq_alloc(unsigned long real_irq); #ifdef CONFIG_PCI_MSI extern void virt_irq_free(unsigned int virt_irq); #endif -- cgit v1.2.1