PM: Added support for self-restore in case of LE Hypervisor - stage 2.

    - Special Handling of MSR Restoration.
    - Handles switchover of endianess after MSR update and accomplishes
      completion of self restoration as a little endian core.
    - Adds attention at all the architected interrupt vectors.
    - Minor update in SRESET handler

Change-Id: I8dfae1cc3fd6aa98182d9adf0ff0f4fabb23ac2e
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/39417
Reviewed-by: Michael S. Floyd <mfloyd@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Reviewed-by: AMIT KUMAR <akumar3@us.ibm.com>
Reviewed-by: Gregory S. Still <stillgs@us.ibm.com>

4 files changed, 386 insertions, 67 deletions

diff --git a/import/chips/p9/procedures/utils/stopreg/p9_core_restore_routines.S b/import/chips/p9/procedures/utils/stopreg/p9_core_restore_routines.S
index 8ba35f39..2ecefe4b 100755
--- a/import/chips/p9/procedures/utils/stopreg/p9_core_restore_routines.S
+++ b/import/chips/p9/procedures/utils/stopreg/p9_core_restore_routines.S
@@ -147,12 +147,21 @@
                 .set THREAD_ID_REG,                 r6
                 .set CORE_RESTORE_BASE_ADDR_REG,    r7
                 .set TEMP_REG,                      r8
-                .set FUSED_MODE_CONST,              0xBB #Chose this flag to be an arbitrary unique value
                 .set HRMOR_RESTORE_REG,             r10
                 .set THREAD_ACTIVE_STATE_REG,       r11
                 .set CORE_SCOPE_RESTORE_ADDR_REG,   r12
+                .set HRMOR_INIT_REG,                r20
+                .set MSR_INIT_REG,                  r21
+                .set SKIP_HRMOR_UPDATE_OFFSET,      4
                 .set STOP,                          0x4C0002E4
                 .set ATTN,                          0x00000200
+                .set MSR_LE0_RESTORE_OFFSET,        0x1200  # Big Endian restore area
+                .set MSR_LE1_RESTORE_OFFSET,        0x1280  # Little Endian restore area
+                .set SWIZZLE_MTHRMOR,               0xA64B597D
+                .set SWIZZLE_SLBIA,                 0xE403007C
+                .set SWIZZLE_STOP,                  0xE402004C
+                .set SWIZZLE_ATTN,                  0x00020000
+
 #--------------------------------------------------------------------#
 #
 #                   Interrupt Vectors
@@ -221,73 +230,230 @@ _start:
 .org _start + SRESET
 
 _sreset_hndlr:
-mfspr r2, HRMOR
-addi  r2, r2, THREAD_LAUNCHER_START_OFFSET
-mtsrr0 r2
-mfmsr r2
-ori r2, r2, MACHINE_CHECK_ENABLE_CONST
-mtsrr1 r2
+mfspr   HRMOR_INIT_REG, HRMOR
+addi    TEMP_REG, HRMOR_INIT_REG, THREAD_LAUNCHER_START_OFFSET
+mtsrr0  TEMP_REG
+mfmsr   MSR_INIT_REG        //saves initial MSR value to a GPR
+ori     MSR_INIT_REG, MSR_INIT_REG, MACHINE_CHECK_ENABLE_CONST
+mtsrr1  MSR_INIT_REG
 rfid
 
 #--------------------------------------------------------------------#
 
 #Error handling for other interrupt vectors.
 
-.org _start + 0x0180
-.long ATTN
-
+#--------------------------------------------------------------------#
+#       Machine Check
+#--------------------------------------------------------------------#
 .org _start + 0x0200
 .long ATTN
 
-.org _start + 0x0280
-.long ATTN
-
+#--------------------------------------------------------------------#
+#       Data Storage
+#--------------------------------------------------------------------#
 .org _start + 0x0300
 .long ATTN
 
+#--------------------------------------------------------------------#
+#       Data Segment
+#--------------------------------------------------------------------#
 .org _start + 0x0380
 .long ATTN
 
+#--------------------------------------------------------------------#
+#       Instruction Storage
+#--------------------------------------------------------------------#
 .org _start + 0x0400
 .long ATTN
 
+#--------------------------------------------------------------------#
+#       Instruction Segment
+#--------------------------------------------------------------------#
 .org _start + 0x0480
 .long ATTN
 
+#--------------------------------------------------------------------#
+#       External
+#--------------------------------------------------------------------#
 .org _start + 0x0500
 .long ATTN
 
-.org _start + 0x0580
+#--------------------------------------------------------------------#
+#       Alignment
+#--------------------------------------------------------------------#
+.org _start + 0x0600
 .long ATTN
 
-.org _start + 0x0600
+#--------------------------------------------------------------------#
+#       Program
+#--------------------------------------------------------------------#
+.org _start + 0x0700
 .long ATTN
 
-.org _start + 0x0680
+#--------------------------------------------------------------------#
+#       Floating Point Unavailable
+#--------------------------------------------------------------------#
+.org _start + 0x0800
 .long ATTN
 
-.org _start + 0x0700
+#--------------------------------------------------------------------#
+#       Decrementer
+#--------------------------------------------------------------------#
+.org _start + 0x0900
 .long ATTN
 
-.org _start + 0x0780
+#--------------------------------------------------------------------#
+#       Hypervisor Decrementer
+#--------------------------------------------------------------------#
+.org _start + 0x0980
 .long ATTN
 
-.org _start + 0x0800
+#--------------------------------------------------------------------#
+#       Directed Priviledged Doorbell
+#--------------------------------------------------------------------#
+.org _start + 0x0A00
 .long ATTN
 
-.org _start + 0x0880
+#--------------------------------------------------------------------#
+#       Reserved
+#--------------------------------------------------------------------#
+.org _start + 0x0B00
 .long ATTN
-.org _start + 0x0900
+
+#--------------------------------------------------------------------#
+#       System Call
+#--------------------------------------------------------------------#
+.org _start + 0x0C00
 .long ATTN
 
-.long _start + 0x0980
+#--------------------------------------------------------------------#
+#       Trace
+#--------------------------------------------------------------------#
+.org _start + 0x0D00
 .long ATTN
 
-#-----------------------------------------------------------------------------
+#--------------------------------------------------------------------#
+#       Hypervisor Data Storage
+#--------------------------------------------------------------------#
+.org _start + 0x0E00
+.long ATTN
+
+#--------------------------------------------------------------------#
+#       Hypervisor Instruction Storage
+#--------------------------------------------------------------------#
+.org _start + 0x0E20
+.long ATTN
+
+#--------------------------------------------------------------------#
+#       Hypervisor Emulation Assistance
+#--------------------------------------------------------------------#
+.org _start + 0x0E40
+.long ATTN
+
+#--------------------------------------------------------------------#
+#       Hypervisor Maintenance
+#--------------------------------------------------------------------#
+.org _start + 0x0E60
+.long ATTN
+
+#--------------------------------------------------------------------#
+#       Directed Hypervisor Doorbell
+#--------------------------------------------------------------------#
+.org _start + 0x0E80
+.long ATTN
+
+#--------------------------------------------------------------------#
+#       Reserved
+#--------------------------------------------------------------------#
+.org _start + 0x0EA0
+.long ATTN
+
+#--------------------------------------------------------------------#
+#       Reserved
+#--------------------------------------------------------------------#
+.org _start + 0x0EC0
+.long ATTN
+
+#--------------------------------------------------------------------#
+#       Reserved
+#--------------------------------------------------------------------#
+.org _start + 0x0EE0
+.long ATTN
+
+#--------------------------------------------------------------------#
+#       Performance Monitoring
+#--------------------------------------------------------------------#
+.org _start + 0x0F00
+.long ATTN
+
+#--------------------------------------------------------------------#
+#       Vector Unavailable
+#--------------------------------------------------------------------#
+.org _start + 0x0F20
+.long ATTN
+
+#--------------------------------------------------------------------#
+#       VSX Unavailable
+#--------------------------------------------------------------------#
+.org _start + 0x0F40
+.long ATTN
+
+#--------------------------------------------------------------------#
+#       Facility Unavailable
+#--------------------------------------------------------------------#
+.org _start + 0x0F60
+.long ATTN
+
+#--------------------------------------------------------------------#
+#       Hypervisor Facility Unavailable
+#--------------------------------------------------------------------#
+.org _start + 0x0F80
+.long ATTN
+
+
+#--------------------------------------------------------------------#
+#       Self Restore Completion - Big Endian Hypervisor
+#--------------------------------------------------------------------#
+.org _start + MSR_LE0_RESTORE_OFFSET
+
+#restore the HRMOR to the value needed by the Hypervisor upon wakeup
+#there can be no future I-fetches after this point, so no more than 7
+#instructions after this
+
+mtspr HRMOR, HRMOR_RESTORE_REG
+
+#necessary to invalidate stale translations in the ERATs that were created
+#during the self-restore code execution.
+
+slbia
+.long STOP
+
+#******* Big Endian Core Entered STOP State **************************
+#Now ready to return execution to the Big Endian Hypervisor
+
+.long ATTN
+
+#--------------------------------------------------------------------#
+#       Self Restore Completion - Little Endian Hypervisor
+#--------------------------------------------------------------------#
+
+#For completion of self restore in case of little endian hypervisor
+.org _start +  MSR_LE1_RESTORE_OFFSET
+
+.long SWIZZLE_MTHRMOR
+.long SWIZZLE_SLBIA
+.long SWIZZLE_STOP
+
+#******* Little Endian Core Entered STOP State ***********************
+#Now ready to return execution to the Little Endian Hypervisor
+
+.long SWIZZLE_ATTN
+
+#---------------------------------------------------------------------
 #
 # common code for thread restoration
 #
-#----------------------------------------------------------------------------
+#---------------------------------------------------------------------
 
 .org _start + THREAD_LAUNCHER_START_OFFSET
 
@@ -374,8 +540,8 @@ blrl                                          #branch to thread register restore
 
 #return here after thread register restoration
 thread_restore_return:
-cmpwi THREAD_ID_REG, 0                        # is it thread 0
-bne enter_stop_state                          # if thread not in STOP 15
+cmpwi  THREAD_ID_REG, 0            # if thread in question is not 0, restore MSR , enter STOP
+bne    core_reg_restore_return     # else wait for other threads to be done then restore core SPR, MSR
 
 wait_until_single_thread:          # wait till all threads are done restoring
 mfspr TEMP_REG, SPRD
@@ -386,29 +552,25 @@ bne wait_until_single_thread
 #**************** all threads restored thread scope register *****************
 
 mtlr CORE_SCOPE_RESTORE_ADDR_REG
-blrl                                # branch to core register restore area
-
-#********************************* enter STOP State **************************
-# following pseudo-op aligns next opcode to 2^4 = 16B boundary. Each opcode
-# takes 4B. So, next four opcodes will always start at 16B boundary and
-# also never cross 128B cache line.
-
-.align 4
-
-core_reg_restore_return:        #so far core reg restored except HRMOR
-mtspr HRMOR, HRMOR_RESTORE_REG
-
-enter_stop_state:
-
-#necessary to invalidate stale translations in the ERATs that were created
-#during the self-restore code execution.
-slbia
-.long STOP
-.long ATTN
+blrl                            # branch to core register restore area
+
+core_reg_restore_return:        #so far core reg restored except HRMOR and MSR
+
+#RFID to set the new MSR.  address determined by MSR[LE] bit 63
+mtsrr1  MSR_INIT_REG
+addi    TEMP_REG, HRMOR_INIT_REG, MSR_LE0_RESTORE_OFFSET
+extrdi. MSR_INIT_REG, MSR_INIT_REG, 1, 63
+beq     skip_le1_offset         # if MSR[LE] eq 0, use LE0 offset
+addi    TEMP_REG, HRMOR_INIT_REG, MSR_LE1_RESTORE_OFFSET
+
+skip_le1_offset:
+cmpwi  THREAD_ID_REG, 0
+beq update_srr0
+addi TEMP_REG, TEMP_REG, SKIP_HRMOR_UPDATE_OFFSET #restore HRMOR only if thread0
+update_srr0:
+mtsrr0  TEMP_REG
+rfid
 
 # THREAD_LAUNCHER_SIZE_OFFSET must be >= (4 * number of instructions between
 # here and thread_launcher_start)
-
-#***************************** finished entered STOP State **************
-
 #********************************* End Thread Launcher **********************
diff --git a/import/chips/p9/procedures/utils/stopreg/self.sh b/import/chips/p9/procedures/utils/stopreg/self.sh
index 290383b6..00ce3f22 100755
--- a/import/chips/p9/procedures/utils/stopreg/self.sh
+++ b/import/chips/p9/procedures/utils/stopreg/self.sh
@@ -23,20 +23,31 @@
 #
 # IBM_PROLOG_END_TAG
 #first first. Remove object file and executable file.
-rm -f *.o
-rm -f *.bin
-SRC=p9_core_restore_routines.S
-OBJ=p9_core_restore_routines.o
-IMG=selfRest
+
+#!/bin/bash
+SELF_REST_PATH=$PROJECT_ROOT/chips/p9/procedures/utils/stopreg
+SRC=$SELF_REST_PATH/p9_core_restore_routines.S
+OBJ=$SELF_REST_PATH/p9_core_restore_routines.o
+IMG=$SELF_REST_PATH/selfRest
+LINK=$SELF_REST_PATH/link.c
+OBJDUMP="/opt/mcp/shared/powerpc64-gcc-20150516/wrappers/powerpc64-unknown-linux-gnu-objdump "
 #set compiler path
-ASMFILE="/opt/mcp/shared/powerpc64-gcc-20150516/wrappers/powerpc64-unknown-linux-gnu-gcc -std=c99 -c -O3 -nostdlib -mcpu=power7 -mbig-endian -ffreestanding -mabi=elfv1"
+ASMFILE="/opt/mcp/shared/powerpc64-gcc-20150516/wrappers/powerpc64-unknown-linux-gnu-gcc "
+ASFLAGS="-save-temps -std=c99 -c -O3 -nostdlib -mcpu=power7 -mbig-endian -ffreestanding -mabi=elfv1"
 #linker script after C preprocessing
-LINK_SCRIPT=selfLink
+LINK_SCRIPT=$SELF_REST_PATH/selfLink
+if [ -f $IMG ] 
+then    
+    rm -f $IMG_MAP
+    rm -f $DIS
+    rm -f $IMG.bin
+fi    
 #set linker path
 LOAD=/opt/mcp/shared/powerpc64-gcc-20150516/wrappers/powerpc64-unknown-linux-gnu-ld
 #passing linker through pre-processor
-$ASMFILE -E -c -P link.c -o $LINK_SCRIPT
+$ASMFILE -E -c -P $LINK -o $LINK_SCRIPT
 #compiling Source file
-$ASMFILE $SRC -o $OBJ
+$ASMFILE $SRC $ASFLAGS -o $OBJ
 #linking
 $LOAD -T$LINK_SCRIPT -Map $IMG.map -Bstatic -o $IMG.bin $OBJ
+$OBJDUMP -d  $OBJ >$IMG.list
diff --git a/import/chips/p9/procedures/utils/stopreg/selfRest.list b/import/chips/p9/procedures/utils/stopreg/selfRest.list
new file mode 100644
index 00000000..56af59a8
--- /dev/null
+++ b/import/chips/p9/procedures/utils/stopreg/selfRest.list
@@ -0,0 +1,146 @@
+
+/gsa/ausgsa/projects/i/indiateam01/premjha2/ekb/chips/p9/procedures/utils/stopreg/p9_core_restore_routines.o:     file format elf64-powerpc
+
+
+Disassembly of section .selfRestore:
+
+0000000000000000 <_start>:
+	...
+
+0000000000000100 <_sreset_hndlr>:
+     100:	7e 99 4a a6 	mfspr   r20,313
+     104:	39 14 20 00 	addi    r8,r20,8192
+     108:	7d 1a 03 a6 	mtsrr0  r8
+     10c:	7e a0 00 a6 	mfmsr   r21
+     110:	62 b5 10 00 	ori     r21,r21,4096
+     114:	7e bb 03 a6 	mtsrr1  r21
+     118:	4c 00 00 24 	rfid
+	...
+     200:	00 00 02 00 	attn
+	...
+     300:	00 00 02 00 	attn
+	...
+     380:	00 00 02 00 	attn
+	...
+     400:	00 00 02 00 	attn
+	...
+     480:	00 00 02 00 	attn
+	...
+     500:	00 00 02 00 	attn
+	...
+     600:	00 00 02 00 	attn
+	...
+     700:	00 00 02 00 	attn
+	...
+     800:	00 00 02 00 	attn
+	...
+     900:	00 00 02 00 	attn
+	...
+     980:	00 00 02 00 	attn
+	...
+     a00:	00 00 02 00 	attn
+	...
+     b00:	00 00 02 00 	attn
+	...
+     c00:	00 00 02 00 	attn
+	...
+     d00:	00 00 02 00 	attn
+	...
+     e00:	00 00 02 00 	attn
+	...
+     e20:	00 00 02 00 	attn
+	...
+     e40:	00 00 02 00 	attn
+	...
+     e60:	00 00 02 00 	attn
+	...
+     e80:	00 00 02 00 	attn
+	...
+     ea0:	00 00 02 00 	attn
+	...
+     ec0:	00 00 02 00 	attn
+	...
+     ee0:	00 00 02 00 	attn
+	...
+     f00:	00 00 02 00 	attn
+	...
+     f20:	00 00 02 00 	attn
+	...
+     f40:	00 00 02 00 	attn
+	...
+     f60:	00 00 02 00 	attn
+	...
+     f80:	00 00 02 00 	attn
+	...
+    1200:	7d 59 4b a6 	mtspr   313,r10
+    1204:	7c 00 03 e4 	slbia
+    1208:	4c 00 02 e4 	.long 0x4c0002e4
+    120c:	00 00 02 00 	attn
+	...
+    1280:	a6 4b 59 7d 	lhzu    r18,22909(r11)
+    1284:	e4 03 00 7c 	lfdp    f0,124(r3)
+    1288:	e4 02 00 4c 	lfdp    f0,76(r2)
+    128c:	00 02 00 00 	.long 0x20000
+	...
+
+0000000000002000 <thread_launcher_start>:
+    2000:	7c 5f fa a6 	mfspr   r2,1023
+    2004:	39 00 01 e0 	li      r8,480
+    2008:	7d 14 43 a6 	mtspr   276,r8
+    200c:	7c 95 42 a6 	mfspr   r4,277
+    2010:	78 84 07 e1 	clrldi. r4,r4,63
+    2014:	41 82 00 1c 	beq     2030 <core_is_not_fused>
+
+0000000000002018 <core_is_fused>:
+    2018:	78 45 ef e2 	rldicl  r5,r2,61,63
+    201c:	78 a5 0f a4 	rldicr  r5,r5,1,62
+    2020:	78 48 07 e0 	clrldi  r8,r2,63
+    2024:	7c a5 42 14 	add     r5,r5,r8
+    2028:	78 46 ff a2 	rldicl  r6,r2,63,62
+    202c:	48 00 00 0c 	b       2038 <jump_thread_restore>
+
+0000000000002030 <core_is_not_fused>:
+    2030:	78 45 f7 a2 	rldicl  r5,r2,62,62
+    2034:	78 46 07 a0 	clrldi  r6,r2,62
+
+0000000000002038 <jump_thread_restore>:
+    2038:	78 48 e7 62 	rldicl  r8,r2,60,61
+    203c:	79 08 17 64 	rldicr  r8,r8,2,61
+    2040:	7c a5 42 14 	add     r5,r5,r8
+    2044:	7c 79 4a a6 	mfspr   r3,313
+    2048:	78 a7 6c a4 	rldicr  r7,r5,13,50
+    204c:	7c e7 1a 14 	add     r7,r7,r3
+    2050:	38 e7 21 00 	addi    r7,r7,8448
+    2054:	78 c8 5d 24 	rldicr  r8,r6,11,52
+    2058:	7d 07 42 14 	add     r8,r7,r8
+    205c:	39 88 04 00 	addi    r12,r8,1024
+    2060:	7d 08 03 a6 	mtlr    r8
+    2064:	4e 80 00 21 	blrl
+
+0000000000002068 <thread_restore_return>:
+    2068:	2c 06 00 00 	cmpwi   r6,0
+    206c:	40 82 00 1c 	bne     2088 <core_reg_restore_return>
+
+0000000000002070 <wait_until_single_thread>:
+    2070:	7d 15 42 a6 	mfspr   r8,277
+    2074:	79 08 e7 22 	rldicl  r8,r8,60,60
+    2078:	2c 08 00 07 	cmpwi   r8,7
+    207c:	40 82 ff f4 	bne     2070 <wait_until_single_thread>
+    2080:	7d 88 03 a6 	mtlr    r12
+    2084:	4e 80 00 21 	blrl
+
+0000000000002088 <core_reg_restore_return>:
+    2088:	7e bb 03 a6 	mtsrr1  r21
+    208c:	39 14 12 00 	addi    r8,r20,4608
+    2090:	7a b5 07 e1 	clrldi. r21,r21,63
+    2094:	41 82 00 08 	beq     209c <skip_le1_offset>
+    2098:	39 14 12 80 	addi    r8,r20,4736
+
+000000000000209c <skip_le1_offset>:
+    209c:	2c 06 00 00 	cmpwi   r6,0
+    20a0:	41 82 00 08 	beq     20a8 <update_srr0>
+    20a4:	39 08 00 04 	addi    r8,r8,4
+
+00000000000020a8 <update_srr0>:
+    20a8:	7d 1a 03 a6 	mtsrr0  r8
+    20ac:	4c 00 00 24 	rfid
diff --git a/import/chips/p9/procedures/utils/stopreg/selfRest.map b/import/chips/p9/procedures/utils/stopreg/selfRest.map
index 50eb65b7..21a7f70b 100644
--- a/import/chips/p9/procedures/utils/stopreg/selfRest.map
+++ b/import/chips/p9/procedures/utils/stopreg/selfRest.map
@@ -11,18 +11,18 @@ Linker script and memory map
                 0x0000000000000000                . = ALIGN (0x80)
                 0x0000000000000000                _SELF_RESTORE_START = .
 
-.selfRestore    0x0000000000000000     0x20a0
+.selfRestore    0x0000000000000000     0x20b0
  *(.selfRestore)
- .selfRestore   0x0000000000000000     0x20a0 p9_core_restore_routines.o
+ .selfRestore   0x0000000000000000     0x20b0 /gsa/ausgsa/projects/i/indiateam01/premjha2/ekb/chips/p9/procedures/utils/stopreg/p9_core_restore_routines.o
                 0x0000000000000000                _start
-LOAD p9_core_restore_routines.o
-OUTPUT(selfRest.bin binary)
+LOAD /gsa/ausgsa/projects/i/indiateam01/premjha2/ekb/chips/p9/procedures/utils/stopreg/p9_core_restore_routines.o
+OUTPUT(/gsa/ausgsa/projects/i/indiateam01/premjha2/ekb/chips/p9/procedures/utils/stopreg/selfRest.bin binary)
 
-.text           0x00000000000020a0        0x0
- .text          0x00000000000020a0        0x0 p9_core_restore_routines.o
+.text           0x00000000000020b0        0x0
+ .text          0x00000000000020b0        0x0 /gsa/ausgsa/projects/i/indiateam01/premjha2/ekb/chips/p9/procedures/utils/stopreg/p9_core_restore_routines.o
 
-.data           0x00000000000020a0        0x0
- .data          0x00000000000020a0        0x0 p9_core_restore_routines.o
+.data           0x00000000000020b0        0x0
+ .data          0x00000000000020b0        0x0 /gsa/ausgsa/projects/i/indiateam01/premjha2/ekb/chips/p9/procedures/utils/stopreg/p9_core_restore_routines.o
 
-.bss            0x00000000000020a0        0x0
- .bss           0x00000000000020a0        0x0 p9_core_restore_routines.o
+.bss            0x00000000000020b0        0x0
+ .bss           0x00000000000020b0        0x0 /gsa/ausgsa/projects/i/indiateam01/premjha2/ekb/chips/p9/procedures/utils/stopreg/p9_core_restore_routines.o