UV Support : Augmented STOP API and self restore for enabling ultravisor.

HW-Image-Coreq: yes
HW-Image-Prereq: Ia9ae0d284398af375f1562efff152a6a12a6eb9a
Change-Id: I67f9ec7a76acc93d89a479bcbae343d2b278ce11
Original-Change-Id: I1f7ca865640dfc0a08aef783fd3595d2f249a672
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/58843
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: HWSV CI <hwsv-ci+hostboot@us.ibm.com>
Tested-by: PPE CI <ppe-ci+hostboot@us.ibm.com>
Tested-by: Hostboot CI <hostboot-ci+hostboot@us.ibm.com>
Reviewed-by: Gregory S. Still <stillgs@us.ibm.com>
Reviewed-by: AMIT J. TENDOLKAR <amit.tendolkar@in.ibm.com>
Reviewed-by: Jennifer A. Stofer <stofer@us.ibm.com>

6 files changed, 202 insertions, 70 deletions

diff --git a/import/chips/p9/procedures/ppe_closed/cme/pk_app_cfg.h b/import/chips/p9/procedures/ppe_closed/cme/pk_app_cfg.h
index 63ad2414..1124ed8a 100644
--- a/import/chips/p9/procedures/ppe_closed/cme/pk_app_cfg.h
+++ b/import/chips/p9/procedures/ppe_closed/cme/pk_app_cfg.h
@@ -57,8 +57,8 @@
 // being enabled on all other dd levels unless this override
 #define DISABLE_CME_DUAL_CAST            0
 
-// NDD23 Secure Memory Support: RAM URMOR
-#if NIMBUS_DD_LEVEL >= 23 || CUMULUS_DD_LEVEL >= 13
+// NDD22 Secure Memory Support: RAM URMOR
+#if NIMBUS_DD_LEVEL >= 22 || CUMULUS_DD_LEVEL >= 13
 
     #define SMF_SUPPORT_ENABLE 1
 
diff --git a/import/chips/p9/procedures/utils/stopreg/link.c b/import/chips/p9/procedures/utils/stopreg/link.c
index 157565b8..f8514c05 100644
--- a/import/chips/p9/procedures/utils/stopreg/link.c
+++ b/import/chips/p9/procedures/utils/stopreg/link.c
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HCODE Project                                                */
 /*                                                                        */
-/* COPYRIGHT 2016,2017                                                    */
+/* COPYRIGHT 2016,2018                                                    */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -23,9 +23,9 @@
 /*                                                                        */
 /* IBM_PROLOG_END_TAG                                                     */
 #define SRESET_HNDLR_OFFSET    0x00
-#define SELF_RESTORE_SIZE      0x2100
+#define SELF_RESTORE_SIZE      0x2400
 #define INT_VECT_AREA          THREAD_LAUNCHER_OFFSET
-#define THREAD_LAUNCHER_SIZE   256
+#define THREAD_LAUNCHER_SIZE   1024
 #define CACHE_LINE_SIZE        128
 
 OUTPUT_FORMAT(binary)
diff --git a/import/chips/p9/procedures/utils/stopreg/self.sh b/import/chips/p9/procedures/utils/stopreg/self.sh
index c4ba7089..d25f5b6c 100755
--- a/import/chips/p9/procedures/utils/stopreg/self.sh
+++ b/import/chips/p9/procedures/utils/stopreg/self.sh
@@ -26,8 +26,8 @@
 
 #!/bin/bash
 SELF_REST_PATH=$PROJECT_ROOT/chips/p9/procedures/utils/stopreg
-SRC=$SELF_REST_PATH/p9_core_restore_routines.S
-OBJ=$SELF_REST_PATH/p9_core_restore_routines.o
+SRC=$SELF_REST_PATH/p9_core_save_restore_routines.S
+OBJ=$SELF_REST_PATH/p9_core_save_restore_routines.o
 IMG=$SELF_REST_PATH/selfRest
 LINK=$SELF_REST_PATH/link.c
 OBJDUMP="/opt/mcp/shared/powerpc64-gcc-20150516/wrappers/powerpc64-unknown-linux-gnu-objdump "
diff --git a/import/chips/p9/procedures/utils/stopreg/selfLink b/import/chips/p9/procedures/utils/stopreg/selfLink
index 6334172b..8b41aec4 100644
--- a/import/chips/p9/procedures/utils/stopreg/selfLink
+++ b/import/chips/p9/procedures/utils/stopreg/selfLink
@@ -1,7 +1,7 @@
 OUTPUT_FORMAT(binary)
 MEMORY
 {
-    selfRestoreRegion (rx) : ORIGIN = 0x00, LENGTH = 0x2100
+    selfRestoreRegion (rx) : ORIGIN = 0x00, LENGTH = 0x2400
     }
     SECTIONS
 {
diff --git a/import/chips/p9/procedures/utils/stopreg/selfRest.list b/import/chips/p9/procedures/utils/stopreg/selfRest.list
index 56af59a8..065855cf 100644
--- a/import/chips/p9/procedures/utils/stopreg/selfRest.list
+++ b/import/chips/p9/procedures/utils/stopreg/selfRest.list
@@ -1,5 +1,5 @@
 
-/gsa/ausgsa/projects/i/indiateam01/premjha2/ekb/chips/p9/procedures/utils/stopreg/p9_core_restore_routines.o:     file format elf64-powerpc
+/esw/san2/premjha2/fapi2_lite/ekb/chips/p9/procedures/utils/stopreg/p9_core_save_restore_routines.o:     file format elf64-powerpc
 
 
 Disassembly of section .selfRestore:
@@ -8,13 +8,38 @@ Disassembly of section .selfRestore:
 	...
 
 0000000000000100 <_sreset_hndlr>:
-     100:	7e 99 4a a6 	mfspr   r20,313
-     104:	39 14 20 00 	addi    r8,r20,8192
-     108:	7d 1a 03 a6 	mtsrr0  r8
-     10c:	7e a0 00 a6 	mfmsr   r21
-     110:	62 b5 10 00 	ori     r21,r21,4096
-     114:	7e bb 03 a6 	mtsrr1  r21
-     118:	4c 00 00 24 	rfid
+     100:	7e a0 00 a6 	mfmsr   r21
+     104:	62 b5 10 00 	ori     r21,r21,4096
+     108:	7a b7 57 e3 	rldicl. r23,r21,42,63
+     10c:	41 82 00 2c 	beq     138 <hv_core_init>
+
+0000000000000110 <uv_core_check>:
+     110:	3a 20 00 01 	li      r17,1
+     114:	7e 1f 7a a6 	mfspr   r16,511
+     118:	7a 08 0f e1 	rldicl. r8,r16,1,63
+     11c:	41 82 00 14 	beq     130 <uv_init_error>
+     120:	3a 20 00 02 	li      r17,2
+     124:	7a 08 07 a0 	clrldi  r8,r16,62
+     128:	2c 08 00 02 	cmpwi   r8,2
+     12c:	41 82 00 20 	beq     14c <uv_core_init>
+
+0000000000000130 <uv_init_error>:
+     130:	7e 39 8b 78 	mr      r25,r17
+     134:	00 00 02 00 	attn
+
+0000000000000138 <hv_core_init>:
+     138:	7e 99 4a a6 	mfspr   r20,313
+     13c:	3a 34 20 00 	addi    r17,r20,8192
+     140:	7e 3a 03 a6 	mtsrr0  r17
+     144:	7e bb 03 a6 	mtsrr1  r21
+     148:	4c 00 00 24 	rfid
+
+000000000000014c <uv_core_init>:
+     14c:	7e 99 7a a6 	mfspr   r20,505
+     150:	39 14 20 00 	addi    r8,r20,8192
+     154:	7d 1a 7b a6 	mtspr   506,r8
+     158:	7e bb 7b a6 	mtspr   507,r21
+     15c:	4c 00 02 64 	.long 0x4c000264
 	...
      200:	00 00 02 00 	attn
 	...
@@ -73,14 +98,17 @@ Disassembly of section .selfRestore:
      f80:	00 00 02 00 	attn
 	...
     1200:	7d 59 4b a6 	mtspr   313,r10
+
+0000000000001204 <save_restore_done>:
     1204:	7c 00 03 e4 	slbia
     1208:	4c 00 02 e4 	.long 0x4c0002e4
     120c:	00 00 02 00 	attn
 	...
-    1280:	a6 4b 59 7d 	lhzu    r18,22909(r11)
-    1284:	e4 03 00 7c 	lfdp    f0,124(r3)
-    1288:	e4 02 00 4c 	lfdp    f0,76(r2)
-    128c:	00 02 00 00 	.long 0x20000
+    1280:	7d 59 4b a6 	mtspr   313,r10
+    1284:	7d 39 7b a6 	mtspr   505,r9
+    1288:	7c 00 03 e4 	slbia
+    128c:	4c 00 02 e4 	.long 0x4c0002e4
+    1290:	00 00 02 00 	attn
 	...
 
 0000000000002000 <thread_launcher_start>:
@@ -97,50 +125,154 @@ Disassembly of section .selfRestore:
     2020:	78 48 07 e0 	clrldi  r8,r2,63
     2024:	7c a5 42 14 	add     r5,r5,r8
     2028:	78 46 ff a2 	rldicl  r6,r2,63,62
-    202c:	48 00 00 0c 	b       2038 <jump_thread_restore>
+    202c:	48 00 00 0c 	b       2038 <thread_restore>
 
 0000000000002030 <core_is_not_fused>:
     2030:	78 45 f7 a2 	rldicl  r5,r2,62,62
     2034:	78 46 07 a0 	clrldi  r6,r2,62
 
-0000000000002038 <jump_thread_restore>:
+0000000000002038 <thread_restore>:
     2038:	78 48 e7 62 	rldicl  r8,r2,60,61
     203c:	79 08 17 64 	rldicr  r8,r8,2,61
     2040:	7c a5 42 14 	add     r5,r5,r8
-    2044:	7c 79 4a a6 	mfspr   r3,313
-    2048:	78 a7 6c a4 	rldicr  r7,r5,13,50
-    204c:	7c e7 1a 14 	add     r7,r7,r3
-    2050:	38 e7 21 00 	addi    r7,r7,8448
-    2054:	78 c8 5d 24 	rldicr  r8,r6,11,52
-    2058:	7d 07 42 14 	add     r8,r7,r8
-    205c:	39 88 04 00 	addi    r12,r8,1024
-    2060:	7d 08 03 a6 	mtlr    r8
-    2064:	4e 80 00 21 	blrl
-
-0000000000002068 <thread_restore_return>:
-    2068:	2c 06 00 00 	cmpwi   r6,0
-    206c:	40 82 00 1c 	bne     2088 <core_reg_restore_return>
-
-0000000000002070 <wait_until_single_thread>:
-    2070:	7d 15 42 a6 	mfspr   r8,277
-    2074:	79 08 e7 22 	rldicl  r8,r8,60,60
-    2078:	2c 08 00 07 	cmpwi   r8,7
-    207c:	40 82 ff f4 	bne     2070 <wait_until_single_thread>
-    2080:	7d 88 03 a6 	mtlr    r12
-    2084:	4e 80 00 21 	blrl
-
-0000000000002088 <core_reg_restore_return>:
-    2088:	7e bb 03 a6 	mtsrr1  r21
-    208c:	39 14 12 00 	addi    r8,r20,4608
-    2090:	7a b5 07 e1 	clrldi. r21,r21,63
-    2094:	41 82 00 08 	beq     209c <skip_le1_offset>
-    2098:	39 14 12 80 	addi    r8,r20,4736
-
-000000000000209c <skip_le1_offset>:
-    209c:	2c 06 00 00 	cmpwi   r6,0
-    20a0:	41 82 00 08 	beq     20a8 <update_srr0>
-    20a4:	39 08 00 04 	addi    r8,r8,4
-
-00000000000020a8 <update_srr0>:
-    20a8:	7d 1a 03 a6 	mtsrr0  r8
-    20ac:	4c 00 00 24 	rfid
+    2044:	78 a7 64 e4 	rldicr  r7,r5,12,51
+    2048:	38 e7 24 00 	addi    r7,r7,9216
+    204c:	7c e7 a2 14 	add     r7,r7,r20
+
+0000000000002050 <calculate_thread_save_addr>:
+    2050:	78 ce 45 e4 	rldicr  r14,r6,8,55
+    2054:	7d ce 3a 14 	add     r14,r14,r7
+    2058:	39 ce 08 00 	addi    r14,r14,2048
+
+000000000000205c <calculate_core_self_save_addr>:
+    205c:	7c ef 3b 78 	mr      r15,r7
+    2060:	39 ef 0e 00 	addi    r15,r15,3584
+
+0000000000002064 <calculate_self_restore_address>:
+    2064:	7c ed 3b 78 	mr      r13,r7
+    2068:	78 c8 4d a4 	rldicr  r8,r6,9,54
+    206c:	7d ad 42 14 	add     r13,r13,r8
+    2070:	39 87 0c 00 	addi    r12,r7,3072
+    2074:	7c a8 2b 78 	mr      r8,r5
+    2078:	71 08 00 01 	andi.   r8,r8,1
+    207c:	28 08 00 00 	cmplwi  r8,0
+    2080:	41 82 00 0c 	beq     208c <set_scratch_reg0>
+
+0000000000002084 <set_scratch_reg1>:
+    2084:	7d 14 43 a6 	mtspr   276,r8
+    2088:	48 00 00 08 	b       2090 <read_scratch_reg>
+
+000000000000208c <set_scratch_reg0>:
+    208c:	7c 14 43 a6 	mtspr   276,r0
+
+0000000000002090 <read_scratch_reg>:
+    2090:	7e 55 42 a6 	mfspr   r18,277
+    2094:	72 48 00 01 	andi.   r8,r18,1
+    2098:	2c 08 00 00 	cmpwi   r8,0
+    209c:	40 82 00 e4 	bne     2180 <find_self_save>
+
+00000000000020a0 <thread_restore_base>:
+    20a0:	39 00 01 e0 	li      r8,480
+    20a4:	7d 14 43 a6 	mtspr   276,r8
+    20a8:	7d a8 03 a6 	mtlr    r13
+    20ac:	4e 80 00 21 	blrl
+
+00000000000020b0 <thread_restore_return>:
+    20b0:	2c 06 00 00 	cmpwi   r6,0
+    20b4:	40 82 00 1c 	bne     20d0 <restore_done>
+
+00000000000020b8 <wait_until_single_thread>:
+    20b8:	7d 15 42 a6 	mfspr   r8,277
+    20bc:	79 08 e7 22 	rldicl  r8,r8,60,60
+    20c0:	2c 08 00 07 	cmpwi   r8,7
+    20c4:	40 82 ff f4 	bne     20b8 <wait_until_single_thread>
+    20c8:	7d 88 03 a6 	mtlr    r12
+    20cc:	4e 80 00 21 	blrl
+
+00000000000020d0 <restore_done>:
+    20d0:	39 00 ff fe 	li      r8,-2
+    20d4:	7e b5 40 39 	and.    r21,r21,r8
+    20d8:	28 17 00 00 	cmplwi  r23,0
+    20dc:	41 82 00 64 	beq     2140 <initiate_hv_compatibility_mode>
+    20e0:	7a 51 e7 e3 	rldicl. r17,r18,60,63
+    20e4:	41 82 00 7c 	beq     2160 <initiate_urmor_restore>
+
+00000000000020e8 <initiate_runtime_hv_wakeup>:
+    20e8:	39 00 00 00 	li      r8,0
+    20ec:	7e 3b 02 a6 	mfsrr1  r17
+    20f0:	79 11 b2 6c 	rldimi  r17,r8,22,41
+    20f4:	7e 3b 03 a6 	mtsrr1  r17
+    20f8:	7e 3b 4a a6 	mfspr   r17,315
+    20fc:	79 11 b2 6c 	rldimi  r17,r8,22,41
+    2100:	7e 3b 4b a6 	mtspr   315,r17
+    2104:	7e 1f 7a a6 	mfspr   r16,511
+    2108:	79 10 f8 0e 	rldimi  r16,r8,63,0
+    210c:	79 15 b2 6c 	rldimi  r21,r8,22,41
+    2110:	39 14 12 00 	addi    r8,r20,4608
+    2114:	28 06 00 00 	cmplwi  r6,0
+    2118:	41 82 00 08 	beq     2120 <update_usrrx>
+    211c:	39 08 00 04 	addi    r8,r8,4
+
+0000000000002120 <update_usrrx>:
+    2120:	7d 1a 7b a6 	mtspr   506,r8
+    2124:	7e bb 7b a6 	mtspr   507,r21
+    2128:	2c 06 00 00 	cmpwi   r6,0
+    212c:	40 82 00 08 	bne     2134 <exit_to_thread_stop>
+    2130:	7e 1f 7b a6 	mtspr   511,r16
+
+0000000000002134 <exit_to_thread_stop>:
+    2134:	4c 00 01 2c 	isync
+    2138:	4c 00 02 64 	.long 0x4c000264
+    213c:	00 00 02 00 	attn
+
+0000000000002140 <initiate_hv_compatibility_mode>:
+    2140:	39 14 12 00 	addi    r8,r20,4608
+    2144:	28 06 00 00 	cmplwi  r6,0
+    2148:	41 82 00 08 	beq     2150 <update_srrx>
+    214c:	39 08 00 04 	addi    r8,r8,4
+
+0000000000002150 <update_srrx>:
+    2150:	7d 1a 03 a6 	mtsrr0  r8
+    2154:	7e bb 03 a6 	mtsrr1  r21
+    2158:	4c 00 00 24 	rfid
+    215c:	00 00 02 00 	attn
+
+0000000000002160 <initiate_urmor_restore>:
+    2160:	39 14 12 80 	addi    r8,r20,4736
+    2164:	28 06 00 00 	cmplwi  r6,0
+    2168:	41 82 00 08 	beq     2170 <update_uv_exit>
+    216c:	39 08 00 08 	addi    r8,r8,8
+
+0000000000002170 <update_uv_exit>:
+    2170:	7d 1a 7b a6 	mtspr   506,r8
+    2174:	7e bb 7b a6 	mtspr   507,r21
+    2178:	4c 00 02 64 	.long 0x4c000264
+    217c:	00 00 02 00 	attn
+
+0000000000002180 <find_self_save>:
+    2180:	3b ed 00 08 	addi    r31,r13,8
+    2184:	7d c8 03 a6 	mtlr    r14
+    2188:	4e 80 00 21 	blrl
+    218c:	2c 06 00 00 	cmpwi   r6,0
+    2190:	40 82 f0 74 	bne     1204 <save_restore_done>
+
+0000000000002194 <save_core_spr>:
+    2194:	3b ec 00 08 	addi    r31,r12,8
+    2198:	7d e8 03 a6 	mtlr    r15
+    219c:	4e 80 00 21 	blrl
+    21a0:	4b ff f0 64 	b       1204 <save_restore_done>
+	...
+    2300:	38 00 00 00 	li      r0,0
+    2304:	64 00 60 00 	oris    r0,r0,24576
+    2308:	90 1f ff fc 	stw     r0,-4(r31)
+    230c:	78 20 84 20 	rldicl  r0,r1,16,48
+    2310:	64 00 64 00 	oris    r0,r0,25600
+    2314:	b4 1f 00 02 	sthu    r0,2(r31)
+    2318:	78 20 04 22 	rldicl  r0,r1,32,48
+    231c:	b4 1f 00 04 	sthu    r0,4(r31)
+    2320:	78 20 84 22 	rldicl  r0,r1,48,48
+    2324:	b4 1f 00 08 	sthu    r0,8(r31)
+    2328:	78 20 04 20 	clrldi  r0,r1,48
+    232c:	b4 1f 00 04 	sthu    r0,4(r31)
+    2330:	3b ff 00 0e 	addi    r31,r31,14
+    2334:	4e 80 00 20 	blr
diff --git a/import/chips/p9/procedures/utils/stopreg/selfRest.map b/import/chips/p9/procedures/utils/stopreg/selfRest.map
index 21a7f70b..9f76c54b 100644
--- a/import/chips/p9/procedures/utils/stopreg/selfRest.map
+++ b/import/chips/p9/procedures/utils/stopreg/selfRest.map
@@ -2,7 +2,7 @@
 Memory Configuration
 
 Name             Origin             Length             Attributes
-selfRestoreRegion 0x0000000000000000 0x0000000000002100 xr
+selfRestoreRegion 0x0000000000000000 0x0000000000002400 xr
 *default*        0x0000000000000000 0xffffffffffffffff
 
 Linker script and memory map
@@ -11,18 +11,18 @@ Linker script and memory map
                 0x0000000000000000                . = ALIGN (0x80)
                 0x0000000000000000                _SELF_RESTORE_START = .
 
-.selfRestore    0x0000000000000000     0x20b0
+.selfRestore    0x0000000000000000     0x2338
  *(.selfRestore)
- .selfRestore   0x0000000000000000     0x20b0 /gsa/ausgsa/projects/i/indiateam01/premjha2/ekb/chips/p9/procedures/utils/stopreg/p9_core_restore_routines.o
+ .selfRestore   0x0000000000000000     0x2338 /esw/san2/premjha2/fapi2_lite/ekb/chips/p9/procedures/utils/stopreg/p9_core_save_restore_routines.o
                 0x0000000000000000                _start
-LOAD /gsa/ausgsa/projects/i/indiateam01/premjha2/ekb/chips/p9/procedures/utils/stopreg/p9_core_restore_routines.o
-OUTPUT(/gsa/ausgsa/projects/i/indiateam01/premjha2/ekb/chips/p9/procedures/utils/stopreg/selfRest.bin binary)
+LOAD /esw/san2/premjha2/fapi2_lite/ekb/chips/p9/procedures/utils/stopreg/p9_core_save_restore_routines.o
+OUTPUT(/esw/san2/premjha2/fapi2_lite/ekb/chips/p9/procedures/utils/stopreg/selfRest.bin binary)
 
-.text           0x00000000000020b0        0x0
- .text          0x00000000000020b0        0x0 /gsa/ausgsa/projects/i/indiateam01/premjha2/ekb/chips/p9/procedures/utils/stopreg/p9_core_restore_routines.o
+.text           0x0000000000002338        0x0
+ .text          0x0000000000002338        0x0 /esw/san2/premjha2/fapi2_lite/ekb/chips/p9/procedures/utils/stopreg/p9_core_save_restore_routines.o
 
-.data           0x00000000000020b0        0x0
- .data          0x00000000000020b0        0x0 /gsa/ausgsa/projects/i/indiateam01/premjha2/ekb/chips/p9/procedures/utils/stopreg/p9_core_restore_routines.o
+.data           0x0000000000002338        0x0
+ .data          0x0000000000002338        0x0 /esw/san2/premjha2/fapi2_lite/ekb/chips/p9/procedures/utils/stopreg/p9_core_save_restore_routines.o
 
-.bss            0x00000000000020b0        0x0
- .bss           0x00000000000020b0        0x0 /gsa/ausgsa/projects/i/indiateam01/premjha2/ekb/chips/p9/procedures/utils/stopreg/p9_core_restore_routines.o
+.bss            0x0000000000002338        0x0
+ .bss           0x0000000000002338        0x0 /esw/san2/premjha2/fapi2_lite/ekb/chips/p9/procedures/utils/stopreg/p9_core_save_restore_routines.o