Sbe Compression Decompression

Change-Id: I8eb691ba7b28a4c7347040d3da5c16c01d5b9697
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/39225
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Reviewed-by: Sachin Gupta <sgupta2m@in.ibm.com>

6 files changed, 317 insertions, 17 deletions

diff --git a/src/boot/loader_l2.c b/src/boot/loader_l2.c
index 003a2af6..1bdc0828 100644
--- a/src/boot/loader_l2.c
+++ b/src/boot/loader_l2.c
@@ -23,20 +23,24 @@
 /*                                                                        */
 /* IBM_PROLOG_END_TAG                                                     */
 #include "sbeXipUtils.H"
+#include "sbeDecompression.h"
 
 // Load section to destination address
 int32_t loadSection( P9XipSection * i_section, uint64_t *i_destAddr )
 {
-    uint32_t idx = 0;
-    uint64_t *seepromAddr = (uint64_t *)( g_headerAddr + i_section->iv_offset);
-    uint32_t sectionSize  = i_section->iv_size;
-    for( idx = 0; idx < sectionSize; idx += 8 )
-    {
-        *i_destAddr = *seepromAddr;
-        i_destAddr++; seepromAddr++;
+    uint32_t rc = 0;
+    do {
 
-    }
-    return 0;
+         uint8_t *seepromAddr = (uint8_t *)( g_headerAddr + i_section->iv_offset);
+
+         uint8_t rc = decompress(seepromAddr, (uint8_t *)i_destAddr);
+
+         if (rc != 0 )
+           break;
+ 
+       } while(0);
+
+    return rc;
 }
 
 // Function to load base image into PIBMEM
diff --git a/src/boot/sbeCompression.py b/src/boot/sbeCompression.py
new file mode 100755
index 00000000..a9ac9a23
--- /dev/null
+++ b/src/boot/sbeCompression.py
@@ -0,0 +1,174 @@
+#!/usr/bin/python
+# IBM_PROLOG_BEGIN_TAG
+# This is an automatically generated prolog.
+#
+# $Source: src/boot/sbeCompression.py $
+#
+# OpenPOWER sbe Project
+#
+# Contributors Listed Below - COPYRIGHT 2017
+# [+] International Business Machines Corp.
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+#
+# IBM_PROLOG_END_TAG
+import os
+import subprocess
+import re
+import random
+import sys
+import binascii
+import fileinput
+import argparse
+import struct
+import operator
+err = False
+
+def compress(inputFile, compressedFile):
+
+    try:
+      f = open(inputFile, "rb")
+    except IOError as e :
+      print "I/O error File for File to be compressed."
+      sys.exit()
+
+    try:
+      fW = open(compressedFile, "wb")
+    except IOError as e :
+      print "I/O error File for compressed file."
+      sys.exit()
+
+    if os.stat(inputFile).st_size < 4 :
+      print "File is less than four bytes."
+      sys.exit()
+
+    instDict = dict()
+    for i in range(0, os.stat(inputFile).st_size / 4 ):
+
+       fourByt = f.read(4)
+
+       if fourByt in instDict:
+
+          iCount = instDict[fourByt]
+          instDict[fourByt] = iCount + 1;
+
+       else :
+
+          iCount = 1
+          instDict[fourByt] = iCount
+
+    sortedList =  sorted(instDict.iteritems(), key=operator.itemgetter(1), reverse = True)
+
+    sortedList[256:] = []
+    instList = []
+    iCount = 0
+    for k, v in sortedList:
+      instList.append(k)
+
+    for x in instList:
+       fW.write(x)
+
+    fileSize = os.stat(inputFile).st_size
+    fW.write(struct.pack(">Q",fileSize))
+
+    f.seek(0, 0)
+    strBits = ""
+    count = 0
+
+    #Create a bitmap for each four bytes of binary. 
+    for i in range(0, os.stat(inputFile).st_size / 4 ):
+
+       fourByt = f.read(4)
+       if fourByt in instList:
+          strBits += '1'
+
+       else :
+          strBits += '0'
+
+       if ((len(strBits) == 32) or (i == (os.stat(inputFile).st_size / 4) - 1)):
+          value = int(strBits, 2)
+          fW.write(struct.pack('>I', value))
+          strBits = ""
+          count = count + 1
+
+    value = 0
+    #To make the bit map eight byte alligned in compressed image.
+    if ((count % 2) == 0):
+      padCount = count
+    else:
+      padCount = count + 1
+
+    for i in range(count, padCount):
+      fW.write(struct.pack('>I', value))
+
+    f.seek(0, 0)
+
+    for i in range(0, os.stat(inputFile).st_size / 4 ):
+
+       fourByt = f.read(4)
+
+       if fourByt in instList:
+          ind = instList.index(fourByt)
+          fW.write(struct.pack('>B', ind))
+
+       else:
+          fW.write(fourByt)
+
+    f.close()
+    fW.close()
+
+def main( argv ):
+
+    parser = argparse.ArgumentParser( description = "SBE Compression Parser" )
+    parser.add_argument( '-l', '--imageLoc', type=str, help = 'Seeprom Binary Location' )
+    parser.add_argument( '-i', '--image', type=str, help = 'Seeprom Binary ' )
+
+    args = parser.parse_args()
+    imagePath = args.imageLoc
+    image = args.image
+
+    #Make a copy of SEEPROM binary.
+    cmd1 = "cp " + imagePath + "/" + image + " " + imagePath +  "/" + image + ".orig"
+    rc = os.system(cmd1)
+    if rc:
+      print "Unable to make copy of seeprom binary"
+      sys.exit()
+
+    #Extract base from SEEPROM binary.
+    cmd2 = imagePath + "/p9_xip_tool " + imagePath + "/" + image + " extract .base " + imagePath + "/" + image + ".base"
+    rc = os.system(cmd2)
+    if rc:
+      print "Unable to extract the base from seeprom binary"
+      sys.exit()
+
+    #Compress the base section
+    compress(imagePath + "/" + image + ".base", imagePath + "/" + image + ".base.compressed")
+
+    #Delete the base section from SEEPEOM binary.
+    cmd3 = imagePath + "/p9_xip_tool " + imagePath + "/" + image + " delete .base"
+    rc = os.system(cmd3)
+    if rc:
+      print "Unable to delete base section from seeprom binary"
+      sys.exit()
+
+    #Append the base section from SEEPEOM binary.
+    cmd4 = imagePath + "/p9_xip_tool " + imagePath +  "/" + image + " append .base " + imagePath + "/" + image + ".base.compressed"
+    rc = os.system(cmd4)
+    if rc:
+      print "Unable to append the base section"
+      sys.exit()
+
+if __name__ == "__main__":
+    main( sys.argv )
+
diff --git a/src/boot/sbeDecompression.h b/src/boot/sbeDecompression.h
new file mode 100644
index 00000000..19030687
--- /dev/null
+++ b/src/boot/sbeDecompression.h
@@ -0,0 +1,120 @@
+/* IBM_PROLOG_BEGIN_TAG                                                   */
+/* This is an automatically generated prolog.                             */
+/*                                                                        */
+/* $Source: src/boot/sbeDecompression.h $                                 */
+/*                                                                        */
+/* OpenPOWER sbe Project                                                  */
+/*                                                                        */
+/* Contributors Listed Below - COPYRIGHT 2017                             */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
+/*                                                                        */
+/* Licensed under the Apache License, Version 2.0 (the "License");        */
+/* you may not use this file except in compliance with the License.       */
+/* You may obtain a copy of the License at                                */
+/*                                                                        */
+/*     http://www.apache.org/licenses/LICENSE-2.0                         */
+/*                                                                        */
+/* Unless required by applicable law or agreed to in writing, software    */
+/* distributed under the License is distributed on an "AS IS" BASIS,      */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or        */
+/* implied. See the License for the specific language governing           */
+/* permissions and limitations under the License.                         */
+/*                                                                        */
+/* IBM_PROLOG_END_TAG                                                     */
+
+uint8_t decompress(uint8_t * compBuffer, uint8_t * decompBuffer)
+{
+    uint8_t rc = 0;
+    do
+    {
+        uint32_t iCount = 0;
+        int32_t jCount = 0;
+
+        uint32_t dict[256];
+        for(iCount = 0; iCount < 256; iCount++)
+        {
+           uint64_t temp = *(uint64_t *)compBuffer;
+           dict[iCount] = (uint32_t)(((temp) & 0xFFFFFFFF00000000LL) >> 32);
+           ++iCount;
+           dict[iCount] = (uint32_t)(((temp) & 0xFFFFFFFFLL));
+           compBuffer = compBuffer + 8;
+        }
+
+        //Get the file size.
+        uint64_t compfileSize = *((uint64_t * )compBuffer);
+        compBuffer = compBuffer + 8;
+        uint32_t quo = compfileSize / 128;
+        uint32_t rem = compfileSize % (128);
+        uint32_t bitCount;
+        if(rem == 0)
+            bitCount = quo;
+        else
+            bitCount = quo + 1;
+
+        uint32_t bitArray[bitCount];
+
+        for(iCount = 0; iCount < bitCount; iCount++)
+        {
+            uint64_t temp = *(uint64_t *)compBuffer;
+            bitArray[iCount] = (uint32_t)(((temp) & 0xFFFFFFFF00000000LL) >> 32);
+            ++iCount;
+            bitArray[iCount] = (uint32_t)((temp) & 0xFFFFFFFFLL);
+            compBuffer = compBuffer + 8;
+        }
+
+        uint64_t eightByte = *(uint64_t *)compBuffer;
+        uint8_t *ptr = (uint8_t *)&eightByte;
+        compBuffer = compBuffer + 8;
+        uint32_t kCount = 0;
+
+        for(iCount = 0; iCount < bitCount ; iCount++)
+        {
+            //Extract a bit from 32 bit integer.
+            int32_t j = 31;
+
+            if((iCount == bitCount - 1) && (rem != 0))
+                j = rem/4 -1;
+
+            for (jCount = j; jCount >= 0; jCount--)
+            {
+                char isCompressed = (bitArray[iCount] >> jCount) & 1;
+                if(isCompressed == 1)
+                {
+                    //Read a byte.
+                    uint8_t index = *(ptr + kCount);
+                    ++kCount;
+                    if(kCount == 8)
+                    {
+                        eightByte = *(uint64_t *)compBuffer;
+                        compBuffer = compBuffer + 8;
+                        kCount = 0;
+                    }
+                    uint32_t value = dict[index];
+	            uint32_t * pTemp = (uint32_t *) decompBuffer;
+                    *pTemp = value; 
+                    decompBuffer = decompBuffer + 4;
+                }
+                else
+                {
+                    //Read four bytes.
+                    uint8_t i;
+                    for(i = 0; i < 4; i++)
+                    {
+                        *decompBuffer = *(ptr + kCount);
+                        ++kCount;
+                        if(kCount == 8)
+                        {
+                            eightByte = *(uint64_t *)compBuffer;
+                            compBuffer = compBuffer + 8;
+                            kCount = 0;
+                        }
+                        ++decompBuffer;
+                    }
+                }// else
+            }// loop ends for (jCount = j; jCount >= 0; jCount--)
+        }//loops ends for (iCount = 0; iCount < bitCount ; iCount++)
+    } while(0);
+   return rc;
+}
+
diff --git a/src/build/Makefile b/src/build/Makefile
index f87c3aae..d798fab2 100644
--- a/src/build/Makefile
+++ b/src/build/Makefile
@@ -169,7 +169,7 @@ all: $(OBJDIR) xml \
 endif
 ifeq ($(img), seeprom)
 all: $(OBJDIR) xml $(IMG_DIR)/$(IMAGE_SEEPROM_NAME).bin \
-	 $(SBE_TOOLS) $(SBE_IPL_TOOLS) normalize defaultset $(IMG_DIR)/fixed.bin \
+	 $(SBE_TOOLS) $(SBE_IPL_TOOLS) normalize compress defaultset $(IMG_DIR)/fixed.bin \
 	 appendloader appendoverrides add_LoaderAddr symbols tracehash buildinfo \
 	 report
 endif
@@ -192,7 +192,7 @@ tar:
 	@rm -rf $(TAR_OBJDIR)
 	@echo "Generated simics.tar in Sbe images Directory"
 
-.PHONY: all normalize defaultset appendloader add_LoaderAddr symbols report \
+.PHONY: all normalize compress defaultset appendloader add_LoaderAddr symbols report \
 		appendoverrides xml tracehash topfixedheaders  $(SUBDIRS) \
 		tar install
 
@@ -279,6 +279,9 @@ $(P9_XIP_TOOL):
 normalize: $(P9_XIP_TOOL) $(IMG_DIR)/$(IMAGE_SEEPROM_NAME).bin
 	$(P9_XIP_TOOL) $(IMG_DIR)/$(IMAGE_SEEPROM_NAME).bin normalize
 
+compress:
+	$(BOOT_SRCDIR)/sbeCompression.py -l $(IMG_DIR) -i $(IMAGE_SEEPROM_NAME).bin
+
 defaultset:$(SBE_TOOLS) $(IMG_DIR)/$(IMAGE_SEEPROM_NAME).bin normalize
 	$(TOOLS_ATTR_DIR)/ppeSetFixed.pl $(IMG_DIR) $(IMG_DIR)/$(IMAGE_SEEPROM_NAME).bin $(IMPORT_XML_DIR)/attribute_info/p9_sbe_attributes.xml $(ATTRFILES)
 
diff --git a/src/build/linkerscripts/linkloader.cmd b/src/build/linkerscripts/linkloader.cmd
index 1e2a20e3..dec33189 100755
--- a/src/build/linkerscripts/linkloader.cmd
+++ b/src/build/linkerscripts/linkloader.cmd
@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER sbe Project                                                  */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2015,2016                        */
+/* Contributors Listed Below - COPYRIGHT 2015,2017                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -27,7 +27,7 @@
 #undef powerpc
 
 #ifndef BASE_LOADER_STACK_SIZE
-#define BASE_LOADER_STACK_SIZE 128
+#define BASE_LOADER_STACK_SIZE 6144
 #endif
 #include "sbe_link.H"
 
@@ -88,8 +88,7 @@ SECTIONS
    .rwdata . : { *(.data) *(.bss) } > sram
 
    _BASE_LOADER_STACK_LIMIT = .;
-   . = . + BASE_LOADER_STACK_SIZE;
-   _BASE_LOADER_STACK_LIMIT = . - 1;
+   _BASE_LOADER_STACK_LIMIT = . + BASE_LOADER_STACK_SIZE - 1;
 
     . = ALIGN(8);
     _loader_end = . - 0;
diff --git a/src/build/utils/sbe_link.H b/src/build/utils/sbe_link.H
index 808e5207..c2c9623b 100644
--- a/src/build/utils/sbe_link.H
+++ b/src/build/utils/sbe_link.H
@@ -79,9 +79,9 @@
 #define SBE_LOADER_BASE_SECTION  SBE_SEEPROM_BASE_ORIGIN + SBE_XIP_TOC_OFFSET \
                                   + 120
 // Base Loader start address
-#define SBE_LOADER_BASE_ORIGIN 0xFFFFFE00
+#define SBE_LOADER_BASE_ORIGIN 0xFFFFE400
 // Base Loader length
-#define SBE_LOADER_BASE_LENGTH 0x200
+#define SBE_LOADER_BASE_LENGTH 0x1C00
 
 // Base Loader entry function offset in header
 #define SBE_LOADER_ENTRY_HEADER_OFFSET 20