diff options
author | spashabk-in <shakeebbk@in.ibm.com> | 2017-02-02 01:10:33 -0600 |
---|---|---|
committer | Sachin Gupta <sgupta2m@in.ibm.com> | 2017-02-24 21:10:00 -0500 |
commit | 7088a779f43b2b3c1b09be0e5fb8adcfdb805fe5 (patch) | |
tree | a73214250816bf891c73883a21a6532c39d6d22a /src/tools/models | |
parent | 36ef073ffc88c8bf2fe179f9f51914c7d8164ee5 (diff) | |
download | talos-sbe-7088a779f43b2b3c1b09be0e5fb8adcfdb805fe5.tar.gz talos-sbe-7088a779f43b2b3c1b09be0e5fb8adcfdb805fe5.zip |
SBE model build tool
Change-Id: I5ed406f7b6adc299f4c8b9242eaa5aef34e3dcb4
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/35775
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Sachin Gupta <sgupta2m@in.ibm.com>
Diffstat (limited to 'src/tools/models')
-rwxr-xr-x | src/tools/models/buildtool/README | 294 | ||||
-rwxr-xr-x | src/tools/models/buildtool/buildSBEImage.py | 634 | ||||
-rwxr-xr-x | src/tools/models/buildtool/imageBuildCommand | 3 | ||||
-rwxr-xr-x | src/tools/models/buildtool/ppeModel.py | 484 | ||||
-rwxr-xr-x | src/tools/models/buildtool/ppeUtility.py | 240 | ||||
-rwxr-xr-x | src/tools/models/buildtool/sbeConstants.py | 101 | ||||
-rwxr-xr-x | src/tools/models/buildtool/sbeContinue.py | 381 | ||||
-rwxr-xr-x | src/tools/models/buildtool/sbeGitUtility.py | 220 | ||||
-rwxr-xr-x | src/tools/models/buildtool/sbeUtility.py | 681 |
9 files changed, 3038 insertions, 0 deletions
diff --git a/src/tools/models/buildtool/README b/src/tools/models/buildtool/README new file mode 100755 index 00000000..a7900446 --- /dev/null +++ b/src/tools/models/buildtool/README @@ -0,0 +1,294 @@ +CONTENTS OF THIS FILE +--------------------- + + * Introduction + * Requirements + * Recommended modules + * Installation + * Configuration + * Troubleshooting + * Maintainers + + +INTRODUCTION +------------ + + This Model tool is designed to take the user supplied inputs such as Model name, commit ID, path where the image to be build and compilation mode + and build the corresponding model SBE image of EKB and verify the image if needed using CI build script methods. + + By default Model name MUST be supplied to the tool as an argument, while rest arguments are optional. + + The tool is designed to take hit due to build break. The build work flow is split into three separate workflow. + Thus checkpoints are maintained to re-start for last known good checkpoint of the build run. + + First part intializing and setting up the sandboxes, Repo and other logs. + + Second part validating the PPE models. + + Third part does the compiling and triggering CI. In case the build break due to ERRORS, a build ACTION file is generated. + This build Action file is used to continue skipping the first part of intialization and directly starts building it with patches if needed. + + + ------------------------------------------------------------------------------------------------------------- + :: Command line USAGE options for building FW SBE image :: + + buildSBEImage.py -m <model_name> -c < Commit log id> -i <abc,xyz> -p < PPE build path> -b <Auto|Manual|Force> + + + +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + | NOTE: Please use VALID input string since validation in code is a tideous task. | + | By default user MUST pass a VALID Model ID. | + +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + ***** Options Supported ***** + -m,--model = Target Model to be used to build the image + -c,--commit = Base commit id of the patch to be checkout + -i,--cherry-pick = refs id of the patch to be cherry pick + -p,--path = Path where user wants the image to be built + -b,--build = Default is Auto, user can pass Manual/Force + Auto : CI build and test + Manual : Only compiles No Simics run + Force : Rebuild Repo + -e,--env = User ENV build exports + + ***** Allowed options example ***** + [ Only Model as an input ] + buildSBEImage.py -m n10_e9027_tp026_soa_sc_u285_01 + + + [ Model and PPE commit id as an input ] + buildSBEImage.py -m n10_e9027_tp026_soa_sc_u285_01 -c c1980f5a4de0bac8922260ed827fe37a124b2b0a + + + [ Model and PPE commit id and cherry pick refs as an input ] + [ ** Coma ',' separated cherry pick patch refs **] + buildSBEImage.py -m n10_e9027_tp026_soa_sc_u285_01 -c c1980f5a4de0bac8922260ed827fe37a124b2b0a -i refs/changes/57/21157/3, refs/changes/35/22535/2 + + + [ Model, PPE commit id and PPE image build path as an input ] + buildSBEImage.py -m n10_e9027_tp026_soa_sc_u285_01 -c c1980f5a4de0bac8922260ed827fe37a124b2b0a -p <user path> + + + [ Choosing to compile ONLY option - By Default it run in Auto mode and trigger CI build and simics run test ] + buildSBEImage.py -m n10_e9027_tp026_soa_sc_u285_01 -b Manual + + + [ Choosing to do fresh Force build ] + buildSBEImage.py -m n10_e9027_tp026_soa_sc_u285_01 -b Force + + [ Facility to -ONLY- Build PPE with ENV passed and run simcis without EKB models ] + buildSBEImage.py -m <model> -c b85fe14bd0c86ff223ac15e261ead4877949084c -i refs/changes/57/21157/3,refs/changes/35/22535/2 -e '__FAPI_DELAY_SIM__=1' + ------------------------------------------------------------------------------------------------------------- + + + [ Model Tool work flow layout ] + ------------------------------ + + ************************************************************************************************************** + [START][ Main module ] <Model Name> < base Commit > <Patches> <Build Path> <Build Mode> + | | | | | | + | | | | | | + buildSBEImage.py -m XXXXXXXX -c YYYYYYYYYYY -i refs/xxx,refs/yyy -p /home/somelocation/ -b Manual/Auto/Force + | + | + [ Pre-req's & Cleanup stale data ] <- ** -> [ Applies by-pass action if pending from previous build failure run] + | | + | | + [ Create Working Sandbox/Repo for EKB/PPE ] <---- uses Library utility | + | | + \ [ Skips & Jumps ahead ] + \ | + sbeGitUtility.py <-- uses sbeConstants.py for constants paths & codes | + | | + | | + [ PPE specific Model check, patches cherry pick ] <-- ppeModel.py | + | | + | | + | / + | / + [ Setup hooks to EKB env.bash ] <-- sbeContinue.py <------------------------/ + | + | + [ Loads the ENV and workon EKB ] + | + \ + Triggers [ sbeContinue.py ] <-- Uses Library utility + | + | + [ Loads sbeGitUtility.py ] <-- To validate Models and comits + | + | + [ Model EKB checkout ] + | + | + [ Stage files for Compilation ] <-- Copy EKB to PPE + | + | + ------------- + | | + | | + [ Manual ] [ CI build ] + | | + | | + [ 'make all' ] [ loads ./src/test/framework/build-script ] + | | + | | + ------------------ + | + | + [ Returns the control to sbeContinue.py ] + | + | + [ Returns the control to buildSBEImage.py ] + | + | + [ Tool Exits ] + | + | + [ END ] + + ************************************************************************************************************** + + +REQUIREMENTS +------------ + + This need python version 2.6 or above to work. + The user is assumed to have access tokens and Gerrit setup in the configuration for EKB and PPE and Sandbox. + + +RECOMMENDED MODULES +------------------- + + This uses in background the EKB mirroring tool and compilers from scl devtoolset to switch between the git Repo during compilation. + +INSTALLATION +------------ + + This has no installable needed by the tool. Checkout the code from production repo for latest updates. + + +CONFIGURATION +------------- + + Permission access location directory to where the image needs to be build. + + Access tokens needed before launching this tool. + Example if you type 'tokens', at minimal, it must have these AFS Austin and AWD tokens + + Tokens held by the Cache Manager: + + User's (AFS ID 69515) tokens for afs@rchland.ibm.com [Expires Dec 8 23:32] + User's (AFS ID 21061) tokens for afs@awd.austin.ibm.com [Expires Dec 8 23:32] + User's (AFS ID 21060) tokens for afs@austin.ibm.com [Expires Dec 8 23:32] + --End of list-- + + Configuration in SSH must be setup for gerrit access. Example + By default : "gerrit-server" is needed + + $ cat ~/.ssh/config + Host hw.gerrit + Hostname gfw160.aus.stglabs.ibm.com + Port 29418 + AFSTokenPassing no + Host gerrit-server + Hostname gfw160.aus.stglabs.ibm.com + Port 29418 + AFSTokenPassing no + Host fsp.gerrit + Hostname fspbld06.in.ibm.com + Port 29418 + AFSTokenPassing no + Host hostboot.gerrit + Hostname gfw160.aus.stglabs.ibm.com + Port 29418 + AFSTokenPassing no + + + !! NOTE : Host hw.gerrit order has to be first in config in case you are using this tool, since the EKB model env needs its that way. !! + To confirm ./workon ekb + echo $GERRIT_SRV + hw.gerrit + $ + + +TROUBLESHOOTING +--------------- + + It is observed during a CI build which in turns starts the NIMBUS simics sometimes failed to start if simic is ran for the first time. + + It is advised to start the simics manaually once and acccept the WindRiver license prompted on the simics console. Post that re-run the + tool and it will allow your simics to be started by the script onwards. + + The error codes are uniquely encoded and displayed when failed. Refer the RC code for identifying the issues. + + At glance these are the RC code setup up and subject to changes and update. + + ''' + Generic RC code + ''' + SUCCESS_EXIT = 0 + SHELL_EXIT = 1 + HELP_EXIT = 10 + NEW_MODEL_EXIT = 11 + NO_PENDING_MODEL_EXIT = 12 + VALIDATE_EXIT = 13 + FILES_EXIST = 14 + DIR_EMPTY = 15 + + BYPASS_EXIT = 20 + PATCH_IGNORE_EXIT = 21 + PPE_MODEL_NOT_EXIST = 22 + PPE_MODEL_SKIP_EXIT = 23 + + EKB_MODEL_NOT_EXIST = 40 + + ERROR_SYS_ERROR_EXIT = 100 + + ERROR_PREP_GIT_FAILED = 1000 + ERROR_PREP_TOKEN_SETUP_FAILED = 1001 + ERROR_PREP_GIT_SETUP_FAILED = 1002 + + ERROR_PREP_COM_FAILED = 2000 + ERROR_PREP_CHANGE_FAILED = 2001 + ERROR_GIT_CHANGEID_FAILED = 2002 + + ERROR_LOAD_ENV = 5000 + ERROR_MODEL_NOT_EXIST = 256 # hacked RC code exit from bash + + # Build failures Manual or CI + BUILD_CI_FAILED = 10000 + BUILD_MANUAL_FAILED = 10001 + + Note : The user exception handling are not done as a part of this changes. For future enhancement. + + Important log files : Check under base ~/GIT_REPO_SB/logs/ + + ----------------- + buildActionHooks : + ----------------- + By default this file will be written for the build script to re-start skipping initialization. + This file is persistent if there is a failure else it will get cleared. + + ---------------------------------- + buildPPE_out_20151202-123355.log : + ---------------------------------- + This is has the date and time run of the build and good for debugging during failure. + This file is persistent until next run. + + ---------------------- + PATCH_ACTION_TEMPLATE: + ---------------------- + This files is generated on build or CI failure. Add extra step needed in the build step. + This is intended to be persistent per model or until cleared manaully. + + +MAINTAINERS +----------- + +Original Tool Designer/Developer: + * George Keishing (Hostboot) - http://faces.tap.ibm.com/bluepages/profile.html?email=gkeishin@in.ibm.com + +Current maintainers: + * Sachin Gupta ( SBE ) - http://faces.tap.ibm.com/bluepages/profile.html?email=sgupta2m@in.ibm.com + * Sunil Kumar ( SBE ) - http://faces.tap.ibm.com/bluepages/profile.html?email=skumar8j@in.ibm.com diff --git a/src/tools/models/buildtool/buildSBEImage.py b/src/tools/models/buildtool/buildSBEImage.py new file mode 100755 index 00000000..79fc34b6 --- /dev/null +++ b/src/tools/models/buildtool/buildSBEImage.py @@ -0,0 +1,634 @@ +#!/usr/bin/python +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/tools/models/buildtool/buildSBEImage.py $ +# +# OpenPOWER sbe Project +# +# Contributors Listed Below - COPYRIGHT 2017 +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG +############################################################## +# @file buildSBEImage.py +# @brief Main Module to support building SBE image +# +# @author George Keishing +# +# @date Nov 23, 2015 +# +# ---------------------------------------------------- +# @version Developer Date Description +# ---------------------------------------------------- +# 1.0 gkeishin 11/12/15 Initial create +############################################################## + +#------------------------- +# Imports +#------------------------- +import os, sys, glob +import getopt +import os.path +import subprocess +import time +import datetime +import threading +from os.path import expanduser # for getting $HOME PATH +# Local imports +import sbeUtility as sbeUtility +import sbeConstants as sbeConstants +#------------------------- + +#------------------------- +# Global Var for reference +#------------------------- + +#------------------------- +# Main Functions +#------------------------- +def main(): + + #----------------------- + # Start time tick tock + #----------------------- + start_time = datetime.datetime.now().replace(microsecond=0) + + #------------------------------------------ + # Usage tool option + #------------------------------------------ + def usage(): + print " ++++++++++++++++++++++++++++++++++++++++++++++++" + print " +++ Refer README file for more information +++" + print " ++++++++++++++++++++++++++++++++++++++++++++++++" + print " -------------------------------------------------------------------------------------------------------------" + print " :: Command line USAGE options for building FW SBE image :: \n" + print " buildSBEImage.py -m <model_name> -c < Commit log id> -i <abc,xyz> -p < PPE build path> -b <Auto|Manual|Force> -e <ENV exports>" + + print " \n" + print " +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" + print " | NOTE: Please use VALID input string since validation in code is a tideous task. |" + print " | By default user MUST pass a VALID Model ID. |" + print " +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" + print " \n ***** Options Supported *****" + print " \t -m,--model = Target Model to be used to build the image" + print " \t -c,--commit = Base commit id of the patch to be checkout" + print " \t -i,--cherry-pick = refs id of the patch to be cherry pick" + print " \t -p,--path = Path where user wants the image to be built" + print " \t -b,--build = Default is Auto, user can pass Manual/Force" + print " \t Auto : CI build and test" + print " \t Manual : Only compiles No Simics run" + print " \t Force : Rebuild Repo" + print " \t -e,--env = User ENV build exports" + print " \n ***** Allowed options example *****" + + print " [ Only Model as an input ]" + print " buildSBEImage.py -m n10_e9027_tp026_soa_sc_u285_01 " + print " \n" + print " [ Model and PPE commit id as an input ]" + print " buildSBEImage.py -m n10_e9027_tp026_soa_sc_u285_01 -c c1980f5a4de0bac8922260ed827fe37a124b2b0a" + print " \n" + print " [ Model and PPE commit id and cherry pick refs as an input ]" + print " [ ** Coma ',' separated cherry pick patch refs **]" + print " buildSBEImage.py -m n10_e9027_tp026_soa_sc_u285_01 -c c1980f5a4de0bac8922260ed827fe37a124b2b0a -i refs/changes/57/21157/3, refs/changes/35/22535/2" + print " \n" + print " [ Model, PPE commit id and PPE image build path as an input ] " + print " buildSBEImage.py -m n10_e9027_tp026_soa_sc_u285_01 -c c1980f5a4de0bac8922260ed827fe37a124b2b0a -p <user path>" + print " \n" + print " [ Choosing to compile ONLY option - By Default it run in Auto mode and trigger CI build and simics run test ]" + print " buildSBEImage.py -m n10_e9027_tp026_soa_sc_u285_01 -b Manual" + print " \n" + print " [ Choosing to do fresh Force build ]" + print " buildSBEImage.py -m n10_e9027_tp026_soa_sc_u285_01 -b Force" + print " \n" + print " [ Facility to -ONLY- Build PPE with ENV passed and run simcis without EKB models ]" + print " buildSBEImage.py -m <model> -c b85fe14bd0c86ff223ac15e261ead4877949084c -i refs/changes/57/21157/3,refs/changes/35/22535/2 -e '__FAPI_DELAY_SIM__=1'" + print " -------------------------------------------------------------------------------------------------------------" + exit_main(sbeConstants.HELP_EXIT) + + #------------------------------------------ + # Exit from this Main + #------------------------------------------ + def exit_main(rc): + if rc == sbeConstants.HELP_EXIT: + print "" + sys.exit(0) + + #------------------------------------- + # Calculate the time different and log + #------------------------------------- + end_time = datetime.datetime.now().replace(microsecond=0) + print "\n *********************************************************" + print " Build Time End : ", end_time + print " [ Total Time taken for build : %s ] " % ( end_time - start_time ) + print "\n *********************************************************" + + if rc: + print "\n [ ERROR - MAIN ] Exiting with error code = ", rc + sys.exit(rc) + else: + print " Build PPE image build process completed [ OK ] " + sys.exit(0) + + #------------------------------------------ + # Local var place name holder's + #------------------------------------------ + model_name = "None" + commit_name = "None" + patch_refs = "None" + build_type = "Force" + build_path = "None" + env_patch = "None" + + # UNCOMMENT WHEN IN PRODUCTION + #l_Log= expanduser("~") + '/GIT_REPO_SB/logs/console_out.log' + #print " Check log : %s" % l_Log + #orig_stdout = sys.stdout + #f = file(l_Log, 'w') + #sys.stdout = f + + #---------------------------- + # Read command line args + #---------------------------- + opts, args = getopt.getopt(sys.argv[1:],"m:c:i:p:b:e:h",['model=', 'commit=', 'cherry-pick=', 'path=', 'build=', 'env=', 'help']) + for opt, arg in opts: + if opt in ('-h', '--help'): + usage() + exit_main(sbeConstants.ERROR_SYS_ERROR_EXIT) + elif opt in ('-m', '--model'): + model_name = arg + elif opt in ('-c', '--commit'): + commit_name = arg + elif opt in ('-i', '--cherry-pick'): + patch_refs = arg + elif opt in ('-p', '--path'): + build_path = arg + elif opt in ('-e', '--env'): + env_patch = arg + elif opt in ('-b', '--build'): + build_type = arg + else: + usage() + exit_main(sbeConstants.ERROR_SYS_ERROR_EXIT) + + #---------------------------------- + # Preping the data for Image build + #---------------------------------- + print " \n" + print " ******************************************************" + print " ****** Preping workspace for PPE image Build *********" + print " ******************************************************" + + #------------------------------------------------------ + # Make sure that it has passed atleast one arg with it + #------------------------------------------------------ + if len(sys.argv)<2: + usage() + exit_main(sbeConstants.ERROR_SYS_ERROR_EXIT) + else: + if build_path == "None": + #--------------- + # Default paths + #--------------- + l_home_ekb_path= expanduser("~") + "/" +sbeConstants.GIT_EKB_REPO_SB + l_home_ppe_path= expanduser("~") + "/" +sbeConstants.GIT_PPE_REPO_SB + l_buildActionHooks = expanduser("~") + '/GIT_REPO_SB/logs/buildActionHooks' + '_' + model_name + l_RepoSuccess = expanduser("~") + '/GIT_REPO_SB/logs/sandboxSkip' + l_action_file = expanduser("~") + '/GIT_REPO_SB/logs/PATCH_ACTION_TEMPLATE' + '_' + model_name + l_log_dir = expanduser("~") + '/GIT_REPO_SB/logs/' + else: + #---------------------- + # User Specified paths + #--------------------- + l_home_ekb_path= build_path + "/" +sbeConstants.GIT_EKB_REPO_SB + l_home_ppe_path= build_path + "/" +sbeConstants.GIT_PPE_REPO_SB + l_buildActionHooks = build_path + '/GIT_REPO_SB/logs/buildActionHooks' + '_' + model_name + l_RepoSuccess = build_path + '/GIT_REPO_SB/logs/sandboxSkip' + l_action_file = build_path + '/GIT_REPO_SB/logs/PATCH_ACTION_TEMPLATE' + '_' + model_name + l_log_dir = build_path + '/GIT_REPO_SB/logs/' + + #------------------------------------------------------------- + # Local functions + #------------------------------------------------------------- + + #------------------------------------------------------------- + # Default main func + #------------------------------------------------------------- + def main(): + print "" + + # This is for future logging purpose.. keep it or re-implement + #logging.basicConfig(filename='test.log', level=logging.INFO) + # create logger + #logger = logging.getLogger('buildSBEImage_log') + #logger.setLevel(logging.DEBUG) + + # create console handler and set level to debug + #ch = logging.StreamHandler() + #ch.setLevel(logging.DEBUG) + + # create formatter + #formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + + # add formatter to ch + #ch.setFormatter(formatter) + + # add ch to logger + #logger.addHandler(ch) + + #------------------------------------------------------------- + # Check user inputs and display + #------------------------------------------------------------- + def input_setting(): + print " ---------------------------------------------------------------------------------" + print " [ Display User Inputs ]" + print " ---------------------------------------------------------------------------------" + print " [ Build Time Start ] \t =", start_time + if not model_name == "None": + print " [ Model ID ] model name\t = %s"%(model_name) + else: + print " [ Model ID ] This can't be empty if EKB model is needed " + print " Proceeding to build -ONLY- PPE image" + #exit_main(0) + + print " [ Optional ] commit ID\t = %s"%(commit_name) + print " [ Optional ] Patch ref\t = %s"%(patch_refs) + print " [ Optional ] Build path\t = %s"%(build_path) + print " [ Optional ] Build Type\t = %s"%(build_type) + print " [ Optional ] ENV PATCH\t = %s"%(env_patch) + print " ---------------------------------------------------------------------------------" + print " ---------------------------------------------------------------------------------" + print " [ ** These Files are generated on Failure/incomplete Build process ** ]" + print " ---------------------------------------------------------------------------------" + print " -> Patch Action File =",l_action_file + print " -> buildActionHooks =",l_buildActionHooks + print " -> Sandbox checkpoint=",l_RepoSuccess + print " ---------------------------------------------------------------------------------" + print " ---------------------------------------------------------------------------------" + print " [ Repo sandbox paths ] " + print " ---------------------------------------------------------------------------------" + print " -> EKB HOME path =", l_home_ekb_path + print " -> PPE HOME path =", l_home_ppe_path + print " ---------------------------------------------------------------------------------" + + #------------------------------------------------------------- + # Check pre-req setup requirements + #------------------------------------------------------------- + def user_setting(): + print "\n [ Checking Pre-requisite settings ]" + token_rc=sbeUtility.utilCheckTokens() + if token_rc: + print " One of the access AWD/Austin Token NOT found [ FAILED ]" + print " Please klog -c to awd and austn to get the access tokens" + exit_main(token_rc) + else: + print "\n Token access grant found [ OK ]\n" + + setup_rc=sbeUtility.utilCheckSetup() + if setup_rc: + print " Gerrit Setup Access for hw.gerrit Not found [ FAILED ]" + print " * Check the README section CONFIGURATION " + exit_main(setup_rc) + else: + print " Gerrit setup found [ OK ]" + + #------------------------------------------------------------- + # Clean up existing setup for fresh build from master + #------------------------------------------------------------- + def cleanup_build(): + if not os.path.exists(l_RepoSuccess): + print "\n [ Cleaning up working Sandboxes/Repo ]" + sbeUtility.utilCleanup(build_path) + print " Clean up sandbox Repo [ OK ]\n" + else: + print " Looks like there is failure post Repo create earlier" + + #------------------------------------------------------------- + # Build default sandboxes for PPE + #------------------------------------------------------------- + def sandbox_build_ppe(): + ret_ppe_code=sbeUtility.utilCreateSandbox_ppe(build_path) + if ret_ppe_code: + print " Creating ppe sandbox [ FAILED ] rc =%s" % ret_ppe_code + exit_main(ret_ppe_code) + else: + print " Creating ppe sandbox [ OK ]" + + #------------------------------------------------------------- + # Build default sandboxes for EKB + #------------------------------------------------------------- + def sandbox_build_ekb(): + ret_ekb_code=sbeUtility.utilCreateSandbox_ekb(build_path) + if ret_ekb_code: + print " Creating ekb sandbox [ FAILED ] rc= %s" % ret_ekb_code + exit_main(ret_ekb_code) + else: + print " Creating ekb sandbox [ OK ]" + + if build_path == "None": + print " Using Default path for EKB Repo" + else: + print " Creating Sandbox using user input path" + + # Sanbox create is good.. Ignore re create if it failed in this run + # This ensure that we can't keep creating repo each run for same model + if not os.path.exists(l_RepoSuccess): + open(l_RepoSuccess, 'w').close() + + #------------------------------------------------------------- + # Is the Saved off Hook file same as the user input + #------------------------------------------------------------- + def ValidateHookFile(): + print "\n [ Validate Action hooks Signature vs inputs ] " + ret_val_code = sbeUtility.utilValidateHookFile(l_buildActionHooks, model_name, commit_name, patch_refs, build_path, build_type, env_patch) + if ret_val_code == sbeConstants.VALIDATE_EXIT: + print " * Saved off action files signature doesn't match the user input" + return sbeConstants.VALIDATE_EXIT + else: + print " * Signature Match 1x1 with Action File saved off earlier" + return sbeConstants.SUCCESS_EXIT + + #------------------------------------------------------------- + # By-pass the main init process if the build failed latter + #------------------------------------------------------------- + def bypass_main(): + print "\n [ Check if by-pass is allowed for this run ]" + ret_bypass_code = sbeUtility.utilBypassCheck(l_buildActionHooks) + if ret_bypass_code == sbeConstants.BYPASS_EXIT: + # Wait don't be in hurry, it may also mean this model + # id is different so check if it has patch action + + print "\n [ Checking if this Model has PATCH action Pending ]" + ret_continue_code = ThisModelPatchExist(model_name) + if ret_continue_code == sbeConstants.NO_PENDING_MODEL_EXIT: + print " There is No Failure noticed in earlier build for this Model " + return sbeConstants.NO_PENDING_MODEL_EXIT + elif ret_continue_code == sbeConstants.NEW_MODEL_EXIT: + print " --> By-pass action pending.. but not for this model" + print " This is a new Model name user input, treating as new build" + return sbeConstants.NEW_MODEL_EXIT + else: + # Make sure the signature matches before loading it + ret_val_code = ValidateHookFile() + if ret_val_code == sbeConstants.VALIDATE_EXIT: + print " * Signature - NOT- matching to continue .. " + return sbeConstants.NEW_MODEL_EXIT + else: + # Load the action file + print " --> By-pass action pending.. Loading" + with open(l_buildActionHooks) as fp: + for line in fp: + # convert the single line entry to command + l_cmd = line.strip() + print "\n Executing : %s" % l_cmd + rc = os.system( l_cmd ) + if rc: + # Fix the things manually and compile + exit_main(rc) + else: + # getta out Exit from here ONLY + exit_main(sbeConstants.SUCCESS_EXIT) + else: + print" No by-pass action pending.. Continuing" + + #------------------------------------------------------------- + # validate PPE Model + #------------------------------------------------------------- + def ValidatePPEModel(): + # Call in PPE Model check code + # Find out from where the main script triggered, + # yank out the last word and replace with the hook script + print "\n [ Validating PPE Model base ]" + l_string=__file__ + l_string=l_string.rsplit('/', 1)[0] + l_string=l_string +'/ppeModel.py' + + ppe_cmd = "python " + l_string + " -m " + model_name + " -c " + commit_name + " -i " + patch_refs + " -p " + build_path + " -b " + build_type + " -e " + env_patch + print " Executing : ", ppe_cmd + + rc_ppe_model = os.system(ppe_cmd) + if not rc_ppe_model == sbeConstants.SUCCESS_EXIT: + print " * PPE build conflict.. Resolve Manually or Review Patches" + #print " * Switch build to Manual to skip strict checking - ONLY- if needed" + exit_main(rc_ppe_model) + else: + return sbeConstants.SUCCESS_EXIT + + #------------------------------------------------------------- + # Workon the EKB and then hook to the env.bash to load ENV + #------------------------------------------------------------- + def LoadEnv(): + print " --------------------------" + print " [ Loading EKB workon ENV ]" + print " --------------------------" + ret_env_load=sbeUtility.utilLoadEnv(model_name , commit_name, patch_refs, build_type, build_path,env_patch) + if ret_env_load == sbeConstants.ERROR_LOAD_ENV: + print " EKB ENV load [ FAILED ] rc= %s" % ret_env_load + exit_main(ret_env_load) + elif ret_env_load == sbeConstants.ERROR_MODEL_NOT_EXIST: + # Remove the action file since its a basic failure + # But check if its for sure ? since bash returns by default this RC + rc_ekb_code = sbeUtility.utilEkbModelExist(model_name, l_home_ekb_path ) + if rc_ekb_code == sbeConstants.EKB_MODEL_NOT_EXIST: + print " EKB Model [%s] Not Found [ FAILED ] rc= %s" % (model_name,rc_ekb_code) + os.remove(l_buildActionHooks) + else: + print " EKB Model [%s] Build [ FAILED ] rc= %s" % (model_name,ret_env_load) + + exit_main(ret_env_load) + else: + print " EKB ENV load and Exited [ OK ] " + + #------------------------------------------------------------- + # is Model patch action of this user input exist ? + #------------------------------------------------------------- + def ThisModelPatchExist(model_name): + # It is possible that the user just ran Model xxx and failed, + # and still wants to try another Model yyy + + # if the patch action exist, then continue else + # allow to do build and jump to corresponding checkpoint + if os.path.exists(l_action_file): + if os.path.exists(l_buildActionHooks): + return sbeConstants.SUCCESS_EXIT + else: + return sbeConstants.NEW_MODEL_EXIT + else: + # This is an indication this is new + return sbeConstants.NO_PENDING_MODEL_EXIT + + #------------------------------------------------------------- + # Cleanup on exit + #------------------------------------------------------------- + def cleanupOnExit(): + if os.path.exists(l_RepoSuccess): + print " Removing Sandbox create log = %s" % l_RepoSuccess + os.remove(l_RepoSuccess) + + # PPE base file + l_base_file = l_log_dir +"/base" + if os.path.exists(l_base_file): + print " Removing PPE base file= %s" % l_base_file + os.remove(l_base_file) + + # Use glob method to remove files starting with same name + if build_path == "None": + l_Hooks = expanduser("~") + '/GIT_REPO_SB/logs/buildActionHooks*' + else: + l_Hooks = build_path + '/GIT_REPO_SB/logs/buildActionHooks*' + + for filename in glob.glob(l_Hooks): + # Don't remember anything if its a clean exit + print " Removing Hooks file = %s" % filename + os.remove(filename) + + #------------------------------------- + # MAIN : Call the sequence func here + #------------------------------------- + + #---------------- + # Pre-req cheks + #---------------- + input_setting() + user_setting() + + #------------------------- + # Force fresh new build + #------------------------- + if build_type == "Force": + print "\n [ Deafult : Forced to rebuild Repo and sandboxes for clean build ]" + cleanupOnExit() + # Job done , now fall back to Auto mode + build_type = "Auto" + + #------------------------- + # Manual new build + #------------------------- + if build_type == "Manual": + print "\n [ Manual Repo and sandboxes for clean build ]" + cleanupOnExit() + # User knows what they are doing this for + build_type = "Manual" + + #---------------- + # By-pass cheks + #---------------- + # is this the first test run ? + ret_log_code = sbeUtility.utilFilesInDirExist(l_log_dir) + if ret_log_code == sbeConstants.FILES_EXIST: + ret_bypass = bypass_main() + # if new model and not the previous run + # clean start fresh + if ret_bypass == sbeConstants.NEW_MODEL_EXIT: + cleanupOnExit() + elif ret_bypass == sbeConstants.NO_PENDING_MODEL_EXIT: + cleanupOnExit() + else: + print "\n [ Clean Build test started ]" + + #--------------------------- + # Build ENV HACK + #--------------------------- + if not env_patch == "None": + # This will compile and run CI internally + if model_name == "None": + cleanup_build() + print "\n [ PPE ENV : Setting up PPE working Sandboxes/Repo ]" + sandbox_build_ppe() + + ret_code_hack = ValidatePPEModel() + if ret_code_hack == sbeConstants.SUCCESS_EXIT: + if build_type == "Auto": + print " Build and CI successfully completed [ OK ] " + else: + print " Build successfully completed [ OK ] " + sbeUtility.utilshowBuildPPEObj(build_path) + else: + print " Build -OR- CI [ FAILED ] rc = ",ret_code_hack + # Don't go beyond this.. just exit + exit_main(0) + + #--------------------------- + # if Model name is empty + #--------------------------- + if model_name == "None": + cleanup_build() + print "\n [ PPE Model : Setting up PPE working Sandboxes/Repo ]" + sandbox_build_ppe() + # This will compile and run CI internally + ret_code_ppe = ValidatePPEModel() + if ret_code_ppe == sbeConstants.SUCCESS_EXIT: + if build_type == "Auto": + print " Build and CI successfully completed [ OK ] " + else: + print " Build successfully completed [ OK ] " + sbeUtility.utilshowBuildPPEObj(build_path) + else: + print " Build -OR- CI [ FAILED ] rc = ",ret_code_ppe + # Don't go beyond this.. just exit + exit_main(0) + + #--------------------------- + # Pre-req sandbox stagging + #--------------------------- + if not os.path.exists(l_RepoSuccess): + cleanup_build() + print "\n [ Setting up working Sandboxes/Repo ]" + sandbox_build_ppe() + + # skip EKB if Model name is empty + if not model_name == "None": + sandbox_build_ekb() + else: + print " * Skipping Repo create [ Previous run hasn't completed ]" + + #--------------------------- + # Pre-req PPE model check + #--------------------------- + ValidatePPEModel() + + #--------------------------- + # if Model name is empty + #--------------------------- + if model_name == "None": + print " PPE build process completed [ OK ]" + cleanupOnExit() + # Since we are not going to get code from EKB + # Exit from here + exit_main(0) + + #--------------------------- + # Load EKB and hook + #--------------------------- + LoadEnv() + + #--------------------------- + # Clean up on exit + #--------------------------- + cleanupOnExit() + + # Close the file + # UNCOMMENT WHEN IN PRODUCTION + #sys.stdout = orig_stdout + #f.close() + + exit_main(0) + + #------------------------------------- + # End of the main.. exit + #------------------------------------- + +if __name__=="__main__": + main() diff --git a/src/tools/models/buildtool/imageBuildCommand b/src/tools/models/buildtool/imageBuildCommand new file mode 100755 index 00000000..7f275baf --- /dev/null +++ b/src/tools/models/buildtool/imageBuildCommand @@ -0,0 +1,3 @@ +#/afs/awd/projects/eclipz/lab/p9/u/srinivan/sbeModel/buildSBEImage.py -m n10_e9067_tp058_ec163uXXa_soa_sc_u190_01 -b Manual -e "__FAPI_DELAY_SIM__=1,SIM_ONLY_OSC_SWC_CHK=1" -i refs/changes/87/11787/7,refs/changes/68/27568/1 -c cac247559332f434263a21de31d39950f6232f81 -p /gsa/boegsa/home/s/r/srinivan +/afs/awd/projects/eclipz/lab/p9/u/srinivan/sbeModel/buildSBEImage.py -b Manual -e "__FAPI_DELAY_SIM__=1,SIM_ONLY_OSC_SWC_CHK=1" -i refs/changes/87/11787/7 -p /gsa/boegsa/home/s/r/srinivan +#/afs/awd/projects/eclipz/lab/p9/u/srinivan/sbeModel/buildSBEImage.py -b Manual -e "__FAPI_DELAY_SIM__=1,SIM_ONLY_OSC_SWC_CHK=1" -i refs/changes/87/11787/7 -p /gsa/boegsa/home/s/r/srinivan/pibmemHealth diff --git a/src/tools/models/buildtool/ppeModel.py b/src/tools/models/buildtool/ppeModel.py new file mode 100755 index 00000000..6065a7f4 --- /dev/null +++ b/src/tools/models/buildtool/ppeModel.py @@ -0,0 +1,484 @@ +#!/usr/bin/python +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/tools/models/buildtool/ppeModel.py $ +# +# OpenPOWER sbe Project +# +# Contributors Listed Below - COPYRIGHT 2017 +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG +########################################################### +# @file ppeModel.py +# @brief Model tool to creat and validate the PPE model +# +# @author George Keishing +# +# @date Dec 07, 2015 +# ---------------------------------------------------- +# @version Developer Date Description +# ---------------------------------------------------- +# 1.0 gkeishin 11/12/15 Initial create +########################################################### + +#------------------------- +# Imports +#------------------------- +import os, sys +import getopt +import time +import subprocess +import threading +from os.path import expanduser # for getting $HOME PATH +# Local Imports +import ppeUtility as ppeUtility +import sbeUtility as sbeUtility +import sbeGitUtility as sbeGitUtility +import sbeConstants as sbeConstants + +#------------------------- +# Global Var for reference +#------------------------- +timestr = time.strftime("%Y%m%d-%H%M%S") +g_ppeLog = expanduser("~") + '/GIT_REPO_SB/logs/ppeModel_out_' + timestr + +#------------------------- +# Main entry +#------------------------- +def main(): + + #------------------------ + # This main exit func + #------------------------ + def exit_main(rc): + if rc: + print " \n [ ERROR - PPE MODEL ] Exiting with error code = %d "% rc + sys.exit(rc) # get outta from here with rc + else: + print " PPE Model and Processing [ OK ]\n " + sys.exit(0) # get outta from here normal + + #------------------------------------------ + # Usage tool option + #------------------------------------------ + def usage(): + print " ++++++++++++++++++++++++++++++++++++++++++++++++" + print " +++ Refer README file for more information +++" + print " ++++++++++++++++++++++++++++++++++++++++++++++++" + print " -------------------------------------------------------------------------------------------------------------" + print " :: Command line USAGE options for building FW SBE image :: \n" + print " ppeModel.py -m <model_name> -c < Commit log id> -i <patch refs> -p < PPE build path> -s <None|Create>" + + print " \n" + print " +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" + print " | NOTE: Please use VALID input string since validation in code is a tideous task. |" + print " | By default user MUST pass a VALID Model ID. |" + print " | This script alone doesn't work properly. |" + print " +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" + print " \n ***** Options Supported *****" + print " \t -m,--model = Target Model to be used to build the image" + print " \t -c,--commit = commit id of the patch to be checkout" + print " \t -i,--cherry-pick = patch refs to be applied" + print " \t -p,--path = Path where user wants the image to be built" + print " \t -s,--stagged = Default is None, user can pass Create" + print " \n ***** Allowed options example *****" + + print " [ Only Model as an input ]" + print " ppeModel.py -m n10_e9027_tp026_soa_sc_u285_01 " + print " \n" + print " [ Model and PPE commit id as an input ]" + print " ppeModel.py -m n10_e9027_tp026_soa_sc_u285_01 -c c1980f5a4de0bac8922260ed827fe37a124b2b0a" + print " \n" + print " -------------------------------------------------------------------------------------------------------------" + print " [ Creating PPE model base directory ]" + print " [ Creates models/n10_e9027_tp026_soa_sc_u285_01/ directory and base file with the commit ]" + print " ppeModel.py -m n10_e9027_tp026_soa_sc_u285_01 -c c1980f5a4de0bac8922260ed827fe37a124b2b0a -s create" + print " -------------------------------------------------------------------------------------------------------------" + exit_main(sbeConstants.HELP_EXIT) + + #----------------------------------- + # Local var input name place holder + #----------------------------------- + model_name ="None" + commit_name ="None" + patch_refs ="None" + build_path ="None" + create_ppe_model ="None" + build_type ="Auto" + env_patch ="None" + + # Local for base checkout from PPE model + l_base_commit ="None" + l_ignore_base =False + + #----------------------------------- + # Read command line args + #----------------------------------- + opts, args = getopt.getopt(sys.argv[1:],"m:c:i:p:s:b:e:h",['model=', 'commit=', 'cherry-pick=', 'path=', 'stagged=', 'build=', 'env', 'help']) + for opt, arg in opts: + if opt in ('-h', '--help'): + usage() + exit_main(1) + elif opt in ('-m', '--model'): + model_name = arg + elif opt in ('-c', '--commit'): + commit_name = arg + elif opt in ('-i', '--cherry-pick'): + patch_refs = arg + elif opt in ('-p', '--path'): + build_path = arg + elif opt in ('-s', '--stagged'): + create_ppe_model = arg + elif opt in ('-b', '--build'): + build_type = arg + elif opt in ('-e', '--env'): + env_patch = arg + else: + usage() + exit_main(1) + + #----------------------------------- + # Preping the data for Image build + #----------------------------------- + print " \n" + print " ******************************************************" + print " ********* ENV Loaded for PPE Model Build ***********" + print " ******************************************************" + + #------------------------------------------------------------- + # Build default sandboxes for EKB and PPE + #------------------------------------------------------------- + def sandbox_build(): + print "\n [ Setting up working Sandboxes/Repo ]" + if build_path == "None": + print " Using Default PPE git Repo" + else: + print " Setting up the PPE repor at = ", build_path + + ret_ppe_code=sbeUtility.utilCreateSandbox_ppe(build_path) + if ret_ppe_code: + print " Creating ppe sandbox [ FAILED ] rc =%s" % ret_ppe_code + exit_main(ret_ppe_code) + else: + print " Creating ppe sandbox [ OK ]" + + #------------------------------------------------------------- + # Build default sandboxes for EKB and PPE + #------------------------------------------------------------- + def change_ppe_dir(): + if build_path == "None": + l_home_path_ppe= expanduser("~") + "/" +sbeConstants.GIT_PPE_REPO_SB + else: + l_home_path_ppe= build_path + "/" +sbeConstants.GIT_PPE_REPO_SB + + os.chdir(l_home_path_ppe) + + #------------------------------------- + # Create Model + #------------------------------------- + def ppeModelCreate(): + # Create Model + rc_create = ppeUtility.utilppeModelCreate(model_name, build_path, commit_name) + if rc_create == sbeConstants.SUCCESS_EXIT: + exit_main(0) + else: + exit_main(rc_create) + + #------------------------------------- + # Staged PPE model + #------------------------------------- + def ppeModelExist(): + # Model name exist ? + rc_exist = ppeUtility.utilppeModelExist(model_name, build_path, commit_name) + if rc_exist == sbeConstants.SUCCESS_EXIT: + print " PPE Model directory exist [ OK ]" + print " \n" + print " ** Proceeding with the PPE Model base commits" + elif rc_exist == sbeConstants.PPE_MODEL_SKIP_EXIT: + print " * Skipping Model check in PPE since it's a user explicit request" + return sbeConstants.PPE_MODEL_SKIP_EXIT + else: + print " PPE Model directory doesn't exist [ FAILED ] rc = ",rc_exist + if build_type == "Auto": + print " * Continuing with PPE master checkout to build" + return sbeConstants.PPE_MODEL_SKIP_EXIT + #exit_main(rc_exist) + else: + print " * Allowing to continue since it's a Manual Build run" + # Do no more of other task in this Model, return to MAIN + exit_main(sbeConstants.SUCCESS_EXIT) + + #------------------------------------- + # Checkout base commit test_model + #------------------------------------- + def ppeUserbaseCheckout(): + print "\n" + print " -------------------------------" + print " [ Checking out user base commit ] " + print " -------------------------------" + l_gitCmd='git checkout ' + commit_name + ' -b test_build' + print " Executing : ", l_gitCmd + rc = os.system(l_gitCmd) + if rc: + return sbeConstants.PPE_BASE_ERROR_EXIT + else: + return sbeConstants.SUCCESS_EXIT + + #------------------------------------- + # Checkout base commit test_model + #------------------------------------- + def ppebaseCheckout(): + # Check out the base commit + print "\n" + print " -------------------------------" + print " [ PPE Model base Check-out ]" + print " -------------------------------" + rc_base = ppeUtility.utilCheckOutCommit(model_name, build_path, l_base_commit) + if rc_base == sbeConstants.SUCCESS_EXIT: + print " Base Commit checkout [ OK ]" + print "\n" + else: + print " Base Commit checkout [ FAILED ] rc = ",rc_base + exit_main(rc_base) + + #------------------------------------- + # Cherry pick the user supplied refs + #------------------------------------- + def ppeUserCherryPick(): + print "\n [ PPE : Applying User Patches on PPE Repo ] " + print " ++++++++++++++++++++++++++++++++++++++" + print " Patch list : ", patch_refs.split(",") + print " ++++++++++++++++++++++++++++++++++++++" + + # Get from remote and then cherry pick + for refs_id in patch_refs.split(","): + print "\n [ Applying Patch : %s ]\n " %refs_id + rc_cherry_code = ppeUtility.utilCherryPick(refs_id) + if rc_cherry_code == sbeConstants.PPE_CHERRY_ERROR_EXIT: + print "\n *** Cherry-pick of %s failed *** " % refs_id + print " Aborting Build process.... [ ABORT ] rc = %s" % rc_cherry_code + print " [ * Please recheck the patch refs data provided ]" + exit_main(1) + else: + print "\n --> Cherry picked [ OK ] for the Patch ",refs_id + + print "\n " + print " ------------------------------------------------------- " + print " ** All patches Cherry picked and applied successfully **" + print " ------------------------------------------------------- " + + #------------------------------------- + # Cherry pick the commits in the PPE model + #------------------------------------- + def ppeFileCherryPick(): + l_patch_found = False + # Check out commits under model + if build_path == "None": + l_model_build_path= expanduser("~") + "/" +sbeConstants.GIT_PPE_REPO_SB + "/src/tools/models/" + model_name + else: + l_model_build_path= build_path + "/" +sbeConstants.GIT_PPE_REPO_SB + "/src/tools/models/" + model_name + + # Loop through the dir files + if os.path.exists(l_model_build_path): + print "\n" + print " ++++++++++++++++++++++++++++++++++++++++++++++++++++++" + print " [ PPE Model Cherry pick if any ... applying patches ]" + print " ++++++++++++++++++++++++++++++++++++++++++++++++++++++" + #list=os.listdir(l_model_build_path) + # Open base file + l_model_ppe_base = l_model_build_path +"/base" + with open(l_model_ppe_base) as fp: + for line in fp: + # If new line or empty + if not line.strip(): continue + # Checkout the base here + if "base:" in line.rstrip(): + print " ", line.rstrip() + l_base_commit = line[5:].rstrip() + print "\n" + print " --------------------------" + print " [ PPE Model base Check-out ]" + print " --------------------------" + rc_base = ppeUtility.utilCheckOutCommit(model_name, build_path, l_base_commit) + if rc_base == sbeConstants.SUCCESS_EXIT: + print " Base Commit checkout [ OK ]" + print "\n" + else: + print " Base Commit checkout [ FAILED ] rc = ",rc_base + exit_main(rc_base) + + + # Loop through the rest + if not "cp:" in line.strip(): continue + + if "cp:" in line: + # Patch found + l_patch_found = True + + print " Reading Line : ", line.strip() + #patch_id=patch_id.split("cp:") + patch_id=line[3:].rstrip() + print " Cherry-Pick : ", patch_id + rc_cherry_code = ppeUtility.utilCherryPick(patch_id) + if rc_cherry_code == sbeConstants.PPE_CHERRY_ERROR_EXIT: + # Reset the what had been done so far + # os.system("git reset HEAD --hard") + print " ** Cherry-pick of %s failed " % patch_id + exit_main(1) + + print " \n" + print " ++++++++++++++++++++++++++++++++++++++++++++++++++++++" + print " ** All PPE Patches applied successfully ** " + print " ++++++++++++++++++++++++++++++++++++++++++++++++++++++" + print " \n" + else: + print " Model doesn't exist in PPE.. Continue with Master branch " + + # If patch found ? + if l_patch_found == False: + print " * Patches Not Found .." + + #------------------------------------------------------------- + # From commit to change id extract + #------------------------------------------------------------- + def git_comitId_to_changeId(): + print "\n [ commit -- to --> Change Id ]" + l_changeId=sbeGitUtility.utilGetChangeID(commit_name) + + if l_changeId == sbeConstants.ERROR_PREP_CHANGE_FAILED: + print " [ ERROR ] Change log id info not found - FAILED " + exit_main(2) + elif l_changeId == sbeConstants.ERROR_PREP_GIT_FAILED: + print " [ ERROR ] git command - FAILED " + exit_main(2) + else: + #print " ", l_changeId.strip('Change-Id: ') + print " ", l_changeId + print " * Found change log id info [ OK ] " + + #------------------------------------------------------------- + # From change id to commit id extract + #------------------------------------------------------------- + def git_changeId_to_commitid(): + print "\n [ Change Id -- to --> Commit Id ]" + #l_comId=sbeGitUtility.utilGetCommitID(l_changeId.strip('Change-Id: ')) + l_comId=sbeGitUtility.utilGetCommitID(commit_name) + if l_comId == sbeConstants.ERROR_PREP_COM_FAILED: + print " [ ERROR ] commit log id info not found - FAILED " + exit_main(2) + elif l_comId == sbeConstants.ERROR_PREP_GIT_FAILED: + print " [ ERROR ] git command - FAILED " + exit_main(2) + else: + #print " ", l_comId.strip('commit ') + print " ", l_comId + print " * Found commit log id info [ OK ] " + + #---------------------------- + # Do stuff here + #---------------------------- + #sandbox_build() + + #-------------------------------------- + # HACK for ENV build and model EMPTY + #-------------------------------------- + if model_name == "None": + change_ppe_dir() + if not commit_name == "None": + ret_base_code = ppeUserbaseCheckout() + if ret_base_code == sbeConstants.PPE_BASE_ERROR_EXIT: + print " * Base commit checkout failed !!! rc =", sbeConstants.PPE_BASE_ERROR_EXIT + exit_main(sbeConstants.PPE_BASE_ERROR_EXIT) + + if not patch_refs == "None": + ppeUserCherryPick() + + print " !!!! ENV loaded for build run for developement !!!! " + ret_code_hack = sbeUtility.utilBuildPPE(build_type, build_path, model_name, env_patch) + if not ret_code_hack == sbeConstants.SUCCESS_EXIT: + exit_main(ret_code_hack) + else: + exit_main(0) + + #---------------------------- + # if Model name is EMPTY + #---------------------------- + #if model_name == "None": + # change_ppe_dir() + # if not commit_name == "None": + # ppeUserbaseCheckout() + # if not patch_refs == "None": + # ppeUserCherryPick() + # print " !!!! PPE SIMICS TEST run build !!!! " + # ret_code_hack = sbeUtility.utilBuildPPE(build_type, build_path, model_name, env_patch) + # if not ret_code_hack == sbeConstants.SUCCESS_EXIT: + # exit_main(ret_code_hack) + # else: + # exit_main(0) + + #---------------------------------- + # For creating PPE model - Offline + #---------------------------------- + if not create_ppe_model == "None": + ppeModelCreate() + + #---------------------------- + # PPE model check and prep + #---------------------------- + # User has not provided the patch refs, PPE model check + if patch_refs == "None": + rc_ppe_code = ppeModelExist() + if rc_ppe_code == sbeConstants.SUCCESS_EXIT: + print " * Using base file entries as source of comits" + elif rc_ppe_code == sbeConstants.PPE_MODEL_SKIP_EXIT: + change_ppe_dir() + os.system("git branch") + + change_ppe_dir() + + #------------------------------------- + # If comit is not passed, look up the + # PPE model and continue + #------------------------------------- + if commit_name == "None": + #ppebaseCheckout() + if patch_refs == "None": + ppeFileCherryPick() + else: + ppeUserCherryPick() + else: + ret_base_code = ppeUserbaseCheckout() + if ret_base_code == sbeConstants.PPE_BASE_ERROR_EXIT: + print " * Base commit checkout failed !!! rc =", sbeConstants.PPE_BASE_ERROR_EXIT + exit_main(sbeConstants.PPE_BASE_ERROR_EXIT) + + # Fixes for ONLY master commit and not patches + if not patch_refs == "None": + ppeUserCherryPick() + + #------------------------------------- + # Exit gracefully and return the control + # to the main script + #------------------------------------- + exit_main(0) + + #------------------------------------- + # End of this main .. exiting + #------------------------------------- + +if __name__=="__main__": + main() diff --git a/src/tools/models/buildtool/ppeUtility.py b/src/tools/models/buildtool/ppeUtility.py new file mode 100755 index 00000000..6776eb86 --- /dev/null +++ b/src/tools/models/buildtool/ppeUtility.py @@ -0,0 +1,240 @@ +#!/usr/bin/python +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/tools/models/buildtool/ppeUtility.py $ +# +# OpenPOWER sbe Project +# +# Contributors Listed Below - COPYRIGHT 2017 +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG +########################################################### +# @file ppeUtility.py +# @brief Utilility Module to support model for PPE +# +# @author George Keishing +# +# @date Dec 07, 2015 +# ---------------------------------------------------- +# @version Developer Date Description +# ---------------------------------------------------- +# 1.0 gkeishin 11/12/15 Initial create +########################################################### + +#------------------------- +# Imports +#------------------------- +import os +import sys +import time +import shutil +import os.path +import subprocess +from subprocess import Popen, PIPE +from os.path import expanduser # for getting $HOME PATH +# Local imports +import sbeConstants as sbeConstants + +########################################################################## +# Function : utilppeModelCreate +# +# @param i_model_name : user supplied command line input model name +# @param i_build_path : user supplied command line input buil Repo path +# @param i_commit_name : user supplied command line input commit id +# +# @brief Check if this Model existing in PPE and create +# +########################################################################## +def utilppeModelCreate(i_model_name, i_build_path, i_commit_name): + + # Use default if user hasn't provided one + if i_build_path == "None": + l_model_ppe_path= expanduser("~") + "/" +sbeConstants.GIT_PPE_REPO_SB + "/models/" + i_model_name + else: + l_model_ppe_path= i_build_path + "/" +sbeConstants.GIT_PPE_REPO_SB + "/models/" + i_model_name + + print " Create Model Name : %s " % (i_model_name) + print " Create Commit tag Name : %s " % (i_commit_name) + print " Create PPE Model Path : %s " % (l_model_ppe_path) + + if not os.path.isdir(l_model_ppe_path): + print " User provided Model doesn't exist in PPE Repo" + print " Creating model in PPE Repo" + os.makedirs(l_model_ppe_path) + # Create an empty base file under the model + l_base_path = l_model_ppe_path + "/base" + print " Creating base file : " , l_base_path + file = open(l_base_path, 'a') + file.write(i_commit_name) + file.close() + + # create models/<name>/include and include/.empty + l_model_ppe_include =l_model_ppe_path + "/include/" + os.makedirs(l_model_ppe_include) + utilCreateFile(l_model_ppe_include,".empty") + +########################################################################## +# Function : utilppeModelExist +# +# @param i_model_name : user supplied command line input model name +# @param i_build_path : user supplied command line input buil Repo path +# @param i_commit_name : user supplied command line input commit id +# +# @brief Check if this Model existing in PPE, Skip if Commit is "None" +# +########################################################################## +def utilppeModelExist(i_model_name, i_build_path, i_commit_name): + + # Use default if user hasn't provided one + if i_build_path == "None": + l_model_ppe_path= expanduser("~") + "/" +sbeConstants.GIT_PPE_REPO_SB + "/src/tools/models/" + i_model_name + else: + l_model_ppe_path= i_build_path + "/" +sbeConstants.GIT_PPE_REPO_SB + "/src/tools/models/" + i_model_name + + print " Model Name : %s " % (i_model_name) + print " Commit tag Name : %s " % (i_commit_name) + print " PPE Model Path : %s " % (l_model_ppe_path) + + if os.path.isdir(l_model_ppe_path): + #print " Model exist in PPE Repo " + return sbeConstants.SUCCESS_EXIT + + if not i_commit_name == "None": + #print " Skipping Model check in PPE since it's a user explicit request" + return sbeConstants.PPE_MODEL_SKIP_EXIT + + return sbeConstants.PPE_MODEL_NOT_EXIST + +########################################################################## +# Function : utilCreateFile +# +# @param i_build_path : user supplied command line input buil Repo path +# @param i_file_name : user supplied input file names +# +# @brief Create the file/ID file under model directoty +# +########################################################################## +def utilCreateFile(i_build_path, i_file_name): + l_file_path= i_build_path + i_file_name + print " Creating file under Model dir : ", l_file_path + open(l_file_path, 'a').close() + +########################################################################## +# Function : utilCheckOutCommit +# +# @param i_model_name : user supplied command line input model name +# @param i_build_path : user supplied command line input buil Repo path +# @param i_commit_name : user supplied command line input commit id +# +# @brief Checkout the commit id user passed or in the model base +# +########################################################################## +def utilCheckOutCommit(i_model_name, i_build_path, i_commit_name): + #print " Model Name : %s " % (i_model_name) + #print " Commit tag Name : %s " % (i_commit_name) + #print " PPE Model Path : %s " % (i_build_path) + + if i_build_path == "None": + l_model_ppe_base= expanduser("~") + "/" +sbeConstants.GIT_PPE_REPO_SB + "/src/tools/models/" + i_model_name +"/base" + else: + l_model_ppe_base=i_build_path + "/" +sbeConstants.GIT_PPE_REPO_SB + "/src/tools/models/" + i_model_name +"/base" + + # Check if the base exisit + if not os.path.exists(l_model_ppe_base): + #print " Model doesn't exist .. skipping base check-out" + return sbeConstants.SUCCESS_EXIT + #else: + # DEBUG - Saved off a copy of the base file to HOME + # Since once we checkout the base , the model wouldn't be viewable + #shutil.copy(l_model_ppe_base,l_model_copy_base) + + # Load base model file and extract the commit base id + #print " Opening base file : ",l_model_ppe_base + #with open(l_model_ppe_base) as fp: + # for line in fp: + # if "base:" in line.rstrip(): + # #l_base_git_id = line.split("base:") + # l_base_git_id = line[5:].rstrip() + # print " Base commit id : ", l_base_git_id + + # Git checkout base + #l_gitCmd='git checkout ' + l_base_git_id + ' -b test_build' + + l_gitCmd='git checkout ' + i_commit_name + ' -b test_build' + print " Executing : ", l_gitCmd + #p_out = subprocess.Popen(l_gitCmd, stdout=subprocess.PIPE, shell=True) + rc = os.system(l_gitCmd) + + # Return error if there is + if rc : + print " [ ERROR - PPE UTIL ] Base checkout failed rc = ",rc + # Intercept and send our own rc + return 1 + + # Just for showing which branch we are in + l_gitbranch='git branch' + p_checkout_out = subprocess.Popen(l_gitbranch, stdout=subprocess.PIPE, shell=True) + for line in p_checkout_out.stdout: + line = line.strip() + print " " ,line + + # Allow time to switch to new branch + time.sleep(3) + + return sbeConstants.SUCCESS_EXIT + +########################################################################## +# Function : utilCherryPick +# +# @param i_refs_name : user supplied command line input refs id +# +# @brief Cherry pick the ref patch ids +# +########################################################################## +def utilCherryPick(i_refs_name): + print " * Cherry Pick ref id : ", i_refs_name + + # Fetch + l_fetch = "git fetch ssh://gerrit-server/hw/ppe " + i_refs_name + + #l_gitCmd="git cherry-pick -x " + i_refs_name + l_gitCmd= l_fetch + " && " + "git cherry-pick FETCH_HEAD" + print " Executing : ", l_gitCmd + #p_out = subprocess.Popen(l_gitCmd, stdout=subprocess.PIPE, shell=True) + rc = os.system(l_gitCmd) + + # Reset the HEAD if cherry pick failed + if rc: + print " * Patch failed .. Cleaning up Patches Applied : git reset HEAD --hard " + os.system("git reset HEAD --hard") + return sbeConstants.PPE_CHERRY_ERROR_EXIT + else: + return sbeConstants.SUCCESS_EXIT + +########################################################################## +# Function : utilScanModelCommits +# +# @param i_model_name : user supplied command line input model name +# @param i_build_path : user supplied command line input buil Repo path +# @param i_commit_name : user supplied command line input commit id +# +# @brief Prepare a list of commits to be cherry pick from the PPE model +# +########################################################################## +def utilScanModelCommits(i_model_name, i_build_path): + print " Listing out which commits needs to be Cherry pick " + diff --git a/src/tools/models/buildtool/sbeConstants.py b/src/tools/models/buildtool/sbeConstants.py new file mode 100755 index 00000000..c888d189 --- /dev/null +++ b/src/tools/models/buildtool/sbeConstants.py @@ -0,0 +1,101 @@ +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/tools/models/buildtool/sbeConstants.py $ +# +# OpenPOWER sbe Project +# +# Contributors Listed Below - COPYRIGHT 2017 +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG +''' +########################################################### +# Created on Nov 23, 2015 +# ---------------------------------------------------- +# @version Developer Date Description +# ---------------------------------------------------- +# 1.0 gkeishin 11/12/15 Initial create +########################################################### +''' + +## +# General Build constants +# +# @file sbeImageConstants.py +# @author: George Keishing <gkeishin@in.ibm.com> +# + +''' +Generic Failure RC code +''' +SUCCESS_EXIT = 0 +SHELL_EXIT = 1 +HELP_EXIT = 10 +NEW_MODEL_EXIT = 11 +NO_PENDING_MODEL_EXIT = 12 +VALIDATE_EXIT = 13 +FILES_EXIST = 14 +DIR_EMPTY = 15 + +BYPASS_EXIT = 20 +PATCH_IGNORE_EXIT = 21 +PPE_MODEL_NOT_EXIST = 22 +PPE_MODEL_SKIP_EXIT = 23 +PPE_BASE_ERROR_EXIT = 24 +PPE_CHERRY_ERROR_EXIT = 25 + +EKB_MODEL_NOT_EXIST = 40 + +ERROR_SYS_ERROR_EXIT = 100 + +ERROR_PREP_GIT_FAILED = 1000 +ERROR_PREP_TOKEN_SETUP_FAILED = 1001 +ERROR_PREP_GIT_SETUP_FAILED = 1002 + +ERROR_PREP_COM_FAILED = 2000 +ERROR_PREP_CHANGE_FAILED = 2001 +ERROR_GIT_CHANGEID_FAILED = 2002 + +ERROR_LOAD_ENV = 5000 +ERROR_MODEL_NOT_EXIST = 256 # hacked RC code exit from bash + +# Build failures Manual or CI +BUILD_CI_FAILED = 10000 +BUILD_MANUAL_FAILED = 10001 + + +''' +git command ekb and ppe +''' + +GIT_EKB_SETUP_CMD='git init;\ + git remote add gerrit ssh://hw.gerrit/hw/ekb;\ + git fetch gerrit;\ + scp -p -P 29418 hw.gerrit:hooks/commit-msg .git/hooks;\ + git checkout -b master gerrit/master' + +GIT_PPE_SETUP_CMD='git init;\ + git remote add gerrit ssh://hw.gerrit/hw/ppe;\ + git fetch gerrit;\ + scp -p -P 29418 hw.gerrit:hooks/commit-msg .git/hooks;\ + git checkout -b master gerrit/master' + +''' +git base repo home dir default +''' + +GIT_EKB_REPO_SB='GIT_REPO_SB/ekb_sb' +GIT_PPE_REPO_SB='GIT_REPO_SB/ppe_sb' diff --git a/src/tools/models/buildtool/sbeContinue.py b/src/tools/models/buildtool/sbeContinue.py new file mode 100755 index 00000000..22d2bcd7 --- /dev/null +++ b/src/tools/models/buildtool/sbeContinue.py @@ -0,0 +1,381 @@ +#!/usr/bin/python +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/tools/models/buildtool/sbeContinue.py $ +# +# OpenPOWER sbe Project +# +# Contributors Listed Below - COPYRIGHT 2017 +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG +########################################################### +# @file sbeContinue.py +# @brief Module to support the main building SBE imag script. +# This is a hook from the env.bash from ekb to allow +# the compilation process to continue. +# +# @author George Keishing +# +# @date Nov 23, 2015 +# ---------------------------------------------------- +# @version Developer Date Description +# ---------------------------------------------------- +# 1.0 gkeishin 11/12/15 Initial create +########################################################### + +#------------------------- +# Imports +#------------------------- +import os, sys +import getopt +import time +import subprocess +import threading +from os.path import expanduser # for getting $HOME PATH +# Local Imports +import sbeUtility as sbeUtility +import sbeGitUtility as sbeGitUtility +import sbeConstants as sbeConstants + +#------------------------- +# Global Var for reference +#------------------------- +timestr = time.strftime("%Y%m%d-%H%M%S") +#g_ppeLog = expanduser("~") + '/GIT_REPO_SB/logs/buildPPE_out_' + timestr + +#------------------------- +# Main entry +#------------------------- +def main(): + + #------------------------------------------ + # Usage tool option + #------------------------------------------ + def usage(): + print " ++++++++++++++++++++++++++++++++++++++++++++++++" + print " +++ Refer README file for more information +++" + print " ++++++++++++++++++++++++++++++++++++++++++++++++" + print " -------------------------------------------------------------------------------------------------------------" + print " :: Command line USAGE options for building FW SBE image :: \n" + print " sbeContinue.py -m <model_name> -c < Commit log id> -p < PPE build path> -b <Auto|Manual>" + + print " \n" + print " +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" + print " | NOTE: Please use VALID input string since validation in code is a tideous task. |" + print " | By default user MUST pass a VALID Model ID. |" + print " | -- This is a clone copy args of MAIN to hook into scripts. -- |" + print " | -- However this script won't work by it's own by nature. -- |" + print " +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" + print " \n ***** Options Supported *****" + print " \t -m,--model = Target Model to be used to build the image" + print " \t -c,--commit = commit id of the patch to be checkout" + print " \t -p,--path = Path where user wants the image to be built" + print " \t -b,--build = Default is Auto, user can pass Manual" + print " \n ***** Allowed options example *****" + + print " [ Only Model as an input ]" + print " sbeContinue.py -m n10_e9027_tp026_soa_sc_u285_01 " + print " \n" + print " [ Model and PPE commit id as an input ]" + print " sbeContinue.py -m n10_e9027_tp026_soa_sc_u285_01 -c c1980f5a4de0bac8922260ed827fe37a124b2b0a" + print " \n" + print " [ Model, PPE commit id and PPE image build path as an input ] " + print " sbeContinue.py -m n10_e9027_tp026_soa_sc_u285_01 -c c1980f5a4de0bac8922260ed827fe37a124b2b0a -p <user path>" + print " \n" + print " [ Choosing to compile ONLY option - By Default it run in Auto mode and trigger CI build and simics run test ]" + print " sbeContinue.py -m n10_e9027_tp026_soa_sc_u285_01 -b Manual" + print " -------------------------------------------------------------------------------------------------------------" + exit_main(sbeConstants.HELP_EXIT) + + #------------------------ + # This is main exit func + #------------------------ + def exit_main(rc): + if rc: + print " \n [ ERROR - CONTINUE ] Exiting with error code = ", rc + sys.exit(rc) # get outta from here with rc + else: + print " Build successful [ OK ]\n " + sys.exit(0) # get outta from here normal + + #------------------------------------- + # Local var input name place holder + #------------------------------------- + model_name ="None" + commit_name ="None" + patch_refs ="None" + build_type ="Auto" + build_path ="None" + env_patch ="None" + + # UNCOMMENT WHEN IN PRODUCTION + #l_Log= expanduser("~") + '/GIT_REPO_SB/logs/console_out.log' + #print " Check log : %s" % l_Log + #orig_stdout = sys.stdout + #f = file(l_Log, 'a+') + #sys.stdout = f + + #--------------------------- + # Read command line args + #--------------------------- + opts, args = getopt.getopt(sys.argv[1:],"m:c:i:b:p:e:h",['model=', 'commit=', 'cherry_pick=', 'build=', 'path=', 'env=', 'help']) + for opt, arg in opts: + if opt in ('-h', '--help'): + usage() + exit_main(1) + elif opt in ('-m', '--model'): + model_name = arg + elif opt in ('-c', '--commit'): + commit_name = arg + elif opt in ('-i', '--cherry-pick'): + patch_refs = arg + elif opt in ('-b', '--build'): + build_type = arg + elif opt in ('-p', '--path'): + build_path = arg + elif opt in ('-e', '--env'): + patch_env = arg + else: + usage() + exit_main(1) + + #--------------------------- + # Path to action file + #--------------------------- + # This file is for Manual hack per build if extra method is needed. + # Basically this file is an EMPTY file and the tool reads in next + # run and executes the commands in the files. + if build_path == "None": + l_ppeLog = expanduser("~") + '/GIT_REPO_SB/logs/buildPPE_out_' + timestr + l_action_file = expanduser("~") + '/GIT_REPO_SB/logs/PATCH_ACTION_TEMPLATE' + '_' + model_name + l_buildActionHooks = expanduser("~") + '/GIT_REPO_SB/logs/buildActionHooks' + '_' + model_name + else: + l_ppeLog = build_path + '/GIT_REPO_SB/logs/buildPPE_out_' + timestr + l_action_file = build_path + '/GIT_REPO_SB/logs/PATCH_ACTION_TEMPLATE' + '_' + model_name + l_buildActionHooks = build_path + '/GIT_REPO_SB/logs/buildActionHooks' + '_' + model_name + + #---------------------------------- + # Preping the data for Image build + #---------------------------------- + print " \n" + print " ******************************************************" + print " ********* ENV Loaded for compilation Build ***********" + print " ******************************************************" + + #--------------------------------------------------- + # Check if Action file pending from last Test run ? + #--------------------------------------------------- + def patch_action(): + # Model name Action exist ? + if os.path.isfile(l_action_file): + print "\n [ Checking if PATCH action exist ]" + # Check if the file is empty or not + if os.stat(l_action_file).st_size == 0: + print " %s is EMPTY.. Add instruction" % l_action_file + return sbeConstants.PATCH_IGNORE_EXIT + else: + print " Executing the line entries as commands in the file" + + # Load the action file and read per line as command and execute + with open(l_action_file) as fp: + for line in fp: + if not line: continue + else: + l_cmd = line.strip() + print "\n ++++++++++++++++++++++++++++++++++++++++++++++++++" + print " Executing command : %s" % l_cmd + print " ++++++++++++++++++++++++++++++++++++++++++++++++++\n" + rc = os.system( l_cmd ) + if rc: + # Fix the things manually and compile + print "\n Error Executing : %s" % l_cmd + exit_main(rc) + else: + continue # next line + print " ++++++++++++++++++++++++++++++++++++++++++++++++++" + return sbeConstants.SUCCESS_EXIT + + #------------------------------------- + # Stagged and Copy model files + #------------------------------------- + def PPE_ModelCopy(): + print "\n [ Stagging Model file's from EKB to PPE ]" + ret_code=sbeUtility.utilCopyModel(build_path) + if ret_code: + print " Copying model files [ FAILED ] rc = %s " % ret_code + exit_main(ret_code) + else: + print " Copying model files [ OK ]" + + #------------------------------------- + # Show a git Status display + #------------------------------------- + def gitPPE_Status(): + print "\n [ PPE Git Status log ]" + sbeUtility.utilGitStatus(build_path) + + #------------------------------------------ + # Show the git commit filesa passed by user + #------------------------------------------ + def gitPPE_Files(): + print "\n [ PPE Git show files for commit = %s ]" % commit_name + git_show_ret=sbeGitUtility.utilGitShowCommitFiles(commit_name, build_path) + if git_show_ret: + print " This commit ID passed is either not Valid or active [ FAILED ] rc = %s" % git_show_ret + # Remove the action file since its a basic failure + os.remove(l_buildActionHooks) + exit_main(git_show_ret) + else: + if commit_name == "None": + print "" + else: + print "\n This commit ID is active and valid [ OK ]" + #------------------------------------- + # Chckout PPE from user input commit + #------------------------------------- + def gitPPE_Checkout(): + print "\n [ PPE Git checkout for commit = %s ]" % commit_name + git_checkout_ret=sbeGitUtility.utilGitCommitCheckout(commit_name, build_path) + if git_checkout_ret: + print " Checkout of commit [ FAILED ] rc = %s" % git_checkout_ret + exit_main(git_checkout_ret) + else: + if commit_name == "None": + if build_path == "None": + l_home_build_path = expanduser("~") + "/" +sbeConstants.GIT_PPE_REPO_SB + else: + l_home_build_path = build_path + "/" +sbeConstants.GIT_PPE_REPO_SB + + os.chdir(l_home_build_path) + print " --> ", l_home_build_path + print " Working on branch checkout \n" + os.system("git branch") + else: + print "\n Checkout commit to test_build [ OK ]" + + #------------------------------------- + # Compile PPE code + #------------------------------------- + def Trigger_buildPPE(): + print "\n [ PPE Repo Build triggered ]" + if build_type == "Auto": + print "\n *** Build and trigger CI in progress... ***" + else: + print "\n *** Sandbox Build in progress... ***" + + build_ret=sbeUtility.utilBuildPPE(build_type, build_path, model_name,patch_env) + if build_ret == sbeConstants.BUILD_MANUAL_FAILED: + print " Sandbox Build Manual Mode [ FAILED ] rc = %s " % build_ret + # Create a template file PATCH_ACTION_TEMPLATE so that user can manually + # add commands or instructions for execution + WriteActionFile() + print " --> Edit Manually : %s" % l_action_file + exit_main(build_ret) + elif build_ret == sbeConstants.BUILD_CI_FAILED: + print " CI Triggered build or simics test [ FAILED ] rc = %s " % build_ret + # Create a template file PATCH_ACTION_TEMPLATE so that user can manually + # add commands or instructions for execution + WriteActionFile() + print " --> Edit Manually : %s" % l_action_file + exit_main(build_ret) + else: + # per model per log + l_ppeLog_open = l_ppeLog + "_" + model_name +".log" + if build_type == "Auto": + # Look up the log and verify + with open(l_ppeLog_open) as fp: + for line in fp: + if not "Finished tests on" in line: + continue + else: + print " --> %s" % line + + print " Build and CI run [ OK ]" + else: + print " Sandbox PPE build compiled [ OK ]" + + # Remove the action file since its left over + os.remove(l_buildActionHooks) + + #------------------------------------- + # Show the PPE obj dir listing build + #------------------------------------- + def show_buildPPEObj(): + # Just show the ppe/obj/ dir listing + print "\n [ Listing PPE obj build directory files ]" + if build_path == "None": + l_home_build_path = expanduser("~") + "/" +sbeConstants.GIT_PPE_REPO_SB + "/obj" + else: + l_home_build_path = build_path + "/" +sbeConstants.GIT_PPE_REPO_SB + "/obj" + + l_listCmd = "ls -lart " + l_home_build_path + print " PPE obj : ", l_home_build_path + os.system(l_listCmd) + + #------------------------------------- + # Compile PPE code + #------------------------------------- + def WriteActionFile(): + print "\n [ Writting Action File for build ]" + sbeUtility.utilWriteActionFile(l_action_file, model_name) + + # Do stuff here in sequence + + + #------------------------------------- + # Call in sequentially for build + #------------------------------------- + patch_ret_code = patch_action() + if patch_ret_code == sbeConstants.PATCH_IGNORE_EXIT: + print " --> Continuing with EMPTY patch instruction" + print " [ !!! Manually add steps in the file & re-run the tool !!! ]" + + #------------------------ + # Stagging files + #------------------------ + gitPPE_Files() + #gitPPE_Checkout() + + #------------------------ + # Copy Model files needed + #------------------------ + PPE_ModelCopy() + gitPPE_Status() + + #------------------------ + # Trigger PPE build + #------------------------ + Trigger_buildPPE() + show_buildPPEObj() + + # Close the file + # UNCOMMENT WHEN IN PRODUCTION + #sys.stdout = orig_stdout + #f.close() + + #---------------------------------- + # Exit gracefully and return the + # control to the main script + #---------------------------------- + exit_main(0) + + #------------------------------------- + # End of this main .. exiting + #------------------------------------- + +if __name__=="__main__": + main() diff --git a/src/tools/models/buildtool/sbeGitUtility.py b/src/tools/models/buildtool/sbeGitUtility.py new file mode 100755 index 00000000..fba1bb17 --- /dev/null +++ b/src/tools/models/buildtool/sbeGitUtility.py @@ -0,0 +1,220 @@ +#!/usr/bin/python +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/tools/models/buildtool/sbeGitUtility.py $ +# +# OpenPOWER sbe Project +# +# Contributors Listed Below - COPYRIGHT 2017 +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG +########################################################### +# @file sbeGitUtility.py +# @brief Git Utilility Module to support building SBE image +# +# @author George Keishing +# +# @date Nov 23, 2015 +# ---------------------------------------------------- +# @version Developer Date Description +# ---------------------------------------------------- +# 1.0 gkeishin 11/12/15 Initial create +########################################################### + +#------------------------- +# Imports +#------------------------- +import os +import sys +import time +import shutil +import os.path +import subprocess +from subprocess import Popen, PIPE +from os.path import expanduser # for getting $HOME PATH +# Local imports +import sbeConstants as sbeConstants + +########################################################################## +# Function : utilGitShowCommitFiles +# +# @param i_commitId : user supplied command line input commit id +# @param i_ppe_path : user supplied command line input path +# +# @brief Check the files associated with this commit id +# +########################################################################## +def utilGitShowCommitFiles(i_commitId, i_ppe_path): + + if i_commitId == "None": + return sbeConstants.SUCCESS_EXIT + + if i_ppe_path == "None": + l_home_ppe_path = expanduser("~") + "/" +sbeConstants.GIT_PPE_REPO_SB + else: + l_home_ppe_path = i_ppe_path + "/" +sbeConstants.GIT_PPE_REPO_SB + + os.chdir(l_home_ppe_path) + print " --> ", l_home_ppe_path + + # Quickly check if this commit id is valid or not + is_valid = "git cat-file -t " + i_commitId + p_valid = subprocess.Popen(is_valid, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, shell=True) + for line in p_valid.stdout: + line = line.strip() + if line == "commit": + print " * This Commit Validated " + else: + #There is an error - command has exited with non-zero code + return sbeConstants.ERROR_GIT_CHANGEID_FAILED + + # Git command to list only the files associatede with this commit + l_gitCmd='git show --pretty="format:" --name-only ' + i_commitId + print " Executing : ", l_gitCmd + + p_out = subprocess.Popen(l_gitCmd, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, shell=True) + for line in p_out.stdout: + line = line.strip() + print " " ,line + + return sbeConstants.SUCCESS_EXIT + +########################################################################## +# Function : utilGitShowCommitFiles +# +# @param i_commitId : user supplied command line input commit id +# @param i_ppe_path : user supplied command line input path +# +# @brief Check the files associated with this commit id +# +########################################################################## +def utilGitCommitCheckout(i_commitId, i_ppe_path): + + if i_commitId == "None": + return sbeConstants.SUCCESS_EXIT + + if i_ppe_path == "None": + l_home_ppe_path = expanduser("~") + "/" +sbeConstants.GIT_PPE_REPO_SB + else: + l_home_ppe_path = i_ppe_path + "/" +sbeConstants.GIT_PPE_REPO_SB + + os.chdir(l_home_ppe_path) + print " --> ", l_home_ppe_path + + # Git command to list only the files associatede with this commit + l_gitCmd='git checkout ' + i_commitId + ' -b test_build' + print " Executing : ", l_gitCmd + p_out = subprocess.Popen(l_gitCmd, stdout=subprocess.PIPE, shell=True) + + # Allow time to sink in the checkout + time.sleep(10) + + l_gitbranch='git branch' + p_checkout_out = subprocess.Popen(l_gitbranch, stdout=subprocess.PIPE, shell=True) + for line in p_checkout_out.stdout: + line = line.strip() + print " " ,line + + # Allow time to switch to new branch + time.sleep(3) + + return sbeConstants.SUCCESS_EXIT + +########################################################################## +# Function : utilGetChangeID +# +# @param i_num : user supplied command line input commit id +# +# @brief Get the git log id based on the changed ID +# +########################################################################## +def utilGetChangeID(i_num): + print " commit",i_num + l_found = False + #data = [] + + # execute the command + l_cmd = 'git log ' + i_num + + p = subprocess.Popen(l_cmd, stdout=subprocess.PIPE, shell=True) + #(output, err) = p.communicate() + + while p.poll() is None: + line = p.stdout.readline() + line = line.strip() + if not line: continue + + if i_num in line: + #print "Commit id marker found" + l_found=True + continue + + if 'Change-Id:' in line: + if l_found: + #print " ", line + #data.append(line) + break + + # Should n't come out without data + if l_found: + return line + else: + return sbeConstants.ERROR_PREP_CHANGE_FAILED + +########################################################################## +# Function : utilGetCommitID +# +# @param i_num : user supplied command line input commit id +# +# @brief Get the git log id based on the changed ID +# +########################################################################## +def utilGetCommitID(i_num): + print " Change-Id: ",i_num + l_found = False + #data = [] + + # execute the command + l_cmd = 'git log' + p = subprocess.Popen(l_cmd, stdout=subprocess.PIPE, shell=True) + + while p.poll() is None: + line = p.stdout.readline() + line = line.strip() + if not line: + continue + + # Saved off the previous entry which is commit id + if line.startswith('commit'): + l_saved_commit_id=line + + if line.startswith('Change-Id:'): + # Check if the Id is found + if i_num in line: + l_found=True + #print " ", l_saved_commit_id + break + else: + continue + + # Should n't come out without data + if l_found: + return l_saved_commit_id + else: + return sbeConstants.ERROR_PREP_COM_FAILED + + diff --git a/src/tools/models/buildtool/sbeUtility.py b/src/tools/models/buildtool/sbeUtility.py new file mode 100755 index 00000000..f3e4e1a1 --- /dev/null +++ b/src/tools/models/buildtool/sbeUtility.py @@ -0,0 +1,681 @@ +#!usr/bin/python +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: src/tools/models/buildtool/sbeUtility.py $ +# +# OpenPOWER sbe Project +# +# Contributors Listed Below - COPYRIGHT 2017 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG +########################################################### +# @file sbeUtility.py +# @brief Utilility Module to support building SBE image +# +# @author George Keishing +# +# @date Nov 23, 2015 +# ---------------------------------------------------- +# @version Developer Date Description +# ---------------------------------------------------- +# 1.0 gkeishin 11/12/15 Initial create +########################################################### + +#------------------------- +# Imports +#------------------------- +import os, sys +import time +import os.path +import subprocess +import shutil +from subprocess import Popen, PIPE +from os.path import expanduser # for getting $HOME PATH +# Local imports +import sbeConstants as sbeConstants + +# Global vars +# Action file to write if found guilty during build +g_action_file = expanduser("~") + '/GIT_REPO_SB/logs/buildActionHooks' + +# log path +timestr = time.strftime("%Y%m%d-%H%M%S") +g_ppeLog = expanduser("~") + '/GIT_REPO_SB/logs/buildPPE_out_' + timestr + +########################################################################## +# Function : utilCheckTokens +# +# @param None +# +# @brief Check if the user work ENV has git setup or not +# +########################################################################## +def utilCheckTokens(): + awd_token = False + austin_token = False + token_cmd='tokens' + token_out = subprocess.Popen(token_cmd, shell=True, stdout=subprocess.PIPE) + for line in token_out.stdout: + line = line.strip() + if not line: continue + if '@awd.austin.ibm.com' in line: + print " ", line + awd_token = True + if '@austin.ibm.com' in line: + print " ", line + austin_token = True + + # Check if the AWD and Austin token exist + if (awd_token != False ) and ( austin_token != False): + return sbeConstants.SUCCESS_EXIT + else: + # should not reach here + return sbeConstants.ERROR_PREP_TOKEN_SETUP_FAILED + +########################################################################## +# Function : utilCheckSetup +# +# @param None +# +# @brief Check if the user work ENV has git setup or not +# +########################################################################## +def utilCheckSetup(): + ENV_SSH = expanduser("~") + '/.ssh/config' + #print ENV_SSH + ssh_file = open(ENV_SSH) + scanlines=ssh_file.readlines() + for line in scanlines: + if 'hw.gerrit' in line: + #if 'gerrit-server' in line: + print " ", line + ssh_file.close() + return sbeConstants.SUCCESS_EXIT + + ssh_file.close() + # should not reach here + return sbeConstants.ERROR_PREP_GIT_SETUP_FAILED + +########################################################################## +# Function : utilCleanup +# +# @param i_build_path : user supplied command line input path +# +# @brief cleanup sandbox +# +########################################################################## +def utilCleanup(i_build_path): + # default + rc = sbeConstants.SUCCESS_EXIT + + if i_build_path == "None": + l_home_ekb_path= expanduser("~") + "/" +sbeConstants.GIT_EKB_REPO_SB + l_home_ppe_path= expanduser("~") + "/" +sbeConstants.GIT_PPE_REPO_SB + + # Print what is to be cleaned + print " EKB path = %s" % l_home_ekb_path + print " PPE path = %s" % l_home_ppe_path + + if os.path.exists(l_home_ekb_path): + shutil.rmtree(l_home_ekb_path) + + if os.path.exists(l_home_ppe_path): + shutil.rmtree(l_home_ppe_path) + + # Make sure the same path doesn't have old stale copy + if i_build_path == "None": + print "" + else: + l_user_ppe_home = i_build_path + "/" +sbeConstants.GIT_PPE_REPO_SB + l_user_ekb_home = i_build_path + "/" +sbeConstants.GIT_EKB_REPO_SB + print " User PPE path = %s" % l_user_ppe_home + print " User EKB path = %s" % l_user_ekb_home + if os.path.exists(l_user_ppe_home): + shutil.rmtree(l_user_ppe_home) + if os.path.exists(l_user_ekb_home): + shutil.rmtree(l_user_ekb_home) + + # Create the directory logs for logging output + if i_build_path == "None": + l_ppeLog= expanduser("~") + '/GIT_REPO_SB/logs/' + else: + l_ppeLog= i_build_path + '/GIT_REPO_SB/logs/' + + if os.path.exists(l_ppeLog): + l_ppeLog= l_ppeLog + 'buildActionHooks' + # If Action file exist + if os.path.exists(l_ppeLog): + print " Removing Action Build Hooks = %s" % l_ppeLog + #shutil.rmtree(l_ppeLog) + # Remove only the Action file + rc = os.remove(l_ppeLog) + else: + # Create the logs/ dir if not exisit + rc = os.makedirs(l_ppeLog) + + # Rc codes + if rc: + return rc + else: + return sbeConstants.SUCCESS_EXIT + +########################################################################## +# Function : utilCreateSandbox_ekb +# +# @param None +# +# @brief Create ekb sandbox +# +########################################################################## +def utilCreateSandbox_ekb(i_build_path): + + if i_build_path == "None": + l_home_path_ekb= expanduser("~") + "/" +sbeConstants.GIT_EKB_REPO_SB + else: + l_home_path_ekb= i_build_path + "/" +sbeConstants.GIT_EKB_REPO_SB + + print " ekb Sandbox Path = %s" % l_home_path_ekb + if not os.path.isdir(l_home_path_ekb): + os.makedirs(l_home_path_ekb) + + os.chdir(l_home_path_ekb) + ekb = Popen(sbeConstants.GIT_EKB_SETUP_CMD , shell=True, stdout=PIPE, stderr=PIPE) + out, err = ekb.communicate() + #print out.rstrip(), err.rstrip() + # Due to git re-init this warning errors may be thrown .. ignore + if ekb.returncode == 128: + return sbeConstants.SUCCESS_EXIT + else: + return ekb.returncode + +########################################################################## +# Function : utilCreateSandbox_ppe +# +# @param i_build_path : user supplied command line input path +# +# @brief Create ppe sandbox +# +########################################################################## +def utilCreateSandbox_ppe(i_build_path): + + if i_build_path == "None": + l_home_path_ppe= expanduser("~") + "/" +sbeConstants.GIT_PPE_REPO_SB + else: + l_home_path_ppe= i_build_path + "/" +sbeConstants.GIT_PPE_REPO_SB + + print " ppe Sandbox Path = %s" % l_home_path_ppe + if not os.path.isdir(l_home_path_ppe): + os.makedirs(l_home_path_ppe) + + os.chdir(l_home_path_ppe) + ppe = Popen(sbeConstants.GIT_PPE_SETUP_CMD , shell=True, stdout=PIPE, stderr=PIPE) + out_ekb, err_ekb = ppe.communicate() + #print out_ekb.rstrip(), err_ekb.rstrip() + + # Due to git re-init this warning errors may be thrown .. ignore + if ppe.returncode == 128: + return sbeConstants.SUCCESS_EXIT + else: + return ppe.returncode + +########################################################################## +# Function : utilLoadEnv +# +# @param i_model : user supplied command line input model id +# @param i_commit : user supplied command line input commit id +# @param i_build_type : user supplied command line input build type +# Default : Auto ( Build and trigger CI) +# Manual : Compilation ONLY +# @param i_build_path : user supplied command line input path +# @param i_env_patch : user supplied command line input export patch +# +# @brief Load ekb work on ENV . Write Hooks to env.bash +# +########################################################################## +def utilLoadEnv(i_model, i_commit, i_patch, i_build_type, i_build_path, i_env_patch): + + if i_build_path == "None": + l_home_path_ekb= expanduser("~") + "/" +sbeConstants.GIT_EKB_REPO_SB + l_action_file = g_action_file + "_" + i_model + else: + l_home_path_ekb= i_build_path + "/" +sbeConstants.GIT_EKB_REPO_SB + l_action_file = i_build_path + '/GIT_REPO_SB/logs/buildActionHooks' + "_" + i_model + + os.chdir(l_home_path_ekb) + + # Now once we load the bash env here the control goes away from the main + # until returned, now append the command needed and spun of another script + # sbeContinue hook method. The only way to mix bash and python right now. + ENV_BASH_PATH=l_home_path_ekb +'/env.bash' + + # Probably safe to make a copy of it + l_cmd ='cp ' + ENV_BASH_PATH +' ' + l_home_path_ekb+'/env.bash_backup' + os.system( l_cmd ) + + # Now open the file and fiddle without breaking it + file = open(ENV_BASH_PATH,'a') + + print " Executing : git reset --hard : Remove untracked files from the current directory\n" + file.write("git reset --hard\n") + print " Executing : ./ekb model fetch\n" + file.write("./ekb model fetch\n") + + # Give a sync time to fetch the model + file.write("sleep 5 \n") + + # Check if this Model nameed directory exist or not + l_ekb_model_exist=l_home_path_ekb + '/models/' + i_model + + model_exist='[ -d ' + l_ekb_model_exist + ' ] && echo " ' + l_ekb_model_exist + ' exists" || exit 1 ' + file.write(model_exist) + file.write("\n") + + cmd_string="./ekb model checkout --model="+i_model + print " Executing : ./ekb model checkout --model= %s\n" % i_model + file.write(cmd_string) + file.write("\n") + file.write("sleep 10 \n") + file.write("\n") + + # Find out from where the main script triggered, + # yank out the last word and replace with the hook script + l_string=__file__ + l_string=l_string.rsplit('/', 1)[0] + l_string= l_string +'/sbeContinue.py' + hooks_cmd="python " + l_string + " -m " + i_model + " -c " + i_commit + " -i " + i_patch + " -b " + i_build_type + " -p " + i_build_path + " -e " + i_env_patch + " || exit 1" + print " hooks appended to env.bash : ", hooks_cmd + + # We want to write to refer when commit or compilation breaks + # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + print " Writing Action file to %s" % l_action_file + a_file = open(l_action_file,'w') + a_file.write(hooks_cmd) + a_file.close() + # +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + file.write(hooks_cmd) + file.write("\n") + + # make the script exit + file.write("\n") + file.write("exit") + file.close() + + # Work on ekb and load the env.bash + rc = os.system("./ekb workon") + + # Update PATH + + #ROOTDIR/tools:ROOTDIR/tools/test + #/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin + #os.environ['PATH'] = ROOTDIR +'/tools' ':' + ROOTDIR+ '/tools/test'\ + # + ':/usr/lib64/qt-3.3/bin:/usr/local/bin:/bin:/usr/bin:'\ + # + '/usr/local/sbin:/usr/sbin:/sbin:/opt/ibm/cmvc/bin:'\ + # + '/afs/austin.ibm.com/projects/esw/bin:/usr/ode/bin:'\ + # + '/usr/lpp/cmvc/bin' + #os.system("echo $PATH") + + # ROOTDIR/output/lib/:/tools/ecmd/ver-14-1/x86_64/lib + #os.environ['LD_LIBRARY_PATH'] = ROOTDIR +'/output/lib/' + ':' + '/tools/ecmd/ver-14-1/x86_64/lib' + #os.system("echo $LD_LIBRARY_PATH") + + + # Read the env file and execute it in loop to simulate loading the env + + #print " DEBUG Exit code from env.bash EKB === %d" % rc + if rc: + return rc + else: + return sbeConstants.SUCCESS_EXIT + +########################################################################## +# Function : utilCopyModel +# +# @param i_build_path : user supplied command line input path +# +# @brief Copy EKB Model files to PPE import +# +########################################################################## +def utilCopyModel(i_build_path): + # EKB and PPE path + if i_build_path == "None": + l_ekbPath = expanduser("~") + "/" +sbeConstants.GIT_EKB_REPO_SB +'/' + l_ppePath = expanduser("~") + "/" +sbeConstants.GIT_PPE_REPO_SB +'/' + l_ppeModelLog = expanduser("~") + '/GIT_REPO_SB/logs/buildModel_out.log' + else: + l_ekbPath = i_build_path + "/" +sbeConstants.GIT_EKB_REPO_SB +'/' + l_ppePath = i_build_path + "/" +sbeConstants.GIT_PPE_REPO_SB +'/' + l_ppeModelLog = i_build_path + '/GIT_REPO_SB/logs/buildModel_out.log' + + l_ppeImportPath = l_ppePath + "src/import/" + + # Good practice to keep a count how many files did we copied + file_count = 0 + + l_cmdStr = "find " + l_ppeImportPath + "chips" + " -type f -follow -print" + cmd = subprocess.Popen(l_cmdStr, shell=True, stdout=subprocess.PIPE) + for line in cmd.stdout: + line = line.strip() + line = line.replace(l_ppeImportPath,""); + srcPath = l_ekbPath + line; + destPath = l_ppeImportPath + line; + #print "Copying %s --> %s\n" % (srcPath,destPath) + if( os.path.isfile(srcPath.strip()) ): + file_count+=1 # increment + cmd = "cp " + srcPath + " " + destPath; + rc = os.system( cmd ) + if ( rc ): + print " [ ERROR ] Copying file %s: %d "%( cmd, rc ) + return rc + + print " [ %s ] Files copied from EKB to PPE" % file_count + return sbeConstants.SUCCESS_EXIT + +########################################################################## +# Function : utilGitStatus +# +# @param i_build_path : user supplied command line input path +# +# @brief Get the git changes status to confirm the copied files +# +########################################################################## +def utilGitStatus(i_build_path): + if i_build_path == "None": + l_ppePath= expanduser("~") + "/" +sbeConstants.GIT_PPE_REPO_SB +'/' + else: + l_ppePath= i_build_path + "/" +sbeConstants.GIT_PPE_REPO_SB +'/' + + os.chdir(l_ppePath) + + # execute the command + l_cmd = 'git status' + git_cmd = subprocess.Popen(l_cmd, stdout=subprocess.PIPE, shell=True) + for line in git_cmd.stdout: + line = line.strip() + if 'modified:' in line: + print line + + # fail or pass doesnt matter, this just for info + return sbeConstants.SUCCESS_EXIT + +########################################################################## +# Function : utilCompileManual +# +# @param i_ppeLog : Default logging path +# @param i_build_path : user supplied command line input path +# @param i_env : user supplied ENV patch string flag +# +# @brief Compile step by step +# +########################################################################## +def utilCompileManual(i_ppeLog, i_build_path, i_env): + + if i_build_path == "None": + l_home_path_ppe = expanduser("~") + "/" +sbeConstants.GIT_PPE_REPO_SB + else: + l_home_path_ppe = i_build_path + "/" +sbeConstants.GIT_PPE_REPO_SB +'/' + + # Go to sbe/image/ folder and compile + l_sbe_img =l_home_path_ppe + os.chdir(l_sbe_img) + + print " * Changing directory to : ", l_sbe_img + + #execute make all + if i_env: + l_make=i_env + "make all > " + i_ppeLog + " 2>&1" + else: + l_make="make all > " + i_ppeLog + " 2>&1" + + print " * Executing : ", l_make + print " * Code Compilation in progress..." + rc = os.system("source ./env.bash && bash -c '"+l_make+" && exit'") + + # Check if the shell returns error + if rc: + # intercept the rc and override with user define err + return sbeConstants.BUILD_MANUAL_FAILED + else: + return sbeConstants.SUCCESS_EXIT + +########################################################################## +# Function : utilBuildPPE +# +# @param i_build_type : user supplied command line input build type +# Default : Auto ( Build and trigger CI) +# Manual : Compilation ONLY +# @param i_build_path : user supplied command line input path +# @param i_model_name : user supplied command line input model name +# @param i_env_patch : user specific work around ENV commands +# +# @brief Trigger build for PPE git repo +# +########################################################################## +def utilBuildPPE(i_build_type, i_build_path, i_model_name , i_env_patch): + if i_build_path == "None": + l_ppePath= expanduser("~") + "/" +sbeConstants.GIT_PPE_REPO_SB +'/' + l_ppeLog = g_ppeLog + "_" + i_model_name +".log" + else: + l_ppePath = i_build_path + "/" +sbeConstants.GIT_PPE_REPO_SB +'/' + l_ppeLog = i_build_path + '/GIT_REPO_SB/logs/buildPPE_out_' + timestr + "_" + i_model_name +".log" + + # Change directory to PPE Repo path + os.chdir(l_ppePath) + + # execute the command + print " --> in %s" % l_ppePath + print "\n" + print " --------------------------------------------" + print " ** In PPE Repo : which Current GIT branch ? " + print " --------------------------------------------" + os.system("git branch") + print "\n" + + # New path with model + print " [ Logging build output : %s ]\n" % l_ppeLog + + + env_load_cmd ="" + print " * If any ENV work around ? ", i_env_patch.split(",") + for env_id in i_env_patch.split(","): + if env_id: + env_load_cmd += "export " + env_id + " ; " + + # Hack running user ENV for simics build + if not i_env_patch == "None": + if i_build_type == "Auto": + l_cmd = env_load_cmd + "./src/test/framework/build-script" + else: + l_cmd = env_load_cmd + " " + else: + if i_build_type == "Auto": + l_cmd = "./src/test/framework/build-script" + else: + l_cmd = "" + + # Compile build strategy Auto CI / Manual + if i_build_type == "Auto": + print " * Executing Build script :", l_cmd + print "\n *** Please Wait : Build and NIMBUS simics Test in progress ***" + rc = os.system("source ./env.bash && bash -c '%s > %s 2>&1 && exit'" % (l_cmd,l_ppeLog)) + print " [ ** CI build and Test Result ** ]" + else: + print " * Executing Build : make all" + rc = utilCompileManual(l_ppeLog, i_build_path, l_cmd) + + # Return the rc codes for failure/success + if rc == sbeConstants.BUILD_MANUAL_FAILED: + return rc + elif rc: + return sbeConstants.BUILD_CI_FAILED + else: + return sbeConstants.SUCCESS_EXIT + +########################################################################## +# Function : utilValidateHookFile +# +# @param i_hook_path : Foot print log of last saved off hooks +# @param i_model : user supplied command line input model id +# @param i_commit : user supplied command line input commit id +# @param i_patch : user supplied command line input patch refs +# @param i_build_path : user supplied command line input path +# @param i_build_type : user supplied command line input build type +# Default : Auto ( Build and trigger CI) +# Manual : Compilation ONLY +# +# @brief Validate the input signature vs action file +# +########################################################################## +def utilValidateHookFile(i_hook_path, i_model, i_commit, i_patch, i_build_path, i_build_type, i_env_patch): + # Open the file and walk through + # python <PATH>/sbeContinue.py -m n10_e9025_tp024_soa_sc_u289_01 -c None -b manual -p None + + # Local vars for check match + l_model = " -m " + i_model + l_commit = " -c " + i_commit + l_patch = " -i " + i_patch + l_path = " -p " + i_build_path + l_build = " -b " + i_build_type + l_env = " -e " + i_env_patch + + act_file = open(i_hook_path,'r') + scanlines=act_file.readlines() + for line in scanlines: + if l_model in line: + if l_commit in line: + if l_patch in line: + if l_path in line: + if l_build in line: + if l_env in line: + # All parameters are exactly matching + act_file.close() + return sbeConstants.SUCCESS_EXIT + + act_file.close() + + return sbeConstants.VALIDATE_EXIT + +########################################################################## +# Function : utilBypassCheck +# +# @param i_build_path : Foot print log of last failing build comand +# +# @brief Trigger the build by passing the main script +# +########################################################################## +def utilBypassCheck(i_build_path): + # Check if this file exist + if os.path.exists(i_build_path): + print " [ Build action file exist ]" + print " [ FileName ] : %s " % i_build_path + return sbeConstants.BYPASS_EXIT + else: + return sbeConstants.SUCCESS_EXIT + + +########################################################################## +# Function : utilWriteActionFile +# +# @param i_action_file : Create template action file for user to add +# command to execute as part of build +# @param i_model_name : User input model string +# +# @brief Trigger the build by passing the main script plus patch action +# file needed rules +# +########################################################################## +def utilWriteActionFile(i_action_file, i_model_name): + # Check if this file exist + # logs/PATCH_ACTION_TEMPLATE_n10_e9027_tp026_soa_sc_u285_01 + print " Patch Action : ", i_action_file + if not os.path.exists(i_action_file): + open(i_action_file, 'a').close() + + return sbeConstants.SUCCESS_EXIT + + +########################################################################## +# Function : utilVerifyPPEmodel +# +# @param i_model_name : User input model string +# +# @brief Verify of the model name exist or not in PPE Repo +# +########################################################################## +def utilVerifyPPEmodel(i_model_name): + print " Verify PPE Model existence " + +########################################################################## +# Function : utilEkbModelExist +# +# @param i_model_name : user supplied command line input model name +# @param i_ekb_path : user supplied command line input EKB Repo +# +# @brief Check if this Model existing in PPE, Skip if Commit is "None" +# +########################################################################## +def utilEkbModelExist(i_model_name, i_ekb_path ): + + # Use default + #print " Model Name : %s " % (i_model_name) + + l_model_ekb_path = i_ekb_path + "/models/" + i_model_name + #print " EKB Model Path : %s " % (l_model_ekb_path) + + if os.path.isdir(l_model_ekb_path): + return sbeConstants.SUCCESS_EXIT + else: + return sbeConstants.EKB_MODEL_NOT_EXIST + + +########################################################################## +# Function : utilFilesInDirExist +# +# @param i_folder_name : Path to the dir absolute path +# +# @brief Check if there is files in a folder or not +# +########################################################################## +def utilFilesInDirExist(i_folder_path ): + for dirpath, dirnames, files in os.walk(i_folder_path): + if files: + #print dirpath, 'has files' + return sbeConstants.FILES_EXIST + if not files: + #print dirpath, 'is empty' + return sbeConstants.DIR_EMPTY + +########################################################################## +# Function : utilshow_buildPPEObj +# +# @param i_build_path : user supplied command line input path +# +# @brief List the ppe/images/ after compilation for proof +# +########################################################################## +def utilshowBuildPPEObj(i_build_path): + # Just show the ppe/sbe/obj/ dir listing + print "\n [ Listing PPE obj build directory files ]" + if i_build_path == "None": + l_home_build_path = expanduser("~") + "/" +sbeConstants.GIT_PPE_REPO_SB + "/images" + else: + l_home_build_path = i_build_path + "/" +sbeConstants.GIT_PPE_REPO_SB + "/images" + + l_listCmd = "ls -lart " + l_home_build_path + print " PPE obj : ", l_home_build_path + os.system(l_listCmd) |