From 633d60b56129035978b329c27d7aa4138ae238da Mon Sep 17 00:00:00 2001 From: Sangeetha T S Date: Thu, 14 Jul 2016 00:32:00 -0500 Subject: Facilitate FSP CI Change-Id: I32f4d9cd62a7f2a3c2ace8d66ec7a5dbfa2838c3 Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/27131 Reviewed-by: Shakeeb A. Pasha B K Tested-by: Jenkins Server Reviewed-by: Sachin Gupta --- sbe/build/tools/conv_rel_branch.pl | 340 +++++++++++++++++++ sbe/build/tools/perl.modules/gitUtil.pm | 557 ++++++++++++++++++++++++++++++++ sbe/build/tools/sbeCmvcUtility.py | 49 +++ sbe/build/tools/sbeGitTool.pl | 327 +++++++++++++++++++ sbe/build/tools/sbePrime.py | 81 +++-- 5 files changed, 1332 insertions(+), 22 deletions(-) create mode 100755 sbe/build/tools/conv_rel_branch.pl create mode 100644 sbe/build/tools/perl.modules/gitUtil.pm create mode 100755 sbe/build/tools/sbeGitTool.pl (limited to 'sbe/build/tools') diff --git a/sbe/build/tools/conv_rel_branch.pl b/sbe/build/tools/conv_rel_branch.pl new file mode 100755 index 00000000..e9406e29 --- /dev/null +++ b/sbe/build/tools/conv_rel_branch.pl @@ -0,0 +1,340 @@ +#!/usr/bin/perl +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: sbe/build/tools/conv_rel_branch.pl $ +# +# OpenPOWER sbe Project +# +# Contributors Listed Below - COPYRIGHT 2016 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG + +use strict; +use warnings; +use Data::Dumper; +use Getopt::Long qw(:config pass_through); + +# Globals +my %power_platforms = (); +my %relations = (); +my $latest_power_platform = ""; +my $fsp_ci_jenkins_rel_file = "/gsa/ausgsa/home/f/s/fspcibld/patches/cipatch_xml"; +use constant MASTER => "master"; +my $debug = 0; +my $help = 0; + +# Set local variables based on ENV variables +my $PROJECT_ROOT = `git rev-parse --show-toplevel`; + +# Parse command line parameters +GetOptions("debug!" => \$debug, + "help" => \$help); + +# List of commands tool can run +my %commands = ( "rtob" => \&execute_rel_to_branch, + "btor" => \&execute_branch_to_rel, + ); + +if ($help) +{ + execute_help(); +} +else +{ + my $command = shift @ARGV; + if ($commands{$command}) + { + system("git remote -v | grep gerrit -q"); + die "Gerrit remote DNE, must run in repo with gerrit remote" if $?; + # Build release to branch relation hash. + build_relations(); + &{$commands{$command}}(); + } + else + { + execute_help(); + } +} + +############################## Begin Actions ################################### + +sub execute_rel_to_branch +{ + my $release = ""; + + GetOptions("release:s" => \$release); + die "Missing release" if $release eq ""; + + # Get power platform associated with release + my $power_platform = get_power_platform($release); + # Find release in relations hash. + my $branch = $relations{$power_platform}{$release}; + die "Fips release => $release has no corresponding gerrit branch" if (!$branch); + print "$branch \n"; +} + +sub execute_branch_to_rel +{ + my $branch = ""; + + GetOptions("branch:s" => \$branch); + die "Missing branch" if $branch eq ""; + + # Get power platform associated with branch + my $power_platform = get_power_platform($branch); + + # Find branch in relations hash. + my $release = ""; + if( $power_platform ) + { + foreach my $rel (sort keys %{$relations{$power_platform}}) + { + if ($relations{$power_platform}{$rel} eq "$branch") + { + $release = $rel; + last; + } + } + } + die "Gerrit branch => $branch has no corresponding fips release" if ($release eq ""); + print "$release \n"; +} + +sub execute_help +{ + my $command = shift @ARGV; + + if ($command eq "") + { + print " Usage:\n"; + print " conv_rel_branch [options]\n\n"; + print " Tool to convert fips release to branches and vice versa\n\n"; + print " Requires:\n"; + print " Tool to run in git repo that has gerrit remote\n\n"; + print " Available subtools:\n"; + foreach my $key (sort keys %commands) + { + print " $key\n"; + } + print "\n"; + print " Global options:\n"; + print " --debug Enable debug mode.\n"; + print " --help Display help on a specific tool.\n"; + } + elsif (not defined $commands{$command}) + { + die "Unknown subcommand: $command.\n"; + } + else + { + my %help = ( + "rtob" => +q( + Convert release to branch + + Options: + --release= Fips release name (e.g. fips810) [required]. +), + "btor" => +q( + Convert branch to fips release + + Options: + --branch= Remote gerrit branch (e.g. release-fips910, master) [required]. +), + ); + + print "rel_branch $command:"; + print $help{$command}; + } +} + +############################## Begin Sub Routines ############################## + +# sub get_release_branches +# +# Get all branches in gerrit that are prefixed 'release-' and remove the prefix +# *Note branches with string 'master' are removed from this to result in direct +# matches of fips releases only. Master branches will be dervied later. +# +# @return array - sorted names of branches (e.g release-fips810 ...) +# +sub get_release_branches +{ + chdir($PROJECT_ROOT); + die $? if ($?); + + # Parse for remote gerrit branches associated directly with a release + my $cmd = "git branch -a | grep -e remotes/gerrit/release"; + $cmd .= " | sed -e 's/^[ \\t]*remotes\\/gerrit\\///'"; + + my @release_branches = sort (split('\n',`$cmd`)); + print "Release Branches:\n" if $debug; + print Dumper \@release_branches if $debug; + return @release_branches; +} + +# sub get_fips_releases +# +# Get all fips releases that fsp-ci-jenkins uses in sorted order. +# +# @return array - sorted names of releases (e.g fips910, fips920, etc) +# +sub get_fips_releases +{ + chdir($PROJECT_ROOT); + die $? if ($?); + + # Parse fsp-ci-jenkins xml file for releases + my $cmd = "cat $fsp_ci_jenkins_rel_file | grep release | "; + $cmd .= "sed -e 's/^[ \\t]*//' -e 's/<\\/release>[ \\t]*//'"; + + my @fips_releases = sort (split('\n',`$cmd`)); + print "Fips Release:\n" if $debug; + print Dumper \@fips_releases if $debug; + return @fips_releases; +} + +# sub get_power_platform +# +# Takes a references (release or gerrit branch) and determines the power +# platform it belongs to. +# +# @return string - power platform (e.g. p8, p9) +# +sub get_power_platform +{ + my $reference = shift; + + my $power_platform = ""; + if ($reference =~ m/master/) + { + # Remove prefix from older platforms (e.g. master-p8). If nothing is + # replaced then it is the latest power platform. + $reference =~ s/master-//; + if ($reference eq MASTER) + { + $power_platform = $latest_power_platform; + } + else + { + $power_platform = $reference; + } + } + else + { + ($power_platform) = $reference =~ /fips(.*)[0-9][0-9]/; + if ( $power_platform ) + { + $power_platform = "p$power_platform"; + $power_platforms{$power_platform} = 1; + } + } + return $power_platform; +} + + +# sub branchExists + +sub branchExists +{ + my $branch = shift; + chomp($branch); + my $brChk = `git branch -a | grep $branch`; + if ($brChk eq "") + { + return 0; + } + else + { + return 1; + } +} + +# sub build_relations +# +# Build a relationship hash between fips releases and gerrit branches using +# fsp-ci-jenkins xml and git branch command within PPE. +# +# Structure: +# power_platform => +# fips-release => gerrit-branch +# Example: +# p9 => +# fips910 => master +# +sub build_relations +{ + my @releases = get_fips_releases(); + my @branches = get_release_branches(); + + # Fill in fips release keys + foreach my $release (@releases) + { + my $power_platform = get_power_platform($release); + $relations{$power_platform}{$release} = ""; + } + + # Fill in fips release values, which are gerrit release branches. + foreach my $branch (@branches) + { + my $power_platform = get_power_platform($branch); + my $release = $branch; + $release =~ s/release-//; + $relations{$power_platform}{$release} = $branch; + } + + # Handle master branches for each platform + my @platforms = sort keys %power_platforms; + foreach my $i (0 .. $#platforms) + { + my $power_platform = $platforms[$i]; + + # Lastest power platform matches branch master + my $master = MASTER; + # Previous power platforms match branch "master-$platform" + if ($i < $#platforms) + { + $master = MASTER."-$power_platform"; + } + else + { + # Set latest power platform + $latest_power_platform = $power_platform; + } + + # Check for first fips release without a gerrit branch. Due to sort + # order, it will be the next in order release. It is done this way + # to avoid issues when fips releases are ahead of gerrit branches. In + # other words it is possible to have fips releases past gerrit master. + foreach my $release (sort keys %{$relations{$power_platform}}) + { + if ($relations{$power_platform}{$release} eq "") + { + if (branchExists($master)) + { + $relations{$power_platform}{$release} = $master; + } + last; + } + } + } + + print "Relations:\n" if $debug; + print Dumper \%relations if $debug; +} diff --git a/sbe/build/tools/perl.modules/gitUtil.pm b/sbe/build/tools/perl.modules/gitUtil.pm new file mode 100644 index 00000000..d97ae481 --- /dev/null +++ b/sbe/build/tools/perl.modules/gitUtil.pm @@ -0,0 +1,557 @@ +#!/usr/bin/perl +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: sbe/build/tools/perl.modules/gitUtil.pm $ +# +# OpenPOWER sbe Project +# +# Contributors Listed Below - COPYRIGHT 2016 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG + +package gitUtil; + +use strict; + +my %globals = {}; + +# Function : gitRoot +# +# @brief : Determine the root of the GIT repository +# +# @return root : Root of GIT repository +# +sub gitRoot +{ + return $globals{git_root} if (defined $globals{git_root}); + + open COMMAND, "git rev-parse --show-toplevel |"; + my $root = ; + close COMMAND; + chomp $root; + + die "Unable to determine git_root" if ($root eq ""); + + $globals{git_root} = $root; + return $root; +} + +#################### Begin Gerrit JSON Utility Subroutines ##################### + +# @note There are perl modules for doing this but they are not installed on +# the pool machines. The parsing for JSON (at least the content from +# the Gerrit server) isn't so bad... + +# Function : jsonParse +# +# @brief : Parse a line of JSON into an hash-object. +# +# @param[in] line : The JSON content. +# +# @return hash : The parsed object. +# +sub jsonParse +{ + my $line = shift; + + die "Invalid JSON format: $line" unless ($line =~ m/^\{.*\}$/); + $line =~ s/^\{(.*)}$/$1/; + + my %object = (); + + while($line ne "") + { + my $key; + my $value; + + ($key, $line) = jsonGetString($line); + $key =~ s/^"(.*)"$/$1/; + + $line =~ s/^://; + if ($line =~ m/^"/) + { + ($value, $line) = jsonGetString($line); + $value =~ s/^"(.*)"$/$1/; + } + elsif ($line =~ m/^{/) + { + ($value, $line) = jsonGetObject($line); + $value = jsonParse($value); + } + elsif ($line =~ m/^\[/) + { + ($value, $line) = jsonGetArray($line); + $value = jsonParseArray($value); + } + else + { + $line =~ s/([^,]*)//; + $value = $1; + } + + $object{$key} = $value; + } + + return \%object; +} + +# Function : jsonGetString +# +# @brief : Utility function for jsonParse that extracts +# the string data in a given object +# +# @param[in] line : The JSON line containing the strings. +# +# @return strings : The parsed strings. +# +sub jsonGetString +{ + my $line = shift; + + $line =~ /("[^"]*")(.*)/; + my $first = $1; + my $second = $2; + + if ($first =~ m/\\"$/) + { + my ($more, $rest) = jsonGetString($second); + return ($first.$more , $rest); + } + else + { + return ($first, $second); + } +} + +# Function : jsonGetObject +# +# @brief : Utility function for jsonParse that extracts +# the nested JSON object data in a given object +# +# @param[in] line : The JSON line containing the object +# +# @return object : The nested object +# +sub jsonGetObject +{ + my $line = shift; + + $line =~ s/^{//; + my $object = "{"; + my $frag = ""; + + my $found_object = 0; + + until ((not $found_object) && ($object =~ m/}$/)) + { + $found_object = 0; + + if ($line =~ m/^\{/) + { + ($frag, $line) = jsonGetObject($line); + $object = $object.$frag; + $found_object = 1; + } + elsif ($line =~ m/^"/) + { + ($frag, $line) = jsonGetString($line); + $object = $object.$frag; + } + elsif ($line =~ m/^\[/) + { + ($frag, $line) = jsonGetArray($line); + $object = $object.$frag; + } + elsif ($line =~ m/^[:,}]/) + { + $line =~ s/^([:,}])//; + $frag = $1; + $object = $object.$frag; + } + else + { + $line =~ s/([^,}]*)//; + $frag = $1; + $object = $object.$frag; + } + } + + return ($object, $line); +} + +# Function : jsonGetArray +# +# @brief : Utility function for jsonParse that extracts +# the array in a given object +# +# @param[in] line : The JSON line containing the array +# +# @return array : The array object +# +sub jsonGetArray +{ + my $line = shift; + + $line =~ s/^\[//; + my $array = "["; + my $frag = ""; + + my $found_array = 0; + + until ((not $found_array) && ($array =~ m/]$/)) + { + $found_array = 0; + + if ($line =~ m/^\[/) + { + ($frag, $line) = jsonGetArray($line); + $array = $array.$frag; + $found_array; + } + elsif ($line =~ m/^\{/) + { + ($frag, $line) = jsonGetObject($line); + $array = $array.$frag; + } + elsif ($line =~ m/^"/) + { + ($frag, $line) = jsonGetString($line); + $array = $array.$frag; + } + elsif ($line =~ m/^[:,\]]/) + { + $line =~ s/^([:,\]])//; + $frag = $1; + $array = $array.$frag; + } + else + { + $line =~ s/([^,]*)//; + $frag = $1; + $array = $array.$frag; + } + } + + return ($array, $line); +} + +# Function : jsonParseArray +# +# @brief : Utility function for jsonParse that parses +# the array object +# +# @param[in] line : The array +# +# @return array : The parsed array object +# +# +sub jsonParseArray +{ + my $line = shift; + + $line =~ s/^\[(.*)\]$/$1/; + + my @array = (); + + while ($line ne "") + { + my $value; + + if ($line =~ m/^"/) + { + ($value, $line) = jsonGetString($line); + $value =~ s/^"(.*)"$/$1/; + } + elsif ($line =~ m/^\{/) + { + ($value, $line) = jsonGetObject($line); + $value = jsonParse($value); + } + elsif ($line =~ m/^\[/) + { + ($value, $line) = jsonGetArray($line); + $value = jsonParseArray($value); + } + else + { + $line =~ s/([^,]*)//; + $value = $1; + } + + push @array, $value; + $line =~ s/^,//; + } + + return \@array; +} + +#################### End Gerrit JSON Utility Subroutines ####################### + +# Function : gerritIsPatch +# +# @brief : Determine if patch identifier is a Gerrit patch or not. +# +# @param[in] i_patch : The patch to make determination about. +# +# @retval flag : true/false (patch is/not a valid ID) +# +sub gerritIsPatch +{ + my $patch = shift; + return 1 if ($patch =~ m/I[0-9a-f]+/); + return 0; +} + +# Function : configFilename +# +# @brief : Create the file that stroes the GIT server details +# +# @return : Location of the config file +# +sub configFilename +{ + return gitRoot()."/.git/gitRelease.config"; +} + +# Function : configInit +# +# @brief : Fetch & write server details to the config file +# +sub configInit +{ + return if (defined $globals{configInit}); + + unless (-e configFilename()) + { + open COMMAND, "git config --list | grep remote.*ssh |"; + my $url = ; + close COMMAND; + chomp $url; + + die "Undefined git-remote 'gerrit'" if ($url eq ""); + + die "Unexpected url found: $url" if (not ($url =~ m/ssh:\/\/.*\/.*/)); + + my $server = $url; + my $project = $url; + + # match first occurance of '/' after ssh:// + # eg: remote.gerrit.url=ssh://hw.gerrit/hw/ppe + # $2 is 'hw.gerrit' + # $3 is 'hw/ppe' + $server =~ s/(.*)ssh:\/\/(.*?)\/(.*)/$2/; + $project =~ s/(.*)ssh:\/\/(.*?)\/(.*)/$3/; + + open(UNUSED, ">".configFilename()) || die; + close UNUSED; + + system("git config --file ".configFilename(). + " --add releaseLevels.server $server"); + system("git config --file ".configFilename(). + " --add releaseLevels.project $project"); + } + $globals{configInit} = 1; +} + +# Function : configProject +# +# @brief : Fetch the project name of the current configured repository +# +# @return : GIT project name +# +sub configProject +{ + return $globals{config_project} if (defined $globals{config_project}); + + configInit(); + + open COMMAND, "git config --file ".configFilename(). + " --get releaseLevels.project |"; + my $project = ; chomp($project); + close COMMAND; + + die "Project config does not exist" if ($project eq ""); + + $globals{config_project} = $project; + + return $project; +} + +# Function : configServer +# +# @brief : Fetch the server name of the current configured repository +# +# @return : GIT server location +# +sub configServer +{ + return $globals{config_server} if (defined $globals{config_server}); + + configInit(); + + + open COMMAND, "git config --file ".configFilename(). + " --get releaseLevels.server |"; + my $server = ; chomp($server); + close COMMAND; + + die "Server config does not exist" if ($server eq ""); + + $globals{config_server} = $server; + return $server; + +} + +# Function : gerritSSHCommand +# +# @brief : Creates a properly formed ssh command based on the server address +# +# @return : The basic ssh command to connect to the server. +# +sub gerritSSHCommand +{ + return $globals{gerrit_ssh_command} + if (defined $globals{gerrit_ssh_command}); + + my $server = configServer(); + my $port = ""; + + if ($server =~ m/.*:.*/) + { + $port = $server; + $server =~ s/(.*):.*/$1/; + $port =~ s/.*:(.*)/$1/; + + $port = "-p $port"; + } + + my $command = "ssh -qx $port $server gerrit"; + + $globals{gerrit_ssh_command} = $command; + return $command; +} + +# Function : gerritQuery +# +# @brief : Performs a gerrit query and parses the resulting JSON. +# +# @param[in] query : The query to perform. +# +# @return item : A list of items from the JSON query. Each item is a +# hash (key-value pair) for the item attributes. +# +sub gerritQuery +{ + my $query = shift; + my @items = (); + + $query = gerritSSHCommand()." query $query --current-patch-set --patch-sets --format=JSON |"; + + open COMMAND, $query; + while (my $line = ) + { + chomp $line; + push @items, jsonParse($line); + } + + return \@items; +} + +# Function : gerritQueryReference +# +# @brief : Retrieves reference for a patch id, patchset number +# +# @param[in] changeId : Change id of the patch +# @param[in] patchNumber : Patch set number +# +# @return reference : The reference string +# +sub gerritQueryReference +{ + my $changeId = shift; + my $patchNumber = shift; + + my $project = configProject(); + + my $query_result = gerritQuery("$changeId project:$project"); + + foreach my $result (@{$query_result}) + { + if ($result->{id} eq $changeId) + { + # If all patchsets queried, search all of them for the commit + foreach my $patchset (@{$result->{patchSets}}) + { + if ($patchNumber eq "") + { + return $patchset->{currentPatchSet}->{ref}; + } + else + { + if ($patchset->{number} =~ m/$patchNumber/) + { + return $patchset->{ref}; + } + } + } + } + } + die "Cannot find $changeId in $project"; +} + +# Function : gerritQueryCommit +# +# @brief : Retrieves commit for a patch id, patchset number +# +# @param[in] changeId : Change id of the patch +# @param[in] patchNumber : Patch set number +# +# @return commit : The commit string +# +sub gerritQueryCommit +{ + my $changeId = shift; + my $patchNumber = shift; + + my $project = configProject(); + + my $query_result = gerritQuery("$changeId project:$project"); + + foreach my $result (@{$query_result}) + { + if ($result->{id} eq $changeId) + { + # If all patchsets queried, search all of them for the commit + foreach my $patchset (@{$result->{patchSets}}) + { + if ($patchNumber eq "") + { + return $patchset->{currentPatchSet}->{revision}; + } + else + { + if ($patchset->{number} =~ m/$patchNumber/) + { + return $patchset->{revision}; + } + } + } + } + } + die "Cannot find $changeId in $project"; +} diff --git a/sbe/build/tools/sbeCmvcUtility.py b/sbe/build/tools/sbeCmvcUtility.py index 3ccb2659..4df59338 100755 --- a/sbe/build/tools/sbeCmvcUtility.py +++ b/sbe/build/tools/sbeCmvcUtility.py @@ -1,4 +1,28 @@ #!/usr/bin/python +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: sbe/build/tools/sbeCmvcUtility.py $ +# +# OpenPOWER sbe Project +# +# Contributors Listed Below - COPYRIGHT 2016 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG ''' ########################################################### # @file sbeCmvcUtility.py @@ -128,6 +152,10 @@ def utilFindFilePPE(i_filename, i_path, i_sandbox_name): # Ignore the test sandbox files in the PPE if not i_sandbox_name in root: return os.path.join(root, i_filename) + # Consider the path, if the repository is contanied in the sandbox + else: + if i_sandbox_name in i_path: + return os.path.join(root, i_filename) ########################################################################## # Function :utilCmvcCheckout @@ -464,6 +492,27 @@ def utilFind_sb_base(i_sb_name): else: return out_str.strip('SANDBOXBASE=') +########################################################################## +# Function : utilFind_sb_rc +# +# @param i_sb_name : Sandbox RC path +# +# @brief find the sandbox RC path +# +########################################################################## +def utilFind_sb_rc(i_sb_name): + out_str= "None" + + # workon -m ppc sbeisb -c 'env | grep SANDBOXRC' + find_sb_rc = 'workon -m ppc ' + i_sb_name + " -c 'env | grep SANDBOXRC ' | grep SANDBOXRC" + # SANDBOXRC=/gsa/ausgsa/projects/i/indiateam04/gkeishin/.sandboxrc + out_str = os.popen(find_sb_rc).read() + + if not out_str: + return "None" + else: + return os.path.dirname(out_str.strip('SANDBOXRC=')) + ########################################################################## # Function : utilFind_ENV_string # diff --git a/sbe/build/tools/sbeGitTool.pl b/sbe/build/tools/sbeGitTool.pl new file mode 100755 index 00000000..6f7a2b26 --- /dev/null +++ b/sbe/build/tools/sbeGitTool.pl @@ -0,0 +1,327 @@ +#! /usr/bin/perl +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: sbe/build/tools/sbeGitTool.pl $ +# +# OpenPOWER sbe Project +# +# Contributors Listed Below - COPYRIGHT 2016 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG + +use strict; +#use lib "$ENV{'PERLMODULES'}"; +use lib "sbe/build/tools/perl.modules"; +use Getopt::Long qw(:config pass_through); +use Data::Dumper; +use Cwd; +use gitUtil; + +# Global variables & constants +my $debug = 0; +my $help = 0; +my %globals = (); +my @references = (); +my @patchList = (); +my @commits = (); +my $patches = ""; +my $sbbase = ""; +my $sbname = ""; +my $sbrc = ""; +my $release = ""; + +my %commands = ( "extract" => \&execute_extract, + "get-commits" => \&execute_get_commits); + +# Handle input command +GetOptions("debug!" => \$debug, + "help" => \$help); + +if ($help) +{ + execute_help(); +} +else +{ + my $command = shift @ARGV; + if ($commands{$command}) + { + &{$commands{$command}}(); + } + else + { + execute_help(); + } +} + +foreach my $arg (@ARGV) +{ + print "Unprocessed arg: $arg\n" if $debug; +} + +############################## Begin Actions ################################## + +sub execute_help +{ + my $command = shift @ARGV; + + if ($command eq "") + { + print "sbeGitTool\n"; + print " Run FSP-CI from a set of git commits\n"; + print "\n"; + print " Syntax:\n"; + print " sbeGitTool [options]\n"; + print "\n"; + print " Available subtools:\n"; + foreach my $key (sort keys %commands) + { + print " $key\n"; + } + print "\n"; + print " Global options:\n"; + print " --debug Enable debug mode.\n"; + print " --help Display help on a specific tool.\n"; + print "\n"; + print " Note: Generally a can be any git or gerrit\n"; + print " reference. A git commit number, tag, branch, or\n"; + print " a gerrit change-id are all valid.\n"; + } + elsif (not defined $commands{$command}) + { + die "Unknown subcommand: $command.\n"; + } + else + { + my %help = ( + "extract" => +q( + Create the SBE binaries with the patch and its CMVC/GIT dependencies. + + Options: + --patches= CSV of changeId:patch-set's [required]. + --sbbase= Sandbox base for FSP-CI [required]. + --sbname= Sandbox name [required]. + --rc= RC file for the sandbox with absolute path [optional]. +), + "get-commits" => +q( + Given a patch, find the corresponding commit id. + + Options: + --patches= CSV of changeId:patch-set's [required]. + --sbbase= Sandbox base for FSP-CI [required]. + --sbname= Sandbox name [required]. +) + ); + + print "sbeGitTool $command:"; + print $help{$command}; + } +} + +sub execute_get_commits +{ + + # Set GIT environment + git_environ_init(); + + # Obtain the list of patches + retrivePatchList(); + + # Fetch the commits for the patches + fetchCommits(); + + # Prepare commit string + my $commitStr = prepareCommitStr(); + + print "The set of commits: $commitStr" if $debug; + print $commitStr; + return $commitStr; +} + +sub execute_extract +{ + # Set GIT environment + git_environ_init(); + + # Obtain the list of patches + retrivePatchList(); + + # Fetch the references for the patches + fetchRefs(); + + # Apply the patches on the GIT repo + applyRefs(); + + # Compile the SBE and copy binaries to sandbox + compileAndCopy(); +} + +sub git_environ_init +{ + # Handle the i/p to the function + GetOptions("patches:s" => \$patches, + "sbbase:s" => \$sbbase, + "sbname:s" => \$sbname, + "rc:s" => \$sbrc); + + die "Missing patch list" if ($patches eq ""); + die "Missing sandbox base path" if ($sbbase eq ""); + die "Missing sandbox name" if ($sbname eq ""); + + # Set global variables + $globals{sandbox} = $sbbase."/".$sbname; + $globals{sbe_git_root} = $globals{sandbox}."/git-ci"; + + print "Sandbox: $globals{sandbox}\n" if $debug; + print "GIT repository path: $globals{sbe_git_root}\n" if $debug; + + chdir($globals{sbe_git_root}); + die "ERROR $?: Invalid GIT repository path in the sandbox" if $? ; +} + +sub retrivePatchList +{ + # Parse out the CSV patch list + @patchList = split(/,+/, $patches); + + print ">>>Patches\n" if $debug; + print Dumper @patchList if $debug; + print "<< reference) = $patch -> $currentRef\n" if $debug; + } + else + { + print "\n Warning : Patchset $patch is invalid.. Continuing to check if there is any other valid patch \n"; + } + } + die "ERROR: No valid patches given..\n" if (scalar @references == 0); +} + +sub applyRefs +{ + my $statusFile = $globals{sbe_git_root}."/patchApply.status"; + + foreach my $ref (@references) + { + print "Cherrypicking reference $ref \n" if $debug; + open SBWORKON, " | ./sb workon"; + print SBWORKON "git fetch gerrit $ref && echo \"Fetch Done \" > $statusFile \n"; + print SBWORKON "git cherry-pick FETCH_HEAD && echo \"Cherry-pick Done \" >> $statusFile \n"; + print SBWORKON "exit \n"; + close SBWORKON; + + print "\nChecking cherrypick status for $ref...\n" if $debug; + my $ch_status = `cat $statusFile`; + if( ($ch_status =~ m/Fetch/) && ($ch_status =~ m/Cherry-pick/)) + { + print "Cherry Picking successful\n"; + } + else + { + die "ERROR: Cherry Picking of $ref failed\n"; + } + } +} + +sub compileAndCopy +{ + my $statusFile = $globals{sbe_git_root}."/compile.status"; + my $compile_path = $globals{sbe_git_root}."/sbe/image"; + + print "Compiling and copying the generated binaries to sandbox\n" if $debug; + open SBWORKON, " | ./sb workon"; + print SBWORKON "cd $compile_path \n"; + print SBWORKON "make install && echo \"Compile Passed\" > $statusFile \n"; + print SBWORKON "export SANDBOXBASE=$globals{sandbox} \n"; + print SBWORKON "cd $globals{sbe_git_root} \n"; + print SBWORKON "./sb prime --sb $sbname --no_build && echo \"Prime Passed\" >> $statusFile \n"; + print SBWORKON "exit \n"; + close SBWORKON; + + print "\nChecking compile status...\n" if $debug; + my $compile_status = `cat $statusFile | grep "Compile"`; + if ($compile_status =~ m/Compile/) + { + print "SBE compile successful\n"; + } + else + { + die "ERROR: SBE compile failed\n"; + } + + print "\nChecking sandbox status...\n" if $debug; + my $sb_status = `cat $statusFile | grep "Prime"`; + if ($sb_status =~ m/Prime/) + { + print "SBE prime successful\n"; + } + else + { + die "ERROR: SBE prime failed\n"; + } +} + +sub fetchCommits +{ + my $currentCommit = ""; + + foreach my $patch (@patchList) + { + my ($changeId,$patchSet) = split(":",$patch); + if (gitUtil::gerritIsPatch($changeId)) + { + print "Fetching commit for the patch : $patch \n" if $debug; + my $currentCommit = gitUtil::gerritQueryCommit($changeId, $patchSet); + push @commits, $currentCommit; + print "(patchset -> commit) = $patch -> $currentCommit\n" if $debug; + } + else + { + print "\n Warning : Patchset $patch is invalid.. Continuing to check if there is any other valid patch \n"; + } + } + die "ERROR: No valid patches given..\n" if (scalar @commits == 0); +} + +sub prepareCommitStr +{ + my $commitStr = ""; + + foreach my $commit (@commits) + { + $commitStr = $commitStr.",".$commit; + } + $commitStr =~ s/^,//g; + return $commitStr; +} diff --git a/sbe/build/tools/sbePrime.py b/sbe/build/tools/sbePrime.py index d82d4215..021bad74 100755 --- a/sbe/build/tools/sbePrime.py +++ b/sbe/build/tools/sbePrime.py @@ -1,4 +1,28 @@ #!/usr/bin/python +# IBM_PROLOG_BEGIN_TAG +# This is an automatically generated prolog. +# +# $Source: sbe/build/tools/sbePrime.py $ +# +# OpenPOWER sbe Project +# +# Contributors Listed Below - COPYRIGHT 2016 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# IBM_PROLOG_END_TAG ''' ########################################################### # @file sbePrime.py @@ -53,6 +77,8 @@ def main(): print " \t -i,--files = [ Optional ] Firmware Files coma ',' separated input file1,file2" print " \t Only the pre-define listed files bellow:" print " \t sbe_sp_intf.H,simics.tar,sbe_pibmem.bin,sbe_seeprom.bin" + print " \t -r,--rc_file = [ Optional ] The RC file for the sandbox (with absolute path)" + print " \t -n,--no_build = [ Optional ] Flag to determine if sbei component should be compiled" print " \t -h,--help = Help" print " ------------------------------------------------------------------------------------" @@ -85,11 +111,13 @@ def main(): sandbox_name = "None" path_name = "None" # PPE Repo file_name = "None" + rc_file = "None" + build = "1" #---------------------------- # Read command line args #---------------------------- - opts, args = getopt.getopt(sys.argv[1:],"p:s:i:h",['patch=', 'sb=', 'files=', 'help']) + opts, args = getopt.getopt(sys.argv[1:],"p:s:i:h:r:n",['patch=', 'sb=', 'files=', 'help', 'rc_file=', 'no_build']) for opt, arg in opts: if opt in ('-h', '--help'): usage() @@ -100,6 +128,10 @@ def main(): sandbox_name = arg elif opt in ('-i', '--files'): file_name = arg + elif opt in ('-r', '--rc_file'): + rc_file = arg + elif opt in ('--no_build'): + build = "0" else: usage() exit_main(errorcode.ERROR_EXIT) @@ -149,7 +181,8 @@ def main(): if sandbox_name == "None": sandbox_root = utilcode.utilFind_ENV_string("SANDBOXROOT").rstrip('\n') else: - sandbox_root = utilcode.utilFind_ENV_string("SANDBOXRC").rstrip('\n') +# sandbox_root = utilcode.utilFind_ENV_string("SANDBOXRC").rstrip('\n') + sandbox_root = utilcode.utilFind_sb_rc(sandbox_name).rstrip('\n') if sandbox_root == "None": print " ** [ ERROR ] Something Fishy about the ENV set -OR- Option used.. Please check manually ** " @@ -229,27 +262,31 @@ def main(): sb_name=os.path.basename(sandbox_path) print "\n Sandbox :",sb_name - #---------------------------------------- - # 5) Wite the hook file into shell file - #---------------------------------------- - # Write the compile shell hook on the fips sandbox location - hook_file = utilcode.utilWriteShell_hooks(sandbox_path) - - #---------------------------------------- - # 6) Compile the code - #---------------------------------------- - # Use the hook script to compile the code - if sandbox_name == "None": - compile_cmd="workon -m ppc " + sb_name + " -c " + hook_file + " -rc " + sandbox_root +"/sbesandboxrc" - else: - compile_cmd="workon -m ppc " + sb_name + " -c " + hook_file + " -rc " + sandbox_root +"/.sandboxrc" - - print "\n [ COMPILE ] Executing :%s \n"%compile_cmd - rc = os.system(compile_cmd) + if build == "1": + #---------------------------------------- + # 5) Wite the hook file into shell file + #---------------------------------------- + # Write the compile shell hook on the fips sandbox location + hook_file = utilcode.utilWriteShell_hooks(sandbox_path) - print " Compilation returned rc :",rc - if rc != 0: - exit_main(errorcode.ERROR_BUILD_FAILED) + #---------------------------------------- + # 6) Compile the code + #---------------------------------------- + # Use the hook script to compile the code + if sandbox_name == "None": + compile_cmd="workon -m ppc " + sb_name + " -c " + hook_file + " -rc " + sandbox_root +"/sbesandboxrc" + else: + if rc_file == "None": + compile_cmd="workon -m ppc " + sb_name + " -c " + hook_file + " -rc " + sandbox_root +"/.sandboxrc" + else: + print " getting rc file from user \n" + compile_cmd="workon -m ppc " + sb_name + " -c " + hook_file + " -rc " + rc_file + print "\n [ COMPILE ] Executing :%s \n"%compile_cmd + rc = os.system(compile_cmd) + + print " Compilation returned rc :",rc + if rc != 0: + exit_main(errorcode.ERROR_BUILD_FAILED) # Clean exit exit_main(errorcode.SUCCESS_EXIT) -- cgit v1.2.1