diff options
author | Stewart Smith <stewart@linux.ibm.com> | 2018-08-24 13:30:15 +1000 |
---|---|---|
committer | Stewart Smith <stewart@linux.ibm.com> | 2018-08-30 17:29:00 +1000 |
commit | c4b9bf64b047fff3aec6773fd3397268773062e4 (patch) | |
tree | 7a47d58f41afcec74b113bd5c1dd125bf2a85196 /ci | |
parent | fb432100e9473c6bc96e9d7edbd578e70465c285 (diff) | |
download | talos-op-build-c4b9bf64b047fff3aec6773fd3397268773062e4.tar.gz talos-op-build-c4b9bf64b047fff3aec6773fd3397268773062e4.zip |
ci: Build more targets, cache SDKs, eatmydata
Build graph-depends and legal-info along with firmware images
Also, use proper O=/foo for telling buildroot where to build images.
Also, use the magic buildroot config script rather than hacky hacks.
Support building and caching an SDK, meaning you can build simple
defconfigs in dozens of minutes, and greatly reduce the total time
for building all configurations.
Eatmydata will save us a *lot* of fsync() activity when you don't
have infinite IOPS.
Signed-off-by: Stewart Smith <stewart@linux.ibm.com>
Diffstat (limited to 'ci')
-rw-r--r-- | ci/Dockerfile/ubuntu1604 | 2 | ||||
-rwxr-xr-x | ci/build-all-defconfigs.sh | 55 | ||||
-rwxr-xr-x | ci/build-sdk.sh | 33 | ||||
-rwxr-xr-x | ci/build.sh | 48 |
4 files changed, 106 insertions, 32 deletions
diff --git a/ci/Dockerfile/ubuntu1604 b/ci/Dockerfile/ubuntu1604 index 46f68014..2aaa6474 100644 --- a/ci/Dockerfile/ubuntu1604 +++ b/ci/Dockerfile/ubuntu1604 @@ -7,4 +7,4 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update && DEBIAN_FRONTEND=noninteract build-essential g++ git bison flex unzip \ cpio vim-common lsb-release \ libxml-simple-perl libxml-sax-perl libxml2-dev libxml2-utils xsltproc \ - wget bc libssl-dev + wget bc libssl-dev python-matplotlib python-numpy graphviz eatmydata diff --git a/ci/build-all-defconfigs.sh b/ci/build-all-defconfigs.sh index 839d30a5..0ab53bff 100755 --- a/ci/build-all-defconfigs.sh +++ b/ci/build-all-defconfigs.sh @@ -8,18 +8,24 @@ CONFIGTAG="_defconfig" DEFCONFIGS=(); -while getopts "o:p:r" opt; do +SDK_DIR="" + +while getopts "o:p:rs:" opt; do case $opt in o) echo "Output directory: $OPTARG" OUTDIR="$OPTARG" ;; + s) + echo "SDK is in: $OPTARG" + SDK_DIR=$OPTARG + ;; p) echo "Platforms to build: $OPTARG" PLATFORM_LIST="$OPTARG" ;; r) - echo "Build legal-info for release" + echo "Build legal-info etc for release" BUILD_INFO=1 ;; \?) @@ -61,26 +67,35 @@ if [ -n "$DL_DIR" ]; then export BR2_DL_DIR=${DL_DIR} fi -for i in ${DEFCONFIGS[@]}; do - rm -rf output/* - op-build $i - echo 'BR2_CCACHE=y' >> output/.config - echo "BR2_CCACHE_DIR=\"$CCACHE_DIR\"" >> output/.config - echo 'BR2_CCACHE_INITIAL_SETUP=""' >> output/.config +if [ -f $(ldconfig -p | grep libeatmydata.so | tr ' ' '\n' | grep /|head -n1) ]; then + export LD_PRELOAD=${LD_PRELOAD:+"$LD_PRELOAD "}libeatmydata.so +fi - op-build olddefconfig - op-build +for i in ${DEFCONFIGS[@]}; do + export O=${OUTDIR}-$i + rm -rf $O + op-build O=$O $i + ./buildroot/utils/config --file $O/.config --set-val BR2_CCACHE y + ./buildroot/utils/config --file $O/.config --set-str BR2_CCACHE_DIR $CCACHE_DIR + if [ -d "$SDK_DIR" ]; then + ./buildroot/utils/config --file $O/.config --set-val BR2_TOOLCHAIN_EXTERNAL y + ./buildroot/utils/config --file $O/.config --set-str BR2_TOOLCHAIN_EXTERNAL_PATH $SDK_DIR + ./buildroot/utils/config --file $O/.config --set-val BR2_TOOLCHAIN_EXTERNAL_CUSTOM_GLIBC y + ./buildroot/utils/config --file $O/.config --set-val BR2_TOOLCHAIN_EXTERNAL_CXX y + # FIXME: How do we work this out programatically? + ./buildroot/utils/config --file $O/.config --set-val BR2_TOOLCHAIN_EXTERNAL_GCC_6 y + ./buildroot/utils/config --file $O/.config --set-val BR2_TOOLCHAIN_EXTERNAL_HEADERS_4_18 y + fi + op-build O=$O olddefconfig + op-build O=$O r=$? - - if [ ${BUILD_INFO} = 1 ] && [ $r = 0 ]; then - op-build legal-info - mv output/legal-info ${OUTDIR}/$i-legal-info - fi - - mkdir ${OUTDIR}/$i-images - mv output/images/* ${OUTDIR}/$i-images/ - mv output/.config ${OUTDIR}/$i-images/.config - lsb_release -a > ${OUTDIR}/$i-images/lsb_release + if [ ${BUILD_INFO} = 1 ] && [ $r = 0 ]; then + op-build O=$O legal-info + op-build O=$O graph-build + op-build O=$O graph-size + op-build O=$O graph-depends + fi + lsb_release -a > $O/lsb_release if [ $r -ne 0 ]; then exit $r fi diff --git a/ci/build-sdk.sh b/ci/build-sdk.sh new file mode 100755 index 00000000..8267af38 --- /dev/null +++ b/ci/build-sdk.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -ex +set -eo pipefail + +if [ -z "$1" ]; then + echo "No build distro specified" + exit 1; +fi + +if [ -z "$2" ]; then + echo "No defconfig to build SDK from specified" + exit 1; +fi + +if [ -z "$CCACHE_DIR" ]; then + CCACHE_DIR=`pwd`/.op-build_ccache +fi + +shopt -s expand_aliases +source op-build-env + +if [ -n "$DL_DIR" ]; then + unset BR2_DL_DIR + export BR2_DL_DIR=${DL_DIR} +fi + +export O=`pwd`/output-$1-$2/ +op-build O=$O $2 +./buildroot/utils/config --file $O/.config --set-val BR2_CCACHE y +./buildroot/utils/config --file $O/.config --set-str BR2_CCACHE_DIR $CCACHE_DIR +op-build O=$O olddefconfig +op-build O=$O sdk diff --git a/ci/build.sh b/ci/build.sh index b313291e..012aaef0 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -2,8 +2,19 @@ CONTAINERS="ubuntu1604 fedora27" -while getopts ":ab:hp:c:r" opt; do + +SDK_ONLY=0 + +while getopts ":ab:hp:c:rs:S" opt; do case $opt in + s) + echo "SDK Cache dir" + SDK_CACHE="$OPTARG" + ;; + S) + echo "Build SDK Only" + SDK_ONLY=1 + ;; a) echo "Build firmware images for all the platforms" PLATFORMS="" @@ -26,6 +37,8 @@ while getopts ":ab:hp:c:r" opt; do echo "-a Build firmware images for all the platform defconfig's." echo "-b DIR Bind DIR to container." echo "-p List of comma separated platform names to build images for particular platforms." + echo "-s DIR SDK cache dir." + echo "-S Build SDK only" echo "-c Container to run in" echo "Example:DOCKER_PREFIX=sudo ./ci/build.sh -a" echo -e "\tDOCKER_PREFIX=sudo ./ci/build.sh -p firestone" @@ -62,12 +75,12 @@ function run_docker -t $1 $2 } -env +function toolchain_hash +{ + echo -n 'toolchain-'$((git submodule ; cd openpower/configs/; cat `ls -1 |grep '_defconfig$'|sort`)|sha1sum |sed -e 's/ .*//') +} -if [ -d output-images ]; then - echo 'output-images already exists!'; - exit 1; -fi +env for distro in $CONTAINERS; do @@ -108,13 +121,26 @@ ${HTTPS_PROXY_ENV} ${DL_DIR_ENV} ${CCACHE_DIR_ENV} EOF -) + ) $DOCKER_PREFIX docker build --network=host -t openpower/op-build-$distro - <<< "${Dockerfile}" - mkdir -p output-images/$distro - run_docker openpower/op-build-$distro "./ci/build-all-defconfigs.sh -o output-images/$distro -p $PLATFORMS ${release_args}" - if [ $? = 0 ]; then - mv *-images output-$distro/ + if [ -d "$SDK_CACHE" ]; then + SDK_DIR=$SDK_CACHE/$(toolchain_hash)-$distro + if [ ! -d "$SDK_DIR" ]; then + chmod +x ci/build-sdk.sh + run_docker openpower/op-build-$distro "./ci/build-sdk.sh $distro witherspoon_defconfig" + mv output-$distro-witherspoon_defconfig $SDK_DIR + $SDK_DIR/host/relocate-sdk.sh + fi + sdk_args="-s $SDK_DIR/host" else + sdk_args="" + fi + + if [ $SDK_ONLY == 0 ]; then + run_docker openpower/op-build-$distro "./ci/build-all-defconfigs.sh -o `pwd`/output-$distro -p $PLATFORMS ${release_args} ${sdk_args}" + fi + + if [ $? != 0 ]; then exit $?; fi done; |