[MIPS] Script to build a trusted mipsel-linux-gnu cross toolchain from source.

tools/trusted_cross_toolchains/trusted-toolchain-creator.mipsel.squeeze.sh

Index: tools/trusted_cross_toolchains/trusted-toolchain-creator.mipsel.squeeze.sh
diff --git a/tools/trusted_cross_toolchains/trusted-toolchain-creator.mipsel.squeeze.sh b/tools/trusted_cross_toolchains/trusted-toolchain-creator.mipsel.squeeze.sh
index 5f96de16e970b745f51871f7051ee31809b7de90..2774d6c4bb370a9b782b014e2ce1f276b7bfb39a 100755
--- a/tools/trusted_cross_toolchains/trusted-toolchain-creator.mipsel.squeeze.sh
+++ b/tools/trusted_cross_toolchains/trusted-toolchain-creator.mipsel.squeeze.sh
@@ -6,14 +6,41 @@
 #@ This script creates the mips trusted SDK.
 #@ It must be run from the native_client directory.
 
+# This script is intended to build a mipsel-linux-gnu cross compilation
+# toolchain that runs on x86 linux and generates code for a little-endian,
+# hard-float, mips32 target.
+#
+# It expects the host machine to have relatively recent versions of GMP (4.2.0
+# or later), MPFR (2.4.2), and MPC (0.8.1) in order to build the GCC.
+#
+# Common way to get those is:
+# sudo apt-get install libmpfr-dev libmpc-dev libgmp3-dev
+
 ######################################################################
 # Config
 ######################################################################
 
 set -o nounset
 set -o errexit
+set -o xtrace
+
+readonly MAKE_OPTS="-j8"
+readonly ARCH="mips32"
+
+readonly GCC_URL="http://ftp.gnu.org/gnu/gcc/gcc-4.6.3/gcc-4.6.3.tar.bz2"
+readonly GCC_SHA1SUM="ce317ca5c8185b58bc9300182b534608c578637f"
+
+readonly BINUTILS_URL="http://ftp.gnu.org/gnu/binutils/binutils-2.22.tar.bz2"
+readonly BINUTILS_SHA1SUM="65b304a0b9a53a686ce50a01173d1f40f8efe404"
+
+readonly KERNEL_URL="http://www.linux-mips.org/pub/linux/mips/kernel/v2.6/linux-2.6.38.4.tar.gz"
+readonly KERNEL_SHA1SUM="377fa5cf5f1d0c396759b1c4d147330e7e5b6d7f"
+
+readonly GDB_URL="http://ftp.gnu.org/gnu/gdb/gdb-7.5.tar.bz2"
+readonly GDB_SHA1SUM="79b61152813e5730fa670c89e5fc3c04b670b02c"
 
-readonly CS_URL=https://sourcery.mentor.com/sgpp/lite/mips/portal/package9761/public/mips-linux-gnu/mips-2011.09-75-mips-linux-gnu-i686-pc-linux-gnu.tar.bz2
+readonly EGLIBC_SVN_URL="svn://svn.eglibc.org/branches/eglibc-2_14"
+readonly EGLIBC_REVISION="20996"
 
 readonly DOWNLOAD_QEMU_URL="http://download.savannah.gnu.org/releases/qemu/qemu-0.12.5.tar.gz"
 
@@ -21,16 +48,16 @@ readonly INSTALL_ROOT=$(pwd)/toolchain/linux_mips-trusted
 
 readonly TMP=$(pwd)/toolchain/tmp/crosstool-trusted
 
-readonly PATCH_MIPS32=$(readlink -f ../third_party/qemu/qemu-0.12.5.patch_mips)
+readonly BUILD_DIR=${TMP}/build
 
-readonly CS_ROOT=${INSTALL_ROOT}/mips-release
+readonly PATCH_MIPS32=$(readlink -f ../third_party/qemu/qemu-0.12.5.patch_mips)
 
-readonly JAIL_MIPS32=${CS_ROOT}/mips-linux-gnu/libc/el
+readonly JAIL_MIPS32=${INSTALL_ROOT}/sysroot
 
-readonly  MAKE_OPTS="-j8"
-# These are simple compiler wrappers to force 32bit builds
+# These are simple compiler wrappers to force 32bit builds.
 readonly  CC32=$(readlink -f pnacl/scripts/mygcc32)
 readonly  CXX32=$(readlink -f pnacl/scripts/myg++32)
+
 ######################################################################
 # Helper
 ######################################################################
@@ -44,7 +71,7 @@ Banner() {
 SubBanner() {
   echo "......................................................................"
   echo $*
-  echo "...................................................................."
+  echo "......................................................................"
 }
 
 Usage() {
@@ -56,23 +83,49 @@ Usage() {
 }
 
 DownloadOrCopy() {

[robertm, 2012/10/10 15:38:27]

I think it would be clearer to have a separate function for checkouts. this way
you do not have to overload the purpose 
of the second argument.
Maybe change this in a subsequent CL

[petarj, 2012/10/11 00:44:06]

Done.

-  if [[ -f "$2" ]] ; then
-     echo "$2 already in place"
-  elif [[ $1 =~  'http://' || $1 =~  'https://' ]] ; then
-    SubBanner "downloading from $1 -> $2"
-    wget $1 -O $2
+  local url=${1}
+  local filename="${TMP}/${url##*/}"
+  local filetype="${url%%:*}"
+  local checksum=${2}

[robertm, 2012/10/10 15:38:27]

we usually do not use ${} for single char vars but it is ok 

[petarj, 2012/10/11 00:44:06]

Done.

+  local revision=${2}
+
+  if [ "${filename}" == "" ]; then
+    echo "Unknown error occured. Aborting."
+    exit 1
+  fi
+
+  if [ "${filetype}" == "svn" ]; then
+    SubBanner "checkout from ${url} -> ${filename}"
+    svn --force export -r ${revision} ${url} ${filename}
+  elif [[ "${filetype}" ==  "http" || ${filetype} ==  "https" ]] ; then
+    if [ ! -f "${filename}" ]; then
+      SubBanner "downloading from ${url} -> ${filename}"
+      wget ${url} -O ${filename}
+    fi
+    if [ "${checksum}" != "nochecksum" ]; then
+      CheckSHA1Sum ${filename} ${checksum}
+    fi
   else
-    SubBanner "copying from $1"
-    cp $1 $2
+    SubBanner "copying from ${url}"
+    cp ${url} ${filename}
   fi
 }
 
+CheckSHA1Sum() {
+    local calculated_checksum=`sha1sum ${filename} | cut -d ' ' -f 1`

[robertm, 2012/10/10 15:38:27]

use $() instead of backticks

[petarj, 2012/10/11 00:44:06]

Done.

+    if [ "${calculated_checksum}" != "${checksum}" ]; then
+      echo "${filename} sha1sum failed. Deleting file and aborting."
+      rm -f ${filename}
+      exit 1
+    fi
+}
+
 ######################################################################
 #
 ######################################################################
 
 # some sanity checks to make sure this script is run from the right place
-# with the right tools
+# with the right tools.
 SanityCheck() {
   Banner "Sanity Checks"
   if [[ $(basename $(pwd)) != "native_client" ]] ; then
@@ -100,6 +153,10 @@ ClearInstallDir() {
   rm -rf ${INSTALL_ROOT}/*
 }
 
+ClearBuildDir() {
+  Banner "clearing dirs in ${BUILD_DIR}"
+  rm -rf ${BUILD_DIR}/*
+}
 
 CreateTarBall() {
   local tarball=$1
@@ -108,43 +165,147 @@ CreateTarBall() {
 }
 
 
-# try to keep the tarball small
-PruneDirs() {
-  Banner "pruning code sourcery tree"
-  SubBanner "Size before: $(du -msc  ${CS_ROOT})"
-  rm -rf ${CS_ROOT}/share
-  rm -rf ${CS_ROOT}/mips-linux-gnu/lib/uclibc
-  rm -rf ${CS_ROOT}/mips-linux-gnu/lib/soft-float
-  rm -rf ${CS_ROOT}/mips-linux-gnu/lib/micromips
+# Download the toolchain source tarballs or use a local copy when available.
+DownloadOrCopyAndInstallToolchain() {
+  Banner "Installing toolchain"
 
-  rm -rf ${CS_ROOT}/mips-linux-gnu/libc/uclibc
-  rm -rf ${CS_ROOT}/mips-linux-gnu/libc/soft-float
-  rm -rf ${CS_ROOT}/mips-linux-gnu/libc/micromips
+  local tarball="${TMP}/${BINUTILS_URL##*/}"
+  DownloadOrCopy ${BINUTILS_URL} ${BINUTILS_SHA1SUM}
+  SubBanner "extracting from ${tarball}"
+  tar jxf ${tarball} -C ${TMP}
 
-  rm -rf ${CS_ROOT}/lib/gcc/mips-linux-gnu/4.4.1/uclibc
-  rm -rf ${CS_ROOT}/lib/gcc/mips-linux-gnu/4.4.1/soft-float
-  rm -rf ${CS_ROOT}/lib/gcc/mips-linux-gnu/4.4.1/micromips
+  tarball="${TMP}/${GCC_URL##*/}"
+  DownloadOrCopy ${GCC_URL} ${GCC_SHA1SUM}
+  SubBanner "extracting from ${tarball}"
+  tar jxf ${tarball} -C ${TMP}
 
-  rm -rf ${CS_ROOT}/mips-linux-gnu/include/c++/4.4.1/mips-linux-gnu/uclibc
-  rm -rf ${CS_ROOT}/mips-linux-gnu/include/c++/4.4.1/mips-linux-gnu/soft-float
-  rm -rf ${CS_ROOT}/mips-linux-gnu/include/c++/4.4.1/mips-linux-gnu/micromips
+  tarball="${TMP}/${GDB_URL##*/}"
+  DownloadOrCopy ${GDB_URL} ${GDB_SHA1SUM}
+  SubBanner "extracting from ${tarball}"
+  tar jxf ${tarball} -C ${TMP}
 
-  SubBanner "Size after: $(du -msc  ${CS_ROOT})"
-}
+  tarball="${TMP}/${KERNEL_URL##*/}"
+  DownloadOrCopy ${KERNEL_URL} ${KERNEL_SHA1SUM}
+  SubBanner "extracting from ${tarball}"
+  tar zxf ${tarball} -C ${TMP}
 
+  local eglibc_dir="${TMP}/${EGLIBC_SVN_URL##*/}"
+  DownloadOrCopy ${EGLIBC_SVN_URL} ${EGLIBC_REVISION}
 
-# Download the codesourcery tarball or use a local copy when available.
-DownloadOrCopyAndInstallCodeSourceryTarball() {
-  Banner "Installing Codesourcery Toolchain"
-  local tarball="${TMP}/${CS_URL##*/}"
-  DownloadOrCopy ${CS_URL} ${tarball}
+  Banner "Preparing the code"
 
-  SubBanner "Untaring  ${INSTALL_ROOT}/${tarball}"
-  tar jxf ${tarball} -C ${INSTALL_ROOT}
+  cd ${TMP}/eglibc-2_14/libc/ && ln -s ../ports ports && cd -

[robertm, 2012/10/10 15:38:27]

we do not use long chains of &&

for the "ln" case just fix the args to ln

for the sed case, why no define local variables, like
local FILE_NAME="$PWD/tc-mips.c
etc
and the just say
(cd XXX; sed YYY)  # no need for cd -

(cd XXX; a=b c=d 

[petarj, 2012/10/11 00:44:06]

Done.

+  cd ${TMP}/binutils-2.22/gas/config                                           \

[robertm, 2012/10/10 15:38:27]

add a comment for this

[petarj, 2012/10/11 00:44:06]

Done.

+    && OLD_TEXT="as_warn_where (fragp->fr_file, fragp->fr_line, msg);"         \
+    && NEW_TEXT="as_warn_where (fragp->fr_file, fragp->fr_line, \"%s\", msg);" \
+    && FILE_NAME="$PWD/tc-mips.c"                                              \
+    && sed -i "s/$OLD_TEXT/$NEW_TEXT/g" "$FILE_NAME" && cd -
 
-  pushd ${INSTALL_ROOT}
-  mv mips-* mips-release
-  popd
+
+  Banner "Building toolchain"
+
+  mkdir -p ${BUILD_DIR}/binutils/
+  cd ${BUILD_DIR}/binutils/
+
+  ${TMP}/binutils-2.22/configure --prefix=${INSTALL_ROOT}         \

[robertm, 2012/10/10 15:38:27]

arg alignment

[petarj, 2012/10/11 00:44:06]

Done.

+                                        --target=mipsel-linux-gnu       \
+                                        --with-sysroot=${JAIL_MIPS32}
+
+  make ${MAKE_OPTS} all-binutils all-gas all-ld
+  make ${MAKE_OPTS} install-binutils install-gas install-ld
+
+
+  mkdir -p ${BUILD_DIR}/gcc/initial
+  cd ${BUILD_DIR}/gcc/initial
+
+  ${TMP}/gcc-4.6.3/configure --prefix=${INSTALL_ROOT} --disable-libssp   \

[robertm, 2012/10/10 15:38:27]

one arg per line is easier to read (optional)

[petarj, 2012/10/11 00:44:06]

Done.

+    --disable-libgomp --disable-libmudflap --disable-fixed-point        \
+    --disable-decimal-float --with-mips-plt --with-endian=little        \
+    --with-arch=${ARCH} --target=mipsel-linux-gnu --enable-languages=c  \
+    --with-newlib --without-headers --disable-shared --disable-threads  \
+    --disable-libquadmath --disable-libatomic
+
+  make ${MAKE_OPTS} all
+  make ${MAKE_OPTS} install
+
+
+  cd ${TMP}/linux-2.6.38.4
+  make headers_install ARCH=mips INSTALL_HDR_PATH=${JAIL_MIPS32}/usr
+
+  export PATH=${INSTALL_ROOT}/bin:$PATH

[robertm, 2012/10/10 15:38:27]

it  may be better to not export them put just 
prepend them before the configure command like so
X=Y A=B configure ....

[petarj, 2012/10/11 00:44:06]

Done.

+  export BUILD_CC=gcc
+  export AR=mipsel-linux-gnu-ar
+  export RANLIB=mipsel-linux-gnu-ranlib
+  export CC=mipsel-linux-gnu-gcc
+  export CXX=mipsel-linux-gnu-g++
+  mkdir -p ${BUILD_DIR}/eglibc/initial
+  cd ${BUILD_DIR}/eglibc/initial
+
+  ${TMP}/eglibc-2_14/libc/configure --prefix=/usr --enable-add-ons      \
+    --with-headers=${JAIL_MIPS32}/usr/include --build=i686-pc-linux-gnu \
+    --host=mipsel-linux-gnu --disable-profile --without-gd --without-cvs
+
+  make ${MAKE_OPTS} install-headers install_root=${JAIL_MIPS32} install-bootstrap-headers=yes

[robertm, 2012/10/10 15:38:27]

line length

[petarj, 2012/10/11 00:44:06]

Done.

+  mkdir -p ${JAIL_MIPS32}/usr/lib
+
+  make csu/subdir_lib  && cp csu/crt1.o csu/crti.o csu/crtn.o ${JAIL_MIPS32}/usr/lib
+  mipsel-linux-gnu-gcc -nostdlib -nostartfiles -shared -x c /dev/null   \
+                       -o ${JAIL_MIPS32}/usr/lib/libc.so
+
+
+  unset BUILD_CC AR RANLIB CC CXX

[robertm, 2012/10/10 15:38:27]

comment?
(may not be needed anymore if you follow the suggestion above)

+  mkdir -p ${BUILD_DIR}/gcc/intermediate
+  cd ${BUILD_DIR}/gcc/intermediate
+
+  ${TMP}/gcc-4.6.3/configure --prefix=${INSTALL_ROOT} --disable-libssp   \
+    --disable-libgomp --disable-libmudflap --disable-fixed-point        \
+    --disable-decimal-float --with-mips-plt --with-endian=little        \
+    --with-arch=${ARCH} --target=mipsel-linux-gnu --enable-languages=c  \
+    --with-sysroot=${JAIL_MIPS32} --disable-libquadmath --disable-libatomic
+
+  make ${MAKE_OPTS} all
+  make ${MAKE_OPTS} install
+
+
+  export BUILD_CC=gcc

[robertm, 2012/10/10 15:38:27]

see above

[petarj, 2012/10/11 00:44:06]

Done.

+  export AR=mipsel-linux-gnu-ar
+  export RANLIB=mipsel-linux-gnu-ranlib
+  export CC=mipsel-linux-gnu-gcc
+  export CXX=mipsel-linux-gnu-g++
+  mkdir -p ${BUILD_DIR}/eglibc/final
+  cd ${BUILD_DIR}/eglibc/final
+
+  ${TMP}/eglibc-2_14/libc/configure --prefix=/usr --enable-add-ons       \
+    --with-headers=${JAIL_MIPS32}/usr/include --build=i686-pc-linux-gnu  \
+    --host=mipsel-linux-gnu --disable-profile --without-gd --without-cvs
+
+  make ${MAKE_OPTS} all
+  make ${MAKE_OPTS} install install_root=${JAIL_MIPS32}
+
+
+  unset BUILD_CC AR RANLIB CC CXX
+  mkdir -p ${BUILD_DIR}/gcc/final
+  cd ${BUILD_DIR}/gcc/final
+
+  ${TMP}/gcc-4.6.3/configure --prefix=${INSTALL_ROOT} --disable-libssp    \
+    --disable-libgomp --disable-libmudflap --disable-fixed-point         \
+    --disable-decimal-float --with-mips-plt --with-endian=little         \
+    --with-arch=${ARCH} --target=mipsel-linux-gnu --enable-__cxa_atexit  \
+    --with-sysroot=${JAIL_MIPS32} --enable-languages=c,c++
+
+  make ${MAKE_OPTS} all
+  make ${MAKE_OPTS} install
+
+
+  mkdir -p ${BUILD_DIR}/gdb/
+  cd ${BUILD_DIR}/gdb/
+
+  ${TMP}/gdb-7.5/configure --prefix=${INSTALL_ROOT} --target=mipsel-linux-gnu
+
+  make ${MAKE_OPTS} all-gdb
+  make ${MAKE_OPTS} install-gdb
+
+  cd ${INSTALL_ROOT}/../..
 }
 
 
@@ -156,23 +317,13 @@ InstallTrustedLinkerScript() {
   # to move the sel_ldr and other images "out of the way"
   Banner "installing trusted linker script to ${trusted_ld_script}"
 
-  ${CS_ROOT}/bin/mips-linux-gnu-ld  --verbose |\
+  ${INSTALL_ROOT}/bin/mipsel-linux-gnu-ld  --verbose |\
       grep -A 10000 "=======" |\
       grep -v "=======" |\
       sed -e 's/0400000/70000000/g' > ${trusted_ld_script}
 }
 
 
-InstallMissingHeaders() {
-  Banner "installing openssl headers from local system"
-  cp -r /usr/include/openssl ${JAIL_MIPS32}/usr/include/
-}
-
-
-MissingSharedLibCleanup() {
-  Banner "Cleanup dangling symlinks"
-}
-
 # ----------------------------------------------------------------------
 # mips32 deb files to complete our code sourcery jail
 # ----------------------------------------------------------------------
@@ -182,7 +333,7 @@ readonly MIPS32_PACKAGES=${REPO_DEBIAN}/dists/squeeze/main/binary-mipsel/Package
 
 readonly TMP_PACKAGELIST_MIPS32=${TMP}/../packagelist_mipsel.tmp
 
-# These are good enough for native client
+# These are good enough for native client.
 readonly BASE_PACKAGES="\
   libssl0.9.8 \
   libssl-dev \
@@ -215,11 +366,11 @@ readonly BASE_PACKAGES="\
 GeneratePackageLists() {
   Banner "generating package lists for mips32"
   echo -n > ${TMP_PACKAGELIST_MIPS32}
-  DownloadOrCopy ${MIPS32_PACKAGES} ${TMP}/../Packages_mipsel.bz2
-  bzcat ${TMP}/../Packages_mipsel.bz2\
-    | egrep '^(Package:|Filename:)' > ${TMP}/../Packages_mipsel
+  DownloadOrCopy ${MIPS32_PACKAGES} nochecksum
+  bzcat ${TMP}/Packages.bz2\
+    | egrep '^(Package:|Filename:)' > ${TMP}/Packages_mipsel
   for pkg in ${BASE_PACKAGES} ; do
-    grep  -A 1 "${pkg}\$" ${TMP}/../Packages_mipsel\
+    grep  -A 1 "${pkg}\$" ${TMP}/Packages_mipsel\
       | egrep -o "pool/.*" >> ${TMP_PACKAGELIST_MIPS32}
   done
 }
@@ -229,7 +380,7 @@ InstallMissingLibraries() {
   for file in ${DEP_FILES_NEEDED_MIPS32} ; do
     local package="${TMP}/${file##*/}"
     Banner "installing ${file}"
-    DownloadOrCopy ${REPO_DEBIAN}/${file} ${package}
+    DownloadOrCopy ${REPO_DEBIAN}/${file} nochecksum
     SubBanner "extracting to ${JAIL_MIPS32}"
     dpkg --fsys-tarfile ${package}\
       | tar -xvf - --exclude=./usr/share -C ${JAIL_MIPS32}
@@ -263,12 +414,6 @@ FixLibs() {
 
   rm -f libresolv.so
   ln -s ../../lib/libresolv.so.2  libresolv.so
-
-  echo "OUTPUT_FORMAT(elf32-tradlittlemips)" > libc.so
-  echo "GROUP ( libc.so.6 libc_nonshared.a  AS_NEEDED ( ld.so.1 ) )" >> libc.so
-
-  echo "OUTPUT_FORMAT(elf32-tradlittlemips)" > libpthread.so
-  echo "GROUP ( libpthread.so.0 libpthread_nonshared.a )" >> libpthread.so
 }
 
 BuildAndInstallQemu() {
@@ -315,6 +460,7 @@ BuildAndInstallQemu() {
   cp tools/trusted_cross_toolchains/qemu_tool_mips32.sh ${INSTALL_ROOT}
   ln -sf qemu_tool_mips32.sh ${INSTALL_ROOT}/run_under_qemu_mips32
 }
+
 ######################################################################
 # Main
 ######################################################################
@@ -329,12 +475,10 @@ elif [[ $1 == "trusted_sdk" ]]; then
   mkdir -p ${TMP}
   SanityCheck
   ClearInstallDir
-  DownloadOrCopyAndInstallCodeSourceryTarball
-  PruneDirs
+  ClearBuildDir
+  DownloadOrCopyAndInstallToolchain
   GeneratePackageLists
-  InstallMissingHeaders
   InstallMissingLibraries
-  MissingSharedLibCleanup
   InstallTrustedLinkerScript
   BuildAndInstallQemu
   CreateTarBall $1