Bootstrap: docker
From: ppc64le/ubuntu:16.04

%labels

    APPLICATION_NAME CUDA
    APPLICATION_VERSION 9.2
    APPLICATION_URL https://developer.nvidia.com/cuda-zone

    SYSTEM_NAME HAL
    SYSTEM_SINGULARITY_VERSION 2.5.2
    SYSTEM_URL hal.ncsa.illinois.edu

    SINGULARITY_IMAGE_SIZE 32768

    AUTHOR_NAME Dawei Mu
    AUTHOR_EMAIL dmu@illinois.edu

    LAST_UPDATED 2019/04/10

%setup

%environment

    # Set system locale
    export LC_ALL=C
    export PATH="/usr/local/cuda-9.2/bin${PATH:+:${PATH}}"
    export LD_LIBRARY_PATH="/usr/local/cuda-9.2/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
    export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/cuda/extras/CUPTI/lib64"
    export PATH="/opt/openmpi-4.0.0/bin:${PATH}"
    export LD_LIBRARY_PATH="/opt/openmpi-4.0.0/lib:${LD_LIBRARY_PATH}"

%post -c /bin/bash

    # Set system locale
    export LC_ALL=C

    # Upgrade all packages to their latest versions
    apt-get -y update && apt-get -y upgrade

    # Install common packages from 'main'
    apt-get -y install autoconf
    apt-get -y install automake
    apt-get -y install build-essential
    apt-get -y install cmake
    apt-get -y install libtool
    apt-get -y install pkg-config
    apt-get -y install gfortran
    apt-get -y install zip

    # Install expect to automate responses for interactive build questions
    apt-get -y install expect

    # Make filesystem mount points
    # N/A

    # Location of NVIDIA repositories
    declare -r NVIDIA_ROOT_URL='http://developer.download.nvidia.com/compute'
    declare -r CUDA_REPO_URL="${NVIDIA_ROOT_URL}/cuda/repos/ubuntu1604/ppc64el"
    declare -r ML_REPO_URL="${NVIDIA_ROOT_URL}/machine-learning/repos/ubuntu1604/ppc64el/"

    # Install NVIDIA package dependencies
    apt-get -y install x11-common
    apt-get -y install xserver-xorg-core
    apt-get -y install xserver-xorg-legacy
    apt-get -y install cmake
    apt-get -y install make
    apt-get -y install dkms
    apt-get -y install linux-libc-dev
    apt-get -y install libc6-dev
    apt-get -y install lib32gcc1
    apt-get -y install libc6-i386
    apt-get -y install libgl1
    apt-get -y install libwayland-client0
    apt-get -y install libwayland-server0
    apt-get -y install pkg-config
    apt-get -y install screen-resolution-extra
    apt-get -y install libvdpau1
    apt-get -y install libatk1.0-0
    apt-get -y install libcairo-gobject2
    apt-get -y install libcairo2
    apt-get -y install libfontconfig1
    apt-get -y install libgdk-pixbuf2.0-0
    apt-get -y install libgtk-3-0
    apt-get -y install libgtk2.0-0
    apt-get -y install libjansson4
    apt-get -y install freeglut3-dev
    apt-get -y install libx11-dev
    apt-get -y install libxmu-dev
    apt-get -y install libxi-dev
    apt-get -y install libglu1-mesa
    apt-get -y install libglu1-mesa-dev
    apt-get -y install default-jre
    apt-get -y install wget

    # Download NVIDIA drivers, libraries, and packages
    wget --wait=10 "${CUDA_REPO_URL}/nvidia-396_396.26-0ubuntu1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/nvidia-396-dev_396.26-0ubuntu1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/nvidia-modprobe_396.26-0ubuntu1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/libcuda1-396_396.26-0ubuntu1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/nvidia-libopencl1-396_396.26-0ubuntu1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/nvidia-opencl-icd-396_396.26-0ubuntu1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/libxnvctrl0_396.26-0ubuntu1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/nvidia-settings_396.26-0ubuntu1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-drivers_396.26-1_ppc64el.deb"

    wget --wait=10 "${CUDA_REPO_URL}/cuda-license-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-misc-headers-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-nvcc-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-cuobjdump-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-nvprune-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-compiler-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-core-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-cudart-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-driver-dev-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-cudart-dev-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-nvdisasm-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-gdb-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-nvprof-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-memcheck-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-cupti-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-gpu-library-advisor-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-nvtx-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-command-line-tools-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-nvrtc-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-nvrtc-dev-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-nvml-dev-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-nvgraph-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-nvgraph-dev-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-cusolver-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-cusolver-dev-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-cublas-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-cublas-dev-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-cufft-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-cufft-dev-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-curand-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-curand-dev-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-cusparse-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-cusparse-dev-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-npp-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-npp-dev-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-samples-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-documentation-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-nsight-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-nvvp-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-visual-tools-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-tools-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-libraries-dev-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-toolkit-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-libraries-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-runtime-9-2_9.2.88-1_ppc64el.deb"
    wget --wait=10 "${CUDA_REPO_URL}/cuda-demo-suite-9-2_9.2.88-1_ppc64el.deb"

    wget --wait=10 "${ML_REPO_URL}/libcudnn7_7.2.1.38-1+cuda9.2_ppc64el.deb"
    wget --wait=10 "${ML_REPO_URL}/libcudnn7-dev_7.2.1.38-1+cuda9.2_ppc64el.deb"

    wget --wait=10 "${ML_REPO_URL}/libnccl2_2.4.2-1+cuda9.2_ppc64el.deb"
    wget --wait=10 "${ML_REPO_URL}/libnccl-dev_2.4.2-1+cuda9.2_ppc64el.deb"

    # Fix for "update-alternatives: error: error creating symbolic link
    # '/usr/lib/nvidia/alternate-install-present.dpkg-tmp': No such file or directory"
    mkdir -p /usr/lib/nvidia

    wget --wait=10 "http://launchpadlibrarian.net/252926390/acpid_2.0.26-1ubuntu2_ppc64el.deb"
    dpkg -i acpid_2.0.26-1ubuntu2_ppc64el.deb

    # Install NVIDIA drivers, libraries, and packages
    dpkg -i nvidia-396_396.26-0ubuntu1_ppc64el.deb
    dpkg -i nvidia-396-dev_396.26-0ubuntu1_ppc64el.deb
    dpkg -i nvidia-modprobe_396.26-0ubuntu1_ppc64el.deb
    dpkg -i libcuda1-396_396.26-0ubuntu1_ppc64el.deb
    dpkg -i nvidia-libopencl1-396_396.26-0ubuntu1_ppc64el.deb
    dpkg -i nvidia-opencl-icd-396_396.26-0ubuntu1_ppc64el.deb
    dpkg -i libxnvctrl0_396.26-0ubuntu1_ppc64el.deb
    dpkg -i nvidia-settings_396.26-0ubuntu1_ppc64el.deb
    dpkg -i cuda-drivers_396.26-1_ppc64el.deb

    dpkg -i cuda-license-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-misc-headers-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-nvcc-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-cuobjdump-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-nvprune-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-compiler-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-core-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-cudart-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-driver-dev-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-cudart-dev-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-nvdisasm-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-gdb-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-nvprof-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-memcheck-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-cupti-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-gpu-library-advisor-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-nvtx-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-command-line-tools-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-nvrtc-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-nvrtc-dev-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-nvml-dev-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-nvgraph-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-nvgraph-dev-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-cusolver-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-cusolver-dev-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-cublas-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-cublas-dev-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-cufft-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-cufft-dev-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-curand-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-curand-dev-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-cusparse-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-cusparse-dev-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-npp-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-npp-dev-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-samples-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-documentation-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-nsight-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-nvvp-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-visual-tools-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-tools-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-libraries-dev-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-toolkit-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-libraries-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-runtime-9-2_9.2.88-1_ppc64el.deb
    dpkg -i cuda-demo-suite-9-2_9.2.88-1_ppc64el.deb

    dpkg -i libcudnn7_7.2.1.38-1+cuda9.2_ppc64el.deb
    dpkg -i libcudnn7-dev_7.2.1.38-1+cuda9.2_ppc64el.deb

    dpkg -i libnccl2_2.4.2-1+cuda9.2_ppc64el.deb
    dpkg -i libnccl-dev_2.4.2-1+cuda9.2_ppc64el.deb

    # Fixing some TensorFlow and libnccl*.deb path issues ...
    mv /usr/lib/x86_64-linux-gnu/libnccl_static.a /usr/local/cuda-9.2/lib64/libnccl_static.a
    mv /usr/lib/x86_64-linux-gnu/libnccl.so.2.4.2 /usr/local/cuda-9.2/lib64/libnccl.so.2.4.2
    ln -s /usr/local/cuda-9.2/lib64/libnccl.so.2.4.2 /usr/local/cuda-9.2/lib64/libnccl.so.2
    ln -s /usr/local/cuda-9.2/lib64/libnccl.so.2 /usr/local/cuda-9.2/lib64/libnccl.so
    mv /usr/include/nccl.h /usr/local/cuda-9.2/include/nccl.h
    ln -s /usr/local/cuda-9.2/lib64 /usr/local/cuda-9.2/lib
    mv /usr/share/doc/libnccl2/NCCL-SLA.txt.gz /usr/local/cuda-9.2/NCCL-SLA.txt

    # Set paths to CUDA binaries and libraries
    export PATH="/usr/local/cuda-9.2/bin${PATH:+:${PATH}}"
    export LD_LIBRARY_PATH="/usr/local/cuda-9.2/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
    export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/cuda/extras/CUPTI/lib64"

    # Install basic drivers for user space access to Ethernet, RDMA,
    # and Infiniband. See https://community.mellanox.com/docs/DOC-2431
    apt-get -y install dkms
    apt-get -y install infiniband-diags
    apt-get -y install libibverbs-dev
    apt-get -y install ibacm
    apt-get -y install librdmacm-dev
    apt-get -y install libmlx4-dev
    apt-get -y install libmlx5-dev
    apt-get -y install mstflint
    apt-get -y install libibcm-dev
    apt-get -y install libibmad-dev
    apt-get -y install libibumad-dev
    apt-get -y install opensm
    apt-get -y install srptools

    # Install additional tools
    apt-get -y install ibutils
    apt-get -y install ibverbs-utils
    apt-get -y install rdmacm-utils
    apt-get -y install perftest
    apt-get -y install numactl
    apt-get -y install libnuma-dev

    # Install libnl
    apt-get -y install libnl-3-200
    apt-get -y install libnl-route-3-200
    apt-get -y install libnl-route-3-dev
    apt-get -y install libnl-utils

    # Install OpenMPI dependencies
    apt-get -y install zlib1g-dev

    # Download, build, and install CUDA-aware OpenMPI
    wget https://www.open-mpi.org/software/ompi/v4.0/downloads/openmpi-4.0.0.tar.gz
    tar -xzvf openmpi-4.0.0.tar.gz
    cd openmpi-4.0.0
    ./configure --prefix=/opt/openmpi-4.0.0 --with-cuda
    make all install

    # Set paths to OpenMPI binaries and libraries
    export PATH="/opt/openmpi-4.0.0/bin:${PATH}"
    export LD_LIBRARY_PATH="/opt/openmpi-4.0.0/lib:${LD_LIBRARY_PATH}"

%files

%runscript

%test
