diff --git a/stacks/dars/mkl/Dockerfile b/stacks/dars/mkl/Dockerfile index 1787a51..b198a5a 100644 --- a/stacks/dars/mkl/Dockerfile +++ b/stacks/dars/mkl/Dockerfile @@ -1,38 +1,78 @@ -FROM clearlinux -LABEL maintainer="otc-swstacks@intel.com" - +FROM clearlinux:latest AS builder ARG swupd_args -WORKDIR /root +# Move to latest Clear Linux release to ensure +# that the swupd command line arguments are +# correct +RUN swupd update --no-boot-update $swupd_args -# ldconfig configuration -COPY dars.ld.so.conf . -RUN cat dars.ld.so.conf >> /etc/ld.so.conf +# Grab os-release info from the minimal base image so +# that the new content matches the exact OS version +COPY --from=clearlinux/os-core:latest /usr/lib/os-release / -# OS update and bundle installation -RUN swupd update $swupd_args && \ - swupd bundle-add --skip-diskspace-check \ - big-data-basic \ - which +# Install additional content in a target directory +# using the os version from the minimal base +RUN source /os-release && \ + mkdir /install_root \ + && swupd os-install -V ${VERSION_ID} \ + --path /install_root --statedir /swupd-state \ + --bundles=big-data-basic,cpio,os-core-update,which --no-boot-update \ + && rm -rf /install_root/var/lib/swupd/* -COPY profile /etc/profile +# For some Host OS configuration with redirect_dir on, +# extra data are saved on the upper layer when the same +# file exists on different layers. To minimize docker +# image size, remove the overlapped files before copy. +RUN mkdir /os_core_install +COPY --from=clearlinux/os-core:latest / /os_core_install/ +RUN find / os_core_install | sed -e 's/os_core_install/install_root/' | xargs rm -d &> /dev/null || true +FROM clearlinux/os-core:latest +LABEL maintainer=otc-swstacks@intel.com -# start: MKL specific -ENV MKL_INSTALLER=http://registrationcenter-download.intel.com/akdlm/irc_nas/tec/15095/l_mkl_2019.2.187_online.tgz -ENV MKL_WRAPPER=https://github.com/Intel-bigdata/mkl_wrapper_for_non_CDH/raw/master -ENV MKL_TARGET_DIR=/opt/intel/mkl/wrapper +ENV HOME=/root -COPY silent.cfg . -RUN swupd bundle-add curl cpio && \ - curl ${MKL_INSTALLER} -o l_mkl.tgz && \ - mkdir l_mkl && \ - tar -xvf l_mkl.tgz -C l_mkl --strip-components=1 && \ - l_mkl/install.sh -s silent.cfg && \ - rm -rf l_mkl && \ - mkdir -p ${MKL_TARGET_DIR} && \ - curl -L ${MKL_WRAPPER}/mkl_wrapper.jar -o ${MKL_TARGET_DIR}/mkl_wrapper.jar && \ - curl -L ${MKL_WRAPPER}/mkl_wrapper.so -o ${MKL_TARGET_DIR}/mkl_wrapper.so && \ +# Configure openjdk11 +ENV JAVA_HOME=/usr/lib/jvm/java-1.11.0-openjdk +ENV PATH="${JAVA_HOME}/bin:${PATH}" + +# Environment variables to point to Hadoop, +# Spark and YARN installation and configuration +ENV HADOOP_HOME=/usr +ENV HADOOP_CONF_DIR=/etc/hadoop +ENV HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native +ENV HADOOP_DEFAULT_LIBEXEC_DIR=$HADOOP_HOME/libexec +ENV HADOOP_IDENT_STRING=root +ENV HADOOP_LOG_DIR=/var/log/hadoop +ENV HADOOP_PID_DIR=/var/log/hadoop/pid +ENV HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib/native" + +ENV HDFS_DATANODE_USER=root +ENV HDFS_NAMENODE_USER=root +ENV HDFS_SECONDARYNAMENODE_USER=root + +ENV SPARK_HOME=/usr/share/apache-spark +ENV SPARK_CONF_DIR=/etc/spark + +ENV YARN_RESOURCEMANAGER_USER=root +ENV YARN_NODEMANAGER_USER=root + +COPY --from=builder /install_root / + +COPY dars.ld.so.conf /etc/ld.so.conf + +COPY silent.cfg binaries/l_mkl_2019.4.243_online.tgz / + +RUN mkdir mkl && tar -xvf l_mkl_2019.4.243_online.tgz -C mkl --strip-components=1 && \ + mkl/install.sh -s silent.cfg && \ + rm -rf silent.cfg l_mkl_2019.4.243_online.tgz mkl && \ ldconfig -CMD ["/bin/sh"] +COPY binaries/mkl_wrapper.so binaries/mkl_wrapper.jar /opt/intel/mkl/wrapper/ + +RUN mkdir -p /etc/spark /etc/hadoop + +COPY spark_conf/* /etc/spark/ +COPY hadoop_conf/* /etc/hadoop/ + +CMD ["/bin/bash"] diff --git a/stacks/dars/mkl/README.md b/stacks/dars/mkl/README.md index 5628444..a62b851 100644 --- a/stacks/dars/mkl/README.md +++ b/stacks/dars/mkl/README.md @@ -14,4 +14,4 @@ docker build --no-cache -t clearlinux/stacks-dars-mkl . * `swupd_args` specifies [swupd update](https://github.com/clearlinux/swupd-client/blob/master/docs/swupd.1.rst#options) flags passed to the update during build. ->NOTE: An empty `swupd_args` will default to Clear Linux OS latest version. Consider this when building from the Dockerfile, as an OS update will be performed. The docker image in this registry was built and validated using version 29480. +>NOTE: An empty `swupd_args` will default to Clear Linux OS latest version. Consider this when building from the Dockerfile, as an OS update will be performed. The docker image in this registry was built and validated using version `30690`. diff --git a/stacks/dars/mkl/binaries/l_mkl_2019.4.243_online.tgz b/stacks/dars/mkl/binaries/l_mkl_2019.4.243_online.tgz new file mode 100644 index 0000000..a434c46 Binary files /dev/null and b/stacks/dars/mkl/binaries/l_mkl_2019.4.243_online.tgz differ diff --git a/stacks/dars/mkl/binaries/mkl_wrapper.jar b/stacks/dars/mkl/binaries/mkl_wrapper.jar new file mode 100644 index 0000000..1e48f10 Binary files /dev/null and b/stacks/dars/mkl/binaries/mkl_wrapper.jar differ diff --git a/stacks/dars/mkl/binaries/mkl_wrapper.so b/stacks/dars/mkl/binaries/mkl_wrapper.so new file mode 100644 index 0000000..4f96e42 Binary files /dev/null and b/stacks/dars/mkl/binaries/mkl_wrapper.so differ diff --git a/stacks/dars/mkl/hadoop_conf/core-site.xml b/stacks/dars/mkl/hadoop_conf/core-site.xml new file mode 100644 index 0000000..e88f92a --- /dev/null +++ b/stacks/dars/mkl/hadoop_conf/core-site.xml @@ -0,0 +1,6 @@ + + + fs.defaultFS + hdfs://localhost:9000 + + \ No newline at end of file diff --git a/stacks/dars/mkl/hadoop_conf/hdfs-site.xml b/stacks/dars/mkl/hadoop_conf/hdfs-site.xml new file mode 100644 index 0000000..9133e39 --- /dev/null +++ b/stacks/dars/mkl/hadoop_conf/hdfs-site.xml @@ -0,0 +1,6 @@ + + + dfs.replication + 1 + + \ No newline at end of file diff --git a/stacks/dars/mkl/hadoop_conf/mapred-site.xml b/stacks/dars/mkl/hadoop_conf/mapred-site.xml new file mode 100644 index 0000000..ab03e51 --- /dev/null +++ b/stacks/dars/mkl/hadoop_conf/mapred-site.xml @@ -0,0 +1,6 @@ + + + mapreduce.framework.name + yarn + + \ No newline at end of file diff --git a/stacks/dars/mkl/hadoop_conf/workers b/stacks/dars/mkl/hadoop_conf/workers new file mode 100644 index 0000000..d18580b --- /dev/null +++ b/stacks/dars/mkl/hadoop_conf/workers @@ -0,0 +1 @@ +localhost \ No newline at end of file diff --git a/stacks/dars/mkl/hadoop_conf/yarn-site.xml b/stacks/dars/mkl/hadoop_conf/yarn-site.xml new file mode 100644 index 0000000..087b33b --- /dev/null +++ b/stacks/dars/mkl/hadoop_conf/yarn-site.xml @@ -0,0 +1,6 @@ + + + yarn.nodemanager.aux-services + mapreduce_shuffle + + \ No newline at end of file diff --git a/stacks/dars/mkl/spark_conf/spark-defaults.conf b/stacks/dars/mkl/spark_conf/spark-defaults.conf new file mode 100644 index 0000000..8801cc9 --- /dev/null +++ b/stacks/dars/mkl/spark_conf/spark-defaults.conf @@ -0,0 +1,3 @@ +# MKL flags +spark.executor.extraJavaOptions=-Dcom.github.fommil.netlib.BLAS=com.intel.mkl.MKLBLAS -Dcom.github.fommil.netlib.LAPACK=com.intel.mkl.MKLLAPACK +spark.driver.extraJavaOptions=-Dcom.github.fommil.netlib.BLAS=com.intel.mkl.MKLBLAS -Dcom.github.fommil.netlib.LAPACK=com.intel.mkl.MKLLAPACK diff --git a/stacks/dars/mkl/spark_conf/spark-env.sh b/stacks/dars/mkl/spark_conf/spark-env.sh new file mode 100644 index 0000000..ee32da8 --- /dev/null +++ b/stacks/dars/mkl/spark_conf/spark-env.sh @@ -0,0 +1,2 @@ +MKL_NUM_THREADS=1 +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib/native \ No newline at end of file diff --git a/stacks/dars/openblas/Dockerfile b/stacks/dars/openblas/Dockerfile index e308ba6..a278007 100644 --- a/stacks/dars/openblas/Dockerfile +++ b/stacks/dars/openblas/Dockerfile @@ -1,33 +1,71 @@ -FROM clearlinux -LABEL maintainer="otc-swstacks@intel.com" - +FROM clearlinux:latest AS builder ARG swupd_args -WORKDIR /root +# Move to latest Clear Linux release to ensure +# that the swupd command line arguments are +# correct +RUN swupd update --no-boot-update $swupd_args -# ldconfig configuration -COPY dars.ld.so.conf . -RUN cat dars.ld.so.conf >> /etc/ld.so.conf +# Grab os-release info from the minimal base image so +# that the new content matches the exact OS version +COPY --from=clearlinux/os-core:latest /usr/lib/os-release / -# OS update and bundle installation -RUN swupd update $swupd_args && \ - swupd bundle-add --skip-diskspace-check \ - big-data-basic \ - which +# Install additional content in a target directory +# using the os version from the minimal base +RUN source /os-release && \ + mkdir /install_root \ + && swupd os-install -V ${VERSION_ID} \ + --path /install_root --statedir /swupd-state \ + --bundles=big-data-basic,cpio,os-core-update,python-basic-dev,which --no-boot-update \ + && rm -rf /install_root/var/lib/swupd/* -COPY profile /etc/profile +# For some Host OS configuration with redirect_dir on, +# extra data are saved on the upper layer when the same +# file exists on different layers. To minimize docker +# image size, remove the overlapped files before copy. +RUN mkdir /os_core_install +COPY --from=clearlinux/os-core:latest / /os_core_install/ +RUN find / os_core_install | sed -e 's/os_core_install/install_root/' | xargs rm -d &> /dev/null || true +FROM clearlinux/os-core:latest +LABEL maintainer=otc-swstacks@intel.com -# start: OpenBLAS specific -ENV OPENBLAS_AVX512=/usr/lib64/haswell/avx512_1/libopenblas_skylakexp-r0.3.5.so +ENV HOME=/root -# TODO: remove this softlinks once CLR team fix it -RUN swupd bundle-add --skip-diskspace-check \ - python-basic-dev && \ - ln -sf ${OPENBLAS_AVX512} /usr/lib64/haswell/libblas.so && \ - ln -sf ${OPENBLAS_AVX512} /usr/lib64/haswell/libblas.so.3 && \ - ln -sf ${OPENBLAS_AVX512} /usr/lib64/haswell/liblapack.so && \ - ln -sf ${OPENBLAS_AVX512} /usr/lib64/haswell/liblapack.so.3 && \ - ldconfig +# Configure openjdk11 +ENV JAVA_HOME=/usr/lib/jvm/java-1.11.0-openjdk +ENV PATH="${JAVA_HOME}/bin:${PATH}" -CMD ["/bin/sh"] +# Environment variables to point to Hadoop, +# Spark and YARN installation and configuration +ENV HADOOP_HOME=/usr +ENV HADOOP_CONF_DIR=/etc/hadoop +ENV HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native +ENV HADOOP_DEFAULT_LIBEXEC_DIR=$HADOOP_HOME/libexec +ENV HADOOP_IDENT_STRING=root +ENV HADOOP_LOG_DIR=/var/log/hadoop +ENV HADOOP_PID_DIR=/var/log/hadoop/pid +ENV HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib/native" + +ENV HDFS_DATANODE_USER=root +ENV HDFS_NAMENODE_USER=root +ENV HDFS_SECONDARYNAMENODE_USER=root + +ENV SPARK_HOME=/usr/share/apache-spark +ENV SPARK_CONF_DIR=/etc/spark + +ENV YARN_RESOURCEMANAGER_USER=root +ENV YARN_NODEMANAGER_USER=root + +COPY --from=builder /install_root / + +COPY dars.ld.so.conf /etc/ld.so.conf + +RUN ldconfig + +RUN mkdir -p /etc/spark /etc/hadoop + +COPY spark_conf/* /etc/spark/ +COPY hadoop_conf/* /etc/hadoop/ + +CMD ["/bin/bash"] diff --git a/stacks/dars/openblas/README.md b/stacks/dars/openblas/README.md index 9ce358d..9251fa0 100644 --- a/stacks/dars/openblas/README.md +++ b/stacks/dars/openblas/README.md @@ -14,4 +14,4 @@ docker build --no-cache -t clearlinux/stacks-dars-openblas . * `swupd_args` specifies [swupd update](https://github.com/clearlinux/swupd-client/blob/master/docs/swupd.1.rst#options) flags passed to the update during build. ->NOTE: An empty `swupd_args` will default to Clear Linux OS latest version. Consider this when building from the Dockerfile, as an OS update will be performed. The docker image in this registry was built and validated using version 29480. +>NOTE: An empty `swupd_args` will default to Clear Linux OS latest version. Consider this when building from the Dockerfile, as an OS update will be performed. The docker image in this registry was built and validated using version `30690`. diff --git a/stacks/dars/openblas/hadoop_conf/core-site.xml b/stacks/dars/openblas/hadoop_conf/core-site.xml new file mode 100644 index 0000000..e88f92a --- /dev/null +++ b/stacks/dars/openblas/hadoop_conf/core-site.xml @@ -0,0 +1,6 @@ + + + fs.defaultFS + hdfs://localhost:9000 + + \ No newline at end of file diff --git a/stacks/dars/openblas/hadoop_conf/hdfs-site.xml b/stacks/dars/openblas/hadoop_conf/hdfs-site.xml new file mode 100644 index 0000000..9133e39 --- /dev/null +++ b/stacks/dars/openblas/hadoop_conf/hdfs-site.xml @@ -0,0 +1,6 @@ + + + dfs.replication + 1 + + \ No newline at end of file diff --git a/stacks/dars/openblas/hadoop_conf/mapred-site.xml b/stacks/dars/openblas/hadoop_conf/mapred-site.xml new file mode 100644 index 0000000..ab03e51 --- /dev/null +++ b/stacks/dars/openblas/hadoop_conf/mapred-site.xml @@ -0,0 +1,6 @@ + + + mapreduce.framework.name + yarn + + \ No newline at end of file diff --git a/stacks/dars/openblas/hadoop_conf/workers b/stacks/dars/openblas/hadoop_conf/workers new file mode 100644 index 0000000..d18580b --- /dev/null +++ b/stacks/dars/openblas/hadoop_conf/workers @@ -0,0 +1 @@ +localhost \ No newline at end of file diff --git a/stacks/dars/openblas/hadoop_conf/yarn-site.xml b/stacks/dars/openblas/hadoop_conf/yarn-site.xml new file mode 100644 index 0000000..087b33b --- /dev/null +++ b/stacks/dars/openblas/hadoop_conf/yarn-site.xml @@ -0,0 +1,6 @@ + + + yarn.nodemanager.aux-services + mapreduce_shuffle + + \ No newline at end of file diff --git a/stacks/dars/openblas/spark_conf/spark-defaults.conf b/stacks/dars/openblas/spark_conf/spark-defaults.conf new file mode 100644 index 0000000..4ea99cf --- /dev/null +++ b/stacks/dars/openblas/spark_conf/spark-defaults.conf @@ -0,0 +1,3 @@ +# OpenBlas confs +spark.executor.extraJavaOptions=-Dcom.github.fommil.netlib.BLAS=com.github.fommil.netlib.NativeSystemBLAS -Dcom.github.fommil.netlib.LAPACK=com.github.fommil.netlib.NativeSystemLAPACK -Dcom.github.fommil.netlib.ARPACK=com.github.fommil.netlib.NativeSystemARPACK +spark.driver.extraJavaOptions=-Dcom.github.fommil.netlib.BLAS=com.github.fommil.netlib.NativeSystemBLAS -Dcom.github.fommil.netlib.LAPACK=com.github.fommil.netlib.NativeSystemLAPACK -Dcom.github.fommil.netlib.ARPACK=com.github.fommil.netlib.NativeSystemARPACK diff --git a/stacks/dars/openblas/spark_conf/spark-env.sh b/stacks/dars/openblas/spark_conf/spark-env.sh new file mode 100644 index 0000000..bb21bd8 --- /dev/null +++ b/stacks/dars/openblas/spark_conf/spark-env.sh @@ -0,0 +1,2 @@ +OPENBLAS_NUM_THREADS=1 +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib/native \ No newline at end of file