diff --git a/stacks/dars/README.md b/stacks/dars/README.md new file mode 100644 index 0000000..67e640c --- /dev/null +++ b/stacks/dars/README.md @@ -0,0 +1,9 @@ +# Data Analytics Reference Stack + +This provides the Data Analytics Reference Stack. To offer more flexibility, +there are two versions of the Data Analytics Reference Stack: + +* A Clear Linux derived image optimized for [OpenBLAS](https://www.openblas.net/) +* A Clear Linux derived image optimized for IntelĀ® Math Kernel Library [MKL](https://software.intel.com/en-us/mkl) + +Please see the folders in this level about the variants and how to build and use them. diff --git a/stacks/dars/mkl/Dockerfile b/stacks/dars/mkl/Dockerfile new file mode 100644 index 0000000..3e44dda --- /dev/null +++ b/stacks/dars/mkl/Dockerfile @@ -0,0 +1,38 @@ +FROM clearlinux +LABEL maintainer="otc-swstacks@intel.com" + +ARG swupd_args + +# OS update and bundle installation +RUN swupd update $swupd_args && \ + swupd bundle-add --skip-diskspace-check \ + big-data-basic \ + which + +# general settings for all derived images +WORKDIR /root + +# ldconfig configuration +COPY dars.ld.so.conf . +RUN cat dars.ld.so.conf >> /etc/ld.so.conf + +# copy spark-core jar +COPY spark-core_2.12-2.4.0.jar /usr/share/apache-spark/jars/spark-core_2.12-2.4.0.jar + + +RUN swupd bundle-add curl cpio + +# mkl +COPY silent.cfg . +RUN curl http://registrationcenter-download.intel.com/akdlm/irc_nas/tec/15095/l_mkl_2019.2.187_online.tgz -o l_mkl.tgz +RUN mkdir l_mkl && tar -xvf l_mkl.tgz -C l_mkl --strip-components=1 && \ + l_mkl/install.sh -s silent.cfg && rm -rf l_mkl + +# mkl wrapper +RUN mkdir -p /opt/intel/mkl/wrapper && \ + curl -L https://github.com/Intel-bigdata/mkl_wrapper_for_non_CDH/raw/master/mkl_wrapper.jar -o /opt/intel/mkl/wrapper/mkl_wrapper.jar && \ + curl -L https://github.com/Intel-bigdata/mkl_wrapper_for_non_CDH/raw/master/mkl_wrapper.so -o /opt/intel/mkl/wrapper/mkl_wrapper.so + +RUN ldconfig + +CMD ["/bin/bash"] diff --git a/stacks/dars/mkl/README.md b/stacks/dars/mkl/README.md new file mode 100644 index 0000000..166934b --- /dev/null +++ b/stacks/dars/mkl/README.md @@ -0,0 +1,17 @@ +## Data Analytics Reference Stack with IntelĀ® MKL + +[![](https://images.microbadger.com/badges/image/clearlinux/stacks-dars-mkl.svg)](http://microbadger.com/images/clearlinux/stacks-dars-mkl "Get your own image badge on microbadger.com") + +### Building Locally + +Default build args in Docker are on: https://docs.docker.com/engine/reference/builder/#arg + +``` +docker build --no-cache -t clearlinux/stacks-dars-mkl . +``` + +### Build ARGs + +* `swupd_args` specifies [swupd update](https://github.com/clearlinux/swupd-client/blob/master/docs/swupd.1.rst#options) flags passed to the update during build. + +>NOTE: An empty `swupd_args` will default to Clear Linux OS latest version. Consider this when building from the Dockerfile, as an OS update will be performed. The docker image in this registry was built and validated using version 29200. diff --git a/stacks/dars/mkl/dars.ld.so.conf b/stacks/dars/mkl/dars.ld.so.conf new file mode 100644 index 0000000..24eaa80 --- /dev/null +++ b/stacks/dars/mkl/dars.ld.so.conf @@ -0,0 +1,2 @@ +/opt/intel/mkl/lib/intel64_lin +/opt/intel/lib/intel64_lin diff --git a/stacks/dars/mkl/silent.cfg b/stacks/dars/mkl/silent.cfg new file mode 100755 index 0000000..59dfe79 --- /dev/null +++ b/stacks/dars/mkl/silent.cfg @@ -0,0 +1,37 @@ +# Patterns used to check silent configuration file +# +# anythingpat - any string +# filepat - the file location pattern (/file/location/to/license.lic) +# lspat - the license server address pattern (0123@hostname) +# snpat - the serial number pattern (ABCD-01234567) + +# Accept EULA, valid values are: {accept, decline} +ACCEPT_EULA=accept + +# Optional error behavior, valid values are: {yes, no} +CONTINUE_WITH_OPTIONAL_ERROR=yes + +# Install location, valid values are: {/opt/intel, filepat} +PSET_INSTALL_DIR=/opt/intel + +# Continue with overwrite of existing installation directory, valid values are: {yes, no} +CONTINUE_WITH_INSTALLDIR_OVERWRITE=yes + +# List of components to install, valid values are: {ALL, DEFAULTS, anythingpat} +COMPONENTS=DEFAULTS + +# Installation mode, valid values are: {install, repair, uninstall} +PSET_MODE=install + +# Directory for non-RPM database, valid values are: {filepat} +#NONRPM_DB_DIR=filepat + +# Path to the cluster description file, valid values are: {filepat} +#CLUSTER_INSTALL_MACHINES_FILE=filepat + +# Perform validation of digital signatures of RPM files, valid values are: {yes, no} +SIGNING_ENABLED=yes + +# Select target architecture of your applications, valid values are: {IA32, INTEL64, ALL} +ARCH_SELECTED=INTEL64 + diff --git a/stacks/dars/mkl/spark-core_2.12-2.4.0.jar b/stacks/dars/mkl/spark-core_2.12-2.4.0.jar new file mode 100644 index 0000000..0686262 Binary files /dev/null and b/stacks/dars/mkl/spark-core_2.12-2.4.0.jar differ diff --git a/stacks/dars/openblas/Dockerfile b/stacks/dars/openblas/Dockerfile new file mode 100644 index 0000000..7ece43f --- /dev/null +++ b/stacks/dars/openblas/Dockerfile @@ -0,0 +1,35 @@ +FROM clearlinux +LABEL maintainer="otc-swstacks@intel.com" + +ARG swupd_args + +# OS update and bundle installation +RUN swupd update $swupd_args && \ + swupd bundle-add --skip-diskspace-check \ + big-data-basic \ + which + +# general settings for all derived images +WORKDIR /root + +# ldconfig configuration +COPY dars.ld.so.conf . +RUN cat dars.ld.so.conf >> /etc/ld.so.conf + +# copy spark-core jar +COPY spark-core_2.12-2.4.0.jar /usr/share/apache-spark/jars/spark-core_2.12-2.4.0.jar + + +RUN swupd bundle-add --skip-diskspace-check \ + python-basic-dev + +# TODO: remove this softlinks once CLR team fix it +ENV OPENBLAS_AVX512 /usr/lib64/haswell/avx512_1/libopenblas_skylakexp-r0.3.5.so +RUN ln -sf ${OPENBLAS_AVX512} /usr/lib64/haswell/libblas.so && \ + ln -sf ${OPENBLAS_AVX512} /usr/lib64/haswell/libblas.so.3 && \ + ln -sf ${OPENBLAS_AVX512} /usr/lib64/haswell/liblapack.so && \ + ln -sf ${OPENBLAS_AVX512} /usr/lib64/haswell/liblapack.so.3 + +RUN ldconfig + +CMD ["/bin/bash"] diff --git a/stacks/dars/openblas/README.md b/stacks/dars/openblas/README.md new file mode 100644 index 0000000..ce14398 --- /dev/null +++ b/stacks/dars/openblas/README.md @@ -0,0 +1,17 @@ +## Data Analytics Reference Stack with OpenBLAS + +[![](https://images.microbadger.com/badges/image/clearlinux/stacks-dars-openblas.svg)](http://microbadger.com/images/clearlinux/stacks-dars-openblas "Get your own image badge on microbadger.com") + +### Building Locally + +Default build args in Docker are on: https://docs.docker.com/engine/reference/builder/#arg + +``` +docker build --no-cache -t clearlinux/stacks-dars-openblas . +``` + +### Build ARGs + +* `swupd_args` specifies [swupd update](https://github.com/clearlinux/swupd-client/blob/master/docs/swupd.1.rst#options) flags passed to the update during build. + +>NOTE: An empty `swupd_args` will default to Clear Linux OS latest version. Consider this when building from the Dockerfile, as an OS update will be performed. The docker image in this registry was built and validated using version 29200. diff --git a/stacks/dars/openblas/dars.ld.so.conf b/stacks/dars/openblas/dars.ld.so.conf new file mode 100644 index 0000000..01215a3 --- /dev/null +++ b/stacks/dars/openblas/dars.ld.so.conf @@ -0,0 +1 @@ +/usr/lib64/haswell/avx512_1 diff --git a/stacks/dars/openblas/spark-core_2.12-2.4.0.jar b/stacks/dars/openblas/spark-core_2.12-2.4.0.jar new file mode 100644 index 0000000..0686262 Binary files /dev/null and b/stacks/dars/openblas/spark-core_2.12-2.4.0.jar differ