From 23fa7d86544dcb561e98b68fc3e9a0abc0f6a489 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias <rodrigo.arias@bsc.es>
Date: Mon, 17 Aug 2020 18:51:51 +0200
Subject: [PATCH] Update and fix Intel MPI, fixes #9

---
 bsc/intel-mpi/default.nix  | 43 +++++++++++++++++++++++++++++++-------
 bsc/intel-mpi/mpicc.patch  |  4 ++--
 bsc/intel-mpi/mpicxx.patch |  4 ++--
 3 files changed, 39 insertions(+), 12 deletions(-)

diff --git a/bsc/intel-mpi/default.nix b/bsc/intel-mpi/default.nix
index 0b30487..261654a 100644
--- a/bsc/intel-mpi/default.nix
+++ b/bsc/intel-mpi/default.nix
@@ -1,30 +1,39 @@
 { stdenv
 , rpmextract
-, libfabric
 , gcc
 , zlib
+, ucx
+, numactl
+, rdma-core
+, libpsm2
+, patchelf
 , autoPatchelfHook
 , enableDebug ? false
 }:
 
 stdenv.mkDerivation rec {
   name = "intel-mpi-${version}";
-  version = "2019.7.217";
-  dir_nr = "16546";
+  version = "2019.8.254";
+  dir_nr = "16814";
+  internal-ver = "2020.2.254";
 
   lib_variant = (if enableDebug then "debug" else "release");
 
   src = builtins.fetchTarball {
-    url = "http://registrationcenter-download.intel.com/akdlm/IRC_NAS/tec/${dir_nr}/l_mpi_${version}.tgz";
-    sha256 = "19l995aavbn5lkiz9sxl6iwmjsrvjgjp14nn0qi1hjqs705db5li";
+    url = "http://registrationcenter-download.intel.com/akdlm/irc_nas/tec/${dir_nr}/l_mpi_${version}.tgz";
+    sha256 = "1za4zyvxm5bfkrca843na6sxq2gq7qb87s0zysa7dnyqjwa11n45";
   };
 
   buildInputs = [
     rpmextract
-    libfabric
     autoPatchelfHook
     gcc.cc.lib
     zlib
+    ucx
+    numactl
+    rdma-core
+    libpsm2
+    patchelf
   ];
 
   postUnpack = ''
@@ -39,7 +48,7 @@ stdenv.mkDerivation rec {
   ];
 
   postPatch = ''
-    pushd opt/intel/compilers_and_libraries_2020.1.217/linux/mpi/intel64/bin
+    pushd opt/intel/compilers_and_libraries_${internal-ver}/linux/mpi/intel64/bin
       for i in mpi* ; do
         echo "Fixing paths in $i"
         sed -i "s:I_MPI_SUBSTITUTE_INSTALLDIR:$out:g" "$i"
@@ -50,7 +59,7 @@ stdenv.mkDerivation rec {
   dontBuild = true;
 
   installPhase = ''
-    cd opt/intel/compilers_and_libraries_2020.1.217/linux/mpi/intel64
+    cd opt/intel/compilers_and_libraries_${internal-ver}/linux/mpi/intel64
     mkdir -p $out
     mv etc $out
     mv bin $out 
@@ -58,7 +67,25 @@ stdenv.mkDerivation rec {
     mkdir $out/lib
     cp -a lib/lib* $out/lib
     cp -a lib/${lib_variant}_mt/lib* $out/lib
+    cp -a libfabric/lib/* $out/lib
+    cp -a libfabric/lib/prov/* $out/lib
+    cp -a libfabric/bin/* $out/bin
     ln -s . $out/intel64
     rm $out/lib/libmpi.dbg
+
+    # Fixup Intel PSM2 library missing (now located at PSMX2)
+    ln -s $out/lib/libpsmx2-fi.so $out/lib/libpsm2-fi.so
+  '';
+
+  dontAutoPatchelf = true;
+
+  # The rpath of libfabric.so bundled with Intel MPI is patched to include the
+  # rdma-core lib path, as is required for dlopen to find the rdma components.
+  # TODO: Try the upstream libfabric library with rdma support, so we can avoid
+  # this hack.
+  postFixup = ''
+    autoPatchelf -- $out
+    patchelf --set-rpath "$out/lib:${rdma-core}/lib:${libpsm2}/lib" $out/lib/libfabric.so
+    echo "Patched RPATH in libfabric.so to: $(patchelf --print-rpath $out/lib/libfabric.so)"
   '';
 }
diff --git a/bsc/intel-mpi/mpicc.patch b/bsc/intel-mpi/mpicc.patch
index c98dffb..c252490 100644
--- a/bsc/intel-mpi/mpicc.patch
+++ b/bsc/intel-mpi/mpicc.patch
@@ -1,5 +1,5 @@
---- a/opt/intel/compilers_and_libraries_2020.1.217/linux/mpi/intel64/bin/mpicc	2020-06-30 14:22:04.510566478 +0200
-+++ b/opt/intel/compilers_and_libraries_2020.1.217/linux/mpi/intel64/bin/mpicc	2020-06-30 14:24:58.250998988 +0200
+--- a/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin/mpicc	2020-06-30 14:22:04.510566478 +0200
++++ b/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin/mpicc	2020-06-30 14:24:58.250998988 +0200
 @@ -50,7 +50,7 @@
  if [ x"$opt_args" == x"" ]; then
      case "${compiler_short_name}" in
diff --git a/bsc/intel-mpi/mpicxx.patch b/bsc/intel-mpi/mpicxx.patch
index 16602f8..a51d170 100644
--- a/bsc/intel-mpi/mpicxx.patch
+++ b/bsc/intel-mpi/mpicxx.patch
@@ -1,5 +1,5 @@
---- a/opt/intel/compilers_and_libraries_2020.1.217/linux/mpi/intel64/bin/mpicxx	2020-06-30 14:26:14.619723932 +0200
-+++ b/opt/intel/compilers_and_libraries_2020.1.217/linux/mpi/intel64/bin/mpicxx	2020-06-30 14:27:56.644687134 +0200
+--- a/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin/mpicxx	2020-06-30 14:26:14.619723932 +0200
++++ b/opt/intel/compilers_and_libraries_2020.2.254/linux/mpi/intel64/bin/mpicxx	2020-06-30 14:27:56.644687134 +0200
 @@ -49,9 +49,9 @@
  
  if [ x"$opt_args" == x"" ]; then