From effcc2d20be69eb7466f0f5aec1c7cd2c0be0bca Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Mon, 5 Oct 2020 19:13:02 +0200
Subject: [PATCH] Working isolated environment

---
 garlic/exp/nbody/tampi.nix        |  4 ----
 garlic/stages/isolate/default.nix | 10 ++++++++++
 garlic/stages/isolate/stage1      | 16 ++++++++++++----
 overlay.nix                       |  4 ++++
 4 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/garlic/exp/nbody/tampi.nix b/garlic/exp/nbody/tampi.nix
index 153c7c4..398d4b9 100644
--- a/garlic/exp/nbody/tampi.nix
+++ b/garlic/exp/nbody/tampi.nix
@@ -112,10 +112,6 @@ let
 
   argv = {stage, conf, ...}: w.argv {
     program = stageProgram stage;
-    env = ''
-      set -e
-      export I_MPI_THREAD_SPLIT=1
-    '';
     argv = ''( -t ${toString conf.timesteps}
       -p ${toString conf.particles} )'';
   };
diff --git a/garlic/stages/isolate/default.nix b/garlic/stages/isolate/default.nix
index a8d0756..644d2c7 100644
--- a/garlic/stages/isolate/default.nix
+++ b/garlic/stages/isolate/default.nix
@@ -2,6 +2,7 @@
   stdenv
 , nixtools
 , busybox
+, strace
 }:
 
 {
@@ -14,15 +15,24 @@ stdenv.mkDerivation {
   name = "isolate";
   preferLocalBuild = true;
   phases = [ "unpackPhase" "installPhase" ];
+  buildInputs = [
+    #nixtools
+    #strace
+  ];
   src = ./.;
   dontPatchShebangs = true;
   programPath = "/bin/stage1";
   inherit program nixPrefix clusterName nixtools busybox;
   out = "$out";
   installPhase = ''
+
+    echo PATH=$PATH
+
     substituteAllInPlace stage1
     substituteAllInPlace stage2
 
+    sed -i "s|@extraPath@|$PATH|g" stage1
+
     mkdir -p $out/bin
     cp stage* $out/bin/
     chmod +x $out/bin/stage*
diff --git a/garlic/stages/isolate/stage1 b/garlic/stages/isolate/stage1
index 3f2b546..ee8b4d8 100644
--- a/garlic/stages/isolate/stage1
+++ b/garlic/stages/isolate/stage1
@@ -1,4 +1,4 @@
-#!/bin/sh -ex
+#!/bin/bash -ex
 
 >&2 echo Running isolate stage1
 >&2 echo PATH=$PATH
@@ -13,11 +13,19 @@ shell="@busybox@/bin/sh"
 nixjoin="@nixPrefix@@nixtools@/bin/nix-join"
 
 
+env=(
+  PATH="@nixPrefix@@busybox@/bin:@busybox@/bin:@extraPath@"
+  $(env | grep ^SLURM || true)
+  $(env | grep ^GARLIC_OUT || true)
+)
+
 #-m @nixPrefix@ \
-join_flags="
-	-m /etc \
+join_flags="-m /etc \
+	-m /.statelite/tmpfs/etc \
+	-m /sys \
 	-m /var/run/munge \
 	-m /gpfs/projects/bsc15 \
 	-m /bin:@nixPrefix@@busybox@/bin"
 
-exec $nixjoin -i $join_flags $nixhome -- @out@/bin/stage2
+exec $nixjoin -v -i $join_flags $nixhome -- \
+	env -i "${env[@]}" @out@/bin/stage2
diff --git a/overlay.nix b/overlay.nix
index a099497..0c646b8 100644
--- a/overlay.nix
+++ b/overlay.nix
@@ -131,6 +131,10 @@ let
 
     mpptest = callPackage ./bsc/mpptest/default.nix { };
 
+    busybox = self.busybox.override {
+      enableStatic = true;
+    };
+
     nixtools = callPackage ./bsc/nixtools/default.nix {
       targetCluster = "mn4";
       nixPrefix = "/gpfs/projects/bsc15/nix";