From edf429c932ae8f33aa6c45786d60ce93dbfec6cb Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Tue, 22 Sep 2020 17:39:26 +0200
Subject: [PATCH] Avoid loading .bashrc

---
 garlic/exp/saiph/numcomm.nix | 20 ++++++++++++++++++--
 garlic/stages/argv.nix       |  2 +-
 garlic/stages/broom.nix      | 23 +++++++++++++++++++++++
 garlic/stages/control.nix    |  1 -
 garlic/stages/envRecord.nix  | 25 +++++++++++++++++++++++++
 garlic/stages/extrae.nix     |  3 +--
 garlic/stages/nix-setup.nix  |  3 ++-
 garlic/stages/perf.nix       |  2 +-
 garlic/stages/statspy.nix    |  2 +-
 9 files changed, 72 insertions(+), 9 deletions(-)
 create mode 100644 garlic/stages/broom.nix
 create mode 100644 garlic/stages/envRecord.nix

diff --git a/garlic/exp/saiph/numcomm.nix b/garlic/exp/saiph/numcomm.nix
index 541ab0f..7bc37fc 100644
--- a/garlic/exp/saiph/numcomm.nix
+++ b/garlic/exp/saiph/numcomm.nix
@@ -74,6 +74,7 @@ let
 
   nixsetup = {stage, conf, ...}: with conf; w.nixsetup {
     program = stageProgram stage;
+    nixsetup = "${nixPrefix}/bin/nix-setup";
   };
 
   extrae = {stage, conf, ...}:
@@ -102,6 +103,14 @@ let
     ];
   };
 
+  # Print the environment to ensure we don't get anything nasty
+  envRecord = {stage, conf, ...}: w.envRecord {
+    program = stageProgram stage;
+  };
+
+  broom = {stage, conf, ...}: w.broom {
+    program = stageProgram stage;
+  };
   # We may be able to use overlays by invoking the fix function directly, but we
   # have to get the definition of the bsc packages and the garlic ones as
   # overlays.
@@ -119,11 +128,18 @@ let
     };
 
   stages = with common; []
+    # Cleans ALL environment variables
+    ++ [ broom ]
+
     # Use sbatch to request resources first
     ++ optional enableSbatch sbatch
 
-    # Repeats the next stages N times
-    ++ optionals enableControl [ nixsetup control ]
+    # Record the current env vars set by SLURM to verify we don't have something
+    # nasty (like sourcing .bashrc). Take a look at #26
+    ++ [ envRecord ]
+
+    # Repeats the next stages N=30 times
+    ++ optional enableControl control
 
     # Executes srun to launch the program in the requested nodes, and
     # immediately after enters the nix environment again, as slurmstepd launches
diff --git a/garlic/stages/argv.nix b/garlic/stages/argv.nix
index 5060661..e879851 100644
--- a/garlic/stages/argv.nix
+++ b/garlic/stages/argv.nix
@@ -17,7 +17,7 @@ stdenv.mkDerivation {
   phases = [ "installPhase" ];
   installPhase = ''
     cat > $out <<EOF
-    #!${bash}/bin/bash
+    #!${bash}/bin/bash --norc
     # Requires /nix to use bash
     
     ${env}
diff --git a/garlic/stages/broom.nix b/garlic/stages/broom.nix
new file mode 100644
index 0000000..6586170
--- /dev/null
+++ b/garlic/stages/broom.nix
@@ -0,0 +1,23 @@
+{
+  stdenv
+}:
+
+{
+  program
+}:
+
+# This stage provides a clean environment to run experiments
+stdenv.mkDerivation {
+  name = "broom";
+  preferLocalBuild = true;
+  phases = [ "installPhase" ];
+  installPhase = ''
+    cat > $out <<EOF
+    #!/bin/sh
+
+    # Removes all environment variables
+    /usr/bin/env -i ${program}
+    EOF
+    chmod +x $out
+  '';
+}
diff --git a/garlic/stages/control.nix b/garlic/stages/control.nix
index e03f198..93b99b7 100644
--- a/garlic/stages/control.nix
+++ b/garlic/stages/control.nix
@@ -15,7 +15,6 @@ stdenv.mkDerivation {
   installPhase = ''
     cat > $out <<EOF
     #!/bin/sh
-    #set -e
     for n in {1..${toString loops}}; do
       ${program}
     done
diff --git a/garlic/stages/envRecord.nix b/garlic/stages/envRecord.nix
new file mode 100644
index 0000000..4d9a312
--- /dev/null
+++ b/garlic/stages/envRecord.nix
@@ -0,0 +1,25 @@
+{
+  stdenv
+}:
+
+{
+  program
+}:
+
+stdenv.mkDerivation {
+  name = "argv";
+  preferLocalBuild = true;
+  phases = [ "installPhase" ];
+  installPhase = ''
+    cat > $out <<EOF
+    #!/bin/sh
+    
+    echo ----- ENV BEGIN -------
+    /usr/bin/env
+    echo ----- ENV END -------
+
+    exec ${program} \''${argv[@]}
+    EOF
+    chmod +x $out
+  '';
+}
diff --git a/garlic/stages/extrae.nix b/garlic/stages/extrae.nix
index a450ca9..4d2db35 100644
--- a/garlic/stages/extrae.nix
+++ b/garlic/stages/extrae.nix
@@ -24,8 +24,7 @@ stdenv.mkDerivation {
   phases = [ "installPhase" ];
   installPhase = ''
     cat > $out <<EOF
-    #!${bash}/bin/bash
-    # Requires /nix to use bash
+    #!/bin/sh
     
     export EXTRAE_HOME=${extrae}
     export LD_PRELOAD=${extrae}/lib/lib${traceLib}trace.so:$LD_PRELOAD
diff --git a/garlic/stages/nix-setup.nix b/garlic/stages/nix-setup.nix
index e65e4c5..600984c 100644
--- a/garlic/stages/nix-setup.nix
+++ b/garlic/stages/nix-setup.nix
@@ -4,6 +4,7 @@
 
 {
   program
+, nixsetup
 }:
 
 stdenv.mkDerivation {
@@ -18,7 +19,7 @@ stdenv.mkDerivation {
     # We need to enter the nix namespace first, in order to have /nix
     # available, so we use this hack:
     if [ ! -e /nix ]; then
-      exec nix-setup \$0
+      exec ${nixsetup} \$0
     fi
 
     exec ${program}
diff --git a/garlic/stages/perf.nix b/garlic/stages/perf.nix
index f0a44b7..7518381 100644
--- a/garlic/stages/perf.nix
+++ b/garlic/stages/perf.nix
@@ -15,7 +15,7 @@ stdenv.mkDerivation {
   phases = [ "installPhase" ];
   installPhase = ''
     cat > $out <<EOF
-    #!${bash}/bin/bash
+    #!/bin/sh
 
     exec ${perf}/bin/perf ${perfArgs} ${program}
     EOF
diff --git a/garlic/stages/statspy.nix b/garlic/stages/statspy.nix
index 06e75e9..93e6459 100644
--- a/garlic/stages/statspy.nix
+++ b/garlic/stages/statspy.nix
@@ -16,7 +16,7 @@ stdenv.mkDerivation {
   installPhase = ''
     mkdir -p $out/bin
     cat > $out/bin/${name} <<EOF
-    #!${bash}/bin/bash
+    #!/bin/sh
 
     mkdir -p ${outputDir}
     cat /proc/[0-9]*/stat | sort -n > ${outputDir}/statspy.\$(date +%s.%3N).begin