fwi: generate the input at runtime

garlic/exp/fwi/common.nix

@@ -40,41 +40,85 @@ rec {
     ntasksPerNode = hw.cpusPerNode;
   };
 
+  srun = {nextStage, conf, ...}:
+  let
+    fwiParams = bsc.apps.fwi.params.override {
+      inherit (conf) nx ny nz;
+    };
+  in
+    stdexp.stdStages.srun {
+      inherit nextStage conf;
+      # Now we add some commands to execute before calling srun. These will
+      # only run in one rank (the first in the list of allocated nodes)
+      preSrun = ''
+        export GARLIC_FWI_SRUNDIR=$(pwd)
+        export GARLIC_FWI_EXECDIR="${conf.tempDir}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN"
+        mkdir -p "$GARLIC_FWI_EXECDIR"
+
+        export GARLIC_FWI_PARAMS="${fwiParams}/fwi_params.txt"
+        export GARLIC_FWI_FREQ="${fwiParams}/fwi_frequencies.txt"
+
+        # We cannot change the working directory of srun, so we use a
+        # subshell to ignore the cd
+        (
+          # Generate the input dataset
+          >&2 echo "generating the input dataset"
+          cd "$GARLIC_FWI_EXECDIR"
+          ${fwiParams}/bin/ModelGenerator \
+            -m "$GARLIC_FWI_PARAMS" "$GARLIC_FWI_FREQ"
+        )
+      '';
+
+      postSrun = optionalString (conf.enableCTF) ''
+        # Save the traces
+        mv "$GARLIC_FWI_EXECDIR"/trace_* .
+      '' + ''
+        # Remove everything else
+        rm -rf "$GARLIC_FWI_EXECDIR"
+      '';
+    };
+
   exec = {nextStage, conf, ...}: stages.exec {
     inherit nextStage;
 
+    # FIXME: FWI should allow the I/O directory to be specified as a
+    # parameter
     pre = ''
-      CDIR=$(pwd)
-      EXECDIR="${conf.tempDir}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN"
-      mkdir -p "$EXECDIR"
-      cd "$EXECDIR"
-      ln -fs ${conf.fwiInput}/InputModels InputModels || true
+      # Run fwi at the in a directory with fast local storage
+      cd "$GARLIC_FWI_EXECDIR"
+
+      echo >&2 "Current dir: $(pwd)"
+      echo >&2 "Using PARAMS=$GARLIC_FWI_PARAMS and FREQ=$GARLIC_FWI_FREQ"
     '' + optionalString (conf.enableCTF) ''
       export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf"
     '';
 
     argv = [
-      "${conf.fwiInput}/fwi_params.txt"
-      "${conf.fwiInput}/fwi_frequencies.txt"
+      ''"$GARLIC_FWI_PARAMS"''
+      ''"$GARLIC_FWI_FREQ"''
     ] ++ optional (needsBlocksize conf) conf.blocksize ++ [
       "-1" # Fordward steps
       "-1" # Backward steps
       conf.ioFreq # Write/read frequency
     ];
-
-    post = ''
-      rm -rf Results || true
-    '' + optionalString (conf.enableCTF) ''
-      mv trace_* "$CDIR"
-    '';
   };
 
   apps = bsc.garlic.apps;
 
   # FWI program
-  program = {nextStage, conf, ...}: apps.fwi.solver.override {
-    inherit (conf) gitBranch fwiInput;
-  };
+  program = {nextStage, conf, ...}:
+  let
+    fwiParams = bsc.apps.fwi.params.override {
+      inherit (conf) nx ny nz;
+    };
+  in
+    apps.fwi.solver.override {
+      inherit (conf) gitBranch;
+      inherit fwiParams;
+    };
 
-  pipeline = stdexp.stdPipeline ++ [ exec program ];
+  pipeline = stdexp.stdPipelineOverride {
+    # Replace the stdandard srun stage with our own
+    overrides = { inherit srun; };
+  } ++ [ exec program ];
 }

garlic/exp/fwi/granularity.nix

@@ -37,10 +37,6 @@ let
     inherit (c) gitBranch blocksize;
     inherit (c.n) nx ny nz ntpn nodes;
 
-    fwiInput = bsc.apps.fwi.input.override {
-      inherit (c.n) nx ny nz;
-    };
-
     # Other FWI parameters
     ioFreq = -1;
 

garlic/exp/fwi/reuse.nix

@@ -60,10 +60,6 @@ let
     inherit (c) gitBranch blocksize;
     inherit (c.n) nx ny nz;
 
-    fwiInput = bsc.apps.fwi.input.override {
-      inherit (c.n) nx ny nz;
-    };
-
     # Repeat the execution of each unit several times
     loops = 10;
 

garlic/exp/fwi/ss.nix

@@ -59,10 +59,6 @@ let
     inherit (c) gitBranch blocksize;
     inherit (c.n) nx ny nz;
 
-    fwiInput = bsc.apps.fwi.input.override {
-      inherit (c.n) nx ny nz;
-    };
-
     # Other FWI parameters
     ioFreq = -1;
 

garlic/exp/fwi/sync_io.nix

@@ -54,10 +54,6 @@ let
     inherit (c) gitBranch blocksize;
     inherit (c.n) nx ny nz;
 
-    fwiInput = bsc.apps.fwi.input.override {
-      inherit (c.n) nx ny nz;
-    };
-
     # Other FWI parameters
     ioFreq = c.ioFreq;