osu: add mtu and eager experiments

garlic/exp/index.nix

@@ -116,6 +116,8 @@
     bw = callPackage ./osu/bw.nix { };
     impi = callPackage ./osu/impi.nix { };
     bwShm = bw.override { interNode = false; };
+    mtu = callPackage ./osu/mtu.nix { };
+    eager = callPackage ./osu/eager.nix { };
   };
 
   examples = {

garlic/exp/osu/eager.nix

@@ -0,0 +1,83 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+}:
+
+with builtins;
+with stdenv.lib;
+
+let
+
+  machineConfig = targetMachine.config;
+
+  # Initial variable configuration
+  varConf = with bsc; {
+    sizeKB = range 5 25;
+    mpi = [ impi ];
+    #mpi = [ impi bsc.openmpi mpich ]; #psmpi ];
+    PSM2_MQ_EAGER_SDMA_SZ_KB = [ 16 20 24 ];
+    PSM2_MTU_KB = [ 10 ];
+  };
+
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    inherit (machineConfig) hw;
+    nodes = 2;
+    ntasksPerNode = 1;
+    cpusPerTask = 1;
+    time = "00:30:00";
+    qos = "debug";
+    loops = 10;
+    iterations = 50000;
+    #FIXME: Notice the switchover is 16000 and MTU is 10240
+    PSM2_MQ_EAGER_SDMA_SZ = PSM2_MQ_EAGER_SDMA_SZ_KB * 1000;
+    PSM2_MTU = PSM2_MTU_KB * 1024;
+    expName = "osu-bw";
+    unitName = expName +
+      "-size.${toString sizeKB}K" +
+      "-mtu.${toString PSM2_MTU_KB}K" +
+      "-sdma.${toString PSM2_MQ_EAGER_SDMA_SZ_KB}K";
+    jobName = expName;
+    inherit (c) mpi sizeKB
+      PSM2_MQ_EAGER_SDMA_SZ_KB
+      PSM2_MTU_KB;
+
+    size = sizeKB * 1024;
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  exec = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+
+    program = "${nextStage}/bin/osu_bw";
+
+    env = ''
+      export PSM2_MQ_EAGER_SDMA_SZ=${toString PSM2_MQ_EAGER_SDMA_SZ}
+      export PSM2_MTU=${toString PSM2_MTU}
+      export PSM2_TRACEMASK=0x101
+      export PSM2_MQ_PRINT_STATS=-1
+    '';
+
+    argv = [
+      "-m" "${toString size}:${toString size}"
+      "-i" iterations
+    ];
+  };
+
+  program = {nextStage, conf, ...}: bsc.osumb.override {
+    # Use the specified MPI implementation
+    inherit (conf) mpi;
+  };
+
+  pipeline = stdexp.stdPipeline ++ [ exec program ];
+
+in
+
+  stdexp.genExperiment { inherit configs pipeline; }

garlic/exp/osu/mtu.nix

@@ -0,0 +1,83 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+}:
+
+with builtins;
+with stdenv.lib;
+
+let
+
+  machineConfig = targetMachine.config;
+
+  # Initial variable configuration
+  varConf = with bsc; {
+    sizeKB = range 5 25;
+    mpi = [ impi ];
+    #mpi = [ impi bsc.openmpi mpich ]; #psmpi ];
+    PSM2_MQ_EAGER_SDMA_SZ_KB = [ 16 ];
+    PSM2_MTU_KB = [ 8 10 ];
+  };
+
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    inherit (machineConfig) hw;
+    nodes = 2;
+    ntasksPerNode = 1;
+    cpusPerTask = 1;
+    time = "00:30:00";
+    qos = "debug";
+    loops = 10;
+    iterations = 50000;
+    #FIXME: Notice the switchover is 16000 and MTU is 10240
+    PSM2_MQ_EAGER_SDMA_SZ = PSM2_MQ_EAGER_SDMA_SZ_KB * 1000;
+    PSM2_MTU = PSM2_MTU_KB * 1024;
+    expName = "osu-bw";
+    unitName = expName +
+      "-size.${toString sizeKB}K" +
+      "-mtu.${toString PSM2_MTU_KB}K" +
+      "-sdma.${toString PSM2_MQ_EAGER_SDMA_SZ_KB}K";
+    jobName = expName;
+    inherit (c) mpi sizeKB
+      PSM2_MQ_EAGER_SDMA_SZ_KB
+      PSM2_MTU_KB;
+
+    size = sizeKB * 1024;
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  exec = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+
+    program = "${nextStage}/bin/osu_bw";
+
+    env = ''
+      export PSM2_MQ_EAGER_SDMA_SZ=${toString PSM2_MQ_EAGER_SDMA_SZ}
+      export PSM2_MTU=${toString PSM2_MTU}
+      export PSM2_TRACEMASK=0x101
+      export PSM2_MQ_PRINT_STATS=-1
+    '';
+
+    argv = [
+      "-m" "${toString size}:${toString size}"
+      "-i" iterations
+    ];
+  };
+
+  program = {nextStage, conf, ...}: bsc.osumb.override {
+    # Use the specified MPI implementation
+    inherit (conf) mpi;
+  };
+
+  pipeline = stdexp.stdPipeline ++ [ exec program ];
+
+in
+
+  stdexp.genExperiment { inherit configs pipeline; }

garlic/fig/index.nix

@@ -77,6 +77,8 @@ in
     bw    = customPlot ./osu/bw.R (ds.osu.bw bw.result);
     bwShm = customPlot ./osu/bw.R (ds.osu.bw bwShm.result);
     impi  = customPlot ./osu/impi.R (ds.osu.bw impi.result);
+    mtu   = customPlot ./osu/mtu.R (ds.osu.bw mtu.result);
+    eager   = customPlot ./osu/eager.R (ds.osu.bw eager.result);
   };
 
   # The figures used in the article contained in a directory per figure

garlic/fig/osu/eager.R

@@ -0,0 +1,62 @@
+library(ggplot2)
+library(dplyr, warn.conflicts = FALSE)
+library(scales)
+library(jsonlite)
+
+args=commandArgs(trailingOnly=TRUE)
+
+# Read the timetable from args[1]
+input_file = "input.json"
+if (length(args)>0) { input_file = args[1] }
+
+# Load the dataset in NDJSON format
+dataset = jsonlite::stream_in(file(input_file), verbose=FALSE) %>%
+	jsonlite::flatten()
+
+# We only need the nblocks and time
+df = select(dataset,
+	    config.unitName,
+	    config.nodes,
+	    config.ntasksPerNode,
+	    config.cpusPerTask,
+	    config.PSM2_MQ_EAGER_SDMA_SZ,
+	    size, bw, config.iterations) %>%
+	rename(unitName=config.unitName,
+	       iterations=config.iterations,
+	       PSM2_MQ_EAGER_SDMA_SZ=config.PSM2_MQ_EAGER_SDMA_SZ)
+
+nodes = unique(df$config.nodes)
+tasksPerNode = unique(df$config.ntasksPerNode)
+cpusPerTask = unique(df$config.cpusPerTask)
+df$unitName = as.factor(df$unitName)
+df$sizeFactor = as.factor(df$size)
+df$sizeKB = df$size / 1024
+df$PSM2_MQ_EAGER_SDMA_SZ.f = as.factor(df$PSM2_MQ_EAGER_SDMA_SZ)
+
+iterations = unique(df$iterations)
+
+df = group_by(df, unitName, sizeFactor) %>%
+  mutate(medianBw = median(bw)) %>%
+  ungroup()
+
+breaks = 10^(-10:10)
+minor_breaks <- rep(1:9, 21)*(10^rep(-10:10, each=9))
+
+ppi=150
+h=6
+w=8
+
+p = ggplot(data=df, aes(x=sizeKB, y=bw)) +
+  labs(x="Message size (KB)", y="Bandwidth (MB/s)",
+    title=sprintf("OSU benchmark: osu_bw --iterations %d", iterations), 
+    subtitle=input_file) +
+  geom_point(shape=21, size=3) +
+  geom_vline(aes(xintercept = PSM2_MQ_EAGER_SDMA_SZ/1024), color="blue") +
+  geom_vline(xintercept = 10, color="red") +
+  annotate("text", x = 10.2, y = 8.5e3, label = "MTU = 10KB", color="red", hjust=0) +
+  facet_wrap(vars(PSM2_MQ_EAGER_SDMA_SZ.f), nrow=3, labeller = "label_both") +
+  scale_x_continuous(breaks = unique(df$sizeKB), minor_breaks=NULL) +
+  theme_bw()
+
+ggsave("bw.png", plot=p, width=w, height=h, dpi=ppi)
+ggsave("bw.pdf", plot=p, width=w, height=h, dpi=ppi)

garlic/fig/osu/mtu.R

@@ -0,0 +1,65 @@
+library(ggplot2)
+library(dplyr, warn.conflicts = FALSE)
+library(scales)
+library(jsonlite)
+
+args=commandArgs(trailingOnly=TRUE)
+
+# Read the timetable from args[1]
+input_file = "input.json"
+if (length(args)>0) { input_file = args[1] }
+
+# Load the dataset in NDJSON format
+dataset = jsonlite::stream_in(file(input_file), verbose=FALSE) %>%
+	jsonlite::flatten()
+
+# We only need the nblocks and time
+df = select(dataset,
+	    config.unitName,
+	    config.nodes,
+	    config.ntasksPerNode,
+	    config.cpusPerTask,
+	    config.PSM2_MQ_EAGER_SDMA_SZ,
+	    config.PSM2_MTU,
+	    size, bw, config.iterations) %>%
+	rename(unitName=config.unitName,
+	       iterations=config.iterations,
+	       PSM2_MQ_EAGER_SDMA_SZ=config.PSM2_MQ_EAGER_SDMA_SZ,
+	       PSM2_MTU=config.PSM2_MTU)
+
+nodes = unique(df$config.nodes)
+tasksPerNode = unique(df$config.ntasksPerNode)
+cpusPerTask = unique(df$config.cpusPerTask)
+df$unitName = as.factor(df$unitName)
+df$sizeFactor = as.factor(df$size)
+df$sizeKB = df$size / 1024
+df$PSM2_MQ_EAGER_SDMA_SZ.f = as.factor(df$PSM2_MQ_EAGER_SDMA_SZ)
+df$PSM2_MTU.f = as.factor(df$PSM2_MTU)
+
+iterations = unique(df$iterations)
+
+df = group_by(df, unitName, sizeFactor) %>%
+  mutate(medianBw = median(bw)) %>%
+  ungroup()
+
+breaks = 10^(-10:10)
+minor_breaks <- rep(1:9, 21)*(10^rep(-10:10, each=9))
+
+ppi=150
+h=6
+w=8
+
+p = ggplot(data=df, aes(x=sizeKB, y=bw)) +
+  labs(x="Message size (KB)", y="Bandwidth (MB/s)",
+    title=sprintf("OSU benchmark: osu_bw --iterations %d", iterations), 
+    subtitle=input_file) +
+  geom_point(shape=21, size=3) +
+  geom_vline(aes(xintercept = PSM2_MQ_EAGER_SDMA_SZ/1024), color="blue") +
+  geom_vline(aes(xintercept = PSM2_MTU / 1024), color="red") +
+  #annotate("text", x = 10.2, y = 8.5e3, label = "MTU = 10KB", color="red", hjust=0) +
+  facet_wrap(vars(PSM2_MTU.f), nrow=3, labeller = "label_both") +
+  scale_x_continuous(breaks = unique(df$sizeKB), minor_breaks=NULL) +
+  theme_bw()
+
+ggsave("bw.png", plot=p, width=w, height=h, dpi=ppi)
+ggsave("bw.pdf", plot=p, width=w, height=h, dpi=ppi)