Adding scripts to the environment
The command to start the benchmark is quite obscure and cumbersome to type. We would like to create a script to wrap it. However, creating a full Nix package for such a small script is not worth it. Fortunately, Nix provides ways to create reproducible bash (or others) script easily.
Let us create a Nix file called myscripts.nix
.
It will be a function that takes pkgs
as input and return the set of our scripts.
{ pkgs, ... }:
let
# We define some constants here
nfsMountPoint = "/data";
nbProcs = 16;
iorConfig = "/etc/ior_script";
in {
# This function takes the number of compute nodes,
# creates a hostfile for MPI and runs the benchmark
start_ior =
pkgs.writeScriptBin "start_ior" ''
cd ${nfsMountPoint}
NB_NODES=$(cat /etc/hosts | grep node | wc -l)
NB_SLOTS_PER_NODE=$((${builtins.toString nbProcs} / $NB_NODES))
cat /etc/hosts | grep node | awk -v nb_slots="$NB_SLOTS_PER_NODE" '{ print $2 " slots=" nb_slots;}' > my_hosts
mpirun --mca pml ^ucx --mca mtl ^psm2,ofi --mca btl ^ofi,openib --allow-run-as-root -np ${builtins.toString nbProcs} --hostfile my_hosts ior -f ${iorConfig}
'';
}
We can now import these scripts in the composition:
# ...
node = { pkgs, ... }:
let
scripts = import ./myscripts.nix { inherit pkgs; };
in
{
networking.firewall.enable = false;
environment.systemPackages = with pkgs; [ openmpi ior scripts.start_ior ];
environment.etc = {
ior_script = {
text = builtins.readFile ./script.ior;
};
};
fileSystems."/data" = {
device = "server:/";
fsType = "nfs";
};
};
# ...
Building
nxc build -f g5k-nfs-store
Deploying
Reserving the resources
export $(oarsub -l nodes=2,walltime=1:0:0 "$(nxc helper g5k_script) 1h" | grep OAR_JOB_ID)
Getting the machine file
oarstat -u -J | jq --raw-output 'to_entries | .[0].value.assigned_network_address | .[]' > machines
Starting the nodes
nxc start -m machines
Connecting
nxc connect
Running the script
After building and deploying, we can simply run start_ior
from the node to run the benchmark.
start_ior