Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
nixos/rdma-core: add module for soft RoCE and test (#34607)
- Loading branch information
1 parent
58cf763
commit bf53dc6
Showing
4 changed files
with
118 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,63 @@ | ||
{ config, lib, pkgs, ... }: | ||
|
||
with lib; | ||
|
||
let | ||
cfg = config.networking.rxe; | ||
|
||
runRxeCmd = cmd: ifcs: | ||
concatStrings ( map (x: "${pkgs.rdma-core}/bin/rxe_cfg -n ${cmd} ${x};") ifcs); | ||
|
||
startScript = pkgs.writeShellScriptBin "rxe-start" '' | ||
${pkgs.rdma-core}/bin/rxe_cfg -n start | ||
${runRxeCmd "add" cfg.interfaces} | ||
${pkgs.rdma-core}/bin/rxe_cfg | ||
''; | ||
|
||
stopScript = pkgs.writeShellScriptBin "rxe-stop" '' | ||
${runRxeCmd "remove" cfg.interfaces } | ||
${pkgs.rdma-core}/bin/rxe_cfg -n stop | ||
''; | ||
|
||
in { | ||
###### interface | ||
|
||
options = { | ||
networking.rxe = { | ||
enable = mkEnableOption "RDMA over converged ethernet"; | ||
interfaces = mkOption { | ||
type = types.listOf types.str; | ||
default = [ ]; | ||
example = [ "eth0" ]; | ||
description = '' | ||
Enable RDMA on the listed interfaces. The corresponding virtual | ||
RDMA interfaces will be named rxe0 ... rxeN where the ordering | ||
will be as they are named in the list. UDP port 4791 must be | ||
open on the respective ethernet interfaces. | ||
''; | ||
}; | ||
}; | ||
}; | ||
|
||
###### implementation | ||
|
||
config = mkIf cfg.enable { | ||
|
||
systemd.services.rxe = { | ||
path = with pkgs; [ kmod rdma-core ]; | ||
description = "RoCE interfaces"; | ||
|
||
wantedBy = [ "multi-user.target" ]; | ||
after = [ "systemd-modules-load.service" "network-online.target" ]; | ||
wants = [ "network-pre.target" ]; | ||
|
||
serviceConfig = { | ||
Type = "oneshot"; | ||
RemainAfterExit = true; | ||
ExecStart = "${startScript}/bin/rxe-start"; | ||
ExecStop = "${stopScript}/bin/rxe-stop"; | ||
}; | ||
}; | ||
}; | ||
} | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,53 @@ | ||
import ./make-test.nix ({ pkgs, ... } : | ||
|
||
let | ||
node = { config, pkgs, lib, ... } : { | ||
networking = { | ||
firewall = { | ||
allowedUDPPorts = [ 4791 ]; # open RoCE port | ||
allowedTCPPorts = [ 4800 ]; # port for test utils | ||
}; | ||
rxe = { | ||
enable = true; | ||
interfaces = [ "eth1" ]; | ||
}; | ||
}; | ||
|
||
environment.systemPackages = with pkgs; [ rdma-core screen ]; | ||
}; | ||
|
||
in { | ||
name = "rxe"; | ||
|
||
nodes = { | ||
server = node; | ||
client = node; | ||
}; | ||
|
||
testScript = '' | ||
# Test if rxe interface comes up | ||
$server->waitForUnit("default.target"); | ||
$server->succeed("systemctl status rxe.service"); | ||
$server->succeed("ibv_devices | grep rxe0"); | ||
$client->waitForUnit("default.target"); | ||
# ping pong test | ||
$server->succeed("screen -dmS rc_pingpong ibv_rc_pingpong -p 4800 -g0"); | ||
$client->succeed("sleep 2; ibv_rc_pingpong -p 4800 -g0 server"); | ||
$server->succeed("screen -dmS uc_pingpong ibv_uc_pingpong -p 4800 -g0"); | ||
$client->succeed("sleep 2; ibv_uc_pingpong -p 4800 -g0 server"); | ||
$server->succeed("screen -dmS ud_pingpong ibv_ud_pingpong -p 4800 -s 1024 -g0"); | ||
$client->succeed("sleep 2; ibv_ud_pingpong -p 4800 -s 1024 -g0 server"); | ||
$server->succeed("screen -dmS srq_pingpong ibv_srq_pingpong -p 4800 -g0"); | ||
$client->succeed("sleep 2; ibv_srq_pingpong -p 4800 -g0 server"); | ||
$server->succeed("screen -dmS rping rping -s -a server -C 10"); | ||
$client->succeed("sleep 2; rping -c -a server -C 10"); | ||
''; | ||
}) | ||
|
||
|