Category: ‘infiniband’

Dracut Custom Infiniband Module

August 21, 2013 Posted by admin

Dracut the replacement for mkinitrd is the new tool of choice for building ramdisk images. It modular and relatively easy to use once you get the hang of it. I will not boar you with the details but I work in a fast paced HPCC envirnment where we use pxe to boot nodes and go from bare metal to running in minutes. I ran into to an issue where we got new servers that we wanted to pxe boot over infiniband. So I set out to create my own dracut module.

To create your new dracut module change the the dracut module directory and make a directory starting with a number which specifies the order your modules get run in.


cd /usr/share/dracut/modules.d
mkdir 39infiniband

This script is where I define all the infiniband kernel modules that I want included into my ramdisk

cat installkernel
#!/bin/bash
# Include wired net drivers, excluding wireless

net_module_test() {
local net_drivers=’eth_type_trans|register_virtio_device’
local unwanted_drivers=’/(wireless|isdn|uwb)/’
egrep -q $net_drivers “$1” && \
egrep -qv ‘iw_handler_get_spy’ “$1” && \
[[ ! $1 =~ $unwanted_drivers ]]
}

instmods $(filter_kernel_modules net_module_test)

instmods mlx4_core
instmods mlx5_ib
instmods mlx4_ib
instmods ib_ipoib
instmods ib_core
instmods ib_mad
instmods ib_sa
instmods ib_addr
instmods iw_cm
instmods ib_cm
instmods rdma_cm
instmods rdma_ucm
instmods ib_ucm
instmods ib_uverbs
instmods ib_umad
instmods rdma_cm
instmods ib_cm
instmods iw_cm
instmods ib_sa
instmods ib_mad
instmods ipv6
instmods rds
instmods ib_srp
instmods ib_iser
instmods rdma_cm
instmods ib_uverbs
instmods rdma_ucm
instmods mst_pciconf
instmods mst_pci
instmods eth_ipoib
instmods mlx4_core
instmods mlx4_en
instmods mlx5_core

This section here is where the kernel modules get loaded


cat infiniband.sh
#!/bin/bash

load_modules()
{

modprobe mlx4_core
modprobe mlx4_ib
modprobe ib_ipoib
modprobe ib_core
modprobe ib_mad
modprobe ib_sa
modprobe ib_addr
modprobe iw_cm
modprobe ib_cm
modprobe rdma_cm
modprobe rdma_ucm
modprobe ib_ucm
modprobe ib_uverbs
modprobe ib_umad
modprobe rdma_cm
modprobe ib_cm
modprobe iw_cm
modprobe ib_sa
modprobe ib_mad
modprobe ipv6
modprobe rds
modprobe ib_srp
modprobe ib_iser
modprobe rdma_cm
modprobe ib_uverbs
modprobe rdma_ucm

}

load_modules

This script makes sure the newly created module gets run at pre-udev


cat install
#!/bin/bash

inst_hook pre-udev 39 “$moddir/infiniband.sh”

To build you own custom ramdisk run the dracut command with the following options


dracut -v -m "nfs network base infiniband " -f /tmp/test/initramfs-2.6.32-358.el6.x86_64_test.img 2.6.32-358.el6.x86_64