Linux disk performance tweaking parameters

A quick reference and note to self. Will be fleshed out in greater detail in future.

NCQ queue depth: /sys/block/sd*/device/queue_depth
readahead blocks: /sbin/blockdev --getra /dev/md* (or /dev/sd*)
stripe cache size: /sys/block/md*/md/stripe_cache_size

[edit] A fuller version has been posted on Ubuntu forums. The tuneraid script I use is as follows:

#!/bin/bash
###############################################################################
#  simple script to set some parameters to increase performance on a mdadm
# raid5 or raid6. Ajust the ## parameters ##-section to your system!
#
#  WARNING: depending on stripesize and the number of devices the array might
# use QUITE a lot of memory after optimization!
#
#  27may2010 by Alexander Peganz
#  31jul211 modified by Mathias B
###############################################################################


## parameters ##
MDDEV=md127			# e.g. md51 for /dev/md51
CHUNKSIZE=512		# in KB
BLOCKSIZE=4			# of file system in KB
NCQ=enable			# disable, enable. ath. else keeps current setting
NCQDEPTH=31			# 31 should work for almost anyone
FORCECHUNKSIZE=true	# force max sectors kb to chunk size > 512
DOTUNEFS=true		# run tune2fs, ONLY SET TO true IF YOU USE EXT[34]
RAIDLEVEL=raid5		# raid5, raid6
logfile="/home/kureshii/tuneraid.log"

## code ##
# test for priviledges
if [ "$(whoami)" != 'root' ]; then
	echo $(date): Need to be root >> "$logfile"
	exit 1
fi

# set number of parity devices
NUMPARITY=1
[[ $RAIDLEVEL == "raid6" ]] && NUMPARITY=2
echo "Using $NUMPARITY parity devices ($RAIDLEVEL)"
# get all devices
DEVSTR="`grep \"^$MDDEV : \" /proc/mdstat` eol"
while \
 [ -z "`expr match \"$DEVSTR\" '\(\> "$logfile"
	exit 1
fi

echo "Found $NUMDEVS devices with $NUMSPAREDEVS spares"

# set read ahead
RASIZE=$[$NUMDEVS*($NUMDEVS-$NUMPARITY)*2*$CHUNKSIZE]   # in 512b blocks
echo read ahead size per device: $RASIZE blocks \($[$RASIZE/2]kb\)
MDRASIZE=$[$RASIZE*$NUMDEVS]
echo read ahead size of array: $MDRASIZE blocks \($[$MDRASIZE/2]kb\)
blockdev --setra $RASIZE /dev/sd[$DEVS]
#blockdev --setra $RASIZE /dev/sd[$SPAREDEVS]
blockdev --setra $MDRASIZE /dev/$MDDEV

# set stripe cache size
STRCACHESIZE=$[$RASIZE/8]                               # in pages per device
echo stripe cache size of devices: $STRCACHESIZE pages \($[$STRCACHESIZE*4]kb\)
echo $STRCACHESIZE > /sys/block/$MDDEV/md/stripe_cache_size

# set max sectors kb
DEVINDEX=0
MINMAXHWSECKB=$(cat /sys/block/sd${DEVS:0:1}/queue/max_hw_sectors_kb)
until [ $DEVINDEX -ge $NUMDEVS ]; do
	DEVLETTER=${DEVS:$DEVINDEX:1}
	MAXHWSECKB=$(cat /sys/block/sd$DEVLETTER/queue/max_hw_sectors_kb)
	if [ $MAXHWSECKB -lt $MINMAXHWSECKB ]; then
		MINMAXHWSECKB=$MAXHWSECKB
	fi
	DEVINDEX=$[$DEVINDEX+1]
done
if [ $CHUNKSIZE -le $MINMAXHWSECKB ] && ( [ $CHUNKSIZE -le 512 ] || [[ $FORCECHUNKSIZE == "true" ]] ); then
	echo Setting max sectors KB to match chunk size
	DEVINDEX=0
	until [ $DEVINDEX -ge $NUMDEVS ]; do
		DEVLETTER=${DEVS:$DEVINDEX:1}
		echo "Set max sectors KB to $CHUNKSIZE on $DEVLETTER"
		echo $CHUNKSIZE > /sys/block/sd$DEVLETTER/queue/max_sectors_kb
		DEVINDEX=$[$DEVINDEX+1]
	done
	DEVINDEX=0
	until [ $DEVINDEX -ge $NUMSPAREDEVS ]; do
		DEVLETTER=${SPAREDEVS:$DEVINDEX:1}
		echo "Set max sectors KB to $CHUNKSIZE on $DEVLETTER"
		echo $CHUNKSIZE > /sys/block/sd$DEVLETTER/queue/max_sectors_kb
		DEVINDEX=$[$DEVINDEX+1]
	done
fi

# enable/disable NCQ
DEVINDEX=0
if [[ $NCQ == "enable" ]] || [[ $NCQ == "disable" ]]; then
	if [[ $NCQ == "disable" ]]; then
		NCQDEPTH=1
	fi
	until [ $DEVINDEX -ge $NUMDEVS ]; do
		DEVLETTER=${DEVS:$DEVINDEX:1}
		echo Setting NCQ queue depth to $NCQDEPTH on $DEVLETTER
		echo $NCQDEPTH > /sys/block/sd$DEVLETTER/device/queue_depth
		DEVINDEX=$[$DEVINDEX+1]
	done
	DEVINDEX=0
	until [ $DEVINDEX -ge $NUMSPAREDEVS ]; do
		DEVLETTER=${SPAREDEVS:$DEVINDEX:1}
		echo Setting NCQ queue depth to $NCQDEPTH on $DEVLETTER
		echo $NCQDEPTH > /sys/block/sd$DEVLETTER/device/queue_depth
		DEVINDEX=$[$DEVINDEX+1]
	done
fi

# tune2fs
if [[ $DOTUNEFS == "true" ]]; then
	STRIDE=$[$CHUNKSIZE/$BLOCKSIZE]
	STRWIDTH=$[$CHUNKSIZE/$BLOCKSIZE*($NUMDEVS-$NUMPARITY)]
	echo setting stride to $STRIDE blocks \($CHUNKSIZE KB\)
	echo setting stripe-width to $STRWIDTH blocks \($[$STRWIDTH*$BLOCKSIZE] KB\)
	echo tune2fs -E stride=$STRIDE,stripe-width=$STRWIDTH /dev/$MDDEV
fi

# exit
echo $(date): Success >> "$logfile"
exit 0