^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) #!/bin/bash
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) # SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) # NAME
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) # failcmd.sh - run a command with injecting slab/page allocation failures
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) # SYNOPSIS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) # failcmd.sh --help
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) # failcmd.sh [<options>] command [arguments]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) # DESCRIPTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) # Run command with injecting slab/page allocation failures by fault
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) # injection.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) # NOTE: you need to run this script as root.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) usage()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) cat >&2 <<EOF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) Usage: $0 [options] command [arguments]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) OPTIONS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) -p percent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) --probability=percent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) likelihood of failure injection, in percent.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) Default value is 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) -t value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) --times=value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) specifies how many times failures may happen at most.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) Default value is 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) --oom-kill-allocating-task=value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) set /proc/sys/vm/oom_kill_allocating_task to specified value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) before running the command.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) Default value is 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) -h, --help
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) Display a usage message and exit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) --interval=value, --space=value, --verbose=value, --task-filter=value,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) --stacktrace-depth=value, --require-start=value, --require-end=value,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) --reject-start=value, --reject-end=value, --ignore-gfp-wait=value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) See Documentation/fault-injection/fault-injection.rst for more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) information
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) failslab options:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) --cache-filter=value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) fail_page_alloc options:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) --ignore-gfp-highmem=value, --min-order=value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) ENVIRONMENT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) FAILCMD_TYPE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) The following values for FAILCMD_TYPE are recognized:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) failslab
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) inject slab allocation failures
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) fail_page_alloc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) inject page allocation failures
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) If FAILCMD_TYPE is not defined, then failslab is used.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) EOF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) if [ $UID != 0 ]; then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) echo must be run as root >&2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) exit 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) fi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) DEBUGFS=`mount -t debugfs | head -1 | awk '{ print $3}'`
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) if [ ! -d "$DEBUGFS" ]; then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) echo debugfs is not mounted >&2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) exit 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) fi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) FAILCMD_TYPE=${FAILCMD_TYPE:-failslab}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) FAULTATTR=$DEBUGFS/$FAILCMD_TYPE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) if [ ! -d $FAULTATTR ]; then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) echo $FAILCMD_TYPE is not available >&2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) exit 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) fi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) LONGOPTS=probability:,interval:,times:,space:,verbose:,task-filter:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) LONGOPTS=$LONGOPTS,stacktrace-depth:,require-start:,require-end:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) LONGOPTS=$LONGOPTS,reject-start:,reject-end:,oom-kill-allocating-task:,help
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) if [ $FAILCMD_TYPE = failslab ]; then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) LONGOPTS=$LONGOPTS,ignore-gfp-wait:,cache-filter:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) elif [ $FAILCMD_TYPE = fail_page_alloc ]; then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) LONGOPTS=$LONGOPTS,ignore-gfp-wait:,ignore-gfp-highmem:,min-order:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) fi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) TEMP=`getopt -o p:i:t:s:v:h --long $LONGOPTS -n 'failcmd.sh' -- "$@"`
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) if [ $? != 0 ]; then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) usage
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) exit 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) fi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) eval set -- "$TEMP"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) fault_attr_default()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) echo N > $FAULTATTR/task-filter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) echo 0 > $FAULTATTR/probability
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) echo 1 > $FAULTATTR/times
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) fault_attr_default
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) oom_kill_allocating_task_saved=`cat /proc/sys/vm/oom_kill_allocating_task`
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) restore_values()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) fault_attr_default
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) echo $oom_kill_allocating_task_saved \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) > /proc/sys/vm/oom_kill_allocating_task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) #
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) # Default options
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) #
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) declare -i oom_kill_allocating_task=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) declare task_filter=Y
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) declare -i probability=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) declare -i times=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) while true; do
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) case "$1" in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) -p|--probability)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) probability=$2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) shift 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) ;;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) -i|--interval)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) echo $2 > $FAULTATTR/interval
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) shift 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) ;;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) -t|--times)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) times=$2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) shift 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) ;;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) -s|--space)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) echo $2 > $FAULTATTR/space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) shift 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) ;;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) -v|--verbose)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) echo $2 > $FAULTATTR/verbose
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) shift 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) ;;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) --task-filter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) task_filter=$2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) shift 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) ;;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) --stacktrace-depth)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) echo $2 > $FAULTATTR/stacktrace-depth
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) shift 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) ;;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) --require-start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) echo $2 > $FAULTATTR/require-start
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) shift 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) ;;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) --require-end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) echo $2 > $FAULTATTR/require-end
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) shift 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) ;;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) --reject-start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) echo $2 > $FAULTATTR/reject-start
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) shift 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) ;;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) --reject-end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) echo $2 > $FAULTATTR/reject-end
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) shift 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) ;;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) --oom-kill-allocating-task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) oom_kill_allocating_task=$2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) shift 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) ;;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) --ignore-gfp-wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) echo $2 > $FAULTATTR/ignore-gfp-wait
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) shift 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) ;;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) --cache-filter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) echo $2 > $FAULTATTR/cache_filter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) shift 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) ;;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) --ignore-gfp-highmem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) echo $2 > $FAULTATTR/ignore-gfp-highmem
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) shift 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) ;;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) --min-order)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) echo $2 > $FAULTATTR/min-order
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) shift 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) ;;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) -h|--help)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) usage
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) exit 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) shift
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) ;;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) --)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) shift
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) break
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) ;;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) esac
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) [ -z "$1" ] && exit 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) echo $oom_kill_allocating_task > /proc/sys/vm/oom_kill_allocating_task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) echo $task_filter > $FAULTATTR/task-filter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) echo $probability > $FAULTATTR/probability
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) echo $times > $FAULTATTR/times
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) trap "restore_values" SIGINT SIGTERM EXIT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) cmd="echo 1 > /proc/self/make-it-fail && exec $@"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) bash -c "$cmd"