nvme-stas/coverage.sh.in

#!/usr/bin/env bash

PRIMARY_GRP=$( id -ng )
PRIMARY_USR=$( id -nu )
PYTHON_PATH=.:./subprojects/libnvme
AVAHI_PUBLISHER=mdns_publisher.service

file=/tmp/stafd.conf.XXXXXX
stafd_conf_fname=$(mktemp $file)

file=/tmp/stacd.conf.XXXXXX
stacd_conf_fname=$(mktemp $file)

CYAN="[1;36m"
RED="[1;31m"
YELLOW="[1;33m"
NORMAL="[0m"

log() {
	msg="$1"
	printf "%b%s%s%b[0m\n" "\0033" ${CYAN} "${msg}" "\0033"
	sudo logger -t COVERAGE -i "@@@@@ " -p warning -- "${msg}"
}

log_file_contents() {
	rc=$1
	file=$2

	if [ $rc -eq 0 ]; then
		color=${NORMAL}
		level="info"
	else
		color=${YELLOW}
		level="error"
	fi

	while IFS= read -r line; do
		printf "%b%s%s%b[0m\n" "\0033" ${color} "   ${line}" "\0033"
	done < ${file}

        sudo file=${file} level=${level} bash <<'EOF'
while IFS= read -r line; do
	logger -t COVERAGE -i "@@@@@ " -p ${level} -- "   ${line}"
done < ${file}
EOF
}

systemctl-exists() {
	unit="$1"
	[ $(systemctl list-unit-files "${unit}" | wc -l) -gt 3 ]
}

sd_stop() {
	app="$1"
	unit="${app}"-cov.service
	if systemctl-exists "${unit}" >/dev/null 2>&1; then
		log "Stop ${app}"
		sudo systemctl stop "${unit}" >/tmp/output.txt 2>&1
		if [ -s /tmp/output.txt ]; then
			log_file_contents $? /tmp/output.txt
		else
			printf "   sudo systemctl stop %s\n" "${unit}"
		fi
		sudo systemctl reset-failed "${unit}" >/dev/null 2>&1
		printf "\n"
		sleep 1
	fi
}

sd_start() {
	app="$1"
	dbus="$2"
	conf="$3"
	unit="${app}"-cov.service

	if [ -z "${conf}" ]; then
		cmd="${app} --syslog"
	else
		cmd="${app} --syslog -f ${conf}"
	fi

	RUNTIME_DIRECTORY=/tmp/${app}
	rm -rf ${RUNTIME_DIRECTORY}
	mkdir ${RUNTIME_DIRECTORY}

	# Clear previous failure status (if any)
	sudo systemctl reset-failed "${unit}" >/dev/null 2>&1

	log "Start ${app}"
	sudo systemd-run --unit="${unit}" --working-directory=. --property=Type=dbus --property=BusName="${dbus}" --property="SyslogIdentifier=${app}" --setenv=PYTHONPATH=${PYTHON_PATH} --setenv=RUNTIME_DIRECTORY=${RUNTIME_DIRECTORY} coverage run --rcfile=.coveragerc ${cmd}  >/tmp/output.txt 2>&1
	log_file_contents $? /tmp/output.txt
	printf "\n"
	sleep 1
}

sd_restart() {
	app="$1"
	unit="${app}"-cov.service

	if systemctl is-active "${unit}" >/dev/null 2>&1; then
		log "Restart ${app}"
		sudo systemctl restart "${unit}" && printf "systemctl restart %s\n" "${unit}" >/tmp/output.txt 2>&1
		log_file_contents $? /tmp/output.txt
		sleep 1
	else
		msg="Cannot restart ${app}, which is not currently running."
		printf "%b%s%s%b[0m\n\n" "\0033" ${RED} "${msg}" "\0033"
	fi
	printf "\n"
}

reload_cfg() {
	app="$1"
	unit="${app}"-cov.service
	pid=$( systemctl show --property MainPID --value "${unit}" )
	log "Reload config ${app} - SIGHUP ${pid}"
	#sudo systemctl reload "${unit}" && printf "systemctl reload %s\n" "${unit}" >/tmp/output.txt 2>&1
	sudo kill -HUP "${pid}" >/tmp/output.txt 2>&1
	log_file_contents $? /tmp/output.txt
	printf "\n"
	sleep 1
}

run_unit_test() {
	input=$@
	if [ "$1" == "sudo" ]; then
		shift
		COVERAGE="sudo coverage"
	else
		COVERAGE="coverage"
	fi
	args=$@
	log "Run unit test: ${input}"
	PYTHONPATH=${PYTHON_PATH} ${COVERAGE} run --rcfile=.coveragerc "${args}" >/dev/null 2>&1
}

run_cmd_coverage() {
	input=$@
	if [ "$1" == "sudo" ]; then
		shift
		COVERAGE="sudo coverage"
	else
		COVERAGE="coverage"
	fi
	cmd="$@"
	log "Invoke: ${input}"
	${COVERAGE} run --rcfile=.coveragerc ${cmd} >/tmp/output.txt 2>&1
	log_file_contents $? /tmp/output.txt
	printf "\n"
}

run_cmd() {
	cmd="$@"
	${cmd} >/tmp/output.txt 2>&1
	if [ -s /tmp/output.txt ]; then
		log_file_contents $? /tmp/output.txt
	else
		printf "   %s\n" "${cmd}"
	fi
}

prerun_setup() {
	if [ ! -d coverage ]; then
		mkdir coverage
	fi

	for file in staf stac; do
	if [ ! -f "/usr/share/dbus-1/system.d/org.nvmexpress.${file}.conf" -a \
	     ! -f "/etc/dbus-1/system.d/org.nvmexpress.${file}.conf" ]; then
		log "hardlink /etc/dbus-1/system.d/org.nvmexpress.${file}.conf -> @BUILD_DIR@/etc/dbus-1/system.d/org.nvmexpress.${file}.conf"
		sudo ln @BUILD_DIR@/etc/dbus-1/system.d/org.nvmexpress.${file}.conf /etc/dbus-1/system.d/org.nvmexpress.${file}.conf
		if [ $? -ne 0 ]; then
			log "hardlink failed"
			exit 1
		fi
	fi
	done
	sudo systemctl reload dbus.service
}

postrun_cleanup() {
	sd_stop "stafd"
	sd_stop "stacd"

	log "Stop nvmet"
	sudo ../utils/nvmet/nvmet.py clean >/tmp/output.txt 2>&1
	log_file_contents $? /tmp/output.txt
	printf "\n"

	log "nvme disconnect-all"
	run_cmd sudo nvme disconnect-all
	printf "\n"

	log "Remove ${stafd_conf_fname} and ${stacd_conf_fname}"
	rm "${stafd_conf_fname}"
	rm "${stacd_conf_fname}"
	printf "\n"

	for file in staf stac; do
		if [ -f "/etc/dbus-1/system.d/org.nvmexpress.${file}.conf" ]; then
			if [ "$(stat -c %h -- "/etc/dbus-1/system.d/org.nvmexpress.${file}.conf")" -gt 1 ]; then
				log "Remove hardlink /etc/dbus-1/system.d/org.nvmexpress.${file}.conf"
				sudo rm "/etc/dbus-1/system.d/org.nvmexpress.${file}.conf"
			fi
		fi
	done
	sudo systemctl reload dbus.service

	sudo systemctl unmask avahi-daemon.service
	sudo systemctl unmask avahi-daemon.socket
	sudo systemctl start avahi-daemon.service
	sudo systemctl start avahi-daemon.socket

	sudo systemctl stop ${AVAHI_PUBLISHER} >/dev/null 2>&1
	sudo systemctl reset-failed ${AVAHI_PUBLISHER} >/dev/null 2>&1

	log "All done!!!"
	log "FINISHED-FINISHED-FINISHED-FINISHED-FINISHED-FINISHED-FINISHED-FINISHED"
}

trap postrun_cleanup EXIT
trap postrun_cleanup SIGINT

################################################################################
################################################################################
################################################################################

log "START-START-START-START-START-START-START-START-START-START-START-START"

if systemctl is-active stafd.service >/dev/null 2>&1 || systemctl is-active stacd.service >/dev/null 2>&1; then
	msg="Stopping because stafd and/or stacd is/are currently running."
	printf "%b%s%s%b[0m\n" "\0033" ${RED} "${msg}" "\0033"
	exit 1
fi

prerun_setup

#*******************************************************************************
# Load nvme kernel module
log "modprobe nvme_tcp"
run_cmd sudo /usr/sbin/modprobe nvme_tcp

log "nvme disconnect-all"
run_cmd sudo nvme disconnect-all
printf "\n"

sd_stop stafd # make sure it's not running already
sd_stop stacd # make sure it's not running already

#*******************************************************************************
# Create a dummy config file for stafd
log "Create dummy config file ${stafd_conf_fname}"
cat > "${stafd_conf_fname}" <<'EOF'
[Global]
tron              = true
ip-family         = ipv6
johnny            = be-good
queue-size        = 2000000
reconnect-delay   = NaN
ctrl-loss-tmo     = 10
disable-sqflow    = true

[Discovery controller connection management]
persistent-connections           = false
zeroconf-connections-persistence = -1

[Hello]
hello = bye
EOF
log_file_contents 0 "${stafd_conf_fname}"
printf "\n"

#*******************************************************************************
# Create a dummy config file for stacd
log "Create dummy config file ${stacd_conf_fname}"
cat > "${stacd_conf_fname}" <<'EOF'
[Global]
tron              = true
kato              = 10
nr-io-queues      = 4
nr-write-queues   = NaN
nr-poll-queues    = NaN
queue-size        = 2000000
reconnect-delay   = 1
ctrl-loss-tmo     = 1
disable-sqflow    = true

[I/O controller connection management]
disconnect-scope   = blah-blah
disconnect-trtypes = boing-boing
EOF
log_file_contents 0 "${stacd_conf_fname}"
printf "\n"

log "Stop & Mask Avahi daemon"
run_cmd sudo systemctl stop avahi-daemon.service
run_cmd sudo systemctl stop avahi-daemon.socket
run_cmd sudo systemctl mask avahi-daemon.service
run_cmd sudo systemctl mask avahi-daemon.socket
printf "\n"
sleep 1

log ">>>>>>>>>>>>>>>>>>>>> Marker [1] <<<<<<<<<<<<<<<<<<<<<"
printf "\n"

run_cmd_coverage stafctl ls
run_cmd_coverage stafctl invalid-command
run_cmd_coverage stacctl ls
run_cmd_coverage stacctl invalid-command

#*******************************************************************************
# Start nvme target simulator
log "Start nvmet"
sudo ../utils/nvmet/nvmet.py clean >/dev/null 2>&1
sudo ../utils/nvmet/nvmet.py create -f ../utils/nvmet/nvmet.conf >/tmp/output.txt 2>&1
log_file_contents $? /tmp/output.txt
printf "\n"

sleep 2

log ">>>>>>>>>>>>>>>>>>>>> Marker [2] <<<<<<<<<<<<<<<<<<<<<"
printf "\n"

#*******************************************************************************
# Start stafd and stacd
sd_start "stafd" "@STAFD_DBUS_NAME@" "${stafd_conf_fname}"
sd_start "stacd" "@STACD_DBUS_NAME@" "${stacd_conf_fname}"
sleep 2

run_cmd_coverage stafctl status

reload_cfg "stafd"
sleep 1

log "Restart Avahi daemon"
run_cmd sudo systemctl unmask avahi-daemon.socket
run_cmd sudo systemctl unmask avahi-daemon.service
run_cmd sudo systemctl start avahi-daemon.socket
run_cmd sudo systemctl start avahi-daemon.service
printf "\n"
sleep 2

log ">>>>>>>>>>>>>>>>>>>>> Marker [3] <<<<<<<<<<<<<<<<<<<<<"
printf "\n"

log "Change stafd config [1]:"
cat > "${stafd_conf_fname}" <<'EOF'
[Global]
tron = true

[Discovery controller connection management]
persistent-connections           = false
zeroconf-connections-persistence = 0.5

[Service Discovery]
zeroconf = enabled
EOF
log_file_contents 0 "${stafd_conf_fname}"
printf "\n"

reload_cfg "stafd"
sleep 1

log "Change stafd config [2]:"
cat > "${stafd_conf_fname}" <<'EOF'
[Global]
tron              = true
ip-family         = ipv4
queue-size        = 2000000
reconnect-delay   = 1
ctrl-loss-tmo     = 1
disable-sqflow    = true
pleo              = disable

[Discovery controller connection management]
persistent-connections            = false
zeroconf-connections-persistence  = 1:01

[Controllers]
controller = transport = tcp ; traddr = localhost ; ; ; kato=31; dhchap-ctrl-secret=not-so-secret
controller=transport=tcp;traddr=1.1.1.1
controller=transport=tcp;traddr=100.100.100.100
controller=transport=tcp;traddr=2607:f8b0:4002:c2c::71

exclude=transport=tcp;traddr=1.1.1.1
EOF
log_file_contents 0 "${stafd_conf_fname}"
printf "\n"

reload_cfg "stafd"
sleep 5


log "Change stacd config [1]:"
cat > "${stacd_conf_fname}" <<'EOF'
[Global]
tron=true
nr-io-queues=4
nr-write-queues=4
queue-size=2000000
reconnect-delay=1
ctrl-loss-tmo=1
disable-sqflow=true

[I/O controller connection management]
disconnect-scope=all-connections-matching-disconnect-trtypes
disconnect-trtypes=tcp+rdma
EOF
log_file_contents 0 "${stacd_conf_fname}"
printf "\n"

reload_cfg "stacd"
sleep 5

log ">>>>>>>>>>>>>>>>>>>>> Marker [4] <<<<<<<<<<<<<<<<<<<<<"
printf "\n"

run_cmd_coverage stafctl status

#*******************************************************************************
# Fake mDNS packets from a CDC
log "Start Avahi publisher"
run_cmd sudo systemctl stop ${AVAHI_PUBLISHER}
run_cmd sudo systemctl reset-failed ${AVAHI_PUBLISHER}
run_cmd sudo systemd-run --unit=${AVAHI_PUBLISHER} --working-directory=. avahi-publish -s SFSS _nvme-disc._tcp 8009 "p=tcp"
printf "\n"
sleep 1

#*******************************************************************************
run_cmd_coverage stafd --version
run_cmd_coverage stacd --version

#*******************************************************************************
# Stimulate D-Bus activity
run_cmd_coverage sudo stafctl --version
run_cmd_coverage sudo stafctl blah
run_cmd_coverage sudo stafctl troff
run_cmd_coverage stafctl status
run_cmd_coverage sudo stafctl tron
run_cmd_coverage stafctl ls -d
run_cmd_coverage stafctl adlp -d
run_cmd_coverage stafctl dlp -t tcp -a ::1 -s 8009

run_cmd_coverage sudo stacctl --version
run_cmd_coverage sudo stacctl blah
run_cmd_coverage sudo stacctl troff
run_cmd_coverage stacctl status
run_cmd_coverage sudo stacctl tron
run_cmd_coverage stacctl ls -d

log ">>>>>>>>>>>>>>>>>>>>> Marker [5] <<<<<<<<<<<<<<<<<<<<<"
printf "\n"

#*******************************************************************************
# Stimulate AENs activity by removing/restoring namespaces
log "Remove namespace: klingons"
run_cmd sudo ../utils/nvmet/nvmet.py unlink -p 1 -s klingons
printf "\n"
sleep 2
run_cmd_coverage stacctl ls

log "Restore namespace: klingons"
run_cmd sudo ../utils/nvmet/nvmet.py link -p 1 -s klingons
printf "\n"
sleep 2
run_cmd_coverage stacctl ls

log ">>>>>>>>>>>>>>>>>>>>> Marker [6] <<<<<<<<<<<<<<<<<<<<<"
printf "\n"


#*******************************************************************************
log "Restart Avahi publisher with invalid protocol"
run_cmd sudo systemctl stop ${AVAHI_PUBLISHER}
printf "\n"
sleep 1
run_cmd sudo systemd-run --unit=${AVAHI_PUBLISHER} --working-directory=. avahi-publish -s SFSS _nvme-disc._tcp 8009 "p=walmart"
printf "\n"
sleep 2

#*******************************************************************************
log "Restart Avahi publisher with protocol set to RoCE"
run_cmd sudo systemctl stop ${AVAHI_PUBLISHER}
printf "\n"
sleep 1
run_cmd sudo systemd-run --unit=${AVAHI_PUBLISHER} --working-directory=. avahi-publish -s SFSS _nvme-disc._tcp 8009 "p=roce"
printf "\n"
sleep 2

#*******************************************************************************
log "Restart Avahi publisher without specifying protocol"
run_cmd sudo systemctl stop ${AVAHI_PUBLISHER}
printf "\n"
sleep 1
run_cmd sudo systemd-run --unit=${AVAHI_PUBLISHER} --working-directory=. avahi-publish -s SFSS _nvme-disc._tcp 8009
printf "\n"
sleep 2

#*******************************************************************************
log "Restart Avahi publisher with protocol set to TCP"
run_cmd sudo systemctl stop ${AVAHI_PUBLISHER}
printf "\n"
sleep 1
run_cmd sudo systemd-run --unit=${AVAHI_PUBLISHER} --working-directory=. avahi-publish -s SFSS _nvme-disc._tcp 8009 "p=tcp"
printf "\n"
sleep 2

log ">>>>>>>>>>>>>>>>>>>>> Marker [7] <<<<<<<<<<<<<<<<<<<<<"
printf "\n"

#*******************************************************************************
# Make config changes for stafd
log "Change stafd config [3]:"
cat > "${stafd_conf_fname}" <<'EOF'
[Global]
tron              = true
queue-size        = 2000000
reconnect-delay   = 1
ctrl-loss-tmo     = 1
disable-sqflow    = true

[Discovery controller connection management]
persistent-connections=false
zeroconf-connections-persistence=0.5

[Service Discovery]
zeroconf=disabled
EOF
log_file_contents 0 "${stafd_conf_fname}"
printf "\n"

reload_cfg "stafd"
sleep 3

#*******************************************************************************
# Make more config changes for stafd
log "Change stafd config [4]:"
cat > "${stafd_conf_fname}" <<'EOF'
[Global]
tron=true
queue-size=2000000
reconnect-delay=1
ctrl-loss-tmo=0
disable-sqflow=true
ip-family=ipv6
pleo=disabled

[Discovery controller connection management]
persistent-connections=false
zeroconf-connections-persistence=0

[Controllers]
controller=transport=tcp;traddr=localhost;trsvcid=8009
controller=transport=tcp;traddr=abracadabra
controller=transport=tcp;traddr=google.com
controller=
controller=trsvcid
controller=transport=rdma;traddr=!@#$
controller=transport=fc;traddr=21:00:00:00:00:00:00:00;host-traddr=20:00:00:00:00:00:00:00
controller=transport=XM;traddr=2.2.2.2
controller=transport=tcp;traddr=555.555.555.555
EOF
log_file_contents 0 "${stafd_conf_fname}"
printf "\n"

log ">>>>>>>>>>>>>>>>>>>>> Marker [8] <<<<<<<<<<<<<<<<<<<<<"
printf "\n"

reload_cfg "stafd"
sleep 2

#*******************************************************************************
# Stop Avahi Publisher
log "Stop Avahi publisher"
run_cmd sudo systemctl stop ${AVAHI_PUBLISHER}
printf "\n"
sleep 2

log ">>>>>>>>>>>>>>>>>>>>> Marker [9] <<<<<<<<<<<<<<<<<<<<<"
printf "\n"

#*******************************************************************************
# Remove one of the NVMe device's
file=/tmp/getdev-XXX.py
getdev=$(mktemp $file)
cat > "${getdev}" <<'EOF'
import sys
from dasbus.connection import SystemMessageBus

bus = SystemMessageBus()
iface = bus.get_proxy(sys.argv[1], sys.argv[2])
controllers = iface.list_controllers(False)
if len(controllers) > 0:
    controller = controllers[0]
    print(controller['device'])
    sys.exit(0)
sys.exit(1)
EOF

# Find a Discovery Controller and issue a "nvme disconnect"
if dev=$(python3 ${getdev} @STAFD_DBUS_NAME@ @STAFD_DBUS_PATH@); then
	log "Remove connection (disconnect) to Discovery Controller ${dev}"
	run_cmd sudo nvme disconnect -d ${dev}
	printf "\n"
else
	msg="Failed to find a connection to a Discovery Controller"
	printf "%b%s%s%b[0m\n" "\0033" ${RED} "${msg}" "\0033"
	sudo logger -t COVERAGE -i "@@@@@ " -p warning -- "${msg}"
fi

# Find an I/O Controller and issue a "nvme disconnect"
if dev=$(python3 ${getdev} @STACD_DBUS_NAME@ @STACD_DBUS_PATH@); then
	log "Remove connection (disconnect) to I/O Controller ${dev}"
	run_cmd sudo nvme disconnect -d ${dev}
	printf "\n"
else
	msg="Failed to find a connection to an I/O Controller"
	printf "%b%s%s%b[0m\n" "\0033" ${RED} "${msg}" "\0033"
	sudo logger -t COVERAGE -i "@@@@@ " -p warning -- "${msg}"
fi

sleep 3

rm "${getdev}"


#*******************************************************************************
log ">>>>>>>>>>>>>>>>>>>>> Marker [10] <<<<<<<<<<<<<<<<<<<<<"
printf "\n"

sd_restart "stafd"
sd_restart "stacd"
sleep 4

log "Create invalid conditions for saving/loading stafd's last known config"
rm -rf "/tmp/stafd"
sd_restart "stafd"
sleep 2

log "Remove invalid conditions for saving/loading stafd's last known config"
mkdir -p "/tmp/stafd"
sd_restart "stafd"
sleep 2


log ">>>>>>>>>>>>>>>>>>>>> Marker [11] <<<<<<<<<<<<<<<<<<<<<"
printf "\n"

log "Change stafd config [5]:"
cat > "${stafd_conf_fname}" <<'EOF'
[Global]
tron              = true

[Controllers]
controller=transport=tcp;traddr=localhost
controller=transport=tcp;traddr=1.1.1.1
controller=transport=tcp;traddr=2.2.2.2
controller=transport=tcp;traddr=3.3.3.3
controller=transport=tcp;traddr=4.4.4.4
controller=transport=tcp;traddr=5.5.5.5
controller=transport=tcp;traddr=6.6.6.6
EOF
log_file_contents 0 "${stafd_conf_fname}"
printf "\n"

reload_cfg "stafd"
sleep 2

sd_stop "stafd"
sleep 5
sd_start "stafd"


#*******************************************************************************
# Change ownership of files that were created as root
sudo chown -R "${PRIMARY_USR}":"${PRIMARY_GRP}" coverage  >/dev/null 2>&1
sudo chown -R "${PRIMARY_USR}":"${PRIMARY_GRP}" staslib/__pycache__  >/dev/null 2>&1
sudo chown -R "${PRIMARY_USR}":"${PRIMARY_GRP}" subprojects/libnvme/libnvme/__pycache__  >/dev/null 2>&1

#*******************************************************************************
# Run unit tests
TEST_DIR=$( realpath ../test )
run_unit_test ${TEST_DIR}/test-avahi.py
run_unit_test ${TEST_DIR}/test-avahi.py
run_unit_test ${TEST_DIR}/test-config.py
run_unit_test ${TEST_DIR}/test-controller.py
run_unit_test ${TEST_DIR}/test-gtimer.py
run_unit_test ${TEST_DIR}/test-iputil.py
run_unit_test ${TEST_DIR}/test-log.py
run_unit_test ${TEST_DIR}/test-nbft.py
run_unit_test ${TEST_DIR}/test-nbft_conf.py
run_unit_test sudo ${TEST_DIR}/test-nvme_options.py  # Test both with super user...
run_unit_test ${TEST_DIR}/test-nvme_options.py       # ... and with regular user
run_unit_test ${TEST_DIR}/test-service.py
run_unit_test ${TEST_DIR}/test-timeparse.py
run_unit_test ${TEST_DIR}/test-transport_id.py
run_unit_test ${TEST_DIR}/test-defs.py
run_unit_test ${TEST_DIR}/test-gutil.py
run_unit_test ${TEST_DIR}/test-udev.py
run_unit_test ${TEST_DIR}/test-version.py

#*******************************************************************************
# Stop nvme target simulator

log "Collect all coverage data"
coverage combine --rcfile=.coveragerc
printf "\n"

log "Generating coverage report"
coverage report -i --rcfile=.coveragerc
printf "\n"

log "Generating coverage report (HTML)"
coverage html -i --rcfile=.coveragerc
printf "\n"