406 lines
13 KiB
Python
406 lines
13 KiB
Python
|
#!/usr/bin/python3
|
||
|
|
# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
|
||
|
|
# SPDX-License-Identifier: Apache-2.0
|
||
|
|
# See the LICENSE file for details.
|
||
|
|
#
|
||
|
|
# This file is part of NVMe STorage Appliance Services (nvme-stas).
|
||
|
|
#
|
||
|
|
# Authors: Martin Belanger <Martin.Belanger@dell.com>
|
||
|
|
|
||
|
|
# PYTHON_ARGCOMPLETE_OK
|
||
|
|
|
||
|
|
import os
|
||
|
|
import sys
|
||
|
|
import pprint
|
||
|
|
import pathlib
|
||
|
|
import subprocess
|
||
|
|
from argparse import ArgumentParser
|
||
|
|
|
||
|
|
VERSION = 1.0
|
||
|
|
DEFAULT_CONFIG_FILE = './nvmet.conf'
|
||
|
|
|
||
|
|
|
||
|
|
class Fore:
|
||
|
|
RED = '\033[31m'
|
||
|
|
GREEN = '\033[32m'
|
||
|
|
|
||
|
|
|
||
|
|
class Style:
|
||
|
|
RESET_ALL = '\033[0m'
|
||
|
|
|
||
|
|
|
||
|
|
def _get_loaded_nvmet_modules():
|
||
|
|
try:
|
||
|
|
cp = subprocess.run('/usr/sbin/lsmod', capture_output=True, text=True)
|
||
|
|
except TypeError:
|
||
|
|
# For older Python versions that don't support "capture_output" or "text"
|
||
|
|
cp = subprocess.run('/usr/sbin/lsmod', stdout=subprocess.PIPE, universal_newlines=True)
|
||
|
|
|
||
|
|
if cp.returncode != 0 or not cp.stdout:
|
||
|
|
return []
|
||
|
|
|
||
|
|
output = []
|
||
|
|
lines = cp.stdout.split('\n')
|
||
|
|
for line in lines:
|
||
|
|
if 'nvmet_' in line:
|
||
|
|
module = line.split()[0]
|
||
|
|
for end in ('loop', 'tcp', 'fc', 'rdma'):
|
||
|
|
if module.endswith(end):
|
||
|
|
output.append(module)
|
||
|
|
break
|
||
|
|
|
||
|
|
return output
|
||
|
|
|
||
|
|
|
||
|
|
def _runcmd(cmd: list, quiet=False):
|
||
|
|
if not quiet:
|
||
|
|
print(' '.join(cmd))
|
||
|
|
if args.dry_run:
|
||
|
|
return
|
||
|
|
subprocess.run(cmd)
|
||
|
|
|
||
|
|
|
||
|
|
def _modprobe(module: str, args: list = None, quiet=False):
|
||
|
|
cmd = ['/usr/sbin/modprobe', module]
|
||
|
|
if args:
|
||
|
|
cmd.extend(args)
|
||
|
|
_runcmd(cmd, quiet)
|
||
|
|
|
||
|
|
|
||
|
|
def _mkdir(dname: str):
|
||
|
|
print(f'mkdir -p "{dname}"')
|
||
|
|
if args.dry_run:
|
||
|
|
return
|
||
|
|
pathlib.Path(dname).mkdir(parents=True, exist_ok=True)
|
||
|
|
|
||
|
|
|
||
|
|
def _echo(value, fname: str):
|
||
|
|
print(f'echo -n "{value}" > "{fname}"')
|
||
|
|
if args.dry_run:
|
||
|
|
return
|
||
|
|
with open(fname, 'w') as f:
|
||
|
|
f.write(str(value))
|
||
|
|
|
||
|
|
|
||
|
|
def _symlink(port: str, subsysnqn: str):
|
||
|
|
print(
|
||
|
|
f'$( cd "/sys/kernel/config/nvmet/ports/{port}/subsystems" && ln -s "../../../subsystems/{subsysnqn}" "{subsysnqn}" )'
|
||
|
|
)
|
||
|
|
if args.dry_run:
|
||
|
|
return
|
||
|
|
target = os.path.join('/sys/kernel/config/nvmet/subsystems', subsysnqn)
|
||
|
|
link = pathlib.Path(os.path.join('/sys/kernel/config/nvmet/ports', port, 'subsystems', subsysnqn))
|
||
|
|
link.symlink_to(target)
|
||
|
|
|
||
|
|
|
||
|
|
def _create_subsystem(subsysnqn: str) -> str:
|
||
|
|
print(f'###{Fore.GREEN} Create subsystem: {subsysnqn}{Style.RESET_ALL}')
|
||
|
|
dname = os.path.join('/sys/kernel/config/nvmet/subsystems/', subsysnqn)
|
||
|
|
_mkdir(dname)
|
||
|
|
_echo(1, os.path.join(dname, 'attr_allow_any_host'))
|
||
|
|
return dname
|
||
|
|
|
||
|
|
|
||
|
|
def _create_namespace(subsysnqn: str, id: str, node: str) -> str:
|
||
|
|
print(f'###{Fore.GREEN} Add namespace: {id}{Style.RESET_ALL}')
|
||
|
|
dname = os.path.join('/sys/kernel/config/nvmet/subsystems/', subsysnqn, 'namespaces', id)
|
||
|
|
_mkdir(dname)
|
||
|
|
_echo(node, os.path.join(dname, 'device_path'))
|
||
|
|
_echo(1, os.path.join(dname, 'enable'))
|
||
|
|
return dname
|
||
|
|
|
||
|
|
|
||
|
|
def _args_valid(id, traddr, trsvcid, trtype, adrfam):
|
||
|
|
if None in (id, trtype):
|
||
|
|
return False
|
||
|
|
|
||
|
|
if trtype != 'loop' and None in (traddr, trsvcid, adrfam):
|
||
|
|
return False
|
||
|
|
|
||
|
|
return True
|
||
|
|
|
||
|
|
|
||
|
|
def _create_port(port: str, traddr: str, trsvcid: str, trtype: str, adrfam: str):
|
||
|
|
'''@param port: This is a nvmet port and not a tcp port.'''
|
||
|
|
print(f'###{Fore.GREEN} Create port: {port} -> {traddr}:{trsvcid}{Style.RESET_ALL}')
|
||
|
|
dname = os.path.join('/sys/kernel/config/nvmet/ports', port)
|
||
|
|
_mkdir(dname)
|
||
|
|
_echo(trtype, os.path.join(dname, 'addr_trtype'))
|
||
|
|
if traddr:
|
||
|
|
_echo(traddr, os.path.join(dname, 'addr_traddr'))
|
||
|
|
if trsvcid:
|
||
|
|
_echo(trsvcid, os.path.join(dname, 'addr_trsvcid'))
|
||
|
|
if adrfam:
|
||
|
|
_echo(adrfam, os.path.join(dname, 'addr_adrfam'))
|
||
|
|
|
||
|
|
|
||
|
|
def _map_subsystems_to_ports(subsystems: list):
|
||
|
|
print(f'###{Fore.GREEN} Map subsystems to ports{Style.RESET_ALL}')
|
||
|
|
for subsystem in subsystems:
|
||
|
|
subsysnqn, port = subsystem.get('subsysnqn'), str(subsystem.get('port'))
|
||
|
|
if None not in (subsysnqn, port):
|
||
|
|
_symlink(port, subsysnqn)
|
||
|
|
|
||
|
|
|
||
|
|
def _read_config(fname: str) -> dict:
|
||
|
|
try:
|
||
|
|
with open(fname) as f:
|
||
|
|
return eval(f.read())
|
||
|
|
except Exception as e:
|
||
|
|
sys.exit(f'Error reading config file. {e}')
|
||
|
|
|
||
|
|
|
||
|
|
def _read_attr_from_file(fname: str) -> str:
|
||
|
|
try:
|
||
|
|
with open(fname, 'r') as f:
|
||
|
|
return f.read().strip('\n')
|
||
|
|
except Exception as e:
|
||
|
|
sys.exit(f'Error reading attribute. {e}')
|
||
|
|
|
||
|
|
|
||
|
|
################################################################################
|
||
|
|
|
||
|
|
|
||
|
|
def create(args):
|
||
|
|
# Need to be root to run this script
|
||
|
|
if not args.dry_run and os.geteuid() != 0:
|
||
|
|
sys.exit(f'Permission denied. You need root privileges to run {os.path.basename(__file__)}.')
|
||
|
|
|
||
|
|
config = _read_config(args.conf_file)
|
||
|
|
|
||
|
|
print('')
|
||
|
|
|
||
|
|
# Create a dummy null block device (if one doesn't already exist)
|
||
|
|
dev_node = '/dev/nullb0'
|
||
|
|
_modprobe('null_blk', ['nr_devices=1'])
|
||
|
|
|
||
|
|
ports = config.get('ports')
|
||
|
|
if ports is None:
|
||
|
|
sys.exit(f'Config file "{args.conf_file}" missing a "ports" section')
|
||
|
|
|
||
|
|
subsystems = config.get('subsystems')
|
||
|
|
if subsystems is None:
|
||
|
|
sys.exit(f'Config file "{args.conf_file}" missing a "subsystems" section')
|
||
|
|
|
||
|
|
# Extract the list of transport types found in the
|
||
|
|
# config file and load the corresponding kernel module.
|
||
|
|
_modprobe('nvmet')
|
||
|
|
trtypes = {port.get('trtype') for port in ports if port.get('trtype') is not None}
|
||
|
|
for trtype in trtypes:
|
||
|
|
if trtype in ('tcp', 'fc', 'rdma'):
|
||
|
|
_modprobe(f'nvmet_{trtype}')
|
||
|
|
elif trtype == 'loop':
|
||
|
|
_modprobe('nvmet_loop')
|
||
|
|
|
||
|
|
for port in ports:
|
||
|
|
print('')
|
||
|
|
id, traddr, trsvcid, trtype, adrfam = (
|
||
|
|
str(port.get('id')),
|
||
|
|
port.get('traddr'),
|
||
|
|
port.get('trsvcid'),
|
||
|
|
port.get('trtype'),
|
||
|
|
port.get('adrfam'),
|
||
|
|
)
|
||
|
|
if _args_valid(id, traddr, trsvcid, trtype, adrfam):
|
||
|
|
_create_port(id, traddr, trsvcid, trtype, adrfam)
|
||
|
|
else:
|
||
|
|
print(
|
||
|
|
f'{Fore.RED}### Config file "{args.conf_file}" error in "ports" section: id={id}, traddr={traddr}, trsvcid={trsvcid}, trtype={trtype}, adrfam={adrfam}{Style.RESET_ALL}'
|
||
|
|
)
|
||
|
|
|
||
|
|
for subsystem in subsystems:
|
||
|
|
print('')
|
||
|
|
subsysnqn, port, namespaces = (
|
||
|
|
subsystem.get('subsysnqn'),
|
||
|
|
str(subsystem.get('port')),
|
||
|
|
subsystem.get('namespaces'),
|
||
|
|
)
|
||
|
|
if None not in (subsysnqn, port, namespaces):
|
||
|
|
_create_subsystem(subsysnqn)
|
||
|
|
for id in namespaces:
|
||
|
|
_create_namespace(subsysnqn, str(id), dev_node)
|
||
|
|
else:
|
||
|
|
print(
|
||
|
|
f'{Fore.RED}### Config file "{args.conf_file}" error in "subsystems" section: subsysnqn={subsysnqn}, port={port}, namespaces={namespaces}{Style.RESET_ALL}'
|
||
|
|
)
|
||
|
|
|
||
|
|
print('')
|
||
|
|
_map_subsystems_to_ports(subsystems)
|
||
|
|
|
||
|
|
print('')
|
||
|
|
|
||
|
|
|
||
|
|
def clean(args):
|
||
|
|
# Need to be root to run this script
|
||
|
|
if not args.dry_run and os.geteuid() != 0:
|
||
|
|
sys.exit(f'Permission denied. You need root privileges to run {os.path.basename(__file__)}.')
|
||
|
|
|
||
|
|
print('rm -f /sys/kernel/config/nvmet/ports/*/subsystems/*')
|
||
|
|
for dname in pathlib.Path('/sys/kernel/config/nvmet/ports').glob('*/subsystems/*'):
|
||
|
|
_runcmd(['rm', '-f', str(dname)], quiet=True)
|
||
|
|
|
||
|
|
print('rmdir /sys/kernel/config/nvmet/ports/*')
|
||
|
|
for dname in pathlib.Path('/sys/kernel/config/nvmet/ports').glob('*'):
|
||
|
|
_runcmd(['rmdir', str(dname)], quiet=True)
|
||
|
|
|
||
|
|
print('rmdir /sys/kernel/config/nvmet/subsystems/*/namespaces/*')
|
||
|
|
for dname in pathlib.Path('/sys/kernel/config/nvmet/subsystems').glob('*/namespaces/*'):
|
||
|
|
_runcmd(['rmdir', str(dname)], quiet=True)
|
||
|
|
|
||
|
|
print('rmdir /sys/kernel/config/nvmet/subsystems/*')
|
||
|
|
for dname in pathlib.Path('/sys/kernel/config/nvmet/subsystems').glob('*'):
|
||
|
|
_runcmd(['rmdir', str(dname)], quiet=True)
|
||
|
|
|
||
|
|
for module in _get_loaded_nvmet_modules():
|
||
|
|
_modprobe(module, ['--remove'])
|
||
|
|
|
||
|
|
_modprobe('nvmet', ['--remove'])
|
||
|
|
_modprobe('null_blk', ['--remove'])
|
||
|
|
|
||
|
|
|
||
|
|
def link(args):
|
||
|
|
port = str(args.port)
|
||
|
|
subsysnqn = str(args.subnqn)
|
||
|
|
if not args.dry_run:
|
||
|
|
if os.geteuid() != 0:
|
||
|
|
# Need to be root to run this script
|
||
|
|
sys.exit(f'Permission denied. You need root privileges to run {os.path.basename(__file__)}.')
|
||
|
|
|
||
|
|
symlink = os.path.join('/sys/kernel/config/nvmet/ports', port, 'subsystems', subsysnqn)
|
||
|
|
if os.path.exists(symlink):
|
||
|
|
sys.exit(f'Symlink already exists: {symlink}')
|
||
|
|
|
||
|
|
_symlink(port, subsysnqn)
|
||
|
|
|
||
|
|
|
||
|
|
def unlink(args):
|
||
|
|
port = str(args.port)
|
||
|
|
subsysnqn = str(args.subnqn)
|
||
|
|
symlink = os.path.join('/sys/kernel/config/nvmet/ports', port, 'subsystems', subsysnqn)
|
||
|
|
if not args.dry_run:
|
||
|
|
if os.geteuid() != 0:
|
||
|
|
# Need to be root to run this script
|
||
|
|
sys.exit(f'Permission denied. You need root privileges to run {os.path.basename(__file__)}.')
|
||
|
|
|
||
|
|
if not os.path.exists(symlink):
|
||
|
|
sys.exit(f'No such symlink: {symlink}')
|
||
|
|
|
||
|
|
_runcmd(['rm', symlink])
|
||
|
|
|
||
|
|
|
||
|
|
def ls(args):
|
||
|
|
ports = list()
|
||
|
|
for port_path in pathlib.Path('/sys/kernel/config/nvmet/ports').glob('*'):
|
||
|
|
id = port_path.parts[-1]
|
||
|
|
port = {
|
||
|
|
'id': int(id),
|
||
|
|
'traddr': _read_attr_from_file(os.path.join('/sys/kernel/config/nvmet/ports', id, 'addr_traddr')),
|
||
|
|
'trsvcid': _read_attr_from_file(os.path.join('/sys/kernel/config/nvmet/ports', id, 'addr_trsvcid')),
|
||
|
|
'adrfam': _read_attr_from_file(os.path.join('/sys/kernel/config/nvmet/ports', id, 'addr_adrfam')),
|
||
|
|
'trtype': _read_attr_from_file(os.path.join('/sys/kernel/config/nvmet/ports', id, 'addr_trtype')),
|
||
|
|
}
|
||
|
|
|
||
|
|
ports.append(port)
|
||
|
|
|
||
|
|
subsystems = dict()
|
||
|
|
for subsystem_path in pathlib.Path('/sys/kernel/config/nvmet/subsystems').glob('*'):
|
||
|
|
subsysnqn = subsystem_path.parts[-1]
|
||
|
|
namespaces_path = pathlib.Path(os.path.join('/sys/kernel/config/nvmet/subsystems', subsysnqn, 'namespaces'))
|
||
|
|
subsystems[subsysnqn] = {
|
||
|
|
'port': None,
|
||
|
|
'subsysnqn': subsysnqn,
|
||
|
|
'namespaces': sorted([int(namespace_path.parts[-1]) for namespace_path in namespaces_path.glob('*')]),
|
||
|
|
}
|
||
|
|
|
||
|
|
# Find the port that each subsystem is mapped to
|
||
|
|
for subsystem_path in pathlib.Path('/sys/kernel/config/nvmet/ports').glob('*/subsystems/*'):
|
||
|
|
subsysnqn = subsystem_path.parts[-1]
|
||
|
|
if subsysnqn in subsystems:
|
||
|
|
subsystems[subsysnqn]['port'] = int(subsystem_path.parts[-3])
|
||
|
|
|
||
|
|
output = {
|
||
|
|
'ports': ports,
|
||
|
|
'subsystems': list(subsystems.values()),
|
||
|
|
}
|
||
|
|
|
||
|
|
if sys.version_info < (3, 8):
|
||
|
|
print(pprint.pformat(output, width=70))
|
||
|
|
else:
|
||
|
|
print(pprint.pformat(output, width=70, sort_dicts=False))
|
||
|
|
|
||
|
|
print('')
|
||
|
|
|
||
|
|
|
||
|
|
################################################################################
|
||
|
|
|
||
|
|
parser = ArgumentParser(description="Create NVMe-oF Storage Subsystems")
|
||
|
|
parser.add_argument('-v', '--version', action='store_true', help='Print version, then exit', default=False)
|
||
|
|
|
||
|
|
subparser = parser.add_subparsers(title='Commands', description='valid commands')
|
||
|
|
|
||
|
|
prsr = subparser.add_parser('create', help='Create nvme targets')
|
||
|
|
prsr.add_argument(
|
||
|
|
'-f',
|
||
|
|
'--conf-file',
|
||
|
|
action='store',
|
||
|
|
help='Configuration file (default: %(default)s)',
|
||
|
|
default=DEFAULT_CONFIG_FILE,
|
||
|
|
type=str,
|
||
|
|
metavar='FILE',
|
||
|
|
)
|
||
|
|
prsr.add_argument(
|
||
|
|
'-d', '--dry-run', action='store_true', help='Just print what would be done. (default: %(default)s)', default=False
|
||
|
|
)
|
||
|
|
prsr.set_defaults(func=create)
|
||
|
|
|
||
|
|
prsr = subparser.add_parser('clean', help='Remove all previously created nvme targets')
|
||
|
|
prsr.add_argument(
|
||
|
|
'-d', '--dry-run', action='store_true', help='Just print what would be done. (default: %(default)s)', default=False
|
||
|
|
)
|
||
|
|
prsr.set_defaults(func=clean)
|
||
|
|
|
||
|
|
prsr = subparser.add_parser('ls', help='List ports and subsystems')
|
||
|
|
prsr.set_defaults(func=ls)
|
||
|
|
|
||
|
|
prsr = subparser.add_parser('link', help='Map a subsystem to a port')
|
||
|
|
prsr.add_argument(
|
||
|
|
'-d', '--dry-run', action='store_true', help='Just print what would be done. (default: %(default)s)', default=False
|
||
|
|
)
|
||
|
|
prsr.add_argument('-p', '--port', action='store', type=int, help='nvmet port', required=True)
|
||
|
|
prsr.add_argument('-s', '--subnqn', action='store', type=str, help='nvmet subsystem NQN', required=True, metavar='NQN')
|
||
|
|
prsr.set_defaults(func=link)
|
||
|
|
|
||
|
|
prsr = subparser.add_parser('unlink', help='Unmap a subsystem from a port')
|
||
|
|
prsr.add_argument(
|
||
|
|
'-d', '--dry-run', action='store_true', help='Just print what would be done. (default: %(default)s)', default=False
|
||
|
|
)
|
||
|
|
prsr.add_argument('-p', '--port', action='store', type=int, help='nvmet port', required=True)
|
||
|
|
prsr.add_argument('-s', '--subnqn', action='store', type=str, help='nvmet subsystem NQN', required=True, metavar='NQN')
|
||
|
|
prsr.set_defaults(func=unlink)
|
||
|
|
|
||
|
|
|
||
|
|
# =============================
|
||
|
|
# Tab-completion.
|
||
|
|
# MUST BE CALLED BEFORE parser.parse_args() BELOW.
|
||
|
|
# Ref: https://kislyuk.github.io/argcomplete/
|
||
|
|
#
|
||
|
|
# If you do have argcomplete installed, you also need to run
|
||
|
|
# "sudo activate-global-python-argcomplete3" to globally activate
|
||
|
|
# auto-completion. Ref: https://pypi.python.org/pypi/argcomplete#global-completion
|
||
|
|
try:
|
||
|
|
import argcomplete
|
||
|
|
|
||
|
|
argcomplete.autocomplete(parser)
|
||
|
|
except ModuleNotFoundError:
|
||
|
|
# auto-complete is not necessary for the operation of this script. Just nice to have
|
||
|
|
pass
|
||
|
|
|
||
|
|
args = parser.parse_args()
|
||
|
|
|
||
|
|
if args.version:
|
||
|
|
print(f'{os.path.basename(__file__)} {VERSION}')
|
||
|
|
sys.exit(0)
|
||
|
|
|
||
|
|
# Invoke the sub-command
|
||
|
|
args.func(args)
|