linux/nvme-stas
1
0
Fork 0
Code Issues Activity

Adding upstream version 2.2.1.

Browse source 
Signed-off-by: Daniel Baumann <daniel@debian.org>
This commit is contained in:
Benjamin Drung 2025-02-16 12:41:59 +01:00 • committed by Daniel Baumann
parent 1d36de0179
commit 757b718eff
Signed by: daniel
GPG key ID: FBB4F0E80A80222F
129 changed files with 16110 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
.coveragerc.in Normal file
View file

@ -0,0 +1,39 @@
# .coveragerc to control coverage.py for combined stafd/stacd coverage
[run]
data_file = coverage/nvme-stas
parallel=True
concurrency=thread
[report]
omit =
/usr/*
*/test/test-*.py
subprojects/libnvme/*
# Regexes for lines to exclude from consideration
exclude_lines =
# Have to re-enable the standard pragma
pragma: no cover
# Coverage cannot cover code running in threads
def in_thread_exec
# Don't complain if tests don't hit defensive assertion code:
raise AssertionError
raise RuntimeError
# Don't complain if non-runnable code isn't run:
pass
if 0:
if __name__ *== *__main__ *:
sys\.exit\(\)
sys\.exit\(f?'.+\)
# ImportError is usually OK because there will be a workaround import.
except ImportError
skip_empty = True
[html]
directory = coverage
title = nvme-stas coverage report

3
.dockerignore Normal file
View file

@ -0,0 +1,3 @@
.git
.github
.build

11
.github/dependabot.yml vendored Normal file
View file

@ -0,0 +1,11 @@
---
version: 2
updates:
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "weekly"
- package-ecosystem: "docker"
directory: "/"
schedule:
interval: "weekly"

63
.github/workflows/docker-publish.yml vendored Normal file
View file

@ -0,0 +1,63 @@
name: Docker
# This workflow uses actions that are not certified by GitHub.
# They are provided by a third-party and are governed by
# separate terms of service, privacy policy, and support
# documentation.
on:
push:
branches: [ main ]
# Publish semver tags as releases.
tags: [ 'v*.*.*' ]
pull_request:
branches: [ main ]
workflow_dispatch:
env:
# Use docker.io for Docker Hub if empty
REGISTRY: ghcr.io
# github.repository as <account>/<repo>
IMAGE_NAME: ${{ github.repository }}
jobs:
docker-publish:
if: ${{ !github.event.act }} # skip during local actions testing
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout repository
uses: actions/checkout@v3
# Login against a Docker registry except on PR
# https://github.com/docker/login-action
- name: Log into registry ${{ env.REGISTRY }}
if: github.event_name != 'pull_request'
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
# Extract metadata (tags, labels) for Docker
# https://github.com/docker/metadata-action
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
# Build and push Docker image with Buildx (don't push on PR)
# https://github.com/docker/build-push-action
- name: Build and push Docker image
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
with:
context: .
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}

51
.github/workflows/docker-test.yml vendored Normal file
View file

@ -0,0 +1,51 @@
name: Test
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
workflow_dispatch:
jobs:
docker-run:
if: ${{ !github.event.act }} # skip during local actions testing
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Install requirements
# make sure nvme-cli installed (we need it for /etc/nvme/hostnqn and /etc/nvme/hostid)
run: sudo apt-get install --yes --quiet nvme-cli
- name: Load Kernel drivers
run: sudo modprobe -v nvme-fabrics
- name: Fix D-BUS
run: |
sed 's/@STAFD_DBUS_NAME@/org.nvmexpress.staf/g' etc/dbus-1/system.d/org.nvmexpress.staf.in.conf | sudo tee /usr/share/dbus-1/system.d/org.nvmexpress.staf.conf
sed 's/@STACD_DBUS_NAME@/org.nvmexpress.stac/g' etc/dbus-1/system.d/org.nvmexpress.stac.in.conf | sudo tee /usr/share/dbus-1/system.d/org.nvmexpress.stac.conf
sudo systemctl reload dbus.service
- name: Build & Start containers
run: docker-compose -f "docker-compose.yml" up --detach --build
- name: Run tests
run: |
docker-compose ps
docker-compose exec -T stafd stafctl ls
docker-compose exec -T stafd stafctl status
docker-compose exec -T stacd stacctl ls
docker-compose exec -T stacd stacctl status
docker-compose logs
- name: Logs
if: failure()
run: |
docker-compose ps || true
docker-compose logs || true
- name: Stop containers
if: always()
run: docker-compose -f "docker-compose.yml" down

76
.github/workflows/meson-test.yml vendored Normal file
View file

@ -0,0 +1,76 @@
name: Meson
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
workflow_dispatch:
jobs:
meson-build:
runs-on: ubuntu-latest
steps:
- name: "CHECKOUT: nvme-stas"
uses: actions/checkout@v3
- name: "INSTALL: Overall dependencies"
run: |
sudo apt-mark hold grub-efi-amd64-signed # Workaround for upstream issue
sudo apt-get update --yes --quiet
sudo apt-get upgrade --yes --quiet
sudo apt-get install --yes --quiet python3-pip cmake iproute2
sudo python3 -m pip install --upgrade pip
sudo python3 -m pip install --upgrade wheel meson ninja
- name: "INSTALL: nvme-stas dependencies"
run: |
sudo apt-get install --yes --quiet docbook-xml
sudo apt-get install --yes --quiet docbook-xsl
sudo apt-get install --yes --quiet xsltproc
sudo apt-get install --yes --quiet libglib2.0-dev
sudo apt-get install --yes --quiet libgirepository1.0-dev
sudo apt-get install --yes --quiet libsystemd-dev
sudo apt-get install --yes --quiet python3-systemd
sudo apt-get install --yes --quiet python3-pyudev
sudo apt-get install --yes --quiet python3-lxml
python3 -m pip install --upgrade dasbus pylint pyflakes PyGObject
python3 -m pip install --upgrade vermin pyfakefs importlib-resources
- name: "INSTALL: libnvme"
run: |
sudo apt-get install --yes --quiet swig libjson-c-dev
meson subprojects download
meson setup .build subprojects/libnvme -Dlibdbus=disabled -Dopenssl=disabled -Dbuildtype=release -Dprefix=/usr -Dpython=true
ninja -C .build
sudo meson install -C .build
- name: "CONFIG: PYTHONPATH"
run: |
echo "PYTHONPATH=.build:.build/subprojects/libnvme:/usr/lib/python3/dist-packages/" >> $GITHUB_ENV
- name: "TEST: nvme-stas"
uses: BSFishy/meson-build@v1.0.3
with:
action: test
directory: .build
setup-options: -Dman=true -Dhtml=true
options: --print-errorlogs --suite nvme-stas
# Preserve meson's log file on failure
- uses: actions/upload-artifact@v3
if: failure()
with:
name: "Linux_Meson_Testlog"
path: .build/meson-logs/*
- name: "Generate coverage report"
run: |
python3 -m pip install pytest
python3 -m pip install pytest-cov
PYTHONPATH=.build:.build/subprojects/libnvme:/usr/lib/python3/dist-packages/ pytest --cov=./staslib --cov-report=xml test/test-*.py
- uses: codecov/codecov-action@v3
with:
fail_ci_if_error: false

104
.github/workflows/pylint.yml vendored Normal file
View file

@ -0,0 +1,104 @@
name: Linters
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
workflow_dispatch:
jobs:
docker-lint:
if: ${{ !github.event.act }} # skip during local actions testing
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- uses: hadolint/hadolint-action@v3.1.0
with:
recursive: true
ignore: DL3041
python-lint:
runs-on: ubuntu-20.04
strategy:
fail-fast: false
matrix:
python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
steps:
- name: "CHECKOUT: nvme-stas"
uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
# - name: "UPGRADE: existing packages"
# run: |
# sudo apt-get update --yes --quiet || true
# sudo apt-get upgrade --yes --quiet || true
- name: "INSTALL: additional packages"
run: |
sudo apt-get install --yes --quiet python3-pip || true
sudo apt-get install --yes --quiet cmake || true
sudo apt-get install --yes --quiet libgirepository1.0-dev || true
sudo apt-get install --yes --quiet libsystemd-dev || true
sudo apt-get install --yes --quiet python3-systemd || true
sudo python3 -m pip install --upgrade pip
sudo python3 -m pip install --upgrade wheel
sudo python3 -m pip install --upgrade meson
sudo python3 -m pip install --upgrade ninja
python3 -m pip install --upgrade dasbus
python3 -m pip install --upgrade pylint
python3 -m pip install --upgrade pyflakes
python3 -m pip install --upgrade PyGObject
python3 -m pip install --upgrade lxml
python3 -m pip install --upgrade pyudev
- name: "BUILD: libnvme"
run: |
sudo apt-get install --yes --quiet swig libjson-c-dev || true
meson subprojects download
meson setup builddir subprojects/libnvme -Dlibdbus=disabled -Dopenssl=disabled -Dbuildtype=release -Dprefix=/usr -Dpython=true
ninja -C builddir
sudo meson install -C builddir
- name: Set PYTHONPATH
run: |
echo "PYTHONPATH=builddir:builddir/subprojects/libnvme:/usr/lib/python3/dist-packages/" >> $GITHUB_ENV
- name: Show test environment
run: |
echo -e "Build Directory:\n$(ls -laF builddir)"
python3 -VV
python3 -m site
python3 -m pylint --version
echo "pyflakes $(python3 -m pyflakes --version)"
- name: Pylint
run: |
python3 -m pylint --rcfile=test/pylint.rc *.py staslib
- name: Pyflakes
if: always()
run: |
python3 -m pyflakes *.py staslib
python-black:
if: ${{ !github.event.act }} # skip during local actions testing
name: python-black formatter
runs-on: ubuntu-latest
steps:
- name: "CHECKOUT: nvme-stas"
uses: actions/checkout@v3
- name: "BLACK"
uses: psf/black@stable
with:
options: "--check --diff --color --line-length 120 --skip-string-normalization --extend-exclude (subprojects|debian|.build)"
src: "."

9
.gitignore vendored Normal file
View file

@ -0,0 +1,9 @@
.build
obj-x86_64-linux-gnu # DEBs Artifacts
redhat-linux-build # RPMs Artifacts
__pycache__
subprojects/*
!subprojects/*.wrap
.vscode

33
.readthedocs.yaml Normal file
View file

@ -0,0 +1,33 @@
# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See the LICENSE file for details.
#
# This file is part of NVMe STorage Appliance Services (nvme-stas).
#
# Authors: Martin Belanger <Martin.Belanger@dell.com>
#
version: 2
python:
system_packages: true
build:
os: ubuntu-22.04
tools:
python: "3"
apt_packages:
- meson
- python3-lxml
- docbook-xsl
- xsltproc
- pandoc
jobs:
post_install:
- pip3 install lxml
pre_build:
- meson .build -Dreadthedocs=true || cat .build/meson-logs/meson-log.txt
- ninja -C .build
sphinx:
configuration: .build/doc/readthedocs/conf.py

53
CONTRIBUTING.md Normal file
View file

@ -0,0 +1,53 @@
# Contributing guidelines
Thanks for contributing to this project. We'd like to get your feedback and suggestions.
## Issues
Bugs, feature requests, or issues must be reported through GitHub's "[Issues](https://github.com/linux-nvme/nvme-stas/issues)". Make sure there is not an existing open issue (or recently closed) for the same problem or feature. Include all pertinent info: environment, nvme-stas version, how to reproduce, expected result, etc.
## Contribution process
All contributions should be made through pull requests. Before submitting make sure that you followed the coding style (below) and you ran and passed the unit tests.
### How to submit contributions
1. Fork the repo
2. Make changes. Try to split you changes into distinct commits and avoid making big reformatting as it makes it harder to review the changes.
3. If possible, add unit tests for new features.
4. Run `make black` to make sure the changes conform to coding styles. See [Coding styles]() below.
5. Run `make test` and make sure all tests pass.
6. Commit to your fork with descriptive message and use the "`--signoff, -s`" option
7. Send the pull request
8. Check for failures in the automated CI output.
9. Be involved in the conversation (if any).
## Coding style
nvme-stas uses [black](https://black.readthedocs.io/en/stable/), [pylint](https://pylint.readthedocs.io/en/latest/), and [pyflakes](https://pypi.org/project/pyflakes/) to check that the code meets minimum style requirements. We use `black` with the following options.
```bash
black --diff --color --line-length 120 --skip-string-normalization [file or directory]
```
You can also use this convenience make command:
```
make black
```
## Minimum Python version required
nvme-stas must be able to run with Python 3.6. Code changes cannot use Python features not supported by Python 3.6. The only exception is for Python scripts used during the build phase (e.g. scripts to generate the documentation) or test scripts. Those scripts can follow Python 3.8 syntax.
nvme-stas uses [vermin](https://pypi.org/project/vermin/) to verify that the code submitted complies with the minimum version required. Vermin gets executed as part of the tests (see `make test` below).
## Unit tests
Unit tests can be run with this command:
```bash
make test
```
This command not only runs the unit tests, but also pylint, pyflakes, and vermin. Make sure that these programs are installed otherwise the tests will be skipped.

403
DISTROS.md Normal file
View file

@ -0,0 +1,403 @@
# Notes to Linux distributors
This document contains information about the packaging of nvme-stas.
## Compile-time dependencies
nvme-stas is a Python 3 project and does not require compile-time libraries per se. However, we use the meson build system for installation and testing.
| Library / Program | Purpose | Mandatory / Optional |
| ----------------- | ------------------------------------------------- | -------------------- |
| meson | Project configuration, installation, and testing. | Mandatory |
## Unit tests dependencies
nvme-stas provides static code analysis (pylint, pyflakes), which can be run with "`meson test`".
| Library / Program | Purpose | Mandatory / Optional |
| ----------------- | ------------------------------------------------------------ | -------------------- |
| pylint | Static code analysis | Optional |
| python3-pyflakes | Static code analysis | Optional |
| python3-pyfakefs | Static code analysis | Optional |
| vermin | Check that code meets minimum Python version requirement (3.6) | Optional |
## Run-time dependencies
Python 3.6 is the minimum version required to run nvme-stas. nvme-stas is built on top of libnvme, which is used to interact with the kernel's NVMe driver (i.e. `drivers/nvme/host/`). To support all the features of nvme-stas, several changes to the Linux kernel are required. nvme-stas can also operate with older kernels, but with limited functionality. Kernel 5.18 provides all the features needed by nvme-stas. nvme-stas can also work with older kernels that include back-ported changes to the NVMe driver.
The next table shows different features that were added to the NVMe driver and in which version of the Linux kernel they were added (the list of git patches can be found in addendum). Note that the ability to query the NVMe driver to determine what options it supports was added in 5.17. This is needed if nvme-stas is to make the right decision on whether a feature is supported. Otherwise, nvme-stas can only rely on the kernel version to decide what is supported. This can greatly limit the features supported on back-ported kernels.
| Feature | Introduced in kernel version |
| ------------------------------------------------------------ | ---------------------------- |
| **`host-iface` option** - Ability to force TCP connections over a specific interface. Needed for zeroconf provisioning. | 5.14 |
| **TP8013 Support** - Discovery Controller (DC) Unique NQN. Allow the creation of connections to DC with a NQN other than the default `nqn.2014-08.org.nvmexpress.discovery` | 5.16 |
| **Query supported options** - Allow user-space applications to query which options the NVMe driver supports | 5.17 |
| **TP8010 Support** - Ability for a Host to register with a Discovery Controller. This version of the kernel introduces a new event to indicate to user-space apps (e.g. nvme-stas) when a connection to a DC is restored. This is used to trigger a re-registration of the host. This kernel also exposes the DC Type (dctype) attribute through the sysfs, which is needed to determine whether registration is supported. | 5.18 |
| - Print actual source IP address (`src_addr`) through sysfs "address" attr. This is needed to verify that TCP connections were made on the right interface.<br />- Consider also `host_iface` when checking IP options.<br />- Send a rediscover uevent when a persistent discovery controller reconnects. | 6.1 |
nvme-stas also depends on the following run-time libraries and modules. Note that versions listed are the versions that were tested with at the time the code was developed.
| Package / Module | Min version | stafd | stacd | How to determine the currently installed version |
| ---------------------------------------------------------- | ----------- | ------------- | ------------- | ------------------------------------------------------------ |
| python3 | 3.6 | **Mandatory** | **Mandatory** | `python3 --version`<br />`nvme-stas` requires Python 3.6 as a minimum. |
| python3-dasbus | 1.6 | **Mandatory** | **Mandatory** | pip list \| grep dasbus |
| python3-pyudev | 0.22.0 | **Mandatory** | **Mandatory** | `python3 -c 'import pyudev; print(f"{pyudev.__version__}")'` |
| python3-systemd | 240 | **Mandatory** | **Mandatory** | `systemd --version` |
| python3-gi (Debian) OR python3-gobject (Fedora) | 3.36.0 | **Mandatory** | **Mandatory** | `python3 -c 'import gi; print(f"{gi.__version__}")'` |
| nvme-tcp (kernel module) | 5.18 * | **Mandatory** | **Mandatory** | N/A |
| dbus-daemon | 1.12.2 | **Mandatory** | **Mandatory** | `dbus-daemon --version` |
| avahi-daemon | 0.7 | **Mandatory** | Not required | `avahi-daemon --version` |
| python3-libnvme | 1.3 | **Mandatory** | **Mandatory** | `python3 -c 'import libnvme; print(f"{libnvme.__version__}")'` |
| importlib.resources.files() OR importlib_resources.files() | *** | Optional | Optional | `importlib.resources.files()` was introduced in Python 3.9 and backported to earlier versions as `importlib_resources.files()`. If neither modules can be found, `nvme-stas` will default to using the less efficient `pkg_resources.resource_string()` instead. When `nvme-stas` is no longer required to support Python 3.6 and is allowed a minimum of 3.9 or later, only `importlib.resources.files()` will be required. |
* Kernel 5.18 provides full functionality. nvme-stas can work with older kernels, but with limited functionality, unless the kernels contain back-ported features (see Addendum for the list of kernel patches that could be back-ported to an older kernel).
## Things to do post installation
### D-Bus configuration
We install D-Bus configuration files under `/usr/share/dbus-1/system.d`. One needs to run **`systemctl reload dbus-broker.service`** (Fedora) OR **`systemctl reload dbus.service`** (SuSE, Debian) for the new configuration to take effect.
### Configuration shared with `libnvme` and `nvme-cli`
`stafd` and `stacd` use the `libnvme` library to interact with the Linux kernel. And `libnvme` as well as `nvme-cli` rely on two configuration files, `/etc/nvme/hostnqn` and `/etc/nvme/hostid`, to retrieve the Host NQN and ID respectively. These files should be created post installation with the help of the `stadadm` utility. Here's an example for Debian-based systems:
```
if [ "$1" = "configure" ]; then
if [ ! -d "/etc/nvme" ]
mkdir /etc/nvme
fi
if [ ! -s "/etc/nvme/hostnqn" ]; then
stasadm hostnqn -f /etc/nvme/hostnqn
fi
if [ ! -s "/etc/nvme/hostid" ]; then
stasadm hostid -f /etc/nvme/hostid
fi
fi
```
The utility program `stasadm` gets installed with `nvme-stas`. `stasadm` also manages the creation (and updating) of `/etc/stas/sys.conf`, the `nvme-stas` system configuration file.
### Configuration specific to nvme-stas
The [README](./README.md) file defines the following three configuration files:
- `/etc/stas/sys.conf`
- `/etc/stas/stafd.conf`
- `/etc/stas/stacd.conf`
Care should be taken during upgrades to preserve customer configuration and not simply overwrite it. The process to migrate the configuration data and the list of parameters to migrate is still to be defined.
### Enabling and starting the daemons
Lastly, the two daemons, `stafd` and `stacd`, should be enabled (e.g. `systemctl enable stafd.service stacd.service`) and started (e.g. `systemctl start stafd.service stacd.service`).
# Compatibility between nvme-stas and nvme-cli
Udev rules are installed along with `nvme-cli` (e.g. `/usr/lib/udev/rules.d/70-nvmf-autoconnect.rules`). These udev rules allow `nvme-cli` to perform tasks similar to those performed by `nvme-stas`. However, the udev rules in `nvme-cli` version 2.1.2 and prior drop the `host-iface` parameter when making TCP connections to I/O controllers. `nvme-stas`, on the other hand, always makes sure that TCP connections to I/O controllers are made over the right interface using the `host-iface` parameter.
We essentially have a race condition because `nvme-stas` and `nvme-cli` react to the same kernel events. Both try to perform the same task in parallel, which is to connect to I/O controllers. Because `nvme-stas` is written in Python and the udevd daemon (i.e. the process running the udev rules) in C, `nvme-stas` usually loses the race and TCP connections are made by the udev rules without specifying the `host-iface`.
To remedy to this problem, `nvme-stas` disables `nvme-cli` udev rules and assumes the tasks performed by the udev rules. This way, only one process will take action on kernel events eliminating any race conditions. This also ensure that the right `host-iface` is used when making TCP connections.
# Addendum
## Kernel patches for nvme-stas 1.x
Here's the list of kernel patches (added in kernels 5.14 to 5.18) that will enable all features of nvme-stas.
```
commit e3448b134426741902b6e2c07cbaf5f66cfd2ebc
Author: Martin Belanger <martin.belanger@dell.com>
Date: Tue Feb 8 14:18:02 2022 -0500
nvme: Expose cntrltype and dctype through sysfs
TP8010 introduces the Discovery Controller Type attribute (dctype).
The dctype is returned in the response to the Identify command. This
patch exposes the dctype through the sysfs. Since the dctype depends on
the Controller Type (cntrltype), another attribute of the Identify
response, the patch also exposes the cntrltype as well. The dctype will
only be displayed for discovery controllers.
A note about the naming of this attribute:
Although TP8010 calls this attribute the Discovery Controller Type,
note that the dctype is now part of the response to the Identify
command for all controller types. I/O, Discovery, and Admin controllers
all share the same Identify response PDU structure. Non-discovery
controllers as well as pre-TP8010 discovery controllers will continue
to set this field to 0 (which has always been the default for reserved
bytes). Per TP8010, the value 0 now means "Discovery controller type is
not reported" instead of "Reserved". One could argue that this
definition is correct even for non-discovery controllers, and by
extension, exposing it in the sysfs for non-discovery controllers is
appropriate.
Signed-off-by: Martin Belanger <martin.belanger@dell.com>
commit 68c483a105ce7107f1cf8e1ed6c2c2abb5baa551
Author: Martin Belanger <martin.belanger@dell.com>
Date: Thu Feb 3 16:04:29 2022 -0500
nvme: send uevent on connection up
When connectivity with a controller is lost, the driver will keep
trying to reconnect once every 10 sec. When connection is restored,
user-space apps need to be informed so that they can take proper
action. For example, TP8010 introduces the DIM PDU, which is used to
register with a discovery controller (DC). The DIM PDU is sent from
user-space. The DIM PDU must be sent every time a connection is
established with a DC. Therefore, the kernel must tell user-space apps
when connection is restored so that registration can happen.
The uevent sent is a "change" uevent with environmental data
set to: "NVME_EVENT=connected".
Signed-off-by: Martin Belanger <martin.belanger@dell.com>
Reviewed-by: Hannes Reinecke <hare@suse.de>
Reviewed-by: Sagi Grimberg <sagi@grimberg.me>
Reviewed-by: Chaitanya Kulkarni <kch@nvidia.com>
commit f18ee3d988157ebcadc9b7e5fd34811938f50223
Author: Hannes Reinecke <hare@suse.de>
Date: Tue Dec 7 14:55:49 2021 +0100
nvme-fabrics: print out valid arguments when reading from /dev/nvme-fabrics
Currently applications have a hard time figuring out which
nvme-over-fabrics arguments are supported for any given kernel;
the ioctl will return an error code on failure, and the application
has to guess whether this was due to an invalid argument or due
to a connection or controller error.
With this patch applications can read a list of supported
arguments by simply reading from /dev/nvme-fabrics, allowing
them to validate the connection string.
Signed-off-by: Hannes Reinecke <hare@suse.de>
Reviewed-by: Chaitanya Kulkarni <kch@nvidia.com>
Signed-off-by: Christoph Hellwig <hch@lst.de>
commit e5ea42faa773c6a6bb5d9e9f5c2cc808940b5a55
Author: Hannes Reinecke <hare@suse.de>
Date: Wed Sep 22 08:35:25 2021 +0200
nvme: display correct subsystem NQN
With discovery controllers supporting unique subsystem NQNs the
actual subsystem NQN might be different from that one passed in
via the connect args. So add a helper to display the resulting
subsystem NQN.
Signed-off-by: Hannes Reinecke <hare@suse.de>
Reviewed-by: Chaitanya Kulkarni <kch@nvidia.com>
Signed-off-by: Christoph Hellwig <hch@lst.de>
commit 20e8b689c9088027b7495ffd6f80812c11ecc872
Author: Hannes Reinecke <hare@suse.de>
Date: Wed Sep 22 08:35:24 2021 +0200
nvme: Add connect option 'discovery'
Add a connect option 'discovery' to specify that the connection
should be made to a discovery controller, not a normal I/O controller.
With discovery controllers supporting unique subsystem NQNs we
cannot easily distinguish by the subsystem NQN if this should be
a discovery connection, but we need this information to blank out
options not supported by discovery controllers.
Signed-off-by: Hannes Reinecke <hare@suse.de>
Reviewed-by: Chaitanya Kulkarni <kch@nvidia.com>
Signed-off-by: Christoph Hellwig <hch@lst.de>
commit 954ae16681f6bdf684f016ca626329302a38e177
Author: Hannes Reinecke <hare@suse.de>
Date: Wed Sep 22 08:35:23 2021 +0200
nvme: expose subsystem type in sysfs attribute 'subsystype'
With unique discovery controller NQNs we cannot distinguish the
subsystem type by the NQN alone, but need to check the subsystem
type, too.
So expose the subsystem type in a new sysfs attribute 'subsystype'.
Signed-off-by: Hannes Reinecke <hare@suse.de>
Reviewed-by: Chaitanya Kulkarni <kch@nvidia.com>
Signed-off-by: Christoph Hellwig <hch@lst.de>
commit 3ede8f72a9a2825efca23a3552e80a1202ea88fd
Author: Martin Belanger <martin.belanger@dell.com>
Date: Thu May 20 15:09:34 2021 -0400
nvme-tcp: allow selecting the network interface for connections
In our application, we need a way to force TCP connections to go out a
specific IP interface instead of letting Linux select the interface
based on the routing tables.
Add the 'host-iface' option to allow specifying the interface to use.
When the option host-iface is specified, the driver uses the specified
interface to set the option SO_BINDTODEVICE on the TCP socket before
connecting.
This new option is needed in addtion to the existing host-traddr for
the following reasons:
Specifying an IP interface by its associated IP address is less
intuitive than specifying the actual interface name and, in some cases,
simply doesn't work. That's because the association between interfaces
and IP addresses is not predictable. IP addresses can be changed or can
change by themselves over time (e.g. DHCP). Interface names are
predictable [1] and will persist over time. Consider the following
configuration.
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state ...
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 100.0.0.100/24 scope global lo
valid_lft forever preferred_lft forever
2: enp0s3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc ...
link/ether 08:00:27:21:65:ec brd ff:ff:ff:ff:ff:ff
inet 100.0.0.100/24 scope global enp0s3
valid_lft forever preferred_lft forever
3: enp0s8: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc ...
link/ether 08:00:27:4f:95:5c brd ff:ff:ff:ff:ff:ff
inet 100.0.0.100/24 scope global enp0s8
valid_lft forever preferred_lft forever
The above is a VM that I configured with the same IP address
(100.0.0.100) on all interfaces. Doing a reverse lookup to identify the
unique interface associated with 100.0.0.100 does not work here. And
this is why the option host_iface is required. I understand that the
above config does not represent a standard host system, but I'm using
this to prove a point: "We can never know how users will configure
their systems". By te way, The above configuration is perfectly fine
by Linux.
The current TCP implementation for host_traddr performs a
bind()-before-connect(). This is a common construct to set the source
IP address on a TCP socket before connecting. This has no effect on how
Linux selects the interface for the connection. That's because Linux
uses the Weak End System model as described in RFC1122 [2]. On the other
hand, setting the Source IP Address has benefits and should be supported
by linux-nvme. In fact, setting the Source IP Address is a mandatory
FedGov requirement (e.g. connection to a RADIUS/TACACS+ server).
Consider the following configuration.
$ ip addr list dev enp0s8
3: enp0s8: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc ...
link/ether 08:00:27:4f:95:5c brd ff:ff:ff:ff:ff:ff
inet 192.168.56.101/24 brd 192.168.56.255 scope global enp0s8
valid_lft 426sec preferred_lft 426sec
inet 192.168.56.102/24 scope global secondary enp0s8
valid_lft forever preferred_lft forever
inet 192.168.56.103/24 scope global secondary enp0s8
valid_lft forever preferred_lft forever
inet 192.168.56.104/24 scope global secondary enp0s8
valid_lft forever preferred_lft forever
Here we can see that several addresses are associated with interface
enp0s8. By default, Linux always selects the default IP address,
192.168.56.101, as the source address when connecting over interface
enp0s8. Some users, however, want the ability to specify a different
source address (e.g., 192.168.56.102, 192.168.56.103, ...). The option
host_traddr can be used as-is to perform this function.
In conclusion, I believe that we need 2 options for TCP connections.
One that can be used to specify an interface (host-iface). And one that
can be used to set the source address (host-traddr). Users should be
allowed to use one or the other, or both, or none. Of course, the
documentation for host_traddr will need some clarification. It should
state that when used for TCP connection, this option only sets the
source address. And the documentation for host_iface should say that
this option is only available for TCP connections.
References:
[1] https://www.freedesktop.org/wiki/Software/systemd/PredictableNetworkInterfaceNames/
[2] https://tools.ietf.org/html/rfc1122
Tested both IPv4 and IPv6 connections.
Signed-off-by: Martin Belanger <martin.belanger@dell.com>
Reviewed-by: Sagi Grimberg <sagi@grimberg.me>
Reviewed-by: Hannes Reinecke <hare@suse.de>
Signed-off-by: Christoph Hellwig <hch@lst.de>
```
## Kernel patches for nvme-stas 2.x
These patches are not essential for nvme-stas 2.x, but they allow nvme-stas to operate better.
```
commit f46ef9e87c9e8941b7acee45611c7c6a322592bb
Author: Sagi Grimberg <sagi@grimberg.me>
Date: Thu Sep 22 11:15:37 2022 +0300
nvme: send a rediscover uevent when a persistent discovery controller reconnects
When a discovery controller is disconnected, no AENs will arrive to
notify the host about discovery log change events.
In order to solve this, send a uevent notification when a
persistent discovery controller reconnects. We add a new ctrl
flag NVME_CTRL_STARTED_ONCE that will be set on the first
start, and consecutive calls will find it set, and send the
event to userspace if the controller is a discovery controller.
Upon the event reception, userspace will re-read the discovery
log page and will act upon changes as it sees fit.
Signed-off-by: Sagi Grimberg <sagi@grimberg.me>
Reviewed-by: Daniel Wagner <dwagner@suse.de>
Reviewed-by: James Smart <jsmart2021@gmail.com>
Signed-off-by: Christoph Hellwig <hch@lst.de>
commit 02c57a82c0081141abc19150beab48ef47f97f18 (tag: nvme-6.1-2022-09-20)
Author: Martin Belanger <martin.belanger@dell.com>
Date: Wed Sep 7 08:27:37 2022 -0400
nvme-tcp: print actual source IP address through sysfs "address" attr
TCP transport relies on the routing table to determine which source
address and interface to use when making a connection. Currently, there
is no way to tell from userspace where a connection was made. This
patch exposes the actual source address using a new field named
"src_addr=" in the "address" attribute.
This is needed to diagnose and identify connectivity issues. With the
source address we can infer the interface associated with each
connection.
This was tested with nvme-cli 2.0 to verify it does not have any
adverse effect. The new "src_addr=" field will simply be displayed in
the output of the "list-subsys" or "list -v" commands as shown here.
$ nvme list-subsys
nvme-subsys0 - NQN=nqn.2014-08.org.nvmexpress.discovery
\
+- nvme0 tcp traddr=192.168.56.1,trsvcid=8009,src_addr=192.168.56.101 live
Signed-off-by: Martin Belanger <martin.belanger@dell.com>
Reviewed-by: Sagi Grimberg <sagi@grimberg.me>
Reviewed-by: Chaitanya Kulkarni <kch@nvidia.com>
Signed-off-by: Christoph Hellwig <hch@lst.de>
commit 4cde03d82e2d0056d20fd5af6a264c7f5e6a3e76
Author: Daniel Wagner <dwagner@suse.de>
Date: Fri Jul 29 16:26:30 2022 +0200
nvme: consider also host_iface when checking ip options
It's perfectly fine to use the same traddr and trsvcid more than once
as long we use different host interface. This is used in setups where
the host has more than one interface but the target exposes only one
traddr/trsvcid combination.
Use the same acceptance rules for host_iface as we have for
host_traddr.
Signed-off-by: Daniel Wagner <dwagner@suse.de>
Reviewed-by: Chao Leng <lengchao@huawei.com>
Signed-off-by: Christoph Hellwig <hch@lst.de>
```

13
Dockerfile Normal file
View file

@ -0,0 +1,13 @@
FROM fedora:37
WORKDIR /root
# first line for nvme-stas
# second line for libnvme
RUN dnf install -y python3-dasbus python3-pyudev python3-systemd python3-gobject meson \
git gcc g++ cmake openssl-devel libuuid-devel json-c-devel swig python-devel meson && dnf clean all
COPY . .
RUN meson .build && ninja -C .build && meson install -C .build
ENTRYPOINT ["python3"]

201
LICENSE Normal file
View file

@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2021 STFS
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

111
Makefile Normal file
View file

@ -0,0 +1,111 @@
# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See the LICENSE file for details.
#
# This file is part of NVMe STorage Appliance Services (nvme-stas).
#
# Authors: Martin Belanger <Martin.Belanger@dell.com>
#
.DEFAULT_GOAL := stas
BUILD-DIR := .build
DEB-PKG-DIR := ${BUILD-DIR}/deb-pkg
RPM-BUILDROOT-DIR := ${BUILD-DIR}/rpmbuild
ifneq (,$(strip $(filter $(MAKECMDGOALS),rpm deb dist)))
XTRA-MESON-OPTS := --wrap-mode=nodownload
endif
.PHONY: update-subprojects
update-subprojects:
meson subprojects update
${BUILD-DIR}:
BUILD_DIR=${BUILD-DIR} ./configure ${XTRA-MESON-OPTS}
@echo "Configuration located in: $@"
@echo "-------------------------------------------------------"
.PHONY: stas
stas: ${BUILD-DIR}
ninja -C ${BUILD-DIR}
.PHONY: clean
clean:
ifneq ("$(wildcard ${BUILD-DIR})","")
ninja -C ${BUILD-DIR} -t clean
endif
.PHONY: purge
purge:
ifneq ("$(wildcard ${BUILD-DIR})","")
rm -rf ${BUILD-DIR}
endif
.PHONY: install
install: stas
sudo meson $@ -C ${BUILD-DIR} --skip-subprojects
.PHONY: uninstall
uninstall: ${BUILD-DIR}
sudo ninja -C ${BUILD-DIR} uninstall
.PHONY: dist
dist: stas
meson $@ -C ${BUILD-DIR} --formats gztar
.PHONY: test
test: stas
meson $@ -C ${BUILD-DIR} --suite nvme-stas
.PHONY: loc
loc:
@cloc --by-file --exclude-dir=${BUILD-DIR},doc,subprojects,test,utils,debian,obj-x86_64-linux-gnu,.github --exclude-lang=Markdown,"NAnt script",XML,"Bourne Again Shell",make,"Bourne Shell",Meson,YAML,XSLT .
.PHONY: loc-full
loc-full:
@cloc --by-file --exclude-dir=${BUILD-DIR},subprojects,debian,obj-x86_64-linux-gnu,.github .
.PHONY: black
black:
black --diff --color --line-length 120 --skip-string-normalization --extend-exclude="(subprojects|debian|.build)" .
# Coverage requirements:
# pip install coverage
.PHONY: coverage
coverage: stas
cd ${BUILD-DIR} && ./coverage.sh
################################################################################
# Debian (*.deb)
# Use "DEB_BUILD_OPTIONS=nocheck make deb" to skip unit testing.
# This requires: sudo apt install -y debhelper dh-python
ifeq (deb,$(strip $(MAKECMDGOALS)))
ifneq (SUCCESS,$(shell dpkg -s debhelper dh-python > /dev/null 2>&1 && echo "SUCCESS" || echo "FAIL"))
$(error Missing packages. Run -> "sudo apt install -y debhelper dh-python")
endif
endif
.PHONY: deb
deb: ${BUILD-DIR}
mkdir -p ${DEB-PKG-DIR}
dpkg-buildpackage -us -uc
@mv ../nvme-stas_*.deb ${DEB-PKG-DIR}
@mv ../nvme-stas_*.buildinfo ${DEB-PKG-DIR}
@mv ../nvme-stas_*.changes ${DEB-PKG-DIR}
@mv ../nvme-stas_*.dsc ${DEB-PKG-DIR}
@mv ../nvme-stas_*.tar.gz ${DEB-PKG-DIR}
@echo "======================================================="
@echo "Debian packages located in: ${DEB-PKG-DIR}/"
################################################################################
# RedHat (*.rpm)
${BUILD-DIR}/nvme-stas.spec: ${BUILD-DIR} nvme-stas.spec.in
meson --wrap-mode=nodownload --reconfigure ${BUILD-DIR}
${RPM-BUILDROOT-DIR}: ${BUILD-DIR}/nvme-stas.spec
rpmbuild -ba $< --build-in-place --clean --nocheck --define "_topdir $(abspath ${BUILD-DIR}/rpmbuild)"
@echo "======================================================="
@echo "RPM packages located in: ${RPM-BUILDROOT-DIR}/"
.PHONY: rpm
rpm: ${RPM-BUILDROOT-DIR}

143
NEWS.md Normal file
View file

@ -0,0 +1,143 @@
# STorage Appliance Services (STAS)
## Changes with release 2.2.1
Added a few more unit and coverage tests. Fixed the following bugs.
Bug fixes:
* Fix errors with some debug commands (e.g. `stafctl ls --detailed`)
* Fix setting controller DHCHAP key (this requires [corresponding changes in libnvme](https://github.com/linux-nvme/libnvme/pull/597))
## Changes with release 2.2
Support for in-band authentication.
## Changes with release 2.1.3
This release is all about `udev rules`. As explained in [DISTROS.md](./DISTROS.md), `nvme-stas` and `nvme-cli` compete for the same kernel events (a.k.a. uevents or udev events). Those are events generated by the kernel related to Discovery Controller (DC) state changes. For example, an AEN indicating a change of Discovery Log Page (DLP), or an event indicating that the the connection to a DC was restored (event = `connected` or `rediscover`), which means that the DLP needs to be refreshed and connections to controllers listed in the DLP need to be updated.
When both `nvme-stas` and `nvme-cli` are allowed to react and process these events, we have a race condition where both processes try to perform the same connections at the same time. Since the kernel will not allow duplicate connections, then one process will get an error. This is not a real problem since the connection does succeed, but the kernel will log an error and this can be irritating to users.
We tried different ways to fix this issue. The simplest was to disable the `udev rules` installed by `nvme-cli`. This prevents `nvme-cli` from reacting to udev events and only `nvme-stas` gets to process the events. The downside to this is that `nvme-stas` only expects udev events from DCs that it manages. If a DC connection is made outside of `nvme-stas` (e.g. using `nvme-cli`) and `nvme-stas` receives an event for that DC, it won't know what to do with it and will simply ignore it.
To solve this issue, and to eliminate the race condition, this release of `nvme-stas` includes changes that allows `nvme-stas` to react and process events even for DCs that are not managed by `nvme-stas`. In that case, `nvme-stas` invokes `nvme-cli's` standard event handler. While `nvme-stas` is running, `nvme-cli's` `udev rules` will be disabled and all event handling will be performed by `nvme-stas`. `nvme-cli's` `udev rules` are restored when `nvme-stas` is stopped.
With this change we no longer need to provide the configuration parameter `udev-rule=[enabled|disabled]` in `stacd.conf`. This parameter is therefore deprecated.
This release also adds the "[black](https://github.com/psf/black)" code formatter to the GitHub actions. From now on, code submitted as a pull request with GitHub must comply to black's code format. A new command, `make black`, has been added to allow users to verify their code before submitting a pull request.
## Changes with release 2.1.2
* Bug fixes:
* Add support for RoCE and iWARP protocols in mDNS TXT field (i.e. `p=roce`, `p=iwarp`)
* Add `_nvme-disc._udp` to the list of supported mDNS service types (stype)
## Changes with release 2.1.1
* Bug fixes:
* Fix handling of unresponsive zeroconf-discovered Discovery Controllers. Sometimes we could have a timeout during twice as long as normal.
* Set default value of legacy "[Global] persistent-connections=false"
* Add `ControllerTerminator` entity to deal with potential (rare) cases where Connect/Disconnect operations could be performed in reverse order.
* Add more unit tests
* Increase code coverage
* Improve name resolution algorithm
* Set udev event priority to high (for faster handling)
## Changes with release 2.1
* Bug fixes:
* Immediately remove existing connection to Discovery Controllers (DC) discovered through zeroconf (mDNS) when added to `exclude=` in `stafd.conf`. Previously, adding DCs to `exclude=` would only take effect on new connections and would not apply to existing connections.
* When handling "key=value" pairs in the TXT field from Avahi, "keys" need to be case insensitive.
* Strip spaces from Discovery Log Page Entries (DLPE). Some DCs may append extra spaces to DLPEs (e.g. IP addresses with trailing spaces). The kernel driver does not expect extra spaces and therefore they need to be removed.
* In `stafd.conf` and `stacd.conf`, added new configuration parameters to provide parity with `nvme-cli`:
* `nr-io-queues`, `nr-write-queues`, `nr-poll-queues`, `queue-size`, `reconnect-delay`, `ctrl-loss-tmo`, `duplicate-connect`, `disable-sqflow`
* Changes to `stafd.conf`:
* Move `persistent-connections` from the `[Global]` section to a new section named `[Discovery controller connection management]`. `persistent-connections` will still be recognized from the `[Global]` section, but will be deprecated over time.
* Add new configuration parameter `zeroconf-connections-persistence` to section `[Discovery controller connection management]`. This parameter allows one to age Discovery Controllers discovered through zeroconf (mDNS) when they are no longer reachable and should be purged from the configuration.
* Added more configuration validation to identify invalid Sections and Options in configuration files (`stafd.conf` and `stacd.conf`).
* Improve dependencies in meson build environment so that missing subprojects won't prevent distros from packaging the `nvme-stas` (i.e. needed when invoking meson with the `--wrap-mode=nodownload` option)
* Improve Read-The-Docs documentation format.
## Changes with release 2.0
Because of incompatibilities between 1.1.6 and 1.2 (ref. `sticky-connections`), it was decided to skip release 1.2 and have a 2.0 release instead. Release 2.0 contains everything listed in 1.2 (below) plus the following:
* Add support for PLEO - Port-Local Entries Only, see TP8010.
* Add new configuration parameter to stafd.conf: `pleo=[enabled|disabled]`
* This requires `libnvme` 1.2 or later although nvme-stas can still operate with 1.1 (but PLEO will not be supported).
* Although `blacklist=` is deprecated, keep supporting it for a while.
* Target `udev-rule=` at TCP connections only.
* Read-the-docs will now build directly from source (instead of using a possibly stale copy)
* More unit tests were added
* Refactored the code that handles pyudev events in an effort to fix spurious lost events.
## ~~Changes with release 1.2~~ (never released - use 2.0 instead)
- In `stacd.conf`, add a new configuration section, `[I/O controller connection management]`.
- This is to replace `sticky-connections` by `disconnect-scope` and `disconnect-trtypes`, which is needed so that hosts can better react to Fabric Zoning changes at the CDC.
- Add `connect-attempts-on-ncc` to control how stacd will react to the NCC bit (Not Connected to CDC).
- When the host's symbolic name is changed in `sys.conf`, allow re-issuing the DIM command (register with DC) on a `reload` signal (`systemctl reload stafd`).
- Replace `blacklist=` by `exclude=` is `stafd.conf` and `stacd.conf`. Warning: this may create an incompatibility for people that were using `blacklist=`. They will need to manually migrate their configuration files.
- Change `TID.__eq__()` and `TID.__ne__()` to recognize a TID object even when the `host-iface` is not set. This is to fix system audits where `nvme-stas` would not recognize connections made by `nvme-cli`. The TID object, or Transport ID, contains all the parameters needed to establish a connection with a controller, e.g. (`trtype`, `traddr`, `trsvcid`, `nqn`, `host-traddr`, and `host-iface`). `nvme-stas` can scan the `sysfs` (`/sys/class/nvme/`) to find exiting NVMe connections. It relies on the `address` and other attributes for that. For example the attribute `/sys/class/nvme/nvme0/address` may contain something like: `traddr=192.168.56.1,trsvcid=8009,host_iface=enp0s8`.
`nvme-stas` always specify the `host-iface` when making connections but `nvme-cli` typically does not. Instead, `nvme-cli` relies on the routing table to select the interface. This creates a discrepancy between the `address` attribute of connections made by `nvme-cli` and those made by `nvme-stas` (i.e. `host_iface=` is missing for `nvme-cli` connections). And this results in `nvme-stas` not being able to recognize connections made by `nvme-cli`. Two solutions have been proposed to workaround this problem:
- First, a short term solution changes `TID.__eq__()` and `TID.__ne__()` so that the `host-iface` has a lesser weight when comparing two TIDs. This way, the TID of a connection created by `nvme-cli` can be compared to the TID of a connection made with `nvme-stas` and still result in a match. The downside to this approach is that a connection made with `nvme-cli` that is going over the wrong interface (e.g. bad routing table entry), will now be accepted by `nvme-stas` as a valid connection.
- Second, a long term solution that involves a change to the kernel NVMe driver will allow being able to determine the host interface for any NVMe connections, even those made without specifying the `host-iface` parameter. The kernel driver will now expose the source address of all NVMe connections through the `sysfs`. This will be identified by the key=value pair "`src-addr=[ip-address]`" in the `address` attribute. And from the source address one can infer the actual host interface. This actually will solve the shortcomings of the "short term" solution discussed above. Unfortunately, it may take several months before this kernel addition is available in a stock Distribution OS. So, the short term solution will need to suffice for now.
## Changes with release 1.1.6
- Fix issues with I/O controller connection audits
- Eliminate pcie devices from list of I/O controller connections to audit
- Add soaking timer to workaround race condition between kernel and user-space applications on "add" uevents. When the kernel adds a new nvme device (e.g. `/dev/nvme7`) and sends a "add" uevent to notify user-space applications, the attributes associated with that device (e.g. `/sys/class/nvme/nvme7/cntrltype`) may not be fully initialized which can lead `stacd` to dismiss a device that should get audited.
- Make `sticky-connections=enabled` the default (see `stacd.conf`)
## Changes with release 1.1.5
- Fix issues introduced in 1.1.3 when enabling Fibre Channel (FC) support.
- Eliminate pcie devices from discovery log pages. When enabling FC, pcie was accidentally enabled as well.
- Fix I/O controller scan and detect algorithm. Again, while adding support for FC, the I/O scan & detect algorithm was modified, but we accidentally made it detect Discovery Controllers as well as I/O controllers.
## ~~Changes with release 1.1.4~~ USE 1.1.5 INSTEAD.
- Fix issues for Fibre Channel (FC) support.
- Add TESTING.md
## Changes with release 1.1.3
**stacd**: Add I/O controller connection audits. Audits are enabled when the configuration parameter "`sticky-connections`" is disabled.
**stafd**: Preserve and Reload last known configuration on restarts. This is for warm restarts of the `stafd` daemon. This does not apply to system reboots (cold restarts). This is needed to avoid deleting I/O controller (IOC) connections by mistake when restarting `stafd`. It prevents momentarily losing previously acquired Discovery Log Page Entries (DLPE). Since `stacd` relies on acquired DLPEs to determine which connection should be created or deleted, it's important that the list of DLPEs survives a `stafd` restart. Eventually, after `stafd` has restarted and reconnected to all Discovery Controllers (DC), the list will get refreshed and the DLPE cache will get updated. And as the cache gets updated, `stacd` will be able to determine which connections should remain and which one should get deleted.
**`stafd`/`stacd`**: Fixed crash caused by `stafd`/`stacd` calling the wrong callback function during the normal disconnect of a controller. There are two callback functions that can be called after a controller is disconnected, but one of them must only be called on a final disconnect just before the process (`stafd` or `stacd`) exits. The wrong callback was being called on a normal disconnect, which led the process to think it was shutting down.
## ~~Changes with release 1.1.2~~ USE 1.1.3 INSTEAD.
stacd: Bug fix. Check that self._cfg_soak_tmr is not None before dereferencing it.
## Changes with release 1.1.1
Make `sticky-connections=disabled` the default (see `stacd.conf`)
## Changes with release 1.1
- Add `udev-rule` configuration parameter to `stacd.conf`.
- Add `sticky-connections` configuration parameter to `stacd.conf`.
- Add coverage testing (`make coverage`)
- Add `make uninstall`
- To `README.md`, add mDNS troubleshooting section.
## Changes with release 1.0.1
- Install staslib as pure python package instead of arch-specific.
## Changes with release 1.0
- First public release following TP8009 / TP8010 ratification and publication.
## Changes with release 0.1:
- Initial release

288
README.md Normal file
View file

@ -0,0 +1,288 @@
# STorage Appliance Services (STAS)
![Build](https://github.com/linux-nvme/nvme-stas/actions/workflows/meson-test.yml/badge.svg)
![GitHub](https://img.shields.io/github/license/linux-nvme/nvme-stas)
[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
[![Release](https://img.shields.io/github/v/release/linux-nvme/nvme-stas?include_prereleases&style=)](https://github.com/linux-nvme/nvme-stas/releases)
[![GitHub commits](https://img.shields.io/github/commits-since/linux-nvme/nvme-stas/latest.svg)](https://GitHub.com/linux-nvme/nvme-stas/commit/)
[![Read the Docs](https://img.shields.io/readthedocs/nvme-stas)](https://nvme-stas.readthedocs.io/en/latest/)
[![codecov](https://codecov.io/gh/linux-nvme/nvme-stas/branch/main/graph/badge.svg)](https://codecov.io/gh/linux-nvme/nvme-stas)
[![Minimum Python Version](https://img.shields.io/badge/python-3.6+-blue.svg)](https://www.python.org/downloads/)
What does nvme-stas provide?
- A Central Discovery Controller (CDC) client for Linux
- Asynchronous Event Notifications (AEN) handling
- Automated NVMe subsystem connection controls
- Error handling and reporting
- Automatic (zeroconf) and Manual configuration
## Overview
STAS is composed of two services, STAF and STAC, running on the Host computer.
**STAF** - **STorage Appliance Finder**. The tasks performed by STAF include:
- Register with the Avahi daemon for service type `_nvme-disc._tcp`. This allows STAF to locate Central or Direct Discovery Controllers (CDC, DDC) with zero-touch provisioning (ZTP). STAF also allows users to manually enter CDCs and DDCs in a configuration file (`/etc/stas/stafd.conf`) when users prefer not to use ZTP.
- Connect to discovered or configured CDCs or DDCs.
- Retrieve the list of storage subsystems using the "get log page" command.
- Maintain a cache of the discovered storage subsystems.
- Provide a D-Bus interface where 3rd party applications can retrieve the data about the Discovery Controller connections (e.g. log pages).
**STAC** - **STorage Appliance Connector**. The tasks performed by STAC include:
- Read the list of storage subsystems from STAF over D-Bus.
- Similar to STAF, STAC can also read a list of storage subsystems to connect to from a configuration file.
- Set up the I/O controller connections.
- Provide a D-Bus interface where 3rd party applications can retrieve data about the I/O controller connections.
![Definition](./doc/images/STAF-STAC-libnvme.png)
## Design
**`stafd`** and **`stacd`** use the [GLib main loop](https://docs.gtk.org/glib/main-loop.html). The [GLib](https://docs.gtk.org/glib/index.html) Python module provides several low-level building blocks that are needed by **`stafd`** and **`stacd`**. In addition, many Python modules "play nice" with GLib such as `dasbus` and `pyudev`. GLib also provides additional components such as timers, signal handlers, and much more.
**`stafd`** connects to the `avahi-daemon`, which it uses to detect Central Discovery Controllers (CDC) and Direct Discovery Controllers (DDC). When Discovery Controllers (DC) are found with Avahi's help, **`stafd`** uses `libnvme` to set up persistent connections and retrieve the discovery log pages.
## Daemonization
**`stafd`** and **`stacd`** are managed as `systemd` services. The following operations are supported (here showing only `stafd`, but the same operations apply to `stacd`):
- `systemctl start stafd`. Start daemon.
- `systemctl stop stafd`. Stop daemon. The `SIGTERM` signal is used to tell the daemon to stop.
- `systemctl restart stafd`. Effectively a `stop` + `start`.
- `systemctl reload stafd`. Reload configuration. This is done in real time without restarting the daemon. The `SIGHUP` signal is used to tell the daemon to reload its configuration file.
## Configuration
As stated before, **`stafd`** can automatically locate discovery controllers with the help of Avahi and connect to them, and **`stacd`** can automatically set up the I/O connections to discovered storage subsystems. However, **`stafd`** and **`stacd`** can also operate in a non-automatic mode based on manually entered configuration. In other words, discovery controllers and/or storage subsystems can be entered manually. This is to provide customers with more flexibility. The configuration for each daemon is found in **`/etc/stas/stafd.conf`** and **`/etc/stas/stacd.conf`** respectively. The configuration files also provide additional parameters, such as log-level attributes used mainly for debugging purposes.
The following configuration files are defined:
| File | Consumer | Purpose |
| ---------------------- | ----------------- | ------------------------------------------------------------ |
| `/etc/stas/sys.conf` | `stafd` + `stacd` | Contains system-wide (i.e. host) configuration such as the Host NQN, the Host ID, and the Host Symbolic Name. Changes to this file can be made manually or with the help of the `stasadm` utility as described in the previous section. <br /><br />For example, `stasadm hostnqn -f /etc/nvme/hostnqn` writes the Host NQN to the file `/etc/nvme/hostnqn`, but also adds an entry to `/etc/stas/sys.conf` to indicate where the Host NQN has been saved. <br /><br />This gives nvme-stas the flexibility of defining its own Host parameters or to use the same parameters defined by `libnvme` and `nvme-cli`. |
| `/etc/stas/stafd.conf` | `stafd` | Contains configuration specific to `stafd`. Discovery controllers can be manually added or excluded in this file. |
| `/etc/stas/stacd.conf` | `stacd` | Contains configuration specific to `stacd`. I/O controllers can be manually added or excluded in this file. |
## D-Bus interface
The interface to **`stafd`** and **`stacd`** is D-Bus. This allows other programs, such as **`stafctl`** and **`stacctl`**, to communicate with the daemons. This also provides third parties the ability to write their own applications that can interact with **`stafd`** and **`stacd`**. For example, someone could decide to write a GUI where they would display the discovery controllers as well as the all the discovery log pages in a "pretty" window. The next table provides info about the two D-Bus interfaces.
| Component | D-Bus address |
| --------- | ------------------------------ |
| `stafd` | **`org.nvmexpress.staf.conf`** |
| `stacd` | **`org.nvmexpress.stac.conf`** |
## Companion programs: `stafctl` and `stacctl`
**`stafctl`** and **`stacctl`** are utilities that allow users to interact with **`stafd`** and **`stacd`** respectively. This is a model used by several programs, such as `systemctl` with `systemd`.
At a minimum, these utilities provide debug tools, but they could also provide some configuration capabilities (TBD).
## Packages
**`stafd`** and **`stacd`** as well as their companion programs **`stafctl`** and **`stacctl`** are released together in a package called "**`nvme-stas`**" for **ST**orage **A**pplicance **S**ervices (e.g. `stas-1.0.0-1.x86_64.rpm` or `stas_1.0.0_amd64.deb`).
## Dependencies
**`stafd`**/**`stacd`** require Linux kernel 5.14 or later.
The following packages must be installed to use **`stafd`**/**`stacd`**
**Debian packages (tested on Ubuntu 20.04):**
```bash
sudo apt-get install -y python3-pyudev python3-systemd python3-gi
sudo apt-get install -y python3-dasbus # Ubuntu 22.04
OR:
sudo pip3 install dasbus # Ubuntu 20.04
```
**RPM packages (tested on Fedora 34..35 and SLES15):**
```bash
sudo dnf install -y python3-dasbus python3-pyudev python3-systemd python3-gobject
```
# STAF - STorage Appliance Finder
| Component | Description |
| --------------- | -------------------------------------------------------- |
| **`/usr/sbin/stafd`** | A daemon that finds (discovers) NVMe storage appliances. |
| **`/usr/bin/stafctl`** | A companion shell utility for `stafd`. |
| **`/etc/stas/stafd.conf`** | Configuration file |
## stafd configuration file
The configuration file is named `/etc/stas/stafd.conf`. This file contains configuration parameters for the **`stafd`** daemon. One of the things you may want to configure is the IP address of the discovery controller(s) you want **`stafd`** to connect to. The configuration file contains a description of all the parameters that can be configured.
## Service discovery with Avahi
**`stafd`** can automatically find and set up connections to Discovery Controllers. To do this, **`stafd`** registers with the [Avahi](https://www.avahi.org/), the mDNS/DNS-SD (Service Discovery) daemon. Discovery Controllers that advertise themselves with service type `_nvme-disc._tcp` will be recognized by Avahi, which will inform **`stafd`**.
### Not receiving mDNS packets?
If **`stafd`** is not detecting any discovery controllers through Avahi, it could simply be that the mDNS packets are being suppressed by your firewall. If you know for a fact that the discovery controllers are advertizing themselves with mDNS packets, make sure that the Avahi daemon is receiving them as follows:
```bash
avahi-browse -t -r _nvme-disc._tcp
```
If you're not seeing anything, then check whether your firewall allows mDNS packets.
### Why is Avahi failing to discover services on some interfaces?
Linux limits the number of multicast group memberships that a host can belong to. The default is 20. For Avahi to monitor mDNS (multicast DNS) packets on all interfaces, the host computer must be able to register one multicast group per interface. This can be physical or logical interfaces. For example, configuring 10 VLANs on a physical interface increases the total number of interfaces by 10. If the total number of interfaces is greater than the limit of 20, then Avahi won't be able to monitor all interfaces.
The limit can be changed by configuring the variable **`igmp_max_memberships`**. This variable is defined [here](https://sysctl-explorer.net/net/ipv4/igmp_max_memberships/) in the kernel documentation. And this [StackExchange page](https://unix.stackexchange.com/questions/23832/is-there-a-way-to-increase-the-20-multicast-group-limit-per-socket) describes how one can increase the limit.
# STAC - STorage Appliance Connector
| File name | Description |
| -------------------------- | -------------------------------------------------- |
| **`/usr/sbin/stacd`** | A daemon that connects to NVMe storage appliances. |
| **`/usr/bin/stacctl`** | A companion shell utility for `stacd`. |
| **`/etc/stas/stacd.conf`** | Configuration file |
## stacd configuration file
The configuration file is named `/etc/stas/stacd.conf`. In this file you can configure storage appliances that **`stacd`** will connect to. By default, **`stacd`** uses information (log pages) collected from **`stafd`** to connect to storage appliances. However, you can also manually enter IP addresses of storage appliances in this file.
# System configuration
A host must be provided with a Host NQN and a Host ID. `nvme-stas` will not run without these two mandatory configuration parameters. To follow in the footsteps of `nvme-cli` and `libnvme`, `nvme-stas` will use the same Host NQN and ID that `nvme-cli` and `libnvme` use by default. In other words, `nvme-stas` will read the Host NQN and ID from these two files by default:
1. `/etc/nvme/hostnqn`
2. `/etc/nvme/hostid`
Using the same configuration files will ensure consistency between `nvme-stas`, `nvme-cli`, and `libnvme`. On the other hand, `nvme-stas` can operate with a different Host NQN and/or ID. In that case, one can specify them in `/etc/stas/sys.conf`.
A new optional configuration parameters introduced in TP8010, the Host Symbolic Name, can also be specified in `/etc/stas/sys.conf`. The schema/documentation for `/etc/stas/sys.conf` can be found [`/etc/stas/sys.conf.doc`](./etc/stas/sys.conf.doc).
# Build, install, unit tests
STAS uses the `meson` build system. Since STAS is a Python project, there is no code to build. However, the code needs to be installed using `meson`. Unit tests can also be run with `meson`.
## Using meson
Invoke `meson` to configure the project:
```bash
meson .build
```
The command `meson .build` need only be called once. This analyzes the project and the host computer to determine if all the necessary tools and dependencies are available. The result is saved to the directory named `.build`.
To compile the code:
```bash
cd .build
ninja
```
To install / uninstall the code:
```bash
cd .build
meson install
ninja uninstall
```
To run the unit tests:
```bash
cd .build
meson test
```
For more information about testing, please refer to: [TESTING.md](./TESTING.md)
## Alternate approach using Good-ole make
Recognizing that many people are not familiar with `meson`, we're providing a second way to install the code using the more familiar `configure` script combined with a `make`.
```bash
./configure
make
```
This performs the same operations as the meson approach described above. The `configure` script is automatically invoked when running `make` by itself.
| make command | Description |
| -------------------- | :----------------------------------------------------------- |
| **`make`** | Invoke the `.configure` script and build the code. |
| **`make install`** | Install the code. Requires root privileges (you will be asked to enter your password). |
| **`make uninstall`** | Uninstall the code. Requires root privileges (you will be asked to enter your password). |
| **`make test`** | Run the unit tests |
| **`make clean`** | Clean build artifacts, but does not remove the meson's configuration. That is, the configuration in `.build` is preserved. |
| **`make purge`** | Remove all build artifacts including the `.build` directory. |
## Compiling and running nvme-stas in a docker container
Use published image (optional)
```bash
docker pull ghcr.io/linux-nvme/nvme-stas:main
```
Build your own image (optional)
```bash
docker-compose up --build
```
Run services using docker-compose like this
```bash
docker-compose up
```
Run companion programs **`stafctl`** and **`stacctl`** like this
```bash
docker-compose exec stafd stafctl ls
docker-compose exec stafd stafctl status
docker-compose exec stacd stacctl ls
docker-compose exec stacd stacctl status
```
dependencies: dbus, avahi.
## Generating man and html pages
nvme-stas uses the following programs to generate the documentation. These can be installed as shown in the "dependencies" section below.
- `xsltproc` - Used to convert DocBook XML notation to "man pages" and "html pages".
- `gdbus-codegen` - Used to convert D-Bus IDL to DocBook XML notation.
### Dependencies
The following packages must be installed to generate the documentation
**Debian packages (tested on Ubuntu 20.04):**
```bash
sudo apt-get install -y docbook-xml docbook-xsl xsltproc libglib2.0-dev
```
**RPM packages (tested on Fedora 34..35 and SLES15):**
```bash
sudo dnf install -y docbook-style-xsl libxslt glib2-devel
```
### Configuring and building the man and html pages
By default, the documentation is not built. You need to run the `configure` as follows to tell meson that you want to build the documentation. You may need to first purge any previous configuration.
```bash
make purge
./configure -Dman=true -Dhtml=true
make
```
## Generating RPM and/or DEB packages
```bash
make rpm
make deb
```

221
TESTING.md Normal file
View file

@ -0,0 +1,221 @@
---
author: Martin Belanger
title: Testing nvme-stas
---
# Overview
For quick an easy testing, it's possible to run a storage subsystem simulator using the `nvmet` driver. This is how most of the testing was done during `nvme-stas` development. The main feature that cannot be tested this way is mDNS discovery.
There are two ways to run the tests.
- The first one involves starting all the components manually and using the nvmet driver as the storage appliance.
- The second one is fully automated and can be invoked simply by running `make coverage`.
[toc]
# Manual testing using the nvmet driver
A script is provided (`utils/nvmet/nvmet.py`) to simplify the configuration of the `nvmet` driver. The script comes with a companion configuration file (`utils/nvmet/nvmet.conf`). The configuration file is where you configure the port(s) and subsystem(s) to create. The default configuration will create 3 subsystems under port 1. This is mapped to the local IPv6 loopback address (`::1`).
Since nvmet doesn't provide a mDNS responder, you will need to manually configure `stafd` (`/etc/stas/stafd.conf`) so that it connects to the DDC that the nvmet driver creates by adding the DDC's address under the `[Controllers]` section. For example:
```bash
[Controllers]
controller=transport=tcp;traddr=localhost
```
## Monitoring
While testing it's a good idea to follow the journal in real time to see how `stafd` and `stacd` are performing. In a terminal (e.g. `bash`) do:
```bash
$ sudo journalctl --system --full -o short-precise --follow
```
You probably don't really need all these options, but they will give you full view of the messages with a millisecond time resolution. I personally define an alias `alias j='sudo journalctl --system --full -o short-precise'` and then I need only invoke `j -f`. Or even better, I add my user to the `systemd-journal` group so that I don't have to use `sudo` to see system-level log messages (Ref: [systemd-journal.service](https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html#Access%20Control)).
## Test startup
Here's a step-by-step guide to start `stafd` and `stacd` and connect to the `nvmet` driver. Open a second terminal and enter the following commands (these commands assume that nvme-stas will be cloned under `~/work/nvme-stas` referred to as `$STAS_DIR`):
### Clone nvme-stas (if not done already)
```bash
$ mkdir ~/work
$ cd ~/work
$ git clone https://github.com/linux-nvme/nvme-stas.git
$ STAS_DIR=~/work/nvme-stas
```
### Build and install nvme-stas
```bash
$ cd $STAS_DIR
$ make install
```
### Create a post-install script
Create an executable shell script (call it `stas-config.sh`) with the following contents. These are post-installation configuration steps required every time `nvme-stas` is reinstalled. Place the script in a directory that is in the search `$PATH` so that it can be invoked easily.
```bash
#!/usr/bin/env bash
#####################################################################
# Must run daemon-reload after installing nvme-stas
sudo systemctl daemon-reload
#####################################################################
# Make sure Host NQN and ID are configured
if [ ! -d "/etc/nvme" ]; then
sudo mkdir /etc/nvme
fi
if [ ! -s /etc/nvme/hostnqn ]; then
sudo stasadm hostnqn -f /etc/nvme/hostnqn
fi
if [ ! -s /etc/nvme/hostid ]; then
sudo stasadm hostid -f /etc/nvme/hostid
fi
#####################################################################
# Edit /etc/stas/stafd.conf to enable tracing and add the local
# nvmet driver as the Discovery Controller to connect to.
FILES="stafd.conf stacd.conf"
for file in ${FILES}; do
sudo sed -i '/^#tron=false/a tron=true' /etc/stas/${file}
done
sudo sed -i '/^#controller=$/a controller=transport=tcp;traddr=localhost' /etc/stas/stafd.conf
```
### Run the post-install script
```bash
$ stas-config.sh
```
### Start the nvmet driver
```bash
$ cd $STAS_DIR/utils/nvmet
$ sudo ./nvmet.py create
```
### Start stafd and stacd
```bash
$ sudo systemctl start stafd stacd
```
## So, is it running yet?
You should have seen `stafd` and `stacd` starting in the first terminal where `journalctl` is following the system log. At this point `stafd` should have connected to the `nvmet` discovery controller and retrieved the discovery log page entries (DLPE). And `stacd` should have retrieved the DLPEs from `stafd` and connected to the 3 subsystems defined in `nvmet.conf`. This can be confirmed as follows:
```bash
$ stafctl ls
[{'device': 'nvme0',
'host-iface': '',
'host-traddr': '',
'subsysnqn': 'nqn.2014-08.org.nvmexpress.discovery',
'traddr': '::1',
'transport': 'tcp',
'trsvcid': '8009'}]
```
And:
```bash
$ stacctl ls
[{'device': 'nvme1',
'host-iface': '',
'host-traddr': '',
'subsysnqn': 'klingons',
'traddr': '::1',
'transport': 'tcp',
'trsvcid': '8009'},
{'device': 'nvme2',
'host-iface': '',
'host-traddr': '',
'subsysnqn': 'nqn.1988-11.com.dell:PowerSANxxx:01:20210225100113-454f73093ceb4847a7bdfc6e34ae8e28',
'traddr': '::1',
'transport': 'tcp',
'trsvcid': '8009'},
{'device': 'nvme3',
'host-iface': '',
'host-traddr': '',
'subsysnqn': 'starfleet',
'traddr': '::1',
'transport': 'tcp',
'trsvcid': '8009'}]
```
You can also use `nvme-cli` to list the connections. For example: `nvme list -v`.
## Generating Asynchronous Event Notifications (AEN)
You can use the `nvmet.py` script to simulate the removal of a subsystem, which results in an AEN being sent to indicate a "Change of Discovery Log Page". Here's how:
```bash
$ cd $STAS_DIR/utils/nvmet
$ sudo ./nvmet.py unlink -p 1 -s klingons
```
Observe what happens in the journal. `stafd` will receive the AEN and update the DLPEs by performing a Get Discovery Log Page command. And `stacd` will disconnect from the "`klingons`" subsystem (use `stacctl ls` to confirm).
Then, add the subsystem back as follows:
```bash
$ sudo ./nvmet.py link -p 1 -s klingons
```
**NOTE**: I know, "`klingons`" is not a valid NQN, but it sure is easier to remember and to type than a valid NQN. Fortunately, the `nvmet` driver doesn't care what the actual subsystem's NQN looks like. :smile:
## Stopping nvmet
```bash
$ cd $STAS_DIR/utils/nvmet
$ sudo ./nvmet.py clean
```
# Automated testing using the coverage test
This requires the [Python coverage package](https://coverage.readthedocs.io/en/6.4.1/), which can be installed as follows:
```bash
$ sudo pip install coverage
```
Note that this test cannot be run while `stafd` and `stacd` are running. Make sure to stop `stafd` and `stacd` if they are running (`systemctl stop [stafd|stacd]`). You may also need to mask those services (`systemctl mask [stafd|stacd]`) if coverage fails to start.
To run the coverage test, from the root of the `nvme-stas` git repo:
```bash
$ make coverage
```
This will start `stafd`, `stacd`, and the `nvmet` target. At the end, if all goes well, you should get an output similar to this:
```bash
Name Stmts Miss Cover
----------------------------------------
stacctl 53 0 100%
stacd 190 3 98%
stafctl 75 0 100%
stafd 246 21 91%
staslib/avahi.py 185 19 90%
staslib/defs.py 22 0 100%
staslib/stas.py 858 51 94%
staslib/version.py 31 0 100%
----------------------------------------
TOTAL 1660 94 94%
```
Note that the Python coverage package has trouble tracking code executed in threads. And since nvme-stas uses threads, some of the code will not be accounted for (in other words, you'll never get 100% coverage).
Also note, that some of the code (e.g. explicit registration per TP8010) only gets executed when connected to a CDC (not a DDC). So, depending on your environment you will most likely get different coverage result. The above test was done on a system where mDNS discovery with a CDC was available, which provides more coverage than using the `nvmet` driver alone.
An HTML output is also available where you can click on each file and which lines of code got executed and which ones were missed. In your web browser, simply type `file:///[$STAS_DIR]/.build/coverage/index.html` (you must replace `[$STAS_DIR]` by the actual location of the nvme-stas repo where `make coverage` was run) . You should get something like this:
![](./doc/images/Coverage.png)

17
configure vendored Executable file
View file

@ -0,0 +1,17 @@
#!/bin/bash -e
# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See the LICENSE file for details.
#
# This file is part of NVMe STorage Appliance Services (nvme-stas).
#
# Authors: Martin Belanger <Martin.Belanger@dell.com>
#
BUILD_DIR="${BUILD_DIR:-.build}"
if [ ! -d ${BUILD_DIR} ]; then
exec meson ${BUILD_DIR} "$@"
else
exec meson configure ${BUILD_DIR} "$@"
fi

653
coverage.sh.in Executable file
View file

@ -0,0 +1,653 @@
#!/usr/bin/env bash
PRIMARY_GRP=$( id -ng )
PRIMARY_USR=$( id -nu )
PYTHON_PATH=.:./subprojects/libnvme
AVAHI_PUBLISHER=mdns_publisher.service
file=/tmp/stafd.conf.XXXXXX
stafd_conf_fname=$(mktemp $file)
file=/tmp/stacd.conf.XXXXXX
stacd_conf_fname=$(mktemp $file)
CYAN="[1;36m"
RED="[1;31m"
YELLOW="[1;33m"
NORMAL="[0m"
log() {
msg="$1"
printf "%b%s%s%b[0m\n" "\0033" ${CYAN} "${msg}" "\0033"
sudo logger -t COVERAGE -i "@@@@@ " -p warning -- "${msg}"
}
log_file_contents() {
rc=$1
file=$2
if [ $rc -eq 0 ]; then
color=${NORMAL}
level="info"
else
color=${YELLOW}
level="error"
fi
while IFS= read -r line; do
msg=" ${line}"
printf "%b%s%s%b[0m\n" "\0033" ${color} "${msg}" "\0033"
sudo logger -t COVERAGE -i "@@@@@ " -p ${level} -- "${msg}"
done < ${file}
}
systemctl-exists() {
unit="$1"
[ $(systemctl list-unit-files "${unit}" | wc -l) -gt 3 ]
}
sd_stop() {
app="$1"
unit="${app}"-cov.service
if systemctl-exists "${unit}" >/dev/null 2>&1; then
log "Stop ${app}"
sudo systemctl stop "${unit}" >/tmp/output.txt 2>&1
if [ -s /tmp/output.txt ]; then
log_file_contents $? /tmp/output.txt
else
printf " sudo systemctl stop %s\n" "${unit}"
fi
sudo systemctl reset-failed "${unit}" >/dev/null 2>&1
printf "\n"
sleep 1
fi
}
sd_start() {
app="$1"
dbus="$2"
conf="$3"
unit="${app}"-cov.service
if [ -z "${conf}" ]; then
cmd="${app} --syslog"
else
cmd="${app} --syslog -f ${conf}"
fi
RUNTIME_DIRECTORY=/tmp/${app}
rm -rf ${RUNTIME_DIRECTORY}
mkdir ${RUNTIME_DIRECTORY}
# Clear previous failure status (if any)
sudo systemctl reset-failed "${unit}" >/dev/null 2>&1
log "Start ${app}"
sudo systemd-run --unit="${unit}" --working-directory=. --property=Type=dbus --property=BusName="${dbus}" --property="SyslogIdentifier=${app}" --property="ExecReload=/bin/kill -HUP \$MAINPID" --setenv=PYTHONPATH=${PYTHON_PATH} --setenv=RUNTIME_DIRECTORY=${RUNTIME_DIRECTORY} coverage run --rcfile=.coveragerc ${cmd} >/tmp/output.txt 2>&1
log_file_contents $? /tmp/output.txt
printf "\n"
sleep 1
}
sd_restart() {
app="$1"
unit="${app}"-cov.service
if systemctl is-active "${unit}" >/dev/null 2>&1; then
log "Restart ${app}"
sudo systemctl restart "${unit}" && printf "systemctl restart %s\n" "${unit}" >/tmp/output.txt 2>&1
log_file_contents $? /tmp/output.txt
sleep 1
else
msg="Cannot restart ${app}, which is not currently running."
printf "%b%s%s%b[0m\n\n" "\0033" ${RED} "${msg}" "\0033"
fi
printf "\n"
}
reload_cfg() {
app="$1"
unit="${app}"-cov.service
log "Reload config ${app}"
sudo systemctl reload "${unit}" && printf "systemctl reload %s\n" "${unit}" >/tmp/output.txt 2>&1
#pid=$( systemctl show --property MainPID --value "${unit}" )
#sudo kill -HUP "${pid}" >/tmp/output.txt 2>&1
log_file_contents $? /tmp/output.txt
printf "\n"
sleep 1
}
run_unit_test() {
input=$@
if [ "$1" == "sudo" ]; then
shift
COVERAGE="sudo coverage"
else
COVERAGE="coverage"
fi
test=$@
log "Run unit test: ${input}"
PYTHONPATH=${PYTHON_PATH} ${COVERAGE} run --rcfile=.coveragerc ../test/${test} >/dev/null 2>&1
}
run_cmd_coverage() {
input=$@
if [ "$1" == "sudo" ]; then
shift
COVERAGE="sudo coverage"
else
COVERAGE="coverage"
fi
cmd="$@"
log "Invoke: ${input}"
${COVERAGE} run --rcfile=.coveragerc ${cmd} >/tmp/output.txt 2>&1
log_file_contents $? /tmp/output.txt
printf "\n"
}
run_cmd() {
cmd="$@"
${cmd} >/tmp/output.txt 2>&1
if [ -s /tmp/output.txt ]; then
log_file_contents $? /tmp/output.txt
else
printf " %s\n" "${cmd}"
fi
}
prerun_setup() {
if [ ! -d coverage ]; then
mkdir coverage
fi
for file in staf stac; do
if [ ! -f "/usr/share/dbus-1/system.d/org.nvmexpress.${file}.conf" -a \
! -f "/etc/dbus-1/system.d/org.nvmexpress.${file}.conf" ]; then
log "hardlink /etc/dbus-1/system.d/org.nvmexpress.${file}.conf -> @BUILD_DIR@/etc/dbus-1/system.d/org.nvmexpress.${file}.conf"
sudo ln @BUILD_DIR@/etc/dbus-1/system.d/org.nvmexpress.${file}.conf /etc/dbus-1/system.d/org.nvmexpress.${file}.conf
if [ $? -ne 0 ]; then
log "hardlink failed"
exit 1
fi
fi
done
sudo systemctl reload dbus.service
}
postrun_cleanup() {
sd_stop "stafd"
sd_stop "stacd"
log "Stop nvmet"
sudo ../utils/nvmet/nvmet.py clean >/tmp/output.txt 2>&1
log_file_contents $? /tmp/output.txt
printf "\n"
log "nvme disconnect-all"
run_cmd sudo nvme disconnect-all
printf "\n"
log "Remove ${stafd_conf_fname} and ${stacd_conf_fname}"
rm "${stafd_conf_fname}"
rm "${stacd_conf_fname}"
printf "\n"
for file in staf stac; do
if [ -f "/etc/dbus-1/system.d/org.nvmexpress.${file}.conf" ]; then
if [ "$(stat -c %h -- "/etc/dbus-1/system.d/org.nvmexpress.${file}.conf")" -gt 1 ]; then
log "Remove hardlink /etc/dbus-1/system.d/org.nvmexpress.${file}.conf"
sudo rm "/etc/dbus-1/system.d/org.nvmexpress.${file}.conf"
fi
fi
done
sudo systemctl reload dbus.service
sudo systemctl unmask avahi-daemon.service
sudo systemctl unmask avahi-daemon.socket
sudo systemctl start avahi-daemon.service
sudo systemctl start avahi-daemon.socket
sudo systemctl stop ${AVAHI_PUBLISHER} >/dev/null 2>&1
sudo systemctl reset-failed ${AVAHI_PUBLISHER} >/dev/null 2>&1
log "All done!!!"
log "FINISHED-FINISHED-FINISHED-FINISHED-FINISHED-FINISHED-FINISHED-FINISHED"
}
trap postrun_cleanup EXIT
trap postrun_cleanup SIGINT
################################################################################
################################################################################
################################################################################
log "START-START-START-START-START-START-START-START-START-START-START-START"
if systemctl is-active stafd.service >/dev/null 2>&1 || systemctl is-active stacd.service >/dev/null 2>&1; then
msg="Stopping because stafd and/or stacd is/are currently running."
printf "%b%s%s%b[0m\n" "\0033" ${RED} "${msg}" "\0033"
exit 1
fi
prerun_setup
#*******************************************************************************
# Load nvme kernel module
log "modprobe nvme_tcp"
run_cmd sudo /usr/sbin/modprobe nvme_tcp
log "nvme disconnect-all"
run_cmd sudo nvme disconnect-all
printf "\n"
sd_stop stafd # make sure it's not running already
sd_stop stacd # make sure it's not running already
#*******************************************************************************
# Create a dummy config file for stafd
log "Create dummy config file ${stafd_conf_fname}"
cat > "${stafd_conf_fname}" <<'EOF'
[Global]
tron = true
ip-family = ipv6
johnny = be-good
queue-size = 2000000
reconnect-delay = NaN
ctrl-loss-tmo = 10
disable-sqflow = true
[Discovery controller connection management]
persistent-connections = false
zeroconf-connections-persistence = -1
[Hello]
hello = bye
EOF
log_file_contents 0 "${stafd_conf_fname}"
printf "\n"
#*******************************************************************************
# Create a dummy config file for stacd
log "Create dummy config file ${stacd_conf_fname}"
cat > "${stacd_conf_fname}" <<'EOF'
[Global]
tron = true
kato = 10
nr-io-queues = 4
nr-write-queues = NaN
nr-poll-queues = NaN
queue-size = 2000000
reconnect-delay = 1
ctrl-loss-tmo = 1
disable-sqflow = true
[I/O controller connection management]
disconnect-scope = blah-blah
disconnect-trtypes = boing-boing
EOF
log_file_contents 0 "${stacd_conf_fname}"
printf "\n"
log "Stop & Mask Avahi daemon"
run_cmd sudo systemctl stop avahi-daemon.service
run_cmd sudo systemctl stop avahi-daemon.socket
run_cmd sudo systemctl mask avahi-daemon.service
run_cmd sudo systemctl mask avahi-daemon.socket
printf "\n"
sleep 1
log ">>>>>>>>>>>>>>>>>>>>> Marker [1] <<<<<<<<<<<<<<<<<<<<<"
printf "\n"
run_cmd_coverage stafctl ls
run_cmd_coverage stafctl invalid-command
run_cmd_coverage stacctl ls
run_cmd_coverage stacctl invalid-command
#*******************************************************************************
# Start nvme target simulator
log "Start nvmet"
sudo ../utils/nvmet/nvmet.py clean >/dev/null 2>&1
sudo ../utils/nvmet/nvmet.py create -f ../utils/nvmet/nvmet.conf >/tmp/output.txt 2>&1
log_file_contents $? /tmp/output.txt
printf "\n"
sleep 2
log ">>>>>>>>>>>>>>>>>>>>> Marker [2] <<<<<<<<<<<<<<<<<<<<<"
printf "\n"
#*******************************************************************************
# Start stafd and stacd
sd_start "stafd" "@STAFD_DBUS_NAME@" "${stafd_conf_fname}"
sd_start "stacd" "@STACD_DBUS_NAME@" "${stacd_conf_fname}"
sleep 2
run_cmd_coverage stafctl status
reload_cfg "stafd"
sleep 1
log "Restart Avahi daemon"
run_cmd sudo systemctl unmask avahi-daemon.socket
run_cmd sudo systemctl unmask avahi-daemon.service
run_cmd sudo systemctl start avahi-daemon.socket
run_cmd sudo systemctl start avahi-daemon.service
printf "\n"
sleep 2
log ">>>>>>>>>>>>>>>>>>>>> Marker [3] <<<<<<<<<<<<<<<<<<<<<"
printf "\n"
log "Change stafd config [1]:"
cat > "${stafd_conf_fname}" <<'EOF'
[Global]
tron = true
[Discovery controller connection management]
persistent-connections = false
zeroconf-connections-persistence = 0.5
[Service Discovery]
zeroconf = enabled
EOF
log_file_contents 0 "${stafd_conf_fname}"
printf "\n"
reload_cfg "stafd"
sleep 1
log "Change stafd config [2]:"
cat > "${stafd_conf_fname}" <<'EOF'
[Global]
tron = true
ip-family = ipv4
queue-size = 2000000
reconnect-delay = 1
ctrl-loss-tmo = 1
disable-sqflow = true
pleo = disable
[Discovery controller connection management]
persistent-connections = false
zeroconf-connections-persistence = 1:01
[Controllers]
controller = transport = tcp ; traddr = localhost ; ; ; kato=31; dhchap-ctrl-secret=not-so-secret
controller=transport=tcp;traddr=1.1.1.1
controller=transport=tcp;traddr=100.100.100.100
controller=transport=tcp;traddr=2607:f8b0:4002:c2c::71
exclude=transport=tcp;traddr=1.1.1.1
EOF
log_file_contents 0 "${stafd_conf_fname}"
printf "\n"
reload_cfg "stafd"
sleep 5
log "Change stacd config [1]:"
cat > "${stacd_conf_fname}" <<'EOF'
[Global]
tron=true
nr-io-queues=4
nr-write-queues=4
queue-size=2000000
reconnect-delay=1
ctrl-loss-tmo=1
disable-sqflow=true
[I/O controller connection management]
disconnect-scope=all-connections-matching-disconnect-trtypes
disconnect-trtypes=tcp+rdma
EOF
log_file_contents 0 "${stacd_conf_fname}"
printf "\n"
reload_cfg "stacd"
sleep 5
log ">>>>>>>>>>>>>>>>>>>>> Marker [4] <<<<<<<<<<<<<<<<<<<<<"
printf "\n"
run_cmd_coverage stafctl status
#*******************************************************************************
# Fake mDNS packets from a CDC
log "Start Avahi publisher"
run_cmd sudo systemctl stop ${AVAHI_PUBLISHER}
run_cmd sudo systemctl reset-failed ${AVAHI_PUBLISHER}
run_cmd sudo systemd-run --unit=${AVAHI_PUBLISHER} --working-directory=. avahi-publish -s SFSS _nvme-disc._tcp 8009 "p=tcp"
printf "\n"
sleep 1
#*******************************************************************************
run_cmd_coverage stafd --version
run_cmd_coverage stacd --version
#*******************************************************************************
# Stimulate D-Bus activity
run_cmd_coverage sudo stafctl --version
run_cmd_coverage sudo stafctl blah
run_cmd_coverage sudo stafctl troff
run_cmd_coverage stafctl status
run_cmd_coverage sudo stafctl tron
run_cmd_coverage stafctl ls -d
run_cmd_coverage stafctl adlp -d
run_cmd_coverage stafctl dlp -t tcp -a ::1 -s 8009
run_cmd_coverage sudo stacctl --version
run_cmd_coverage sudo stacctl blah
run_cmd_coverage sudo stacctl troff
run_cmd_coverage stacctl status
run_cmd_coverage sudo stacctl tron
run_cmd_coverage stacctl ls -d
log ">>>>>>>>>>>>>>>>>>>>> Marker [5] <<<<<<<<<<<<<<<<<<<<<"
printf "\n"
#*******************************************************************************
# Stimulate AENs activity by removing/restoring namespaces
log "Remove namespace: klingons"
run_cmd sudo ../utils/nvmet/nvmet.py unlink -p 1 -s klingons
printf "\n"
sleep 2
run_cmd_coverage stacctl ls
log "Restore namespace: klingons"
run_cmd sudo ../utils/nvmet/nvmet.py link -p 1 -s klingons
printf "\n"
sleep 2
run_cmd_coverage stacctl ls
log ">>>>>>>>>>>>>>>>>>>>> Marker [6] <<<<<<<<<<<<<<<<<<<<<"
printf "\n"
#*******************************************************************************
# Stop Avahi Publisher
log "Stop Avahi publisher"
run_cmd sudo systemctl stop ${AVAHI_PUBLISHER}
printf "\n"
sleep 1
#*******************************************************************************
log "Restart Avahi publisher"
run_cmd sudo systemd-run --unit=${AVAHI_PUBLISHER} --working-directory=. avahi-publish -s SFSS _nvme-disc._tcp 8009 "p=tcp"
printf "\n"
sleep 2
log ">>>>>>>>>>>>>>>>>>>>> Marker [7] <<<<<<<<<<<<<<<<<<<<<"
printf "\n"
#*******************************************************************************
# Make config changes for stafd
log "Change stafd config [3]:"
cat > "${stafd_conf_fname}" <<'EOF'
[Global]
tron = true
queue-size = 2000000
reconnect-delay = 1
ctrl-loss-tmo = 1
disable-sqflow = true
[Discovery controller connection management]
persistent-connections=false
zeroconf-connections-persistence=0.5
[Service Discovery]
zeroconf=disabled
EOF
log_file_contents 0 "${stafd_conf_fname}"
printf "\n"
reload_cfg "stafd"
sleep 3
#*******************************************************************************
# Make more config changes for stafd
log "Change stafd config [4]:"
cat > "${stafd_conf_fname}" <<'EOF'
[Global]
tron=true
queue-size=2000000
reconnect-delay=1
ctrl-loss-tmo=0
disable-sqflow=true
ip-family=ipv6
[Discovery controller connection management]
persistent-connections=false
zeroconf-connections-persistence=0
[Controllers]
controller=transport=tcp;traddr=localhost;trsvcid=8009
controller=transport=tcp;traddr=abracadabra
controller=transport=tcp;traddr=google.com
controller=
controller=trsvcid
controller=transport=rdma;traddr=!@#$
controller=transport=fc;traddr=21:00:00:00:00:00:00:00;host-traddr=20:00:00:00:00:00:00:00
controller=transport=XM;traddr=2.2.2.2
controller=transport=tcp;traddr=555.555.555.555
EOF
log_file_contents 0 "${stafd_conf_fname}"
printf "\n"
log ">>>>>>>>>>>>>>>>>>>>> Marker [8] <<<<<<<<<<<<<<<<<<<<<"
printf "\n"
reload_cfg "stafd"
sleep 2
#*******************************************************************************
# Stop Avahi Publisher
log "Stop Avahi publisher"
run_cmd sudo systemctl stop ${AVAHI_PUBLISHER}
printf "\n"
sleep 2
log ">>>>>>>>>>>>>>>>>>>>> Marker [9] <<<<<<<<<<<<<<<<<<<<<"
printf "\n"
#*******************************************************************************
# Remove one of the NVMe device's
file=/tmp/getdev-XXX.py
getdev=$(mktemp $file)
cat > "${getdev}" <<'EOF'
import sys
from dasbus.connection import SystemMessageBus
bus = SystemMessageBus()
iface = bus.get_proxy(sys.argv[1], sys.argv[2])
controllers = iface.list_controllers(False)
if len(controllers) > 0:
controller = controllers[0]
print(controller['device'])
sys.exit(0)
sys.exit(1)
EOF
# Find a Discovery Controller and issue a "nvme disconnect"
if dev=$(python3 ${getdev} @STAFD_DBUS_NAME@ @STAFD_DBUS_PATH@); then
log "Remove connection (disconnect) to Discovery Controller ${dev}"
run_cmd sudo nvme disconnect -d ${dev}
printf "\n"
else
msg="Failed to find a connection to a Discovery Controller"
printf "%b%s%s%b[0m\n" "\0033" ${RED} "${msg}" "\0033"
sudo logger -t COVERAGE -i "@@@@@ " -p warning -- "${msg}"
fi
# Find an I/O Controller and issue a "nvme disconnect"
if dev=$(python3 ${getdev} @STACD_DBUS_NAME@ @STACD_DBUS_PATH@); then
log "Remove connection (disconnect) to I/O Controller ${dev}"
run_cmd sudo nvme disconnect -d ${dev}
printf "\n"
else
msg="Failed to find a connection to an I/O Controller"
printf "%b%s%s%b[0m\n" "\0033" ${RED} "${msg}" "\0033"
sudo logger -t COVERAGE -i "@@@@@ " -p warning -- "${msg}"
fi
sleep 3
rm "${getdev}"
#*******************************************************************************
log ">>>>>>>>>>>>>>>>>>>>> Marker [10] <<<<<<<<<<<<<<<<<<<<<"
printf "\n"
sd_restart "stafd"
sd_restart "stacd"
sleep 4
log "Create invalid conditions for saving/loading stafd's last known config"
rm -rf "/tmp/stafd"
sd_restart "stafd"
sleep 2
log "Remove invalid conditions for saving/loading stafd's last known config"
mkdir -p "/tmp/stafd"
sd_restart "stafd"
sleep 2
#*******************************************************************************
# Change ownership of files that were created as root
sudo chown -R "${PRIMARY_USR}":"${PRIMARY_GRP}" coverage >/dev/null 2>&1
sudo chown -R "${PRIMARY_USR}":"${PRIMARY_GRP}" staslib/__pycache__ >/dev/null 2>&1
sudo chown -R "${PRIMARY_USR}":"${PRIMARY_GRP}" subprojects/libnvme/libnvme/__pycache__ >/dev/null 2>&1
#*******************************************************************************
# Run unit tests
run_unit_test test-avahi.py
run_unit_test test-avahi.py
run_unit_test test-config.py
run_unit_test test-controller.py
run_unit_test test-gtimer.py
run_unit_test test-iputil.py
run_unit_test test-log.py
run_unit_test sudo test-nvme_options.py # Test both with super user...
run_unit_test test-nvme_options.py # ... and with regular user
run_unit_test test-service.py
run_unit_test test-timeparse.py
run_unit_test test-transport_id.py
run_unit_test test-udev.py
run_unit_test test-version.py
#*******************************************************************************
# Stop nvme target simulator
log "Collect all coverage data"
coverage combine --rcfile=.coveragerc
printf "\n"
log "Generating coverage report"
coverage report -i --rcfile=.coveragerc
printf "\n"
log "Generating coverage report (HTML)"
coverage html -i --rcfile=.coveragerc
printf "\n"

1
doc/.gitignore vendored Normal file
View file

@ -0,0 +1 @@
*.xml~

118
doc/dbus-idl-to-docbooks.py Executable file
View file

@ -0,0 +1,118 @@
#!/usr/bin/python3
import os
import sys
import pathlib
import tempfile
import subprocess
from argparse import ArgumentParser
from lxml import etree
def parse_args():
parser = ArgumentParser(description='Generate DocBook documentation from D-Bus IDL.')
parser.add_argument(
'--idl',
action='store',
help='IDL file',
required=True,
type=str,
metavar='FILE',
)
parser.add_argument(
'--output-directory',
action='store',
help='Output directory where DocBook files will be saved',
required=True,
type=str,
metavar='DIR',
)
parser.add_argument(
'--tmp',
action='store',
help='Temporary directory for intermediate files',
required=True,
type=str,
metavar='DIR',
)
return parser.parse_args()
ARGS = parse_args()
pathlib.Path(ARGS.output_directory).mkdir(parents=True, exist_ok=True)
REF_ENTRY_INFO = '''\
<refentryinfo>
<title>stafctl</title>
<productname>nvme-stas</productname>
<author>
<personname>
<honorific>Mr</honorific>
<firstname>Martin</firstname>
<surname>Belanger</surname>
</personname>
<affiliation>
<orgname>Dell, Inc.</orgname>
</affiliation>
</author>
</refentryinfo>
'''
MANVOLNUM = '<manvolnum>5</manvolnum>'
PURPOSE = '<refpurpose>DBus interface</refpurpose>'
PARSER = etree.XMLParser(remove_blank_text=True)
def add_missing_info(fname, stem):
xml = etree.parse(fname, PARSER)
root = xml.getroot()
if root.tag != 'refentry':
return
if xml.find('refentryinfo'):
return
root.insert(0, etree.fromstring(REF_ENTRY_INFO))
refmeta = xml.find('refmeta')
if refmeta is not None:
if refmeta.find('refentrytitle') is None:
refmeta.append(etree.fromstring(f'<refentrytitle>{stem}</refentrytitle>'))
refmeta.append(etree.fromstring(MANVOLNUM))
refnamediv = xml.find('refnamediv')
if refnamediv is not None:
refpurpose = refnamediv.find('refpurpose')
if refpurpose is not None:
refnamediv.remove(refpurpose)
refnamediv.append(etree.fromstring(PURPOSE))
et = etree.ElementTree(root)
et.write(fname, pretty_print=True)
FILE_PREFIX = 'nvme-stas'
FINAL_PREFIX = FILE_PREFIX + '-'
pathlib.Path(ARGS.tmp).mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(dir=ARGS.tmp) as tmpdirname:
try:
subprocess.run(['gdbus-codegen', '--output-directory', tmpdirname, '--generate-docbook', FILE_PREFIX, ARGS.idl])
except subprocess.CalledProcessError as ex:
sys.exit(f'Failed to generate DocBook file. {ex}')
stems = []
with os.scandir(tmpdirname) as it:
for entry in it:
if entry.is_file() and entry.name.endswith('.xml') and entry.name.startswith(FINAL_PREFIX):
fname = entry.name[len(FINAL_PREFIX) :] # Strip prefix
stem = fname[0:-4] # Strip '.xml' suffix
stems.append(stem)
tmp_file = os.path.join(tmpdirname, entry.name)
add_missing_info(tmp_file, stem)
os.replace(tmp_file, os.path.join(ARGS.output_directory, fname))
print(';'.join(stems))

36
doc/genlist-from-docbooks.py Executable file
View file

@ -0,0 +1,36 @@
#!/usr/bin/python3
import glob
from lxml import etree
exclude_list = list(glob.glob('standard-*.xml'))
PARSER = etree.XMLParser(remove_blank_text=True)
def extract_data(fname):
et = etree.parse(fname, PARSER)
manvolnum = et.find('./refmeta/manvolnum')
manvolnum = manvolnum.text if manvolnum is not None else 0
deps = set()
for elem in et.iter():
keys = elem.keys()
if 'href' in keys and 'xpointer' in keys:
dep = elem.values()[0]
if dep in exclude_list:
deps.add(dep)
return manvolnum, list(deps)
output = list()
file_list = glob.glob('*.xml')
for fname in file_list:
if fname not in exclude_list:
stem = fname[0:-4]
manvolnum, deps = extract_data(fname)
deps = ':'.join(deps) if deps else 'None'
output.append(','.join([stem, manvolnum, fname, deps]))
print(';'.join(output))

63
doc/html.xsl Normal file
View file

@ -0,0 +1,63 @@
<?xml version='1.0'?> <!--*-nxml-*-->
<!--
SPDX-License-Identifier: Apache-2.0
Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
-->
<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
<xsl:import href="http://docbook.sourceforge.net/release/xsl/current/html/docbook.xsl"/>
<xsl:template match="citerefentry[not(@project)]">
<a>
<xsl:attribute name="href">
<xsl:value-of select="refentrytitle"/>
<xsl:text>.html#</xsl:text>
<xsl:value-of select="refentrytitle/@target"/>
</xsl:attribute>
<xsl:call-template name="inline.charseq"/>
</a>
</xsl:template>
<xsl:template name="user.header.content">
<style>
a.headerlink {
color: #c60f0f;
font-size: 0.8em;
padding: 0 4px 0 4px;
text-decoration: none;
visibility: hidden;
}
a.headerlink:hover {
background-color: #c60f0f;
color: white;
}
h1:hover > a.headerlink, h2:hover > a.headerlink, h3:hover > a.headerlink, dt:hover > a.headerlink {
visibility: visible;
}
</style>
<a>
<xsl:text xml:space="preserve" white-space="pre"> &#160; </xsl:text>
</a>
<span style="float:right">
<xsl:text>nvme-stas </xsl:text>
<xsl:value-of select="$nvme-stas.version"/>
</span>
<hr/>
</xsl:template>
<xsl:template match="literal">
<xsl:text>"</xsl:text>
<xsl:call-template name="inline.monoseq"/>
<xsl:text>"</xsl:text>
</xsl:template>
<xsl:output method="html" encoding="UTF-8" indent="no"/>
</xsl:stylesheet>

BIN
doc/images/Coverage.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 124 KiB

BIN
doc/images/STAF-STAC-libnvme.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 80 KiB

41
doc/man.xsl Normal file
View file

@ -0,0 +1,41 @@
<?xml version='1.0'?> <!--*-nxml-*-->
<!--
SPDX-License-Identifier: Apache-2.0
Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
-->
<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
xmlns:exsl="http://exslt.org/common"
extension-element-prefixes="exsl"
version="1.0">
<xsl:import href="http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl"/>
<xsl:template name="TH.title.line">
<xsl:param name="title"/>
<xsl:param name="section"/>
<xsl:call-template name="mark.subheading"/>
<xsl:text>.TH "</xsl:text>
<xsl:call-template name="string.upper">
<xsl:with-param name="string">
<xsl:value-of select="normalize-space($title)"/>
</xsl:with-param>
</xsl:call-template>
<xsl:text>" "</xsl:text>
<xsl:value-of select="normalize-space($section)"/>
<xsl:text>" "" "nvme-stas </xsl:text>
<xsl:value-of select="$nvme-stas.version"/>
<xsl:text>" "</xsl:text>
<xsl:text>"&#10;</xsl:text>
<xsl:call-template name="mark.subheading"/>
</xsl:template>
</xsl:stylesheet>

126
doc/meson.build Normal file
View file

@ -0,0 +1,126 @@
# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See the LICENSE file for details.
#
# This file is part of NVMe STorage Appliance Services (nvme-stas).
#
# Authors: Martin Belanger <Martin.Belanger@dell.com>
#
if want_man or want_html or want_readthedocs
docbklst = find_program('genlist-from-docbooks.py')
dbus2doc = find_program('dbus-idl-to-docbooks.py')
dbusgen = find_program('gdbus-codegen', required: false) # Needed by dbus2doc
if not dbusgen.found()
error('gdbus-codegen missing: Install libglib2.0-dev (deb) / glib2-devel (rpm)')
endif
# Get the list of DocBook files to process. The result will
# be saved to variable docbooks as a list of tuples as follows:
# docbooks = [ ['file1', 'manvolnum-from-file1.xml', 'file1.xml'],
# ['file2', 'manvolnum-from-file2.xml', 'file2.xml'], ... ]
docbooks = []
rr = run_command(docbklst, check: true)
output = rr.stdout().strip()
if output != ''
foreach item : output.split(';')
items = item.split(',')
stem = items[0]
manvolnum = items[1]
fname = items[2]
deps = items[3]
if deps == 'None'
deps = []
else
deps = deps.split(':')
endif
docbooks += [ [stem, manvolnum, fname, deps] ]
endforeach
endif
# Generate DocBooks from IDL queried directly from the D-Bus services.
out_dir = conf.get('BUILD_DIR') / 'man-tmp'
env = environment({'PYTHONPATH': PYTHONPATH})
idls = [ 'stafd.idl', 'stacd.idl' ]
foreach idl : idls
rr = run_command(
dbus2doc,
'--idl', conf.get('BUILD_DIR') / 'staslib' / idl,
'--output-directory', out_dir,
'--tmp', meson.current_build_dir(),
env: env,
check: true)
output = rr.stdout().strip()
if output != ''
foreach stem : output.split(';')
docbooks += [ [stem, '5', out_dir / stem + '.xml', []] ]
endforeach
endif
endforeach
xsltproc = find_program('xsltproc')
if xsltproc.found()
manpage_style = 'http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl'
if run_command(xsltproc, '--nonet', manpage_style, check: false).returncode() != 0
error('Docbook style sheet missing: Install docbook-xsl (deb) / docbook-style-xsl (rpm)')
endif
endif
xslt_cmd = [
xsltproc,
'--nonet',
'--xinclude',
'--stringparam', 'man.output.quietly', '1',
'--stringparam', 'funcsynopsis.style', 'ansi',
'--stringparam', 'man.th.extra1.suppress', '1',
'--stringparam', 'man.authors.section.enabled', '0',
'--stringparam', 'man.copyright.section.enabled', '0',
'--stringparam', 'nvme-stas.version', '@0@'.format(meson.project_version()),
'-o', '@OUTPUT@',
]
man_xsl = files('man.xsl')
html_xsl = files('html.xsl')
html_files = [] # Will be used as input to readthedocs
foreach tuple: docbooks
stem = tuple[0]
sect = tuple[1]
file = files(tuple[2])
deps = tuple[3]
if want_man
man = stem + '.' + sect
custom_target(
man,
input: file,
output: man,
depend_files: deps,
command: xslt_cmd + [man_xsl, '@INPUT@'],
install: true,
install_dir: mandir / ('man' + sect)
)
endif
if want_html or want_readthedocs
html = stem + '.html'
html_file = custom_target(
html,
input: file,
output: html,
depend_files: deps,
command: xslt_cmd + [html_xsl, '@INPUT@'],
install: want_html,
install_dir: docdir / 'html'
)
html_files += [ [stem, html_file ] ]
endif
endforeach
endif
if want_readthedocs
subdir('readthedocs')
endif

185
doc/nvme-stas.xml Normal file
View file

@ -0,0 +1,185 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
<!--
SPDX-License-Identifier: Apache-2.0
Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
-->
<refentry id="nvme-stas">
<refentryinfo>
<title>nvme-stas</title>
<productname>nvme-stas</productname>
<author>
<personname>
<honorific>Mr</honorific>
<firstname>Martin</firstname>
<surname>Belanger</surname>
</personname>
<affiliation>
<orgname>Dell, Inc.</orgname>
</affiliation>
</author>
</refentryinfo>
<refmeta>
<refentrytitle>nvme-stas</refentrytitle>
<manvolnum>7</manvolnum>
</refmeta>
<refnamediv>
<refname>nvme-stas</refname>
<refpurpose>NVMe over Fabrics STorage Appliance Services</refpurpose>
</refnamediv>
<refsect1>
<title>Introduction</title>
<para>
This page describes the services provided by the <code>nvme-stas</code> package.
</para>
<para>
<code>nvme-stas</code> is composed of two services, <citerefentry><refentrytitle>stafd</refentrytitle><manvolnum>8</manvolnum></citerefentry>
and <citerefentry><refentrytitle>stacd</refentrytitle><manvolnum>8</manvolnum></citerefentry>,
running on a host computer (the NVMe Host).
</para>
<refsect2>
<title>STorage Appliance Finder (<code>stafd</code>)</title>
<para>
The tasks performed by <code>stafd</code> include:
</para>
<itemizedlist mark='opencircle'>
<listitem>
<para>
Register for mDNS service type <parameter>_nvme-disc._tcp</parameter>
with Avahi, the service discovery daemon. This allows <code>stafd</code>
to automatically locate Central or Direct Discovery
Controllers (CDC, DDC) with zero-configuration networking
(zeroconf). <code>stafd</code> also allows users to manually enter CDCs
and DDCs in a configuration file
(<filename>/etc/stas/stafd.conf</filename>) when users
prefer not to enable mDNS-based zeroconf.
</para>
</listitem>
<listitem>
<para>
Connect to discovered or configured CDCs or DDCs.
</para>
</listitem>
<listitem>
<para>
Retrieve the list of NVMe subsystem IO Controllers or
Discovery Controller referrals from the Discovery Log Page
using the NVMe command "Get Log Page".
</para>
</listitem>
<listitem>
<para>
Maintain a cache of the discovery log pages.
</para>
</listitem>
<listitem>
<para>
Provide a D-Bus API where other applications can interact
with <code>stafd</code>. This API can be used, for example, to retrieve
the list of cached discovery log pages.
</para>
</listitem>
</itemizedlist>
</refsect2>
<refsect2>
<title>STorage Appliance Connector (<code>stacd</code>)</title>
<para>
The tasks performed by <code>stacd</code> include:
</para>
<itemizedlist mark='opencircle'>
<listitem>
<para>
Read the list of storage subsystems (i.e., discovery log pages)
from <code>stafd</code> over the D-Bus API.
</para>
</listitem>
<listitem>
<para>
Similar to <code>stafd</code>, <code>stacd</code> can also read a list of storage
subsystems to connect to from a configuration
file: (<filename>/etc/stas/stacd.conf</filename>).
</para>
</listitem>
<listitem>
<para>
Set up the I/O controller connections to each storage subsystem.
</para>
</listitem>
<listitem>
<para>
Provide a D-Bus API where other applications can interact
with <code>stacd</code>. For example, an application could retrieve the
list of I/O controllers that <code>stacd</code> connected to.
</para>
</listitem>
</itemizedlist>
</refsect2>
<refsect2>
<title>System configuration</title>
<para>
A host must be provided with a Host NQN and a Host ID. <code>nvme-stas</code>
will not run without these two mandatory configuration parameters.
To follow in the footsteps of <code>nvme-cli</code> and <code>libnvme</code>,
<code>nvme-stas</code> will use the same Host NQN and ID that
<code>nvme-cli</code> and <code>libnvme</code> use by default.
In other words, <code>nvme-stas</code> will read the Host NQN and ID
from these two files by default:
</para>
<itemizedlist mark='opencircle'>
<listitem>
<para>
<filename>/etc/nvme/hostnqn</filename>
</para>
</listitem>
<listitem>
<para>
<filename>/etc/nvme/hostid</filename>
</para>
</listitem>
</itemizedlist>
<para>
Using the same configuration files will ensure consistency between
<code>nvme-stas</code>, <code>nvme-cli</code>, and <code>libnvme</code>.
On the other hand, <code>nvme-stas</code> can operate with a
different Host NQN and/or ID. In that case, one can specify them
in <filename>/etc/stas/sys.conf</filename>.
</para>
A new optional configuration parameters introduced in TP8010, the
Host Symbolic Name, can also be specified in <filename>/etc/stas/sys.conf</filename>.
The documentation for <filename>/etc/stas/sys.conf</filename>
can be found <filename>/etc/stas/sys.conf.doc</filename>.
<para>
</para>
</refsect2>
</refsect1>
<refsect1>
<title>See Also</title>
<para>
<citerefentry><refentrytitle>stacctl</refentrytitle><manvolnum>1</manvolnum></citerefentry>,
<citerefentry><refentrytitle>stacd.conf</refentrytitle><manvolnum>5</manvolnum></citerefentry>,
<citerefentry><refentrytitle>stacd.service</refentrytitle><manvolnum>8</manvolnum></citerefentry>,
<citerefentry><refentrytitle>stacd</refentrytitle><manvolnum>8</manvolnum></citerefentry>,
<citerefentry><refentrytitle>stafctl</refentrytitle><manvolnum>1</manvolnum></citerefentry>,
<citerefentry><refentrytitle>stafd.conf</refentrytitle><manvolnum>5</manvolnum></citerefentry>,
<citerefentry><refentrytitle>stafd.service</refentrytitle><manvolnum>8</manvolnum></citerefentry>,
<citerefentry><refentrytitle>stafd</refentrytitle><manvolnum>8</manvolnum></citerefentry>,
</para>
</refsect1>
</refentry>

22
doc/readthedocs/Makefile Normal file
View file

@ -0,0 +1,22 @@
# Minimal makefile for Sphinx documentation
#
.DEFAULT_GOAL := help
# You can set these variables from the command line, and also
# from the environment for the first two.
SPHINXOPTS ?=
SPHINXBUILD ?= sphinx-build
SOURCEDIR = .
BUILDDIR = _build
# Put it first so that "make" without argument is like "make help".
.PHONY: help
help:
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
# Catch-all target: route all unknown targets to Sphinx using the new
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
.PHONY: Makefile
%: Makefile
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

34
doc/readthedocs/conf.py Normal file
View file

@ -0,0 +1,34 @@
# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html
# -- Project information -----------------------------------------------------
project = 'nvme-stas'
copyright = 'Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved.'
author = 'Martin Belanger <martin.belanger@dell.com>'
master_doc = 'index'
version = '@VERSION@'
release = '@VERSION@'
# -- General configuration ---------------------------------------------------
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
'sphinx.ext.autosummary',
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = ['Thumbs.db', '.DS_Store']

1
doc/readthedocs/environment.txt Normal file
View file

@ -0,0 +1 @@
sphinx==5.3.0

30
doc/readthedocs/index.rst Normal file
View file

@ -0,0 +1,30 @@
Welcome to nvme-stas's documentation!
=====================================
What does nvme-stas provide?
* A Central Discovery Controller (CDC) client for Linux
* Asynchronous Event Notifications (AEN) handling
* Automated NVMe subsystem connection controls
* Error handling and reporting
* Automatic (zeroconf) and Manual configuration
.. toctree::
:maxdepth: 2
:caption: Contents:
installation.rst
nvme-stas.rst
stafd-index.rst
stacd-index.rst
stasadm.rst
sys.conf.rst
stas-config.target.rst
stas-config@.service.rst
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`

28
doc/readthedocs/installation.rst Normal file
View file

@ -0,0 +1,28 @@
Installation
============
Debian / Ubuntu:
----------------
.. code-block:: sh
$ apt-get install nvme-stas
Fedora / Red Hat:
-----------------
.. code-block:: sh
$ dnf install nvme-stas
Python Version
--------------
The latest Python 3 version is always recommended, since it has all the latest bells and
whistles. libnvme supports Python 3.6 and above.
Dependencies
------------
nvme-stas is built on top of libnvme, which is used to interact with the kernel's NVMe driver (i.e. drivers/nvme/host/). To support all the features of nvme-stas, several changes to the Linux kernel are required. nvme-stas can also operate with older kernels, but with limited functionality. Kernel 5.18 provides all the features needed by nvme-stas. nvme-stas can also work with older kernels that include back-ported changes to the NVMe driver.

35
doc/readthedocs/make.bat Normal file
View file

@ -0,0 +1,35 @@
@ECHO OFF
pushd %~dp0
REM Command file for Sphinx documentation
if "%SPHINXBUILD%" == "" (
set SPHINXBUILD=sphinx-build
)
set SOURCEDIR=.
set BUILDDIR=_build
if "%1" == "" goto help
%SPHINXBUILD% >NUL 2>NUL
if errorlevel 9009 (
echo.
echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
echo.installed, then set the SPHINXBUILD environment variable to point
echo.to the full path of the 'sphinx-build' executable. Alternatively you
echo.may add the Sphinx directory to PATH.
echo.
echo.If you don't have Sphinx installed, grab it from
echo.https://www.sphinx-doc.org/
exit /b 1
)
%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
goto end
:help
%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
:end
popd

64
doc/readthedocs/meson.build Normal file
View file

@ -0,0 +1,64 @@
# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See the LICENSE file for details.
#
# This file is part of NVMe STorage Appliance Services (nvme-stas).
#
# Authors: Martin Belanger <Martin.Belanger@dell.com>
#
pandoc = find_program('pandoc', required: true)
components = [
'conf.py',
'Makefile',
'make.bat',
'index.rst',
'environment.txt',
'installation.rst',
'nvme-stas.rst',
'org.nvmexpress.stac.debug.rst',
'org.nvmexpress.stac.rst',
'org.nvmexpress.staf.debug.rst',
'org.nvmexpress.staf.rst',
'stacctl.rst',
'stacd-index.rst',
'stacd.conf.rst',
'stacd.rst',
'stacd.service.rst',
'stafctl.rst',
'stafd-index.rst',
'stafd.conf.rst',
'stafd.rst',
'stafd.service.rst',
'stas-config.target.rst',
'stas-config@.service.rst',
'stasadm.rst',
'sys.conf.rst',
]
foreach component : components
configure_file(
input: component,
output: component,
configuration: conf,
)
endforeach
foreach tuple: html_files
stem = tuple[0]
html_file = tuple[1]
rst = '_' + stem + '.rst'
custom_target(
rst,
input: html_file,
output: rst,
build_by_default: true,
command: [
pandoc,
'-f', 'html',
'-t', 'rst',
'-o', '@OUTPUT@',
'@INPUT@'
]
)
endforeach

5
doc/readthedocs/nvme-stas.rst Normal file
View file

@ -0,0 +1,5 @@
==========================
STorage Appliance Services
==========================
.. include:: _nvme-stas.rst

7
doc/readthedocs/org.nvmexpress.stac.debug.rst Normal file
View file

@ -0,0 +1,7 @@
=========================
org.nvmexpress.stac.debug
=========================
.. module:: org.nvmexpress.stac.debug
.. include:: _org.nvmexpress.stac.debug.rst

7
doc/readthedocs/org.nvmexpress.stac.rst Normal file
View file

@ -0,0 +1,7 @@
===================
org.nvmexpress.stac
===================
.. module:: org.nvmexpress.stac
.. include:: _org.nvmexpress.stac.rst

7
doc/readthedocs/org.nvmexpress.staf.debug.rst Normal file
View file

@ -0,0 +1,7 @@
=========================
org.nvmexpress.staf.debug
=========================
.. module:: org.nvmexpress.staf.debug
.. include:: _org.nvmexpress.staf.debug.rst

7
doc/readthedocs/org.nvmexpress.staf.rst Normal file
View file

@ -0,0 +1,7 @@
===================
org.nvmexpress.staf
===================
.. module:: org.nvmexpress.staf
.. include:: _org.nvmexpress.staf.rst

7
doc/readthedocs/stacctl.rst Normal file
View file

@ -0,0 +1,7 @@
=======
stacctl
=======
.. module:: stacctl
.. include:: _stacctl.rst

13
doc/readthedocs/stacd-index.rst Normal file
View file

@ -0,0 +1,13 @@
STorage Appliance Connector
===========================
.. toctree::
:maxdepth: 1
stacd.rst
stacd.conf.rst
stacd.service.rst
stacctl.rst
org.nvmexpress.stac.rst
org.nvmexpress.stac.debug.rst

7
doc/readthedocs/stacd.conf.rst Normal file
View file

@ -0,0 +1,7 @@
==========
stacd.conf
==========
.. module:: stacd.conf
.. include:: _stacd.conf.rst

8
doc/readthedocs/stacd.rst Normal file
View file

@ -0,0 +1,8 @@
=====
stacd
=====
.. module:: stacd
.. include:: _stacd.rst

7
doc/readthedocs/stacd.service.rst Normal file
View file

@ -0,0 +1,7 @@
=============
stacd.service
=============
.. module:: stacd.service
.. include:: _stacd.service.rst

7
doc/readthedocs/stafctl.rst Normal file
View file

@ -0,0 +1,7 @@
=======
stafctl
=======
.. module:: stafctl
.. include:: _stafctl.rst

13
doc/readthedocs/stafd-index.rst Normal file
View file

@ -0,0 +1,13 @@
STorage Appliance Finder
========================
.. toctree::
:maxdepth: 1
stafd.rst
stafd.conf.rst
stafd.service.rst
stafctl.rst
org.nvmexpress.staf.rst
org.nvmexpress.staf.debug.rst

7
doc/readthedocs/stafd.conf.rst Normal file
View file

@ -0,0 +1,7 @@
==========
stafd.conf
==========
.. module:: stafd.conf
.. include:: _stafd.conf.rst

7
doc/readthedocs/stafd.rst Normal file
View file

@ -0,0 +1,7 @@
=====
stafd
=====
.. module:: stafd
.. include:: _stafd.rst

7
doc/readthedocs/stafd.service.rst Normal file
View file

@ -0,0 +1,7 @@
=============
stafd.service
=============
.. module:: stafd.service
.. include:: _stafd.service.rst

7
doc/readthedocs/stas-config.target.rst Normal file
View file

@ -0,0 +1,7 @@
==================
stas-config.target
==================
.. module:: stas-config.target
.. include:: _stas-config.target.rst

7
doc/readthedocs/stas-config@.service.rst Normal file
View file

@ -0,0 +1,7 @@
====================
stas-config@.service
====================
.. module:: stas-config@.service
.. include:: _stas-config@.service.rst

7
doc/readthedocs/stasadm.rst Normal file
View file

@ -0,0 +1,7 @@
=======
stasadm
=======
.. module:: stasadm
.. include:: _stasadm.rst

7
doc/readthedocs/sys.conf.rst Normal file
View file

@ -0,0 +1,7 @@
========
sys.conf
========
.. module:: sys.conf
.. include:: _sys.conf.rst

223
doc/stacctl.xml Normal file
View file

@ -0,0 +1,223 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--*-nxml-*-->
<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
<!--
SPDX-License-Identifier: Apache-2.0
Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
-->
<refentry id="stacctl" xmlns:xi="http://www.w3.org/2001/XInclude">
<refentryinfo>
<title>stacctl</title>
<productname>nvme-stas</productname>
<author>
<personname>
<honorific>Mr</honorific>
<firstname>Martin</firstname>
<surname>Belanger</surname>
</personname>
<affiliation>
<orgname>Dell, Inc.</orgname>
</affiliation>
</author>
</refentryinfo>
<refmeta>
<refentrytitle>stacctl</refentrytitle>
<manvolnum>1</manvolnum>
</refmeta>
<refnamediv>
<refname>stacctl</refname>
<refpurpose>STorage Appliance Connector (STAC) utility program</refpurpose>
</refnamediv>
<refsynopsisdiv>
<cmdsynopsis>
<command>stacctl</command>
<arg choice="opt" rep="repeat">OPTIONS</arg>
<arg choice="req">COMMAND</arg>
<arg choice="opt" rep="repeat">OPTIONS</arg>
</cmdsynopsis>
</refsynopsisdiv>
<refsect1>
<title>Description</title>
<para>
<command>stacctl</command> is a tool that can be used to communicate
with the <citerefentry><refentrytitle>stacd</refentrytitle>
<manvolnum>8</manvolnum></citerefentry> daemon to retrieve
operational data.
</para>
</refsect1>
<refsect1>
<title>Commands</title>
<para>The following commands are understood:</para>
<variablelist>
<xi:include href="standard-options.xml" xpointer="tron"/>
<xi:include href="standard-options.xml" xpointer="troff"/>
<xi:include href="standard-options.xml" xpointer="status"/>
<varlistentry>
<term><command>ls</command></term>
<listitem>
<para>
Show the list of I/O controllers. This will list
all the I/O controllers configured in
<citerefentry>
<refentrytitle>stacd.conf</refentrytitle>
<manvolnum>5</manvolnum>
</citerefentry> as well as those discovered by the
<citerefentry>
<refentrytitle>stafd</refentrytitle>
<manvolnum>8</manvolnum>
</citerefentry> daemon.
</para>
</listitem>
</varlistentry>
</variablelist>
</refsect1>
<refsect1>
<title>Options</title>
<para>The following options are understood:</para>
<variablelist>
<xi:include href="standard-options.xml" xpointer="help"/>
<xi:include href="standard-options.xml" xpointer="version"/>
<xi:include href="standard-options.xml" xpointer="detailed"/>
</variablelist>
</refsect1>
<refsect1>
<title>Exit status</title>
<para>
On success, 0 is returned; otherwise, a non-zero failure code is
returned.
</para>
</refsect1>
<refsect1>
<title>Examples</title>
<example>
<title>List I/O controllers</title>
<programlisting>$ stacctl ls --detailed
[{'connect attempts': 0,
'device': 'nvme1',
'host-iface': '',
'host-traddr': '',
'hostid': '3e518ec3-72ec-46a5-a603-2510e3140e29',
'hostnqn': 'nqn.2014-08.org.nvmexpress:uuid:13730573-e8d7-446e-81f6-042a497846d5',
'model': 'Linux',
'retry connect timer': '60.0s [off]',
'serial': '8d22fa96da912fb13f5a',
'subsysnqn': 'nqn.1988-11.com.dell:PowerSANxxx:01:20210225100113-454f73093ceb4847a7bdfc6e34aedead',
'traddr': '::1',
'transport': 'tcp',
'trsvcid': '8009'},
{'connect attempts': 0,
'device': 'nvme2',
'host-iface': '',
'host-traddr': '',
'hostid': '3e518ec3-72ec-46a5-a603-2510e3140e29',
'hostnqn': 'nqn.2014-08.org.nvmexpress:uuid:13730573-e8d7-446e-81f6-042a497846d5',
'model': 'Linux',
'retry connect timer': '60.0s [off]',
'serial': 'a9987ae2fd173d100fd0',
'subsysnqn': 'nqn.1988-11.com.dell:PowerSANxxx:01:20210225100113-454f73093ceb4847a7bdfc6e34aebeef',
'traddr': '::1',
'transport': 'tcp',
'trsvcid': '8009'},
{'connect attempts': 0,
'device': 'nvme3',
'host-iface': '',
'host-traddr': '',
'hostid': '3e518ec3-72ec-46a5-a603-2510e3140e29',
'hostnqn': 'nqn.2014-08.org.nvmexpress:uuid:13730573-e8d7-446e-81f6-042a497846d5',
'model': 'Linux',
'retry connect timer': '60.0s [off]',
'serial': '13e122f1a8122bed5a8d',
'subsysnqn': 'nqn.1988-11.com.dell:PowerSANxxx:01:20210225100113-454f73093ceb4847a7bdfc6e34ae8e28',
'traddr': '::1',
'transport': 'tcp',
'trsvcid': '8009'}]</programlisting>
</example>
<example>
<title>Disable tracing</title>
<programlisting>$ stacctl troff</programlisting>
</example>
<example>
<title>
Show <citerefentry><refentrytitle>stacd</refentrytitle>
<manvolnum>8</manvolnum></citerefentry> operational status.
</title>
<programlisting>$ stacctl status
{'config soak timer': '1.5s [off]',
'controllers': [{'connect attempts': 0,
'device': 'nvme1',
'host-iface': '',
'host-traddr': '',
'hostid': '3e518ec3-72ec-46a5-a603-2510e3140e29',
'hostnqn': 'nqn.2014-08.org.nvmexpress:uuid:13730573-e8d7-446e-81f6-042a497846d5',
'model': 'Linux',
'retry connect timer': '60.0s [off]',
'serial': '8d22fa96da912fb13f5a',
'subsysnqn': 'nqn.1988-11.com.dell:PowerSANxxx:01:20210225100113-454f73093ceb4847a7bdfc6e34aedead',
'traddr': '::1',
'transport': 'tcp',
'trsvcid': '8009'},
{'connect attempts': 0,
'device': 'nvme2',
'host-iface': '',
'host-traddr': '',
'hostid': '3e518ec3-72ec-46a5-a603-2510e3140e29',
'hostnqn': 'nqn.2014-08.org.nvmexpress:uuid:13730573-e8d7-446e-81f6-042a497846d5',
'model': 'Linux',
'retry connect timer': '60.0s [off]',
'serial': 'a9987ae2fd173d100fd0',
'subsysnqn': 'nqn.1988-11.com.dell:PowerSANxxx:01:20210225100113-454f73093ceb4847a7bdfc6e34aebeef',
'traddr': '::1',
'transport': 'tcp',
'trsvcid': '8009'},
{'connect attempts': 0,
'device': 'nvme3',
'host-iface': '',
'host-traddr': '',
'hostid': '3e518ec3-72ec-46a5-a603-2510e3140e29',
'hostnqn': 'nqn.2014-08.org.nvmexpress:uuid:13730573-e8d7-446e-81f6-042a497846d5',
'model': 'Linux',
'retry connect timer': '60.0s [off]',
'serial': '13e122f1a8122bed5a8d',
'subsysnqn': 'nqn.1988-11.com.dell:PowerSANxxx:01:20210225100113-454f73093ceb4847a7bdfc6e34ae8e28',
'traddr': '::1',
'transport': 'tcp',
'trsvcid': '8009'}],
'log-level': 'DEBUG',
'tron': True}</programlisting>
</example>
</refsect1>
<refsect1>
<title>See Also</title>
<para>
<citerefentry>
<refentrytitle>stacd.conf</refentrytitle>
<manvolnum>5</manvolnum>
</citerefentry>,
<citerefentry>
<refentrytitle>stacd</refentrytitle>
<manvolnum>8</manvolnum>
</citerefentry>
<citerefentry project="man-pages"/>
</para>
</refsect1>
</refentry>

652
doc/stacd.conf.xml Normal file
View file

@ -0,0 +1,652 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--*-nxml-*-->
<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
<!--
SPDX-License-Identifier: Apache-2.0
Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
-->
<refentry id="stacd.conf" xmlns:xi="http://www.w3.org/2001/XInclude">
<refentryinfo>
<title>stacd.conf</title>
<productname>nvme-stas</productname>
<author>
<personname>
<honorific>Mr</honorific>
<firstname>Martin</firstname>
<surname>Belanger</surname>
</personname>
<affiliation>
<orgname>Dell, Inc.</orgname>
</affiliation>
</author>
</refentryinfo>
<refmeta>
<refentrytitle>stacd.conf</refentrytitle>
<manvolnum>5</manvolnum>
</refmeta>
<refnamediv>
<refname>stacd.conf</refname>
<refpurpose>
<citerefentry project="man-pages">
<refentrytitle>stacd</refentrytitle>
<manvolnum>8</manvolnum>
</citerefentry>
configuration file
</refpurpose>
</refnamediv>
<refsynopsisdiv>
<para>
<filename>/etc/stas/stacd.conf</filename>
</para>
</refsynopsisdiv>
<refsect1>
<title>Description</title>
<para>
When <citerefentry project="man-pages"><refentrytitle>stacd</refentrytitle>
<manvolnum>8</manvolnum></citerefentry> starts up, it reads its
configuration from <filename>stacd.conf</filename>.
</para>
</refsect1>
<refsect1>
<title>Configuration File Format</title>
<para>
<filename>stacd.conf</filename> is a plain text file divided into
sections, with configuration entries in the style
<replaceable>key</replaceable>=<replaceable>value</replaceable>.
Spaces immediately before or after the <literal>=</literal> are
ignored. Empty lines are ignored as well as lines starting with
<literal>#</literal>, which may be used for commenting.
</para>
</refsect1>
<refsect1>
<title>Options</title>
<refsect2>
<title>[Global] section</title>
<para>
The following options are available in the
<literal>[Global]</literal> section:
</para>
<variablelist>
<xi:include href="standard-conf.xml" xpointer="tron"/>
<xi:include href="standard-conf.xml" xpointer="hdr-digest"/>
<xi:include href="standard-conf.xml" xpointer="data-digest"/>
<xi:include href="standard-conf.xml" xpointer="kato"/>
<xi:include href="standard-conf.xml" xpointer="ip-family"/>
<varlistentry>
<term><varname>nr-io-queues=</varname></term>
<listitem>
<para>
Takes a value in the range 1...N. Overrides the
default number of I/O queues create by the driver.
</para>
<para>Note: This parameter is identical to that provided by nvme-cli.</para>
<para>
Default: Depends on kernel and other run
time factors (e.g. number of CPUs).
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><varname>nr-write-queues=</varname></term>
<listitem>
<para>
Takes a value in the range 1...N. Adds additional
queues that will be used for write I/O.
</para>
<para>Note: This parameter is identical to that provided by nvme-cli.</para>
<para>
Default: Depends on kernel and other run
time factors (e.g. number of CPUs).
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><varname>nr-poll-queues=</varname></term>
<listitem>
<para>
Takes a value in the range 1...N. Adds additional
queues that will be used for polling latency
sensitive I/O.
</para>
<para>Note: This parameter is identical to that provided by nvme-cli.</para>
<para>
Default: Depends on kernel and other run
time factors (e.g. number of CPUs).
</para>
</listitem>
</varlistentry>
<xi:include href="standard-conf.xml" xpointer="queue-size"/>
<xi:include href="standard-conf.xml" xpointer="reconnect-delay"/>
<xi:include href="standard-conf.xml" xpointer="ctrl-loss-tmo"/>
<xi:include href="standard-conf.xml" xpointer="disable-sqflow"/>
<varlistentry>
<term><varname>ignore-iface=</varname></term>
<listitem>
<para>
Takes a boolean argument. This option controls how
connections with I/O Controllers (IOC) are made.
</para>
<para>
There is no guarantee that there will be a route to
reach that IOC. However, we can use the socket
option SO_BINDTODEVICE to force the connection to be
made on a specific interface instead of letting the
routing tables decide where to make the connection.
</para>
<para>
This option determines whether <code>stacd</code> will use
SO_BINDTODEVICE to force connections on an interface
or just rely on the routing tables. The default is
to use SO_BINDTODEVICE, in other words, <code>stacd</code> does
not ignore the interface.
</para>
<para>
BACKGROUND:
By default, <code>stacd</code> will connect to IOCs on the same
interface that was used to retrieve the discovery
log pages. If stafd discovers a DC on an interface
using mDNS, and stafd connects to that DC and
retrieves the log pages, it is expected that the
storage subsystems listed in the log pages are
reachable on the same interface where the DC was
discovered.
</para>
<para>
For example, let's say a DC is discovered on
interface ens102. Then all the subsystems listed in
the log pages retrieved from that DC must be
reachable on interface ens102. If this doesn't work,
for example you cannot "ping -I ens102 [storage-ip]",
then the most likely explanation is that proxy arp
is not enabled on the switch that the host is
connected to on interface ens102. Whatever you do,
resist the temptation to manually set up the routing
tables or to add alternate routes going over a
different interface than the one where the DC is
located. That simply won't work. Make sure proxy arp
is enabled on the switch first.
</para>
<para>
Setting routes won't work because, by default, <code>stacd</code>
uses the SO_BINDTODEVICE socket option when it
connects to IOCs. This option is used to force a
socket connection to be made on a specific interface
instead of letting the routing tables decide where
to connect the socket. Even if you were to manually
configure an alternate route on a different interface,
the connections (i.e. host to IOC) will still be
made on the interface where the DC was discovered by
stafd.
</para>
<para>
Defaults to <parameter>false</parameter>.
</para>
</listitem>
</varlistentry>
</variablelist>
</refsect2>
<refsect2>
<title>[I/O controller connection management] section</title>
<para>
Connectivity between hosts and subsystems in a fabric is
controlled by Fabric Zoning. Entities that share a common
zone (i.e., are zoned together) are allowed to discover each
other and establish connections between them. Fabric Zoning is
configured on Discovery Controllers (DC). Users can add/remove
controllers and/or hosts to/from zones.
</para>
<para>
Hosts have no direct knowledge of the Fabric Zoning configuration
that is active on a given DC. As a result, if a host is impacted
by a Fabric Zoning configuration change, it will be notified of
the connectivity configuration change by the DC via Asynchronous
Event Notifications (AEN).
</para>
<table frame='all'>
<title>List of terms used in this section:</title>
<tgroup cols="2" align='left' colsep='1' rowsep='1'>
<thead>
<row>
<entry>Term</entry>
<entry>Description</entry>
</row>
</thead>
<tbody>
<row>
<entry>AEN</entry>
<entry>Asynchronous Event Notification. A CQE (Completion Queue Entry) for an Asynchronous Event Request that was previously transmitted by the host to a Discovery Controller. AENs are used by DCs to notify hosts that a change (e.g., a connectivity configuration change) has occurred.</entry>
</row>
<row>
<entry>DC</entry>
<entry>Discovery Controller.</entry>
</row>
<row>
<entry>DLP</entry>
<entry>Discovery Log Page. A host will issue a Get Log Page command to retrieve the list of controllers it may connect to.</entry>
</row>
<row>
<entry>DLPE</entry>
<entry><simpara>
Discovery Log Page Entry. The response
to a Get Log Page command contains a list of DLPEs identifying
each controller that the host is allowed to connect with.
</simpara><simpara>
Note that DLPEs may contain both I/O Controllers (IOCs)
and Discovery Controllers (DCs). DCs listed in DLPEs
are called referrals. <code>stacd</code> only deals with IOCs.
Referrals (DCs) are handled by <code>stafd</code>.
</simpara>
</entry>
</row>
<row>
<entry>IOC</entry>
<entry>I/O Controller.</entry>
</row>
<row>
<entry>Manual Config</entry>
<entry>Refers to manually adding entries to <filename>stacd.conf</filename> with the <varname>controller=</varname> parameter.</entry>
</row>
<row>
<entry>Automatic Config</entry>
<entry>Refers to receiving configuration from a DC as DLPEs</entry>
</row>
<row>
<entry>External Config</entry>
<entry>Refers to configuration done outside of the <code>nvme-stas</code> framework, for example using <code>nvme-cli</code> commands</entry>
</row>
</tbody>
</tgroup>
</table>
<para>
DCs notify hosts of connectivity configuration changes by sending
AENs indicating a "Discovery Log" change. The host uses these AENs as
a trigger to issue a Get Log Page command. The response to this command
is used to update the list of DLPEs containing the controllers
the host is allowed to access.
Upon reception of the current DLPEs, the host will determine
whether DLPEs were added and/or removed, which will trigger the
addition and/or removal of controller connections. This happens in real time
and may affect active connections to controllers including controllers
that support I/O operations (IOCs). A host that was previously
connected to an IOC may suddenly be told that it is no longer
allowed to connect to that IOC and should disconnect from it.
</para>
<formalpara><title>IOC connection creation</title>
<para>
There are 3 ways to configure IOC connections on a host:
</para>
<orderedlist>
<listitem>
<para>
Manual Config by adding <varname>controller=</varname> entries
to the <literal>[Controllers]</literal> section (see below).
</para>
</listitem>
<listitem>
<para>
Automatic Config received in the form of
DLPEs from a remote DC.
</para>
</listitem>
<listitem>
<para>
External Config using <code>nvme-cli</code> (e.g. "<code>nvme connect</code>")
</para>
</listitem>
</orderedlist>
</formalpara>
<formalpara><title>IOC connection removal/prevention</title>
<para>
There are 3 ways to remove (or prevent) connections to an IOC:
</para>
<orderedlist>
<listitem>
<para>
Manual Config.
<orderedlist numeration='lowerroman'>
<listitem>
<para>
by adding <varname>exclude=</varname> entries to
the <literal>[Controllers]</literal> section (see below).
</para>
</listitem>
<listitem>
<para>
by removing <varname>controller=</varname> entries
from the <literal>[Controllers]</literal> section.
</para>
</listitem>
</orderedlist>
</para>
</listitem>
<listitem>
<para>
Automatic Config. As explained above, a host gets a
new list of DLPEs upon connectivity configuration
changes. On DLPE removal, the host should remove the
connection to the IOC matching that DLPE. This
behavior is configurable using the
<varname>disconnect-scope=</varname> parameter
described below.
</para>
</listitem>
<listitem>
<para>
External Config using <code>nvme-cli</code> (e.g. "<code>nvme
disconnect</code>" or "<code>nvme disconnect-all</code>")
</para>
</listitem>
</orderedlist>
</formalpara>
<para>
The decision by the host to automatically disconnect from an
IOC following connectivity configuration changes is controlled
by two parameters: <code>disconnect-scope</code>
and <code>disconnect-trtypes</code>.
</para>
<variablelist>
<varlistentry>
<term><varname>disconnect-scope=</varname></term>
<listitem>
<para>
Takes one of: <parameter>only-stas-connections</parameter>,
<parameter>all-connections-matching-disconnect-trtypes</parameter>, or <parameter>no-disconnect</parameter>.
</para>
<para>
In theory, hosts should only connect to IOCs that have
been zoned for them. Connections to IOCs that a host
is not zoned to have access to should simply not exist.
In practice, however, users may not want hosts to
disconnect from all IOCs in reaction to connectivity
configuration changes (or at least for some of the IOC
connections).
</para>
<para>
Some users may prefer for IOC connections to be "sticky"
and only be removed manually (<code>nvme-cli</code> or
<varname>exclude=</varname>) or removed by a system
reboot. Specifically, they don't want IOC connections
to be removed unexpectedly on DLPE removal. These users
may want to set <varname>disconnect-scope</varname>
to <parameter>no-disconnect</parameter>.
</para>
<para>
It is important to note that when IOC connections
are removed, ongoing I/O transactions will be
terminated immediately. There is no way to tell what
happens to the data being exchanged when such an abrupt
termination happens. If a host was in the middle of writing
to a storage subsystem, there is a chance that outstanding
I/O operations may not successfully complete.
</para>
<refsect3>
<title>Values:</title>
<variablelist>
<varlistentry>
<term><parameter>only-stas-connections</parameter></term>
<listitem>
<para>
Only remove connections previously made by <code>stacd</code>.
</para>
<para>
In this mode, when a DLPE is removed as a result of
connectivity configuration changes, the corresponding
IOC connection will be removed by <code>stacd</code>.
</para>
<para>
Connections to IOCs made externally, e.g. using <code>nvme-cli</code>,
will not be affected, unless they happen to be duplicates
of connections made by <code>stacd</code>. It's simply not
possible for <code>stacd</code> to tell that a connection
was previously made with <code>nvme-cli</code> (or any other external tool).
So, it's good practice to avoid duplicating
configuration between <code>stacd</code> and external tools.
</para>
<para>
Users wanting to persist some of their IOC connections
regardless of connectivity configuration changes should not use
<code>nvme-cli</code> to make those connections. Instead,
they should hard-code them in <filename>stacd.conf</filename>
with the <varname>controller=</varname> parameter. Using the
<varname>controller=</varname> parameter is the only way for a user
to tell <code>stacd</code> that a connection must be made and
not be deleted "<emphasis>no-matter-what</emphasis>".
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><parameter>all-connections-matching-disconnect-trtypes</parameter></term>
<listitem>
<para>
All connections that match the transport type specified by
<varname>disconnect-trtypes=</varname>, whether they were
made automatically by <code>stacd</code> or externally
(e.g., <code>nvme-cli</code>), will be audited and are
subject to removal on DLPE removal.
</para>
<para>
In this mode, as DLPEs are removed as a result of
connectivity configuration changes, the corresponding
IOC connections will be removed by the host immediately
whether they were made by <code>stacd</code>, <code>nvme-cli</code>,
or any other way. Basically, <code>stacd</code> audits
<emphasis>all</emphasis> IOC connections matching the
transport type specified by <varname>disconnect-trtypes=</varname>.
</para>
<formalpara><title><emphasis>NOTE</emphasis></title>
<para>
This mode implies that <code>stacd</code> will
only allow Manually Configured or Automatically
Configured IOC connections to exist. Externally
Configured connections using <code>nvme-cli</code>
(or other external mechanism)
that do not match any Manual Config
(<filename>stacd.conf</filename>)
or Automatic Config (DLPEs) will get deleted
immediately by <code>stacd</code>.
</para>
</formalpara>
</listitem>
</varlistentry>
<varlistentry>
<term><parameter>no-disconnect</parameter></term>
<listitem>
<para>
<code>stacd</code> does not disconnect from IOCs
when a DPLE is removed or a <varname>controller=</varname>
entry is removed from <filename>stacd.conf</filename>.
All IOC connections are "sticky".
</para>
<para>
Instead, users can remove connections
by issuing the <code>nvme-cli</code>
command "<code>nvme disconnect</code>", add an
<varname>exclude=</varname> entry to
<filename>stacd.conf</filename>, or wait
until the next system reboot at which time all
connections will be removed.
</para>
</listitem>
</varlistentry>
</variablelist>
</refsect3>
<para>
Defaults to <parameter>only-stas-connections</parameter>.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><varname>disconnect-trtypes=</varname></term>
<listitem>
<para>
This parameter only applies when <varname>disconnect-scope</varname>
is set to <parameter>all-connections-matching-disconnect-trtypes</parameter>.
It limits the scope of the audit to specific transport types.
</para>
<para>
Can take the values <parameter>tcp</parameter>,
<parameter>rdma</parameter>, <parameter>fc</parameter>, or
a combination thereof by separating them with a plus (+) sign.
For example: <parameter>tcp+fc</parameter>. No spaces
are allowed between values and the plus (+) sign.
</para>
<refsect3>
<title>Values:</title>
<variablelist>
<varlistentry>
<term><parameter>tcp</parameter></term>
<listitem>
<para>
Audit TCP connections.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><parameter>rdma</parameter></term>
<listitem>
<para>
Audit RDMA connections.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><parameter>fc</parameter></term>
<listitem>
<para>
Audit Fibre Channel connections.
</para>
</listitem>
</varlistentry>
</variablelist>
</refsect3>
<para>
Defaults to <parameter>tcp</parameter>.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><varname>connect-attempts-on-ncc=</varname></term>
<listitem>
<para>
The NCC bit (Not Connected to CDC) is a bit returned
by the CDC in the EFLAGS field of the DLPE. Only CDCs
will set the NCC bit. DDCs will always clear NCC to
0. The NCC bit is a way for the CDC to let hosts
know that the subsystem is currently not reachable
by the CDC. This may indicate that the subsystem is
currently down or that there is an outage on the
section of the network connecting the CDC to the
subsystem.
</para>
<para>
If a host is currently failing to connect to an I/O
controller and if the NCC bit associated with that
I/O controller is asserted, the host can decide to
stop trying to connect to that subsystem until
connectivity is restored. This will be indicated by
the CDC when it clears the NCC bit.
</para>
<para>
The parameter <varname>connect-attempts-on-ncc=</varname>
controls whether <code>stacd</code> will take the
NCC bit into account when attempting to connect to
an I/O Controller. Setting <varname>connect-attempts-on-ncc=</varname>
to 0 means that <code>stacd</code> will ignore
the NCC bit and will keep trying to connect. Setting
<varname>connect-attempts-on-ncc=</varname> to a
non-zero value indicates the number of connection
attempts that will be made before <code>stacd</code>
gives up trying. Note that this value should be set
to a value greater than 1. In fact, when set to 1,
<code>stacd</code> will automatically use 2 instead.
The reason for this is simple. It is possible that a
first connect attempt may fail.
</para>
<para>
Defaults to <parameter>0</parameter>.
</para>
</listitem>
</varlistentry>
</variablelist>
</refsect2>
<xi:include href="standard-conf.xml" xpointer="controller"/>
</refsect1>
<refsect1>
<title>See Also</title>
<para>
<citerefentry>
<refentrytitle>stacd</refentrytitle>
<manvolnum>8</manvolnum>
</citerefentry>
</para>
</refsect1>
</refentry>

74
doc/stacd.service.xml Normal file
View file

@ -0,0 +1,74 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--*-nxml-*-->
<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
<!--
SPDX-License-Identifier: Apache-2.0
Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
-->
<refentry id="stacd.service">
<refentryinfo>
<title>stacd.service</title>
<productname>nvme-stas</productname>
<author>
<personname>
<honorific>Mr</honorific>
<firstname>Martin</firstname>
<surname>Belanger</surname>
</personname>
<affiliation>
<orgname>Dell, Inc.</orgname>
</affiliation>
</author>
</refentryinfo>
<refmeta>
<refentrytitle>stacd.service</refentrytitle>
<manvolnum>8</manvolnum>
</refmeta>
<refnamediv>
<refname>stacd.service</refname>
<refpurpose>Systemd unit file for the stacd service</refpurpose>
</refnamediv>
<refsynopsisdiv>
<para>
<filename>/usr/lib/systemd/system/stacd.service</filename>
</para>
</refsynopsisdiv>
<refsect1>
<title>Description</title>
<para>
<citerefentry>
<refentrytitle>stacd</refentrytitle>
<manvolnum>8</manvolnum>
</citerefentry>
is a system service used to automatically connect to I/O controllers
discovered by <citerefentry>
<refentrytitle>stafd</refentrytitle>
<manvolnum>8</manvolnum>
</citerefentry>.
</para>
</refsect1>
<refsect1>
<title>See Also</title>
<para>
<citerefentry>
<refentrytitle>stacd</refentrytitle>
<manvolnum>8</manvolnum>
</citerefentry>,
<citerefentry>
<refentrytitle>stafd</refentrytitle>
<manvolnum>8</manvolnum>
</citerefentry>,
<citerefentry>
<refentrytitle>stas-config.target</refentrytitle>
<manvolnum>8</manvolnum>
</citerefentry>
</para>
</refsect1>
</refentry>

229
doc/stacd.xml Normal file
View file

@ -0,0 +1,229 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd" [
<!ENTITY daemon "stacd">
<!ENTITY deamondesc "STorage Appliance Connector">
<!ENTITY control "stacctl">
]>
<!--
SPDX-License-Identifier: Apache-2.0
Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
-->
<refentry id="&daemon;" xmlns:xi="http://www.w3.org/2001/XInclude">
<refentryinfo>
<title>&daemon;</title>
<productname>nvme-stas</productname>
<author>
<personname>
<honorific>Mr</honorific>
<firstname>Martin</firstname>
<surname>Belanger</surname>
</personname>
<affiliation>
<orgname>Dell, Inc.</orgname>
</affiliation>
</author>
</refentryinfo>
<refmeta>
<refentrytitle>&daemon;</refentrytitle>
<manvolnum>8</manvolnum>
</refmeta>
<refnamediv>
<refname>&daemon;</refname>
<refpurpose>&deamondesc;</refpurpose>
</refnamediv>
<refsynopsisdiv>
<cmdsynopsis>
<command>&daemon;</command>
<arg choice="opt" rep="repeat">OPTIONS</arg>
</cmdsynopsis>
</refsynopsisdiv>
<refsect1>
<title>Description</title>
<para>
<command>&daemon;</command>
is a system daemon that can be used to automatically connect to
NVMe-oF I/O Controllers using the discovery log pages collected by
<citerefentry>
<refentrytitle>stafd</refentrytitle>
<manvolnum>8</manvolnum>
</citerefentry>. It can also be manually configured with
<citerefentry>
<refentrytitle>&daemon;.conf</refentrytitle>
<manvolnum>5</manvolnum>
</citerefentry>
to connect to I/O Controllers that otherwise cannot be found
automatically.
</para>
</refsect1>
<refsect1>
<title>Options</title>
<para>The following options are understood:</para>
<variablelist>
<xi:include href="standard-options.xml" xpointer="help"/>
<xi:include href="standard-options.xml" xpointer="version"/>
</variablelist>
<varlistentry>
<term><option>-fFILE</option></term>
<term><option>--conf-file=FILE</option></term>
<listitem>
<para>
Specify a different configuration file than
<citerefentry>
<refentrytitle>&daemon;.conf</refentrytitle>
<manvolnum>5</manvolnum>
</citerefentry>
(default: <filename>/etc/stas/&daemon;.conf</filename>).
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><option>-s</option></term>
<term><option>--syslog</option></term>
<listitem>
<para>
Send messages to syslog instead of stdout. Use this when
running &daemon; as a daemon. (default: <literal>false</literal>).
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><option>--tron</option></term>
<listitem>
<para>Trace ON. (default: <literal>false</literal>)</para>
</listitem>
</varlistentry>
<varlistentry>
<term><option>--idl=FILE</option></term>
<listitem>
<para>Print D-Bus IDL to FILE and exit.</para>
</listitem>
</varlistentry>
</refsect1>
<refsect1>
<title>Exit status</title>
<para>
On success, 0 is returned, a non-zero failure code otherwise.
</para>
</refsect1>
<refsect1>
<title>Daemonization</title>
<para>
&daemon; is managed by <code>systemd</code>. The following
operations are supported:
</para>
<table align='left' frame='all'>
<tgroup cols="2" align='left' colsep='1' rowsep='1'>
<thead>
<row>
<entry>Command</entry>
<entry>Description</entry>
</row>
</thead>
<tbody>
<row>
<entry><programlisting>$ systemctl start &daemon; </programlisting></entry>
<entry>Start daemon.</entry>
</row>
<row>
<entry><programlisting>$ systemctl stop &daemon; </programlisting></entry>
<entry>Stop daemon. The <code>SIGTERM</code> signal is used to tell the daemon to stop.</entry>
</row>
<row>
<entry><programlisting>$ systemctl restart &daemon; </programlisting></entry>
<entry>Effectively a <code>stop</code> + <code>start</code>.</entry>
</row>
<row>
<entry><programlisting>$ systemctl reload &daemon; </programlisting></entry>
<entry>Reload configuration. This is done in real time without restarting the daemon. The <code>SIGHUP</code> signal is used to tell the daemon to reload its configuration file. Note that configuration parameters that affect connections (e.g. <code>kato</code>), will not apply to existing connections. Only connections established after the configuration was changed will utilize the new configuration parameters.</entry>
</row>
</tbody>
</tgroup>
</table>
</refsect1>
<refsect1>
<title>Design</title>
<para>
<command>&daemon;</command> use the <code>GLib</code> main loop.
The <code>GLib</code> Python module provides several low-level
building blocks that <command>&daemon;</command> requires. In
addition, many Python modules "play nice" with <code>GLib</code>
such as <code>dasbus</code> (D-Bus package) and <code>pyudev</code>
(UDev package). <code>GLib</code> also provides additional components
such as timers, signal handlers, and much more.
</para>
</refsect1>
<refsect1>
<title>Configuration</title>
<para>
<command>&daemon;</command> can automatically set up the I/O
connections to discovered storage subsystems. However,
<command>&daemon;</command> can also operate in a non-automatic
mode based on manually entered configuration. In other words,
storage subsystems can be entered in a configuration file named
<filename>/etc/stas/&daemon;.conf</filename>.
This configuration file also provides additional parameters,
as log-level attributes used for debugging purposes.
</para>
</refsect1>
<refsect1>
<title>D-Bus API</title>
<para>
The interface to <command>&daemon;</command> is D-Bus.
This allows other programs, such as <command>&control;</command>,
to communicate with <command>&daemon;</command>. The D-Bus address
is <code>org.nvmexpress.stac</code>.
</para>
</refsect1>
<refsect1>
<title>See Also</title>
<para>
<citerefentry>
<refentrytitle>&daemon;.conf</refentrytitle>
<manvolnum>5</manvolnum>
</citerefentry>,
<citerefentry>
<refentrytitle>&daemon;.service</refentrytitle>
<manvolnum>8</manvolnum>
</citerefentry>,
<citerefentry>
<refentrytitle>stacctl</refentrytitle>
<manvolnum>1</manvolnum>
</citerefentry>,
<citerefentry>
<refentrytitle>stafd</refentrytitle>
<manvolnum>8</manvolnum>
</citerefentry>,
<citerefentry>
<refentrytitle>org.nvmexpress.stac</refentrytitle>
<manvolnum>5</manvolnum>
</citerefentry>.
</para>
</refsect1>
</refentry>

205
doc/stafctl.xml Normal file
View file

@ -0,0 +1,205 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--*-nxml-*-->
<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
<!--
SPDX-License-Identifier: Apache-2.0
Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
-->
<refentry id="stafctl" xmlns:xi="http://www.w3.org/2001/XInclude">
<refentryinfo>
<title>stafctl</title>
<productname>nvme-stas</productname>
<author>
<personname>
<honorific>Mr</honorific>
<firstname>Martin</firstname>
<surname>Belanger</surname>
</personname>
<affiliation>
<orgname>Dell, Inc.</orgname>
</affiliation>
</author>
</refentryinfo>
<refmeta>
<refentrytitle>stafctl</refentrytitle>
<manvolnum>1</manvolnum>
</refmeta>
<refnamediv>
<refname>stafctl</refname>
<refpurpose>STorage Appliance Finder (STAF) utility program</refpurpose>
</refnamediv>
<refsynopsisdiv>
<cmdsynopsis>
<command>stafctl</command>
<arg choice="opt" rep="repeat">OPTIONS</arg>
<arg choice="req">COMMAND</arg>
<arg choice="opt" rep="repeat">OPTIONS</arg>
</cmdsynopsis>
</refsynopsisdiv>
<refsect1>
<title>Description</title>
<para>
<command>stafctl</command> is a tool that can be used to communicate
with the <citerefentry><refentrytitle>stafd</refentrytitle>
<manvolnum>8</manvolnum></citerefentry> daemon to retrieve
operational data.
</para>
</refsect1>
<refsect1>
<title>Commands</title>
<para>The following commands are understood:</para>
<variablelist>
<xi:include href="standard-options.xml" xpointer="tron"/>
<xi:include href="standard-options.xml" xpointer="troff"/>
<xi:include href="standard-options.xml" xpointer="status"/>
<varlistentry>
<term><command>ls</command></term>
<listitem>
<para>
Show the list of discovery controllers. This will list
all the controllers configured in
<citerefentry><refentrytitle>stafd.conf</refentrytitle>
<manvolnum>5</manvolnum></citerefentry> as well as those
discovered with mDNS service discovery.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><command>dlp</command></term>
<listitem>
<para>Show discovery log pages.</para>
</listitem>
</varlistentry>
<varlistentry>
<term><command>adlp</command></term>
<listitem>
<para>Show all discovery log pages.</para>
</listitem>
</varlistentry>
</variablelist>
</refsect1>
<refsect1>
<title>Options</title>
<para>The following options are understood:</para>
<variablelist>
<xi:include href="standard-options.xml" xpointer="help"/>
<xi:include href="standard-options.xml" xpointer="version"/>
<xi:include href="standard-options.xml" xpointer="detailed"/>
<xi:include href="standard-options.xml" xpointer="transport"/>
<xi:include href="standard-options.xml" xpointer="traddr"/>
<xi:include href="standard-options.xml" xpointer="trsvcid"/>
<xi:include href="standard-options.xml" xpointer="host-traddr"/>
<xi:include href="standard-options.xml" xpointer="host-iface"/>
<xi:include href="standard-options.xml" xpointer="nqn"/>
</variablelist>
</refsect1>
<refsect1>
<title>Values</title>
<variablelist>
<xi:include href="standard-options.xml" xpointer="TRTYPE-value"/>
<xi:include href="standard-options.xml" xpointer="TRADDR-value"/>
<xi:include href="standard-options.xml" xpointer="TRSVCID-value"/>
<xi:include href="standard-options.xml" xpointer="IFACE-value"/>
<xi:include href="standard-options.xml" xpointer="NQN-value"/>
</variablelist>
</refsect1>
<refsect1>
<title>Exit status</title>
<para>
On success, 0 is returned; otherwise, a non-zero failure code is
returned.
</para>
</refsect1>
<refsect1>
<title>Examples</title>
<example>
<title>List all the discovery controllers</title>
<programlisting>$ stafctl ls
[{'device': 'nvme0',
'host-iface': '',
'host-traddr': '',
'subsysnqn': 'nqn.2014-08.org.nvmexpress.discovery',
'traddr': '::1',
'transport': 'tcp',
'trsvcid': '8009'}]</programlisting>
</example>
<example>
<title>Enable tracing</title>
<programlisting>$ stafctl tron</programlisting>
</example>
<example>
<title>
Show discovery log pages from a specific discovery controller
</title>
<programlisting>$ stafctl dlp --transport tcp --traddr ::1 --trsvcid 8009
[{'adrfam': 'ipv6',
'asqsz': '32',
'cntlid': '65535',
'portid': '1',
'subnqn': 'nqn.1988-11.com.dell:PowerSANxxx:01:20210225100113-454f73093ceb4847a7bdfc6e34ae8e28',
'subtype': 'nvme',
'traddr': '::1',
'treq': 'disable sqflow',
'trsvcid': '8009',
'trtype': 'tcp'},
{'adrfam': 'ipv6',
'asqsz': '32',
'cntlid': '65535',
'portid': '1',
'subnqn': 'nqn.1988-11.com.dell:PowerSANxxx:01:20210225100113-454f73093ceb4847a7bdfc6e34aedead',
'subtype': 'nvme',
'traddr': '::1',
'treq': 'disable sqflow',
'trsvcid': '8009',
'trtype': 'tcp'},
{'adrfam': 'ipv6',
'asqsz': '32',
'cntlid': '65535',
'portid': '1',
'subnqn': 'nqn.1988-11.com.dell:PowerSANxxx:01:20210225100113-454f73093ceb4847a7bdfc6e34aebeef',
'subtype': 'nvme',
'traddr': '::1',
'treq': 'disable sqflow',
'trsvcid': '8009',
'trtype': 'tcp'}]</programlisting>
</example>
</refsect1>
<refsect1>
<title>See Also</title>
<para>
<citerefentry>
<refentrytitle>stafd.conf</refentrytitle>
<manvolnum>5</manvolnum>
</citerefentry>,
<citerefentry>
<refentrytitle>stafd</refentrytitle>
<manvolnum>8</manvolnum>
</citerefentry>
<citerefentry project="man-pages"/>
</para>
</refsect1>
</refentry>

280
doc/stafd.conf.xml Normal file
View file

@ -0,0 +1,280 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--*-nxml-*-->
<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
<!--
SPDX-License-Identifier: Apache-2.0
Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
-->
<refentry id="stafd.conf" xmlns:xi="http://www.w3.org/2001/XInclude">
<refentryinfo>
<title>stafd.conf</title>
<productname>nvme-stas</productname>
<author>
<personname>
<honorific>Mr</honorific>
<firstname>Martin</firstname>
<surname>Belanger</surname>
</personname>
<affiliation>
<orgname>Dell, Inc.</orgname>
</affiliation>
</author>
</refentryinfo>
<refmeta>
<refentrytitle>stafd.conf</refentrytitle>
<manvolnum>5</manvolnum>
</refmeta>
<refnamediv>
<refname>stafd.conf</refname>
<refpurpose>
<citerefentry project="man-pages">
<refentrytitle>stafd</refentrytitle>
<manvolnum>8</manvolnum>
</citerefentry>
configuration file
</refpurpose>
</refnamediv>
<refsynopsisdiv>
<para>
<filename>/etc/stas/stafd.conf</filename>
</para>
</refsynopsisdiv>
<refsect1>
<title>Description</title>
<para>
When <citerefentry project="man-pages"><refentrytitle>stafd</refentrytitle>
<manvolnum>8</manvolnum></citerefentry> starts up, it reads its
configuration from <filename>stafd.conf</filename>.
</para>
</refsect1>
<refsect1>
<title>Configuration File Format</title>
<para>
<filename>stafd.conf</filename> is a plain text file divided into
sections, with configuration entries in the style
<replaceable>key</replaceable>=<replaceable>value</replaceable>.
Spaces immediately before or after the <literal>=</literal> are
ignored. Empty lines are ignored as well as lines starting with
<literal>#</literal>, which may be used for commenting.
</para>
</refsect1>
<refsect1>
<title>Options</title>
<refsect2>
<title>[Global] section</title>
<para>
The following options are available in the
<literal>[Global]</literal> section:
</para>
<variablelist>
<xi:include href="standard-conf.xml" xpointer="tron"/>
<xi:include href="standard-conf.xml" xpointer="hdr-digest"/>
<xi:include href="standard-conf.xml" xpointer="data-digest"/>
<xi:include href="standard-conf.xml" xpointer="kato"/>
<xi:include href="standard-conf.xml" xpointer="ip-family"/>
<xi:include href="standard-conf.xml" xpointer="queue-size"/>
<xi:include href="standard-conf.xml" xpointer="reconnect-delay"/>
<xi:include href="standard-conf.xml" xpointer="ctrl-loss-tmo"/>
<xi:include href="standard-conf.xml" xpointer="disable-sqflow"/>
<varlistentry>
<term><varname>ignore-iface=</varname></term>
<listitem>
<para>
Takes a boolean argument. This option controls how
connections with Discovery Controllers (DC) are made.
</para>
<para>
DCs are automatically discovered using DNS-SD/mDNS.
mDNS provides the DC's IP address and the interface
on which the DC was discovered.
</para>
<para>
There is no guarantee that there will be a route to
reach that DC. However, we can use the socket option
SO_BINDTODEVICE to force the connection to be made
on a specific interface instead of letting the
routing tables decide where to make the connection.
</para>
<para>
This option determines whether <code>stafd</code>
will use SO_BINDTODEVICE to force connections on an
interface or just rely on the routing tables. The
default is to use SO_BINDTODEVICE, in other words,
<code>stafd</code> does not ignore the interface by
default.
</para>
<para>
Defaults to <parameter>false</parameter>.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><varname>pleo=</varname></term>
<listitem>
<para>
Port Local Entries Only. Takes a string argument
<parameter>enabled</parameter> or
<parameter>disabled</parameter>. This option is sent in
the LSP field (Log SPecific) of the Get Discovery Log
Page (DLP) command. It is used by <code>stafd</code> to
tell Discovery Controllers (DC) whether the response to
a Get DLP command should contain all the NVM subsystems
or only those reachable by the host on the interface
where the Get DLP command was issued by the host.
</para>
<para>
This parameter was introduced in TP8010. When
<varname>pleo=</varname><parameter>enabled</parameter>,
then the DC shall return records for only NVM subsystem
ports that are presented through the same NVM subsystem
port that received the Get Log Page command. When
<varname>pleo=</varname><parameter>disabled</parameter>,
then the DC may return all the NVM subsystem ports
that it holds, even those that can only be reached
on NVM subsystem ports that did not receive the Get
Log Page command. In other words, the host may not
even be able to reach those subsystems.
</para>
<para>
Defaults to <parameter>enabled</parameter>.
</para>
</listitem>
</varlistentry>
</variablelist>
</refsect2>
<refsect2>
<title>[Service Discovery] section</title>
<para>
The following options are available in the
<literal>[Service Discovery]</literal> section:
</para>
<variablelist>
<varlistentry>
<term><varname>zeroconf=</varname></term>
<listitem>
<para>
Enable zeroconf provisioning using DNS-SD/mDNS.
Takes a string argument <parameter>enabled</parameter> or
<parameter>disabled</parameter>.
</para>
<para>
When <parameter>enabled</parameter>, the default,
<code>stafd</code> makes a request with the
Avahi daemon to locate Discovery Controllers using
DNS-SD/mDNS.
</para>
<para>
Discovery Controllers that support zeroconf advertize
themselves over mDNS with the service type
<literal>_nvme-disc._tcp</literal>.
</para>
<para>
Defaults to <parameter>true</parameter>.
</para>
</listitem>
</varlistentry>
</variablelist>
</refsect2>
<refsect2>
<title>[Discovery controller connection management] section</title>
<para>
The following options are available in the
<literal>[Discovery controller connection management]</literal> section:
</para>
<varlistentry>
<term><varname>persistent-connections=</varname></term>
<listitem>
<para>
Takes a boolean argument. Whether connections to
Discovery Controllers (DC) are persistent. When
true, connections initiated by stafd will persists
even when stafd is stopped. When
<parameter>false</parameter>, <code>stafd</code>
will disconnect from all DCs it is connected to on
exit.
</para>
<para>
Defaults to <parameter>false</parameter>.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><varname>zeroconf-connections-persistence=</varname></term>
<listitem>
<para>
Takes a unit-less value in seconds, or a time span value
such as "72hours" or "5days". A value of 0 means no
persistence. In other words, configuration acquired through
zeroconf (mDNS service discovery) will be removed
immediately when mDNS no longer reports the presence of
a Discovery Controller (DC) and connectivity to that DC
is lost. A value of -1 means that configuration acquired
through zeroconf will persist forever.
</para>
<para>
This is used for the case where a DC that was discovered
through mDNS service discovery no longer advertises
itself through mDNS and can no longer be connected to.
For example, the DC had some catastrophic failure
(e.g. power surge) and needs to be replaced. In that
case, the connection to that DC can never be restored
and a replacement DC will be needed. The replacement
DC will likely have a different NQN (or IP address).
In that scenario, the host won't be able to determine
that the old DC is not coming back. It won't know either
that a newly discovered DC is really the replacement for
the old one. For that reason, the host needs a way to
"age" zeroconf-acquired configuration and remove it
automatically after a certain amount of time. This is
what this parameter is for.
</para>
<para>
Defaults to <parameter>72hours</parameter>.
</para>
</listitem>
</varlistentry>
</refsect2>
<xi:include href="standard-conf.xml" xpointer="controller"/>
</refsect1>
<refsect1>
<title>See Also</title>
<para>
<citerefentry>
<refentrytitle>stafd</refentrytitle>
<manvolnum>8</manvolnum>
</citerefentry>
</para>
</refsect1>
</refentry>

67
doc/stafd.service.xml Normal file
View file

@ -0,0 +1,67 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--*-nxml-*-->
<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
<!--
SPDX-License-Identifier: Apache-2.0
Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
-->
<refentry id="stafd.service">
<refentryinfo>
<title>stafd.service</title>
<productname>nvme-stas</productname>
<author>
<personname>
<honorific>Mr</honorific>
<firstname>Martin</firstname>
<surname>Belanger</surname>
</personname>
<affiliation>
<orgname>Dell, Inc.</orgname>
</affiliation>
</author>
</refentryinfo>
<refmeta>
<refentrytitle>stafd.service</refentrytitle>
<manvolnum>8</manvolnum>
</refmeta>
<refnamediv>
<refname>stafd.service</refname>
<refpurpose>Systemd unit file for the stafd service</refpurpose>
</refnamediv>
<refsynopsisdiv>
<para>
<filename>/usr/lib/systemd/system/stafd.service</filename>
</para>
</refsynopsisdiv>
<refsect1>
<title>Description</title>
<para>
<citerefentry>
<refentrytitle>stafd</refentrytitle>
<manvolnum>8</manvolnum>
</citerefentry>
is a system service used to automatically locate NVMe-oF Discovery
Controllers using mDNS service discovery.
</para>
</refsect1>
<refsect1>
<title>See Also</title>
<para>
<citerefentry>
<refentrytitle>stafd</refentrytitle>
<manvolnum>8</manvolnum>
</citerefentry>,
<citerefentry>
<refentrytitle>stas-config.target</refentrytitle>
<manvolnum>8</manvolnum>
</citerefentry>
</para>
</refsect1>
</refentry>

237
doc/stafd.xml Normal file
View file

@ -0,0 +1,237 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd" [
<!ENTITY daemon "stafd">
<!ENTITY deamondesc "STorage Appliance Finder">
<!ENTITY control "stafctl">
]>
<!--
SPDX-License-Identifier: Apache-2.0
Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
-->
<refentry id="&daemon;" xmlns:xi="http://www.w3.org/2001/XInclude">
<refentryinfo>
<title>&daemon;</title>
<productname>nvme-stas</productname>
<author>
<personname>
<honorific>Mr</honorific>
<firstname>Martin</firstname>
<surname>Belanger</surname>
</personname>
<affiliation>
<orgname>Dell, Inc.</orgname>
</affiliation>
</author>
</refentryinfo>
<refmeta>
<refentrytitle>&daemon;</refentrytitle>
<manvolnum>8</manvolnum>
</refmeta>
<refnamediv>
<refname>&daemon;</refname>
<refpurpose>&deamondesc;</refpurpose>
</refnamediv>
<refsynopsisdiv>
<cmdsynopsis>
<command>&daemon;</command>
<arg choice="opt" rep="repeat">OPTIONS</arg>
</cmdsynopsis>
</refsynopsisdiv>
<refsect1>
<title>Description</title>
<para>
<command>&daemon;</command>
is a system daemon that can be used to automatically locate and
connect to NVMe-oF Discovery Controllers using mDNS service discovery.
It can also be manually configured with
<citerefentry>
<refentrytitle>&daemon;.conf</refentrytitle>
<manvolnum>5</manvolnum>
</citerefentry>
to connect to Discovery Controllers that cannot be located using
mDNS.
</para>
</refsect1>
<refsect1>
<title>Options</title>
<para>The following options are understood:</para>
<variablelist>
<xi:include href="standard-options.xml" xpointer="help"/>
<xi:include href="standard-options.xml" xpointer="version"/>
</variablelist>
<varlistentry>
<term><option>-fFILE</option></term>
<term><option>--conf-file=FILE</option></term>
<listitem>
<para>
Specify a different configuration file than
<citerefentry>
<refentrytitle>&daemon;.conf</refentrytitle>
<manvolnum>5</manvolnum>
</citerefentry>
(default: <filename>/etc/stas/&daemon;.conf</filename>).
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><option>-s</option></term>
<term><option>--syslog</option></term>
<listitem>
<para>
Send messages to syslog instead of stdout. Use this when
running &daemon; as a daemon. (default: <literal>false</literal>).
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><option>--tron</option></term>
<listitem>
<para>Trace ON. (default: <literal>false</literal>)</para>
</listitem>
</varlistentry>
<varlistentry>
<term><option>--idl=FILE</option></term>
<listitem>
<para>Print D-Bus IDL to FILE and exit.</para>
</listitem>
</varlistentry>
</refsect1>
<refsect1>
<title>Exit status</title>
<para>
On success, 0 is returned, a non-zero failure code otherwise.
</para>
</refsect1>
<refsect1>
<title>Daemonization</title>
<para>
&daemon; is managed by <code>systemd</code>. The following
operations are supported:
</para>
<table frame='all'>
<tgroup cols="2" align='left' colsep='1' rowsep='1'>
<thead>
<row>
<entry>Command</entry>
<entry>Description</entry>
</row>
</thead>
<tbody>
<row>
<entry><programlisting>$ systemctl start &daemon; </programlisting></entry>
<entry>Start daemon.</entry>
</row>
<row>
<entry><programlisting>$ systemctl stop &daemon; </programlisting></entry>
<entry>Stop daemon. The <code>SIGTERM</code> signal is used to tell the daemon to stop.</entry>
</row>
<row>
<entry><programlisting>$ systemctl restart &daemon; </programlisting></entry>
<entry>Effectively a <code>stop</code> + <code>start</code>.</entry>
</row>
<row>
<entry><programlisting>$ systemctl reload &daemon; </programlisting></entry>
<entry>Reload configuration. This is done in real time without restarting the daemon. The <code>SIGHUP</code> signal is used to tell the daemon to reload its configuration file. Note that configuration parameters that affect connections (e.g. <code>kato</code>), will not apply to existing connections. Only connections established after the configuration was changed will utilize the new configuration parameters.</entry>
</row>
</tbody>
</tgroup>
</table>
</refsect1>
<refsect1>
<title>Design</title>
<para>
<command>&daemon;</command> use the <code>GLib</code> main loop.
The <code>GLib</code> Python module provides several low-level
building blocks that <command>&daemon;</command> requires. In
addition, many Python modules "play nice" with <code>GLib</code>
such as <code>dasbus</code> (D-Bus package) and <code>pyudev</code>
(UDev package). <code>GLib</code> also provides additional components
such as timers, signal handlers, and much more.
</para>
<para>
<command>&daemon;</command> connects to the <code>avahi-daemon</code>
using D-Bus. The <code>avahi-daemon</code>, or simply
<emphasis>Avahi</emphasis>, is an mDNS discovery service used for
zero-configuration networking (zeroconf). <command>&daemon;</command>
registers with Avahi to automatically locate Central Discovery
Controllers (CDC) and Direct Discovery Controllers (DDC). When Avahi
finds Discovery Controllers (DC), it notifies <command>&daemon;</command>
which connects to the DC with the help of the <code>libnvme</code> library.
Once a connection to a DC is established, <command>&daemon;</command>
can retrieve the <emphasis>discovery log pages</emphasis> from
that DC and cache them in memory.
</para>
</refsect1>
<refsect1>
<title>Configuration</title>
<para>
<command>&daemon;</command> can automatically locate discovery
controllers (DC) with the help of Avahi and connect to them. However,
<command>&daemon;</command> can also operate in a non-automatic
mode based on manually entered configuration. In other words,
DCs can be entered in a configuration named
<filename>/etc/stas/&daemon;.conf</filename>.
This configuration file also provides additional parameters, such
as log-level attributes used for debugging purposes.
</para>
</refsect1>
<refsect1>
<title>D-Bus API</title>
<para>
The interface to <command>&daemon;</command> is D-Bus.
This allows other programs, such as <command>&control;</command>,
to communicate with <command>&daemon;</command>. The D-Bus address
is <code>org.nvmexpress.staf</code>.
</para>
</refsect1>
<refsect1>
<title>See Also</title>
<para>
<citerefentry>
<refentrytitle>&daemon;.conf</refentrytitle>
<manvolnum>5</manvolnum>
</citerefentry>,
<citerefentry>
<refentrytitle>&daemon;.service</refentrytitle>
<manvolnum>8</manvolnum>
</citerefentry>,
<citerefentry>
<refentrytitle>stafctl</refentrytitle>
<manvolnum>1</manvolnum>
</citerefentry>,
<citerefentry>
<refentrytitle>org.nvmexpress.staf</refentrytitle>
<manvolnum>5</manvolnum>
</citerefentry>.
</para>
</refsect1>
</refentry>

562
doc/standard-conf.xml Normal file
View file

@ -0,0 +1,562 @@
<?xml version="1.0"?>
<!DOCTYPE refsection PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
<!--
SPDX-License-Identifier: Apache-2.0
Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
-->
<root>
<variablelist>
<varlistentry id='tron'>
<term><varname>tron=</varname></term>
<listitem id='tron-text'>
<para>
Trace ON. Takes a boolean argument. If <parameter>true</parameter>,
enables full code tracing. The trace will be displayed in
the system log such as systemd's journal. Defaults to
<parameter>false</parameter>.
</para>
</listitem>
</varlistentry>
<varlistentry id='hdr-digest'>
<term><varname>hdr-digest=</varname></term>
<listitem id='hdr-digest-text'>
<para>
Enable Protocol Data Unit (PDU) Header Digest. Takes a
boolean argument. NVMe/TCP facilitates an optional PDU
Header digest. Digests are calculated using the CRC32C
algorithm. If <parameter>true</parameter>, Header Digests
are inserted in PDUs and checked for errors. Defaults to
<parameter>false</parameter>.
</para>
</listitem>
</varlistentry>
<varlistentry id='data-digest'>
<term><varname>data-digest=</varname></term>
<listitem id='data-digest-text'>
<para>
Enable Protocol Data Unit (PDU) Data Digest. Takes a
boolean argument. NVMe/TCP facilitates an optional PDU
Data digest. Digests are calculated using the CRC32C
algorithm. If <parameter>true</parameter>, Data Digests
are inserted in PDUs and checked for errors. Defaults to
<parameter>false</parameter>.
</para>
</listitem>
</varlistentry>
<varlistentry id='kato'>
<term><varname>kato=</varname></term>
<listitem id='kato-text'>
<para>
Keep Alive Timeout (KATO) in seconds. Takes an unsigned
integer. This field specifies the timeout value for the Keep
Alive feature in seconds. Defaults to 30 seconds for
Discovery Controller connections and 120 seconds for I/O
Controller connections.
</para>
</listitem>
</varlistentry>
<varlistentry id='ip-family'>
<term><varname>ip-family=</varname></term>
<listitem id='ip-family-text'>
<para>
Takes a string argument. With this you can specify
whether IPv4, IPv6, or both are supported when
connecting to a Controller. Connections will not be
attempted to IP addresses (whether discovered or
manually configured with <varname>controller=</varname>)
disabled by this option. If an invalid value
is entered, then the default (see below) will apply.
</para>
<para>
Choices are <parameter>ipv4</parameter>, <parameter>ipv6</parameter>, or <parameter>ipv4+ipv6</parameter>.
</para>
<para>
Defaults to <parameter>ipv4+ipv6</parameter>.
</para>
</listitem>
</varlistentry>
<varlistentry id='queue-size'>
<term><varname>queue-size=</varname></term>
<listitem id='queue-size-text'>
<para>
Takes a value in the range 16...1024.
</para>
<para>
Overrides the default number of elements in the I/O queues
created by the driver. This option will be ignored for
discovery, but will be passed on to the subsequent connect
call.
</para>
<para>Note: This parameter is identical to that provided by nvme-cli.</para>
<para>
Defaults to <parameter>128</parameter>.
</para>
</listitem>
</varlistentry>
<varlistentry id='reconnect-delay'>
<term><varname>reconnect-delay=</varname></term>
<listitem id='reconnect-delay-text'>
<para>
Takes a value in the range 1 to N seconds.
</para>
<para>
Overrides the default delay before reconnect is attempted
after a connect loss.
</para>
<para>Note: This parameter is identical to that provided by nvme-cli.</para>
<para>
Defaults to <parameter>10</parameter>. Retry to connect every 10 seconds.
</para>
</listitem>
</varlistentry>
<varlistentry id='ctrl-loss-tmo'>
<term><varname>ctrl-loss-tmo=</varname></term>
<listitem id='ctrl-loss-tmo-text'>
<para>
Takes a value in the range -1, 0, ..., N seconds. -1 means
retry forever. 0 means do not retry.
</para>
<para>
Overrides the default controller loss timeout period (in seconds).
</para>
<para>Note: This parameter is identical to that provided by nvme-cli.</para>
<para>
Defaults to <parameter>600</parameter> seconds (10 minutes).
</para>
</listitem>
</varlistentry>
<varlistentry id='disable-sqflow'>
<term><varname>disable-sqflow=</varname></term>
<listitem id='disable-sqflow-text'>
<para>
Takes a boolean argument. Disables SQ flow control to omit
head doorbell update for submission queues when sending nvme
completions.
</para>
<para>Note: This parameter is identical to that provided by nvme-cli.</para>
<para>
Defaults to <parameter>false</parameter>.
</para>
</listitem>
</varlistentry>
</variablelist>
<refsect2 id='controller'>
<title>[Controllers] section</title>
<para>The following options are available in the
<literal>[Controllers]</literal> section:</para>
<varlistentry>
<term><varname>controller=</varname></term>
<listitem id='controller-text'>
<para>
Controllers are specified with the <varname>controller</varname>
option. This option may be specified more than once to specify
more than one controller. The format is one line per Controller
composed of a series of fields separated by semi-colons as follows:
</para>
<programlisting>controller=transport=[trtype];traddr=[traddr];trsvcid=[trsvcid];host-traddr=[traddr],host-iface=[iface];nqn=[nqn]
</programlisting>
<refsect3>
<title>Fields</title>
<variablelist>
<varlistentry id='transport'>
<term><varname>transport=</varname></term>
<listitem id='transport-text'>
<para>
This is a mandatory field that specifies the
network fabric being used for a
NVMe-over-Fabrics network. Current
<parameter>trtype</parameter> values understood
are:
</para>
<table id='transport-types'>
<title>Transport type</title>
<tgroup cols="2">
<thead>
<row>
<entry>trtype</entry>
<entry>Definition</entry>
</row>
</thead>
<tbody>
<row>
<entry>rdma</entry>
<entry>
The network fabric is an rdma network (RoCE, iWARP, Infiniband, basic rdma, etc)
</entry>
</row>
<row>
<entry>fc</entry>
<entry>
The network fabric is a Fibre Channel network.
</entry>
</row>
<row>
<entry>tcp</entry>
<entry>
The network fabric is a TCP/IP network.
</entry>
</row>
<row>
<entry>loop</entry>
<entry>
Connect to a NVMe over Fabrics target on the local host
</entry>
</row>
</tbody>
</tgroup>
</table>
</listitem>
</varlistentry>
<varlistentry id='tradd'>
<term>
<varname>traddr=</varname>
</term>
<listitem>
<para>
This is a mandatory field that specifies the
network address of the Controller. For
transports using IP addressing (e.g. rdma)
this should be an IP-based address (ex.
IPv4, IPv6). It could also be a resolvable
host name (e.g. localhost).
</para>
</listitem>
</varlistentry>
<varlistentry id='trsvcid'>
<term>
<varname>trsvcid=</varname>
</term>
<listitem>
<para>
This is an optional field that specifies the
transport service id. For transports using
IP addressing (e.g. rdma, tcp) this field is
the port number.
</para>
<para>
Depending on the transport type, this field
will default to either 8009 or 4420 as
follows.
</para>
<para>
UDP port 4420 and TCP port 4420 have been
assigned by IANA for use by NVMe over
Fabrics. NVMe/RoCEv2 controllers use UDP
port 4420 by default. NVMe/iWARP controllers
use TCP port 4420 by default.
</para>
<para>
TCP port 4420 has been assigned for use by
NVMe over Fabrics and TCP port 8009 has been
assigned by IANA for use by NVMe over
Fabrics discovery. TCP port 8009 is the
default TCP port for NVMe/TCP discovery
controllers. There is no default TCP port
for NVMe/TCP I/O controllers, the Transport
Service Identifier (TRSVCID) field in the
Discovery Log Entry indicates the TCP port
to use.
</para>
<para>
The TCP ports that may be used for NVMe/TCP
I/O controllers include TCP port 4420, and
the Dynamic and/or Private TCP ports (i.e.,
ports in the TCP port number range from
49152 to 65535). NVMe/TCP I/O controllers
should not use TCP port 8009. TCP port 4420
shall not be used for both NVMe/iWARP and
NVMe/TCP at the same IP address on the same
network.
</para>
<para>
Ref:
<ulink
url="https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xhtml?search=nvme">
IANA Service names port numbers
</ulink>
</para>
</listitem>
</varlistentry>
<varlistentry id='nqn'>
<term><varname>nqn=</varname></term>
<listitem>
<para>
This field specifies the Controller's NVMe
Qualified Name.
</para>
<para>
This field is mandatory for I/O Controllers, but is optional for
Discovery Controllers (DC). For the latter, the NQN will default
to the well-known DC NQN: <literal>nqn.2014-08.org.nvmexpress.discovery</literal>
if left undefined.
</para>
</listitem>
</varlistentry>
<varlistentry id='host-traddr'>
<term><varname>host-traddr=</varname></term>
<listitem>
<para>
This is an optional field that specifies the
network address used on the host to connect
to the Controller. For TCP, this sets the
source address on the socket.
</para>
</listitem>
</varlistentry>
<varlistentry id='host-iface'>
<term><varname>host-iface=</varname></term>
<listitem>
<para>
This is an optional field that specifies the
network interface used on the host to
connect to the Controller (e.g. IP eth1,
enp2s0, enx78e7d1ea46da). This forces the
connection to be made on a specific
interface instead of letting the system
decide.
</para>
</listitem>
</varlistentry>
<varlistentry id='dhchap-ctrl-secret'>
<term><varname>dhchap-ctrl-secret=</varname></term>
<listitem>
<para>
This is an optional field that specifies the
NVMe In-band authentication controller secret
(i.e. key) for bi-directional authentication;
needs to be in ASCII format as specified in
NVMe 2.0 section 8.13.5.8 'Secret representation'.
Bi-directional authentication will be attempted
when present.
</para>
</listitem>
</varlistentry>
<varlistentry id='hdr-digest-override'>
<term><varname>hdr-digest=</varname></term>
<listitem>
<para>
See definition in [Global] section. This is
an optional field used to override the value
specified in the [Global] section.
</para>
</listitem>
</varlistentry>
<varlistentry id='data-digest-override'>
<term><varname>data-digest=</varname></term>
<listitem>
<para>
See definition in [Global] section. This is
an optional field used to override the value
specified in the [Global] section.
</para>
</listitem>
</varlistentry>
<varlistentry id='nr-io-queues-override'>
<term><varname>nr-io-queues=</varname></term>
<listitem>
<para>
See definition in [Global] section. This is
an optional field used to override the value
specified in the [Global] section.
</para>
</listitem>
</varlistentry>
<varlistentry id='nr-write-queues-override'>
<term><varname>nr-write-queues=</varname></term>
<listitem>
<para>
See definition in [Global] section. This is
an optional field used to override the value
specified in the [Global] section.
</para>
</listitem>
</varlistentry>
<varlistentry id='nr-poll-queues-override'>
<term><varname>nr-poll-queues=</varname></term>
<listitem>
<para>
See definition in [Global] section. This is
an optional field used to override the value
specified in the [Global] section.
</para>
</listitem>
</varlistentry>
<varlistentry id='queue-size-override'>
<term><varname>queue-size=</varname></term>
<listitem>
<para>
See definition in [Global] section. This is
an optional field used to override the value
specified in the [Global] section.
</para>
</listitem>
</varlistentry>
<varlistentry id='kato-override'>
<term><varname>kato=</varname></term>
<listitem>
<para>
See definition in [Global] section. This is
an optional field used to override the value
specified in the [Global] section.
</para>
</listitem>
</varlistentry>
<varlistentry id='reconnect-delay-override'>
<term><varname>reconnect-delay=</varname></term>
<listitem>
<para>
See definition in [Global] section. This is
an optional field used to override the value
specified in the [Global] section.
</para>
</listitem>
</varlistentry>
<varlistentry id='ctrl-loss-tmo-override'>
<term><varname>ctrl-loss-tmo=</varname></term>
<listitem>
<para>
See definition in [Global] section. This is
an optional field used to override the value
specified in the [Global] section.
</para>
</listitem>
</varlistentry>
<varlistentry id='disable-sqflow-override'>
<term><varname>disable-sqflow=</varname></term>
<listitem>
<para>
See definition in [Global] section. This is
an optional field used to override the value
specified in the [Global] section.
</para>
</listitem>
</varlistentry>
</variablelist>
</refsect3>
<para>
Examples:
<programlisting>controller = transport=tcp;traddr=localhost;trsvcid=8009
controller = transport=tcp;traddr=2001:db8::370:7334;host-iface=enp0s8
controller = transport=fc;traddr=nn-0x204600a098cbcac6:pn-0x204700a098cbcac6
</programlisting>
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><varname>exclude=</varname></term>
<listitem id='exclude-text'>
<para>
Controllers that should be excluded can be specified with the
<varname>exclude=</varname> option. Using mDNS to
automatically discover and connect to controllers, can result
in unintentional connections being made. This keyword allows
configuring the controllers that should not be connected to.
</para>
<para>
The syntax is the same as for "controller", except that only
<parameter>transport</parameter>, <parameter>traddr</parameter>,
<parameter>trsvcid</parameter>, <parameter>nqn</parameter>, and
<parameter>host-iface</parameter> apply. Multiple
<varname>exclude=</varname> keywords may appear in the config
file to specify more than 1 excluded controller.
</para>
<para>
Note 1: A minimal match approach is used to eliminate unwanted
controllers. That is, you do not need to specify all the
parameters to identify a controller. Just specifying the
<parameter>host-iface</parameter>, for example, can be used to
exclude all controllers on an interface.
</para>
<para>
Note 2: <varname>exclude=</varname> takes precedence over
<varname>controller</varname>. A controller specified by the
<varname>controller</varname> keyword, can be eliminated by
the <varname>exclude=</varname> keyword.
</para>
<para>
Examples:
<programlisting>exclude = transport=tcp;traddr=fe80::2c6e:dee7:857:26bb # Eliminate a specific address
exclude = host-iface=enp0s8 # Eliminate everything on this interface
</programlisting>
</para>
</listitem>
</varlistentry>
</refsect2>
</root>

163
doc/standard-options.xml Normal file
View file

@ -0,0 +1,163 @@
<?xml version="1.0"?>
<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
<!--
SPDX-License-Identifier: Apache-2.0
Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
-->
<variablelist>
<varlistentry id='help'>
<term><option>-h</option></term>
<term><option>--help</option></term>
<listitem id='help-text'>
<para>Print the help text and exit.
</para></listitem>
</varlistentry>
<varlistentry id='version'>
<term><option>--version</option></term>
<listitem id='version-text'>
<para>Print the version string and exit.</para>
</listitem>
</varlistentry>
<varlistentry id='tron'>
<term><command>tron</command></term>
<listitem>
<para>Trace ON. Enable code tracing, which is to say that lots of
debug information will be printed to the syslog
(e.g. systemd-journal).</para>
</listitem>
</varlistentry>
<varlistentry id='troff'>
<term><command>troff</command></term>
<listitem>
<para>Trace OFF. Disable code tracing.</para>
</listitem>
</varlistentry>
<varlistentry id='status'>
<term><command>status</command></term>
<listitem>
<para>Show runtime status information.</para>
</listitem>
</varlistentry>
<varlistentry id='detailed'>
<term><option>-d</option></term>
<term><option>--detailed</option></term>
<listitem>
<para>Print additional details.</para>
</listitem>
</varlistentry>
<varlistentry id='transport'>
<term><option>-tTRTYPE</option></term>
<term><option>--transport=TRTYPE</option></term>
<listitem>
<para>NVMe-over-Fabrics fabric type (default: "tcp").</para>
</listitem>
</varlistentry>
<varlistentry id='traddr'>
<term><option>-aTRADDR</option></term>
<term><option>--traddr=TRADDR</option></term>
<listitem>
<para>Discovery controller's network address.</para>
</listitem>
</varlistentry>
<varlistentry id='trsvcid'>
<term><option>-sTRSVCID</option></term>
<term><option>--trsvcid=TRSVCID</option></term>
<listitem>
<para>
Transport service id (for IP addressing, e.g. tcp, rdma,
this field is the port number).
</para>
</listitem>
</varlistentry>
<varlistentry id='host-traddr'>
<term><option>-wTRADDR</option></term>
<term><option>--host-traddr=TRADDR</option></term>
<listitem>
<para>
Network source address used on the host to connect to
the controller.
</para>
</listitem>
</varlistentry>
<varlistentry id='host-iface'>
<term><option>-fIFACE</option></term>
<term><option>--host-iface=IFACE</option></term>
<listitem>
<para>
This field specifies the network interface used on the
host to connect to the controller.
</para>
</listitem>
</varlistentry>
<varlistentry id='nqn'>
<term><option>-nNQN</option></term>
<term><option>--nqn=NQN</option></term>
<listitem>
<para>
This field specifies the Controller's NVMe Qualified Name.
</para>
<para>
This field is mandatory for I/O Controllers, but is optional for
Discovery Controllers (DC). For the latter, the NQN will default
to the well-known DC NQN: <literal>nqn.2014-08.org.nvmexpress.discovery</literal>
if left undefined.
</para>
</listitem>
</varlistentry>
<varlistentry id='TRTYPE-value'>
<term><replaceable>TRTYPE</replaceable></term>
<listitem>
<para>rdma, fc, tcp, loop.</para>
</listitem>
</varlistentry>
<varlistentry id='TRADDR-value'>
<term><replaceable>TRADDR</replaceable></term>
<listitem>
<para>IP or Fibre Channel address. E.g. 10.10.0.100.</para>
</listitem>
</varlistentry>
<varlistentry id='TRSVCID-value'>
<term><replaceable>TRSVCID</replaceable></term>
<listitem>
<para>E.g., 8009.</para>
</listitem>
</varlistentry>
<varlistentry id='IFACE-value'>
<term><replaceable>IFACE</replaceable></term>
<listitem>
<para>
Network interface name. E.g., eth1, enp0s8, wlp0s20f3.
</para>
</listitem>
</varlistentry>
<varlistentry id='NQN-value'>
<term><replaceable>NQN</replaceable></term>
<listitem>
<para>
NVMe Qualified Name.
</para>
</listitem>
</varlistentry>
</variablelist>

73
doc/stas-config.target.xml Normal file
View file

@ -0,0 +1,73 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--*-nxml-*-->
<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
<!--
SPDX-License-Identifier: Apache-2.0
Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved.
-->
<refentry id="stas-config.target">
<refentryinfo>
<title>stas-config.target</title>
<productname>nvme-stas</productname>
<author>
<personname>
<honorific>Mr</honorific>
<firstname>Martin</firstname>
<surname>Belanger</surname>
</personname>
<affiliation>
<orgname>Dell, Inc.</orgname>
</affiliation>
</author>
</refentryinfo>
<refmeta>
<refentrytitle>stas-config.target</refentrytitle>
<manvolnum>8</manvolnum>
</refmeta>
<refnamediv>
<refname>stas-config.target</refname>
<refpurpose>Used to synchronize the start of nvme-stas processes</refpurpose>
</refnamediv>
<refsynopsisdiv>
<para>
<filename>/usr/lib/systemd/system/stas-config.target</filename>
</para>
</refsynopsisdiv>
<refsect1>
<title>Description</title>
<para>
This target is used as a synchronization point before starting
<citerefentry><refentrytitle>stacd.service</refentrytitle><manvolnum>8</manvolnum></citerefentry> and
<citerefentry><refentrytitle>stafd.service</refentrytitle><manvolnum>8</manvolnum></citerefentry>.
</para>
<para>
It ensures that <filename>/etc/nvme/hostnqn</filename> and
<filename>/etc/nvme/hostid</filename> are present before starting
<citerefentry><refentrytitle>stacd.service</refentrytitle><manvolnum>8</manvolnum></citerefentry> and
<citerefentry><refentrytitle>stafd.service</refentrytitle><manvolnum>8</manvolnum></citerefentry>.
</para>
</refsect1>
<refsect1>
<title>See Also</title>
<para>
<citerefentry>
<refentrytitle>stacd</refentrytitle>
<manvolnum>8</manvolnum>
</citerefentry>
<citerefentry>
<refentrytitle>stafd</refentrytitle>
<manvolnum>8</manvolnum>
</citerefentry>
</para>
</refsect1>
</refentry>

67
doc/stas-config@.service.xml Normal file
View file

@ -0,0 +1,67 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--*-nxml-*-->
<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
<!--
SPDX-License-Identifier: Apache-2.0
Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved.
-->
<refentry id="stas-config@.service">
<refentryinfo>
<title>stas-config@.service</title>
<productname>nvme-stas</productname>
<author>
<personname>
<honorific>Mr</honorific>
<firstname>Martin</firstname>
<surname>Belanger</surname>
</personname>
<affiliation>
<orgname>Dell, Inc.</orgname>
</affiliation>
</author>
</refentryinfo>
<refmeta>
<refentrytitle>stas-config@.service</refentrytitle>
<manvolnum>8</manvolnum>
</refmeta>
<refnamediv>
<refname>stas-config@.service</refname>
<refpurpose>Used for auto-generation of nvme-stas configuration files.</refpurpose>
</refnamediv>
<refsynopsisdiv>
<para>
<filename>/usr/lib/systemd/system/stas-config@.service</filename>
</para>
</refsynopsisdiv>
<refsect1>
<title>Description</title>
<para>
This service is used for the automatic run-time generation of
NVMe configuration located in <filename>/etc/nvme</filename>
(e.g. <filename>/etc/nvme/hostnqn</filename>). This is needed by
<citerefentry><refentrytitle>stacd.service</refentrytitle><manvolnum>8</manvolnum></citerefentry> and
<citerefentry><refentrytitle>stafd.service</refentrytitle><manvolnum>8</manvolnum></citerefentry>.
</para>
</refsect1>
<refsect1>
<title>See Also</title>
<para>
<citerefentry>
<refentrytitle>stacd</refentrytitle>
<manvolnum>8</manvolnum>
</citerefentry>
<citerefentry>
<refentrytitle>stafd</refentrytitle>
<manvolnum>8</manvolnum>
</citerefentry>
</para>
</refsect1>
</refentry>

201
doc/stasadm.xml Normal file
View file

@ -0,0 +1,201 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--*-nxml-*-->
<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
<!--
SPDX-License-Identifier: Apache-2.0
Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
-->
<refentry id="stasadm" xmlns:xi="http://www.w3.org/2001/XInclude">
<refentryinfo>
<title>stasadm</title>
<productname>nvme-stas</productname>
<author>
<personname>
<honorific>Mr</honorific>
<firstname>Martin</firstname>
<surname>Belanger</surname>
</personname>
<affiliation>
<orgname>Dell, Inc.</orgname>
</affiliation>
</author>
</refentryinfo>
<refmeta>
<refentrytitle>stasadm</refentrytitle>
<manvolnum>1</manvolnum>
</refmeta>
<refnamediv>
<refname>stasadm</refname>
<refpurpose>STorage Appliance Services admin functions</refpurpose>
</refnamediv>
<refsynopsisdiv>
<cmdsynopsis>
<command>stasadm</command>
<arg choice="opt" rep="repeat">OPTIONS</arg>
<arg choice="req">COMMAND</arg>
<arg choice="opt" rep="repeat">OPTIONS</arg>
</cmdsynopsis>
</refsynopsisdiv>
<refsect1>
<title>Description</title>
<para>
<command>stasadm</command> is used to configure <code>nvme-stas</code>. The
configuration is saved to <filename>/etc/stas/sys.conf</filename>.
</para>
<para>
Although <code>nvme-stas</code>' configuration is saved to
<filename>/etc/stas/sys.conf</filename>, it's still possible to
interoperate with the configuration of <command>nvme-cli</command>
and <command>libnvme</command>. <code>nvme-stas</code> allows one to
save individual parameters such as the Host NQN and ID outside of
<filename>/etc/stas/sys.conf</filename>. This allows, for example,
using the same default Host NQN and ID defined by
<command>nvme-cli</command> and <command>libnvme</command> in
<filename>/etc/nvme/hostnqn</filename> and <filename>/etc/nvme/hostid</filename>
respectively. To tell <code>nvme-stas</code> that you want to use the
those files, simply use <command>stasadm</command>'s
<option>--file=FILE</option> option.
</para>
</refsect1>
<refsect1>
<title>Commands</title>
<para>The following commands are understood:</para>
<variablelist>
<varlistentry>
<term><command>hostnqn</command></term>
<listitem>
<para>
Generate the Host NQN. This is typically used as a post
installation step to generate <filename>/etc/nvme/hostnqn</filename>.
</para>
<para>
The NVMe base specifications says: <quote>An NQN is
permanent for the lifetime of the host</quote>. For
this reason, the host NQN should only be generated
if <filename>/etc/nvme/hostnqn</filename> does not exist
already.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><command>hostid</command></term>
<listitem>
<para>
Generate the Host ID. This is typically used as a post
installation step to generate <filename>/etc/nvme/hostid</filename>.
</para>
<para>
Although not explicitly specified in the NVMe
specifications, the Host ID, like the Host NQN, should
be permanent for the lifetime of the host. Only generate
the Host ID if <filename>/etc/nvme/hostid</filename>
does not exist.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><command>set-symname [SYMNAME]</command></term>
<listitem>
<para>
Set the host symbolic name.
</para>
<para>
The symbolic name is an optional parameter that can be
used for explicit registration with a discovery controller.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><command>clear-symname</command></term>
<listitem>
<para>
Clear the host symbolic name.
</para>
</listitem>
</varlistentry>
</variablelist>
</refsect1>
<refsect1>
<title>Options</title>
<para>The following options are understood:</para>
<variablelist>
<xi:include href="standard-options.xml" xpointer="help"/>
<xi:include href="standard-options.xml" xpointer="version"/>
<varlistentry>
<term><option>-fFILE</option></term>
<term><option>--file=FILE</option></term>
<listitem>
<para>
By default, <command>hostnqn</command> and <command>hostid</command>
save the values to <filename>/etc/stas/sys.conf</filename>.
This option allows saving to a separate file.
</para>
<para>
Traditionally, <command>nvme-cli</command> and
<command>libnvme</command> retrieve the default Host NQN
and ID from <filename>/etc/nvme/hostnqn</filename> and
<filename>/etc/nvme/hostid</filename> respectively. The
<option>--file=FILE</option> option can be
used to tell <code>nvme-stas</code> that it should
use those same configuration files.
</para>
</listitem>
</varlistentry>
</variablelist>
</refsect1>
<refsect1>
<title>Exit status</title>
<para>
On success, 0 is returned; otherwise, a non-zero failure code is
returned.
</para>
</refsect1>
<refsect1>
<title>Examples</title>
<example>
<title>Generate <filename>/etc/nvme/hostnqn</filename></title>
<programlisting>$ stasadm hostnqn --file /etc/nvme/hostnqn</programlisting>
</example>
<example>
<title>Generate <filename>/etc/nvme/hostid</filename></title>
<programlisting>$ stasadm hostid -f /etc/nvme/hostid</programlisting>
</example>
<example>
<title>Configure the host's symbolic name</title>
<programlisting>$ stasadm set-symname LukeSkywalker</programlisting>
</example>
</refsect1>
<refsect1>
<title>See Also</title>
<para>
<citerefentry>
<refentrytitle>nvme-stas</refentrytitle>
<manvolnum>7</manvolnum>
</citerefentry>
</para>
</refsect1>
</refentry>

141
doc/sys.conf.xml Normal file
View file

@ -0,0 +1,141 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--*-nxml-*-->
<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
<!--
SPDX-License-Identifier: Apache-2.0
Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
-->
<refentry id="sys.conf" xmlns:xi="http://www.w3.org/2001/XInclude">
<refentryinfo>
<title>sys.conf</title>
<productname>nvme-stas</productname>
<author>
<personname>
<honorific>Mr</honorific>
<firstname>Martin</firstname>
<surname>Belanger</surname>
</personname>
<affiliation>
<orgname>Dell, Inc.</orgname>
</affiliation>
</author>
</refentryinfo>
<refmeta>
<refentrytitle>sys.conf</refentrytitle>
<manvolnum>5</manvolnum>
</refmeta>
<refnamediv>
<refname>sys.conf</refname>
<refpurpose>
<citerefentry project="man-pages">
<refentrytitle>nvme-stas</refentrytitle>
<manvolnum>7</manvolnum>
</citerefentry>
configuration file
</refpurpose>
</refnamediv>
<refsynopsisdiv>
<para>
<filename>/etc/stas/sys.conf</filename>
</para>
</refsynopsisdiv>
<refsect1>
<title>Description</title>
<para>
When <citerefentry project="man-pages"><refentrytitle>stafd</refentrytitle>
<manvolnum>8</manvolnum></citerefentry> and
<citerefentry project="man-pages"><refentrytitle>stacd</refentrytitle>
<manvolnum>8</manvolnum></citerefentry> start up, they read the
system configuration from <filename>sys.conf</filename>.
</para>
</refsect1>
<refsect1>
<title>Configuration File Format</title>
<para>
<filename>sys.conf</filename> is a plain text file divided into
sections, with configuration entries in the style
<replaceable>key</replaceable>=<replaceable>value</replaceable>.
Whitespace immediately before or after the <literal>=</literal> is
ignored. Empty lines and lines starting with <literal>#</literal>
are ignored, which may be used for commenting.
</para>
</refsect1>
<refsect1>
<title>Options</title>
<refsect2>
<title>[Host] section</title>
<para>
The following options are available in the
<literal>[Host]</literal> section:
</para>
<variablelist>
<varlistentry>
<term><varname>nqn=</varname></term>
<listitem>
<para>
Takes a string argument identifying the Host NQN.
A value starting with <code>file://</code>
indicates that the Host NQN can be retrieved from
a separate file. This is a mandatory parameter.
Defaults to: <literal>file:///etc/nvme/hostnqn</literal>.
</para>
</listitem>
</varlistentry>
</variablelist>
<variablelist>
<varlistentry>
<term><varname>id=</varname></term>
<listitem>
<para>
Takes a string argument identifying the Host ID.
A value starting with <code>file://</code>
indicates that the Host ID can be retrieved from
a separate file. This is a mandatory parameter.
Defaults to: <literal>file:///etc/nvme/hostid</literal>.
</para>
</listitem>
</varlistentry>
</variablelist>
<variablelist>
<varlistentry>
<term><varname>symname=</varname></term>
<listitem>
<para>
Takes a string argument identifying the Host symbolic name.
A value starting with <code>file://</code>
indicates that the symbolic name can be retrieved from
a separate file. This is an optional parameter.
There is no default value.
</para>
</listitem>
</varlistentry>
</variablelist>
</refsect2>
</refsect1>
<refsect1>
<title>See Also</title>
<para>
<citerefentry>
<refentrytitle>nvme-stas</refentrytitle>
<manvolnum>7</manvolnum>
</citerefentry>
</para>
</refsect1>
</refentry>

27
docker-compose.yml Normal file
View file

@ -0,0 +1,27 @@
version: '3.7'
x-stas: &default-stas
image: ghcr.io/linux-nvme/nvme-stas:main
build:
context: .
volumes:
- /run/dbus:/run/dbus
- /etc/nvme:/etc/nvme
privileged: true
network_mode: host
services:
stafd:
<<: *default-stas
environment:
RUNTIME_DIRECTORY: /run/stafd
XDG_CACHE_HOME: /var/cache/stafd
PYTHONUNBUFFERED: 1
command: -u /usr/sbin/stafd
stacd:
<<: *default-stas
environment:
RUNTIME_DIRECTORY: /run/stacd
XDG_CACHE_HOME: /var/cache/stacd
PYTHONUNBUFFERED: 1
command: -u /usr/sbin/stacd

23
etc/dbus-1/system.d/meson.build Normal file
View file

@ -0,0 +1,23 @@
# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See the LICENSE file for details.
#
# This file is part of NVMe STorage Appliance Services (nvme-stas).
#
# Authors: Martin Belanger <Martin.Belanger@dell.com>
#
dbus_conf_dir = datadir / 'dbus-1' / 'system.d'
configure_file(
input: 'org.nvmexpress.staf.in.conf',
output: 'org.nvmexpress.staf.conf',
configuration: conf,
install_dir: dbus_conf_dir,
)
configure_file(
input: 'org.nvmexpress.stac.in.conf',
output: 'org.nvmexpress.stac.conf',
configuration: conf,
install_dir: dbus_conf_dir,
)

37
etc/dbus-1/system.d/org.nvmexpress.stac.in.conf Normal file
View file

@ -0,0 +1,37 @@
<!--
Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
SPDX-License-Identifier: Apache-2.0
See the LICENSE file for details.
This file is part of NVMe STorage Appliance Services (nvme-stas).
Authors: Martin Belanger <Martin.Belanger@dell.com>
-->
<!DOCTYPE busconfig PUBLIC
"-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN"
"http://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
<busconfig>
<!-- Only allow root to own the bus. -->
<policy user="root">
<allow own_prefix="@STACD_DBUS_NAME@"/>
</policy>
<!-- Allow anyone to invoke most methods on the bus, but deny setting properties. -->
<policy context="default">
<allow send_destination="@STACD_DBUS_NAME@"/>
<deny send_destination="@STACD_DBUS_NAME@"
send_interface="org.freedesktop.DBus.Properties"
send_member="Set"
send_type="method_call"/>
</policy>
<!-- Allow root to invoke everything on the bus. -->
<policy user="root">
<allow send_destination="@STACD_DBUS_NAME@"/>
</policy>
</busconfig>

37
etc/dbus-1/system.d/org.nvmexpress.staf.in.conf Normal file
View file

@ -0,0 +1,37 @@
<!--
Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
SPDX-License-Identifier: Apache-2.0
See the LICENSE file for details.
This file is part of NVMe STorage Appliance Services (nvme-stas).
Authors: Martin Belanger <Martin.Belanger@dell.com>
-->
<!DOCTYPE busconfig PUBLIC
"-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN"
"http://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
<busconfig>
<!-- Only allow root to own the bus. -->
<policy user="root">
<allow own_prefix="@STAFD_DBUS_NAME@"/>
</policy>
<!-- Allow anyone to invoke most methods on the bus, but deny setting properties. -->
<policy context="default">
<allow send_destination="@STAFD_DBUS_NAME@"/>
<deny send_destination="@STAFD_DBUS_NAME@"
send_interface="org.freedesktop.DBus.Properties"
send_member="Set"
send_type="method_call"/>
</policy>
<!-- Allow root to invoke everything on the bus. -->
<policy user="root">
<allow send_destination="@STAFD_DBUS_NAME@"/>
</policy>
</busconfig>

322
etc/stas/stacd.conf Normal file
View file

@ -0,0 +1,322 @@
# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See the LICENSE file for details.
#
# This file is part of NVMe STorage Appliance Services (nvme-stas).
# ==============================================================================
# STorage Appliance Connector Daemon (stacd) - configuration file
#
# In this file, options that are commented represent the default values used.
# Uncommented options override the default value.
[Global]
# tron: Trace-ON. Enable additional debug info
# Type: boolean
# Range: [false, true]
#tron=false
# hdr-digest: Protocol Data Unit (PDU) Header Digest. NVMe/TCP facilitates an
# optional PDU Header digest. Digests are calculated using the
# CRC32C algorithm.
# Type: boolean
# Range: [false, true]
#hdr-digest=false
# data-digest: Protocol Data Unit (PDU) Data Digest. NVMe/TCP facilitates an
# optional PDU Data digest. Digests are calculated using the
# CRC32C algorithm.
# Type: boolean
# Range: [false, true]
#data-digest=false
# kato: Keep Alive Timeout (KATO): This field specifies the timeout value
# for the Keep Alive feature in seconds. The default value for this
# field is 30 seconds (2 minutes).
# Type: Unsigned integer
# Range: 0..N
# Unit: Seconds
#kato=30
# nr-io-queues: Overrides the default number of I/O queues create by the
# driver.
# Type: Unsigned integer
# Range: 1..N
# Default: Depends on kernel and other run time
# factors (e.g. number of CPUs).
# nr-write-queues: Adds additional queues that will be used for write I/O.
# Type: Unsigned integer
# Range: 1..N
# Default: Depends on kernel and other run time
# factors (e.g. number of CPUs).
# nr-poll-queues: Adds additional queues that will be used for polling
# latency sensitive I/O.
# Type: Unsigned integer
# Range: 1..N
# Default: Depends on kernel and other run time
# factors (e.g. number of CPUs).
# queue-size: Overrides the default number of elements in the I/O queues
# created by the driver.
# Type: Unsigned integer
# Range: 16..1024
# Default: 128
#queue-size=128
# reconnect-delay: Overrides the default delay, in seconds, before reconnect
# is attempted after a connect loss.
# Type: Unsigned integer
# Range: 1..N
# Unit: Seconds
# Default: 10 (retry to connect every 10 seconds)
#reconnect-delay=10
# ctrl-loss-tmo: Overrides the default controller loss timeout period in
# seconds.
# Type: Unsigned integer
# Range: -1, 0..N where -1 means retry forever
# Unit: Seconds
# Default: 600 (retry to connect for up to 10 minutes)
#ctrl-loss-tmo=600
# disable-sqflow: Disables SQ flow control to omit head doorbell update for
# submission queues when sending nvme completions.
# Type: boolean
# Range: [false, true]
# Default: false
#disable-sqflow=false
# ignore-iface: This option controls whether connections with I/O Controllers
# (IOC) will be forced on a specific interface or will rely on
# the routing tables to determine the interface.
#
# See the man pages for details: man stacd.conf
#
# Type: boolean
# Range: [false, true]
# Default: true
#ignore-iface=false
# ip-family: With this you can specify whether stacd will support IPv4, IPv6,
# or both when connecting to I/O Controllers (IOC).
#
# See the man pages for details: man stacd.conf
#
# Type: String
# Range: [ipv4, ipv6, ipv4+ipv6]
# Default: ipv4+ipv6
#ip-family=ipv4+ipv6
# ==============================================================================
[I/O controller connection management]
# This section contains parameters to manage I/O controller connections.
# For example, parameters are provided to control disconnect policy. In other
# words, whether stacd will disconnect from IOCs on DLPE removal and which
# connections will be affected.
#
# Also, what should stacd do when a DLPE NCC bit (Not Connected to CDC) is
# asserted. Should stacd stop trying to connect to an I/O controller after a
# certain number of unsuccessful attempts.
#
# See the man pages for details: man stacd.conf
# disconnect-scope: Determines which connections, if any, will be the target of
# a potential disconnect on DLPE removal.
#
# Type: String
# Range: [only-stas-connections | all-connections-matching-disconnect-trtypes | no-disconnect]
# Default: only-stas-connections
#disconnect-scope=only-stas-connections
# disconnect-trtypes: Specify which connections should be audited based on the
# transport type. This parameter only applies when
# "disconnect-scope = all-connections-matching-disconnect-trtypes".
#
# Type: String
# Range: [tcp, rdma, fc, tcp+rdma, tcp+fc, rdma+fc, tcp+rdma+fc]
# Default: tcp
#disconnect-trtypes=tcp
# connect-attempts-on-ncc: The NCC bit (Not Connected to CDC) returned in a
# DLPE indicates whether a connection is currently
# established between the CDC and the subsystem listed
# in the DLPE.
#
# When the NCC bit is asserted, it may mean that the
# subsystem is offline or that fabrics connectivity is
# momentarily lost. If the host is also unable to
# connect to the subsystem, then there is no point in
# continuing to try to connect. In fact, the CDC will
# monitor the situation an inform the host (AEN) when
# connectivity is restored.
#
# This field is used to tell stacd how to behave when
# the NCC bit is asserted. How many times should it try
# to connect before give-up, or whether to keep trying
# indefinitely.
#
# Type: Integer
# Range: [0, 2..N], 0 means "Never stop trying". A
# non-0 value indicates the number of attempts
# before giving up. This value should never be
# set to 1. A value of 1 will automatically be
# increased to 2. That's because a single
# failure may be normal and a mimimum of 2
# attempts is required to conclude that a
# connection is not possible.
# Default: 0
#connect-attempts-on-ncc=0
# ==============================================================================
[Controllers]
# controller: I/O Controllers (IOC) are specified with this keyword.
#
# Syntax:
# controller = transport=<trtype>;traddr=<traddr>;trsvcid=<trsvcid>;host-traddr=<traddr>;host-iface=<iface>,nqn=<subnqn>
#
# transport=<trtype> [MANDATORY]
# This field specifies the network fabric being used for a NVMe-over-
# Fabrics network. Current string values include:
#
# Value Definition
# ------- -----------------------------------------------------------
# rdma The network fabric is an rdma network (RoCE, iWARP, Infiniband, basic rdma, etc)
# fc The network fabric is a Fibre Channel network.
# tcp The network fabric is a TCP/IP network.
# loop Connect to a NVMe over Fabrics target on the local host
#
# traddr=<traddr> [MANDATORY]
# This field specifies the network address of the Controller. For
# transports using IP addressing (e.g. rdma) this should be an IP-
# based address (ex. IPv4, IPv6). It could also be a resolvable host
# name (e.g. localhost).
#
# nqn=<subnqn> [MANDATORY]
# This field specifies the Subsystem's NVMe Qualified Name.
#
# trsvcid=<trsvcid> [OPTIONAL]
# This field specifies the transport service id. For transports using
# IP addressing (e.g. rdma) this field is the port number.
#
# Depending on the transport type, this field will default to either
# 8009 or 4420 as follows.
#
# UDP port 4420 and TCP port 4420 have been assigned by IANA
# for use by NVMe over Fabrics. NVMe/RoCEv2 controllers use UDP port
# 4420 by default. NVMe/iWARP controllers use TCP port 4420 by
# default.
#
# TCP port 4420 has been assigned for use by NVMe over Fabrics and TCP
# port 8009 has been assigned by IANA for use by NVMe over Fabrics
# discovery. TCP port 8009 is the default TCP port for NVMe/TCP
# discovery controllers. There is no default TCP port for NVMe/TCP I/O
# controllers, the Transport Service Identifier (TRSVCID) field in the
# Discovery Log Entry indicates the TCP port to use.
#
# The TCP ports that may be used for NVMe/TCP I/O controllers include
# TCP port 4420, and the Dynamic and/or Private TCP ports (i.e., ports
# in the TCP port number range from 49152 to 65535). NVMe/TCP I/O
# controllers should not use TCP port 8009. TCP port 4420 shall not be
# used for both NVMe/iWARP and NVMe/TCP at the same IP address on the
# same network.
#
# host-traddr=<traddr> [OPTIONAL]
# This field specifies the network address used on the host to connect
# to the Controller. For TCP, this sets the source address on the
# socket.
#
# host-iface=<iface> [OPTIONAL]
# This field specifies the network interface used on the host to
# connect to the Controller (e.g. IP eth1, enp2s0, enx78e7d1ea46da).
# This forces the connection to be made on a specific interface
# instead of letting the system decide.
#
# dhchap-ctrl-secret [OPTIONAL]
# NVMe In-band authentication controller secret (i.e. key) for
# bi-directional authentication; needs to be in ASCII format as
# specified in NVMe 2.0 section 8.13.5.8 'Secret representation'.
# Bi-directional authentication will be attempted when present.
#
# hdr-digest [OPTIONAL]
# See definition in [Global] section. This is used to override
# the value specified in the [Global] section.
#
# data-digest [OPTIONAL]
# See definition in [Global] section. This is used to override
# the value specified in the [Global] section.
#
# nr-io-queues [OPTIONAL]
# See definition in [Global] section. This is used to override
# the value specified in the [Global] section.
#
# nr-write-queues [OPTIONAL]
# See definition in [Global] section. This is used to override
# the value specified in the [Global] section.
#
# nr-poll-queues [OPTIONAL]
# See definition in [Global] section. This is used to override
# the value specified in the [Global] section.
#
# queue-size [OPTIONAL]
# See definition in [Global] section. This is used to override
# the value specified in the [Global] section.
#
# kato [OPTIONAL]
# See definition in [Global] section. This is used to override
# the value specified in the [Global] section.
#
# reconnect-delay [OPTIONAL]
# See definition in [Global] section. This is used to override
# the value specified in the [Global] section.
#
# ctrl-loss-tmo [OPTIONAL]
# See definition in [Global] section. This is used to override
# the value specified in the [Global] section.
#
# disable-sqflow [OPTIONAL]
# See definition in [Global] section. This is used to override
# the value specified in the [Global] section.
#
# Multiple DCs may be specified on separate lines like this (this is
# just an example and does not represent default values):
#
# controller = transport=tcp;traddr=localhost;nqn=nqn.1988-11.com.dell:PowerSANxxx:01:20210225100113-454f73093ceb4847a7bdfc6e34ae8e28
# controller = transport=tcp;traddr=2001:db8::370:7334;host-iface=enp0s8;nqn=nqn.starship-enterprise
# controller = transport=fc;traddr=nn-0x204600a098cbcac6:pn-0x204700a098cbcac6;nqn=nqn.romulan-empire
# ...
# Type: String
#
# Default: There is no default controller. STAC will not try to
# connect to a default I/O Controller.
#controller=
# exclude: Excluded controllers. This keyword allows configuring I/O
# controllers that should not be connected to (whatever the
# reason may be).
#
# The syntax is the same as for "controller=", except that the key
# host-traddr does not apply. Multiple "exclude=" keywords may
# appear in the config file to define the exclusion list.
#
# Note 1: A minimal match approach is used to eliminate unwanted
# controllers. That is, you do not need to specify all the
# parameters to identify a controller. Just specifying the
# host-iface, for example, can be used to exclude all controllers
# on an interface.
#
# Note 2: "exclude=" takes precedence over "controller=". A
# controller specified by the "controller=" keyword, can be
# eliminated by the "exclude=" keyword.
#
# Syntax: Same as "controller=" above.
# Type: String
#
# Example:
# exclude = transport=tcp;traddr=fe80::2c6e:dee7:857:26bb # Eliminate a specific address
# exclude = host-iface=enp0s8 # Eliminate everything on this interface
#exclude=

277
etc/stas/stafd.conf Normal file
View file

@ -0,0 +1,277 @@
# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See the LICENSE file for details.
#
# This file is part of NVMe STorage Appliance Services (nvme-stas).
# ==============================================================================
# STorage Appliance Finder Daemon (stafd) - configuration file
#
# In this file, options that are commented represent the default values used.
# Uncommented options override the default value.
[Global]
# tron: Trace-ON. Enable additional debug info
# Type: boolean
# Range: [false, true]
#tron=false
# hdr-digest: Protocol Data Unit (PDU) Header Digest. NVMe/TCP facilitates an
# optional PDU Header digest. Digests are calculated using the
# CRC32C algorithm.
# Type: boolean
# Range: [false, true]
#hdr-digest=false
# data-digest: Protocol Data Unit (PDU) Data Digest. NVMe/TCP facilitates an
# optional PDU Data digest. Digests are calculated using the
# CRC32C algorithm.
# Type: boolean
# Range: [false, true]
#data-digest=false
# kato: Keep Alive Timeout (KATO): This field specifies the timeout value
# for the Keep Alive feature in seconds. The default value for this
# field is 30 seconds.
# Type: Unsigned integer
# Range: 0..N
# Unit: Seconds
#kato=30
# queue-size: Overrides the default number of elements in the I/O queues
# created by the driver.
# Type: Unsigned integer
# Range: 16..1024
# Default: 128
#queue-size=128
# reconnect-delay: Overrides the default delay, in seconds, before reconnect
# is attempted after a connect loss.
# Type: Unsigned integer
# Range: 1..N
# Unit: Seconds
# Default: 10 (retry to connect every 10 seconds)
#reconnect-delay=10
# ctrl-loss-tmo: Overrides the default controller loss timeout period in
# seconds.
# Type: Unsigned integer
# Range: -1, 0..N where -1 means retry forever
# Unit: Seconds
# Default: 600 (retry to connect for up to 10 minutes)
#ctrl-loss-tmo=600
# disable-sqflow: Disables SQ flow control to omit head doorbell update for
# submission queues when sending nvme completions.
# Type: boolean
# Range: [false, true]
# Default: false
#disable-sqflow=false
# ignore-iface: This option controls whether connections with Discovery
# Controllers (DC) will be forced on a specific interface or
# will rely on the routing tables to determine the interface.
#
# See the man pages for details: man stafd.conf
#
# Type: boolean
# Range: [false, true]
# Default: true
#ignore-iface=false
# ip-family: With this you can specify whether stafd will support IPv4, IPv6,
# or both when connecting to Discovery Controllers (DC).
#
# See the man pages for details: man stafd.conf
#
# Type: String
# Range: [ipv4, ipv6, ipv4+ipv6]
# Default: ipv4+ipv6
#ip-family=ipv4+ipv6
# pleo: Port Local Entries Only. If enabled and supported, when connected to a
# Direct Discovery Controller (DDC), stafd will ask the DDC to return
# records for only NVM subsystem ports that are presented through the same
# NVM subsystem port that received the Get Log Page command. When disabled
# or not supported by the DDC, the DDC may return NVM subsystems that are
# not even reachable by the host, including those using a transport
# different from the transport used for the Get Log Page command (e.g. Get
# Log Page using TCP and receiving FC subsystems). This configuration
# parameter has no effect if the DDC does not support PLEO (see PLEOS).
#
# Type: String
# Range: [disabled, enabled]
# Default: enabled
#pleo=enabled
# ==============================================================================
[Service Discovery]
# zeroconf: Control whether DNS-SD/mDNS automatic discovery is enabled. This is
# used to enable or disable automatic discovery of Discovery
# Controllers using DNS-SD/mDNS.
#
# Type: String
# Range: [disabled, enabled]
# Default: enabled
#zeroconf=enabled
# ==============================================================================
[Discovery controller connection management]
# persistent-connections: Whether connections to Discovery Controllers (DC)
# are persistent. If stafd is stopped, the connections
# will persists. When this is set to false, stafd will
# disconnect from all DCs it is connected to when stafd
# is stopped.
# Type: boolean
# Range: [false, true]
#persistent-connections=true
# zeroconf-connections-persistence: DCs that are discovered with mDNS service
# discovery which are later lost (i.e. no mDNS
# and TCP connection fails), will be purged from
# the configuration after this amount of time.
# Type: Time specs.
# Unit: Takes a unit-less value in seconds,
# or a time span (TS) value such as
# "3 days 5 hours".
# Range: -1, 0, TS.
# With "-1" equal to "no timeout" and
# 0 equal to timeout immediately.
# Default: 72 hours (3 days)
#zeroconf-connections-persistence=72hours
# ==============================================================================
[Controllers]
# controller: Discovery Controllers (DC) are specified with this keyword.
#
# Syntax:
# controller = transport=[trtype];traddr=[traddr];trsvcid=[trsvcid];host-traddr=[traddr];host-iface=[iface];nqn=<dcnqn>
#
# transport=<trtype> [MANDATORY]
# This field specifies the network fabric being used for a NVMe-over-
# Fabrics network. Current string values include:
#
# Value Definition
# ------- -----------------------------------------------------------
# rdma The network fabric is an rdma network (RoCE, iWARP,
# Infiniband, basic rdma, etc)
# fc The network fabric is a Fibre Channel network.
# tcp The network fabric is a TCP/IP network.
# loop Connect to a NVMe over Fabrics target on the local host
#
# traddr=<traddr> [MANDATORY]
# This field specifies the network address of the Controller. For
# transports using IP addressing (e.g. rdma) this should be an IP-
# based address (ex. IPv4, IPv6). It could also be a resolvable host
# name (e.g. localhost).
#
# nqn=<dcnqn> [OPTIONAL]
# This field specifies the Discovery Controller's NVMe Qualified
# Name. If not specified, this will default to the well-known DC
# NQN: "nqn.2014-08.org.nvmexpress.discovery".
#
# trsvcid=<trsvcid> [OPTIONAL]
# This field specifies the transport service id. For transports using
# IP addressing (e.g. rdma) this field is the port number.
#
# Depending on the transport type, this field will default to either
# 8009 or 4420 as follows.
#
# UDP port 4420 and TCP port 4420 have been assigned by IANA
# for use by NVMe over Fabrics. NVMe/RoCEv2 controllers use UDP port
# 4420 by default. NVMe/iWARP controllers use TCP port 4420 by
# default.
#
# TCP port 4420 has been assigned for use by NVMe over Fabrics and TCP
# port 8009 has been assigned by IANA for use by NVMe over Fabrics
# discovery. TCP port 8009 is the default TCP port for NVMe/TCP
# discovery controllers. There is no default TCP port for NVMe/TCP I/O
# controllers, the Transport Service Identifier (TRSVCID) field in the
# Discovery Log Entry indicates the TCP port to use.
#
# The TCP ports that may be used for NVMe/TCP I/O controllers include
# TCP port 4420, and the Dynamic and/or Private TCP ports (i.e., ports
# in the TCP port number range from 49152 to 65535). NVMe/TCP I/O
# controllers should not use TCP port 8009. TCP port 4420 shall not be
# used for both NVMe/iWARP and NVMe/TCP at the same IP address on the
# same network.
#
# Ref: https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xhtml?search=nvme
#
# host-traddr=<traddr> [OPTIONAL]
# This field specifies the network address used on the host to connect
# to the Controller. For TCP, this sets the source address on the
# socket.
#
# host-iface=<iface> [OPTIONAL]
# This field specifies the network interface used on the host to
# connect to the Controller (e.g. IP eth1, enp2s0, enx78e7d1ea46da).
# This forces the connection to be made on a specific interface
# instead of letting the system decide.
#
# hdr-digest [OPTIONAL]
# See definition in [Global] section. This is used to override
# the value specified in the [Global] section.
#
# data-digest [OPTIONAL]
# See definition in [Global] section. This is used to override
# the value specified in the [Global] section.
#
# kato [OPTIONAL]
# See definition in [Global] section. This is used to override
# the value specified in the [Global] section.
#
# reconnect-delay [OPTIONAL]
# See definition in [Global] section. This is used to override
# the value specified in the [Global] section.
#
# ctrl-loss-tmo [OPTIONAL]
# See definition in [Global] section. This is used to override
# the value specified in the [Global] section.
#
# disable-sqflow [OPTIONAL]
# See definition in [Global] section. This is used to override
# the value specified in the [Global] section.
#
# Multiple DCs may be specified on separate lines like this (this is
# just an example and does not represent default values):
#
# controller = transport=tcp;traddr=localhost;trsvcid=8009
# controller = transport=tcp;traddr=2001:db8::370:7334;host-iface=enp0s8
# controller = transport=fc;traddr=nn-0x204600a098cbcac6:pn-0x204700a098cbcac6
# ...
#
# Type: String
#
# Default: There is no default controller. STAF will not try to
# connect to a default Discovery Controller.
#controller=
# exclude: Excluded controllers. Using mDNS to automatically detect
# and connect controllers, can result in unintentional connections
# being made. This keyword allows configuring the controllers that
# should not be connected to (whatever the reason may be).
#
# The syntax is the same as for "controller=", except that the key
# host-traddr does not apply. Multiple "exclude=" keywords may
# appear in the config file to define the exclusion list.
#
# Note 1: A minimal match approach is used to eliminate unwanted
# controllers. That is, you do not need to specify all the
# parameters to identify a controller. Just specifying the
# host-iface, for example, can be used to exclude all controllers
# on an interface.
#
# Note 2: "exclude=" takes precedence over "controller=". A
# controller specified by the "controller=" keyword, can be
# eliminated by the "exclude=" keyword.
#
# Syntax: Same as "controller=" above.
# Type: String
#
# Example:
# exclude = transport=tcp;traddr=fe80::2c6e:dee7:857:26bb # Eliminate a specific address
# exclude = host-iface=enp0s8 # Eliminate everything on this interface
#exclude=

62
etc/stas/sys.conf.doc Normal file
View file

@ -0,0 +1,62 @@
# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See the LICENSE file for details.
#
# This file is part of NVMe STorage Appliance Services (nvme-stas).
# ==============================================================================
# STorage Appliance Services (stas) - System configuration file
#
# In this file, options that are commented represent the default values used.
# Uncommented options override the default value.
[Host]
# nqn: The host's unique Non-Qualified Name. A value starting with "file://"
# indicates that the Host NQN can be retrieved from a separate file.
# Typically, nvme-cli saves the Host NQN in /etc/nvme/hostnqn. For
# compatibility with nvme-cli, nvme-stas defaults to looking for the
# existance of this file and will read the NQN from it. Otherwise, you
# can overwrite the default NQN by specifying its value here or
# specifying another file that contains the Host NQN to use.
# Type: string
# Default: file:///etc/nvme/hostnqn
#nqn=file:///etc/nvme/hostnqn
# id: The host's unique Identifier (ID). A value starting with "file://"
# indicates that the Host ID can be retrieved from a separate file.
# Typically, nvme-cli saves the Host ID in /etc/nvme/hostid. For
# compatibility with nvme-cli, nvme-stas defaults to looking for the
# existance of this file and will read the ID from it. Otherwise, you
# can overwrite the default ID by specifying its value here or
# specifying another file that contains the Host ID to use.
# Type: string
# Default: file:///etc/nvme/hostid
#id=file:///etc/nvme/hostid
# key: The host's DHCHAP key to be used for authentication. This is an
# optional parameter only required when authentication is needed.
# A value starting with "file://" indicates that the Host Key can
# be retrieved from a separate file. Typically, nvme-cli saves the
# Host Key in /etc/nvme/hostkey. For compatibility with nvme-cli,
# nvme-stas defaults to looking for the existance of this file and
# will read the Key from it. Otherwise, you can overwrite the default
# Key by specifying its value here or specifying another file that
# contains an alternate Host Key to use.
# Type: string
# Default: file:///etc/nvme/hostkey
#key=file:///etc/nvme/hostkey
# symname: The host's symbolic name. This can be a string or the name of a file
# containing the symbolic name. A value starting with "file://"
# indicates that the Symbolic Name can be retrieved from a separate
# file.
# Type: string
# Default: There is no default. The symbolic name is undefined by
# default.
#symname=

182
meson.build Normal file
View file

@ -0,0 +1,182 @@
# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See the LICENSE file for details.
#
# This file is part of NVMe STorage Appliance Services (nvme-stas).
#
# Authors: Martin Belanger <Martin.Belanger@dell.com>
#
project(
'nvme-stas',
meson_version: '>= 0.53.0',
version: '2.2.1',
license: 'Apache-2.0',
default_options: [
'buildtype=release',
'prefix=/usr',
'sysconfdir=/etc',
]
)
fs = import('fs')
#===============================================================================
prefix = get_option('prefix')
datadir = prefix / get_option('datadir')
etcdir = prefix / get_option('sysconfdir')
bindir = prefix / get_option('bindir')
sbindir = prefix / get_option('sbindir')
mandir = prefix / get_option('mandir')
docdir = datadir / 'doc' / 'nvme-stas'
cnfdir = etcdir / 'stas'
want_man = get_option('man')
want_html = get_option('html')
want_readthedocs = get_option('readthedocs')
buildtime_modules = []
if want_man or want_html or want_readthedocs
buildtime_modules += ['lxml']
endif
python3 = import('python').find_installation('python3', modules:buildtime_modules)
python_version = python3.language_version()
python_version_req = '>=3.6'
if not python_version.version_compare(python_version_req)
error('Python @0@ required. Found @1@ instead'.format(python_version_req, python_version))
endif
# Check if the runtime Python modules are present. These are not needed
# to build nvme-stas, but will be needed to run the tests.
missing_runtime_mods = false
py_modules_reqd = [
['libnvme', 'Install python3-libnvme (deb/rpm)'],
['dasbus', 'Install python3-dasbus (deb/rpm) OR pip3 install dasbus'],
['pyudev', 'Install python3-pyudev (deb/rpm)'],
['systemd', 'Install python3-systemd (deb/rpm)'],
['gi', 'Install python3-gi (deb) OR python3-gobject (rpm)'],
]
foreach p : py_modules_reqd
if run_command(python3, '-c', 'import @0@'.format(p[0]), check: false).returncode() != 0
warning('Missing runtime module "@0@". @1@'.format(p[0], p[1]))
missing_runtime_mods = true
endif
endforeach
if missing_runtime_mods and get_option('rt_pymods_reqd')
error('Please install missing runtime modules')
endif
#===============================================================================
conf = configuration_data()
conf.set('VERSION', meson.project_version())
conf.set('LICENSE', meson.project_license()[0])
conf.set('BUILD_DIR', meson.current_build_dir())
conf.set('STAFD_DBUS_NAME', 'org.nvmexpress.staf')
conf.set('STAFD_DBUS_PATH', '/org/nvmexpress/staf')
conf.set('STACD_DBUS_NAME', 'org.nvmexpress.stac')
conf.set('STACD_DBUS_PATH', '/org/nvmexpress/stac')
#===============================================================================
stafd = configure_file(
input: 'stafd.py',
output: 'stafd',
install_dir: sbindir,
copy: true,
)
stacd = configure_file(
input: 'stacd.py',
output: 'stacd',
install_dir: sbindir,
copy: true,
)
stafctl = configure_file(
input: 'stafctl.py',
output: 'stafctl',
install_dir: bindir,
copy: true,
)
stacctl = configure_file(
input: 'stacctl.py',
output: 'stacctl',
install_dir: bindir,
copy: true,
)
stasadm = configure_file(
input: 'stasadm.py',
output: 'stasadm',
install_dir: bindir,
copy: true,
)
#===========================================================================
install_subdir(
'etc/stas',
install_dir: etcdir,
)
#===========================================================================
foreach component : [ 'nvme-stas.spec', '.coveragerc', 'coverage.sh', ]
configure_file(
input: component + '.in',
output: component,
configuration: conf,
)
endforeach
#===========================================================================
# Make a list of modules to lint
modules_to_lint = [stafd, stafctl, stacd, stacctl, stasadm]
# Point Python Path to Current Build Dir.
# This is used by other meson.build files.
PYTHON_SEARCH_PATHS = [
conf.get('BUILD_DIR'),
conf.get('BUILD_DIR') / 'subprojects' / 'libnvme',
]
PYTHONPATH = ':'.join(PYTHON_SEARCH_PATHS)
#===========================================================================
subdir('staslib')
subdir('etc/dbus-1/system.d')
subdir('usr/lib/systemd/system')
subdir('test')
subdir('doc')
#===========================================================================
summary_dict = {
'prefix ': prefix,
'etcdir ': etcdir,
'cnfdir ': cnfdir,
'bindir ': bindir,
'sbindir ': sbindir,
'datadir ': datadir,
'mandir ': mandir,
'docdir ': docdir,
'dbus_conf_dir ': dbus_conf_dir,
'sd_unit_dir ': sd_unit_dir,
'build location ': meson.current_build_dir(),
'libnvme for tests ': libnvme_location,
}
summary(summary_dict, section: 'Directories')
summary_dict = {
'want_man ': want_man,
'want_html ': want_html,
'want_readthedocs ': want_readthedocs,
}
if meson.version().version_compare('>=0.57.0') # conf.keys()
foreach key : conf.keys()
if key not in ['BUILD_DIR', 'VERSION', 'LICENSE']
summary_dict += { key + ' ': conf.get(key) }
endif
endforeach
endif
summary(summary_dict, section: 'Configuration', bool_yn: true)

7
meson_options.txt Normal file
View file

@ -0,0 +1,7 @@
# -*- mode: meson -*-
option('man', type: 'boolean', value: false, description: 'build and install man pages')
option('html', type: 'boolean', value: false, description: 'build and install html pages')
option('readthedocs', type: 'boolean', value: false, description: 'to be used by Read-The-Docs documentation builder')
option('libnvme-sel', type: 'combo', value: 'subproject', choices: ['subproject', 'pre-installed'], description: 'Select the libnvme to be used for testing. Either libnvme built as a "subproject", or libnvme already installed on the system.')
option('rt_pymods_reqd', type: 'boolean', value: false, description: 'Make sure all run-time python modules are installed')

100
nvme-stas.spec.in Normal file
View file

@ -0,0 +1,100 @@
Name: nvme-stas
Summary: NVMe STorage Appliance Services
Version: @VERSION@
Release: 1%{?dist}
License: @LICENSE@
URL: https://github.com/linux-nvme/nvme-stas
BuildArch: noarch
BuildRequires: meson
BuildRequires: glib2-devel
#BuildRequires: libnvme-devel
BuildRequires: libxslt
BuildRequires: docbook-style-xsl
#BuildRequires: systemd-devel
BuildRequires: systemd-rpm-macros
BuildRequires: python3
#BuildRequires: python3-devel
#BuildRequires: python3-pyflakes
#BuildRequires: python3-pylint
#BuildRequires: pylint
#BuildRequires: python3-libnvme
#BuildRequires: python3-dasbus
#BuildRequires: python3-pyudev
#BuildRequires: python3-systemd
#BuildRequires: python3-gobject-devel
BuildRequires: python3-lxml
Requires: avahi
Requires: python3-libnvme
Requires: python3-dasbus
Requires: python3-pyudev
Requires: python3-systemd
Requires: python3-gobject
%description
nvme-stas is a Central Discovery Controller (CDC) client for Linux. It
handles Asynchronous Event Notifications (AEN), Automated NVMe subsystem
connection controls, Error handling and reporting, and Automatic (zeroconf)
and Manual configuration. nvme-stas is composed of two daemons:
stafd (STorage Appliance Finder) and stacd (STorage Appliance Connector).
%prep
%autosetup -p1 -n %{name}-%{version}
%build
%meson --wrap-mode=nodownload -Dman=true -Dhtml=true
%meson_build
%install
%meson_install
%check
%meson_test
%define services stacd.service stafd.service
%pre
%service_add_pre %services
%post
%service_add_post %services
%preun
%service_del_preun %services
%postun
%service_del_postun %services
%files
%license LICENSE
%doc README.md
%dir %{_sysconfdir}/stas
%config(noreplace) %{_sysconfdir}/stas/stacd.conf
%config(noreplace) %{_sysconfdir}/stas/stafd.conf
%{_sysconfdir}/stas/sys.conf.doc
%{_datadir}/dbus-1/system.d/org.nvmexpress.*.conf
%{_bindir}/stacctl
%{_bindir}/stafctl
%{_bindir}/stasadm
%{_sbindir}/stacd
%{_sbindir}/stafd
%{_unitdir}/stacd.service
%{_unitdir}/stafd.service
%{_unitdir}/stas-config.target
%{_unitdir}/stas-config@.service
%dir %{python3_sitelib}/staslib
%{python3_sitelib}/staslib/*
%doc %{_pkgdocdir}
%{_mandir}/man1/sta*.1*
%{_mandir}/man5/*.5*
%{_mandir}/man7/nvme*.7*
%{_mandir}/man8/sta*.8*
%changelog
* Wed May 18 2022 Martin Belanger <martin.belanger@dell.com> - Release 1.1
* Thu Mar 24 2022 Martin Belanger <martin.belanger@dell.com> - Release 1.0-rc4
-

101
stacctl.py Executable file
View file

@ -0,0 +1,101 @@
#!/usr/bin/python3
# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See the LICENSE file for details.
#
# This file is part of NVMe STorage Appliance Services (nvme-stas).
#
# Authors: Martin Belanger <Martin.Belanger@dell.com>
#
''' STorage Appliance Connector Control Utility
'''
import sys
import json
import pprint
from argparse import ArgumentParser
import dasbus.error
from dasbus.connection import SystemMessageBus
from staslib import defs
def tron(args): # pylint: disable=unused-argument
'''@brief Trace ON'''
bus = SystemMessageBus()
iface = bus.get_proxy(defs.STACD_DBUS_NAME, defs.STACD_DBUS_PATH)
iface.tron = True # pylint: disable=assigning-non-slot
print(f'tron = {iface.tron}') # Read value back from stacd and print
def troff(args): # pylint: disable=unused-argument
'''@brief Trace OFF'''
bus = SystemMessageBus()
iface = bus.get_proxy(defs.STACD_DBUS_NAME, defs.STACD_DBUS_PATH)
iface.tron = False # pylint: disable=assigning-non-slot
print(f'tron = {iface.tron}') # Read value back from stacd and print
def _extract_cid(ctrl):
return (
ctrl['transport'],
ctrl['traddr'],
ctrl['trsvcid'],
ctrl['host-traddr'],
ctrl['host-iface'],
ctrl['subsysnqn'],
)
def status(args): # pylint: disable=unused-argument
'''@brief retrieve stacd's status information'''
bus = SystemMessageBus()
iface = bus.get_proxy(defs.STACD_DBUS_NAME, defs.STACD_DBUS_PATH)
info = json.loads(iface.process_info())
info['controllers'] = iface.list_controllers(True)
for controller in info['controllers']:
transport, traddr, trsvcid, host_traddr, host_iface, subsysnqn = _extract_cid(controller)
controller.update(
json.loads(iface.controller_info(transport, traddr, trsvcid, host_traddr, host_iface, subsysnqn))
)
print(pprint.pformat(info, width=120))
def ls(args):
'''@brief list the I/O controller's that stacd is
connected (or trying to connect) to.
'''
bus = SystemMessageBus()
iface = bus.get_proxy(defs.STACD_DBUS_NAME, defs.STACD_DBUS_PATH)
info = iface.list_controllers(args.detailed)
print(pprint.pformat(info, width=120))
PARSER = ArgumentParser(description='STorage Appliance Connector (STAC)')
PARSER.add_argument('-v', '--version', action='store_true', help='Print version, then exit', default=False)
SUBPARSER = PARSER.add_subparsers(title='Commands')
PRSR = SUBPARSER.add_parser('tron', help='Trace ON')
PRSR.set_defaults(func=tron)
PRSR = SUBPARSER.add_parser('troff', help='Trace OFF')
PRSR.set_defaults(func=troff)
PRSR = SUBPARSER.add_parser('status', help='Show runtime status information about stacd')
PRSR.set_defaults(func=status)
PRSR = SUBPARSER.add_parser('ls', help='List I/O controllers')
PRSR.add_argument(
'-d', '--detailed', action='store_true', help='Print detailed info (default: "%(default)s")', default=False
)
PRSR.set_defaults(func=ls)
ARGS = PARSER.parse_args()
if ARGS.version:
print(f'nvme-stas {defs.VERSION}')
sys.exit(0)
try:
ARGS.func(ARGS)
except dasbus.error.DBusError:
sys.exit('Unable to communicate with stacd over D-Bus. Is stacd running?')

115
stacd.py Executable file
View file

@ -0,0 +1,115 @@
#!/usr/bin/python3
# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See the LICENSE file for details.
#
# This file is part of NVMe STorage Appliance Services (nvme-stas).
#
# Authors: Martin Belanger <Martin.Belanger@dell.com>
#
''' STorage Appliance Connector Daemon
'''
import sys
from argparse import ArgumentParser
from staslib import defs
# ******************************************************************************
def parse_args(conf_file: str):
'''Parse command line options'''
parser = ArgumentParser(description='STorage Appliance Connector (STAC). Must be root to run this program.')
parser.add_argument(
'-f',
'--conf-file',
action='store',
help='Configuration file (default: %(default)s)',
default=conf_file,
type=str,
metavar='FILE',
)
parser.add_argument(
'-s',
'--syslog',
action='store_true',
help='Send messages to syslog instead of stdout. Use this when running %(prog)s as a daemon. (default: %(default)s)',
default=False,
)
parser.add_argument('--tron', action='store_true', help='Trace ON. (default: %(default)s)', default=False)
parser.add_argument('-v', '--version', action='store_true', help='Print version, then exit', default=False)
return parser.parse_args()
ARGS = parse_args(defs.STACD_CONF_FILE)
if ARGS.version:
print(f'nvme-stas {defs.VERSION}')
print(f'libnvme {defs.LIBNVME_VERSION}')
sys.exit(0)
# ******************************************************************************
if __name__ == '__main__':
import json
import logging
from staslib import log, service, stas, udev # pylint: disable=ungrouped-imports
# Before going any further, make sure the script is allowed to run.
stas.check_if_allowed_to_continue()
class Dbus:
'''This is the DBus interface that external programs can use to
communicate with stacd.
'''
__dbus_xml__ = stas.load_idl('stacd.idl')
@property
def tron(self):
'''@brief Get Trace ON property'''
return STAC.tron
@tron.setter
def tron(self, value): # pylint: disable=no-self-use
'''@brief Set Trace ON property'''
STAC.tron = value
@property
def log_level(self) -> str:
'''@brief Get Log Level property'''
return log.level()
def process_info(self) -> str:
'''@brief Get status info (for debug)
@return A string representation of a json object.
'''
info = {
'tron': STAC.tron,
'log-level': self.log_level,
}
info.update(STAC.info())
return json.dumps(info)
def controller_info( # pylint: disable=too-many-arguments,no-self-use
self, transport, traddr, trsvcid, host_traddr, host_iface, subsysnqn
) -> str:
'''@brief D-Bus method used to return information about a controller'''
controller = STAC.get_controller(transport, traddr, trsvcid, host_traddr, host_iface, subsysnqn)
return json.dumps(controller.info()) if controller else '{}'
def list_controllers(self, detailed) -> list: # pylint: disable=no-self-use
'''@brief Return the list of I/O controller IDs'''
return [
controller.details() if detailed else controller.controller_id_dict()
for controller in STAC.get_controllers()
]
log.init(ARGS.syslog)
STAC = service.Stac(ARGS, Dbus())
STAC.run()
STAC = None
ARGS = None
udev.shutdown()
logging.shutdown()

184
stafctl.py Executable file
View file

@ -0,0 +1,184 @@
#!/usr/bin/python3
# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See the LICENSE file for details.
#
# This file is part of NVMe STorage Appliance Services (nvme-stas).
#
# Authors: Martin Belanger <Martin.Belanger@dell.com>
#
''' STorage Appliance Finder Control Utility
'''
import sys
import json
import pprint
from argparse import ArgumentParser
import dasbus.error
from dasbus.connection import SystemMessageBus
from staslib import defs
def tron(args): # pylint: disable=unused-argument
'''@brief Trace ON'''
bus = SystemMessageBus()
iface = bus.get_proxy(defs.STAFD_DBUS_NAME, defs.STAFD_DBUS_PATH)
iface.tron = True # pylint: disable=assigning-non-slot
print(f'tron = {iface.tron}') # Read value back from stafd and print
def troff(args): # pylint: disable=unused-argument
'''@brief Trace OFF'''
bus = SystemMessageBus()
iface = bus.get_proxy(defs.STAFD_DBUS_NAME, defs.STAFD_DBUS_PATH)
iface.tron = False # pylint: disable=assigning-non-slot
print(f'tron = {iface.tron}') # Read value back from stafd and print
def _extract_cid(ctrl):
return (
ctrl['transport'],
ctrl['traddr'],
ctrl['trsvcid'],
ctrl['host-traddr'],
ctrl['host-iface'],
ctrl['subsysnqn'],
)
def status(args): # pylint: disable=unused-argument
'''@brief retrieve stafd's status information'''
bus = SystemMessageBus()
iface = bus.get_proxy(defs.STAFD_DBUS_NAME, defs.STAFD_DBUS_PATH)
info = json.loads(iface.process_info())
info['controllers'] = iface.list_controllers(True)
for controller in info['controllers']:
transport, traddr, trsvcid, host_traddr, host_iface, subsysnqn = _extract_cid(controller)
controller['log_pages'] = iface.get_log_pages(transport, traddr, trsvcid, host_traddr, host_iface, subsysnqn)
controller.update(
json.loads(iface.controller_info(transport, traddr, trsvcid, host_traddr, host_iface, subsysnqn))
)
print(pprint.pformat(info, width=120))
def ls(args):
'''@brief list the discovery controller's that stafd is
connected (or trying to connect) to.
'''
bus = SystemMessageBus()
iface = bus.get_proxy(defs.STAFD_DBUS_NAME, defs.STAFD_DBUS_PATH)
info = iface.list_controllers(args.detailed)
print(pprint.pformat(info, width=120))
def dlp(args):
'''@brief retrieve a controller's discovery log pages from stafd'''
bus = SystemMessageBus()
iface = bus.get_proxy(defs.STAFD_DBUS_NAME, defs.STAFD_DBUS_PATH)
info = iface.get_log_pages(args.transport, args.traddr, args.trsvcid, args.host_traddr, args.host_iface, args.nqn)
print(pprint.pformat(info, width=120))
def adlp(args):
'''@brief retrieve all of the controller's discovery log pages from stafd'''
bus = SystemMessageBus()
iface = bus.get_proxy(defs.STAFD_DBUS_NAME, defs.STAFD_DBUS_PATH)
info = json.loads(iface.get_all_log_pages(args.detailed))
print(pprint.pformat(info, width=120))
PARSER = ArgumentParser(description='STorage Appliance Finder (STAF)')
PARSER.add_argument('-v', '--version', action='store_true', help='Print version, then exit', default=False)
SUBPARSER = PARSER.add_subparsers(title='Commands')
PRSR = SUBPARSER.add_parser('tron', help='Trace ON')
PRSR.set_defaults(func=tron)
PRSR = SUBPARSER.add_parser('troff', help='Trace OFF')
PRSR.set_defaults(func=troff)
PRSR = SUBPARSER.add_parser('status', help='Show runtime status information about stafd')
PRSR.set_defaults(func=status)
PRSR = SUBPARSER.add_parser('ls', help='List discovery controllers')
PRSR.add_argument(
'-d',
'--detailed',
action='store_true',
help='Print detailed info (default: "%(default)s")',
default=False,
)
PRSR.set_defaults(func=ls)
PRSR = SUBPARSER.add_parser('dlp', help='Show discovery log pages')
PRSR.add_argument(
'-t',
'--transport',
metavar='<trtype>',
action='store',
help='NVMe-over-Fabrics fabric type (default: "%(default)s")',
choices=['tcp', 'rdma', 'fc', 'loop'],
default='tcp',
)
PRSR.add_argument(
'-a',
'--traddr',
metavar='<traddr>',
action='store',
help='Discovery Controller\'s network address',
required=True,
)
PRSR.add_argument(
'-s',
'--trsvcid',
metavar='<trsvcid>',
action='store',
help='Transport service id (for IP addressing, e.g. tcp, rdma, this field is the port number)',
required=True,
)
PRSR.add_argument(
'-w',
'--host-traddr',
metavar='<traddr>',
action='store',
help='Network address used on the host to connect to the Controller (default: "%(default)s")',
default='',
)
PRSR.add_argument(
'-f',
'--host-iface',
metavar='<iface>',
action='store',
help='This field specifies the network interface used on the host to connect to the Controller (default: "%(default)s")',
default='',
)
PRSR.add_argument(
'-n',
'--nqn',
metavar='<nqn>',
action='store',
help='This field specifies the discovery controller\'s NQN. When not specified this option defaults to "%(default)s"',
default=defs.WELL_KNOWN_DISC_NQN,
)
PRSR.set_defaults(func=dlp)
PRSR = SUBPARSER.add_parser('adlp', help='Show all discovery log pages')
PRSR.add_argument(
'-d',
'--detailed',
action='store_true',
help='Print detailed info (default: "%(default)s")',
default=False,
)
PRSR.set_defaults(func=adlp)
ARGS = PARSER.parse_args()
if ARGS.version:
print(f'nvme-stas {defs.VERSION}')
sys.exit(0)
try:
ARGS.func(ARGS)
except dasbus.error.DBusError:
sys.exit('Unable to communicate with stafd over D-Bus. Is stafd running?')

152
stafd.py Executable file
View file

@ -0,0 +1,152 @@
#!/usr/bin/python3
# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See the LICENSE file for details.
#
# This file is part of NVMe STorage Appliance Services (nvme-stas).
#
# Authors: Martin Belanger <Martin.Belanger@dell.com>
#
''' STorage Appliance Finder Daemon
'''
import sys
from argparse import ArgumentParser
from staslib import defs
# ******************************************************************************
def parse_args(conf_file: str):
'''Parse command line options'''
parser = ArgumentParser(description='STorage Appliance Finder (STAF). Must be root to run this program.')
parser.add_argument(
'-f',
'--conf-file',
action='store',
help='Configuration file (default: %(default)s)',
default=conf_file,
type=str,
metavar='FILE',
)
parser.add_argument(
'-s',
'--syslog',
action='store_true',
help='Send messages to syslog instead of stdout. Use this when running %(prog)s as a daemon. (default: %(default)s)',
default=False,
)
parser.add_argument('--tron', action='store_true', help='Trace ON. (default: %(default)s)', default=False)
parser.add_argument('-v', '--version', action='store_true', help='Print version, then exit', default=False)
return parser.parse_args()
ARGS = parse_args(defs.STAFD_CONF_FILE)
if ARGS.version:
print(f'nvme-stas {defs.VERSION}')
print(f'libnvme {defs.LIBNVME_VERSION}')
sys.exit(0)
# ******************************************************************************
if __name__ == '__main__':
import json
import logging
import dasbus.server.interface
from staslib import log, service, stas, udev # pylint: disable=ungrouped-imports
# Before going any further, make sure the script is allowed to run.
stas.check_if_allowed_to_continue()
class Dbus:
'''This is the DBus interface that external programs can use to
communicate with stafd.
'''
__dbus_xml__ = stas.load_idl('stafd.idl')
@dasbus.server.interface.dbus_signal
def log_pages_changed( # pylint: disable=too-many-arguments
self,
transport: str,
traddr: str,
trsvcid: str,
host_traddr: str,
host_iface: str,
subsysnqn: str,
device: str,
):
'''@brief Signal sent when log pages have changed.'''
@dasbus.server.interface.dbus_signal
def dc_removed(self):
'''@brief Signal sent when log pages have changed.'''
@property
def tron(self):
'''@brief Get Trace ON property'''
return STAF.tron
@tron.setter
def tron(self, value): # pylint: disable=no-self-use
'''@brief Set Trace ON property'''
STAF.tron = value
@property
def log_level(self) -> str:
'''@brief Get Log Level property'''
return log.level()
def process_info(self) -> str:
'''@brief Get status info (for debug)
@return A string representation of a json object.
'''
info = {
'tron': STAF.tron,
'log-level': self.log_level,
}
info.update(STAF.info())
return json.dumps(info)
def controller_info( # pylint: disable=no-self-use,too-many-arguments
self, transport, traddr, trsvcid, host_traddr, host_iface, subsysnqn
) -> str:
'''@brief D-Bus method used to return information about a controller'''
controller = STAF.get_controller(transport, traddr, trsvcid, host_traddr, host_iface, subsysnqn)
return json.dumps(controller.info()) if controller else '{}'
def get_log_pages( # pylint: disable=no-self-use,too-many-arguments
self, transport, traddr, trsvcid, host_traddr, host_iface, subsysnqn
) -> list:
'''@brief D-Bus method used to retrieve the discovery log pages from one controller'''
controller = STAF.get_controller(transport, traddr, trsvcid, host_traddr, host_iface, subsysnqn)
return controller.log_pages() if controller else list()
def get_all_log_pages(self, detailed) -> str: # pylint: disable=no-self-use
'''@brief D-Bus method used to retrieve the discovery log pages from all controllers'''
log_pages = list()
for controller in STAF.get_controllers():
log_pages.append(
{
'discovery-controller': controller.details() if detailed else controller.controller_id_dict(),
'log-pages': controller.log_pages(),
}
)
return json.dumps(log_pages)
def list_controllers(self, detailed) -> list: # pylint: disable=no-self-use
'''@brief Return the list of discovery controller IDs'''
return [
controller.details() if detailed else controller.controller_id_dict()
for controller in STAF.get_controllers()
]
log.init(ARGS.syslog)
STAF = service.Staf(ARGS, Dbus())
STAF.run()
STAF = None
ARGS = None
udev.shutdown()
logging.shutdown()

202
stasadm.py Executable file
View file

@ -0,0 +1,202 @@
#!/usr/bin/python3
# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See the LICENSE file for details.
#
# This file is part of NVMe STorage Appliance Services (nvme-stas).
#
# Authors: Martin Belanger <Martin.Belanger@dell.com>
#
''' STorage Appliance Services Admin Tool '''
import os
import sys
import uuid
import configparser
from argparse import ArgumentParser
from staslib import defs
try:
import hmac
import hashlib
except (ImportError, ModuleNotFoundError):
hmac = None
hashlib = None
def read_from_file(fname, size): # pylint: disable=missing-function-docstring
try:
with open(fname) as f: # pylint: disable=unspecified-encoding
data = f.read(size)
if len(data) == size:
return data
except FileNotFoundError:
pass
return None
def get_machine_app_specific(app_id):
'''@brief Get a machine ID specific to an application. We use the
value retrieved from /etc/machine-id. The documentation states that
/etc/machine-id:
"should be considered "confidential", and must not be exposed in
untrusted environments, in particular on the network. If a stable
unique identifier that is tied to the machine is needed for some
application, the machine ID or any part of it must not be used
directly. Instead the machine ID should be hashed with a crypto-
graphic, keyed hash function, using a fixed, application-specific
key. That way the ID will be properly unique, and derived in a
constant way from the machine ID but there will be no way to
retrieve the original machine ID from the application-specific one"
@note systemd's C function sd_id128_get_machine_app_specific() was the
inspiration for this code.
@ref https://www.freedesktop.org/software/systemd/man/machine-id.html
'''
if not hmac:
return None
data = read_from_file('/etc/machine-id', 32)
if not data:
return None
hmac_obj = hmac.new(app_id, uuid.UUID(data).bytes, hashlib.sha256)
id128_bytes = hmac_obj.digest()[0:16]
return str(uuid.UUID(bytes=id128_bytes, version=4))
def get_uuid_from_system():
'''@brief Try to find system UUID in the following order:
1) /etc/machine-id
2) /sys/class/dmi/id/product_uuid
3) /proc/device-tree/ibm,partition-uuid
'''
uuid_str = get_machine_app_specific(b'$nvmexpress.org$')
if uuid_str:
return uuid_str
# The following files are only readable by root
if os.geteuid() != 0:
sys.exit('Permission denied. Root privileges required.')
id128 = read_from_file('/sys/class/dmi/id/product_uuid', 36)
if id128:
# Swap little-endian to network order per
# DMTF SMBIOS 3.0 Section 7.2.1 System UUID.
swapped = ''.join([id128[x] for x in (6, 7, 4, 5, 2, 3, 0, 1, 8, 11, 12, 9, 10, 13, 16, 17, 14, 15)])
return swapped + id128[18:]
return read_from_file('/proc/device-tree/ibm,partition-uuid', 36)
def save(section, option, string, conf_file, fname):
'''@brief Save configuration
@param section: section in @conf_file where @option will be added
@param option: option to be added under @section in @conf_file
@param string: Text to be saved to @fname
@param conf_file: Configuration file name
@param fname: Optional file where @string will be saved
'''
if fname and string is not None:
with open(fname, 'w') as f: # pylint: disable=unspecified-encoding
print(string, file=f)
if conf_file:
config = configparser.ConfigParser(
default_section=None, allow_no_value=True, delimiters=('='), interpolation=None, strict=False
)
if os.path.isfile(conf_file):
config.read(conf_file)
try:
config.add_section(section)
except configparser.DuplicateSectionError:
pass
if fname:
string = 'file://' + fname
if string is not None:
config.set(section, option, string)
else:
config.remove_option(section, option)
with open(conf_file, 'w') as f: # pylint: disable=unspecified-encoding
config.write(f)
def hostnqn(args):
'''@brief Configure the host NQN'''
uuid_str = get_uuid_from_system() or str(uuid.uuid4())
uuid_str = f'nqn.2014-08.org.nvmexpress:uuid:{uuid_str}'
save('Host', 'nqn', uuid_str, args.conf_file, args.file)
def hostid(args):
'''@brief Configure the host ID'''
save('Host', 'id', str(uuid.uuid4()), args.conf_file, args.file)
def set_symname(args):
'''@brief Define the host Symbolic Name'''
save('Host', 'symname', args.symname, args.conf_file, args.file)
def clr_symname(args):
'''@brief Undefine the host NQN'''
save('Host', 'symname', None, args.conf_file, None)
def get_parser(): # pylint: disable=missing-function-docstring
parser = ArgumentParser(description='Configuration utility for STAS.')
parser.add_argument('-v', '--version', action='store_true', help='Print version, then exit', default=False)
parser.add_argument(
'-c',
'--conf-file',
action='store',
help='Configuration file. Default %(default)s.',
default=defs.SYS_CONF_FILE,
type=str,
metavar='FILE',
)
subparser = parser.add_subparsers(title='Commands')
prsr = subparser.add_parser('hostnqn', help='Configure the host NQN. The NQN is auto-generated.')
prsr.add_argument(
'-f', '--file', action='store', help='Optional file where to save the NQN.', type=str, metavar='FILE'
)
prsr.set_defaults(cmd=hostnqn)
prsr = subparser.add_parser('hostid', help='Configure the host ID. The ID is auto-generated.')
prsr.add_argument(
'-f', '--file', action='store', help='Optional file where to save the ID.', type=str, metavar='FILE'
)
prsr.set_defaults(cmd=hostid)
prsr = subparser.add_parser('set-symname', help='Set the host symbolic')
prsr.add_argument(
'-f', '--file', action='store', help='Optional file where to save the symbolic name.', type=str, metavar='FILE'
)
prsr.add_argument('symname', action='store', help='Symbolic name', default=None, metavar='SYMNAME')
prsr.set_defaults(cmd=set_symname)
prsr = subparser.add_parser('clear-symname', help='Clear the host symbolic')
prsr.set_defaults(cmd=clr_symname)
return parser
PARSER = get_parser()
ARGS = PARSER.parse_args()
if ARGS.version:
print(f'nvme-stas {defs.VERSION}')
sys.exit(0)
try:
ARGS.cmd(ARGS)
except AttributeError as ex:
print(str(ex))
PARSER.print_usage()

1
staslib/.gitignore vendored Normal file
View file

@ -0,0 +1 @@
__pycache__

11
staslib/__init__.py Normal file
View file

@ -0,0 +1,11 @@
# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See the LICENSE file for details.
#
# This file is part of NVMe STorage Appliance Services (nvme-stas).
#
# Authors: Martin Belanger <Martin.Belanger@dell.com>
#
'''STorage Appliance Services'''
__version__ = '@VERSION@'

456
staslib/avahi.py Normal file
View file

@ -0,0 +1,456 @@
# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See the LICENSE file for details.
#
# This file is part of NVMe STorage Appliance Services (nvme-stas).
#
# Authors: Martin Belanger <Martin.Belanger@dell.com>
#
''' Module that provides a way to retrieve discovered
services from the Avahi daemon over D-Bus.
'''
import socket
import typing
import logging
import functools
import dasbus.error
import dasbus.connection
import dasbus.client.proxy
import dasbus.client.observer
from gi.repository import GLib
from staslib import defs, conf, gutil
def _txt2dict(txt: list):
'''@param txt: A list of list of integers. The integers are the ASCII value
of printable text characters.
'''
the_dict = dict()
for list_of_chars in txt:
try:
string = functools.reduce(lambda accumulator, c: accumulator + chr(c), list_of_chars, '')
key, val = string.split("=")
the_dict[key.lower()] = val
except Exception: # pylint: disable=broad-except
pass
return the_dict
def _proto2trans(protocol):
'''Return the matching transport for the given protocol.'''
if protocol is None:
return None
protocol = protocol.strip().lower()
if protocol == 'tcp':
return 'tcp'
if protocol in ('roce', 'iwarp', 'rdma'):
return 'rdma'
return None
# ******************************************************************************
class Avahi: # pylint: disable=too-many-instance-attributes
'''@brief Avahi Server proxy. Set up the D-Bus connection to the Avahi
daemon and register to be notified when services of a certain
type (stype) are discovered or lost.
'''
DBUS_NAME = 'org.freedesktop.Avahi'
DBUS_INTERFACE_SERVICE_BROWSER = DBUS_NAME + '.ServiceBrowser'
DBUS_INTERFACE_SERVICE_RESOLVER = DBUS_NAME + '.ServiceResolver'
LOOKUP_USE_MULTICAST = 2
IF_UNSPEC = -1
PROTO_INET = 0
PROTO_INET6 = 1
PROTO_UNSPEC = -1
LOOKUP_RESULT_LOCAL = 8 # This record/service resides on and was announced by the local host
LOOKUP_RESULT_CACHED = 1 # This response originates from the cache
LOOKUP_RESULT_STATIC = 32 # The returned data has been defined statically by some configuration option
LOOKUP_RESULT_OUR_OWN = 16 # This service belongs to the same local client as the browser object
LOOKUP_RESULT_WIDE_AREA = 2 # This response originates from wide area DNS
LOOKUP_RESULT_MULTICAST = 4 # This response originates from multicast DNS
result_flags = {
LOOKUP_RESULT_LOCAL: 'local',
LOOKUP_RESULT_CACHED: 'cache',
LOOKUP_RESULT_STATIC: 'static',
LOOKUP_RESULT_OUR_OWN: 'own',
LOOKUP_RESULT_WIDE_AREA: 'wan',
LOOKUP_RESULT_MULTICAST: 'mcast',
}
protos = {PROTO_INET: 'IPv4', PROTO_INET6: 'IPv6', PROTO_UNSPEC: 'uspecified'}
@classmethod
def result_flags_as_string(cls, flags):
'''Convert flags to human-readable string'''
return '+'.join((value for flag, value in Avahi.result_flags.items() if (flags & flag) != 0))
@classmethod
def protocol_as_string(cls, proto):
'''Convert protocol codes to human-readable strings'''
return Avahi.protos.get(proto, 'unknown')
# ==========================================================================
def __init__(self, sysbus, change_cb):
self._change_cb = change_cb
self._services = dict()
self._sysbus = sysbus
self._stypes = set()
self._service_browsers = dict()
# Avahi is an on-demand service. If, for some reason, the avahi-daemon
# were to stop, we need to try to contact it for it to restart. For
# example, when installing the avahi-daemon package on a running system,
# the daemon doesn't get started right away. It needs another process to
# access it over D-Bus to wake it up. The following timer is used to
# periodically query the avahi-daemon until we successfully establish
# first contact.
self._kick_avahi_tmr = gutil.GTimer(60, self._on_kick_avahi)
# Subscribe for Avahi signals (i.e. events). This must be done before
# any Browser or Resolver is created to avoid race conditions and
# missed events.
self._subscriptions = [
self._sysbus.connection.signal_subscribe(
Avahi.DBUS_NAME,
Avahi.DBUS_INTERFACE_SERVICE_BROWSER,
'ItemNew',
None,
None,
0,
self._service_discovered,
),
self._sysbus.connection.signal_subscribe(
Avahi.DBUS_NAME,
Avahi.DBUS_INTERFACE_SERVICE_BROWSER,
'ItemRemove',
None,
None,
0,
self._service_removed,
),
self._sysbus.connection.signal_subscribe(
Avahi.DBUS_NAME, Avahi.DBUS_INTERFACE_SERVICE_BROWSER, 'Failure', None, None, 0, self._failure_handler
),
self._sysbus.connection.signal_subscribe(
Avahi.DBUS_NAME, Avahi.DBUS_INTERFACE_SERVICE_RESOLVER, 'Found', None, None, 0, self._service_identified
),
self._sysbus.connection.signal_subscribe(
Avahi.DBUS_NAME, Avahi.DBUS_INTERFACE_SERVICE_RESOLVER, 'Failure', None, None, 0, self._failure_handler
),
]
self._avahi = self._sysbus.get_proxy(Avahi.DBUS_NAME, '/')
self._avahi_watcher = dasbus.client.observer.DBusObserver(self._sysbus, Avahi.DBUS_NAME)
self._avahi_watcher.service_available.connect(self._avahi_available)
self._avahi_watcher.service_unavailable.connect(self._avahi_unavailable)
self._avahi_watcher.connect_once_available()
def kill(self):
'''@brief Clean up object'''
logging.debug('Avahi.kill()')
self._kick_avahi_tmr.kill()
self._kick_avahi_tmr = None
for subscription in self._subscriptions:
self._sysbus.connection.signal_unsubscribe(subscription)
self._subscriptions = list()
self._disconnect()
self._avahi_watcher.service_available.disconnect()
self._avahi_watcher.service_unavailable.disconnect()
self._avahi_watcher.disconnect()
self._avahi_watcher = None
dasbus.client.proxy.disconnect_proxy(self._avahi)
self._avahi = None
self._change_cb = None
self._sysbus = None
def info(self) -> dict:
'''@brief return debug info about this object'''
services = dict()
for service, obj in self._services.items():
interface, protocol, name, stype, domain = service
key = f'({socket.if_indextoname(interface)}, {Avahi.protos.get(protocol, "unknown")}, {name}.{domain}, {stype})'
services[key] = obj.get('data', {})
info = {
'avahi wake up timer': str(self._kick_avahi_tmr),
'service types': list(self._stypes),
'services': services,
}
return info
def get_controllers(self) -> list:
'''@brief Get the discovery controllers as a list of dict()
as follows:
[
{
'transport': tcp,
'traddr': str(),
'trsvcid': str(),
'host-iface': str(),
'subsysnqn': 'nqn.2014-08.org.nvmexpress.discovery',
},
{
'transport': tcp,
'traddr': str(),
'trsvcid': str(),
'host-iface': str(),
'subsysnqn': 'nqn.2014-08.org.nvmexpress.discovery',
},
[...]
]
'''
return [service['data'] for service in self._services.values() if len(service['data'])]
def config_stypes(self, stypes: list):
'''@brief Configure the service types that we want to discover.
@param stypes: A list of services types, e.g. ['_nvme-disc._tcp']
'''
self._stypes = set(stypes)
success = self._configure_browsers()
if not success:
self._kick_avahi_tmr.start()
def kick_start(self):
'''@brief We use this to kick start the Avahi
daemon (i.e. socket activation).
'''
self._kick_avahi_tmr.clear()
def _disconnect(self):
logging.debug('Avahi._disconnect()')
for service in self._services.values():
resolver = service.pop('resolver', None)
if resolver is not None:
try:
resolver.Free()
dasbus.client.proxy.disconnect_proxy(resolver)
except (AttributeError, dasbus.error.DBusError) as ex:
logging.debug('Avahi._disconnect() - Failed to Free() resolver. %s', ex)
self._services = dict()
for browser in self._service_browsers.values():
try:
browser.Free()
dasbus.client.proxy.disconnect_proxy(browser)
except (AttributeError, dasbus.error.DBusError) as ex:
logging.debug('Avahi._disconnect() - Failed to Free() browser. %s', ex)
self._service_browsers = dict()
def _on_kick_avahi(self):
try:
# try to contact avahi-daemon. This is just a wake
# up call in case the avahi-daemon was sleeping.
self._avahi.GetVersionString()
except dasbus.error.DBusError:
return GLib.SOURCE_CONTINUE
return GLib.SOURCE_REMOVE
def _avahi_available(self, _avahi_watcher):
'''@brief Hook up DBus signal handlers for signals from stafd.'''
logging.info('avahi-daemon service available, zeroconf supported.')
success = self._configure_browsers()
if not success:
self._kick_avahi_tmr.start()
def _avahi_unavailable(self, _avahi_watcher):
self._disconnect()
logging.warning('avahi-daemon not available, zeroconf not supported.')
self._kick_avahi_tmr.start()
def _configure_browsers(self):
stypes_cur = set(self._service_browsers.keys())
stypes_to_add = self._stypes - stypes_cur
stypes_to_rm = stypes_cur - self._stypes
logging.debug('Avahi._configure_browsers() - stypes_to_rm = %s', list(stypes_to_rm))
logging.debug('Avahi._configure_browsers() - stypes_to_add = %s', list(stypes_to_add))
for stype_to_rm in stypes_to_rm:
browser = self._service_browsers.pop(stype_to_rm, None)
if browser is not None:
try:
browser.Free()
dasbus.client.proxy.disconnect_proxy(browser)
except (AttributeError, dasbus.error.DBusError) as ex:
logging.debug('Avahi._configure_browsers() - Failed to Free() browser. %s', ex)
# Find the cached services corresponding to stype_to_rm and remove them
services_to_rm = [service for service in self._services if service[3] == stype_to_rm]
for service in services_to_rm:
resolver = self._services.pop(service, {}).pop('resolver', None)
if resolver is not None:
try:
resolver.Free()
dasbus.client.proxy.disconnect_proxy(resolver)
except (AttributeError, dasbus.error.DBusError) as ex:
logging.debug('Avahi._configure_browsers() - Failed to Free() resolver. %s', ex)
for stype in stypes_to_add:
try:
obj_path = self._avahi.ServiceBrowserNew(
Avahi.IF_UNSPEC, Avahi.PROTO_UNSPEC, stype, 'local', Avahi.LOOKUP_USE_MULTICAST
)
self._service_browsers[stype] = self._sysbus.get_proxy(Avahi.DBUS_NAME, obj_path)
except dasbus.error.DBusError as ex:
logging.debug('Avahi._configure_browsers() - Failed to contact avahi-daemon. %s', ex)
logging.warning('avahi-daemon not available, operating w/o mDNS discovery.')
return False
return True
def _service_discovered(
self,
_connection,
_sender_name: str,
_object_path: str,
_interface_name: str,
_signal_name: str,
args: typing.Tuple[int, int, str, str, str, int],
*_user_data,
):
(interface, protocol, name, stype, domain, flags) = args
logging.debug(
'Avahi._service_discovered() - interface=%s (%s), protocol=%s, stype=%s, domain=%s, flags=%s %-14s name=%s',
interface,
socket.if_indextoname(interface),
Avahi.protocol_as_string(protocol),
stype,
domain,
flags,
'(' + Avahi.result_flags_as_string(flags) + '),',
name,
)
service = (interface, protocol, name, stype, domain)
if service not in self._services:
try:
obj_path = self._avahi.ServiceResolverNew(
interface, protocol, name, stype, domain, Avahi.PROTO_UNSPEC, Avahi.LOOKUP_USE_MULTICAST
)
self._services[service] = {
'resolver': self._sysbus.get_proxy(Avahi.DBUS_NAME, obj_path),
'data': {},
}
except dasbus.error.DBusError as ex:
logging.warning('Failed to create resolver: "%s", "%s", "%s". %s', interface, name, stype, ex)
def _service_removed(
self,
_connection,
_sender_name: str,
_object_path: str,
_interface_name: str,
_signal_name: str,
args: typing.Tuple[int, int, str, str, str, int],
*_user_data,
):
(interface, protocol, name, stype, domain, flags) = args
logging.debug(
'Avahi._service_removed() - interface=%s (%s), protocol=%s, stype=%s, domain=%s, flags=%s %-14s name=%s',
interface,
socket.if_indextoname(interface),
Avahi.protocol_as_string(protocol),
stype,
domain,
flags,
'(' + Avahi.result_flags_as_string(flags) + '),',
name,
)
service = (interface, protocol, name, stype, domain)
resolver = self._services.pop(service, {}).pop('resolver', None)
if resolver is not None:
try:
resolver.Free()
dasbus.client.proxy.disconnect_proxy(resolver)
except (AttributeError, dasbus.error.DBusError) as ex:
logging.debug('Avahi._service_removed() - Failed to Free() resolver. %s', ex)
self._change_cb()
def _service_identified( # pylint: disable=too-many-locals
self,
_connection,
_sender_name: str,
_object_path: str,
_interface_name: str,
_signal_name: str,
args: typing.Tuple[int, int, str, str, str, str, int, str, int, list, int],
*_user_data,
):
(interface, protocol, name, stype, domain, host, aprotocol, address, port, txt, flags) = args
txt = _txt2dict(txt)
logging.debug(
'Avahi._service_identified() - interface=%s (%s), protocol=%s, stype=%s, domain=%s, flags=%s %-14s name=%s, host=%s, aprotocol=%s, address=%s, port=%s, txt=%s',
interface,
socket.if_indextoname(interface),
Avahi.protocol_as_string(protocol),
stype,
domain,
flags,
'(' + Avahi.result_flags_as_string(flags) + '),',
name,
host,
Avahi.protocol_as_string(aprotocol),
address,
port,
txt,
)
service = (interface, protocol, name, stype, domain)
if service in self._services:
transport = _proto2trans(txt.get('p'))
if transport is not None:
self._services[service]['data'] = {
'transport': transport,
'traddr': address.strip(),
'trsvcid': str(port).strip(),
# host-iface permitted for tcp alone and not rdma
'host-iface': socket.if_indextoname(interface).strip() if transport == 'tcp' else '',
'subsysnqn': txt.get('nqn', defs.WELL_KNOWN_DISC_NQN).strip()
if conf.NvmeOptions().discovery_supp
else defs.WELL_KNOWN_DISC_NQN,
}
self._change_cb()
else:
logging.error(
'Received invalid/undefined protocol in mDNS TXT field: address=%s, iface=%s, TXT=%s',
address,
socket.if_indextoname(interface).strip(),
txt,
)
def _failure_handler( # pylint: disable=no-self-use
self,
_connection,
_sender_name: str,
_object_path: str,
interface_name: str,
_signal_name: str,
args: typing.Tuple[str],
*_user_data,
):
(error,) = args
if 'ServiceResolver' not in interface_name or 'TimeoutError' not in error:
# ServiceResolver may fire a timeout event after being Free'd(). This seems to be normal.
logging.error('Avahi._failure_handler() - name=%s, error=%s', interface_name, error)

703
staslib/conf.py Normal file
View file

@ -0,0 +1,703 @@
# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See the LICENSE file for details.
#
# This file is part of NVMe STorage Appliance Services (nvme-stas).
#
# Authors: Martin Belanger <Martin.Belanger@dell.com>
#
'''nvme-stas configuration module'''
import re
import os
import sys
import logging
import functools
import configparser
from staslib import defs, singleton, timeparse
__TOKEN_RE = re.compile(r'\s*;\s*')
__OPTION_RE = re.compile(r'\s*=\s*')
class InvalidOption(Exception):
'''Exception raised when an invalid option value is detected'''
def _parse_controller(controller):
'''@brief Parse a "controller" entry. Controller entries are strings
composed of several configuration parameters delimited by
semi-colons. Each configuration parameter is specified as a
"key=value" pair.
@return A dictionary of key-value pairs.
'''
options = dict()
tokens = __TOKEN_RE.split(controller)
for token in tokens:
if token:
try:
option, val = __OPTION_RE.split(token)
options[option.strip()] = val.strip()
except ValueError:
pass
return options
def _parse_single_val(text):
if isinstance(text, str):
return text
if not isinstance(text, list) or len(text) == 0:
return None
return text[-1]
def _parse_list(text):
return text if isinstance(text, list) else [text]
def _to_int(text):
try:
return int(_parse_single_val(text))
except (ValueError, TypeError):
raise InvalidOption # pylint: disable=raise-missing-from
def _to_bool(text, positive='true'):
return _parse_single_val(text).lower() == positive
def _to_ncc(text):
value = _to_int(text)
if value == 1: # 1 is invalid. A minimum of 2 is required (with the exception of 0, which is valid).
value = 2
return value
def _to_ip_family(text):
return tuple((4 if text == 'ipv4' else 6 for text in _parse_single_val(text).split('+')))
# ******************************************************************************
class OrderedMultisetDict(dict):
'''This class is used to change the behavior of configparser.ConfigParser
and allow multiple configuration parameters with the same key. The
result is a list of values.
'''
def __setitem__(self, key, value):
if key in self and isinstance(value, list):
self[key].extend(value)
else:
super().__setitem__(key, value)
def __getitem__(self, key):
value = super().__getitem__(key)
if isinstance(value, str):
return value.split('\n')
return value
class SvcConf(metaclass=singleton.Singleton): # pylint: disable=too-many-public-methods
'''Read and cache configuration file.'''
OPTION_CHECKER = {
'Global': {
'tron': {
'convert': _to_bool,
'default': False,
'txt-chk': lambda text: _parse_single_val(text).lower() in ('false', 'true'),
},
'kato': {
'convert': _to_int,
},
'pleo': {
'convert': functools.partial(_to_bool, positive='enabled'),
'default': True,
'txt-chk': lambda text: _parse_single_val(text).lower() in ('disabled', 'enabled'),
},
'ip-family': {
'convert': _to_ip_family,
'default': (4, 6),
'txt-chk': lambda text: _parse_single_val(text) in ('ipv4', 'ipv6', 'ipv4+ipv6', 'ipv6+ipv4'),
},
'queue-size': {
'convert': _to_int,
'rng-chk': lambda value: None if value in range(16, 1025) else range(16, 1025),
},
'hdr-digest': {
'convert': _to_bool,
'default': False,
'txt-chk': lambda text: _parse_single_val(text).lower() in ('false', 'true'),
},
'data-digest': {
'convert': _to_bool,
'default': False,
'txt-chk': lambda text: _parse_single_val(text).lower() in ('false', 'true'),
},
'ignore-iface': {
'convert': _to_bool,
'default': False,
'txt-chk': lambda text: _parse_single_val(text).lower() in ('false', 'true'),
},
'nr-io-queues': {
'convert': _to_int,
},
'ctrl-loss-tmo': {
'convert': _to_int,
},
'disable-sqflow': {
'convert': _to_bool,
'txt-chk': lambda text: _parse_single_val(text).lower() in ('false', 'true'),
},
'nr-poll-queues': {
'convert': _to_int,
},
'nr-write-queues': {
'convert': _to_int,
},
'reconnect-delay': {
'convert': _to_int,
},
### BEGIN: LEGACY SECTION TO BE REMOVED ###
'persistent-connections': {
'convert': _to_bool,
'default': False,
'txt-chk': lambda text: _parse_single_val(text).lower() in ('false', 'true'),
},
### END: LEGACY SECTION TO BE REMOVED ###
},
'Service Discovery': {
'zeroconf': {
'convert': functools.partial(_to_bool, positive='enabled'),
'default': True,
'txt-chk': lambda text: _parse_single_val(text).lower() in ('disabled', 'enabled'),
},
},
'Discovery controller connection management': {
'persistent-connections': {
'convert': _to_bool,
'default': True,
'txt-chk': lambda text: _parse_single_val(text).lower() in ('false', 'true'),
},
'zeroconf-connections-persistence': {
'convert': lambda text: timeparse.timeparse(_parse_single_val(text)),
'default': timeparse.timeparse('72hours'),
},
},
'I/O controller connection management': {
'disconnect-scope': {
'convert': _parse_single_val,
'default': 'only-stas-connections',
'txt-chk': lambda text: _parse_single_val(text)
in ('only-stas-connections', 'all-connections-matching-disconnect-trtypes', 'no-disconnect'),
},
'disconnect-trtypes': {
# Use set() to eliminate potential duplicates
'convert': lambda text: set(_parse_single_val(text).split('+')),
'default': [
'tcp',
],
'lst-chk': ('tcp', 'rdma', 'fc'),
},
'connect-attempts-on-ncc': {
'convert': _to_ncc,
'default': 0,
},
},
'Controllers': {
'controller': {
'convert': _parse_list,
'default': [],
},
'exclude': {
'convert': _parse_list,
'default': [],
},
### BEGIN: LEGACY SECTION TO BE REMOVED ###
'blacklist': {
'convert': _parse_list,
'default': [],
},
### END: LEGACY SECTION TO BE REMOVED ###
},
}
def __init__(self, default_conf=None, conf_file='/dev/null'):
self._config = None
self._defaults = default_conf if default_conf else {}
if self._defaults is not None and len(self._defaults) != 0:
self._valid_conf = {}
for section, option in self._defaults:
self._valid_conf.setdefault(section, set()).add(option)
else:
self._valid_conf = None
self._conf_file = conf_file
self.reload()
def reload(self):
'''@brief Reload the configuration file.'''
self._config = self._read_conf_file()
@property
def conf_file(self):
'''Return the configuration file name'''
return self._conf_file
def set_conf_file(self, fname):
'''Set the configuration file name and reload config'''
self._conf_file = fname
self.reload()
def get_option(self, section, option, ignore_default=False): # pylint: disable=too-many-locals
'''Retrieve @option from @section, convert raw text to
appropriate object type, and validate.'''
try:
checker = self.OPTION_CHECKER[section][option]
except KeyError:
logging.error('Requesting invalid section=%s and/or option=%s', section, option)
raise
default = checker.get('default', None)
try:
text = self._config.get(section=section, option=option)
except (configparser.NoSectionError, configparser.NoOptionError, KeyError):
return None if ignore_default else self._defaults.get((section, option), default)
return self._check(text, section, option, default)
tron = property(functools.partial(get_option, section='Global', option='tron'))
kato = property(functools.partial(get_option, section='Global', option='kato'))
ip_family = property(functools.partial(get_option, section='Global', option='ip-family'))
queue_size = property(functools.partial(get_option, section='Global', option='queue-size'))
hdr_digest = property(functools.partial(get_option, section='Global', option='hdr-digest'))
data_digest = property(functools.partial(get_option, section='Global', option='data-digest'))
ignore_iface = property(functools.partial(get_option, section='Global', option='ignore-iface'))
pleo_enabled = property(functools.partial(get_option, section='Global', option='pleo'))
nr_io_queues = property(functools.partial(get_option, section='Global', option='nr-io-queues'))
ctrl_loss_tmo = property(functools.partial(get_option, section='Global', option='ctrl-loss-tmo'))
disable_sqflow = property(functools.partial(get_option, section='Global', option='disable-sqflow'))
nr_poll_queues = property(functools.partial(get_option, section='Global', option='nr-poll-queues'))
nr_write_queues = property(functools.partial(get_option, section='Global', option='nr-write-queues'))
reconnect_delay = property(functools.partial(get_option, section='Global', option='reconnect-delay'))
zeroconf_enabled = property(functools.partial(get_option, section='Service Discovery', option='zeroconf'))
zeroconf_persistence_sec = property(
functools.partial(
get_option, section='Discovery controller connection management', option='zeroconf-connections-persistence'
)
)
disconnect_scope = property(
functools.partial(get_option, section='I/O controller connection management', option='disconnect-scope')
)
disconnect_trtypes = property(
functools.partial(get_option, section='I/O controller connection management', option='disconnect-trtypes')
)
connect_attempts_on_ncc = property(
functools.partial(get_option, section='I/O controller connection management', option='connect-attempts-on-ncc')
)
@property
def stypes(self):
'''@brief Get the DNS-SD/mDNS service types.'''
return ['_nvme-disc._tcp', '_nvme-disc._udp'] if self.zeroconf_enabled else list()
@property
def persistent_connections(self):
'''@brief return the "persistent-connections" config parameter'''
section = 'Discovery controller connection management'
option = 'persistent-connections'
value = self.get_option(section, option, ignore_default=True)
legacy = self.get_option('Global', 'persistent-connections', ignore_default=True)
if value is None and legacy is None:
return self._defaults.get((section, option), True)
return value or legacy
def get_controllers(self):
'''@brief Return the list of controllers in the config file.
Each controller is in the form of a dictionary as follows.
Note that some of the keys are optional.
{
'transport': [TRANSPORT],
'traddr': [TRADDR],
'trsvcid': [TRSVCID],
'host-traddr': [TRADDR],
'host-iface': [IFACE],
'subsysnqn': [NQN],
'dhchap-ctrl-secret': [KEY],
'hdr-digest': [BOOL]
'data-digest': [BOOL]
'nr-io-queues': [NUMBER]
'nr-write-queues': [NUMBER]
'nr-poll-queues': [NUMBER]
'queue-size': [SIZE]
'kato': [KATO]
'reconnect-delay': [SECONDS]
'ctrl-loss-tmo': [SECONDS]
'disable-sqflow': [BOOL]
}
'''
controller_list = self.get_option('Controllers', 'controller')
cids = [_parse_controller(controller) for controller in controller_list]
for cid in cids:
try:
# replace 'nqn' key by 'subsysnqn', if present.
cid['subsysnqn'] = cid.pop('nqn')
except KeyError:
pass
# Verify values of the options used to overload the matching [Global] options
for option in cid:
if option in self.OPTION_CHECKER['Global']:
value = self._check(cid[option], 'Global', option, None)
if value is not None:
cid[option] = value
return cids
def get_excluded(self):
'''@brief Return the list of excluded controllers in the config file.
Each excluded controller is in the form of a dictionary
as follows. All the keys are optional.
{
'transport': [TRANSPORT],
'traddr': [TRADDR],
'trsvcid': [TRSVCID],
'host-iface': [IFACE],
'subsysnqn': [NQN],
}
'''
controller_list = self.get_option('Controllers', 'exclude')
# 2022-09-20: Look for "blacklist". This is for backwards compatibility
# with releases 1.0 to 1.1.6. This is to be phased out (i.e. remove by 2024)
controller_list += self.get_option('Controllers', 'blacklist')
excluded = [_parse_controller(controller) for controller in controller_list]
for controller in excluded:
controller.pop('host-traddr', None) # remove host-traddr
try:
# replace 'nqn' key by 'subsysnqn', if present.
controller['subsysnqn'] = controller.pop('nqn')
except KeyError:
pass
return excluded
def _check(self, text, section, option, default):
checker = self.OPTION_CHECKER[section][option]
text_checker = checker.get('txt-chk', None)
if text_checker is not None and not text_checker(text):
logging.warning(
'File:%s [%s]: %s - Text check found invalid value "%s". Default will be used',
self.conf_file,
section,
option,
text,
)
return self._defaults.get((section, option), default)
converter = checker.get('convert', None)
try:
value = converter(text)
except InvalidOption:
logging.warning(
'File:%s [%s]: %s - Data converter found invalid value "%s". Default will be used',
self.conf_file,
section,
option,
text,
)
return self._defaults.get((section, option), default)
value_in_range = checker.get('rng-chk', None)
if value_in_range is not None:
expected_range = value_in_range(value)
if expected_range is not None:
logging.warning(
'File:%s [%s]: %s - "%s" is not within range %s..%s. Default will be used',
self.conf_file,
section,
option,
value,
min(expected_range),
max(expected_range),
)
return self._defaults.get((section, option), default)
list_checker = checker.get('lst-chk', None)
if list_checker:
values = set()
for item in value:
if item not in list_checker:
logging.warning(
'File:%s [%s]: %s - List checker found invalid item "%s" will be ignored.',
self.conf_file,
section,
option,
item,
)
else:
values.add(item)
if len(values) == 0:
return self._defaults.get((section, option), default)
value = list(values)
return value
def _read_conf_file(self):
'''@brief Read the configuration file if the file exists.'''
config = configparser.ConfigParser(
default_section=None,
allow_no_value=True,
delimiters=('='),
interpolation=None,
strict=False,
dict_type=OrderedMultisetDict,
)
if self._conf_file and os.path.isfile(self._conf_file):
config.read(self._conf_file)
# Parse Configuration and validate.
if self._valid_conf is not None:
invalid_sections = set()
for section in config.sections():
if section not in self._valid_conf:
invalid_sections.add(section)
else:
invalid_options = set()
for option in config.options(section):
if option not in self._valid_conf.get(section, []):
invalid_options.add(option)
if len(invalid_options) != 0:
logging.error(
'File:%s [%s] contains invalid options: %s',
self.conf_file,
section,
invalid_options,
)
if len(invalid_sections) != 0:
logging.error(
'File:%s contains invalid sections: %s',
self.conf_file,
invalid_sections,
)
return config
# ******************************************************************************
class SysConf(metaclass=singleton.Singleton):
'''Read and cache the host configuration file.'''
def __init__(self, conf_file=defs.SYS_CONF_FILE):
self._config = None
self._conf_file = conf_file
self.reload()
def reload(self):
'''@brief Reload the configuration file.'''
self._config = self._read_conf_file()
@property
def conf_file(self):
'''Return the configuration file name'''
return self._conf_file
def set_conf_file(self, fname):
'''Set the configuration file name and reload config'''
self._conf_file = fname
self.reload()
def as_dict(self):
'''Return configuration as a dictionary'''
return {
'hostnqn': self.hostnqn,
'hostid': self.hostid,
'hostkey': self.hostkey,
'symname': self.hostsymname,
}
@property
def hostnqn(self):
'''@brief return the host NQN
@return: Host NQN
@raise: Host NQN is mandatory. The program will terminate if a
Host NQN cannot be determined.
'''
try:
value = self.__get_value('Host', 'nqn', defs.NVME_HOSTNQN)
except FileNotFoundError as ex:
sys.exit(f'Error reading mandatory Host NQN (see stasadm --help): {ex}')
if value is not None and not value.startswith('nqn.'):
sys.exit(f'Error Host NQN "{value}" should start with "nqn."')
return value
@property
def hostid(self):
'''@brief return the host ID
@return: Host ID
@raise: Host ID is mandatory. The program will terminate if a
Host ID cannot be determined.
'''
try:
value = self.__get_value('Host', 'id', defs.NVME_HOSTID)
except FileNotFoundError as ex:
sys.exit(f'Error reading mandatory Host ID (see stasadm --help): {ex}')
return value
@property
def hostkey(self):
'''@brief return the host key
@return: Host key
@raise: Host key is optional, but mandatory if authorization will be performed.
'''
try:
value = self.__get_value('Host', 'key', defs.NVME_HOSTKEY)
except FileNotFoundError as ex:
logging.info('Host key undefined: %s', ex)
value = None
return value
@property
def hostsymname(self):
'''@brief return the host symbolic name (or None)
@return: symbolic name or None
'''
try:
value = self.__get_value('Host', 'symname')
except FileNotFoundError as ex:
logging.warning('Error reading host symbolic name (will remain undefined): %s', ex)
value = None
return value
def _read_conf_file(self):
'''@brief Read the configuration file if the file exists.'''
config = configparser.ConfigParser(
default_section=None, allow_no_value=True, delimiters=('='), interpolation=None, strict=False
)
if os.path.isfile(self._conf_file):
config.read(self._conf_file)
return config
def __get_value(self, section, option, default_file=None):
'''@brief A configuration file consists of sections, each led by a
[section] header, followed by key/value entries separated
by a equal sign (=). This method retrieves the value
associated with the key @option from the section @section.
If the value starts with the string "file://", then the value
will be retrieved from that file.
@param section: Configuration section
@param option: The key to look for
@param default_file: A file that contains the default value
@return: On success, the value associated with the key. On failure,
this method will return None is a default_file is not
specified, or will raise an exception if a file is not
found.
@raise: This method will raise the FileNotFoundError exception if
the value retrieved is a file that does not exist.
'''
try:
value = self._config.get(section=section, option=option)
if not value.startswith('file://'):
return value
file = value[7:]
except (configparser.NoSectionError, configparser.NoOptionError, KeyError):
if default_file is None:
return None
file = default_file
try:
with open(file) as f: # pylint: disable=unspecified-encoding
return f.readline().split()[0]
except IndexError:
return None
# ******************************************************************************
class NvmeOptions(metaclass=singleton.Singleton):
'''Object used to read and cache contents of file /dev/nvme-fabrics.
Note that this file was not readable prior to Linux 5.16.
'''
def __init__(self):
# Supported options can be determined by looking at the kernel version
# or by reading '/dev/nvme-fabrics'. The ability to read the options
# from '/dev/nvme-fabrics' was only introduced in kernel 5.17, but may
# have been backported to older kernels. In any case, if the kernel
# version meets the minimum version for that option, then we don't
# even need to read '/dev/nvme-fabrics'.
self._supported_options = {
'discovery': defs.KERNEL_VERSION >= defs.KERNEL_TP8013_MIN_VERSION,
'host_iface': defs.KERNEL_VERSION >= defs.KERNEL_IFACE_MIN_VERSION,
'dhchap_secret': defs.KERNEL_VERSION >= defs.KERNEL_HOSTKEY_MIN_VERSION,
'dhchap_ctrl_secret': defs.KERNEL_VERSION >= defs.KERNEL_CTRLKEY_MIN_VERSION,
}
# If some of the options are False, we need to check wether they can be
# read from '/dev/nvme-fabrics'. This method allows us to determine that
# an older kernel actually supports a specific option because it was
# backported to that kernel.
if not all(self._supported_options.values()): # At least one option is False.
try:
with open('/dev/nvme-fabrics') as f: # pylint: disable=unspecified-encoding
options = [option.split('=')[0].strip() for option in f.readline().rstrip('\n').split(',')]
except PermissionError: # Must be root to read this file
raise
except (OSError, FileNotFoundError):
logging.warning('Cannot determine which NVMe options the kernel supports')
else:
for option, supported in self._supported_options.items():
if not supported:
self._supported_options[option] = option in options
def __str__(self):
return f'supported options: {self._supported_options}'
def get(self):
'''get the supported options as a dict'''
return self._supported_options
@property
def discovery_supp(self):
'''This option adds support for TP8013'''
return self._supported_options['discovery']
@property
def host_iface_supp(self):
'''This option allows forcing connections to go over
a specific interface regardless of the routing tables.
'''
return self._supported_options['host_iface']
@property
def dhchap_hostkey_supp(self):
'''This option allows specifying the host DHCHAP key used for authentication.'''
return self._supported_options['dhchap_secret']
@property
def dhchap_ctrlkey_supp(self):
'''This option allows specifying the controller DHCHAP key used for authentication.'''
return self._supported_options['dhchap_ctrl_secret']

850
staslib/ctrl.py Normal file
View file

@ -0,0 +1,850 @@
# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See the LICENSE file for details.
#
# This file is part of NVMe STorage Appliance Services (nvme-stas).
#
# Authors: Martin Belanger <Martin.Belanger@dell.com>
#
'''This module defines the base Controller object from which the
Dc (Discovery Controller) and Ioc (I/O Controller) objects are derived.'''
import time
import inspect
import logging
from gi.repository import GLib
from libnvme import nvme
from staslib import conf, defs, gutil, trid, udev, stas
DLP_CHANGED = (
(nvme.NVME_LOG_LID_DISCOVER << 16) | (nvme.NVME_AER_NOTICE_DISC_CHANGED << 8) | nvme.NVME_AER_NOTICE
) # 0x70f002
def get_eflags(dlpe):
'''@brief Return eflags field of dlpe'''
return int(dlpe.get('eflags', 0)) if dlpe else 0
def get_ncc(eflags: int):
'''@brief Return True if Not Connected to CDC bit is asserted, False otherwise'''
return eflags & nvme.NVMF_DISC_EFLAGS_NCC != 0
def dlp_supp_opts_as_string(dlp_supp_opts: int):
'''@brief Return the list of options supported by the Get
discovery log page command.
'''
data = {
nvme.NVMF_LOG_DISC_LID_EXTDLPES: "EXTDLPES",
nvme.NVMF_LOG_DISC_LID_PLEOS: "PLEOS",
nvme.NVMF_LOG_DISC_LID_ALLSUBES: "ALLSUBES",
}
return [txt for msk, txt in data.items() if dlp_supp_opts & msk]
# ******************************************************************************
class Controller(stas.ControllerABC): # pylint: disable=too-many-instance-attributes
'''@brief Base class used to manage the connection to a controller.'''
def __init__(self, tid: trid.TID, service, discovery_ctrl: bool = False):
sysconf = conf.SysConf()
self._nvme_options = conf.NvmeOptions()
self._root = nvme.root()
self._host = nvme.host(
self._root, hostnqn=sysconf.hostnqn, hostid=sysconf.hostid, hostsymname=sysconf.hostsymname
)
self._host.dhchap_key = sysconf.hostkey if self._nvme_options.dhchap_hostkey_supp else None
self._udev = udev.UDEV
self._device = None # Refers to the nvme device (e.g. /dev/nvme[n])
self._ctrl = None # libnvme's nvme.ctrl object
self._connect_op = None
super().__init__(tid, service, discovery_ctrl)
def _release_resources(self):
logging.debug('Controller._release_resources() - %s | %s', self.id, self.device)
if self._udev:
self._udev.unregister_for_device_events(self._on_udev_notification)
self._kill_ops()
super()._release_resources()
self._ctrl = None
self._udev = None
self._host = None
self._root = None
self._nvme_options = None
@property
def device(self) -> str:
'''@brief return the Linux nvme device id (e.g. nvme3) or empty
string if no device is associated with this controller'''
if not self._device and self._ctrl and self._ctrl.name:
self._device = self._ctrl.name
return self._device or 'nvme?'
def all_ops_completed(self) -> bool:
'''@brief Returns True if all operations have completed. False otherwise.'''
return self._connect_op is None or self._connect_op.completed()
def connected(self):
'''@brief Return whether a connection is established'''
return self._ctrl and self._ctrl.connected()
def controller_id_dict(self) -> dict:
'''@brief return the controller ID as a dict.'''
cid = super().controller_id_dict()
cid['device'] = self.device
return cid
def details(self) -> dict:
'''@brief return detailed debug info about this controller'''
details = super().details()
details.update(
self._udev.get_attributes(self.device, ('hostid', 'hostnqn', 'model', 'serial', 'dctype', 'cntrltype'))
)
details['connected'] = str(self.connected())
return details
def info(self) -> dict:
'''@brief Get the controller info for this object'''
info = super().info()
if self._connect_op:
info['connect operation'] = str(self._connect_op.as_dict())
return info
def cancel(self):
'''@brief Used to cancel pending operations.'''
super().cancel()
if self._connect_op:
self._connect_op.cancel()
def _kill_ops(self):
if self._connect_op:
self._connect_op.kill()
self._connect_op = None
def set_level_from_tron(self, tron):
'''Set log level based on TRON'''
if self._root:
self._root.log_level("debug" if tron else "err")
def _on_udev_notification(self, udev_obj):
if self._alive():
if udev_obj.action == 'change':
nvme_aen = udev_obj.get('NVME_AEN')
nvme_event = udev_obj.get('NVME_EVENT')
if isinstance(nvme_aen, str):
logging.info('%s | %s - Received AEN: %s', self.id, udev_obj.sys_name, nvme_aen)
self._on_aen(int(nvme_aen, 16))
if isinstance(nvme_event, str):
self._on_nvme_event(nvme_event)
elif udev_obj.action == 'remove':
logging.info('%s | %s - Received "remove" event', self.id, udev_obj.sys_name)
self._on_ctrl_removed(udev_obj)
else:
logging.debug(
'Controller._on_udev_notification() - %s | %s: Received "%s" event',
self.id,
udev_obj.sys_name,
udev_obj.action,
)
else:
logging.debug(
'Controller._on_udev_notification() - %s | %s: Received event on dead object. udev_obj %s: %s',
self.id,
self.device,
udev_obj.action,
udev_obj.sys_name,
)
def _on_ctrl_removed(self, udev_obj): # pylint: disable=unused-argument
if self._udev:
self._udev.unregister_for_device_events(self._on_udev_notification)
self._kill_ops() # Kill all pending operations
self._ctrl = None
# Defer removal of this object to the next main loop's idle period.
GLib.idle_add(self._serv.remove_controller, self, True)
def _get_cfg(self):
'''Get configuration parameters. These may either come from the [Global]
section or from a "controller" entry in the configuration file. A
definition found in a "controller" entry overrides the same definition
found in the [Global] section.
'''
cfg = {}
service_conf = conf.SvcConf()
for option, keyword in (
('kato', 'keep_alive_tmo'),
('queue-size', 'queue_size'),
('hdr-digest', 'hdr_digest'),
('data-digest', 'data_digest'),
('nr-io-queues', 'nr_io_queues'),
('ctrl-loss-tmo', 'ctrl_loss_tmo'),
('disable-sqflow', 'disable_sqflow'),
('nr-poll-queues', 'nr_poll_queues'),
('nr-write-queues', 'nr_write_queues'),
('reconnect-delay', 'reconnect_delay'),
):
# Check if the value is defined as a "controller" entry (i.e. override)
ovrd_val = self.tid.cfg.get(option, None)
if ovrd_val is not None:
cfg[keyword] = ovrd_val
else:
# Check if the value is found in the [Global] section.
glob_val = service_conf.get_option('Global', option)
if glob_val is not None:
cfg[keyword] = glob_val
return cfg
def _do_connect(self):
service_conf = conf.SvcConf()
host_iface = (
self.tid.host_iface
if (self.tid.host_iface and not service_conf.ignore_iface and self._nvme_options.host_iface_supp)
else None
)
self._ctrl = nvme.ctrl(
self._root,
subsysnqn=self.tid.subsysnqn,
transport=self.tid.transport,
traddr=self.tid.traddr,
trsvcid=self.tid.trsvcid if self.tid.trsvcid else None,
host_traddr=self.tid.host_traddr if self.tid.host_traddr else None,
host_iface=host_iface,
)
self._ctrl.discovery_ctrl_set(self._discovery_ctrl)
# Set the DHCHAP key on the controller
# NOTE that this will eventually have to
# change once we have support for AVE (TP8019)
ctrl_dhchap_key = self.tid.cfg.get('dhchap-ctrl-secret')
if ctrl_dhchap_key and self._nvme_options.dhchap_ctrlkey_supp:
has_dhchap_key = hasattr(self._ctrl, 'dhchap_key')
if not has_dhchap_key:
logging.warning(
'%s | %s - libnvme-%s does not allow setting the controller DHCHAP key. Please upgrade libnvme.',
self.id,
self.device,
defs.LIBNVME_VERSION,
)
else:
self._ctrl.dhchap_key = ctrl_dhchap_key
# Audit existing nvme devices. If we find a match, then
# we'll just borrow that device instead of creating a new one.
udev_obj = self._find_existing_connection()
if udev_obj is not None:
# A device already exists.
self._device = udev_obj.sys_name
logging.debug(
'Controller._do_connect() - %s Found existing control device: %s', self.id, udev_obj.sys_name
)
self._connect_op = gutil.AsyncTask(
self._on_connect_success, self._on_connect_fail, self._ctrl.init, self._host, int(udev_obj.sys_number)
)
else:
cfg = self._get_cfg()
logging.debug(
'Controller._do_connect() - %s Connecting to nvme control with cfg=%s', self.id, cfg
)
self._connect_op = gutil.AsyncTask(
self._on_connect_success, self._on_connect_fail, self._ctrl.connect, self._host, cfg
)
self._connect_op.run_async()
# --------------------------------------------------------------------------
def _on_connect_success(self, op_obj: gutil.AsyncTask, data):
'''@brief Function called when we successfully connect to the
Controller.
'''
op_obj.kill()
self._connect_op = None
if self._alive():
self._device = self._ctrl.name
logging.info('%s | %s - Connection established!', self.id, self.device)
self._connect_attempts = 0
self._udev.register_for_device_events(self._device, self._on_udev_notification)
else:
logging.debug(
'Controller._on_connect_success() - %s | %s: Received event on dead object. data=%s',
self.id,
self.device,
data,
)
def _on_connect_fail(self, op_obj: gutil.AsyncTask, err, fail_cnt): # pylint: disable=unused-argument
'''@brief Function called when we fail to connect to the Controller.'''
op_obj.kill()
self._connect_op = None
if self._alive():
if self._connect_attempts == 1:
# Do a fast re-try on the first failure.
self._retry_connect_tmr.set_timeout(self.FAST_CONNECT_RETRY_PERIOD_SEC)
elif self._connect_attempts == 2:
# If the fast connect re-try fails, then we can print a message to
# indicate the failure, and start a slow re-try period.
self._retry_connect_tmr.set_timeout(self.CONNECT_RETRY_PERIOD_SEC)
logging.error('%s Failed to connect to controller. %s %s', self.id, err.domain, err.message)
if self._should_try_to_reconnect():
logging.debug(
'Controller._on_connect_fail() - %s %s. Retry in %s sec.',
self.id,
err,
self._retry_connect_tmr.get_timeout(),
)
self._retry_connect_tmr.start()
else:
logging.debug(
'Controller._on_connect_fail() - %s Received event on dead object. %s %s',
self.id,
err.domain,
err.message,
)
def disconnect(self, disconnected_cb, keep_connection):
'''@brief Issue an asynchronous disconnect command to a Controller.
Once the async command has completed, the callback 'disconnected_cb'
will be invoked. If a controller is already disconnected, then the
callback will be added to the main loop's next idle slot to be executed
ASAP.
@param disconnected_cb: Callback to be called when disconnect has
completed. the callback must have this signature:
def cback(controller: Controller, success: bool)
@param keep_connection: Whether the underlying connection should remain
in the kernel.
'''
logging.debug(
'Controller.disconnect() - %s | %s: keep_connection=%s', self.id, self.device, keep_connection
)
if self._ctrl and self._ctrl.connected() and not keep_connection:
logging.info('%s | %s - Disconnect initiated', self.id, self.device)
op = gutil.AsyncTask(self._on_disconn_success, self._on_disconn_fail, self._ctrl.disconnect)
op.run_async(disconnected_cb)
else:
# Defer callback to the next main loop's idle period. The callback
# cannot be called directly as the current Controller object is in the
# process of being disconnected and the callback will in fact delete
# the object. This would invariably lead to unpredictable outcome.
GLib.idle_add(disconnected_cb, self, True)
def _on_disconn_success(self, op_obj: gutil.AsyncTask, data, disconnected_cb): # pylint: disable=unused-argument
logging.debug('Controller._on_disconn_success() - %s | %s', self.id, self.device)
op_obj.kill()
# Defer callback to the next main loop's idle period. The callback
# cannot be called directly as the current Controller object is in the
# process of being disconnected and the callback will in fact delete
# the object. This would invariably lead to unpredictable outcome.
GLib.idle_add(disconnected_cb, self, True)
def _on_disconn_fail(
self, op_obj: gutil.AsyncTask, err, fail_cnt, disconnected_cb
): # pylint: disable=unused-argument
logging.debug('Controller._on_disconn_fail() - %s | %s: %s', self.id, self.device, err)
op_obj.kill()
# Defer callback to the next main loop's idle period. The callback
# cannot be called directly as the current Controller object is in the
# process of being disconnected and the callback will in fact delete
# the object. This would invariably lead to unpredictable outcome.
GLib.idle_add(disconnected_cb, self, False)
# ******************************************************************************
class Dc(Controller):
'''@brief This object establishes a connection to one Discover Controller (DC).
It retrieves the discovery log pages and caches them.
It also monitors udev events associated with that DC and updates
the cached discovery log pages accordingly.
'''
GET_LOG_PAGE_RETRY_RERIOD_SEC = 20
REGISTRATION_RETRY_RERIOD_SEC = 5
GET_SUPPORTED_RETRY_RERIOD_SEC = 5
def __init__(self, staf, tid: trid.TID, log_pages=None, origin=None):
super().__init__(tid, staf, discovery_ctrl=True)
self._register_op = None
self._get_supported_op = None
self._get_log_op = None
self._origin = origin
self._log_pages = log_pages if log_pages else list() # Log pages cache
# For Avahi-discovered DCs that later become unresponsive, monitor how
# long the controller remains unresponsive and if it does not return for
# a configurable soak period (_ctrl_unresponsive_tmr), remove that
# controller. Only Avahi-discovered controllers need this timeout-based
# cleanup.
self._ctrl_unresponsive_time = None # The time at which connectivity was lost
self._ctrl_unresponsive_tmr = gutil.GTimer(0, self._serv.controller_unresponsive, self.tid)
def _release_resources(self):
logging.debug('Dc._release_resources() - %s | %s', self.id, self.device)
super()._release_resources()
if self._ctrl_unresponsive_tmr is not None:
self._ctrl_unresponsive_tmr.kill()
self._log_pages = list()
self._ctrl_unresponsive_tmr = None
def _kill_ops(self):
super()._kill_ops()
if self._get_log_op:
self._get_log_op.kill()
self._get_log_op = None
if self._register_op:
self._register_op.kill()
self._register_op = None
if self._get_supported_op:
self._get_supported_op.kill()
self._get_supported_op = None
def all_ops_completed(self) -> bool:
'''@brief Returns True if all operations have completed. False otherwise.'''
return (
super().all_ops_completed()
and (self._get_log_op is None or self._get_log_op.completed())
and (self._register_op is None or self._register_op.completed())
and (self._get_supported_op is None or self._get_supported_op.completed())
)
@property
def origin(self):
'''@brief Return how this controller came into existance. Was it
"discovered" through mDNS service discovery (TP8009), was it manually
"configured" in stafd.conf, or was it a "referral".
'''
return self._origin
@origin.setter
def origin(self, value):
'''@brief Set the origin of this controller.'''
if value in ('discovered', 'configured', 'referral'):
self._origin = value
self._handle_lost_controller()
else:
logging.error('%s | %s - Trying to set invalid origin to %s', self.id, self.device, value)
def reload_hdlr(self):
'''@brief This is called when a "reload" signal is received.'''
logging.debug('Dc.reload_hdlr() - %s | %s', self.id, self.device)
self._handle_lost_controller()
self._resync_with_controller()
def info(self) -> dict:
'''@brief Get the controller info for this object'''
timeout = conf.SvcConf().zeroconf_persistence_sec
unresponsive_time = (
time.asctime(self._ctrl_unresponsive_time) if self._ctrl_unresponsive_time is not None else '---'
)
info = super().info()
info['origin'] = self.origin
if self.origin == 'discovered':
# The code that handles "unresponsive" DCs only applies to
# discovered DCs. So, let's only print that info when it's relevant.
info['unresponsive timer'] = str(self._ctrl_unresponsive_tmr)
info['unresponsive timeout'] = f'{timeout} sec' if timeout >= 0 else 'forever'
info['unresponsive time'] = unresponsive_time
if self._get_log_op:
info['get log page operation'] = str(self._get_log_op.as_dict())
if self._register_op:
info['register operation'] = str(self._register_op.as_dict())
if self._get_supported_op:
info['get supported log page operation'] = str(self._get_supported_op.as_dict())
return info
def cancel(self):
'''@brief Used to cancel pending operations.'''
super().cancel()
if self._get_log_op:
self._get_log_op.cancel()
if self._register_op:
self._register_op.cancel()
if self._get_supported_op:
self._get_supported_op.cancel()
def log_pages(self) -> list:
'''@brief Get the cached log pages for this object'''
return self._log_pages
def referrals(self) -> list:
'''@brief Return the list of referrals'''
return [page for page in self._log_pages if page['subtype'] == 'referral']
def _is_ddc(self):
return self._ctrl and self._ctrl.dctype != 'cdc'
def _on_aen(self, aen: int):
if aen == DLP_CHANGED and self._get_log_op:
self._get_log_op.run_async()
def _handle_lost_controller(self):
if self.origin == 'discovered': # Only apply to mDNS-discovered DCs
if not self._serv.is_avahi_reported(self.tid) and not self.connected():
timeout = conf.SvcConf().zeroconf_persistence_sec
if timeout >= 0:
if self._ctrl_unresponsive_time is None:
self._ctrl_unresponsive_time = time.localtime()
self._ctrl_unresponsive_tmr.start(timeout)
logging.info(
'%s | %s - Controller is not responding. Will be removed by %s unless restored',
self.id,
self.device,
time.ctime(time.mktime(self._ctrl_unresponsive_time) + timeout),
)
return
logging.info(
'%s | %s - Controller not responding. Retrying...',
self.id,
self.device,
)
self._ctrl_unresponsive_time = None
self._ctrl_unresponsive_tmr.stop()
self._ctrl_unresponsive_tmr.set_timeout(0)
def is_unresponsive(self):
'''@brief For "discovered" DC, return True if DC is unresponsive,
False otherwise.
'''
return (
self.origin == 'discovered'
and not self._serv.is_avahi_reported(self.tid)
and not self.connected()
and self._ctrl_unresponsive_time is not None
and self._ctrl_unresponsive_tmr.time_remaining() <= 0
)
def _resync_with_controller(self):
'''Communicate with DC to resync the states'''
if self._register_op:
self._register_op.run_async()
elif self._get_supported_op:
self._get_supported_op.run_async()
elif self._get_log_op:
self._get_log_op.run_async()
def _on_nvme_event(self, nvme_event: str):
if nvme_event in ('connected', 'rediscover'):
# This event indicates that the kernel
# driver re-connected to the DC.
logging.debug(
'Dc._on_nvme_event() - %s | %s: Received "%s" event',
self.id,
self.device,
nvme_event,
)
self._resync_with_controller()
def _find_existing_connection(self):
return self._udev.find_nvme_dc_device(self.tid)
def _post_registration_actions(self):
# Need to check that supported_log_pages() is available (introduced in libnvme 1.2)
has_supported_log_pages = hasattr(self._ctrl, 'supported_log_pages')
if not has_supported_log_pages:
logging.warning(
'%s | %s - libnvme-%s does not support "Get supported log pages". Please upgrade libnvme.',
self.id,
self.device,
defs.LIBNVME_VERSION,
)
if conf.SvcConf().pleo_enabled and self._is_ddc() and has_supported_log_pages:
self._get_supported_op = gutil.AsyncTask(
self._on_get_supported_success, self._on_get_supported_fail, self._ctrl.supported_log_pages
)
self._get_supported_op.run_async()
else:
self._get_log_op = gutil.AsyncTask(self._on_get_log_success, self._on_get_log_fail, self._ctrl.discover)
self._get_log_op.run_async()
# --------------------------------------------------------------------------
def _on_connect_success(self, op_obj: gutil.AsyncTask, data):
'''@brief Function called when we successfully connect to the
Discovery Controller.
'''
super()._on_connect_success(op_obj, data)
if self._alive():
self._ctrl_unresponsive_time = None
self._ctrl_unresponsive_tmr.stop()
self._ctrl_unresponsive_tmr.set_timeout(0)
if self._ctrl.is_registration_supported():
self._register_op = gutil.AsyncTask(
self._on_registration_success,
self._on_registration_fail,
self._ctrl.registration_ctlr,
nvme.NVMF_DIM_TAS_REGISTER,
)
self._register_op.run_async()
else:
self._post_registration_actions()
def _on_connect_fail(self, op_obj: gutil.AsyncTask, err, fail_cnt):
'''@brief Function called when we fail to connect to the Controller.'''
super()._on_connect_fail(op_obj, err, fail_cnt)
if self._alive():
self._handle_lost_controller()
# --------------------------------------------------------------------------
def _on_registration_success(self, op_obj: gutil.AsyncTask, data): # pylint: disable=unused-argument
'''@brief Function called when we successfully register with the
Discovery Controller. See self._register_op object
for details.
NOTE: The name _on_registration_success() may be misleading. "success"
refers to the fact that a successful exchange was made with the DC.
It doesn't mean that the registration itself succeeded.
'''
if self._alive():
if data is not None:
logging.warning('%s | %s - Registration error. %s.', self.id, self.device, data)
else:
logging.debug('Dc._on_registration_success() - %s | %s', self.id, self.device)
self._post_registration_actions()
else:
logging.debug(
'Dc._on_registration_success() - %s | %s: Received event on dead object.', self.id, self.device
)
def _on_registration_fail(self, op_obj: gutil.AsyncTask, err, fail_cnt):
'''@brief Function called when we fail to register with the
Discovery Controller. See self._register_op object
for details.
'''
if self._alive():
logging.debug(
'Dc._on_registration_fail() - %s | %s: %s. Retry in %s sec',
self.id,
self.device,
err,
Dc.REGISTRATION_RETRY_RERIOD_SEC,
)
if fail_cnt == 1: # Throttle the logs. Only print the first time the command fails
logging.error('%s | %s - Failed to register with Discovery Controller. %s', self.id, self.device, err)
op_obj.retry(Dc.REGISTRATION_RETRY_RERIOD_SEC)
else:
logging.debug(
'Dc._on_registration_fail() - %s | %s: Received event on dead object. %s',
self.id,
self.device,
err,
)
op_obj.kill()
# --------------------------------------------------------------------------
def _on_get_supported_success(self, op_obj: gutil.AsyncTask, data): # pylint: disable=unused-argument
'''@brief Function called when we successfully retrieved the supported
log pages from the Discovery Controller. See self._get_supported_op object
for details.
NOTE: The name _on_get_supported_success() may be misleading. "success"
refers to the fact that a successful exchange was made with the DC.
It doesn't mean that the Get Supported Log Page itself succeeded.
'''
if self._alive():
try:
dlp_supp_opts = data[nvme.NVME_LOG_LID_DISCOVER] >> 16
except (TypeError, IndexError):
dlp_supp_opts = 0
logging.debug(
'Dc._on_get_supported_success() - %s | %s: supported options = 0x%04X = %s',
self.id,
self.device,
dlp_supp_opts,
dlp_supp_opts_as_string(dlp_supp_opts),
)
if 'lsp' in inspect.signature(self._ctrl.discover).parameters:
lsp = nvme.NVMF_LOG_DISC_LSP_PLEO if dlp_supp_opts & nvme.NVMF_LOG_DISC_LID_PLEOS else 0
self._get_log_op = gutil.AsyncTask(
self._on_get_log_success, self._on_get_log_fail, self._ctrl.discover, lsp
)
else:
logging.warning(
'%s | %s - libnvme-%s does not support setting PLEO bit. Please upgrade.',
self.id,
self.device,
defs.LIBNVME_VERSION,
)
self._get_log_op = gutil.AsyncTask(self._on_get_log_success, self._on_get_log_fail, self._ctrl.discover)
self._get_log_op.run_async()
else:
logging.debug(
'Dc._on_get_supported_success() - %s | %s: Received event on dead object.', self.id, self.device
)
def _on_get_supported_fail(self, op_obj: gutil.AsyncTask, err, fail_cnt):
'''@brief Function called when we fail to retrieve the supported log
page from the Discovery Controller. See self._get_supported_op object
for details.
'''
if self._alive():
logging.debug(
'Dc._on_get_supported_fail() - %s | %s: %s. Retry in %s sec',
self.id,
self.device,
err,
Dc.GET_SUPPORTED_RETRY_RERIOD_SEC,
)
if fail_cnt == 1: # Throttle the logs. Only print the first time the command fails
logging.error(
'%s | %s - Failed to Get supported log pages from Discovery Controller. %s',
self.id,
self.device,
err,
)
op_obj.retry(Dc.GET_SUPPORTED_RETRY_RERIOD_SEC)
else:
logging.debug(
'Dc._on_get_supported_fail() - %s | %s: Received event on dead object. %s',
self.id,
self.device,
err,
)
op_obj.kill()
# --------------------------------------------------------------------------
def _on_get_log_success(self, op_obj: gutil.AsyncTask, data): # pylint: disable=unused-argument
'''@brief Function called when we successfully retrieve the log pages
from the Discovery Controller. See self._get_log_op object
for details.
'''
if self._alive():
# Note that for historical reasons too long to explain, the CDC may
# return invalid addresses ('0.0.0.0', '::', or ''). Those need to
# be filtered out.
referrals_before = self.referrals()
self._log_pages = (
[
{k.strip(): str(v).strip() for k, v in dictionary.items()}
for dictionary in data
if dictionary.get('traddr', '').strip() not in ('0.0.0.0', '::', '')
]
if data
else list()
)
logging.info(
'%s | %s - Received discovery log pages (num records=%s).', self.id, self.device, len(self._log_pages)
)
referrals_after = self.referrals()
self._serv.log_pages_changed(self, self.device)
if referrals_after != referrals_before:
logging.debug(
'Dc._on_get_log_success() - %s | %s: Referrals before = %s',
self.id,
self.device,
referrals_before,
)
logging.debug(
'Dc._on_get_log_success() - %s | %s: Referrals after = %s',
self.id,
self.device,
referrals_after,
)
self._serv.referrals_changed()
else:
logging.debug(
'Dc._on_get_log_success() - %s | %s: Received event on dead object.', self.id, self.device
)
def _on_get_log_fail(self, op_obj: gutil.AsyncTask, err, fail_cnt):
'''@brief Function called when we fail to retrieve the log pages
from the Discovery Controller. See self._get_log_op object
for details.
'''
if self._alive():
logging.debug(
'Dc._on_get_log_fail() - %s | %s: %s. Retry in %s sec',
self.id,
self.device,
err,
Dc.GET_LOG_PAGE_RETRY_RERIOD_SEC,
)
if fail_cnt == 1: # Throttle the logs. Only print the first time the command fails
logging.error('%s | %s - Failed to retrieve log pages. %s', self.id, self.device, err)
op_obj.retry(Dc.GET_LOG_PAGE_RETRY_RERIOD_SEC)
else:
logging.debug(
'Dc._on_get_log_fail() - %s | %s: Received event on dead object. %s',
self.id,
self.device,
err,
)
op_obj.kill()
# ******************************************************************************
class Ioc(Controller):
'''@brief This object establishes a connection to one I/O Controller.'''
def __init__(self, stac, tid: trid.TID):
self._dlpe = None
super().__init__(tid, stac)
def _find_existing_connection(self):
return self._udev.find_nvme_ioc_device(self.tid)
def _on_aen(self, aen: int):
pass
def _on_nvme_event(self, nvme_event):
pass
def reload_hdlr(self):
'''@brief This is called when a "reload" signal is received.'''
if not self.connected() and self._retry_connect_tmr.time_remaining() == 0:
self._try_to_connect_deferred.schedule()
@property
def eflags(self):
'''@brief Return the eflag field of the DLPE'''
return get_eflags(self._dlpe)
@property
def ncc(self):
'''@brief Return Not Connected to CDC status'''
return get_ncc(self.eflags)
def details(self) -> dict:
'''@brief return detailed debug info about this controller'''
details = super().details()
details['dlpe'] = str(self._dlpe)
details['dlpe.eflags.ncc'] = str(self.ncc)
return details
def update_dlpe(self, dlpe):
'''@brief This method is called when a new DLPE associated
with this controller is received.'''
new_ncc = get_ncc(get_eflags(dlpe))
old_ncc = self.ncc
self._dlpe = dlpe
if old_ncc and not new_ncc: # NCC bit cleared?
if not self.connected():
self._connect_attempts = 0
self._try_to_connect_deferred.schedule()
def _should_try_to_reconnect(self):
'''@brief This is used to determine when it's time to stop trying toi connect'''
max_connect_attempts = conf.SvcConf().connect_attempts_on_ncc if self.ncc else 0
return max_connect_attempts == 0 or self._connect_attempts < max_connect_attempts

51
staslib/defs.py Normal file
View file

@ -0,0 +1,51 @@
# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See the LICENSE file for details.
#
# This file is part of NVMe STorage Appliance Services (nvme-stas).
#
# Authors: Martin Belanger <Martin.Belanger@dell.com>
''' @brief This file gets automagically configured by meson at build time.
'''
import os
import sys
import shutil
import platform
from staslib.version import KernelVersion
try:
import libnvme
LIBNVME_VERSION = libnvme.__version__
except (AttributeError, ModuleNotFoundError):
LIBNVME_VERSION = '?.?'
VERSION = '@VERSION@'
LICENSE = '@LICENSE@'
STACD_DBUS_NAME = '@STACD_DBUS_NAME@'
STACD_DBUS_PATH = '@STACD_DBUS_PATH@'
STAFD_DBUS_NAME = '@STAFD_DBUS_NAME@'
STAFD_DBUS_PATH = '@STAFD_DBUS_PATH@'
KERNEL_VERSION = KernelVersion(platform.release())
KERNEL_IFACE_MIN_VERSION = KernelVersion('5.14')
KERNEL_TP8013_MIN_VERSION = KernelVersion('5.16')
KERNEL_HOSTKEY_MIN_VERSION = KernelVersion('5.20')
KERNEL_CTRLKEY_MIN_VERSION = KernelVersion('5.20')
WELL_KNOWN_DISC_NQN = 'nqn.2014-08.org.nvmexpress.discovery'
PROG_NAME = os.path.basename(sys.argv[0])
NVME_HOSTID = '/etc/nvme/hostid'
NVME_HOSTNQN = '/etc/nvme/hostnqn'
NVME_HOSTKEY = '/etc/nvme/hostkey'
SYS_CONF_FILE = '/etc/stas/sys.conf'
STAFD_CONF_FILE = '/etc/stas/stafd.conf'
STACD_CONF_FILE = '/etc/stas/stacd.conf'
SYSTEMCTL = shutil.which('systemctl')

418
staslib/gutil.py Normal file
View file

@ -0,0 +1,418 @@
# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See the LICENSE file for details.
#
# This file is part of NVMe STorage Appliance Services (nvme-stas).
#
# Authors: Martin Belanger <Martin.Belanger@dell.com>
#
'''This module provides utility functions/classes to provide easier to use
access to GLib/Gio/Gobject resources.
'''
import logging
from gi.repository import Gio, GLib, GObject
from staslib import conf, iputil, trid
# ******************************************************************************
class GTimer:
'''@brief Convenience class to wrap GLib timers'''
def __init__(
self, interval_sec: float = 0, user_cback=lambda: GLib.SOURCE_REMOVE, *user_data, priority=GLib.PRIORITY_DEFAULT
): # pylint: disable=keyword-arg-before-vararg
self._source = None
self._interval_sec = float(interval_sec)
self._user_cback = user_cback
self._user_data = user_data
self._priority = priority if priority is not None else GLib.PRIORITY_DEFAULT
def _release_resources(self):
self.stop()
self._user_cback = None
self._user_data = None
def kill(self):
'''@brief Used to release all resources associated with a timer.'''
self._release_resources()
def __str__(self):
if self._source is not None:
return f'{self._interval_sec}s [{self.time_remaining()}s]'
return f'{self._interval_sec}s [off]'
def _callback(self, *_):
retval = self._user_cback(*self._user_data)
if retval == GLib.SOURCE_REMOVE:
self._source = None
return retval
def stop(self):
'''@brief Stop timer'''
if self._source is not None:
self._source.destroy()
self._source = None
def start(self, new_interval_sec: float = -1.0):
'''@brief Start (or restart) timer'''
if new_interval_sec >= 0:
self._interval_sec = float(new_interval_sec)
if self._source is not None:
self._source.set_ready_time(
self._source.get_time() + (self._interval_sec * 1000000)
) # ready time is in micro-seconds (monotonic time)
else:
if self._interval_sec.is_integer():
self._source = GLib.timeout_source_new_seconds(int(self._interval_sec)) # seconds resolution
else:
self._source = GLib.timeout_source_new(self._interval_sec * 1000.0) # mili-seconds resolution
self._source.set_priority(self._priority)
self._source.set_callback(self._callback)
self._source.attach()
def clear(self):
'''@brief Make timer expire now. The callback function
will be invoked immediately by the main loop.
'''
if self._source is not None:
self._source.set_ready_time(0) # Expire now!
def set_callback(self, user_cback, *user_data):
'''@brief set the callback function to invoke when timer expires'''
self._user_cback = user_cback
self._user_data = user_data
def set_timeout(self, new_interval_sec: float):
'''@brief set the timer's duration'''
if new_interval_sec >= 0:
self._interval_sec = float(new_interval_sec)
def get_timeout(self):
'''@brief get the timer's duration'''
return self._interval_sec
def time_remaining(self) -> float:
'''@brief Get how much time remains on a timer before it fires.'''
if self._source is not None:
delta_us = self._source.get_ready_time() - self._source.get_time() # monotonic time in micro-seconds
if delta_us > 0:
return delta_us / 1000000.0
return 0
# ******************************************************************************
class NameResolver: # pylint: disable=too-few-public-methods
'''@brief DNS resolver to convert host names to IP addresses.'''
def __init__(self):
self._resolver = Gio.Resolver.get_default()
def resolve_ctrl_async(self, cancellable, controllers_in: list, callback):
'''@brief The traddr fields may specify a hostname instead of an IP
address. We need to resolve all the host names to addresses.
Resolving hostnames may take a while as a DNS server may need
to be contacted. For that reason, we're using async APIs with
callbacks to resolve all the hostnames.
The callback @callback will be called once all hostnames have
been resolved.
@param controllers: List of trid.TID
'''
pending_resolution_count = 0
controllers_out = []
service_conf = conf.SvcConf()
def addr_resolved(resolver, result, controller):
try:
addresses = resolver.lookup_by_name_finish(result) # List of Gio.InetAddress objects
except GLib.GError as err:
# We don't need to report "cancellation" errors.
if err.matches(Gio.io_error_quark(), Gio.IOErrorEnum.CANCELLED):
# pylint: disable=no-member
logging.debug('NameResolver.resolve_ctrl_async() - %s %s', err.message, controller)
else:
logging.error('%s', err.message) # pylint: disable=no-member
# if err.matches(Gio.resolver_error_quark(), Gio.ResolverError.TEMPORARY_FAILURE):
# elif err.matches(Gio.resolver_error_quark(), Gio.ResolverError.NOT_FOUND):
# elif err.matches(Gio.resolver_error_quark(), Gio.ResolverError.INTERNAL):
else:
traddr = None
# If multiple addresses are returned (which is often the case),
# prefer IPv4 addresses over IPv6.
if 4 in service_conf.ip_family:
for address in addresses:
# There may be multiple IPv4 addresses. Pick 1st one.
if address.get_family() == Gio.SocketFamily.IPV4:
traddr = address.to_string()
break
if traddr is None and 6 in service_conf.ip_family:
for address in addresses:
# There may be multiple IPv6 addresses. Pick 1st one.
if address.get_family() == Gio.SocketFamily.IPV6:
traddr = address.to_string()
break
if traddr is not None:
logging.debug(
'NameResolver.resolve_ctrl_async() - resolved \'%s\' -> %s', controller.traddr, traddr
)
cid = controller.as_dict()
cid['traddr'] = traddr
nonlocal controllers_out
controllers_out.append(trid.TID(cid))
# Invoke callback after all hostnames have been resolved
nonlocal pending_resolution_count
pending_resolution_count -= 1
if pending_resolution_count == 0:
callback(controllers_out)
for controller in controllers_in:
if controller.transport in ('tcp', 'rdma'):
hostname_or_addr = controller.traddr
if not hostname_or_addr:
logging.error('Invalid traddr: %s', controller)
else:
# Try to convert to an ipaddress object. If this
# succeeds, then we don't need to call the resolver.
ip = iputil.get_ipaddress_obj(hostname_or_addr)
if ip is None:
logging.debug('NameResolver.resolve_ctrl_async() - resolving \'%s\'', hostname_or_addr)
pending_resolution_count += 1
self._resolver.lookup_by_name_async(hostname_or_addr, cancellable, addr_resolved, controller)
elif ip.version in service_conf.ip_family:
controllers_out.append(controller)
else:
logging.warning(
'Excluding configured IP address %s based on "ip-family" setting', hostname_or_addr
)
else:
controllers_out.append(controller)
if pending_resolution_count == 0: # No names are pending asynchronous resolution
callback(controllers_out)
# ******************************************************************************
class _TaskRunner(GObject.Object):
'''@brief This class allows running methods asynchronously in a thread.'''
def __init__(self, user_function, *user_args):
'''@param user_function: function to run inside a thread
@param user_args: arguments passed to @user_function
'''
super().__init__()
self._user_function = user_function
self._user_args = user_args
def communicate(self, cancellable, cb_function, *cb_args):
'''@param cancellable: A Gio.Cancellable object that can be used to
cancel an in-flight async command.
@param cb_function: User callback function to call when the async
command has completed. The callback function
will be passed these arguments:
(runner, result, *cb_args)
Where:
runner: This _TaskRunner object instance
result: A GObject.Object instance that contains the result
cb_args: The cb_args arguments passed to communicate()
@param cb_args: User arguments to pass to @cb_function
'''
def in_thread_exec(task, self, task_data, cancellable): # pylint: disable=unused-argument
if task.return_error_if_cancelled():
return # Bail out if task has been cancelled
try:
value = GObject.Object()
value.result = self._user_function(*self._user_args)
task.return_value(value)
except Exception as ex: # pylint: disable=broad-except
task.return_error(GLib.Error(message=str(ex), domain=type(ex).__name__))
task = Gio.Task.new(self, cancellable, cb_function, *cb_args)
task.set_return_on_cancel(False)
task.run_in_thread(in_thread_exec)
return task
def communicate_finish(self, result): # pylint: disable=no-self-use
'''@brief Use this function in your callback (see @cb_function) to
extract data from the result object.
@return On success (True, data, None),
On failure (False, None, err: GLib.Error)
'''
try:
success, value = result.propagate_value()
return success, value.result, None
except GLib.Error as err:
return False, None, err
# ******************************************************************************
class AsyncTask: # pylint: disable=too-many-instance-attributes
'''Object used to manage an asynchronous GLib operation. The operation
can be cancelled or retried.
'''
def __init__(self, on_success_callback, on_failure_callback, operation, *op_args):
'''@param on_success_callback: Callback method invoked when @operation completes successfully
@param on_failure_callback: Callback method invoked when @operation fails
@param operation: Operation (i.e. a function) to execute asynchronously
@param op_args: Arguments passed to operation
'''
self._cancellable = Gio.Cancellable()
self._operation = operation
self._op_args = op_args
self._success_cb = on_success_callback
self._fail_cb = on_failure_callback
self._retry_tmr = None
self._errmsg = None
self._task = None
self._fail_cnt = 0
def _release_resources(self):
if self._alive():
self._cancellable.cancel()
if self._retry_tmr is not None:
self._retry_tmr.kill()
self._operation = None
self._op_args = None
self._success_cb = None
self._fail_cb = None
self._retry_tmr = None
self._errmsg = None
self._task = None
self._fail_cnt = None
self._cancellable = None
def __str__(self):
return str(self.as_dict())
def as_dict(self):
'''Return object members as a dictionary'''
info = {
'fail count': self._fail_cnt,
'completed': self._task.get_completed(),
'alive': self._alive(),
}
if self._retry_tmr:
info['retry timer'] = str(self._retry_tmr)
if self._errmsg:
info['error'] = self._errmsg
return info
def _alive(self):
return self._cancellable and not self._cancellable.is_cancelled()
def completed(self):
'''@brief Returns True if the task has completed, False otherwise.'''
return self._task is not None and self._task.get_completed()
def cancel(self):
'''@brief cancel async operation'''
if self._alive():
self._cancellable.cancel()
def kill(self):
'''@brief kill and clean up this object'''
self._release_resources()
def run_async(self, *args):
'''@brief
Method used to initiate an asynchronous operation with the
Controller. When the operation completes (or fails) the
callback method @_on_operation_complete() will be invoked.
'''
runner = _TaskRunner(self._operation, *self._op_args)
self._task = runner.communicate(self._cancellable, self._on_operation_complete, *args)
def retry(self, interval_sec, *args):
'''@brief Tell this object that the async operation is to be retried
in @interval_sec seconds.
'''
if self._retry_tmr is None:
self._retry_tmr = GTimer()
self._retry_tmr.set_callback(self._on_retry_timeout, *args)
self._retry_tmr.start(interval_sec)
def _on_retry_timeout(self, *args):
'''@brief
When an operation fails, the application has the option to
retry at a later time by calling the retry() method. The
retry() method starts a timer at the end of which the operation
will be executed again. This is the method that is called when
the timer expires.
'''
if self._alive():
self.run_async(*args)
return GLib.SOURCE_REMOVE
def _on_operation_complete(self, runner, result, *args):
'''@brief
This callback method is invoked when the operation with the
Controller has completed (be it successful or not).
'''
# The operation might have been cancelled.
# Only proceed if it hasn't been cancelled.
if self._operation is None or not self._alive():
return
success, data, err = runner.communicate_finish(result)
if success:
self._errmsg = None
self._fail_cnt = 0
self._success_cb(self, data, *args)
else:
self._errmsg = str(err)
self._fail_cnt += 1
self._fail_cb(self, err, self._fail_cnt, *args)
# ******************************************************************************
class Deferred:
'''Implement a deferred function call. A deferred is a function that gets
added to the main loop to be executed during the next idle slot.'''
def __init__(self, func, *user_data):
self._source = None
self._func = func
self._user_data = user_data
def schedule(self):
'''Schedule the function to be called by the main loop. If the
function is already scheduled, then do nothing'''
if not self.is_scheduled():
srce_id = GLib.idle_add(self._func, *self._user_data)
self._source = GLib.main_context_default().find_source_by_id(srce_id)
def is_scheduled(self):
'''Check if deferred is currently schedules to run'''
return self._source and not self._source.is_destroyed()
def cancel(self):
'''Remove deferred from main loop'''
if self.is_scheduled():
self._source.destroy()
self._source = None

169
staslib/iputil.py Normal file
View file

@ -0,0 +1,169 @@
# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See the LICENSE file for details.
#
# This file is part of NVMe STorage Appliance Services (nvme-stas).
#
# Authors: Martin Belanger <Martin.Belanger@dell.com>
'''A collection of IP address and network interface utilities'''
import socket
import logging
import ipaddress
from staslib import conf
RTM_NEWADDR = 20
RTM_GETADDR = 22
NLM_F_REQUEST = 0x01
NLM_F_ROOT = 0x100
NLMSG_DONE = 3
IFLA_ADDRESS = 1
NLMSGHDR_SZ = 16
IFADDRMSG_SZ = 8
RTATTR_SZ = 4
# Netlink request (Get address command)
GETADDRCMD = (
# BEGIN: struct nlmsghdr
b'\0' * 4 # nlmsg_len (placeholder - actual length calculated below)
+ (RTM_GETADDR).to_bytes(2, byteorder='little', signed=False) # nlmsg_type
+ (NLM_F_REQUEST | NLM_F_ROOT).to_bytes(2, byteorder='little', signed=False) # nlmsg_flags
+ b'\0' * 2 # nlmsg_seq
+ b'\0' * 2 # nlmsg_pid
# END: struct nlmsghdr
+ b'\0' * 8 # struct ifaddrmsg
)
GETADDRCMD = len(GETADDRCMD).to_bytes(4, byteorder='little') + GETADDRCMD[4:] # nlmsg_len
# ******************************************************************************
def get_ipaddress_obj(ipaddr):
'''@brief Return a IPv4Address or IPv6Address depending on whether @ipaddr
is a valid IPv4 or IPv6 address. Return None otherwise.'''
try:
ip = ipaddress.ip_address(ipaddr)
except ValueError:
return None
return ip
# ******************************************************************************
def _data_matches_ip(data_family, data, ip):
if data_family == socket.AF_INET:
try:
other_ip = ipaddress.IPv4Address(data)
except ValueError:
return False
if ip.version == 6:
ip = ip.ipv4_mapped
elif data_family == socket.AF_INET6:
try:
other_ip = ipaddress.IPv6Address(data)
except ValueError:
return False
if ip.version == 4:
other_ip = other_ip.ipv4_mapped
else:
return False
return other_ip == ip
# ******************************************************************************
def iface_of(src_addr):
'''@brief Find the interface that has src_addr as one of its assigned IP addresses.
@param src_addr: The IP address to match
@type src_addr: Instance of ipaddress.IPv4Address or ipaddress.IPv6Address
'''
with socket.socket(socket.AF_NETLINK, socket.SOCK_RAW) as sock:
sock.sendall(GETADDRCMD)
nlmsg = sock.recv(8192)
nlmsg_idx = 0
while True:
if nlmsg_idx >= len(nlmsg):
nlmsg += sock.recv(8192)
nlmsg_type = int.from_bytes(nlmsg[nlmsg_idx + 4 : nlmsg_idx + 6], byteorder='little', signed=False)
if nlmsg_type == NLMSG_DONE:
break
if nlmsg_type != RTM_NEWADDR:
break
nlmsg_len = int.from_bytes(nlmsg[nlmsg_idx : nlmsg_idx + 4], byteorder='little', signed=False)
if nlmsg_len % 4: # Is msg length not a multiple of 4?
break
ifaddrmsg_indx = nlmsg_idx + NLMSGHDR_SZ
ifa_family = nlmsg[ifaddrmsg_indx]
ifa_index = int.from_bytes(nlmsg[ifaddrmsg_indx + 4 : ifaddrmsg_indx + 8], byteorder='little', signed=False)
rtattr_indx = ifaddrmsg_indx + IFADDRMSG_SZ
while rtattr_indx < (nlmsg_idx + nlmsg_len):
rta_len = int.from_bytes(nlmsg[rtattr_indx : rtattr_indx + 2], byteorder='little', signed=False)
rta_type = int.from_bytes(nlmsg[rtattr_indx + 2 : rtattr_indx + 4], byteorder='little', signed=False)
if rta_type == IFLA_ADDRESS:
data = nlmsg[rtattr_indx + RTATTR_SZ : rtattr_indx + rta_len]
if _data_matches_ip(ifa_family, data, src_addr):
return socket.if_indextoname(ifa_index)
rta_len = (rta_len + 3) & ~3 # Round up to multiple of 4
rtattr_indx += rta_len # Move to next rtattr
nlmsg_idx += nlmsg_len # Move to next Netlink message
return ''
# ******************************************************************************
def get_interface(src_addr):
'''Get interface for given source address
@param src_addr: The source address
@type src_addr: str
'''
if not src_addr:
return ''
src_addr = src_addr.split('%')[0] # remove scope-id (if any)
src_addr = get_ipaddress_obj(src_addr)
return '' if src_addr is None else iface_of(src_addr)
# ******************************************************************************
def remove_invalid_addresses(controllers: list):
'''@brief Remove controllers with invalid addresses from the list of controllers.
@param controllers: List of TIDs
'''
service_conf = conf.SvcConf()
valid_controllers = list()
for controller in controllers:
if controller.transport in ('tcp', 'rdma'):
# Let's make sure that traddr is
# syntactically a valid IPv4 or IPv6 address.
ip = get_ipaddress_obj(controller.traddr)
if ip is None:
logging.warning('%s IP address is not valid', controller)
continue
# Let's make sure the address family is enabled.
if ip.version not in service_conf.ip_family:
logging.debug(
'%s ignored because IPv%s is disabled in %s',
controller,
ip.version,
service_conf.conf_file,
)
continue
valid_controllers.append(controller)
elif controller.transport in ('fc', 'loop'):
# At some point, need to validate FC addresses as well...
valid_controllers.append(controller)
else:
logging.warning('Invalid transport %s', controller.transport)
return valid_controllers

53
staslib/log.py Normal file
View file

@ -0,0 +1,53 @@
# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See the LICENSE file for details.
#
# This file is part of NVMe STorage Appliance Services (nvme-stas).
#
# Authors: Martin Belanger <Martin.Belanger@dell.com>
#
'''nvme-stas logging module'''
import sys
import logging
from staslib import defs
def init(syslog: bool):
'''Init log module
@param syslog: True to send messages to the syslog,
False to send messages to stdout.
'''
log = logging.getLogger()
log.propagate = False
if syslog:
try:
# Try journal logger first
import systemd.journal # pylint: disable=import-outside-toplevel
handler = systemd.journal.JournalHandler(SYSLOG_IDENTIFIER=defs.PROG_NAME)
except ModuleNotFoundError:
# Go back to standard syslog handler
from logging.handlers import SysLogHandler # pylint: disable=import-outside-toplevel
handler = SysLogHandler(address="/dev/log")
handler.setFormatter(logging.Formatter(f'{defs.PROG_NAME}: %(message)s'))
else:
# Log to stdout
handler = logging.StreamHandler(stream=sys.stdout)
log.addHandler(handler)
log.setLevel(logging.INFO if syslog else logging.DEBUG)
def level() -> str:
'''@brief return current log level'''
logger = logging.getLogger()
return str(logging.getLevelName(logger.getEffectiveLevel()))
def set_level_from_tron(tron):
'''Set log level based on TRON'''
logger = logging.getLogger()
logger.setLevel(logging.DEBUG if tron else logging.INFO)

60
staslib/meson.build Normal file
View file

@ -0,0 +1,60 @@
# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See the LICENSE file for details.
#
# This file is part of NVMe STorage Appliance Services (nvme-stas).
#
# Authors: Martin Belanger <Martin.Belanger@dell.com>
#
files_to_configure = [ 'defs.py', '__init__.py', 'stafd.idl', 'stacd.idl' ]
configured_files = []
foreach file : files_to_configure
configured_files += configure_file(
input: file,
output: file,
configuration: conf
)
endforeach
files_to_copy = [
'avahi.py',
'conf.py',
'ctrl.py',
'gutil.py',
'iputil.py',
'log.py',
'service.py',
'singleton.py',
'stas.py',
'timeparse.py',
'trid.py',
'udev.py',
'version.py'
]
copied_files = []
foreach file : files_to_copy
copied_files += configure_file(
input: file,
output: file,
copy: true,
)
endforeach
files_to_install = copied_files + configured_files
python3.install_sources(
files_to_install,
pure: true,
subdir: 'staslib',
)
#===============================================================================
# Make a list of modules to lint
skip = ['stafd.idl', 'stacd.idl']
foreach file: files_to_install
fname = fs.name('@0@'.format(file))
if fname not in skip
modules_to_lint += file
endif
endforeach

878
staslib/service.py Normal file
View file

@ -0,0 +1,878 @@
# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See the LICENSE file for details.
#
# This file is part of NVMe STorage Appliance Services (nvme-stas).
#
# Authors: Martin Belanger <Martin.Belanger@dell.com>
#
'''This module defines the base Service object from
which the Staf and the Stac objects are derived.'''
import json
import logging
import pathlib
import subprocess
from itertools import filterfalse
import dasbus.error
import dasbus.client.observer
import dasbus.client.proxy
from gi.repository import GLib
from systemd.daemon import notify as sd_notify
from staslib import avahi, conf, ctrl, defs, gutil, iputil, stas, timeparse, trid, udev
# ******************************************************************************
class CtrlTerminator:
'''The Controller Terminator is used to gracefully disconnect from
controllers. All communications with controllers is handled by the kernel.
Once we make a request to the kernel to perform an operation (e.g. connect),
we have to wait for it to complete before requesting another operation. This
is particularly important when we want to disconnect from a controller while
there are pending operations, especially a pending connect.
The "connect" operation is especially unpredictable because all connect
requests are made through the blocking interface "/dev/nvme-fabrics". This
means that once a "connect" operation has been submitted, and depending on
how many connect requests are made concurrently, it can take several seconds
for a connect to be processed by the kernel.
While connect or other operations are being performed, it is possible
that a disconnect may be requested (e.g. someone or something changes the
configuration to remove a controller). Because it is not possible to
terminate a pending operation request, we have to wait for it to complete
before we can issue a disconnect. Failure to do that will result in
operations being performed by the kernel in reverse order. For example,
a disconnect may be executed before a pending connect has had a chance to
complete. And this will result in controllers that are supposed to be
disconnected to be connected without nvme-stas knowing about it.
The Controller Terminator is used when we need to disconnect from a
controller. It will make sure that there are no pending operations before
issuing a disconnect.
'''
DISPOSAL_AUDIT_PERIOD_SEC = 30
def __init__(self):
self._udev = udev.UDEV
self._controllers = list() # The list of controllers to dispose of.
self._audit_tmr = gutil.GTimer(self.DISPOSAL_AUDIT_PERIOD_SEC, self._on_disposal_check)
def dispose(self, controller: ctrl.Controller, on_controller_removed_cb, keep_connection: bool):
'''Invoked by a service (stafd or stacd) to dispose of a controller'''
if controller.all_ops_completed():
logging.debug(
'CtrlTerminator.dispose() - %s | %s: Invoke disconnect()', controller.tid, controller.device
)
controller.disconnect(on_controller_removed_cb, keep_connection)
else:
logging.debug(
'CtrlTerminator.dispose() - %s | %s: Add controller to garbage disposal',
controller.tid,
controller.device,
)
self._controllers.append((controller, keep_connection, on_controller_removed_cb, controller.tid))
self._udev.register_for_action_events('add', self._on_kernel_events)
self._udev.register_for_action_events('remove', self._on_kernel_events)
if self._audit_tmr.time_remaining() == 0:
self._audit_tmr.start()
def pending_disposal(self, tid):
'''Check whether @tid is pending disposal'''
for controller in self._controllers:
if controller.tid == tid:
return True
return False
def info(self):
'''@brief Get info about this object (used for debug)'''
info = {
'terminator.audit timer': str(self._audit_tmr),
}
for controller, _, _, tid in self._controllers:
info[f'terminator.controller.{tid}'] = str(controller.info())
return info
def kill(self):
'''Stop Controller Terminator and release resources.'''
self._audit_tmr.stop()
self._audit_tmr = None
if self._udev:
self._udev.unregister_for_action_events('add', self._on_kernel_events)
self._udev.unregister_for_action_events('remove', self._on_kernel_events)
self._udev = None
for controller, keep_connection, on_controller_removed_cb, _ in self._controllers:
controller.disconnect(on_controller_removed_cb, keep_connection)
self._controllers.clear()
def _on_kernel_events(self, udev_obj):
logging.debug('CtrlTerminator._on_kernel_events() - %s event received', udev_obj.action)
self._disposal_check()
def _on_disposal_check(self, *_user_data):
logging.debug('CtrlTerminator._on_disposal_check()- Periodic audit')
return GLib.SOURCE_REMOVE if self._disposal_check() else GLib.SOURCE_CONTINUE
@staticmethod
def _keep_or_terminate(args):
'''Return False if controller is to be kept. True if controller
was terminated and can be removed from the list.'''
controller, keep_connection, on_controller_removed_cb, tid = args
if controller.all_ops_completed():
logging.debug(
'CtrlTerminator._keep_or_terminate()- %s | %s: Disconnecting controller',
tid,
controller.device,
)
controller.disconnect(on_controller_removed_cb, keep_connection)
return True
return False
def _disposal_check(self):
# Iterate over the list, terminating (disconnecting) those controllers
# that have no pending operations, and remove those controllers from the
# list (only keep controllers that still have operations pending).
self._controllers[:] = filterfalse(self._keep_or_terminate, self._controllers)
disposal_complete = len(self._controllers) == 0
if disposal_complete:
logging.debug('CtrlTerminator._disposal_check() - Disposal complete')
self._audit_tmr.stop()
self._udev.unregister_for_action_events('add', self._on_kernel_events)
self._udev.unregister_for_action_events('remove', self._on_kernel_events)
else:
self._audit_tmr.start() # Restart timer
return disposal_complete
# ******************************************************************************
class Service(stas.ServiceABC):
'''@brief Base class used to manage a STorage Appliance Service'''
def __init__(self, args, default_conf, reload_hdlr):
self._udev = udev.UDEV
self._terminator = CtrlTerminator()
super().__init__(args, default_conf, reload_hdlr)
def _release_resources(self):
logging.debug('Service._release_resources()')
super()._release_resources()
if self._terminator:
self._terminator.kill()
self._udev = None
self._terminator = None
def _disconnect_all(self):
'''Tell all controller objects to disconnect'''
keep_connections = self._keep_connections_on_exit()
controllers = self._controllers.values()
logging.debug(
'Service._stop_hdlr() - Controller count = %s, keep_connections = %s',
len(controllers),
keep_connections,
)
for controller in controllers:
self._terminator.dispose(controller, self._on_final_disconnect, keep_connections)
def info(self) -> dict:
'''@brief Get the status info for this object (used for debug)'''
info = super().info()
if self._terminator:
info.update(self._terminator.info())
return info
@stas.ServiceABC.tron.setter
def tron(self, value):
'''@brief Set Trace ON property'''
super(__class__, self.__class__).tron.__set__(self, value)
# ******************************************************************************
class Stac(Service):
'''STorage Appliance Connector (STAC)'''
CONF_STABILITY_LONG_SOAK_TIME_SEC = 10 # pylint: disable=invalid-name
ADD_EVENT_SOAK_TIME_SEC = 1
def __init__(self, args, dbus):
default_conf = {
('Global', 'tron'): False,
('Global', 'hdr-digest'): False,
('Global', 'data-digest'): False,
('Global', 'kato'): None, # None to let the driver decide the default
('Global', 'nr-io-queues'): None, # None to let the driver decide the default
('Global', 'nr-write-queues'): None, # None to let the driver decide the default
('Global', 'nr-poll-queues'): None, # None to let the driver decide the default
('Global', 'queue-size'): None, # None to let the driver decide the default
('Global', 'reconnect-delay'): None, # None to let the driver decide the default
('Global', 'ctrl-loss-tmo'): None, # None to let the driver decide the default
('Global', 'disable-sqflow'): None, # None to let the driver decide the default
('Global', 'ignore-iface'): False,
('Global', 'ip-family'): (4, 6),
('Controllers', 'controller'): list(),
('Controllers', 'exclude'): list(),
('I/O controller connection management', 'disconnect-scope'): 'only-stas-connections',
('I/O controller connection management', 'disconnect-trtypes'): ['tcp'],
('I/O controller connection management', 'connect-attempts-on-ncc'): 0,
}
super().__init__(args, default_conf, self._reload_hdlr)
self._add_event_soak_tmr = gutil.GTimer(self.ADD_EVENT_SOAK_TIME_SEC, self._on_add_event_soaked)
self._config_connections_audit()
# Create the D-Bus instance.
self._config_dbus(dbus, defs.STACD_DBUS_NAME, defs.STACD_DBUS_PATH)
# Connect to STAF D-Bus interface
self._staf = None
self._staf_watcher = dasbus.client.observer.DBusObserver(self._sysbus, defs.STAFD_DBUS_NAME)
self._staf_watcher.service_available.connect(self._connect_to_staf)
self._staf_watcher.service_unavailable.connect(self._disconnect_from_staf)
self._staf_watcher.connect_once_available()
def _release_resources(self):
logging.debug('Stac._release_resources()')
if self._add_event_soak_tmr:
self._add_event_soak_tmr.kill()
if self._udev:
self._udev.unregister_for_action_events('add', self._on_add_event)
self._destroy_staf_comlink(self._staf_watcher)
if self._staf_watcher is not None:
self._staf_watcher.disconnect()
super()._release_resources()
self._staf = None
self._staf_watcher = None
self._add_event_soak_tmr = None
def _dump_last_known_config(self, controllers):
config = list(controllers.keys())
logging.debug('Stac._dump_last_known_config() - IOC count = %s', len(config))
self._write_lkc(config)
def _load_last_known_config(self):
config = self._read_lkc() or list()
logging.debug('Stac._load_last_known_config() - IOC count = %s', len(config))
controllers = {}
for tid in config:
# Only create Ioc objects if there is already a connection in the kernel
# First, regenerate the TID (in case of soft. upgrade and TID object
# has changed internally)
tid = trid.TID(tid.as_dict())
if udev.UDEV.find_nvme_ioc_device(tid) is not None:
controllers[tid] = ctrl.Ioc(self, tid)
return controllers
def _audit_all_connections(self, tids):
'''A host should only connect to I/O controllers that have been zoned
for that host or a manual "controller" entry exists in stacd.conf.
A host should disconnect from an I/O controller when that I/O controller
is removed from the zone or a "controller" entry is manually removed
from stacd.conf. stacd will audit connections if "disconnect-scope=
all-connections-matching-disconnect-trtypes". stacd will delete any
connection that is not supposed to exist.
'''
logging.debug('Stac._audit_all_connections() - tids = %s', tids)
num_controllers = len(self._controllers)
for tid in tids:
if tid not in self._controllers and not self._terminator.pending_disposal(tid):
self._controllers[tid] = ctrl.Ioc(self, tid)
if num_controllers != len(self._controllers):
self._cfg_soak_tmr.start(self.CONF_STABILITY_SOAK_TIME_SEC)
def _on_add_event(self, udev_obj):
'''@brief This function is called when a "add" event is received from
the kernel for an NVMe device. This is used to trigger an audit and make
sure that the connection to an I/O controller is allowed.
WARNING: There is a race condition with the "add" event from the kernel.
The kernel sends the "add" event a bit early and the sysfs attributes
associated with the nvme object are not always fully initialized.
To workaround this problem we use a soaking timer to give time for the
sysfs attributes to stabilize.
'''
logging.debug('Stac._on_add_event(() - Received "add" event: %s', udev_obj.sys_name)
self._add_event_soak_tmr.start()
def _on_add_event_soaked(self):
'''@brief After the add event has been soaking for ADD_EVENT_SOAK_TIME_SEC
seconds, we can audit the connections.
'''
if self._alive():
svc_conf = conf.SvcConf()
if svc_conf.disconnect_scope == 'all-connections-matching-disconnect-trtypes':
self._audit_all_connections(self._udev.get_nvme_ioc_tids(svc_conf.disconnect_trtypes))
return GLib.SOURCE_REMOVE
def _config_connections_audit(self):
'''This function checks the "disconnect_scope" parameter to determine
whether audits should be performed. Audits are enabled when
"disconnect_scope == all-connections-matching-disconnect-trtypes".
'''
svc_conf = conf.SvcConf()
if svc_conf.disconnect_scope == 'all-connections-matching-disconnect-trtypes':
if not self._udev.is_action_cback_registered('add', self._on_add_event):
self._udev.register_for_action_events('add', self._on_add_event)
self._audit_all_connections(self._udev.get_nvme_ioc_tids(svc_conf.disconnect_trtypes))
else:
self._udev.unregister_for_action_events('add', self._on_add_event)
def _keep_connections_on_exit(self):
'''@brief Determine whether connections should remain when the
process exits.
'''
return True
def _reload_hdlr(self):
'''@brief Reload configuration file. This is triggered by the SIGHUP
signal, which can be sent with "systemctl reload stacd".
'''
if not self._alive():
return GLib.SOURCE_REMOVE
sd_notify('RELOADING=1')
service_cnf = conf.SvcConf()
service_cnf.reload()
self.tron = service_cnf.tron
self._config_connections_audit()
self._cfg_soak_tmr.start(self.CONF_STABILITY_SOAK_TIME_SEC)
for controller in self._controllers.values():
controller.reload_hdlr()
sd_notify('READY=1')
return GLib.SOURCE_CONTINUE
def _get_log_pages_from_stafd(self):
if self._staf:
try:
return json.loads(self._staf.get_all_log_pages(True))
except dasbus.error.DBusError:
pass
return list()
def _config_ctrls_finish(self, configured_ctrl_list: list): # pylint: disable=too-many-locals
'''@param configured_ctrl_list: list of TIDs'''
# This is a callback function, which may be called after the service
# has been signalled to stop. So let's make sure the service is still
# alive and well before continuing.
if not self._alive():
logging.debug('Stac._config_ctrls_finish() - Exiting because service is no longer alive')
return
# Eliminate invalid entries from stacd.conf "controller list".
configured_ctrl_list = [
tid for tid in configured_ctrl_list if '' not in (tid.transport, tid.traddr, tid.trsvcid, tid.subsysnqn)
]
logging.debug('Stac._config_ctrls_finish() - configured_ctrl_list = %s', configured_ctrl_list)
discovered_ctrls = dict()
for staf_data in self._get_log_pages_from_stafd():
host_traddr = staf_data['discovery-controller']['host-traddr']
host_iface = staf_data['discovery-controller']['host-iface']
for dlpe in staf_data['log-pages']:
if dlpe.get('subtype') == 'nvme': # eliminate discovery controllers
tid = stas.tid_from_dlpe(dlpe, host_traddr, host_iface)
discovered_ctrls[tid] = dlpe
discovered_ctrl_list = list(discovered_ctrls.keys())
logging.debug('Stac._config_ctrls_finish() - discovered_ctrl_list = %s', discovered_ctrl_list)
controllers = stas.remove_excluded(configured_ctrl_list + discovered_ctrl_list)
controllers = iputil.remove_invalid_addresses(controllers)
new_controller_tids = set(controllers)
cur_controller_tids = set(self._controllers.keys())
controllers_to_add = new_controller_tids - cur_controller_tids
controllers_to_del = cur_controller_tids - new_controller_tids
logging.debug('Stac._config_ctrls_finish() - controllers_to_add = %s', list(controllers_to_add))
logging.debug('Stac._config_ctrls_finish() - controllers_to_del = %s', list(controllers_to_del))
svc_conf = conf.SvcConf()
no_disconnect = svc_conf.disconnect_scope == 'no-disconnect'
match_trtypes = svc_conf.disconnect_scope == 'all-connections-matching-disconnect-trtypes'
logging.debug(
'Stac._config_ctrls_finish() - no_disconnect=%s, match_trtypes=%s, svc_conf.disconnect_trtypes=%s',
no_disconnect,
match_trtypes,
svc_conf.disconnect_trtypes,
)
for tid in controllers_to_del:
controller = self._controllers.pop(tid, None)
if controller is not None:
keep_connection = no_disconnect or (match_trtypes and tid.transport not in svc_conf.disconnect_trtypes)
self._terminator.dispose(controller, self.remove_controller, keep_connection)
for tid in controllers_to_add:
self._controllers[tid] = ctrl.Ioc(self, tid)
for tid, controller in self._controllers.items():
if tid in discovered_ctrls:
dlpe = discovered_ctrls[tid]
controller.update_dlpe(dlpe)
self._dump_last_known_config(self._controllers)
def _connect_to_staf(self, _):
'''@brief Hook up DBus signal handlers for signals from stafd.'''
if not self._alive():
return
try:
self._staf = self._sysbus.get_proxy(defs.STAFD_DBUS_NAME, defs.STAFD_DBUS_PATH)
self._staf.log_pages_changed.connect(self._log_pages_changed)
self._staf.dc_removed.connect(self._dc_removed)
self._cfg_soak_tmr.start()
# Make sure timer is set back to its normal value.
self._cfg_soak_tmr.set_timeout(self.CONF_STABILITY_SOAK_TIME_SEC)
logging.debug('Stac._connect_to_staf() - Connected to staf')
except dasbus.error.DBusError:
logging.error('Failed to connect to staf')
def _destroy_staf_comlink(self, watcher): # pylint: disable=unused-argument
if self._staf:
self._staf.log_pages_changed.disconnect(self._log_pages_changed)
self._staf.dc_removed.disconnect(self._dc_removed)
dasbus.client.proxy.disconnect_proxy(self._staf)
self._staf = None
def _disconnect_from_staf(self, watcher):
self._destroy_staf_comlink(watcher)
# When we lose connectivity with stafd, the most logical explanation
# is that stafd restarted. In that case, it may take some time for stafd
# to re-populate its log pages cache. So let's give stafd plenty of time
# to update its log pages cache and send log pages change notifications
# before triggering a stacd re-config. We do this by momentarily
# increasing the config soak timer to a longer period.
if self._cfg_soak_tmr:
self._cfg_soak_tmr.set_timeout(self.CONF_STABILITY_LONG_SOAK_TIME_SEC)
logging.debug('Stac._disconnect_from_staf() - Disconnected from staf')
def _log_pages_changed( # pylint: disable=too-many-arguments
self, transport, traddr, trsvcid, host_traddr, host_iface, subsysnqn, device
):
if not self._alive():
return
logging.debug(
'Stac._log_pages_changed() - transport=%s, traddr=%s, trsvcid=%s, host_traddr=%s, host_iface=%s, subsysnqn=%s, device=%s',
transport,
traddr,
trsvcid,
host_traddr,
host_iface,
subsysnqn,
device,
)
if self._cfg_soak_tmr:
self._cfg_soak_tmr.start(self.CONF_STABILITY_SOAK_TIME_SEC)
def _dc_removed(self):
if not self._alive():
return
logging.debug('Stac._dc_removed()')
if self._cfg_soak_tmr:
self._cfg_soak_tmr.start(self.CONF_STABILITY_SOAK_TIME_SEC)
# ******************************************************************************
# Only keep legacy FC rule (not even sure this is still in use today, but just to be safe).
UDEV_RULE_OVERRIDE = r'''
ACTION=="change", SUBSYSTEM=="fc", ENV{FC_EVENT}=="nvmediscovery", \
ENV{NVMEFC_HOST_TRADDR}=="*", ENV{NVMEFC_TRADDR}=="*", \
RUN+="%s --no-block start nvmf-connect@--transport=fc\t--traddr=$env{NVMEFC_TRADDR}\t--trsvcid=none\t--host-traddr=$env{NVMEFC_HOST_TRADDR}.service"
'''
def _udev_rule_ctrl(suppress):
'''@brief We override the standard udev rule installed by nvme-cli, i.e.
'/usr/lib/udev/rules.d/70-nvmf-autoconnect.rules', with a copy into
/run/udev/rules.d. The goal is to suppress the udev rule that controls TCP
connections to I/O controllers. This is to avoid race conditions between
stacd and udevd. This is configurable. See "udev-rule" in stacd.conf
for details.
@param enable: When True, override nvme-cli's udev rule and prevent TCP I/O
Controller connections by nvme-cli. When False, allow nvme-cli's udev rule
to make TCP I/O connections.
@type enable: bool
'''
udev_rule_file = pathlib.Path('/run/udev/rules.d', '70-nvmf-autoconnect.rules')
if suppress:
if not udev_rule_file.exists():
pathlib.Path('/run/udev/rules.d').mkdir(parents=True, exist_ok=True)
text = UDEV_RULE_OVERRIDE % (defs.SYSTEMCTL)
udev_rule_file.write_text(text) # pylint: disable=unspecified-encoding
else:
try:
udev_rule_file.unlink()
except FileNotFoundError:
pass
def _is_dlp_changed_aen(udev_obj):
'''Check whether we received a Change of Discovery Log Page AEN'''
nvme_aen = udev_obj.get('NVME_AEN')
if not isinstance(nvme_aen, str):
return False
aen = int(nvme_aen, 16)
if aen != ctrl.DLP_CHANGED:
return False
logging.info(
'%s - Received AEN: Change of Discovery Log Page (%s)',
udev_obj.sys_name,
nvme_aen,
)
return True
def _event_matches(udev_obj, nvme_events):
'''Check whether we received an NVMe Event matching
one of the events listed in @nvme_events'''
nvme_event = udev_obj.get('NVME_EVENT')
if nvme_event not in nvme_events:
return False
logging.info('%s - Received "%s" event', udev_obj.sys_name, nvme_event)
return True
# ******************************************************************************
class Staf(Service):
'''STorage Appliance Finder (STAF)'''
def __init__(self, args, dbus):
default_conf = {
('Global', 'tron'): False,
('Global', 'hdr-digest'): False,
('Global', 'data-digest'): False,
('Global', 'kato'): 30,
('Global', 'queue-size'): None, # None to let the driver decide the default
('Global', 'reconnect-delay'): None, # None to let the driver decide the default
('Global', 'ctrl-loss-tmo'): None, # None to let the driver decide the default
('Global', 'disable-sqflow'): None, # None to let the driver decide the default
('Global', 'persistent-connections'): False, # Deprecated
('Discovery controller connection management', 'persistent-connections'): True,
('Discovery controller connection management', 'zeroconf-connections-persistence'): timeparse.timeparse(
'72hours'
),
('Global', 'ignore-iface'): False,
('Global', 'ip-family'): (4, 6),
('Global', 'pleo'): True,
('Service Discovery', 'zeroconf'): True,
('Controllers', 'controller'): list(),
('Controllers', 'exclude'): list(),
}
super().__init__(args, default_conf, self._reload_hdlr)
self._avahi = avahi.Avahi(self._sysbus, self._avahi_change)
self._avahi.config_stypes(conf.SvcConf().stypes)
# Create the D-Bus instance.
self._config_dbus(dbus, defs.STAFD_DBUS_NAME, defs.STAFD_DBUS_PATH)
self._udev.register_for_action_events('change', self._nvme_cli_interop)
_udev_rule_ctrl(True)
def info(self) -> dict:
'''@brief Get the status info for this object (used for debug)'''
info = super().info()
info['avahi'] = self._avahi.info()
return info
def _release_resources(self):
logging.debug('Staf._release_resources()')
if self._udev:
self._udev.unregister_for_action_events('change', self._nvme_cli_interop)
super()._release_resources()
_udev_rule_ctrl(False)
if self._avahi:
self._avahi.kill()
self._avahi = None
def _dump_last_known_config(self, controllers):
config = {tid: {'log_pages': dc.log_pages(), 'origin': dc.origin} for tid, dc in controllers.items()}
logging.debug('Staf._dump_last_known_config() - DC count = %s', len(config))
self._write_lkc(config)
def _load_last_known_config(self):
config = self._read_lkc() or dict()
logging.debug('Staf._load_last_known_config() - DC count = %s', len(config))
controllers = {}
for tid, data in config.items():
if isinstance(data, dict):
log_pages = data.get('log_pages')
origin = data.get('origin')
else:
log_pages = data
origin = None
# Regenerate the TID (in case of soft. upgrade and TID object
# has changed internally)
tid = trid.TID(tid.as_dict())
controllers[tid] = ctrl.Dc(self, tid, log_pages, origin)
return controllers
def _keep_connections_on_exit(self):
'''@brief Determine whether connections should remain when the
process exits.
'''
return conf.SvcConf().persistent_connections
def _reload_hdlr(self):
'''@brief Reload configuration file. This is triggered by the SIGHUP
signal, which can be sent with "systemctl reload stafd".
'''
if not self._alive():
return GLib.SOURCE_REMOVE
sd_notify('RELOADING=1')
service_cnf = conf.SvcConf()
service_cnf.reload()
self.tron = service_cnf.tron
self._avahi.kick_start() # Make sure Avahi is running
self._avahi.config_stypes(service_cnf.stypes)
self._cfg_soak_tmr.start()
for controller in self._controllers.values():
controller.reload_hdlr()
sd_notify('READY=1')
return GLib.SOURCE_CONTINUE
def is_avahi_reported(self, tid):
'''@brief Return whether @tid is being reported by the Avahi daemon.
@return: True if the Avahi daemon is reporting it, False otherwise.
'''
for cid in self._avahi.get_controllers():
if trid.TID(cid) == tid:
return True
return False
def log_pages_changed(self, controller, device):
'''@brief Function invoked when a controller's cached log pages
have changed. This will emit a D-Bus signal to inform
other applications that the cached log pages have changed.
'''
self._dbus_iface.log_pages_changed.emit(
controller.tid.transport,
controller.tid.traddr,
controller.tid.trsvcid,
controller.tid.host_traddr,
controller.tid.host_iface,
controller.tid.subsysnqn,
device,
)
def dc_removed(self):
'''@brief Function invoked when a controller's cached log pages
have changed. This will emit a D-Bus signal to inform
other applications that the cached log pages have changed.
'''
self._dbus_iface.dc_removed.emit()
def _referrals(self) -> list:
return [
stas.tid_from_dlpe(dlpe, controller.tid.host_traddr, controller.tid.host_iface)
for controller in self.get_controllers()
for dlpe in controller.referrals()
]
def _config_ctrls_finish(self, configured_ctrl_list: list):
'''@brief Finish discovery controllers configuration after
hostnames (if any) have been resolved. All the logic associated
with discovery controller creation/deletion is found here. To
avoid calling this algorith repetitively for each and every events,
it is called after a soaking period controlled by self._cfg_soak_tmr.
@param configured_ctrl_list: List of TIDs configured in stafd.conf with
all hostnames resolved to their corresponding IP addresses.
'''
# This is a callback function, which may be called after the service
# has been signalled to stop. So let's make sure the service is still
# alive and well before continuing.
if not self._alive():
logging.debug('Staf._config_ctrls_finish() - Exiting because service is no longer alive')
return
# Eliminate invalid entries from stafd.conf "controller list".
controllers = list()
for tid in configured_ctrl_list:
if '' in (tid.transport, tid.traddr, tid.trsvcid):
continue
if not tid.subsysnqn:
cid = tid.as_dict()
cid['subsysnqn'] = defs.WELL_KNOWN_DISC_NQN
controllers.append(trid.TID(cid))
else:
controllers.append(tid)
configured_ctrl_list = controllers
# Get the Avahi-discovered list and the referrals.
discovered_ctrl_list = [trid.TID(cid) for cid in self._avahi.get_controllers()]
referral_ctrl_list = self._referrals()
logging.debug('Staf._config_ctrls_finish() - configured_ctrl_list = %s', configured_ctrl_list)
logging.debug('Staf._config_ctrls_finish() - discovered_ctrl_list = %s', discovered_ctrl_list)
logging.debug('Staf._config_ctrls_finish() - referral_ctrl_list = %s', referral_ctrl_list)
all_ctrls = configured_ctrl_list + discovered_ctrl_list + referral_ctrl_list
controllers = stas.remove_excluded(all_ctrls)
controllers = iputil.remove_invalid_addresses(controllers)
new_controller_tids = set(controllers)
cur_controller_tids = set(self._controllers.keys())
controllers_to_add = new_controller_tids - cur_controller_tids
controllers_to_del = cur_controller_tids - new_controller_tids
# Make a list list of excluded and invalid controllers
must_remove_list = set(all_ctrls) - new_controller_tids
# Find "discovered" controllers that have not responded
# in a while and add them to controllers that must be removed.
must_remove_list.update({tid for tid, controller in self._controllers.items() if controller.is_unresponsive()})
# Do not remove Avahi-discovered DCs from controllers_to_del unless
# marked as "must-be-removed" (must_remove_list). This is to account for
# the case where mDNS discovery is momentarily disabled (e.g. Avahi
# daemon restarts). We don't want to delete connections because of
# temporary mDNS impairments. Removal of Avahi-discovered DCs will be
# handled differently and only if the connection cannot be established
# for a long period of time.
logging.debug('Staf._config_ctrls_finish() - must_remove_list = %s', list(must_remove_list))
controllers_to_del = {
tid
for tid in controllers_to_del
if tid in must_remove_list or self._controllers[tid].origin != 'discovered'
}
logging.debug('Staf._config_ctrls_finish() - controllers_to_add = %s', list(controllers_to_add))
logging.debug('Staf._config_ctrls_finish() - controllers_to_del = %s', list(controllers_to_del))
# Delete controllers
for tid in controllers_to_del:
controller = self._controllers.pop(tid, None)
if controller is not None:
self._terminator.dispose(controller, self.remove_controller, keep_connection=False)
if len(controllers_to_del) > 0:
self.dc_removed() # Let other apps (e.g. stacd) know that discovery controllers were removed.
# Add controllers
for tid in controllers_to_add:
self._controllers[tid] = ctrl.Dc(self, tid)
# Update "origin" on all DC objects
for tid, controller in self._controllers.items():
origin = (
'configured'
if tid in configured_ctrl_list
else 'referral'
if tid in referral_ctrl_list
else 'discovered'
if tid in discovered_ctrl_list
else None
)
if origin is not None:
controller.origin = origin
self._dump_last_known_config(self._controllers)
def _avahi_change(self):
if self._alive() and self._cfg_soak_tmr is not None:
self._cfg_soak_tmr.start()
def controller_unresponsive(self, tid):
'''@brief Function invoked when a controller becomes unresponsive and
needs to be removed.
'''
if self._alive() and self._cfg_soak_tmr is not None:
logging.debug('Staf.controller_unresponsive() - tid = %s', tid)
self._cfg_soak_tmr.start()
def referrals_changed(self):
'''@brief Function invoked when a controller's cached referrals
have changed.
'''
if self._alive() and self._cfg_soak_tmr is not None:
logging.debug('Staf.referrals_changed()')
self._cfg_soak_tmr.start()
def _nvme_cli_interop(self, udev_obj):
'''Interoperability with nvme-cli:
stafd will invoke nvme-cli's connect-all the same way nvme-cli's udev
rules would do normally. This is for the case where a user has an hybrid
configuration where some controllers are configured through nvme-stas
and others through nvme-cli. This is not an optimal configuration. It
would be better if everything was configured through nvme-stas, however
support for hybrid configuration was requested by users (actually only
one user requested this).'''
# Looking for 'change' events only
if udev_obj.action != 'change':
return
# Looking for events from Discovery Controllers only
if not udev.Udev.is_dc_device(udev_obj):
return
# Is the controller already being monitored by stafd?
for controller in self.get_controllers():
if controller.device == udev_obj.sys_name:
return
# Did we receive a Change of DLP AEN or an NVME Event indicating 'connect' or 'rediscover'?
if not _is_dlp_changed_aen(udev_obj) and not _event_matches(udev_obj, ('connected', 'rediscover')):
return
# We need to invoke "nvme connect-all" using nvme-cli's nvmf-connect@.service
# NOTE: Eventually, we'll be able to drop --host-traddr and --host-iface from
# the parameters passed to nvmf-connect@.service. A fix was added to connect-all
# to infer these two values from the device used to connect to the DC.
# Ref: https://github.com/linux-nvme/nvme-cli/pull/1812
cnf = [
('--device', udev_obj.sys_name),
('--host-traddr', udev_obj.properties.get('NVME_HOST_TRADDR', None)),
('--host-iface', udev_obj.properties.get('NVME_HOST_IFACE', None)),
]
# Use systemd's escaped syntax (i.e. '=' is replaced by '\x3d', '\t' by '\x09', etc.
options = r'\x09'.join(
[fr'{option}\x3d{value}' for option, value in cnf if value not in (None, 'none', 'None', '')]
)
logging.info('Invoking: systemctl start nvmf-connect@%s.service', options)
cmd = [defs.SYSTEMCTL, '--quiet', '--no-block', 'start', fr'nvmf-connect@{options}.service']
subprocess.run(cmd, check=False)

23
staslib/singleton.py Normal file
View file

@ -0,0 +1,23 @@
# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See the LICENSE file for details.
#
# This file is part of NVMe STorage Appliance Services (nvme-stas).
#
# Authors: Martin Belanger <Martin.Belanger@dell.com>
#
'''Implementation of a singleton pattern'''
class Singleton(type):
'''metaclass implementation of a singleton pattern'''
_instances = {}
def __call__(cls, *args, **kwargs):
if cls not in cls._instances:
# This variable declaration is required to force a
# strong reference on the instance.
instance = super(Singleton, cls).__call__(*args, **kwargs)
cls._instances[cls] = instance
return cls._instances[cls]

27
staslib/stacd.idl Normal file
View file

@ -0,0 +1,27 @@
<node>
<interface name="@STACD_DBUS_NAME@.debug">
<property name="tron" type="b" access="readwrite"/>
<property name="log_level" type="s" access="read"/>
<method name="process_info">
<arg direction="out" type="s" name="info_json"/>
</method>
<method name="controller_info">
<arg direction="in" type="s" name="transport"/>
<arg direction="in" type="s" name="traddr"/>
<arg direction="in" type="s" name="trsvcid"/>
<arg direction="in" type="s" name="host_traddr"/>
<arg direction="in" type="s" name="host_iface"/>
<arg direction="in" type="s" name="subsysnqn"/>
<arg direction="out" type="s" name="info_json"/>
</method>
</interface>
<interface name="@STACD_DBUS_NAME@">
<method name="list_controllers">
<arg direction="in" type="b" name="detailed"/>
<arg direction="out" type="aa{ss}" name="controller_list"/>
</method>
</interface>
</node>

49
staslib/stafd.idl Normal file
View file

@ -0,0 +1,49 @@
<node>
<interface name="@STAFD_DBUS_NAME@.debug">
<property name="tron" type="b" access="readwrite"/>
<property name="log_level" type="s" access="read"/>
<method name="process_info">
<arg direction="out" type="s" name="info_json"/>
</method>
<method name="controller_info">
<arg direction="in" type="s" name="transport"/>
<arg direction="in" type="s" name="traddr"/>
<arg direction="in" type="s" name="trsvcid"/>
<arg direction="in" type="s" name="host_traddr"/>
<arg direction="in" type="s" name="host_iface"/>
<arg direction="in" type="s" name="subsysnqn"/>
<arg direction="out" type="s" name="info_json"/>
</method>
</interface>
<interface name="@STAFD_DBUS_NAME@">
<method name="list_controllers">
<arg direction="in" type="b" name="detailed"/>
<arg direction="out" type="aa{ss}" name="controller_list"/>
</method>
<method name="get_log_pages">
<arg direction="in" type="s" name="transport"/>
<arg direction="in" type="s" name="traddr"/>
<arg direction="in" type="s" name="trsvcid"/>
<arg direction="in" type="s" name="host_traddr"/>
<arg direction="in" type="s" name="host_iface"/>
<arg direction="in" type="s" name="subsysnqn"/>
<arg direction="out" type="aa{ss}" name="log_pages"/>
</method>
<method name="get_all_log_pages">
<arg direction="in" type="b" name="detailed"/>
<arg direction="out" type="s" name="log_pages_json"/>
</method>
<signal name="log_pages_changed">
<arg direction="out" type="s" name="transport"/>
<arg direction="out" type="s" name="traddr"/>
<arg direction="out" type="s" name="trsvcid"/>
<arg direction="out" type="s" name="host_traddr"/>
<arg direction="out" type="s" name="host_iface"/>
<arg direction="out" type="s" name="subsysnqn"/>
<arg direction="out" type="s" name="device"/>
</signal>
<signal name="dc_removed"></signal>
</interface>
</node>

554
staslib/stas.py Normal file
View file

@ -0,0 +1,554 @@
# Copyright (c) 2021, Dell Inc. or its subsidiaries. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See the LICENSE file for details.
#
# This file is part of NVMe STorage Appliance Services (nvme-stas).
#
# Authors: Martin Belanger <Martin.Belanger@dell.com>
#
'''Library for staf/stac. You will find here common code for stafd and stacd
including the Abstract Base Classes (ABC) for Controllers and Services'''
import os
import sys
import abc
import signal
import pickle
import logging
import dasbus.connection
from gi.repository import Gio, GLib
from systemd.daemon import notify as sd_notify
from staslib import conf, defs, gutil, log, trid
try:
# Python 3.9 or later
# This is the preferred way, but may not be available before Python 3.9
from importlib.resources import files
except ImportError:
try:
# Pre Python 3.9 backport of importlib.resources (if installed)
from importlib_resources import files
except ImportError:
# Less efficient, but avalable on older versions of Python
import pkg_resources
def load_idl(idl_fname):
'''@brief Load D-Bus Interface Description Language File'''
try:
return pkg_resources.resource_string('staslib', idl_fname).decode()
except (FileNotFoundError, AttributeError):
pass
return ''
else:
def load_idl(idl_fname):
'''@brief Load D-Bus Interface Description Language File'''
try:
return files('staslib').joinpath(idl_fname).read_text() # pylint: disable=unspecified-encoding
except FileNotFoundError:
pass
return ''
else:
def load_idl(idl_fname):
'''@brief Load D-Bus Interface Description Language File'''
try:
return files('staslib').joinpath(idl_fname).read_text() # pylint: disable=unspecified-encoding
except FileNotFoundError:
pass
return ''
# ******************************************************************************
def check_if_allowed_to_continue():
'''@brief Let's perform some basic checks before going too far. There are
a few pre-requisites that need to be met before this program
is allowed to proceed:
1) The program needs to have root privileges
2) The nvme-tcp kernel module must be loaded
@return This function will only return if all conditions listed above
are met. Otherwise the program exits.
'''
# 1) Check root privileges
if os.geteuid() != 0:
sys.exit(f'Permission denied. You need root privileges to run {defs.PROG_NAME}.')
# 2) Check that nvme-tcp kernel module is running
if not os.path.exists('/dev/nvme-fabrics'):
# There's no point going any further if the kernel module hasn't been loaded
sys.exit('Fatal error: missing nvme-tcp kernel module')
# ******************************************************************************
def tid_from_dlpe(dlpe, host_traddr, host_iface):
'''@brief Take a Discovery Log Page Entry and return a Controller ID as a dict.'''
cid = {
'transport': dlpe['trtype'],
'traddr': dlpe['traddr'],
'trsvcid': dlpe['trsvcid'],
'host-traddr': host_traddr,
'host-iface': host_iface,
'subsysnqn': dlpe['subnqn'],
}
return trid.TID(cid)
# ******************************************************************************
def _excluded(excluded_ctrl_list, controller: dict):
'''@brief Check if @controller is excluded.'''
for excluded_ctrl in excluded_ctrl_list:
test_results = [val == controller.get(key, None) for key, val in excluded_ctrl.items()]
if all(test_results):
return True
return False
# ******************************************************************************
def remove_excluded(controllers: list):
'''@brief Remove excluded controllers from the list of controllers.
@param controllers: List of TIDs
'''
excluded_ctrl_list = conf.SvcConf().get_excluded()
if excluded_ctrl_list:
logging.debug('remove_excluded() - excluded_ctrl_list = %s', excluded_ctrl_list)
controllers = [
controller for controller in controllers if not _excluded(excluded_ctrl_list, controller.as_dict())
]
return controllers
# ******************************************************************************
class ControllerABC(abc.ABC):
'''@brief Base class used to manage the connection to a controller.'''
CONNECT_RETRY_PERIOD_SEC = 60
FAST_CONNECT_RETRY_PERIOD_SEC = 3
def __init__(self, tid: trid.TID, service, discovery_ctrl: bool = False):
self._tid = tid
self._serv = service # Refers to the parent service (either Staf or Stac)
self.set_level_from_tron(self._serv.tron)
self._cancellable = Gio.Cancellable()
self._connect_attempts = 0
self._retry_connect_tmr = gutil.GTimer(self.CONNECT_RETRY_PERIOD_SEC, self._on_try_to_connect)
self._discovery_ctrl = discovery_ctrl
self._try_to_connect_deferred = gutil.Deferred(self._try_to_connect)
self._try_to_connect_deferred.schedule()
def _release_resources(self):
# Remove pending deferred from main loop
if self._try_to_connect_deferred:
self._try_to_connect_deferred.cancel()
if self._retry_connect_tmr is not None:
self._retry_connect_tmr.kill()
if self._alive():
self._cancellable.cancel()
self._tid = None
self._serv = None
self._cancellable = None
self._retry_connect_tmr = None
self._try_to_connect_deferred = None
@property
def id(self) -> str:
'''@brief Return the Transport ID as a printable string'''
return str(self.tid)
@property
def tid(self):
'''@brief Return the Transport ID object'''
return self._tid
def controller_id_dict(self) -> dict:
'''@brief return the controller ID as a dict.'''
return {k: str(v) for k, v in self.tid.as_dict().items()}
def details(self) -> dict:
'''@brief return detailed debug info about this controller'''
return self.info()
def info(self) -> dict:
'''@brief Get the controller info for this object'''
info = self.controller_id_dict()
info['connect attempts'] = str(self._connect_attempts)
info['retry connect timer'] = str(self._retry_connect_tmr)
return info
def cancel(self):
'''@brief Used to cancel pending operations.'''
if self._alive():
logging.debug('ControllerABC.cancel() - %s', self.id)
self._cancellable.cancel()
def kill(self):
'''@brief Used to release all resources associated with this object.'''
logging.debug('ControllerABC.kill() - %s', self.id)
self._release_resources()
def _alive(self):
'''There may be race condition where a queued event gets processed
after the object is no longer configured (i.e. alive). This method
can be used by callback functions to make sure the object is still
alive before processing further.
'''
return self._cancellable and not self._cancellable.is_cancelled()
def _on_try_to_connect(self):
if self._alive():
self._try_to_connect_deferred.schedule()
return GLib.SOURCE_REMOVE
def _should_try_to_reconnect(self): # pylint: disable=no-self-use
return True
def _try_to_connect(self):
if not self._alive():
return GLib.SOURCE_REMOVE
# This is a deferred function call. Make sure
# the source of the deferred is still good.
source = GLib.main_current_source()
if source and source.is_destroyed():
return GLib.SOURCE_REMOVE
self._connect_attempts += 1
self._do_connect()
return GLib.SOURCE_REMOVE
@abc.abstractmethod
def set_level_from_tron(self, tron):
'''Set log level based on TRON'''
@abc.abstractmethod
def _do_connect(self):
'''Perform connection'''
@abc.abstractmethod
def _on_aen(self, aen: int):
'''Event handler when an AEN is received'''
@abc.abstractmethod
def _on_nvme_event(self, nvme_event):
'''Event handler when an nvme_event is received'''
@abc.abstractmethod
def _on_ctrl_removed(self, udev_obj):
'''Called when the associated nvme device (/dev/nvmeX) is removed
from the system by the kernel.
'''
@abc.abstractmethod
def _find_existing_connection(self):
'''Check if there is an existing connection that matches this Controller's TID'''
@abc.abstractmethod
def all_ops_completed(self) -> bool:
'''@brief Returns True if all operations have completed. False otherwise.'''
@abc.abstractmethod
def connected(self):
'''@brief Return whether a connection is established'''
@abc.abstractmethod
def disconnect(self, disconnected_cb, keep_connection):
'''@brief Issue an asynchronous disconnect command to a Controller.
Once the async command has completed, the callback 'disconnected_cb'
will be invoked. If a controller is already disconnected, then the
callback will be added to the main loop's next idle slot to be executed
ASAP.
'''
@abc.abstractmethod
def reload_hdlr(self):
'''@brief This is called when a "reload" signal is received.'''
# ******************************************************************************
class ServiceABC(abc.ABC): # pylint: disable=too-many-instance-attributes
'''@brief Base class used to manage a STorage Appliance Service'''
CONF_STABILITY_SOAK_TIME_SEC = 1.5
def __init__(self, args, default_conf, reload_hdlr):
service_conf = conf.SvcConf(default_conf=default_conf)
service_conf.set_conf_file(args.conf_file) # reload configuration
self._tron = args.tron or service_conf.tron
log.set_level_from_tron(self._tron)
self._lkc_file = os.path.join(
os.environ.get('RUNTIME_DIRECTORY', os.path.join('/run', defs.PROG_NAME)), 'last-known-config.pickle'
)
self._loop = GLib.MainLoop()
self._cancellable = Gio.Cancellable()
self._resolver = gutil.NameResolver()
self._controllers = self._load_last_known_config()
self._dbus_iface = None
self._cfg_soak_tmr = gutil.GTimer(self.CONF_STABILITY_SOAK_TIME_SEC, self._on_config_ctrls)
self._sysbus = dasbus.connection.SystemMessageBus()
GLib.unix_signal_add(GLib.PRIORITY_HIGH, signal.SIGINT, self._stop_hdlr) # CTRL-C
GLib.unix_signal_add(GLib.PRIORITY_HIGH, signal.SIGTERM, self._stop_hdlr) # systemctl stop stafd
GLib.unix_signal_add(GLib.PRIORITY_HIGH, signal.SIGHUP, reload_hdlr) # systemctl reload stafd
nvme_options = conf.NvmeOptions()
if not nvme_options.host_iface_supp or not nvme_options.discovery_supp:
logging.warning(
'Kernel does not appear to support all the options needed to run this program. Consider updating to a later kernel version.'
)
# We don't want to apply configuration changes right away.
# Often, multiple changes will occur in a short amount of time (sub-second).
# We want to wait until there are no more changes before applying them
# to the system. The following timer acts as a "soak period". Changes
# will be applied by calling self._on_config_ctrls() at the end of
# the soak period.
self._cfg_soak_tmr.start()
def _release_resources(self):
logging.debug('ServiceABC._release_resources()')
if self._alive():
self._cancellable.cancel()
if self._cfg_soak_tmr is not None:
self._cfg_soak_tmr.kill()
self._controllers.clear()
if self._sysbus:
self._sysbus.disconnect()
self._cfg_soak_tmr = None
self._cancellable = None
self._resolver = None
self._lkc_file = None
self._sysbus = None
def _config_dbus(self, iface_obj, bus_name: str, obj_name: str):
self._dbus_iface = iface_obj
self._sysbus.publish_object(obj_name, iface_obj)
self._sysbus.register_service(bus_name)
@property
def tron(self):
'''@brief Get Trace ON property'''
return self._tron
@tron.setter
def tron(self, value):
'''@brief Set Trace ON property'''
self._tron = value
log.set_level_from_tron(self._tron)
for controller in self._controllers.values():
controller.set_level_from_tron(self._tron)
def run(self):
'''@brief Start the main loop execution'''
try:
self._loop.run()
except Exception as ex: # pylint: disable=broad-except
logging.critical('exception: %s', ex)
self._loop = None
def info(self) -> dict:
'''@brief Get the status info for this object (used for debug)'''
nvme_options = conf.NvmeOptions()
info = conf.SysConf().as_dict()
info['last known config file'] = self._lkc_file
info['config soak timer'] = str(self._cfg_soak_tmr)
info['kernel support.TP8013'] = str(nvme_options.discovery_supp)
info['kernel support.host_iface'] = str(nvme_options.host_iface_supp)
return info
def get_controllers(self) -> dict:
'''@brief return the list of controller objects'''
return self._controllers.values()
def get_controller(
self, transport: str, traddr: str, trsvcid: str, host_traddr: str, host_iface: str, subsysnqn: str
): # pylint: disable=too-many-arguments
'''@brief get the specified controller object from the list of controllers'''
cid = {
'transport': transport,
'traddr': traddr,
'trsvcid': trsvcid,
'host-traddr': host_traddr,
'host-iface': host_iface,
'subsysnqn': subsysnqn,
}
return self._controllers.get(trid.TID(cid))
def _remove_ctrl_from_dict(self, controller, shutdown=False):
tid_to_pop = controller.tid
if not tid_to_pop:
# Being paranoid. This should not happen, but let's say the
# controller object has been purged, but it is somehow still
# listed in self._controllers.
for tid, _controller in self._controllers.items():
if _controller is controller:
tid_to_pop = tid
break
if tid_to_pop:
logging.debug('ServiceABC._remove_ctrl_from_dict()- %s | %s', tid_to_pop, controller.device)
popped = self._controllers.pop(tid_to_pop, None)
if not shutdown and popped is not None and self._cfg_soak_tmr:
self._cfg_soak_tmr.start()
else:
logging.debug('ServiceABC._remove_ctrl_from_dict()- already removed')
def remove_controller(self, controller, success): # pylint: disable=unused-argument
'''@brief remove the specified controller object from the list of controllers
@param controller: the controller object
@param success: whether the disconnect was successful'''
logging.debug('ServiceABC.remove_controller()')
if isinstance(controller, ControllerABC):
self._remove_ctrl_from_dict(controller)
controller.kill()
def _alive(self):
'''It's a good idea to check that this object hasn't been
cancelled (i.e. is still alive) when entering a callback function.
Callback functrions can be invoked after, for example, a process has
been signalled to stop or restart, in which case it makes no sense to
proceed with the callback.
'''
return self._cancellable and not self._cancellable.is_cancelled()
def _cancel(self):
logging.debug('ServiceABC._cancel()')
if self._alive():
self._cancellable.cancel()
for controller in self._controllers.values():
controller.cancel()
def _stop_hdlr(self):
logging.debug('ServiceABC._stop_hdlr()')
sd_notify('STOPPING=1')
self._cancel() # Cancel pending operations
self._dump_last_known_config(self._controllers)
if len(self._controllers) == 0:
GLib.idle_add(self._exit)
else:
self._disconnect_all()
return GLib.SOURCE_REMOVE
def _on_final_disconnect(self, controller, success):
'''Callback invoked after a controller is disconnected.
THIS IS USED DURING PROCESS SHUTDOWN TO WAIT FOR ALL CONTROLLERS TO BE
DISCONNECTED BEFORE EXITING THE PROGRAM. ONLY CALL ON SHUTDOWN!
@param controller: the controller object
@param success: whether the disconnect operation was successful
'''
logging.debug(
'ServiceABC._on_final_disconnect() - %s | %s: disconnect %s',
controller.id,
controller.device,
'succeeded' if success else 'failed',
)
self._remove_ctrl_from_dict(controller, True)
controller.kill()
# When all controllers have disconnected, we can finish the clean up
if len(self._controllers) == 0:
# Defer exit to the next main loop's idle period.
GLib.idle_add(self._exit)
def _exit(self):
logging.debug('ServiceABC._exit()')
self._release_resources()
self._loop.quit()
def _on_config_ctrls(self, *_user_data):
if self._alive():
self._config_ctrls()
return GLib.SOURCE_REMOVE
def _config_ctrls(self):
'''@brief Start controllers configuration.'''
# The configuration file may contain controllers and/or excluded
# controllers with traddr specified as hostname instead of IP address.
# Because of this, we need to remove those excluded elements before
# running name resolution. And we will need to remove excluded
# elements after name resolution is complete (i.e. in the calback
# function _config_ctrls_finish)
logging.debug('ServiceABC._config_ctrls()')
configured_controllers = [trid.TID(cid) for cid in conf.SvcConf().get_controllers()]
configured_controllers = remove_excluded(configured_controllers)
self._resolver.resolve_ctrl_async(self._cancellable, configured_controllers, self._config_ctrls_finish)
def _read_lkc(self):
'''@brief Read Last Known Config from file'''
try:
with open(self._lkc_file, 'rb') as file:
return pickle.load(file)
except (FileNotFoundError, AttributeError, EOFError):
return None
def _write_lkc(self, config):
'''@brief Write Last Known Config to file, and if config is empty
make sure the file is emptied.'''
try:
# Note that if config is empty we still
# want to open/close the file to empty it.
with open(self._lkc_file, 'wb') as file:
if config:
pickle.dump(config, file)
except FileNotFoundError as ex:
logging.error('Unable to save last known config: %s', ex)
@abc.abstractmethod
def _disconnect_all(self):
'''Tell all controller objects to disconnect'''
@abc.abstractmethod
def _keep_connections_on_exit(self):
'''@brief Determine whether connections should remain when the
process exits.
NOTE) This is the base class method used to define the interface.
It must be overloaded by a child class.
'''
@abc.abstractmethod
def _config_ctrls_finish(self, configured_ctrl_list):
'''@brief Finish controllers configuration after hostnames (if any)
have been resolved.
Configuring controllers must be done asynchronously in 2 steps.
In the first step, host names get resolved to find their IP addresses.
Name resolution can take a while, especially when an external name
resolution server is used. Once that step completed, the callback
method _config_ctrls_finish() (i.e. this method), gets invoked to
complete the controller configuration.
NOTE) This is the base class method used to define the interface.
It must be overloaded by a child class.
'''
@abc.abstractmethod
def _load_last_known_config(self):
'''Load last known config from file (if any)'''
@abc.abstractmethod
def _dump_last_known_config(self, controllers):
'''Save last known config to file'''

139
staslib/timeparse.py Normal file
View file

@ -0,0 +1,139 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
This module was borrowed and modified from: https://github.com/wroberts/pytimeparse
timeparse.py
(c) Will Roberts <wildwilhelm@gmail.com> 1 February, 2014
Implements a single function, `timeparse`, which can parse various
kinds of time expressions.
'''
# MIT LICENSE
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation files
# (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge,
# publish, distribute, sublicense, and/or sell copies of the Software,
# and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import re
SIGN = r'(?P<sign>[+|-])?'
DAYS = r'(?P<days>[\d.]+)\s*(?:d|dys?|days?)'
HOURS = r'(?P<hours>[\d.]+)\s*(?:h|hrs?|hours?)'
MINS = r'(?P<mins>[\d.]+)\s*(?:m|(mins?)|(minutes?))'
SECS = r'(?P<secs>[\d.]+)\s*(?:s|secs?|seconds?)'
SEPARATORS = r'[,/]'
SECCLOCK = r':(?P<secs>\d{2}(?:\.\d+)?)'
MINCLOCK = r'(?P<mins>\d{1,2}):(?P<secs>\d{2}(?:\.\d+)?)'
HOURCLOCK = r'(?P<hours>\d+):(?P<mins>\d{2}):(?P<secs>\d{2}(?:\.\d+)?)'
DAYCLOCK = r'(?P<days>\d+):(?P<hours>\d{2}):(?P<mins>\d{2}):(?P<secs>\d{2}(?:\.\d+)?)'
def _opt(string):
return f'(?:{string})?'
def _optsep(string):
return fr'(?:{string}\s*(?:{SEPARATORS}\s*)?)?'
TIMEFORMATS = [
fr'{_optsep(DAYS)}\s*{_optsep(HOURS)}\s*{_optsep(MINS)}\s*{_opt(SECS)}',
f'{MINCLOCK}',
fr'{_optsep(DAYS)}\s*{HOURCLOCK}',
f'{DAYCLOCK}',
f'{SECCLOCK}',
]
COMPILED_SIGN = re.compile(r'\s*' + SIGN + r'\s*(?P<unsigned>.*)$')
COMPILED_TIMEFORMATS = [re.compile(r'\s*' + timefmt + r'\s*$', re.I) for timefmt in TIMEFORMATS]
MULTIPLIERS = {
'days': 60 * 60 * 24,
'hours': 60 * 60,
'mins': 60,
'secs': 1,
}
def timeparse(sval):
'''
Parse a time expression, returning it as a number of seconds. If
possible, the return value will be an `int`; if this is not
possible, the return will be a `float`. Returns `None` if a time
expression cannot be parsed from the given string.
Arguments:
- `sval`: the string value to parse
>>> timeparse('1:24')
84
>>> timeparse(':22')
22
>>> timeparse('1 minute, 24 secs')
84
>>> timeparse('1m24s')
84
>>> timeparse('1.2 minutes')
72
>>> timeparse('1.2 seconds')
1.2
Time expressions can be signed.
>>> timeparse('- 1 minute')
-60
>>> timeparse('+ 1 minute')
60
'''
try:
return float(sval)
except TypeError:
pass
except ValueError:
match = COMPILED_SIGN.match(sval)
sign = -1 if match.groupdict()['sign'] == '-' else 1
sval = match.groupdict()['unsigned']
for timefmt in COMPILED_TIMEFORMATS:
match = timefmt.match(sval)
if match and match.group(0).strip():
mdict = match.groupdict()
# if all of the fields are integer numbers
if all(v.isdigit() for v in list(mdict.values()) if v):
return sign * sum((MULTIPLIERS[k] * int(v, 10) for (k, v) in list(mdict.items()) if v is not None))
# if SECS is an integer number
if 'secs' not in mdict or mdict['secs'] is None or mdict['secs'].isdigit():
# we will return an integer
return sign * int(
sum(
(
MULTIPLIERS[k] * float(v)
for (k, v) in list(mdict.items())
if k != 'secs' and v is not None
)
)
) + (int(mdict['secs'], 10) if mdict['secs'] else 0)
# SECS is a float, we will return a float
return sign * sum((MULTIPLIERS[k] * float(v) for (k, v) in list(mdict.items()) if v is not None))
return None

137
staslib/trid.py Normal file
View file

@ -0,0 +1,137 @@
# Copyright (c) 2022, Dell Inc. or its subsidiaries. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# See the LICENSE file for details.
#
# This file is part of NVMe STorage Appliance Services (nvme-stas).
#
# Authors: Martin Belanger <Martin.Belanger@dell.com>
#
'''This module defines the Transport Identifier Object, which is used
throughout nvme-stas to uniquely identify a Controller'''
import hashlib
from staslib import conf
class TID: # pylint: disable=too-many-instance-attributes
'''Transport Identifier'''
RDMA_IP_PORT = '4420'
DISC_IP_PORT = '8009'
def __init__(self, cid: dict):
'''@param cid: Controller Identifier. A dictionary with the following
contents.
{
# Transport parameters
'transport': str, # [mandatory]
'traddr': str, # [mandatory]
'subsysnqn': str, # [mandatory]
'trsvcid': str, # [optional]
'host-traddr': str, # [optional]
'host-iface': str, # [optional]
# Connection parameters
'dhchap-ctrl-secret': str, # [optional]
'hdr-digest': str, # [optional]
'data-digest': str, # [optional]
'nr-io-queues': str, # [optional]
'nr-write-queues': str, # [optional]
'nr-poll-queues': str, # [optional]
'queue-size': str, # [optional]
'kato': str, # [optional]
'reconnect-delay': str, # [optional]
'ctrl-loss-tmo': str, # [optional]
'disable-sqflow': str, # [optional]
}
'''
self._cfg = {
k: v
for k, v in cid.items()
if k not in ('transport', 'traddr', 'subsysnqn', 'trsvcid', 'host-traddr', 'host-iface')
}
self._transport = cid.get('transport', '')
self._traddr = cid.get('traddr', '')
self._trsvcid = ''
if self._transport in ('tcp', 'rdma'):
trsvcid = cid.get('trsvcid', None)
self._trsvcid = (
trsvcid if trsvcid else (TID.RDMA_IP_PORT if self._transport == 'rdma' else TID.DISC_IP_PORT)
)
self._host_traddr = cid.get('host-traddr', '')
self._host_iface = '' if conf.SvcConf().ignore_iface else cid.get('host-iface', '')
self._subsysnqn = cid.get('subsysnqn', '')
self._shortkey = (self._transport, self._traddr, self._trsvcid, self._subsysnqn, self._host_traddr)
self._key = (self._transport, self._traddr, self._trsvcid, self._subsysnqn, self._host_traddr, self._host_iface)
self._hash = int.from_bytes(
hashlib.md5(''.join(self._key).encode('utf-8')).digest(), 'big'
) # We need a consistent hash between restarts
self._id = f'({self._transport}, {self._traddr}, {self._trsvcid}{", " + self._subsysnqn if self._subsysnqn else ""}{", " + self._host_iface if self._host_iface else ""}{", " + self._host_traddr if self._host_traddr else ""})' # pylint: disable=line-too-long
@property
def transport(self): # pylint: disable=missing-function-docstring
return self._transport
@property
def traddr(self): # pylint: disable=missing-function-docstring
return self._traddr
@property
def trsvcid(self): # pylint: disable=missing-function-docstring
return self._trsvcid
@property
def host_traddr(self): # pylint: disable=missing-function-docstring
return self._host_traddr
@property
def host_iface(self): # pylint: disable=missing-function-docstring
return self._host_iface
@property
def subsysnqn(self): # pylint: disable=missing-function-docstring
return self._subsysnqn
@property
def cfg(self): # pylint: disable=missing-function-docstring
return self._cfg
def as_dict(self):
'''Return object members as a dictionary'''
data = {
'transport': self.transport,
'traddr': self.traddr,
'subsysnqn': self.subsysnqn,
'trsvcid': self.trsvcid,
'host-traddr': self.host_traddr,
'host-iface': self.host_iface,
}
data.update(self._cfg)
return data
def __str__(self):
return self._id
def __repr__(self):
return self._id
def __eq__(self, other):
if not isinstance(other, self.__class__):
return False
if self._host_iface and other._host_iface:
return self._key == other._key
return self._shortkey == other._shortkey
def __ne__(self, other):
if not isinstance(other, self.__class__):
return True
if self._host_iface and other._host_iface:
return self._key != other._key
return self._shortkey != other._shortkey
def __hash__(self):
return self._hash

Some files were not shown because too many files have changed in this diff Show more