diff --git a/.github/workflows/docker-publish.yml.disabled b/.github/workflows/docker-publish.yml similarity index 95% rename from .github/workflows/docker-publish.yml.disabled rename to .github/workflows/docker-publish.yml index ca5d24f..d04aacd 100644 --- a/.github/workflows/docker-publish.yml.disabled +++ b/.github/workflows/docker-publish.yml @@ -48,7 +48,7 @@ jobs: # https://github.com/docker/metadata-action - name: Extract Docker metadata id: meta - uses: docker/metadata-action@dbef88086f6cef02e264edb7dbf63250c17cef6c + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} diff --git a/.github/workflows/docker-test.yml.disabled b/.github/workflows/docker-test.yml similarity index 100% rename from .github/workflows/docker-test.yml.disabled rename to .github/workflows/docker-test.yml diff --git a/.github/workflows/meson-test.yml b/.github/workflows/meson-test.yml index 14bbc63..18db019 100644 --- a/.github/workflows/meson-test.yml +++ b/.github/workflows/meson-test.yml @@ -10,33 +10,26 @@ on: jobs: meson-build: - runs-on: ubuntu-24.04 + runs-on: ubuntu-latest steps: - name: "CHECKOUT: nvme-stas" uses: actions/checkout@v4 - - name: "INSTALL: build packages" + - name: "INSTALL: Overall dependencies" run: | sudo apt update - sudo apt-get install --yes --quiet meson ninja-build cmake + sudo apt-get install --yes --quiet python3-pip cmake iproute2 + sudo python3 -m pip install --upgrade pip + sudo python3 -m pip install --upgrade wheel meson ninja - - name: "INSTALL: python packages" + - name: "INSTALL: nvme-stas dependencies" run: | - sudo apt-get install --yes --quiet python3-pip python3-wheel pylint pyflakes3 python3-systemd python3-pyudev python3-lxml python3-dasbus python3-gi python3-importlib-resources python3-pyfakefs + sudo apt-get install --yes --quiet docbook-xml docbook-xsl xsltproc libglib2.0-dev libgirepository1.0-dev libsystemd-dev + sudo apt-get install --yes --quiet python3-systemd python3-pyudev python3-lxml + python3 -m pip install --upgrade dasbus pylint==2.17.7 pyflakes PyGObject + python3 -m pip install --upgrade vermin pyfakefs importlib-resources - - name: "INSTALL: documentation packages" - run: | - sudo apt-get install --yes --quiet docbook-xml docbook-xsl xsltproc - - - name: "INSTALL: remaining debian packages" - run: | - sudo apt-get install --yes --quiet iproute2 libglib2.0-dev libgirepository1.0-dev libsystemd-dev - - - name: "INSTALL: pip packages" - run: | - pip install vermin - - - name: "INSTALL: libnvme packages (needed to build libnvme)" + - name: "INSTALL: libnvme dependencies" run: | sudo apt-get install --yes --quiet swig libjson-c-dev @@ -53,7 +46,7 @@ jobs: options: --print-errorlogs --suite nvme-stas # Preserve meson's log file on failure - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v3 if: failure() with: name: "Linux_Meson_Testlog" @@ -61,11 +54,12 @@ jobs: - name: "Generate coverage report" run: | - sudo apt-get install python3-pytest python3-pytest-cov + python3 -m pip install --upgrade pytest + python3 -m pip install --upgrade pytest-cov echo $( pwd ) cp -r .build/staslib/* ./staslib/. pytest --cov=./staslib --cov-report=xml test/test-*.py - - uses: codecov/codecov-action@v5 + - uses: codecov/codecov-action@v3 with: fail_ci_if_error: false diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 985a929..4ca014b 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -20,6 +20,58 @@ jobs: recursive: true ignore: DL3041 + python-lint: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + python-version: ["3.7", "3.8", "3.9", "3.10"] + + steps: + - name: "CHECKOUT: nvme-stas" + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: "INSTALL: additional packages" + run: | + sudo apt update + sudo apt-get install --yes --quiet python3-pip cmake libgirepository1.0-dev libsystemd-dev python3-systemd swig libjson-c-dev || true + sudo python3 -m pip install --upgrade pip wheel meson ninja + python3 -m pip install --upgrade dasbus pylint==2.17.7 pyflakes PyGObject lxml pyudev + + - name: "BUILD: [libnvme, nvme-stas]" + uses: BSFishy/meson-build@v1.0.3 + with: + action: build + directory: .build + setup-options: --buildtype=release --sysconfdir=/etc --prefix=/usr -Dlibnvme:buildtype=release -Dlibnvme:sysconfdir=/etc -Dlibnvme:prefix=/usr -Dlibnvme:python=enabled -Dlibnvme:libdbus=disabled -Dlibnvme:openssl=disabled -Dlibnvme:json-c=disabled -Dlibnvme:keyutils=disabled + + - name: Set PYTHONPATH + run: | + echo "PYTHONPATH=.build:.build/subprojects/libnvme:/usr/lib/python3/dist-packages" >> $GITHUB_ENV + + - name: Show test environment + run: | + echo -e "Build Directory:\n$(ls -laF .build)" + python3 -VV + python3 -m site + python3 -m pylint --version + echo "pyflakes $(python3 -m pyflakes --version)" + + - name: Pylint + run: | + python3 -m pylint -j 0 --rcfile=test/pylint.rc .build/stacctl .build/stacd .build/stafctl .build/stafd .build/stasadm .build/staslib + + - name: Pyflakes + if: always() + run: | + python3 -m pyflakes .build/stacctl .build/stacd .build/stafctl .build/stafd .build/stasadm .build/staslib + python-black: if: ${{ !github.event.act }} # skip during local actions testing name: python-black formatter @@ -29,119 +81,7 @@ jobs: uses: actions/checkout@v4 - name: "BLACK" - uses: psf/black@25.1.0 + uses: psf/black@stable with: options: "--check --diff --color --line-length 120 --skip-string-normalization --extend-exclude (subprojects|debian|.build)" src: "." - - python-lint-Jammy: - runs-on: ubuntu-22.04 - - strategy: - fail-fast: false - matrix: - python-version: ["3.9", "3.10"] - - steps: - - name: "CHECKOUT: nvme-stas" - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: "INSTALL: apt-get packages" - run: | - sudo apt update - sudo apt-get install --yes --quiet meson ninja-build cmake libgirepository1.0-dev libsystemd-dev swig libjson-c-dev - sudo apt-get install --yes --quiet python3-wheel python3-systemd python3-pyudev python3-dasbus python3-gi python3-lxml pyflakes3 python3-tomli - - - name: "INSTALL: pip packages" - run: | - pip install pylint - # pip install PyGObject - - - name: "BUILD: [libnvme, nvme-stas]" - uses: BSFishy/meson-build@v1.0.3 - with: - action: build - directory: .build - setup-options: --buildtype=release --sysconfdir=/etc --prefix=/usr -Dlibnvme:buildtype=release -Dlibnvme:sysconfdir=/etc -Dlibnvme:prefix=/usr -Dlibnvme:python=enabled -Dlibnvme:libdbus=disabled -Dlibnvme:openssl=disabled -Dlibnvme:json-c=disabled -Dlibnvme:keyutils=disabled - - - name: Set PYTHONPATH - run: | - echo "PYTHONPATH=.build:.build/subprojects/libnvme:/usr/lib/python3/dist-packages" >> $GITHUB_ENV - - - name: Show test environment - run: | - echo -e "Build Directory:\n$(ls -laF .build)" - python3 -VV - python3 -m site - pylint --version - echo "pyflakes3 $(pyflakes3 --version)" - - #- name: Pylint - # run: | - # pylint --rcfile=test/pylint.rc .build/stacctl .build/stacd .build/stafctl .build/stafd .build/stasadm .build/staslib - - - name: Pyflakes - if: always() - run: | - pyflakes3 .build/stacctl .build/stacd .build/stafctl .build/stafd .build/stasadm .build/staslib - - python-lint-Noble: - runs-on: ubuntu-24.04 - - strategy: - fail-fast: false - matrix: - python-version: ["3.11", "3.12", "3.13"] - - steps: - - name: "CHECKOUT: nvme-stas" - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5.3.0 - with: - python-version: ${{ matrix.python-version }} - - - name: "INSTALL: apt-get packages" - run: | - sudo apt update - sudo apt-get install --yes --quiet meson ninja-build cmake libgirepository1.0-dev libsystemd-dev swig libjson-c-dev - sudo apt-get install --yes --quiet python3-wheel python3-systemd python3-pyudev python3-dasbus python3-gi python3-lxml pylint pyflakes3 python3-tomli - - #- name: "INSTALL: pip packages" - # run: | - # pip install pylint - # pip install PyGObject - - - name: "BUILD: [libnvme, nvme-stas]" - uses: BSFishy/meson-build@v1.0.3 - with: - action: build - directory: .build - setup-options: --buildtype=release --sysconfdir=/etc --prefix=/usr -Dlibnvme:buildtype=release -Dlibnvme:sysconfdir=/etc -Dlibnvme:prefix=/usr -Dlibnvme:python=enabled -Dlibnvme:libdbus=disabled -Dlibnvme:openssl=disabled -Dlibnvme:json-c=disabled -Dlibnvme:keyutils=disabled - - - name: Set PYTHONPATH - run: | - echo "PYTHONPATH=.build:.build/subprojects/libnvme:/usr/lib/python3/dist-packages" >> $GITHUB_ENV - - - name: Show test environment - run: | - echo -e "Build Directory:\n$(ls -laF .build)" - python3 -VV - python3 -m site - pylint --version - echo "pyflakes3 $(pyflakes3 --version)" - - - name: Pylint - run: | - pylint --jobs=0 --rcfile=test/pylint.rc .build/stacctl .build/stacd .build/stafctl .build/stafd .build/stasadm .build/staslib - - - name: Pyflakes - if: always() - run: | - pyflakes3 .build/stacctl .build/stacd .build/stafctl .build/stafd .build/stasadm .build/staslib diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 2a20488..e3e10f1 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -21,7 +21,7 @@ build: - pandoc jobs: post_install: - - pip install lxml + - pip3 install lxml pre_build: - meson .build -Dreadthedocs=true || cat .build/meson-logs/meson-log.txt - ninja -C .build diff --git a/Dockerfile b/Dockerfile index d770999..e1d7b43 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM fedora:41 +FROM fedora:39 WORKDIR /root diff --git a/Makefile b/Makefile index 93f7b94..c02ac77 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,6 @@ endif purge: ifneq ("$(wildcard ${BUILD-DIR})","") rm -rf ${BUILD-DIR} - meson subprojects purge --confirm endif .PHONY: install @@ -47,8 +46,7 @@ install: stas .PHONY: uninstall uninstall: ${BUILD-DIR} - cd ${BUILD-DIR} && sudo meson --internal uninstall - + sudo ninja $@ -C ${BUILD-DIR} .PHONY: dist dist: stas @@ -72,7 +70,7 @@ black: black --diff --color --line-length 120 --skip-string-normalization --extend-exclude="(subprojects|debian|.build)" . # Coverage requirements: -# apt-get install python3-coverage +# pip install coverage .PHONY: coverage coverage: stas cd ${BUILD-DIR} && ./coverage.sh diff --git a/NEWS.md b/NEWS.md index 01c6eb4..6b08878 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,15 +1,5 @@ # STorage Appliance Services (STAS) -## Changes with release 2.4 - -New features: - -Support for authentication - -Bug fix: - -* Various fixes related to unit testing and GitHub Actions - ## Changes with release 2.3.1 Bug fix: diff --git a/README.md b/README.md index b67bc7e..4938d49 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ The following packages must be installed to use **`stafd`**/**`stacd`** sudo apt-get install -y python3-pyudev python3-systemd python3-gi sudo apt-get install -y python3-dasbus # Ubuntu 22.04 OR: -sudo pip install dasbus # Ubuntu 20.04 (may require --break-system-packages) +sudo pip3 install dasbus # Ubuntu 20.04 ``` **RPM packages (tested on Fedora 34..35 and SLES15):** diff --git a/TESTING.md b/TESTING.md index 6e52e04..0fa928f 100644 --- a/TESTING.md +++ b/TESTING.md @@ -184,7 +184,7 @@ $ sudo ./nvmet.py clean This requires the [Python coverage package](https://coverage.readthedocs.io/en/6.4.1/), which can be installed as follows: ```bash -$ sudo apt-get install python3-coverage +$ sudo pip install coverage ``` Note that this test cannot be run while `stafd` and `stacd` are running. Make sure to stop `stafd` and `stacd` if they are running (`systemctl stop [stafd|stacd]`). You may also need to mask those services (`systemctl mask [stafd|stacd]`) if coverage fails to start. diff --git a/coverage.sh.in b/coverage.sh.in index 0d2478e..51d1106 100755 --- a/coverage.sh.in +++ b/coverage.sh.in @@ -377,7 +377,7 @@ persistent-connections = false zeroconf-connections-persistence = 1:01 [Controllers] -controller = transport = tcp ; traddr = localhost ; ; ; kato=31; dhchap-ctrl-secret=DHHC-1:00:not-so-secret/not-so-secret/not-so-secret/not-so: ; dhchap-secret=DHHC-1:00:very-secret/very-secret/very-secret/very-secret/: +controller = transport = tcp ; traddr = localhost ; ; ; kato=31; dhchap-ctrl-secret=not-so-secret controller=transport=tcp;traddr=1.1.1.1 controller=transport=tcp;traddr=100.100.100.100 controller=transport=tcp;traddr=2607:f8b0:4002:c2c::71 diff --git a/debian/changelog b/debian/changelog index 5468e32..dd0d57c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,14 +1,3 @@ -nvme-stas (2.4-1) sid; urgency=medium - - * Updating to standards version 4.7.1. - * Updating to standards version 4.7.2. - * Merging upstream version 2.4. - * Updating year in upstream copyright for 2025. - * Removing test-dummy-interfaces.patch, included upstream. - * Removing skip-esoteric-interfaces.patch, included upstream. - - -- Daniel Baumann Mon, 17 Mar 2025 07:20:31 +0100 - nvme-stas (2.3.1-4) sid; urgency=medium * Updating source url in copyright. diff --git a/debian/copyright b/debian/copyright index ca59153..d206bfe 100644 --- a/debian/copyright +++ b/debian/copyright @@ -4,7 +4,7 @@ Upstream-Contact: https://github.com/linux-nvme/nvme-stas/issues Source: https://github.com/linux-nvme/nvme-stas/tags Files: * -Copyright: 2022-2025 Dell Inc. +Copyright: 2022-2023 Dell Inc. License: Apache-2.0 Files: * diff --git a/debian/patches/series b/debian/patches/series new file mode 100644 index 0000000..9ebe5cf --- /dev/null +++ b/debian/patches/series @@ -0,0 +1,2 @@ +upstream/0001-test-dummy-interfaces.patch +upstream/0002-skip-esoteric-interfaces.patch diff --git a/debian/patches/upstream/0001-test-dummy-interfaces.patch b/debian/patches/upstream/0001-test-dummy-interfaces.patch new file mode 100644 index 0000000..55eee35 --- /dev/null +++ b/debian/patches/upstream/0001-test-dummy-interfaces.patch @@ -0,0 +1,20 @@ +Author: Martin Belanger +Description: test: Fix test-udev with dummy interfaces + https://github.com/linux-nvme/nvme-stas/issues/407 + https://github.com/linux-nvme/nvme-stas/commit/371c1e875a9a660adca241265e4cd460ee7e5e5c + +diff -Naurp nvme-stas.orig/test/test-udev.py nvme-stas/test/test-udev.py +--- nvme-stas.orig/test/test-udev.py ++++ nvme-stas/test/test-udev.py +@@ -295,6 +295,11 @@ class Test(unittest.TestCase): + def test__cid_matches_tid(self): + ifaces = iputil.net_if_addrs() + for ifname, addrs in self.ifaces.items(): ++ # contains a subset of the interfaces found in . ++ # So, let's make sure that we only test with the interfaces found in both. ++ if ifname not in ifaces: ++ continue ++ + ############################################## + # IPV4 + diff --git a/debian/patches/upstream/0002-skip-esoteric-interfaces.patch b/debian/patches/upstream/0002-skip-esoteric-interfaces.patch new file mode 100644 index 0000000..4994093 --- /dev/null +++ b/debian/patches/upstream/0002-skip-esoteric-interfaces.patch @@ -0,0 +1,70 @@ +Author: Olivier Gayot +Description: Skip mac2iface test for esoteric interfaces + mac2iface takes a MAC address as argument and returns the corresponding + interface (if any). + The mac2iface tests will however invoke mac2iface with invalid MAC addresses + when esoteric network interfaces are present on the system. As an example, + armhf autopkgtest runners in Ubuntu have gre interfaces configured so the + test-suite fails. + . + We now ensure that the test-suite calls mac2iface with only valid MAC + addresses. + . + Furthermore, sometimes the same MAC address is assigned to more than one + interface on the system (this is true for VLAN interfaces for instance). This + confuses mac2iface, which returns only the first match. This scenario is + possibly more of a nvme-stas bug than a test-suite bug, but for now we will + just skip the interfaces that have a duplicate MAC address. + . + https://github.com/linux-nvme/nvme-stas/pull/411 + https://github.com/linux-nvme/nvme-stas/commit/2336eab5b4e4e2f9fd28b7efe425f86e6a23ab91 + +diff -Naurp nvme-stas.orig/test/test-iputil.py nvme-stas/test/test-iputil.py +--- nvme-stas.orig/test/test-iputil.py ++++ nvme-stas/test/test-iputil.py +@@ -43,11 +43,41 @@ class Test(unittest.TestCase): + self.assertEqual('', iputil.get_interface(ifaces, '')) + self.assertEqual('', iputil.get_interface(ifaces, None)) + ++ @staticmethod ++ def _is_ok_for_mac2iface(iface) -> bool: ++ ''' mac2iface can only work with interfaces that have a proper MAC ++ address. One can use this function to filter out other interfaces ++ configured on the system. ''' ++ if iface['link_type'] != 'ether': ++ # Some esoteric interface types (e.g., gre) use the address ++ # field to store something that is not a MAC address. Skip ++ # them. ++ return False ++ if 'address' not in iface: ++ return False ++ if iface['address'] == '00:00:00:00:00:00': ++ # All 0's is an invalid MAC address so do not bother. ++ # In practice, it often appears as the address of the loopback ++ # interface but it can also appear for other things like a gretap ++ # or erspan interface. ++ return False ++ return True ++ + def test_mac2iface(self): +- for iface in self.ifaces: +- address = iface.get('address', None) +- if address: +- self.assertEqual(iface['ifname'], iputil.mac2iface(address)) ++ # We only test the interfaces that have a MAC address, and a valid one. ++ candidate_ifaces = [iface for iface in self.ifaces if self._is_ok_for_mac2iface(iface)] ++ ++ for iface in candidate_ifaces: ++ if len([x for x in candidate_ifaces if x['address'] == iface['address']]) >= 2: ++ # We need to be careful, sometimes we can have the same MAC address ++ # on multiple interfaces. This happens with VLAN interfaces for ++ # instance. mac2iface will obviously be confused when dealing with ++ # those so let's skip the interfaces that have duplicate MAC. ++ logging.warning('[%s] is not the only interface with address [%s]', ++ iface['ifname'], iface['address']) ++ continue ++ ++ self.assertEqual(iface['ifname'], iputil.mac2iface(iface['address'])) + + def test_remove_invalid_addresses(self): + good_tcp = trid.TID({'transport': 'tcp', 'traddr': '1.1.1.1', 'subsysnqn': '', 'trsvcid': '8009'}) diff --git a/doc/standard-conf.xml b/doc/standard-conf.xml index 36cb19f..50d4fe5 100644 --- a/doc/standard-conf.xml +++ b/doc/standard-conf.xml @@ -376,21 +376,6 @@ - - dhchap-secret= - - - NVMe In-band authentication host secret (i.e. key); - needs to be in ASCII format as specified in NVMe 2.0 - section 8.13.5.8 Secret representation. If this - option is not specified, the default is read - from /etc/stas/sys.conf (see the 'key' parameter - under the [Host] section). In-band authentication - is attempted when this is present. - - - - dhchap-ctrl-secret= diff --git a/etc/stas/stacd.conf b/etc/stas/stacd.conf index c1bcbed..9fbc1c3 100644 --- a/etc/stas/stacd.conf +++ b/etc/stas/stacd.conf @@ -235,14 +235,6 @@ # This forces the connection to be made on a specific interface # instead of letting the system decide. # -# dhchap-secret [OPTIONAL] -# NVMe In-band authentication host secret (i.e. key); needs to be -# in ASCII format as specified in NVMe 2.0 section 8.13.5.8 Secret -# representation. If this option is not specified, the default is -# read from /etc/stas/sys.conf (see the 'key' parameter under the -# [Host] section). In-band authentication is attempted when this -# is present. -# # dhchap-ctrl-secret [OPTIONAL] # NVMe In-band authentication controller secret (i.e. key) for # bi-directional authentication; needs to be in ASCII format as diff --git a/meson.build b/meson.build index 59f5929..5b42856 100644 --- a/meson.build +++ b/meson.build @@ -9,7 +9,7 @@ project( 'nvme-stas', meson_version: '>= 0.53.0', - version: '2.4', + version: '2.3.1', license: 'Apache-2.0', default_options: [ 'buildtype=release', @@ -39,13 +39,7 @@ if want_man or want_html or want_readthedocs buildtime_modules += ['lxml'] endif -# On older systems we had to invoke Python 3 as "python3". On newer systems, -# Python 2 has been completely deprecated and Python 3 is simply named "python". -pymod = import('python') -python3 = pymod.find_installation('python3', modules:buildtime_modules, required:false) -if not python3.found() - python3 = pymod.find_installation('python', modules:buildtime_modules) -endif +python3 = import('python').find_installation('python3', modules:buildtime_modules) python_version = python3.language_version() python_version_req = '>=3.6' if not python_version.version_compare(python_version_req) @@ -57,7 +51,7 @@ endif missing_runtime_mods = false py_modules_reqd = [ ['libnvme', 'Install python3-libnvme (deb/rpm)'], - ['dasbus', 'Install python3-dasbus (deb/rpm) OR pip install dasbus'], + ['dasbus', 'Install python3-dasbus (deb/rpm) OR pip3 install dasbus'], ['pyudev', 'Install python3-pyudev (deb/rpm)'], ['systemd', 'Install python3-systemd (deb/rpm)'], ['gi', 'Install python3-gi (deb) OR python3-gobject (rpm)'], @@ -171,7 +165,7 @@ summary_dict = { 'dbus_conf_dir ': dbus_conf_dir, 'sd_unit_dir ': sd_unit_dir, 'build location ': meson.current_build_dir(), - 'libnvme location ': libnvme_location, + 'libnvme for tests ': libnvme_location, } summary(summary_dict, section: 'Directories') diff --git a/stacctl.py b/stacctl.py index 38f45fd..9375af4 100755 --- a/stacctl.py +++ b/stacctl.py @@ -7,7 +7,8 @@ # # Authors: Martin Belanger # -'''STorage Appliance Connector Control Utility''' +''' STorage Appliance Connector Control Utility +''' import sys import json import pprint diff --git a/stacd.py b/stacd.py index 024e1f1..60bcab2 100755 --- a/stacd.py +++ b/stacd.py @@ -7,7 +7,8 @@ # # Authors: Martin Belanger # -'''STorage Appliance Connector Daemon''' +''' STorage Appliance Connector Daemon +''' import sys from argparse import ArgumentParser from staslib import defs diff --git a/stafctl.py b/stafctl.py index 0a61556..a9f94ec 100755 --- a/stafctl.py +++ b/stafctl.py @@ -7,7 +7,8 @@ # # Authors: Martin Belanger # -'''STorage Appliance Finder Control Utility''' +''' STorage Appliance Finder Control Utility +''' import sys import json import pprint diff --git a/stafd.py b/stafd.py index 3a1e60e..6701499 100755 --- a/stafd.py +++ b/stafd.py @@ -7,7 +7,8 @@ # # Authors: Martin Belanger # -'''STorage Appliance Finder Daemon''' +''' STorage Appliance Finder Daemon +''' import sys from argparse import ArgumentParser from staslib import defs diff --git a/stasadm.py b/stasadm.py index 3f1538f..294fdde 100755 --- a/stasadm.py +++ b/stasadm.py @@ -7,7 +7,7 @@ # # Authors: Martin Belanger # -'''STorage Appliance Services Admin Tool''' +''' STorage Appliance Services Admin Tool ''' import os import sys import uuid diff --git a/staslib/avahi.py b/staslib/avahi.py index b8a30da..cd4d1f9 100644 --- a/staslib/avahi.py +++ b/staslib/avahi.py @@ -6,8 +6,8 @@ # # Authors: Martin Belanger # -'''Module that provides a way to retrieve discovered -services from the Avahi daemon over D-Bus. +''' Module that provides a way to retrieve discovered + services from the Avahi daemon over D-Bus. ''' import socket import typing @@ -167,11 +167,9 @@ class Service: # pylint: disable=too-many-instance-attributes 'trsvcid': trsvcid, # host-iface permitted for tcp alone and not rdma 'host-iface': host_iface, - 'subsysnqn': ( - txt.get('nqn', defs.WELL_KNOWN_DISC_NQN).strip() - if conf.NvmeOptions().discovery_supp - else defs.WELL_KNOWN_DISC_NQN - ), + 'subsysnqn': txt.get('nqn', defs.WELL_KNOWN_DISC_NQN).strip() + if conf.NvmeOptions().discovery_supp + else defs.WELL_KNOWN_DISC_NQN, } self._ip = iputil.get_ipaddress_obj(traddr, ipv4_mapped_convert=True) diff --git a/staslib/conf.py b/staslib/conf.py index 3be6f50..4dd411c 100644 --- a/staslib/conf.py +++ b/staslib/conf.py @@ -289,6 +289,8 @@ class SvcConf(metaclass=singleton.Singleton): # pylint: disable=too-many-public nr_write_queues = property(functools.partial(get_option, section='Global', option='nr-write-queues')) reconnect_delay = property(functools.partial(get_option, section='Global', option='reconnect-delay')) + zeroconf_enabled = property(functools.partial(get_option, section='Service Discovery', option='zeroconf')) + zeroconf_persistence_sec = property( functools.partial( get_option, section='Discovery controller connection management', option='zeroconf-connections-persistence' @@ -305,11 +307,6 @@ class SvcConf(metaclass=singleton.Singleton): # pylint: disable=too-many-public functools.partial(get_option, section='I/O controller connection management', option='connect-attempts-on-ncc') ) - @property # pylint chokes on this when defined as zeroconf_enabled=property(...). Works fine using a decorator... - def zeroconf_enabled(self): - '''Return whether zeroconf is enabled''' - return self.get_option(section='Service Discovery', option='zeroconf') - @property def stypes(self): '''@brief Get the DNS-SD/mDNS service types.''' @@ -341,7 +338,6 @@ class SvcConf(metaclass=singleton.Singleton): # pylint: disable=too-many-public 'host-traddr': [TRADDR], 'host-iface': [IFACE], 'host-nqn': [NQN], - 'dhchap-secret': [KEY], 'dhchap-ctrl-secret': [KEY], 'hdr-digest': [BOOL] 'data-digest': [BOOL] @@ -711,7 +707,7 @@ class NvmeOptions(metaclass=singleton.Singleton): # ****************************************************************************** -class NbftConf(metaclass=singleton.Singleton): # pylint: disable=too-few-public-methods +class NbftConf(metaclass=singleton.Singleton): '''Read and cache configuration file.''' def __init__(self, root_dir=defs.NBFT_SYSFS_PATH): diff --git a/staslib/ctrl.py b/staslib/ctrl.py index e18414e..e4cda6b 100644 --- a/staslib/ctrl.py +++ b/staslib/ctrl.py @@ -221,39 +221,23 @@ class Controller(stas.ControllerABC): # pylint: disable=too-many-instance-attri host_traddr=self.tid.host_traddr if self.tid.host_traddr else None, host_iface=host_iface, ) - self._ctrl.discovery_ctrl = self._discovery_ctrl + self._ctrl.discovery_ctrl_set(self._discovery_ctrl) - # Set the DHCHAP host key on the controller - # NOTE that this may eventually have to + # Set the DHCHAP key on the controller + # NOTE that this will eventually have to # change once we have support for AVE (TP8019) - # This is used for in-band authentication - dhchap_host_key = self.tid.cfg.get('dhchap-secret') - if dhchap_host_key and self._nvme_options.dhchap_hostkey_supp: - try: - self._ctrl.dhchap_host_key = dhchap_host_key - except AttributeError: + ctrl_dhchap_key = self.tid.cfg.get('dhchap-ctrl-secret') + if ctrl_dhchap_key and self._nvme_options.dhchap_ctrlkey_supp: + has_dhchap_key = hasattr(self._ctrl, 'dhchap_key') + if not has_dhchap_key: logging.warning( - '%s | %s - libnvme-%s does not allow setting the host DHCHAP key on the controller. Please upgrade libnvme.', - self.id, - self.device, - defs.LIBNVME_VERSION, - ) - - # Set the DHCHAP controller key on the controller - # NOTE that this may eventually have to - # change once we have support for AVE (TP8019) - # This is used for bidirectional authentication - dhchap_ctrl_key = self.tid.cfg.get('dhchap-ctrl-secret') - if dhchap_ctrl_key and self._nvme_options.dhchap_ctrlkey_supp: - try: - self._ctrl.dhchap_key = dhchap_ctrl_key - except AttributeError: - logging.warning( - '%s | %s - libnvme-%s does not allow setting the controller DHCHAP key on the controller. Please upgrade libnvme.', + '%s | %s - libnvme-%s does not allow setting the controller DHCHAP key. Please upgrade libnvme.', self.id, self.device, defs.LIBNVME_VERSION, ) + else: + self._ctrl.dhchap_key = ctrl_dhchap_key # Audit existing nvme devices. If we find a match, then # we'll just borrow that device instead of creating a new one. diff --git a/staslib/defs.py b/staslib/defs.py index 164c366..fc1135b 100644 --- a/staslib/defs.py +++ b/staslib/defs.py @@ -6,7 +6,8 @@ # # Authors: Martin Belanger -'''@brief This file gets automagically configured by meson at build time.''' +''' @brief This file gets automagically configured by meson at build time. +''' import os import sys import shutil diff --git a/staslib/gutil.py b/staslib/gutil.py index 0922fce..4cdc087 100644 --- a/staslib/gutil.py +++ b/staslib/gutil.py @@ -123,7 +123,7 @@ class NameResolver: # pylint: disable=too-few-public-methods The callback @callback will be called once all hostnames have been resolved. - @param controllers_in: List of trid.TID + @param controllers: List of trid.TID ''' pending_resolution_count = 0 controllers_out = [] diff --git a/staslib/service.py b/staslib/service.py index 0badc53..19acec3 100644 --- a/staslib/service.py +++ b/staslib/service.py @@ -510,7 +510,7 @@ class Stac(Service): UDEV_RULE_OVERRIDE = r''' ACTION=="change", SUBSYSTEM=="fc", ENV{FC_EVENT}=="nvmediscovery", \ ENV{NVMEFC_HOST_TRADDR}=="*", ENV{NVMEFC_TRADDR}=="*", \ - RUN+="%s --no-block restart nvmf-connect@--device\x3dnone\x09--transport\x3dfc\x09--traddr\x3d$env{NVMEFC_TRADDR}\x09--trsvcid\x3dnone\x09--host-traddr\x3d$env{NVMEFC_HOST_TRADDR}.service" + RUN+="%s --no-block start nvmf-connect@--transport=fc\t--traddr=$env{NVMEFC_TRADDR}\t--trsvcid=none\t--host-traddr=$env{NVMEFC_HOST_TRADDR}.service" ''' @@ -809,7 +809,11 @@ class Staf(Service): origin = ( 'configured' if tid in configured_ctrl_list - else 'referral' if tid in referral_ctrl_list else 'discovered' if tid in discovered_ctrl_list else None + else 'referral' + if tid in referral_ctrl_list + else 'discovered' + if tid in discovered_ctrl_list + else None ) if origin is not None: controller.origin = origin @@ -864,12 +868,10 @@ class Staf(Service): return # We need to invoke "nvme connect-all" using nvme-cli's nvmf-connect@.service - # NOTE 1: Eventually, we'll be able to drop --host-traddr and --host-iface from + # NOTE: Eventually, we'll be able to drop --host-traddr and --host-iface from # the parameters passed to nvmf-connect@.service. A fix was added to connect-all # to infer these two values from the device used to connect to the DC. # Ref: https://github.com/linux-nvme/nvme-cli/pull/1812 - # - # NOTE 2:--transport, --traddr, and --trsvcid, not needed when using --device cnf = [ ('--device', udev_obj.sys_name), ('--host-traddr', udev_obj.properties.get('NVME_HOST_TRADDR', None)), diff --git a/staslib/trid.py b/staslib/trid.py index cb4a5de..e814f4e 100644 --- a/staslib/trid.py +++ b/staslib/trid.py @@ -33,7 +33,6 @@ class TID: # pylint: disable=too-many-instance-attributes 'host-nqn': str, # [optional] # Connection parameters - 'dhchap-secret': str, # [optional] 'dhchap-ctrl-secret': str, # [optional] 'hdr-digest': str, # [optional] 'data-digest': str, # [optional] diff --git a/staslib/version.py b/staslib/version.py index 2a4c515..999d916 100644 --- a/staslib/version.py +++ b/staslib/version.py @@ -6,12 +6,12 @@ # # Authors: Martin Belanger # -'''distutils (and hence LooseVersion) is being deprecated. None of the -suggested replacements (e.g. from pkg_resources import parse_version) quite -work with Linux kernel versions the way LooseVersion does. +''' distutils (and hence LooseVersion) is being deprecated. None of the + suggested replacements (e.g. from pkg_resources import parse_version) quite + work with Linux kernel versions the way LooseVersion does. -It was suggested to simply lift the LooseVersion code and vendor it in, -which is what this module is about. + It was suggested to simply lift the LooseVersion code and vendor it in, + which is what this module is about. ''' import re diff --git a/test/meson.build b/test/meson.build index 31068b2..ef31c63 100644 --- a/test/meson.build +++ b/test/meson.build @@ -9,8 +9,6 @@ srce_dir = meson.current_source_dir() test_env = environment({'MALLOC_PERTURB_': '0'}) -test_env.append('PYTHONMALLOC', 'malloc') -test_list = modules_to_lint + packages_to_lint libnvme_location = '?' @@ -25,9 +23,9 @@ if get_option('libnvme-sel') == 'pre-installed' rr = run_command(python3, '-c', 'import libnvme; print(f"{libnvme.__path__[0]}")', check: false, env: test_env) if rr.returncode() == 0 libnvme_location = rr.stdout().strip() - libnvme_path = fs.parent(libnvme_location) - PYTHONPATH = ':'.join([libnvme_path, PYTHONPATH]) - test_env.prepend('PYTHONPATH', PYTHONPATH) + pythonpath = fs.parent(libnvme_location) + test_env.prepend('PYTHONPATH', pythonpath) # Look in standard location first + test_env.append('PYTHONPATH', PYTHONPATH) # Look in the build directory second endif endif @@ -48,7 +46,13 @@ if libnvme_location == '?' else #--------------------------------------------------------------------------- # pylint and pyflakes - if test_list.length() != 0 + + # There's a bug with pylint 3.X. Tests should be run with pylint + # 2.17.7 (or less), which can be installed with: + # python3 -m pip install --upgrade pylint==2.17.7 + + + if modules_to_lint.length() != 0 pylint = find_program('pylint', required: false) pyflakes = find_program('pyflakes3', required: false) if not pyflakes.found() @@ -61,12 +65,12 @@ else rcfile = srce_dir / 'pylint.rc' if pylint.found() - test('pylint', pylint, args: ['--rcfile=' + rcfile] + test_list, env: test_env) + test('pylint', pylint, args: ['--rcfile=' + rcfile] + modules_to_lint + packages_to_lint, env: test_env) else warning('Skiping some of the tests because "pylint" is missing.') endif if pyflakes.found() - test('pyflakes', pyflakes, args: test_list, env: test_env) + test('pyflakes', pyflakes, args: modules_to_lint, env: test_env) else warning('Skiping some of the tests because "pyflakes" is missing.') endif @@ -152,8 +156,8 @@ tools = [ ] vermin = find_program('vermin', required: false) if vermin.found() - if test_list.length() != 0 - test('vermin code', vermin, args: ['--config-file', srce_dir / 'vermin.conf'] + test_list, env: test_env) + if modules_to_lint.length() != 0 + test('vermin code', vermin, args: ['--config-file', srce_dir / 'vermin.conf'] + modules_to_lint, env: test_env) endif test('vermin tools', vermin, args: ['--config-file', srce_dir / 'vermin-tools.conf'] + tools, env: test_env) else diff --git a/test/test-iputil.py b/test/test-iputil.py index 822dbaf..b0a5448 100755 --- a/test/test-iputil.py +++ b/test/test-iputil.py @@ -43,40 +43,11 @@ class Test(unittest.TestCase): self.assertEqual('', iputil.get_interface(ifaces, '')) self.assertEqual('', iputil.get_interface(ifaces, None)) - @staticmethod - def _is_ok_for_mac2iface(iface) -> bool: - '''mac2iface can only work with interfaces that have a proper MAC - address. One can use this function to filter out other interfaces - configured on the system.''' - if iface['link_type'] != 'ether': - # Some esoteric interface types (e.g., gre) use the address - # field to store something that is not a MAC address. Skip - # them. - return False - if 'address' not in iface: - return False - if iface['address'] == '00:00:00:00:00:00': - # All 0's is an invalid MAC address so do not bother. - # In practice, it often appears as the address of the loopback - # interface but it can also appear for other things like a gretap - # or erspan interface. - return False - return True - def test_mac2iface(self): - # We only test the interfaces that have a MAC address, and a valid one. - candidate_ifaces = [iface for iface in self.ifaces if self._is_ok_for_mac2iface(iface)] - - for iface in candidate_ifaces: - if len([x for x in candidate_ifaces if x['address'] == iface['address']]) >= 2: - # We need to be careful, sometimes we can have the same MAC address - # on multiple interfaces. This happens with VLAN interfaces for - # instance. mac2iface will obviously be confused when dealing with - # those so let's skip the interfaces that have duplicate MAC. - logging.warning('[%s] is not the only interface with address [%s]', iface['ifname'], iface['address']) - continue - - self.assertEqual(iface['ifname'], iputil.mac2iface(iface['address'])) + for iface in self.ifaces: + address = iface.get('address', None) + if address: + self.assertEqual(iface['ifname'], iputil.mac2iface(address)) def test_remove_invalid_addresses(self): good_tcp = trid.TID({'transport': 'tcp', 'traddr': '1.1.1.1', 'subsysnqn': '', 'trsvcid': '8009'}) diff --git a/test/test-udev.py b/test/test-udev.py index 2f29471..ba484e0 100755 --- a/test/test-udev.py +++ b/test/test-udev.py @@ -200,7 +200,8 @@ def get_tids_to_test(family, src_ip, ifname): ] -class DummyDevice: ... +class DummyDevice: + ... class Test(unittest.TestCase): @@ -294,11 +295,6 @@ class Test(unittest.TestCase): def test__cid_matches_tid(self): ifaces = iputil.net_if_addrs() for ifname, addrs in self.ifaces.items(): - # contains a subset of the interfaces found in . - # So, let's make sure that we only test with the interfaces found in both. - if ifname not in ifaces: - continue - ############################################## # IPV4 diff --git a/usr/lib/systemd/system/stacd.in.service b/usr/lib/systemd/system/stacd.in.service index dae6620..77a4ad5 100644 --- a/usr/lib/systemd/system/stacd.in.service +++ b/usr/lib/systemd/system/stacd.in.service @@ -28,14 +28,5 @@ RuntimeDirectory=stacd CacheDirectory=stacd RuntimeDirectoryPreserve=yes -ProtectHome=true -ProtectKernelModules=true -ProtectKernelLogs=true -ProtectControlGroups=true -ProtectProc=invisible -RestrictRealtime=true -LockPersonality=yes -MemoryDenyWriteExecute=yes - [Install] WantedBy=multi-user.target diff --git a/usr/lib/systemd/system/stafd.in.service b/usr/lib/systemd/system/stafd.in.service index 9e31685..01ddc2b 100644 --- a/usr/lib/systemd/system/stafd.in.service +++ b/usr/lib/systemd/system/stafd.in.service @@ -31,14 +31,5 @@ RuntimeDirectory=stafd CacheDirectory=stafd RuntimeDirectoryPreserve=yes -ProtectHome=true -ProtectKernelModules=true -ProtectKernelLogs=true -ProtectControlGroups=true -ProtectProc=invisible -RestrictRealtime=true -LockPersonality=yes -MemoryDenyWriteExecute=yes - [Install] WantedBy=multi-user.target diff --git a/utils/nvmet/auth.conf b/utils/nvmet/auth.conf deleted file mode 100644 index b4c23e6..0000000 --- a/utils/nvmet/auth.conf +++ /dev/null @@ -1,34 +0,0 @@ -# Config file format: Python, i.e. dict(), list(), int, str, etc... -# port ids (id) are integers 0...N -# namespaces are integers 0..N -# subsysnqn can be integers or strings -{ - 'ports': [ - { - 'id': 1, - #'adrfam': 'ipv6', - #'traddr': '::', - 'adrfam': 'ipv4', - 'traddr': '0.0.0.0', - 'trsvcid': 4420, - 'trtype': 'tcp', - } - ], - - 'subsystems': [ - { - 'subsysnqn': 'nqn.1988-11.com.dell:PowerSANxxx:01:20210225100113-454f73093ceb4847a7bdfc6e34ae8e28', - 'port': 1, - 'namespaces': [1], - 'allowed_hosts': [ - { - # Must match with the NQN and key configured on the host - # Key was generated with: - # nvme gen-dhchap-key ... - 'nqn': 'nqn.2014-08.org.nvmexpress:uuid:46ba5037-7ce5-41fa-9452-48477bf00080', - 'key': 'DHHC-1:00:2kx1hDTUPdvwtxHYUXFRl8pzn5hYZH7K3Z77IYM4hNN6/fQT:', - }, - ], - }, - ] -} diff --git a/utils/nvmet/nvmet.conf b/utils/nvmet/nvmet.conf index ba3d40a..d3288e9 100644 --- a/utils/nvmet/nvmet.conf +++ b/utils/nvmet/nvmet.conf @@ -6,10 +6,10 @@ 'ports': [ { 'id': 1, - #'adrfam': 'ipv6', - #'traddr': '::', - 'adrfam': 'ipv4', - 'traddr': '0.0.0.0', + 'adrfam': 'ipv6', + 'traddr': '::', + #'adrfam': 'ipv4', + #'traddr': '0.0.0.0', 'trsvcid': 8009, 'trtype': 'tcp', } diff --git a/utils/nvmet/nvmet.py b/utils/nvmet/nvmet.py index 9049d63..baf6560 100755 --- a/utils/nvmet/nvmet.py +++ b/utils/nvmet/nvmet.py @@ -52,26 +52,19 @@ def _get_loaded_nvmet_modules(): return output -def _runcmd(cmd: list, quiet=False, capture_output=False): +def _runcmd(cmd: list, quiet=False): if not quiet: print(' '.join(cmd)) if args.dry_run: return - - try: - cp = subprocess.run(cmd, capture_output=capture_output, text=True) - except TypeError: - # For older Python versions that don't support "capture_output" or "text" - cp = subprocess.run(cmd, stdout=subprocess.PIPE, universal_newlines=True) - - return cp.stdout if capture_output else None + subprocess.run(cmd) def _modprobe(module: str, args: list = None, quiet=False): cmd = ['/usr/sbin/modprobe', module] if args: cmd.extend(args) - _runcmd(cmd, quiet=quiet) + _runcmd(cmd, quiet) def _mkdir(dname: str): @@ -100,32 +93,12 @@ def _symlink(port: str, subsysnqn: str): link.symlink_to(target) -def _symlink_allowed_hosts(hostnqn: str, subsysnqn: str): - print( - f'$( cd "/sys/kernel/config/nvmet/subsystems/{subsysnqn}/allowed_hosts" && ln -s "../../../hosts/{hostnqn}" "{hostnqn}" )' - ) - if args.dry_run: - return - target = os.path.join('/sys/kernel/config/nvmet/hosts', hostnqn) - link = pathlib.Path(os.path.join('/sys/kernel/config/nvmet/subsystems', subsysnqn, 'allowed_hosts', hostnqn)) - link.symlink_to(target) - - -def _create_subsystem(subsysnqn: str, allowed_hosts: list) -> str: +def _create_subsystem(subsysnqn: str) -> str: print(f'###{Fore.GREEN} Create subsystem: {subsysnqn}{Style.RESET_ALL}') dname = os.path.join('/sys/kernel/config/nvmet/subsystems/', subsysnqn) _mkdir(dname) - _echo(0 if allowed_hosts else 1, os.path.join(dname, 'attr_allow_any_host')) - - # Configure all the hosts that are allowed to access this subsystem - for host in allowed_hosts: - hostnqn = host.get('nqn') - hostkey = host.get('key') - if all([hostnqn, hostkey]): - dname = os.path.join('/sys/kernel/config/nvmet/hosts/', hostnqn) - _mkdir(dname) - _echo(hostkey, os.path.join(dname, 'dhchap_key')) - _symlink_allowed_hosts(hostnqn, subsysnqn) + _echo(1, os.path.join(dname, 'attr_allow_any_host')) + return dname def _create_namespace(subsysnqn: str, id: str, node: str) -> str: @@ -134,6 +107,7 @@ def _create_namespace(subsysnqn: str, id: str, node: str) -> str: _mkdir(dname) _echo(node, os.path.join(dname, 'device_path')) _echo(1, os.path.join(dname, 'enable')) + return dname def _args_valid(id, traddr, trsvcid, trtype, adrfam): @@ -241,9 +215,8 @@ def create(args): str(subsystem.get('port')), subsystem.get('namespaces'), ) - if None not in (subsysnqn, port, namespaces): - _create_subsystem(subsysnqn, subsystem.get('allowed_hosts', [])) + _create_subsystem(subsysnqn) for id in namespaces: _create_namespace(subsysnqn, str(id), dev_node) else: @@ -262,16 +235,10 @@ def clean(args): if not args.dry_run and os.geteuid() != 0: sys.exit(f'Permission denied. You need root privileges to run {os.path.basename(__file__)}.') - print(f'###{Fore.GREEN} 1st) Remove the symlinks{Style.RESET_ALL}') print('rm -f /sys/kernel/config/nvmet/ports/*/subsystems/*') for dname in pathlib.Path('/sys/kernel/config/nvmet/ports').glob('*/subsystems/*'): _runcmd(['rm', '-f', str(dname)], quiet=True) - print('rm -f /sys/kernel/config/nvmet/subsystems/*/allowed_hosts/*') - for dname in pathlib.Path('/sys/kernel/config/nvmet/subsystems').glob('*/allowed_hosts/*'): - _runcmd(['rm', '-f', str(dname)], quiet=True) - - print(f'###{Fore.GREEN} 2nd) Remove directories{Style.RESET_ALL}') print('rmdir /sys/kernel/config/nvmet/ports/*') for dname in pathlib.Path('/sys/kernel/config/nvmet/ports').glob('*'): _runcmd(['rmdir', str(dname)], quiet=True) @@ -284,11 +251,6 @@ def clean(args): for dname in pathlib.Path('/sys/kernel/config/nvmet/subsystems').glob('*'): _runcmd(['rmdir', str(dname)], quiet=True) - print('rmdir /sys/kernel/config/nvmet/hosts/*') - for dname in pathlib.Path('/sys/kernel/config/nvmet/hosts').glob('*'): - _runcmd(['rmdir', str(dname)], quiet=True) - - print(f'###{Fore.GREEN} 3rd) Unload kernel modules{Style.RESET_ALL}') for module in _get_loaded_nvmet_modules(): _modprobe(module, ['--remove'])