"""dsc_datatool
The main Python module for the command line tool `dsc-datatool`, see
`man dsc-datatool` on how to run it.
On runtime it will load all plugins under the following module path:
- dsc_datatool.input
- dsc_datatool.output
- dsc_datatool.generator
- dsc_datatool.transformer
Each plugin category should base it class on one of the follow superclasses:
- dsc_datatool.Input
- dsc_datatool.Output
- dsc_datatool.Generator
- dsc_datatool.Transformer
Doing so it will be automatically registered as available and indexed in
the following public dicts using the class name:
- inputs
- outputs
- generators
- transformers
Example of an output:
from dsc_datatool import Output
class ExampleOutput(Output):
def process(self, datasets)
...
:copyright: 2024 OARC, Inc.
"""
__version__ = '1.4.2'
import argparse
import logging
import os
import importlib
import pkgutil
import sys
import traceback
import re
args = argparse.Namespace()
inputs = {}
outputs = {}
generators = {}
transformers = {}
process_dataset = {}
encoding = 'utf-8'
class Dataset(object):
"""A representation of a DSC dataset
A DSC dataset is one to two dimensional structure where the last
dimension holds an array of values and counters.
It is based on the XML structure of DSC:
Attributes:
- name: The name of the dataset
- start_time: The start time of the dataset in seconds
- stop_time: The stop time of the dataset in seconds
- dimensions: An array with `Dimension`, the first dimension
"""
name = None
start_time = None
stop_time = None
dimensions = None
def __init__(self):
self.dimensions = []
def __repr__(self):
return '' % (self.name, self.dimensions)
class Dimension(object):
"""A representation of a DSC dimension
A DSC dataset dimension which can be the first or second dimension,
see `Dataset` for more information.
Attributes:
- name: The name of the dimension
- value: Is set to the value of the dimension if it's the first dimension
- values: A dict of values with corresponding counters if it's the second dimension
"""
name = None
value = None
values = None
dimensions = None
def __init__(self, name):
self.name = name
self.values = {}
self.dimensions = []
def __repr__(self):
return '' % (self.name, self.values or self.value, self.dimensions)
class Input(object):
"""Base class of an input plugin"""
def process(self, file):
"""Input.process(...) -> [ Dataset, ... ]
Called to process a file and return an array of `Dataset`'s found in it.
"""
raise Exception('process() not overloaded')
def __init_subclass__(cls):
"""This method is called when a class is subclassed and it will
register the input plugin in `inputs`."""
global inputs
if cls.__name__ in inputs:
raise Exception('Duplicate input module: %s already exists' % cls.__name__)
inputs[cls.__name__] = cls
class Output(object):
"""Base class of an output plugin"""
def process(self, datasets):
"""Output.process([ Dataset, ... ])
Called to output the `Dataset`'s in the given array."""
raise Exception('process() not overloaded')
def __init__(self, opts):
"""instance = Output({ 'opt': value, ... })
Called to create an instance of the output plugin, will get a dict
with options provided on command line."""
pass
def __init_subclass__(cls):
"""This method is called when a class is subclassed and it will
register the output plugin in `outputs`."""
global outputs
if cls.__name__ in outputs:
raise Exception('Duplicate output module: %s already exists' % cls.__name__)
outputs[cls.__name__] = cls
class Generator(object):
"""Base class of a generator plugin"""
def process(self, datasets):
"""Generator.process([ Dataset, ... ]) -> [ Dataset, ... ]
Called to generate additional `Dataset`'s based on the given array
of `Dataset`'s."""
raise Exception('process() not overloaded')
def __init__(self, opts):
"""instance = Generator({ 'opt': value, ... })
Called to create an instance of the generator plugin, will get a dict
with options provided on command line."""
pass
def __init_subclass__(cls):
"""This method is called when a class is subclassed and it will
register the generator plugin in `generators`."""
global generators
if cls.__name__ in generators:
raise Exception('Duplicate generator module: %s already exists' % cls.__name__)
generators[cls.__name__] = cls
class Transformer(object):
"""Base class of a transformer plugin"""
def process(self, datasets):
"""Transformer.process([ Dataset, ... ])
Called to do transformation of the given `Dataset`'s, as in modifying
them directly."""
raise Exception('process() not overloaded')
def __init__(self, opts):
"""instance = Transformer({ 'opt': value, ... })
Called to create an instance of the transformer plugin, will get a dict
with options provided on command line."""
pass
def __init_subclass__(cls):
"""This method is called when a class is subclassed and it will
register the transformer plugin in `transformers`."""
global transformers
if cls.__name__ in transformers:
raise Exception('Duplicate transformer module: %s already exists' % cls.__name__)
transformers[cls.__name__] = cls
def main():
"""Called when running `dsc-datatool`."""
def iter_namespace(ns_pkg):
return pkgutil.iter_modules(ns_pkg.__path__, ns_pkg.__name__ + ".")
def split_arg(arg, num=1):
sep = arg[0]
p = arg.split(sep)
p.pop(0)
ret = ()
while num > 0:
ret += (p.pop(0),)
num -= 1
ret += (p,)
return ret
def parse_opts(opts):
ret = {}
for opt in opts:
p = opt.split('=', maxsplit=1)
if len(p) > 1:
if p[0] in ret:
if isinstance(ret[p[0]], list):
ret[p[0]].append(p[1])
else:
ret[p[0]] = [ ret[p[0]], p[1] ]
else:
ret[p[0]] = p[1]
elif len(p) > 0:
ret[p[0]] = True
return ret
def _process(datasets, generators, transformers, outputs):
gen_datasets = []
for generator in generators:
try:
gen_datasets += generator.process(datasets)
except Exception as e:
logging.warning('Generator %s failed: %s' % (generator, e))
exc_type, exc_value, exc_traceback = sys.exc_info()
for tb in traceback.format_tb(exc_traceback):
logging.warning(str(tb))
return 2
datasets += gen_datasets
if '*' in transformers:
for transformer in transformers['*']:
try:
transformer.process(datasets)
except Exception as e:
logging.warning('Transformer %s failed: %s' % (transformer, e))
exc_type, exc_value, exc_traceback = sys.exc_info()
for tb in traceback.format_tb(exc_traceback):
logging.warning(str(tb))
return 2
for dataset in datasets:
if dataset.name in transformers:
for transformer in transformers[dataset.name]:
try:
transformer.process([dataset])
except Exception as e:
logging.warning('Transformer %s failed: %s' % (transformer, e))
exc_type, exc_value, exc_traceback = sys.exc_info()
for tb in traceback.format_tb(exc_traceback):
logging.warning(str(tb))
return 2
for output in outputs:
try:
output.process(datasets)
except Exception as e:
logging.warning('Output %s failed: %s' % (output, e))
exc_type, exc_value, exc_traceback = sys.exc_info()
for tb in traceback.format_tb(exc_traceback):
logging.warning(str(tb))
return 2
return 0
global args, inputs, outputs, generators, transformers, process_dataset
parser = argparse.ArgumentParser(prog='dsc-datatool',
description='Export DSC data into various formats and databases.',
epilog='See man-page dsc-datatool(1) and dsc-datatool-[generator|transformer|output] (5) for more information')
parser.add_argument('-c', '--conf', nargs=1,
help='Not implemented')
# help='Specify the YAML configuration file to use (default to ~/.dsc-datatool.conf), any command line option will override the options in the configuration file. See dsc-datatool.conf(5)for more information.')
parser.add_argument('-s', '--server', nargs=1,
help='Specify the server for where the data comes from. (required)')
parser.add_argument('-n', '--node', nargs=1,
help='Specify the node for where the data comes from. (required)')
parser.add_argument('-x', '--xml', action='append',
help='Read DSC data from the given file or directory, can be specified multiple times. If a directory is given then all files ending with .xml will be read.')
parser.add_argument('-d', '--dat', action='append',
help='Read DSC data from the given directory, can be specified multiple times. Note that the DAT format is depended on the filename to know what type of data it is.')
parser.add_argument('--dataset', action='append',
help='Specify that only the list of datasets will be processed, the list is comma separated and the option can be given multiple times.')
parser.add_argument('-o', '--output', action='append',
help='"