diff --git a/.github/workflows/manual-pages-deploy.yml b/.github/workflows/manual-pages-deploy.yml index 1cdf6ca..5cd3b98 100644 --- a/.github/workflows/manual-pages-deploy.yml +++ b/.github/workflows/manual-pages-deploy.yml @@ -18,10 +18,10 @@ jobs: - name: Update docs configuration with version (tag) name run: | - sed -i "s/PACKAGE_VERSION/${{ github.ref_name }}/" setup.py - sed -i "s/PACKAGE_VERSION/${{ github.ref_name }}/" CHAP/__init__.py - sed -i "s/PACKAGE_VERSION/${{ github.ref_name }}/" docs/conf.py - sed -i "s/PACKAGE_VERSION/${{ github.ref_name }}/" docs/installation.md + sed -i "s/PACKAGE_VERSION/0.0.1/" setup.py + sed -i "s/PACKAGE_VERSION/0.0.1/" CHAP/__init__.py + sed -i "s/PACKAGE_VERSION/0.0.1/" docs/conf.py + sed -i "s/PACKAGE_VERSION/0.0.1/" docs/installation.md - name: Setup Miniconda uses: conda-incubator/setup-miniconda@v3 diff --git a/CHAP/__init__.py b/CHAP/__init__.py index 5873478..b6cb0b2 100755 --- a/CHAP/__init__.py +++ b/CHAP/__init__.py @@ -41,9 +41,4 @@ Module defining the base `Writer` class to derive all others from. """ -#from CHAP.models import CHAPBaseModel -#from CHAP.reader import Reader -#from CHAP.processor import Processor -#from CHAP.writer import Writer - version = 'PACKAGE_VERSION' diff --git a/CHAP/common/__init__.py b/CHAP/common/__init__.py index 17ac704..c48d504 100755 --- a/CHAP/common/__init__.py +++ b/CHAP/common/__init__.py @@ -27,75 +27,3 @@ Module for generic Writers used in multiple experiment-specific workflows. """ -#from CHAP.common.processor import ( -## AnimationProcessor, -# AsyncProcessor, -# BinarizeProcessor, -# ConvertStructuredProcessor, -# ExpressionProcessor, -# ImageProcessor, -# MapProcessor, -# MPICollectProcessor, -# MPIMapProcessor, -# MPISpawnMapProcessor, -# NexusToNumpyProcessor, -## NexusToTiffsprocessor, -# NexusToXarrayProcessor, -# NexusToZarrProcessor, -# NormalizeNexusProcessor, -# NormalizeMapProcessor, -# NumpyStackProcessor, -# NumpySumProcessor, -# NumpyToNXfieldProcessor, -# PandasToXarrayProcessor, -# PrintProcessor, -# PyfaiAzimuthalIntegrationProcessor, -# RawDetectorDataMapProcessor, -# SetupNXdataProcessor, -# UpdateNXvalueProcessor, -# UpdateNXdataProcessor, -# UnstructuredToStructuredProcessor, -# NXdataToDataPointsProcessor, -# XarrayToNexusProcessor, -# XarrayToNumpyProcessor, -# ZarrToNexusProcessor, -#) -#from CHAP.common.reader import ( -# BinaryFileReader, -# ConfigReader, -# DetectorDataReader, -# FabioImageReader, -# H5Reader, -# LinkamReader, -# MapReader, -# PandasReader, -# NexusReader, -# NXdataReader, -# NXfieldReader, -# SpecReader, -# URLReader, -# YAMLReader, -# ZarrReader, -#) -#from CHAP.common.writer import ( -# ExtractArchiveWriter, -# FileTreeWriter, -# H5Writer, -# ImageWriter, -# MatplotlibAnimationWriter, -# MatplotlibFigureWriter, -# NexusWriter, -# NexusValuesWriter, -# PyfaiResultsWriter, -# YAMLWriter, -# TXTWriter, -# ZarrValuesWriter, -# ZarrWriter, -#) -#from CHAP.common.map_utils import ( -# MapSliceProcessor, -# SpecScanToMapConfigProcessor, -#) -#from CHAP.common.nexus_utils import ( -# NexusMakeLinkProcessor, -#) diff --git a/CHAP/common/map_utils.py b/CHAP/common/map_utils.py index 39f4ff8..1e2c870 100755 --- a/CHAP/common/map_utils.py +++ b/CHAP/common/map_utils.py @@ -11,11 +11,11 @@ ) # Local modules -from CHAP import Processor from CHAP.common.models.map import ( Detector, MapConfig, ) +from CHAP.processor import Processor def get_axes(nxdata, skip_axes=None): """Get the axes of a NeXus style diff --git a/CHAP/common/models/__init__.py b/CHAP/common/models/__init__.py index 043c5ea..c0dd629 100755 --- a/CHAP/common/models/__init__.py +++ b/CHAP/common/models/__init__.py @@ -12,8 +12,13 @@ Map related Pydantic model configuration classes. """ -from CHAP.common.models.common import ( - BinarizeConfig, - ImageProcessorConfig, - UnstructuredToStructuredConfig, -) +# System modules +import typing + +# Local modules +from CHAP.common.models.map import DetectorConfig + +# Avoid Pydantic "Class not fully defined" in sphinx autodoc as a +# result of lazy importing by using DetectorConfig within a default +# value of a pydantic instance variable +DetectorConfig.model_rebuild(_types_namespace=vars(typing)) diff --git a/CHAP/common/models/integration.py b/CHAP/common/models/integration.py index 3dc2345..fd88bb4 100755 --- a/CHAP/common/models/integration.py +++ b/CHAP/common/models/integration.py @@ -26,8 +26,8 @@ from pyFAI.integrator.azimuthal import AzimuthalIntegrator # Local modules -from CHAP import CHAPBaseModel from CHAP.common.models.map import Detector +from CHAP.models import CHAPBaseModel class AzimuthalIntegratorConfig(Detector, CHAPBaseModel): @@ -196,7 +196,7 @@ def validate_ais(cls, ais): return ais -class IntegrateConfig(CHAPBaseModel): +class _IntegrateConfig(CHAPBaseModel): """Class with the input parameters to perform various integrations with `pyFAI `__. @@ -219,7 +219,7 @@ class IntegrateConfig(CHAPBaseModel): attrs: Optional[dict] = {} -class Integrate1dConfig(IntegrateConfig): +class Integrate1dConfig(_IntegrateConfig): """Class with the input parameters to perform 1D azimuthal integration with `pyFAI `__. @@ -242,7 +242,7 @@ class Integrate1dConfig(IntegrateConfig): npt: Optional[conint(gt=0)] = 1800 -class Integrate2dConfig(IntegrateConfig): +class Integrate2dConfig(_IntegrateConfig): """Class with the input parameters to perform 2D azimuthal (cake) integration with `pyFAI `__. @@ -264,7 +264,7 @@ class Integrate2dConfig(IntegrateConfig): npt_rad: Optional[conint(gt=0)] = 1800 -class IntegrateRadialConfig(IntegrateConfig, MultiGeometryConfig): +class IntegrateRadialConfig(_IntegrateConfig, MultiGeometryConfig): """Class with the input parameters to perform radial integration with `pyFAI `__. diff --git a/CHAP/common/nexus_utils.py b/CHAP/common/nexus_utils.py index 4789c02..71d501b 100755 --- a/CHAP/common/nexus_utils.py +++ b/CHAP/common/nexus_utils.py @@ -5,7 +5,7 @@ """ # Local modules -from CHAP import Processor +from CHAP.processor import Processor class NexusMakeLinkProcessor(Processor): diff --git a/CHAP/common/processor.py b/CHAP/common/processor.py index 1f835a9..0d198c1 100755 --- a/CHAP/common/processor.py +++ b/CHAP/common/processor.py @@ -20,13 +20,13 @@ ) # Local modules -from CHAP import Processor from CHAP.common.models.common import ImageProcessorConfig from CHAP.common.models.map import ( DetectorConfig, MapConfig, ) from CHAP.pipeline import PipelineData +from CHAP.processor import Processor class AsyncProcessor(Processor): diff --git a/CHAP/common/reader.py b/CHAP/common/reader.py index b552f87..398fe18 100755 --- a/CHAP/common/reader.py +++ b/CHAP/common/reader.py @@ -22,14 +22,14 @@ ) # Local modules -from CHAP.reader import ( - Reader, - validate_reader_model, -) from CHAP.common.models.map import ( DetectorConfig, SpecConfig, ) +from CHAP.reader import ( + Reader, + validate_reader_model, +) def validate_model(model): """Validate the `model` configuration. diff --git a/CHAP/edd/__init__.py b/CHAP/edd/__init__.py index d57397d..2ae96ce 100755 --- a/CHAP/edd/__init__.py +++ b/CHAP/edd/__init__.py @@ -29,24 +29,19 @@ Writers unique to the EDD workflow. """ -#from CHAP.edd.processor import ( -# DiffractionVolumeLengthProcessor, -# LatticeParameterRefinementProcessor, -# HKLProcessor, -# MCAEnergyCalibrationProcessor, -# MCATthCalibrationProcessor, -# ReducedDataProcessor, -# StrainAnalysisProcessor, -#) -#from CHAP.edd.reader import ( -# EddMapReader, -# EddMPIMapReader, -# ScanToMapReader, -# SetupNXdataReader, -# UpdateNXdataReader, -# NXdataSliceReader, -# SliceNXdataReader, -#) -#from CHAP.edd.writer import ( -# StrainAnalysisUpdateWriter, -#) +# System modules +import typing + +# Local modules +from CHAP.edd.models import ( + DiffractionVolumeLengthConfig, + MCACalibrationConfig, + StrainAnalysisConfig, +) + +# Avoid Pydantic "Class not fully defined" in sphinx autodoc as a +# result of lazy importing by using any of these within a default +# value of a pydantic instance variable +DiffractionVolumeLengthConfig.model_rebuild(_types_namespace=vars(typing)) +MCACalibrationConfig.model_rebuild(_types_namespace=vars(typing)) +StrainAnalysisConfig.model_rebuild(_types_namespace=vars(typing)) diff --git a/CHAP/edd/models.py b/CHAP/edd/models.py index 68cd19e..d224a69 100755 --- a/CHAP/edd/models.py +++ b/CHAP/edd/models.py @@ -5,6 +5,7 @@ # System modules from copy import deepcopy import os +import typing from typing import ( Literal, Optional, @@ -63,7 +64,7 @@ class BaselineConfig(CHAPBaseModel): # Fit configuration class -class FitConfig(CHAPBaseModel): +class _FitConfig(CHAPBaseModel): """Fit parameters configuration class for peak fitting. :ivar background: Background model for peak fitting, defaults @@ -233,7 +234,12 @@ def validate_materialconfig_after(self): # Detector configuration classes -class MCADetectorCalibration(Detector, FitConfig): +# Avoid Pydantic "Class not fully defined" in sphinx autodoc as a +# result of lazy importing by using in an _exclude pydantic instance +# variable +_FitConfig.model_rebuild(_types_namespace=vars(typing)) + +class MCADetectorCalibration(Detector, _FitConfig): """Class representing the configuration for a single MCA detector element to perform detector calibration. @@ -541,7 +547,7 @@ def get_tth_map(self, map_shape): ] -class MCADetectorConfig(FitConfig): +class MCADetectorConfig(_FitConfig): """Class representing metadata required to configure a full MCA detector. @@ -553,7 +559,7 @@ class MCADetectorConfig(FitConfig): 'calibration', 'diffractionvolumelength', 'strainanalysis'] detectors: Optional[conlist(min_length=1, item_type=MCADetector)] = [] - _exclude = set(vars(FitConfig()).keys()) + _exclude = set(vars(_FitConfig()).keys()) @model_validator(mode='before') @classmethod @@ -605,7 +611,7 @@ def update_detectors(self): # Processor configuration classes -class DiffractionVolumeLengthConfig(FitConfig): +class DiffractionVolumeLengthConfig(_FitConfig): """Configuration for the differential volume length processor :class:`~CHAP.edd.processor.DiffractionVolumeLengthProcessor` for an EDD setup using a steel-foil raster scan. @@ -633,7 +639,7 @@ class DiffractionVolumeLengthConfig(FitConfig): sample_thickness: Optional[confloat(gt=0, allow_inf_nan=False)] = None sigma_to_dvl_factor: Optional[Literal[2.0, 3.5, 4.0]] = 3.5 - _exclude = set(vars(FitConfig()).keys()) + _exclude = set(vars(_FitConfig()).keys()) @model_validator(mode='after') def validate_diffractionvolumelengthconfig_after(self): @@ -648,6 +654,11 @@ def validate_diffractionvolumelengthconfig_after(self): return self +# Avoid Pydantic "Class not fully defined" in sphinx autodoc as a +# result of lazy importing by using MaterialConfig within a default +# value of a pydantic instance variable +MaterialConfig.model_rebuild(_types_namespace=vars(typing)) + class MCACalibrationConfig(CHAPBaseModel): """Base class configuration for energy and 2&theta calibration processors. diff --git a/CHAP/edd/processor.py b/CHAP/edd/processor.py index 332b068..2309ff7 100755 --- a/CHAP/edd/processor.py +++ b/CHAP/edd/processor.py @@ -24,12 +24,12 @@ # Local modules from CHAP.common.models.map import DetectorConfig from CHAP.edd.models import ( + DiffractionVolumeLengthConfig, + MCACalibrationConfig, MCADetectorCalibration, + MCADetectorConfig, MCADetectorDiffractionVolumeLength, MCADetectorStrainAnalysis, - MCADetectorConfig, - DiffractionVolumeLengthConfig, - MCACalibrationConfig, MCAEnergyCalibrationConfig, MCATthCalibrationConfig, StrainAnalysisConfig, @@ -44,7 +44,7 @@ # 0, 2, 3, 5, 6, 7, 8, 10, 13, 14, 16, 17, 18, 19, 21, 22 -class BaseEddProcessor(Processor): +class _BaseEddProcessor(Processor): """Base processor for the EDD processors. :ivar save_figures: Save .pngs of plots for checking inputs and @@ -313,7 +313,7 @@ def _subtract_baselines(self): mean_data -= baseline -class BaseStrainProcessor(BaseEddProcessor): +class _BaseStrainProcessor(_BaseEddProcessor): """Base processor for :class:`~CHAP.edd.processor.LatticeParameterRefinementProcessor` and :class:`~CHAP.edd.processor.StrainAnalysisProcessor`. @@ -472,7 +472,7 @@ def _setup_detector_data(self, nxobject, **kwargs): f'mean_data shape: {np.asarray(self._mean_data).shape}') -class DiffractionVolumeLengthProcessor(BaseEddProcessor): +class DiffractionVolumeLengthProcessor(_BaseEddProcessor): """A Processor using a steel foil raster scan to calculate the diffraction volume length for an EDD setup. @@ -807,7 +807,7 @@ def _measure_dvl(self, scanned_vals): schema='common.write.ImageWriter') -class HKLProcessor(BaseStrainProcessor): +class HKLProcessor(_BaseStrainProcessor): """Processor that plots the HKLs for a given material against the calibrated energy channels. @@ -892,7 +892,7 @@ def process(self, data): -class LatticeParameterRefinementProcessor(BaseStrainProcessor): +class LatticeParameterRefinementProcessor(_BaseStrainProcessor): """Processor to get a refined estimate for a sample's lattice parameters. @@ -1154,7 +1154,7 @@ def _refine_lattice_parameters(self): # a_uniform, a_uniform, a_uniform, 90., 90., 90.] -class MCAEnergyCalibrationProcessor(BaseEddProcessor): +class MCAEnergyCalibrationProcessor(_BaseEddProcessor): """Processor to return parameters for linearly transforming MCA channel indices to energies (in keV). Procedure: provide a spectrum from the MCA element to be calibrated and the theoretical @@ -1712,7 +1712,7 @@ def select_peaks(): return buf, peak_indices -class MCATthCalibrationProcessor(BaseEddProcessor): +class MCATthCalibrationProcessor(_BaseEddProcessor): """Processor to calibrate the 2&theta angle and fine tune the energy calibration coefficients for an EDD experimental setup. @@ -2351,7 +2351,7 @@ def _select_tth_init(self): f'{detector.tth_initial_guess}') -class ReducedDataProcessor(BaseStrainProcessor): +class ReducedDataProcessor(_BaseStrainProcessor): """Processor that takes a map of MCA data and returns a map of reduced data. """ @@ -2518,7 +2518,7 @@ def _linkdims( nxgroup.attrs['unstructured_axes'] = unstructured_axes -class StrainAnalysisProcessor(BaseStrainProcessor): +class StrainAnalysisProcessor(_BaseStrainProcessor): """Processor that takes a map of MCA data and returns a map of sample strains. diff --git a/CHAP/edd/reader.py b/CHAP/edd/reader.py index 038a9f7..fe59755 100755 --- a/CHAP/edd/reader.py +++ b/CHAP/edd/reader.py @@ -19,8 +19,8 @@ ) # Local modules -from CHAP.reader import Reader from CHAP.common.models.map import DetectorConfig +from CHAP.reader import Reader class EddMapReader(Reader): diff --git a/CHAP/edd/writer.py b/CHAP/edd/writer.py index 3e73f4a..6730c00 100755 --- a/CHAP/edd/writer.py +++ b/CHAP/edd/writer.py @@ -3,7 +3,7 @@ """Module for Writers unique to the EDD workflow.""" # Local modules -from CHAP import Writer +from CHAP.writer import Writer class StrainAnalysisUpdateWriter(Writer): """Writer to add or update the strain analysis for a set of map diff --git a/CHAP/foxden/__init__.py b/CHAP/foxden/__init__.py index a17a066..1af2f02 100755 --- a/CHAP/foxden/__init__.py +++ b/CHAP/foxden/__init__.py @@ -25,19 +25,13 @@ Writers unique to the FOXDEN pipeline items. """ -#from CHAP.foxden.processor import ( -## FoxdenMetadataProcessor, -## FoxdenProvenanceProcessor, -# ProvenanceFileProcessor, -#) -#from CHAP.foxden.reader import ( -# FoxdenDataDiscoveryReader, -# FoxdenMetadataReader, -# FoxdenProvenanceReader, -# FoxdenSpecScansReader, -#) -#from CHAP.foxden.writer import ( -# FoxdenDoiWriter, -# FoxdenMetadataWriter, -# FoxdenProvenanceWriter, -#) +# System modules +import typing + +# Local modules +from CHAP.foxden.models import FoxdenRequestConfig + +# Avoid Pydantic "Class not fully defined" in sphinx autodoc as a +# result of lazy importing by using FoxdenRequestConfig within a +# default value of a pydantic instance variable +FoxdenRequestConfig.model_rebuild(_types_namespace=vars(typing)) diff --git a/CHAP/foxden/models.py b/CHAP/foxden/models.py index ac99ae3..0af1937 100755 --- a/CHAP/foxden/models.py +++ b/CHAP/foxden/models.py @@ -17,7 +17,7 @@ ) # Local modules -from CHAP import CHAPBaseModel +from CHAP.models import CHAPBaseModel class FoxdenRequestConfig(CHAPBaseModel): diff --git a/CHAP/giwaxs/__init__.py b/CHAP/giwaxs/__init__.py index 7f524f3..ad38184 100755 --- a/CHAP/giwaxs/__init__.py +++ b/CHAP/giwaxs/__init__.py @@ -24,10 +24,3 @@ Writers unique to the GIWAXS workflow. """ -##from CHAP.giwaxs.models import PyfaiIntegrationConfig -#from CHAP.giwaxs.processor import ( -# GiwaxsConversionProcessor, -# PyfaiIntegrationProcessor, -#) -## from CHAP.giwaxs.reader import -## from CHAP.giwaxs.writer import diff --git a/CHAP/giwaxs/models.py b/CHAP/giwaxs/models.py index fcef8df..4f5d11c 100755 --- a/CHAP/giwaxs/models.py +++ b/CHAP/giwaxs/models.py @@ -26,8 +26,8 @@ from pyFAI.integrator.fiber import FiberIntegrator # Local modules -from CHAP import CHAPBaseModel from CHAP.common.models.map import Detector +from CHAP.models import CHAPBaseModel @model_validator(mode='before') diff --git a/CHAP/hdrm/__init__.py b/CHAP/hdrm/__init__.py index b267c53..9e43a16 100755 --- a/CHAP/hdrm/__init__.py +++ b/CHAP/hdrm/__init__.py @@ -2,9 +2,3 @@ processing workflows. """ -#from CHAP.hdrm.processor import ( -# HdrmOrmfinderProcessor, -# HdrmPeakfinderProcessor, -#) -## from CHAP.hdrm.reader import -## from CHAP.hdrm.writer import diff --git a/CHAP/hdrm/models.py b/CHAP/hdrm/models.py index 9fdacf4..c92727b 100755 --- a/CHAP/hdrm/models.py +++ b/CHAP/hdrm/models.py @@ -10,9 +10,9 @@ ) # Local modules -from CHAP import CHAPBaseModel from CHAP.edd.models import MaterialConfig from CHAP.giwaxs.models import AzimuthalIntegratorConfig +from CHAP.models import CHAPBaseModel class HdrmOrmfinderConfig(CHAPBaseModel): diff --git a/CHAP/inference/__init__.py b/CHAP/inference/__init__.py index 92b9cf3..c21cf56 100755 --- a/CHAP/inference/__init__.py +++ b/CHAP/inference/__init__.py @@ -2,6 +2,3 @@ [TFaaS inference server](https://github.com/vkuznet/TFaaS/). """ -#from CHAP.inference.processor import TFaaSImageProcessor -## from CHAP.inference.reader import -## from CHAP.inference.writer import diff --git a/CHAP/inference/processor.py b/CHAP/inference/processor.py index e3aaff7..84360e3 100755 --- a/CHAP/inference/processor.py +++ b/CHAP/inference/processor.py @@ -10,7 +10,7 @@ from time import time # Local modules -from CHAP import Processor +from CHAP.processor import Processor class TFaaSImageProcessor(Processor): diff --git a/CHAP/pipeline.py b/CHAP/pipeline.py index ba3ef87..ce854e1 100755 --- a/CHAP/pipeline.py +++ b/CHAP/pipeline.py @@ -15,7 +15,6 @@ # Third party modules from pydantic import ( -# ConfigDict, Field, # FilePath, PrivateAttr, @@ -68,8 +67,6 @@ class PipelineItem(RunConfig): _status: Literal[ 'read', 'write_pending', 'written'] = PrivateAttr(default=None) - #FIX model_config = ConfigDict(arbitrary_types_allowed=True) - @model_validator(mode='after') def validate_pipelineitem_after(self): """Validate the `PipelineItem` configuration. @@ -496,8 +493,6 @@ class Pipeline(CHAPBaseModel): # default={'application': 'CHAP', 'user_metadata': {}}) # _provenance: dict = PrivateAttr(default={}) - #FIX model_config = ConfigDict(arbitrary_types_allowed=True) - @model_validator(mode='after') def validate_pipeline_after(self): """Validate the `Pipeline` configuration and initialize and diff --git a/CHAP/reader.py b/CHAP/reader.py index c0946da..488a9c4 100755 --- a/CHAP/reader.py +++ b/CHAP/reader.py @@ -81,7 +81,7 @@ def read(self): return data -class OptionParser(): +class _OptionParser(): """User based option parser.""" def __init__(self): @@ -97,11 +97,11 @@ def __init__(self): dest='log_level', default='INFO', help='logging level') -def main(opt_parser=OptionParser): +def main(opt_parser=_OptionParser): """Main function. :param opt_parser: User based option parser. - :type opt_parser: OptionParser + :type opt_parser: CHAP.reader._OptionParser """ optmgr = opt_parser() opts = optmgr.parser.parse_args() diff --git a/CHAP/runner.py b/CHAP/runner.py index b143849..9e191d1 100755 --- a/CHAP/runner.py +++ b/CHAP/runner.py @@ -308,6 +308,9 @@ def run( config['outputdir'] = outputdir else: name = item + split_name = name.split('.') + cls_name = split_name[-1] + mod_name = '.'.join(split_name[:-1]) # Initialize the object's identifiers if 'users' in name: @@ -323,15 +326,28 @@ def run( if logger is not None: logger.error(f'Unable to load {name}') continue - cls_name = name.split('.')[-1] - mod_name = '.'.join(name.split('.')[:-1]) - module = __import__(mod_name, fromlist=[cls_name]) + module_name = __import__(mod_name, fromlist=[cls_name]) else: - cls_name = name.split('.')[-1] - mod_name = '.'.join(name.split('.')[:-1]) - module = __import__(f'CHAP.{mod_name}', fromlist=[cls_name]) + module_name = __import__(f'CHAP.{mod_name}', fromlist=[cls_name]) - pipeline_mmcs.append(getattr(module, cls_name)) + try: + module = getattr(module_name, cls_name) + except: + if ((cls_name.endswith('Processor') + or cls_name == 'TomoCHESSMapConverter') + and split_name[-2] != 'processor'): + mod_name += '.processor' + if cls_name.endswith('Reader') and split_name[-2] != 'reader': + mod_name += '.reader' + if cls_name.endswith('Writer') and split_name[-2] != 'writer': + mod_name += '.writer' + if 'users' in name: + module_name = __import__(mod_name, fromlist=[cls_name]) + else: + module_name = __import__( + f'CHAP.{mod_name}', fromlist=[cls_name]) + module = getattr(module_name, cls_name) + pipeline_mmcs.append(module) # Initialize the object's runtime arguments item_args['comm'] = comm #FIX make comm a field in RunConfig? diff --git a/CHAP/saxswaxs/__init__.py b/CHAP/saxswaxs/__init__.py index 15cd819..6ecede4 100755 --- a/CHAP/saxswaxs/__init__.py +++ b/CHAP/saxswaxs/__init__.py @@ -22,16 +22,3 @@ Writers unique to the SAXSWAXS workflow. """ -#from CHAP.saxswaxs.processor import ( -# CfProcessor, -# FluxCorrectionProcessor, -# FluxAbsorptionCorrectionProcessor, -# FluxAbsorptionBackgroundCorrectionProcessor, -# PyfaiIntegrationProcessor, -# SetupResultsProcessor, -# SetupProcessor, -# UnstructuredToStructuredProcessor, -# UpdateValuesProcessor, -#) -## from CHAP.saxswaxs.reader import () -## from CHAP.saxswaxs.writer import () diff --git a/CHAP/saxswaxs/processor.py b/CHAP/saxswaxs/processor.py index d11e5da..f2933d8 100755 --- a/CHAP/saxswaxs/processor.py +++ b/CHAP/saxswaxs/processor.py @@ -22,13 +22,13 @@ import numpy as np # Local modules -from CHAP import Processor -from CHAP.common.processor import ExpressionProcessor from CHAP.common.models.map import ( Detector, MapConfig, ) from CHAP.common.models.integration import PyfaiIntegrationConfig +from CHAP.common.processor import ExpressionProcessor +from CHAP.processor import Processor class CfProcessor(Processor): diff --git a/CHAP/sin2psi/__init__.py b/CHAP/sin2psi/__init__.py index a5ee473..ddc596d 100755 --- a/CHAP/sin2psi/__init__.py +++ b/CHAP/sin2psi/__init__.py @@ -2,6 +2,3 @@ processing workflows. """ -# from CHAP.sin2psi.processor import -# from CHAP.sin2psi.reader import -# from CHAP.sin2psi.writer import diff --git a/CHAP/test/__init__.py b/CHAP/test/__init__.py index 5f3c337..08f9502 100644 --- a/CHAP/test/__init__.py +++ b/CHAP/test/__init__.py @@ -1,3 +1,2 @@ """This subpackage contains unit test routines.""" -#from CHAP.test.common import * diff --git a/CHAP/tomo/__init__.py b/CHAP/tomo/__init__.py index 9b3d70a..c8cb415 100755 --- a/CHAP/tomo/__init__.py +++ b/CHAP/tomo/__init__.py @@ -24,17 +24,21 @@ Writers unique to the tomography workflow. """ -#from CHAP.tomo.processor import ( -# TomoMetadataProcessor, -# TomoCHESSMapConverter, -# TomoReduceProcessor, -# TomoFindCenterProcessor, -# TomoReconstructProcessor, -# TomoCombineProcessor, -# TomoSimFieldProcessor, -# TomoDarkFieldProcessor, -# TomoBrightFieldProcessor, -# TomoSpecProcessor, -#) -## from CHAP.tomo.reader import -#from CHAP.tomo.writer import TomoWriter +# System modules +import typing + +# Local modules +from CHAP.tomo.models import ( + TomoCombineConfig, + TomoFindCenterConfig, + TomoReconstructConfig, + TomoReduceConfig, +) + +# Avoid Pydantic "Class not fully defined" in sphinx autodoc as a +# result of lazy importing by using any of these within a default +# value of a pydantic instance variable +TomoCombineConfig.model_rebuild(_types_namespace=vars(typing)) +TomoFindCenterConfig.model_rebuild(_types_namespace=vars(typing)) +TomoReconstructConfig.model_rebuild(_types_namespace=vars(typing)) +TomoReduceConfig.model_rebuild(_types_namespace=vars(typing)) diff --git a/CHAP/tomo/models.py b/CHAP/tomo/models.py index 873f6b3..3df8177 100755 --- a/CHAP/tomo/models.py +++ b/CHAP/tomo/models.py @@ -8,7 +8,6 @@ Optional, ) from pydantic import ( -# ConfigDict, conint, conlist, confloat, @@ -230,5 +229,3 @@ class TomoSimConfig(CHAPBaseModel): beam_intensity: Optional[confloat(gt=0, allow_inf_nan=False)] = 1.e9 background_intensity: Optional[confloat(gt=0, allow_inf_nan=False)] = 20 slit_size: Optional[confloat(gt=0, allow_inf_nan=False)] = 1.0 - - #FIX model_config = ConfigDict(arbitrary_types_allowed=True) diff --git a/CHAP/tomo/processor.py b/CHAP/tomo/processor.py index 024cddc..18993aa 100755 --- a/CHAP/tomo/processor.py +++ b/CHAP/tomo/processor.py @@ -26,7 +26,6 @@ from json import loads import numpy as np from pydantic import ( -# ConfigDict, Field, PrivateAttr, SkipValidation, @@ -44,8 +43,6 @@ DetectorConfig, MapConfig, ) -from CHAP.pipeline import PipelineData -from CHAP.processor import Processor from CHAP.tomo.models import ( TomoReduceConfig, TomoFindCenterConfig, @@ -53,6 +50,8 @@ TomoCombineConfig, TomoSimConfig, ) +from CHAP.pipeline import PipelineData +from CHAP.processor import Processor from CHAP.utils.general import ( #input_num, #input_yesno, @@ -1797,8 +1796,6 @@ class TomoFindCenterGui(Processor): _exclude = {'tk_root'} - #FIX model_config = ConfigDict(arbitrary_types_allowed=True) - @property def center_offsets(self): """Return the selected centers at the specified or selected diff --git a/CHAP/tomo/writer.py b/CHAP/tomo/writer.py index 4aeca3e..174b02e 100755 --- a/CHAP/tomo/writer.py +++ b/CHAP/tomo/writer.py @@ -14,7 +14,7 @@ from pydantic import model_validator # Local modules -from CHAP import Writer +from CHAP.writer import Writer class TomoWriter(Writer): diff --git a/CHAP/utils/__init__.py b/CHAP/utils/__init__.py index 1196248..b29acf9 100755 --- a/CHAP/utils/__init__.py +++ b/CHAP/utils/__init__.py @@ -27,4 +27,3 @@ as input. """ -#from CHAP.utils.fit import FitProcessor diff --git a/CHAP/writer.py b/CHAP/writer.py index 25942e6..b4b5b88 100755 --- a/CHAP/writer.py +++ b/CHAP/writer.py @@ -101,7 +101,7 @@ def write(self, data): # add a sort of modification flag later -class OptionParser(): +class _OptionParser(): """User based option parser.""" def __init__(self): @@ -120,11 +120,11 @@ def __init__(self): dest='log_level', default='INFO', help='logging level') -def main(opt_parser=OptionParser): +def main(opt_parser=_OptionParser): """Main function. :param opt_parser: User based option parser. - :type opt_parser: OptionParser + :type opt_parser: CHAP.writer._OptionParser """ optmgr = opt_parser() opts = optmgr.parser.parse_args() diff --git a/examples/tomo/pipeline_id1a3_pyramid_sim.yaml b/examples/tomo/pipeline_id1a3_pyramid_sim.yaml index 695f797..799e2a9 100644 --- a/examples/tomo/pipeline_id1a3_pyramid_sim.yaml +++ b/examples/tomo/pipeline_id1a3_pyramid_sim.yaml @@ -3,26 +3,26 @@ config: inputdir: config outputdir: raw/hollow_pyramid interactive: true - log_level: error + log_level: info profile: false pipeline: # Create a simulated stack of tomography images - - common.reader.YAMLReader: + - common.YAMLReader: filename: tomo_sim_id1a3_pyramid.yaml schema: tomo.models.TomoSimConfig - - tomo.processor.TomoSimFieldProcessor: + - tomo.TomoSimFieldProcessor: schema: tomo.models.TomoSimField # Create the dark field for the simulation - - tomo.processor.TomoDarkFieldProcessor: + - tomo.TomoDarkFieldProcessor: schema: tomo.models.TomoDarkField # Create the bright field for the simulation - - tomo.processor.TomoBrightFieldProcessor: + - tomo.TomoBrightFieldProcessor: num_image: 10 schema: tomo.models.TomoBrightField # Create the SPEC file for the simulation - tomo.processor.TomoSpecProcessor: # scan_numbers: [1, 2, 3, 4, 5] - - common.writer.FileTreeWriter: + - common.FileTreeWriter: force_overwrite: true