Cannot customize logging behaviour in Hydra

[YerePhy]

My purpose is to customize logging according to this example.

I have the following folder structure for configs:

configs/
├── dataset_structurizers
│   ├── config.yaml
│   └── datasets
│       └── brats_2021.yaml
└── hydra
    └── job_logging
        └── custom.yaml

configs/datasets_structurizers/config.yaml has the structure:

defaults:
  - override hydra/job_logging: custom
  - datasets:
    - brats_2021

And hydra/job_logging/custom.yaml:

version: 1
formatters:
  simple:
    format: '[%(levelname)s] : [%(filename)s] : [%(funcName)s] : %(message)s'
handlers:
  console:
    class: logging.StreamHandler
    formatter: simple
    level: DEBUG
    stream: ext://sys.stdout
loggers:
  dataset_structurizers:
    level: DEBUG
    handlers: [console]
    propagate: no
root:
  level: DEBUG
  handlers: [console]
  propagate: no

If I comment - override hydra/job_logging: custom the code works fine. It also works if I move everything in configs/dataset_structurizers/ to configs. However, with my current folder structure if I run the code I get the following error:

Traceback (most recent call last):
  File "/home/yere/anaconda3/envs/brain_tumor/lib/python3.8/site-packages/hydra/_internal/utils.py", line 211, in run_and_report
    return func()
  File "/home/yere/anaconda3/envs/brain_tumor/lib/python3.8/site-packages/hydra/_internal/utils.py", line 378, in <lambda>
    lambda: hydra.run(
  File "/home/yere/anaconda3/envs/brain_tumor/lib/python3.8/site-packages/hydra/_internal/hydra.py", line 88, in run
    cfg = self.compose_config(
  File "/home/yere/anaconda3/envs/brain_tumor/lib/python3.8/site-packages/hydra/_internal/hydra.py", line 559, in compose_config
    cfg = self.config_loader.load_configuration(
  File "/home/yere/anaconda3/envs/brain_tumor/lib/python3.8/site-packages/hydra/_internal/config_loader_impl.py", line 141, in load_configuration
    return self._load_configuration_impl(
  File "/home/yere/anaconda3/envs/brain_tumor/lib/python3.8/site-packages/hydra/_internal/config_loader_impl.py", line 239, in _load_configuration_impl
    defaults_list = create_defaults_list(
  File "/home/yere/anaconda3/envs/brain_tumor/lib/python3.8/site-packages/hydra/_internal/defaults_list.py", line 748, in create_defaults_list
    defaults, tree = _create_defaults_list(
  File "/home/yere/anaconda3/envs/brain_tumor/lib/python3.8/site-packages/hydra/_internal/defaults_list.py", line 718, in _create_defaults_list
    defaults_tree = _create_defaults_tree(
  File "/home/yere/anaconda3/envs/brain_tumor/lib/python3.8/site-packages/hydra/_internal/defaults_list.py", line 356, in _create_defaults_tree
    ret = _create_defaults_tree_impl(
  File "/home/yere/anaconda3/envs/brain_tumor/lib/python3.8/site-packages/hydra/_internal/defaults_list.py", line 455, in _create_defaults_tree_impl
    return _expand_virtual_root(repo, root, overrides, skip_missing)
  File "/home/yere/anaconda3/envs/brain_tumor/lib/python3.8/site-packages/hydra/_internal/defaults_list.py", line 280, in _expand_virtual_root
    subtree = _create_defaults_tree_impl(
  File "/home/yere/anaconda3/envs/brain_tumor/lib/python3.8/site-packages/hydra/_internal/defaults_list.py", line 512, in _create_defaults_tree_impl
    _update_overrides(defaults_list, overrides, parent, interpolated_subtree)
  File "/home/yere/anaconda3/envs/brain_tumor/lib/python3.8/site-packages/hydra/_internal/defaults_list.py", line 396, in _update_overrides
    In {pcp}: Override '{okey} : {oval}' is defined before '{d.get_override_key()}: {d.get_name()}'.
  File "/home/yere/anaconda3/envs/brain_tumor/lib/python3.8/site-packages/hydra/core/default_element.py", line 495, in get_name
    assert self.value is None or isinstance(self.value, str)
AssertionError

What is the proper way to access hydra/job_logging/custom.yaml from configs/datasets_structurizers/config.yaml?

[jieru-hu]

try - override /hydra/job_logging: custom ?
This looks like absolute vs relative override issue...

[YerePhy]

Thanks for your answer but I finally moved to dataclasses and everyting is working fine. My code now looks like (I leave it as an example):

@dataclass
class CustomLogging:
    version: int = 1
    formatters: Dict[str, Any] = field(default_factory=lambda: {
        'simple': {
            'format': '[%(levelname)s] - [%(filename)s] - [%(funcName)s] - %(message)s'
        }
    })
    handlers: Dict[str, Any] = field(default_factory=lambda: {
        'console': {
            'class': 'logging.StreamHandler', 
            'formatter': 'simple',
            'stream': 'ext://sys.stdout',
            'level': 'DEBUG'
        }
    })
    root: Any = field(default_factory=lambda: {
        'level': 'DEBUG', 
        'handlers': ['console']
    })
    disable_existing_loggers: bool = False


@dataclass
class ImageFileConfig:
    regex: str 
    dest_file: str


@dataclass
class HydraGetMethodConfig:
    _target_: str
    path: str


@dataclass
class MultiModalFiles: 
    T1: Optional[ImageFileConfig] 
    FLAIR: Optional[ImageFileConfig]
    T1CE: Optional[ImageFileConfig]
    T2: Optional[ImageFileConfig]
    SEG: Optional[ImageFileConfig]


@dataclass
class BraTS2021Files(MultiModalFiles):
    T1: ImageFileConfig = ImageFileConfig('.+_t1.nii.gz', 'T1.nii.gz')
    FLAIR: ImageFileConfig = ImageFileConfig('.+_flair.nii.gz', 'FLAIR.nii.gz')
    T1CE: ImageFileConfig = ImageFileConfig('.+_t1ce.nii.gz', 'T1CE.nii.gz')
    T2: ImageFileConfig = ImageFileConfig('.+_t2.nii.gz', 'T2.nii.gz')
    SEG: ImageFileConfig = ImageFileConfig('.+_seg.nii.gz','SEG.nii.gz') 


@dataclass
class DatasetStructurizerConfig:
    dataset_path: str 
    dest_path: str 
    file_ext: str
    overwrite: bool 
    file_formats: BraTS2021Files 
    metadata_fn: HydraGetMethodConfig 


@dataclass
class BraTS2021StructurizerConfig(DatasetStructurizerConfig):
    dataset_path: str = '/media/yere/TOSHIBA EXT/data/folders/brain_tumor/RSNA_ASNR_MICCAI_BraTS2021_TrainingData'
    dest_path: str = '/home/yere/Repositories/brain_tumor/datasets/dataset_0'
    file_ext: str = '.nii.gz'
    overwrite: bool = True
    file_formats: BraTS2021Files = BraTS2021Files()
    metadata_fn: HydraGetMethodConfig = HydraGetMethodConfig(
        'hydra.utils.get_method', 
        'src.data.metadata_fns.metadata_fn_brats2021'
    )


@dataclass
class StructurizersConfig:
    defaults: List[Any] = field(default_factory=lambda: [{'override hydra/job_logging': 'custom'}])
    datasets: Dict[str, BraTS2021StructurizerConfig] = field(
        default_factory=lambda: {
        'brats2021': BraTS2021StructurizerConfig()
    })


def register_configs() -> None:
    '''
    Register configuration to Hydra ConfigStore.
    '''

    cs = ConfigStore.instance()
    cs.store(name='custom', group='hydra/job_logging', node=CustomLogging)
    cs.store(name="config", node=StructurizersConfig)

Maybe it's a little bit more complicated than .yaml files but it has a well defined structured (reviews and comments are welcome since I am a begginer in software desing).

[YerePhy]

If you are someone who has facing the same problem with the .yaml approach, it can be finally solved if you add the search path where /hydra/job_logging is located, in my case:

hydra:
  searchpath:
    - file:///home/yere/Repositories/brain_tumor/configs

This solves the problem for me, thanks @jieru-hu for your support!

[alihasan757t-creator]

@DataClass
class CustomLogging:
version: int = 1
formatters: Dict[str, Any] = field(default_factory=lambda: {
'simple': {
'format': '[%(levelname)s] - [%(filename)s] - [%(funcName)s] - %(message)s'
}
})
handlers: Dict[str, Any] = field(default_factory=lambda: {
'console': {
'class': 'logging.StreamHandler',
'formatter': 'simple',
'stream': 'ext://sys.stdout',
'level': 'DEBUG'
}
})
root: Any = field(default_factory=lambda: {
'level': 'DEBUG',
'handlers': ['console']
})
disable_existing_loggers: bool = False

@DataClass
class ImageFileConfig:
regex: str
dest_file: str

@DataClass
class HydraGetMethodConfig:
target: str
path: str

@DataClass
class MultiModalFiles:
T1: Optional[ImageFileConfig]
FLAIR: Optional[ImageFileConfig]
T1CE: Optional[ImageFileConfig]
T2: Optional[ImageFileConfig]
SEG: Optional[ImageFileConfig]

@DataClass
class BraTS2021Files(MultiModalFiles):
T1: ImageFileConfig = ImageFileConfig('.+_t1.nii.gz', 'T1.nii.gz')
FLAIR: ImageFileConfig = ImageFileConfig('.+_flair.nii.gz', 'FLAIR.nii.gz')
T1CE: ImageFileConfig = ImageFileConfig('.+_t1ce.nii.gz', 'T1CE.nii.gz')
T2: ImageFileConfig = ImageFileConfig('.+_t2.nii.gz', 'T2.nii.gz')
SEG: ImageFileConfig = ImageFileConfig('.+_seg.nii.gz','SEG.nii.gz')

@DataClass
class DatasetStructurizerConfig:
dataset_path: str
dest_path: str
file_ext: str
overwrite: bool
file_formats: BraTS2021Files
metadata_fn: HydraGetMethodConfig

@DataClass
class BraTS2021StructurizerConfig(DatasetStructurizerConfig):
dataset_path: str = '/media/yere/TOSHIBA EXT/data/folders/brain_tumor/RSNA_ASNR_MICCAI_BraTS2021_TrainingData'
dest_path: str = '/home/yere/Repositories/brain_tumor/datasets/dataset_0'
file_ext: str = '.nii.gz'
overwrite: bool = True
file_formats: BraTS2021Files = BraTS2021Files()
metadata_fn: HydraGetMethodConfig = HydraGetMethodConfig(
'hydra.utils.get_method',
'src.data.metadata_fns.metadata_fn_brats2021'
)

@DataClass
class StructurizersConfig:
defaults: List[Any] = field(default_factory=lambda: [{'override hydra/job_logging': 'custom'}])
datasets: Dict[str, BraTS2021StructurizerConfig] = field(
default_factory=lambda: {
'brats2021': BraTS2021StructurizerConfig()
})

def register_configs() -> None:
'''
Register configuration to Hydra ConfigStore.
'''

cs = ConfigStore.instance()
cs.store(name='custom', group='hydra/job_logging', node=CustomLogging)
cs.store(name="config", node=StructurizersConfig)