utils

---

Utils for evaluating model(s) on given data

get_reports(config)

Generate reports with given evaluation config

Parameters:

Name	Type	Description	Default
config	EvalConfig	config for evaluation	required

Returns:

Name	Type	Description
out	Dict[str, dict]	model names as keys and report dicts as values

Source code in conftrainer/evaluation/utils.py

def get_reports(config: EvalConfig) -> Dict:
    """
    Generate reports with given evaluation config

    Parameters
    ----------
    config : EvalConfig
        config for evaluation

    Returns
    -------
    out : Dict[str, dict]
        model names as keys and report dicts as values
    """
    filepaths, labels = read_preprocess_dataframe(csv_path=config.test_csv_path,
                                                  name_col=config.name_col,
                                                  data_dir=config.data_dir,
                                                  classes=config.output_classes,
                                                  clean_dataset=config.clean_dataset)

    test_datagen = ImageDatagen(filepaths=filepaths,
                                labels=labels,
                                classes=config.output_classes)

    metric_dict = get_metrics(metric_config_list=config.metrics)
    metrics = list(metric_dict.values())

    reports = evaluate_multiple_models(model_paths=config.model_paths,
                                       datagen=test_datagen,
                                       metrics=metrics,
                                       batch_size=config.batch_size,
                                       classes=config.output_classes)

    return reports

get_multibranch_reports(config)

Generate reports with given evaluation config

Parameters:

Name	Type	Description	Default
config	MultiBranchEvalConfig	config for evaluation	required

Returns:

Name	Type	Description
out	Dict[str, dict]	model names as keys and report dicts as values

Source code in conftrainer/evaluation/utils.py

def get_multibranch_reports(config: MultiBranchEvalConfig) -> Dict:
    """
    Generate reports with given evaluation config

    Parameters
    ----------
    config : MultiBranchEvalConfig
        config for evaluation

    Returns
    -------
    out : Dict[str, dict]
        model names as keys and report dicts as values
    """
    datagen_config = read_multioutput_dataframe(csv_path=config.test_csv_path,
                                                name_col=config.name_col,
                                                data_dir=config.data_dir,
                                                per_task_data=config.per_task_data,
                                                clean_dataset=config.clean_dataset,
                                                name="evaluation dataset")

    test_datagen = MultiOutputDatagen(config=datagen_config, shape=[224, 224, 3])

    metric_dict = {}
    for task_name, metric_list in config.metrics.items():
        metric_dict[task_name] = get_metrics(metric_config_list=metric_list)
    classes = {data.name: data.classes for data in config.per_task_data}
    reports = evaluate_multiple_models(model_paths=config.model_paths,
                                       datagen=test_datagen,
                                       metrics=metric_dict,
                                       batch_size=config.batch_size,
                                       classes=classes,
                                       input_shape=config.input_shape)

    return reports

evaluate_single_model(model_path, datagen, metrics, batch_size, classes, del_network=False, input_shape=None)

Evaluate a single network and return a report

Parameters:

Name	Type	Description	Default
model_path	str	path to load the network	required
datagen	ImageDatagen	data to evaluate on	required
metrics	dict	names as keys and metric objects as values	required
batch_size	int	size of each batch passed to the model	required
classes	Union[List[str], Dict[str, List[str]]]	classes names corresponding to the models. Can be a list of class names or a dictionary mapping each branch to its classes for multibranch models.	required
del_network	bool	whether delete the network after evaluating to release memory	False
input_shape	List[int]	input shape of the network. Used only if the input shape couldn't be inferred from the network itself	None

Returns:

Name	Type	Description
out	dict	report

Source code in conftrainer/evaluation/utils.py

def evaluate_single_model(model_path: str, datagen: BaseImageDatagen,
                          metrics: Union[Dict[str, Dict[str, Metric]], Dict[str, Metric], List[Metric]],
                          batch_size: int, classes: Union[List[str], Dict[str, List[str]]],
                          del_network: bool = False, input_shape: List[int] = None) -> Dict[
    str, Union[float, Dict[str, float]]]:
    """
    Evaluate a single network and return a report

    Parameters
    ----------
    model_path : str
        path to load the network
    datagen : ImageDatagen
        data to evaluate on
    metrics : dict
        names as keys and metric objects as values
    batch_size : int
        size of each batch passed to the model
    classes : Union[List[str], Dict[str, List[str]]]
        classes names corresponding to the models. Can be a list of class names or a dictionary mapping each branch to
        its classes for multibranch models.
    del_network : bool
        whether delete the network after evaluating to release memory
    input_shape : List[int]
        input shape of the network. Used only if the input shape couldn't be inferred from the network itself

    Returns
    -------
    out: dict
        report
    """
    network = load_model(model_path, compile=False)
    net_input_shape = network.input.shape.as_list()[-3:]
    if not any(dim is None for dim in net_input_shape):
        input_shape = net_input_shape
    dataset = datagen.create_ds(shape=input_shape, batch_size=batch_size)
    network.compile(optimizer="adam", loss="mse", metrics=metrics)
    eval_dict = network.evaluate(dataset, verbose=1, return_dict=True)
    report_dict = {}

    multibranch = False
    if isinstance(classes, Dict):
        multibranch = True
        branch_names = list(classes.keys())

    if del_network:
        del network
    for k, v in eval_dict.items():
        if "loss" in k:
            continue
        if not isinstance(v, np.ndarray):
            report_dict[k] = np.round(v, 3).item()
            continue

        if not multibranch:
            report_dict[k] = {predict_class: np.round(score, 3).item() for predict_class, score in zip(classes, v)}
            continue

        # Determine the branch the metric corresponds to
        metric_branch = None
        for branch in branch_names:
            if k.startswith(branch + "_"):
                metric_branch = branch
                break
        if metric_branch is not None:
            branch_classes = classes[metric_branch]
            report_dict[k] = {predict_class: np.round(score, 3).item() for predict_class, score in
                              zip(branch_classes, v)}

    return report_dict

evaluate_multiple_models(model_paths, datagen, metrics, batch_size, classes=Union[List[str], Dict[str, List[str]]], del_network=False, input_shape=None)

Evaluate a list of models and return dict of reports. For more information see evaluate_single_model

Source code in conftrainer/evaluation/utils.py

def evaluate_multiple_models(model_paths: List[str], datagen: BaseImageDatagen,
                             metrics: Union[Dict[str, Dict[str, Metric]], Dict[str, Metric], List[Metric]],
                             batch_size: int,
                             classes=Union[List[str], Dict[str, List[str]]], del_network: bool = False,
                             input_shape: List[int] = None) -> Dict[
    str, Dict]:
    """
    Evaluate a list of models and return dict of reports. For more information see
    evaluate_single_model
    """
    return {path: evaluate_single_model(model_path=path,
                                        datagen=datagen,
                                        batch_size=batch_size,
                                        classes=classes,
                                        metrics=metrics,
                                        del_network=del_network,
                                        input_shape=input_shape)
            for path in model_paths}

save_report(config)

Generate evaluation reports for several models and save a single report

Parameters:

Name	Type	Description	Default
config	evaluation.config.EvalConfig	configuration for evaluation	required

Source code in conftrainer/evaluation/utils.py

def save_report(config: EvalConfig) -> None:
    """
    Generate evaluation reports for several models and save a single report

    Parameters
    ----------
    config : evaluation.config.EvalConfig
        configuration for evaluation
    """
    reports = get_reports(config=config)
    json_save(config.report_filename, reports)

save_multioutput_report(config)

Generate evaluation report for 1 or several multioutput models and save a single report

Parameters:

Name	Type	Description	Default
config	evaluation.config.MultiBranchEvalConfig	configuration for evaluation	required

Source code in conftrainer/evaluation/utils.py

def save_multioutput_report(config: MultiBranchEvalConfig):
    """
    Generate evaluation report for 1 or several multioutput models and save a single report

    Parameters
    ----------
    config : evaluation.config.MultiBranchEvalConfig
        configuration for evaluation
    """
    reports = get_multibranch_reports(config=config)
    json_save(config.report_filename, reports)