From 1032d673b78d8e54e36ab80f1ee80e4b6ec533f2 Mon Sep 17 00:00:00 2001 From: valosekj Date: Tue, 27 Feb 2024 16:37:44 -0500 Subject: [PATCH 1/9] Add missing 'vol_diff' key to BinaryPairwiseMeasures.measures_dict Otherwise, the following KeyError is raised: Traceback (most recent call last): File "/Users/user/code/MetricsReloaded/MetricsReloaded/metrics/pairwise_measures.py", line 1161, in to_dict_meas if len(self.measures_dict[key]) == 2: KeyError: 'vol_diff' --- MetricsReloaded/metrics/pairwise_measures.py | 1 + 1 file changed, 1 insertion(+) diff --git a/MetricsReloaded/metrics/pairwise_measures.py b/MetricsReloaded/metrics/pairwise_measures.py index 72e25a5..dccc5c4 100755 --- a/MetricsReloaded/metrics/pairwise_measures.py +++ b/MetricsReloaded/metrics/pairwise_measures.py @@ -272,6 +272,7 @@ def __init__( "hd_perc": (self.measured_hausdorff_distance_perc, "HDPerc"), "masd": (self.measured_masd, "MASD"), "nsd": (self.normalised_surface_distance, "NSD"), + "vol_diff": (self.vol_diff, "VolDiff"), } self.pred = pred From 789a6bdab88721a0b4bde4a3e22e999ac3e51058 Mon Sep 17 00:00:00 2001 From: valosekj Date: Wed, 28 Feb 2024 07:30:53 -0500 Subject: [PATCH 2/9] Add 'rel_vol_diff' function to calculate the relative volume difference (RVE) in % between the prediction and the reference. --- MetricsReloaded/metrics/pairwise_measures.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/MetricsReloaded/metrics/pairwise_measures.py b/MetricsReloaded/metrics/pairwise_measures.py index dccc5c4..424792a 100755 --- a/MetricsReloaded/metrics/pairwise_measures.py +++ b/MetricsReloaded/metrics/pairwise_measures.py @@ -273,6 +273,7 @@ def __init__( "masd": (self.measured_masd, "MASD"), "nsd": (self.normalised_surface_distance, "NSD"), "vol_diff": (self.vol_diff, "VolDiff"), + "rel_vol_diff": (self.rel_vol_diff, "RelVolDiff"), } self.pred = pred @@ -886,6 +887,16 @@ def vol_diff(self): """ return np.abs(self.n_pos_ref() - self.n_pos_pred()) / self.n_pos_ref() + def rel_vol_diff(self): + """ + This function calculates the relative volume difference (RVE) in % between the prediction and the reference. + If the prediction is smaller than the reference, the relative volume difference is negative. + If the prediction is larger than the reference, the relative volume difference is positive. + + :return: rel_vol_diff + """ + return ((self.n_pos_pred() - self.n_pos_ref()) / self.n_pos_ref()) * 100 + @CacheFunctionOutput def skeleton_versions(self): """ From b0a6790203ed7b9604e8a51327140b019715d4f4 Mon Sep 17 00:00:00 2001 From: valosekj Date: Wed, 28 Feb 2024 07:31:16 -0500 Subject: [PATCH 3/9] Comment out print for hausdorff distance --- MetricsReloaded/metrics/pairwise_measures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MetricsReloaded/metrics/pairwise_measures.py b/MetricsReloaded/metrics/pairwise_measures.py index 424792a..f5461b6 100755 --- a/MetricsReloaded/metrics/pairwise_measures.py +++ b/MetricsReloaded/metrics/pairwise_measures.py @@ -1083,7 +1083,7 @@ def measured_distance(self): ref_border, pred_border, ) = self.border_distance() - print(ref_border_dist) + #print(ref_border_dist) average_distance = (np.sum(ref_border_dist) + np.sum(pred_border_dist)) / ( np.sum(pred_border + ref_border) ) From dfa701a8212396b65f536cf0ca606821c8d9ec6d Mon Sep 17 00:00:00 2001 From: valosekj Date: Wed, 28 Feb 2024 07:36:18 -0500 Subject: [PATCH 4/9] fix typo --- MetricsReloaded/metrics/pairwise_measures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MetricsReloaded/metrics/pairwise_measures.py b/MetricsReloaded/metrics/pairwise_measures.py index f5461b6..76460db 100755 --- a/MetricsReloaded/metrics/pairwise_measures.py +++ b/MetricsReloaded/metrics/pairwise_measures.py @@ -889,7 +889,7 @@ def vol_diff(self): def rel_vol_diff(self): """ - This function calculates the relative volume difference (RVE) in % between the prediction and the reference. + This function calculates the relative volume error (RVE) in % between the prediction and the reference. If the prediction is smaller than the reference, the relative volume difference is negative. If the prediction is larger than the reference, the relative volume difference is positive. From 11bf9d18648cf078067a038b26658d0191f2d8c9 Mon Sep 17 00:00:00 2001 From: valosekj Date: Wed, 28 Feb 2024 10:31:58 -0500 Subject: [PATCH 5/9] Handle edge cases for 'rel_vol_diff': - if both reference and prediction are empty --> model learned correctly --> setting 0 representing no over- or under-segmentation - if reference is empty, prediction is not empty --> model did not learn correctly --> setting positive value representing overestimation --- MetricsReloaded/metrics/pairwise_measures.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/MetricsReloaded/metrics/pairwise_measures.py b/MetricsReloaded/metrics/pairwise_measures.py index 76460db..f7cd612 100755 --- a/MetricsReloaded/metrics/pairwise_measures.py +++ b/MetricsReloaded/metrics/pairwise_measures.py @@ -895,7 +895,16 @@ def rel_vol_diff(self): :return: rel_vol_diff """ - return ((self.n_pos_pred() - self.n_pos_ref()) / self.n_pos_ref()) * 100 + if self.flag_empty_ref and self.flag_empty_pred: + # Both reference and prediction are empty --> model learned correctly --> setting 0 representing no over- + # or under-segmentation + return 0 + elif self.flag_empty_ref and not self.flag_empty_pred: + # Reference is empty, prediction is not empty --> model did not learn correctly --> setting positive value + # representing overestimation + return 100 + else: + return ((self.n_pos_pred() - self.n_pos_ref()) / self.n_pos_ref()) * 100 @CacheFunctionOutput def skeleton_versions(self): From de5dc96ebf8aed2c04f5aabeb8e6c49da2cd6682 Mon Sep 17 00:00:00 2001 From: valosekj Date: Thu, 29 Feb 2024 11:46:02 -0500 Subject: [PATCH 6/9] rename rel_vol_diff --> rel_vol_error --- MetricsReloaded/metrics/pairwise_measures.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/MetricsReloaded/metrics/pairwise_measures.py b/MetricsReloaded/metrics/pairwise_measures.py index f7cd612..45d6836 100755 --- a/MetricsReloaded/metrics/pairwise_measures.py +++ b/MetricsReloaded/metrics/pairwise_measures.py @@ -273,7 +273,7 @@ def __init__( "masd": (self.measured_masd, "MASD"), "nsd": (self.normalised_surface_distance, "NSD"), "vol_diff": (self.vol_diff, "VolDiff"), - "rel_vol_diff": (self.rel_vol_diff, "RelVolDiff"), + "rel_vol_error": (self.rel_vol_error, "RelVolError"), } self.pred = pred @@ -887,13 +887,13 @@ def vol_diff(self): """ return np.abs(self.n_pos_ref() - self.n_pos_pred()) / self.n_pos_ref() - def rel_vol_diff(self): + def rel_vol_error(self): """ This function calculates the relative volume error (RVE) in % between the prediction and the reference. If the prediction is smaller than the reference, the relative volume difference is negative. If the prediction is larger than the reference, the relative volume difference is positive. - :return: rel_vol_diff + :return: rel_vol_error """ if self.flag_empty_ref and self.flag_empty_pred: # Both reference and prediction are empty --> model learned correctly --> setting 0 representing no over- From 99a35b3a3b9b59e86db273ad06585bc50ac4d6e7 Mon Sep 17 00:00:00 2001 From: Jan Valosek <39456460+valosekj@users.noreply.github.com> Date: Tue, 16 Apr 2024 16:11:06 -0400 Subject: [PATCH 7/9] Update README.rst - Add info that is is a fork - Update clone link to ivadomed - Remove badges Signed-off-by: Jan Valosek <39456460+valosekj@users.noreply.github.com> --- README.rst | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/README.rst b/README.rst index 0714635..d5dbaa0 100644 --- a/README.rst +++ b/README.rst @@ -4,27 +4,9 @@ Metrics Reloaded .. start-description -A Python implementaiton of `Metrics Reloaded `__ - A new recommendation framework for biomedical image analysis validation. +A Python implementation of `Metrics Reloaded `__ - A new recommendation framework for biomedical image analysis validation. -.. start-badges - -|docs| -|testing| -|codecov| - -.. |docs| image:: https://readthedocs.org/projects/metricsreloaded/badge/?style=flat - :target: https://MetricsReloaded.readthedocs.io/en/latest/?badge=latest - :alt: Documentation Status - -.. |testing| image:: https://github.com/csudre/MetricsReloaded/actions/workflows/python-app.yml/badge.svg - :target: https://github.com/csudre/MetricsReloaded/actions - :alt: Testing Status - -.. |codecov| image:: https://codecov.io/gh/csudre/MetricsReloaded/branch/main/graph/badge.svg - :target: https://codecov.io/gh/csudre/MetricsReloaded - :alt: Coverage Status - -.. end-badges +This is a fork of the `Project-MONAI/MetricsReloaded `__ repo. Installation ============ @@ -38,7 +20,7 @@ Create and activate a new `Conda Date: Thu, 30 May 2024 18:34:34 -0400 Subject: [PATCH 8/9] reorder measures_dict in BPM class --- MetricsReloaded/metrics/pairwise_measures.py | 29 +++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/MetricsReloaded/metrics/pairwise_measures.py b/MetricsReloaded/metrics/pairwise_measures.py index 45d6836..bd2b0b4 100755 --- a/MetricsReloaded/metrics/pairwise_measures.py +++ b/MetricsReloaded/metrics/pairwise_measures.py @@ -146,7 +146,7 @@ def matthews_correlation_coefficient(self): cov_pred += np.cov(one_hot_pred[:, f], one_hot_pred[:, f])[0, 1] cov_ref += np.cov(one_hot_ref[:, f], one_hot_ref[:, f])[0, 1] cov_pr += np.cov(one_hot_pred[:, f], one_hot_ref[:, f])[0, 1] - + numerator = cov_pr denominator = np.sqrt(cov_pred * cov_ref) return numerator / denominator @@ -233,7 +233,7 @@ def to_dict_meas(self, fmt="{:.4f}"): for key in self.measures: result = self.measures_dict[key][0]() result_dict[key] = result - return result_dict + return result_dict class BinaryPairwiseMeasures(object): @@ -260,18 +260,21 @@ def __init__( "ba": (self.balanced_accuracy, "BalAcc"), "cohens_kappa": (self.cohens_kappa, "CohensKappa"), "lr+": (self.positive_likelihood_ratio, "LR+"), + "youden_ind": (self.youden_index, "YoudenInd"), + "mcc": (self.matthews_correlation_coefficient, "MCC"), + # overlap-based measures "iou": (self.intersection_over_union, "IoU"), "fbeta": (self.fbeta, "FBeta"), "dsc":(self.dsc, "DSC"), - "youden_ind": (self.youden_index, "YoudenInd"), - "mcc": (self.matthews_correlation_coefficient, "MCC"), + "boundary_iou": (self.boundary_iou, "BoundaryIoU"), "cldice": (self.centreline_dsc, "CentreLineDSC"), + # distance-based measures "assd": (self.measured_average_distance, "ASSD"), - "boundary_iou": (self.boundary_iou, "BoundaryIoU"), "hd": (self.measured_hausdorff_distance, "HD"), "hd_perc": (self.measured_hausdorff_distance_perc, "HDPerc"), "masd": (self.measured_masd, "MASD"), "nsd": (self.normalised_surface_distance, "NSD"), + # other measures "vol_diff": (self.vol_diff, "VolDiff"), "rel_vol_error": (self.rel_vol_error, "RelVolError"), } @@ -672,7 +675,7 @@ def dsc(self): ..math:: DSC = \dfrac{2TP}{2TP+FP+FN} - + This is also F:math:`{\\beta}` for :math:`{\\beta}`=1 """ @@ -832,7 +835,7 @@ def com_dist(self): :return: Euclidean distance between centre of mass when reference and prediction not empty -1 otherwise """ - + if self.flag_empty_pred or self.flag_empty_ref: return -1 else: @@ -976,8 +979,8 @@ def boundary_iou(self): This functions determines the boundary iou Bowen Cheng, Ross Girshick, Piotr Dollár, Alexander C Berg, and Alexander Kirillov. 2021. Boundary IoU: Improving -Object-Centric Image Segmentation Evaluation. In Proceedings of the IEEE/CVF Conference on Computer Vision and -Pattern Recognition. 15334–15342. + Object-Centric Image Segmentation Evaluation. In Proceedings of the IEEE/CVF Conference on Computer Vision and + Pattern Recognition. 15334–15342. .. math:: @@ -1102,7 +1105,7 @@ def measured_distance(self): ) hausdorff_distance = np.max([np.max(ref_border_dist), np.max(pred_border_dist)]) - + hausdorff_distance_perc = np.max( [ np.percentile(ref_border_dist[pred_border > 0], q=perc), @@ -1129,14 +1132,14 @@ def measured_average_distance(self): def measured_masd(self): """ This function returns only the mean average surface distance defined as - + Miroslav Beneš and Barbara Zitová. 2015. Performance evaluation of image segmentation algorithms on microscopic image data. Journal of microscopy 257, 1 (2015), 65–85. .. math:: MASD(A,B) = \dfrac{1}{2}\left(\dfrac{\sum_{a\in A}d(a,B)}{|A|} + \dfrac{\sum_{b\inB}d(b,A)}{|B|}) - + :return: masd """ return self.measured_distance()[3] @@ -1159,7 +1162,7 @@ def measured_hausdorff_distance_perc(self): Daniel P Huttenlocher, Gregory A. Klanderman, and William J Rucklidge. 1993. Comparing images using the Hausdorff distance. IEEE Transactions on pattern analysis and machine intelligence 15, 9 (1993), 850–863. - + :return: hausdorff_distance_perc """ return self.measured_distance()[2] From 1247255a435c07b1c4774cd623bab2108d92be4b Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Thu, 30 May 2024 18:35:41 -0400 Subject: [PATCH 9/9] add lesion-wise F1score, PPV, and Sensitivity measures --- MetricsReloaded/metrics/pairwise_measures.py | 129 ++++++++++++++++++- 1 file changed, 126 insertions(+), 3 deletions(-) diff --git a/MetricsReloaded/metrics/pairwise_measures.py b/MetricsReloaded/metrics/pairwise_measures.py index bd2b0b4..19de73d 100755 --- a/MetricsReloaded/metrics/pairwise_measures.py +++ b/MetricsReloaded/metrics/pairwise_measures.py @@ -274,6 +274,10 @@ def __init__( "hd_perc": (self.measured_hausdorff_distance_perc, "HDPerc"), "masd": (self.measured_masd, "MASD"), "nsd": (self.normalised_surface_distance, "NSD"), + # instance-specific measures + "lesion_ppv": (self.lesion_ppv, "LesionWisePPV"), + "lesion_sensitivity": (self.lesion_sensitivity, "LesionWiseSensitivity"), + "lesion_f1_score": (self.lesion_f1_score, "LesionWiseF1Score"), # other measures "vol_diff": (self.vol_diff, "VolDiff"), "rel_vol_error": (self.rel_vol_error, "RelVolError"), @@ -1167,6 +1171,127 @@ def measured_hausdorff_distance_perc(self): """ return self.measured_distance()[2] + def lesion_wise_tp_fp_fn(self, truth, prediction): + """ + Computes the true positives, false positives, and false negatives two masks. Masks are considered true positives + if at least one voxel overlaps between the truth and the prediction. + Adapted from: https://github.com/npnl/atlas2_grand_challenge/blob/main/isles/scoring.py#L341 + + Parameters + ---------- + truth : array-like, bool + 3D array. If not boolean, will be converted. + prediction : array-like, bool + 3D array with a shape matching 'truth'. If not boolean, will be converted. + empty_value : scalar, float + Optional. Value to which to default if there are no labels. Default: 1.0. + + Returns + ------- + tp (int): 3D connected-component from the ground-truth image that overlaps at least on one voxel with the prediction image. + fp (int): 3D connected-component from the prediction image that has no voxel overlapping with the ground-truth image. + fn (int): 3d connected-component from the ground-truth image that has no voxel overlapping with the prediction image. + + Notes + ----- + This function computes lesion-wise score by defining true positive lesions (tp), false positive lesions (fp) and + false negative lesions (fn) using 3D connected-component-analysis. + + tp: 3D connected-component from the ground-truth image that overlaps at least on one voxel with the prediction image. + fp: 3D connected-component from the prediction image that has no voxel overlapping with the ground-truth image. + fn: 3d connected-component from the ground-truth image that has no voxel overlapping with the prediction image. + """ + tp, fp, fn = 0, 0, 0 + + # For each true lesion, check if there is at least one overlapping voxel. This determines true positives and + # false negatives (unpredicted lesions) + labeled_ground_truth, num_lesions = ndimage.label(truth.astype(bool)) + for idx_lesion in range(1, num_lesions+1): + lesion = labeled_ground_truth == idx_lesion + lesion_pred_sum = lesion + prediction + if(np.max(lesion_pred_sum) > 1): + tp += 1 + else: + fn += 1 + + # For each predicted lesion, check if there is at least one overlapping voxel in the ground truth. + labaled_prediction, num_pred_lesions = ndimage.label(prediction.astype(bool)) + for idx_lesion in range(1, num_pred_lesions+1): + lesion = labaled_prediction == idx_lesion + lesion_pred_sum = lesion + truth + if(np.max(lesion_pred_sum) <= 1): # No overlap + fp += 1 + + return tp, fp, fn + + def lesion_f1_score(self): + """ + Computes the lesion-wise F1-score between two masks by defining true positive lesions (tp), false positive lesions (fp) + and false negative lesions (fn) using 3D connected-component-analysis. + + Masks are considered true positives if at least one voxel overlaps between the truth and the prediction. + + Returns + ------- + f1_score : float + Lesion-wise F1-score as float. + Max score = 1 + Min score = 0 + If both images are empty (tp + fp + fn =0) = empty_value + """ + empty_value = 1.0 # Value to which to default if there are no labels. Default: 1.0. + tp, fp, fn = self.lesion_wise_tp_fp_fn(self.ref, self.pred) + f1_score = empty_value + + # Compute f1_score + denom = tp + (fp + fn)/2 + if(denom != 0): + f1_score = tp / denom + return f1_score + + def lesion_ppv(self): + """ + Computes the lesion-wise positive predictive value (PPV) between two masks + Returns + ------- + ppv (float): Lesion-wise positive predictive value as float. + Max score = 1 + Min score = 0 + If both images are empty (tp + fp + fn =0) = empty_value + """ + empty_value = 1.0 + + tp, fp, fn = self.lesion_wise_tp_fp_fn(self.ref, self.pred) + ppv = empty_value + + # Compute ppv + denom = tp + fp + if(denom != 0): + ppv = tp / denom + return ppv + + def lesion_sensitivity(self): + """ + Computes the lesion-wise sensitivity between two masks + Returns + ------- + sensitivity (float): Lesion-wise sensitivity as float. + Max score = 1 + Min score = 0 + If both images are empty (tp + fp + fn =0) = empty_value + """ + empty_value = 1.0 + + tp, fp, fn = self.lesion_wise_tp_fp_fn(self.ref, self.pred) + sensitivity = empty_value + + # Compute sensitivity + denom = tp + fn + if(denom != 0): + sensitivity = tp / denom + return sensitivity + + # NOTE: it's best to keep this function at the end as it does not explicitly compute any metric def to_dict_meas(self, fmt="{:.4f}"): result_dict = {} for key in self.measures: @@ -1175,6 +1300,4 @@ def to_dict_meas(self, fmt="{:.4f}"): else: result = self.measures_dict[key][0](self.measures_dict[key][2]) result_dict[key] = result - return result_dict - - \ No newline at end of file + return result_dict