Skip to content

cvt.metrics

Module including routines computing metrics.

This module includes the following functions:

MAE(estimate, target, reduction_dims, mask=None, relative=False)

Mean Absolute Error.

Parameters:

Name Type Description Default
estimate

.

 required
target

.

 required
reduction_dims

.

 required
mask

.

 None
relative

.

 False

Returns: .

Source code in src/cvt/metrics.py 
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
def MAE(estimate, target, reduction_dims, mask=None, relative=False):
    """Mean Absolute Error.

    Parameters:
        estimate: .
        target: .
        reduction_dims: .
        mask: .
        relative: .

    Returns: .
    """
    assert(estimate.shape==target.shape)
    error = estimate - target
    if relative:
        error /= target
    error = torch.abs(error)

    if mask != None:
        assert(error.shape==mask.shape)
        error *= mask
        error = (error.sum(dim=reduction_dims) / (mask.sum(dim=reduction_dims)+1e-10)).sum()
    else:
        error =  error.mean()
    return error

RMSE(estimate, target, mask=None, relative=False)

Root Mean Squared Error.

Parameters:

Name Type Description Default
estimate

.

 required
target

.

 required
mask

.

 None
relative

.

 False

Returns: .

Source code in src/cvt/metrics.py 
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
def RMSE(estimate, target, mask=None, relative=False):
    """Root Mean Squared Error.

    Parameters:
        estimate: .
        target: .
        mask: .
        relative: .

    Returns: .
    """
    assert(estimate.shape==target.shape)
    error = estimate - target
    if relative:
        error /= target
    error = torch.square(error)

    assert(error.shape==mask.shape)
    error *= mask

    reduction_dims = tuple(range(1,len(target.shape)))
    error = (error.sum(dim=reduction_dims) / (mask.sum(dim=reduction_dims)+1e-10)).sum()
    return torch.sqrt(error)

abs_error(est_depth, gt_depth)

Computes the absolute error between an estimated and groun-truth depth map.

Parameters:

Name Type Description Default
est_depth ndarray

Estimated depth map.

 required
gt_depth ndarray

Ground-truth depth map.

 required

Returns:

Type Description
ndarray

The absolute error map for the estimated depth map.
Source code in src/cvt/metrics.py 
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
def abs_error(est_depth: np.ndarray, gt_depth: np.ndarray) -> np.ndarray:
    """Computes the absolute error between an estimated and groun-truth depth map.

    Parameters:
        est_depth: Estimated depth map.
        gt_depth: Ground-truth depth map.

    Returns:
       The absolute error map for the estimated depth map. 
    """
    signed_error = est_depth - gt_depth

    # compute gt mask and number of valid pixels
    gt_mask = np.not_equal(gt_depth, 0.0).astype(np.double)
    error = np.abs(signed_error) * gt_mask

    return error

accuracy_eval(est_ply, gt_ply, mask_th, est_filt=None, gt_filt=None)

Computes the accuracy of an estimated point cloud against the provided ground-truth.

Parameters:

Name Type Description Default
est_ply ndarray

Estimated point cloud to be evaluated.

 required
gt_ply ndarray

Ground-truth point cloud.

 required
mask_th float

Masking threshold used to remove points from the evaluation farther than a specified distance value.

 required
est_filt Optional[ndarray]

Optional filter to remove unwanted point from the estimated point cloud in the evaluation

 None
gt_filt Optional[ndarray]

Optional filter to remove unwanted point from the ground-truth point cloud in the evaluation

 None

Returns:

Name Type Description
valid_est_ply PointCloud

Point cloud containing all the valid evaluation points after filtering.
dists_est ndarray

Estimated distances of all valid points in the estimated point cloud to the closest point in the ground-truth point cloud.
colors_est ndarray

Estimated colors for the points in the valid point cloud.
Source code in src/cvt/metrics.py 
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
def accuracy_eval(
        est_ply: np.ndarray,
        gt_ply: np.ndarray,
        mask_th: float,
        est_filt: Optional[np.ndarray] = None,
        gt_filt: Optional[np.ndarray] = None) -> Tuple[o3d.geometry.PointCloud, np.ndarray, np.ndarray]:
    """Computes the accuracy of an estimated point cloud against the provided ground-truth.

    Parameters:
        est_ply: Estimated point cloud to be evaluated.
        gt_ply: Ground-truth point cloud.
        mask_th: Masking threshold used to remove points from the evaluation farther 
                    than a specified distance value.
        est_filt: Optional filter to remove unwanted point from the estimated 
                    point cloud in the evaluation
        gt_filt: Optional filter to remove unwanted point from the ground-truth
                    point cloud in the evaluation

    Returns:
        valid_est_ply: Point cloud containing all the valid evaluation points after filtering.
        dists_est: Estimated distances of all valid points in the estimated point cloud 
                    to the closest point in the ground-truth point cloud.
        colors_est: Estimated colors for the points in the valid point cloud.
    """
    # distance from est to gt
    dists_est = np.asarray(est_ply.compute_point_cloud_distance(gt_ply))

    # extract valid indices
    valid_inds_est = set(np.where(est_filt == 1)[0])
    valid_dists = set(np.where(dists_est <= mask_th)[0])
    valid_inds_est.intersection_update(valid_dists)
    valid_inds_est = np.asarray(list(valid_inds_est))

    # get distances and colors at valid indices
    valid_est_ply = est_ply.select_by_index(valid_inds_est)
    dists_est = dists_est[valid_inds_est]
    colors_est = np.asarray(est_ply.colors)[valid_inds_est]

    return valid_est_ply, dists_est, colors_est

completeness_eval(est_ply, gt_ply, mask_th=20.0, est_filt=None, gt_filt=None)

Computes the completeness of an estimated point cloud against the provided ground-truth.

Parameters:

Name Type Description Default
est_ply PointCloud

Estimated point cloud to be evaluated.

 required
gt_ply PointCloud

Ground-truth point cloud.

 required
mask_th float

Masking threshold used to remove points from the evaluation farther than a specified distance value.

 20.0
est_filt Optional[ndarray]

Optional filter to remove unwanted point from the estimated point cloud in the evaluation

 None
gt_filt Optional[ndarray]

Optional filter to remove unwanted point from the ground-truth point cloud in the evaluation

 None

Returns:

Name Type Description
ply_points ndarray

Point cloud vertices containing all the valid evaluation points after filtering.
dists ndarray

Distances of all valid points in the ground-truth point cloud to the closest point in the estimated point cloud.
colors ndarray

Colors for the points in the valid point cloud.
Source code in src/cvt/metrics.py 
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
def completeness_eval(
        est_ply: o3d.geometry.PointCloud,
        gt_ply: o3d.geometry.PointCloud,
        mask_th: float = 20.0,
        est_filt: Optional[np.ndarray] = None,
        gt_filt: Optional[np.ndarray] = None) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
    """Computes the completeness of an estimated point cloud against the provided ground-truth.

    Parameters:
        est_ply: Estimated point cloud to be evaluated.
        gt_ply: Ground-truth point cloud.
        mask_th: Masking threshold used to remove points from the evaluation farther 
                    than a specified distance value.
        est_filt: Optional filter to remove unwanted point from the estimated 
                    point cloud in the evaluation
        gt_filt: Optional filter to remove unwanted point from the ground-truth
                    point cloud in the evaluation

    Returns:
        ply_points: Point cloud vertices containing all the valid evaluation points after filtering.
        dists: Distances of all valid points in the ground-truth point cloud 
                    to the closest point in the estimated point cloud.
        colors: Colors for the points in the valid point cloud.
    """
    # build KD-Tree of estimated point cloud for querying
    tree = KDTree(np.asarray(est_ply.points), leaf_size=40)
    (dists, inds) = tree.query(np.asarray(gt_ply.points), k=1)

    # extract valid indices
    valid_inds = set(np.where(gt_filt == 1)[0])
    valid_inds.intersection_update(set(np.where(dists <= mask_th)[0]))
    valid_inds = np.asarray(list(valid_inds))

    dists = dists[valid_inds]
    inds = inds[valid_inds]
    ply_points = np.asarray(est_ply.points)[inds]
    colors = np.asarray(est_ply.colors)[inds]

    return ply_points, dists, colors

filter_outlier_points(est_ply, gt_ply, outlier_th)

Filters out points from an estimated point cloud that are farther than some threshold to the ground-truth point cloud.

Parameters:

Name Type Description Default
est_ply PointCloud

Estimated point cloud to filter.

 required
gt_ply PointCloud

Ground-truth point cloud for reference.

 required
outlier_th float

Distance threshold used for filtering.

 required

Returns:

Type Description
PointCloud

The filtered point cloud.
Source code in src/cvt/metrics.py 
164
165
166
167
168
169
170
171
172
173
174
175
176
177
def filter_outlier_points(est_ply: o3d.geometry.PointCloud, gt_ply: o3d.geometry.PointCloud, outlier_th: float) -> o3d.geometry.PointCloud:
    """Filters out points from an estimated point cloud that are farther than some threshold to the ground-truth point cloud.

    Parameters:
        est_ply: Estimated point cloud to filter.
        gt_ply: Ground-truth point cloud for reference.
        outlier_th: Distance threshold used for filtering.

    Returns:
        The filtered point cloud.
    """
    dists_est = np.asarray(est_ply.compute_point_cloud_distance(gt_ply))
    valid_dists = np.where(dists_est <= outlier_th)[0]
    return est_ply.select_by_index(valid_dists)