@article {Liu523, author = {Jinxin Liu and Ziping Liu and Hae Sol Moon and Joyce Mhlanga and Abhinav Jha}, title = {A no-gold-standard technique for objective evaluation of quantitative nuclear-medicine imaging methods in the presence of correlated noise}, volume = {61}, number = {supplement 1}, pages = {523--523}, year = {2020}, publisher = {Society of Nuclear Medicine}, abstract = {523Objectives: Clinical translation of new and improved quantitative imaging (QI) methods requires an objective evaluation of the ability of these methods to provide reliable quantitative measurements. Performing such evaluation on measurements directly obtained from patient images is highly desirable. However, this requires knowledge of the true quantitative value or a gold standard, which is typically time-consuming and expensive to obtain, and even when available, could be unreliable. To address this issue, our objective was to develop a no-gold-standard (NGS) technique to evaluate QI methods in the absence of a gold standard. Methods: Existing techniques for NGS evaluation [1-4] are based on the premise that measurements using different QI methods are the result of an image formation and quantitation process. Thus, the measured and true values are mathematically related, and more specifically linearly related, by a slope, bias, and normally distributed noise term. However, these techniques assume that the noise between the different methods is independent. Since noise arises in the process of measuring the same true value, the noise could be correlated. To address this issue, we extend the NGS framework to model the noise term as multivariate normally distributed and characterized by a covariance matrix. Based on this model, we derive a maximum-likelihood-based technique that, without any knowledge of the true value, yields estimates of the slope, bias, and covariance matrix terms. Similar to previous literature [1-3], the ratio of the diagonal term of the covariance matrix and the slope (noise-to-slope ratio) was used to rank the methods on the basis of precision of the measured QI values. The method was evaluated using numerical experiments and realistic simulation studies. In the numerical experiments, measurements corresponding to three different QI methods were generated with correlated noise. The realistic simulations were conducted in the context of evaluating PET segmentation methods on the task of measuring metabolic tumor volume (MTV). We generated realistic PET images with known ground-truth tumor boundaries using a projection-domain-based lesion-insertion approach [5]. The realism of the tumors was evaluated using a trained-reader-based observer study. After validating realism, tumors in these images were segmented using three methods based on thresholding, snakes, and deep-learning [6], respectively. We evaluated the performance of the NGS technique with multiple datasets and multiple noise realizations, comprehensively studying the efficacy of the technique. Results: In the numerical experiments, the proposed technique yielded accurate rankings of the three QI methods in 96\% (48 out of 50) of the noise realizations (Table 1), and clearly outperformed the existing NGS technique (Fig. 1). In the realistic PET simulation studies, the NGS evaluation method yielded accurate ranking without any knowledge of the true MTV values in 95\% (57 out of 60) of the cases (Fig. 1). Conclusions: The proposed no-gold-standard (NGS) evaluation technique yielded an accurate evaluation of different QI methods, as evaluated using both realistic simulations and numerical experiments. The results demonstrate the ability to perform NGS evaluation even in the presence of correlated noise. View this table:Measured noise-to-slope ratios for three different QI methods (s: standard deviation).}, issn = {0161-5505}, URL = {https://jnm.snmjournals.org/content/61/supplement_1/523}, eprint = {https://jnm.snmjournals.org/content}, journal = {Journal of Nuclear Medicine} }