\documentclass{article}

% \usepackage{xr} % for crossreferencing to tex. file containing additional information
\usepackage[utf8]{inputenc}
\usepackage{amsthm,amsmath}
%PACKAGES ADDED BY AM
\RequirePackage[sort]{natbib}
\RequirePackage{hyperref}
\usepackage[nohyperlinks, nolist, withpage]{acronym}
\usepackage{graphicx}
\usepackage[margin=10pt]{caption}
\usepackage{subcaption}
\usepackage{microtype} % to avoid overfull \hbox
\usepackage{ragged2e}
\usepackage{xr} % for crossreferencing to tex. file containing additional information
\usepackage{scrlayer-scrpage}
\ihead{Morger \textit{et al.}}

%%% HELPER CODE FOR DEALING WITH EXTERNAL REFERENCES
\usepackage{xr}
\makeatletter
\newcommand*{\addFileDependency}[1]{
  \typeout{(#1)}
  \@addtofilelist{#1}
  \IfFileExists{#1}{}{\typeout{No file #1.}}
}
\makeatother

\newcommand*{\myexternaldocument}[1]{
    \externaldocument{#1}
    \addFileDependency{#1.tex}
    \addFileDependency{#1.aux}
}

\newcommand{\beginsupplement}{%
        \setcounter{table}{0}
        \renewcommand{\thetable}{S\arabic{table}}%
        \setcounter{figure}{0}
        \renewcommand{\thefigure}{S\arabic{figure}}%
				}
%%% END HELPER CODE

% put all the external documents here!
\myexternaldocument{main}

% just to see what's happening
\listfiles
\begin{document}
\center{\LARGE{Additional file}}\\
\vspace{5pt}
\Large{Assessing the Calibration in Toxicological in Vitro Models with Conformal Prediction}\\
\vspace{10pt}
\large{Andrea Morger, Fredrik Svensson, Staffan Arvidsson McShane, Niharika Gauraha, Ulf Norinder, Ola Spjuth and Andrea Volkamer}

\beginsupplement

\begin{table}[ht]
    \centering
    \caption{Number of compounds available per Tox21 dataset and endpoint before standardisation.}
    \begin{tabular}{p{24mm}|p{8mm}p{12mm}|p{8mm}p{12mm}|p{8mm}p{12mm}}
        %  &  \\
    & \multicolumn{2}{c}{\acsu{toxtr}} & \multicolumn{2}{c}{\acsu{toxte}} &\multicolumn{2}{c}{\acsu{toxsc}}\\
    endpoint & actives & inactives & actives & inactives & actives & inactives\\
    \hline
    \acsu{ahr} & 950 & 7214 & 30 & 241 & 73 & 537\\
     \hline
    \acsu{ar} & 380 & 8977 & 3 & 288 & 12 & 574\\
     \hline
    \acsu{arlbd} & 303 & 8291 & 4 & 248 & 8 & 574\\
     \hline
    NR\_Aromatase & 360 & 6861 & 18 & 196 & 39 & 489\\
     \hline
    \acsu{er} & 937 & 6756 & 27 & 237 & 51 & 465\\
     \hline
    \acsu{erlbd} & 446 & 8302 & 10 & 276 & 20 & 580\\
     \hline
    \acsu{pparg} & 222 & 7957 & 15 & 251 & 31 & 574\\
     \hline
    \acsu{are} & 1097 & 6067 & 47 & 186 & 93 & 462\\
     \hline
    SR\_ATAD5 & 338 & 8748 & 25 & 246 & 38 & 584\\
     \hline
    \acsu{hse} & 428 & 7718 & 10 & 256 & 22 & 588\\
     \hline
    \acsu{mmp} & 1142 & 6174 & 38 & 199 & 60 & 483\\
     \hline
    SR\_p53 & 537 & 8092 & 28 & 240 & 41 & 575\\ 
     \hline
    \end{tabular}
    
    \label{stab:num_unstd_cpds}
\end{table}

\begin{figure}
    \centering
    \captionsetup{type=figure}
    \includegraphics[width=\textwidth]{cal_plot_all_eps_cv.pdf}
    \caption{\textit{1-internal\_CV:} \acsu{acp} models were trained and calibrated on \acsu{toxtr} and internally validated. \acsp{cep} for all twelve Tox21 endpoints are shown. Class 0: inactive compounds, class 1: active compounds. For a detailed explanation of all the components in the \acsu{cep}, see Figure \ref{fig:calplot}.}
    \label{sfig:sup_cv_results}
\end{figure}

\begin{figure}
    \centering
    \captionsetup{type=figure}
    \includegraphics[width=\textwidth]{cal_plot_all_eps_pred_score.pdf}
    \caption{\textit{2-pred\_score:} \acsu{acp} models were trained and calibrated on \acsu{toxtr} and predictions were made for \acsu{toxsc}. \acsp{cep} for all twelve Tox21 endpoints are shown. Class 0: inactive compounds, class 1: active compounds. For a detailed explanation of all the components in the \acsu{cep}, see Figure \ref{fig:calplot}.}
    \label{sfig:sup_pred_score_results}
\end{figure}

\begin{figure}
    \centering
    \captionsetup{type=figure}
    \includegraphics[width=\textwidth]{cal_plot_all_eps_pred_test.pdf}
    \caption{\textit{pred\_test:} \acsu{acp} models were trained and calibrated on \acsu{toxtr} and predictions were made for \acsu{toxte}. \acsp{cep} for all twelve Tox21 endpoints are shown. Class 0: inactive compounds, class 1: active compounds. For a detailed explanation of all the components in the \acsu{cep}, see Figure \ref{fig:calplot}.}
    \label{sfig:sup_pred_test_results}
\end{figure}


\begin{figure}
    \centering
    \captionsetup{type=figure}
    \includegraphics[width=\textwidth]{cal_plot_all_eps_pred_score_scp.pdf}
    \caption{\textit{3-pred\_score\_SCP:} \acsu{scp} models were trained on \acsu{toxtr} and predictions made for \acsu{toxsc}. \acsp{cep} for all twelve Tox21 endpoints are shown. Class 0: inactive compounds, class 1: active compounds. For a detailed explanation of all the components in the \acsu{cep}, see Figure \ref{fig:calplot}.}
    \label{sfig:sup_scp_results}
\end{figure}

\begin{figure}
    \centering
    \captionsetup{type=figure}
    \includegraphics[width=\textwidth]{cal_plot_all_eps_pred_score_train_update.pdf}
    \caption{\textit{4-train\_update:} The training set from \acsu{toxtr} was updated with \acsu{toxte}. An ACP model was retrained and predictions were made for \acsu{toxsc}. \acsp{cep} for all twelve Tox21 endpoints are shown. Class 0: inactive compounds, class 1: active compounds. For a detailed explanation of all the components in the \acsu{cep}, see Figure \ref{fig:calplot}.}
    \label{sfig:sup_train_update_results}
\end{figure}

\begin{figure}
    \centering
    \captionsetup{type=figure}
    \includegraphics[width=\textwidth]{cal_plot_all_eps_cal_update.pdf}
    \caption{\textit{5-cal\_update:} \acsu{acp} models were trained on \acsu{toxtr} and calibrated on \acsu{toxte}. Predictions were made for \acsu{toxsc}. \acsp{cep} for all twelve Tox21 endpoints are shown. Class 0: inactive compounds, class 1: active compounds. For a detailed explanation of all the components in the \acsu{cep}, see Figure \ref{fig:calplot}.}
    \label{sfig:sup_cal_update_results}
\end{figure}

\begin{figure}
    \centering
    \captionsetup{type=figure}
    \includegraphics[width=\textwidth]{cal_plot_all_eps_cal_update2.pdf}
    \caption{\textit{6-cal\_update\_2:} \acsu{acp} models were trained on \acsu{toxtr} and calibrated on 50\% of \acsu{toxsc}. Predictions were made for the other 50\% of \acsu{toxsc}. \acsp{cep} for all twelve Tox21 endpoints are shown. Class 0: inactive compounds, class 1: active compounds. For a detailed explanation of all the components in the \acsu{cep}, see Figure \ref{fig:calplot}.}
    \label{sfig:sup_cal_update2_results}
\end{figure}


% \begin{figure}[!ht]
%     \centering
%     \captionsetup{type=figure}
%     \begin{subfigure}[b]{0.45\textwidth}
%         \includegraphics[width=\textwidth]{boxplot_rmsd_pos_1.pdf}
%         \caption{rmsd$_{pos}$ actives}
%         \label{sfig:rmsd_pos_cl1}
%     \end{subfigure}
%     \begin{subfigure}[b]{0.45\textwidth}
%         \includegraphics[width=\textwidth]{boxplot_rmsd_pos_0.pdf}
%         \caption{rmsd$_{pos}$ inactives}
%         \label{sfig:rmsd_pos_cl0}
%     \end{subfigure}
%     \caption{rmsd$_{pos}$, i.e \acsu{rmsd} only considering deviations of the error rate above the diagonal line, for all twelve Tox21 endpoints compared amongst the different experiments.}
%     \label{sfig:rmsd_pos}
% \end{figure}


\begin{table}[!ht]
\begin{minipage}{\linewidth}
    \centering
     \caption{Mean $\pm$ standard deviation values over all twelve endpoints for observed error rate and efficiency at \acsu{sl} 0.2 for all experiments.} 
    \begin{tabular}{c| p{23mm}|c|c}
         nr. & name & error rate at SL 0.2 & efficiency at SL 0.2\\
         \hline
         1 & \textit{internal\_CV} & 0.17 $\pm$ 0.01& 0.77 $\pm$ 0.13 \\
         2 & \textit{pred\_score} & 0.31 $\pm$ 0.12 & 0.72 $\pm$ 0.14\\
         $-$ & \textit{pred\_test}* & 0.26 $\pm$ 0.11 & 0.70 $\pm$ 0.15\\
         3 & \textit{pred\_score\_SCP} & 0.27 $\pm$ 0.12 & 0.73 $\pm$ 0.13\\
         4 & \textit{train\_update} & 0.23 $\pm$0.06 & 0.71 $\pm$0.15\\
         5 & \textit{cal\_update} & 0.21 $\pm$ 0.05 & 0.51 $\pm$ 0.18\\
         6 & \textit{cal\_update\_2} & 0.18 $\pm$ 0.01 & 0.50 $\pm$ 0.17\\
    \end{tabular}\\
    \label{stab:sl02}
    \end{minipage}
    *\textit{pred\_test}: the CV-models from \textit{internal\_cv} were used to make predictions on \acsu{toxte}.
   
    
\end{table}


\begin{table}[!ht]
\begin{minipage}{\linewidth}
    \centering
     \caption{Mean RMSD values over all 12 endpoints, calculated for all compounds, as well as for active and inactive compounds, separately.} 
    \begin{tabular}{c| p{23mm}|c|c|c}
         nr. & name & all & actives & inactives\\
         \hline
         1 & \textit{internal\_CV} & 0.022 & 0.032 & 0.022\\
         2 & \textit{pred\_score} & 0.150 & 0.167 & 0.154\\
         $-$ & \textit{pred\_test}* & 0.116 & 0.180 & 0.119\\
         3 & \textit{pred\_score\_SCP} & 0.121 & 0.147 & 0.124 \\
         4 & \textit{train\_update} & 0.090 & 0.135 & 0.089\\
         5 & \textit{cal\_update} & 0.054 & 0.073 & 0.058\\
         6 & \textit{cal\_update\_2} & 0.018 & 0.057 & 0.018\\
    \end{tabular}
    \label{stab:RMSD}
   \end{minipage}
    *\textit{pred\_test}: the CV-models from \textit{internal\_cv} were used to make predictions on \acsu{toxte}.
   
    
\end{table}

\begin{acronym}%[HTS-FP]
	\acro{ml}[ML]{Machine Learning}
	\acro{svm}[SVM]{support vector machine}
	\acro{EP}{endpoint}
	\acro{ahr}[NR\_AhR]{Aryl hydrocarbon receptor}
	\acro{ar}[NR\_AR]{Androgen receptor, full length}
	\acro{er}[NR\_ER]{Estrogen receptor, full length}
	\acro{arlbd}[NR\_AR\_LBD]{Androgen receptor, ligand-binding domain}
	\acro{erlbd}[NR\_ER\_LBD]{Estrogen receptor, ligand-binding domain}
	\acro{pparg}[NR\_PPAR\_$\gamma$]{Peroxisome proliferator-activated receptor gamma}
	\acro{are}[SR\_ARE]{Nuclear factor (erythroid-derived 2)-like 2\/antioxidant responsive element}
	\acro{hse}[SR\_HSE]{Heat shock factor response element}
	\acro{mmp}[SR\_MMP]{Mitochondrial membrane potential}
	\acro{atad5}[SR\_ATAD5]{ATAD5}
	\acro{aromatase}[NR\_Aromatase]{Aromatase}
	\acro{p53}[SR\_p53]{p53}
	\acro{cp}[CP]{conformal prediction}
	\acro{icp}[ICP]{inductive conformal predictor}
	\acro{acp}[ACP]{aggregated conformal prediction}
	\acro{scp}[SCP]{synergy conformal prediction}
	\acro{nc}[nc]{nonconformity score}
	\acro{rmsd}{root-mean-square deviation}
	\acro{toxtr}[Tox21Train]{training data for the Tox21 Data Challenge}
	\acro{toxte}[Tox21Test]{test data for the Tox21 Data Challenge}
	\acro{toxsc}[Tox21Score]{final scoring data for the Tox21 Data Challenge}
	\acro{sl}[SL]{significance level}
	\acro{cep}[CEP]{calibration and efficiency plot}
	\acro{ptrain}{proper training set}
	\acro{cv}[CV]{cross-validation}
\end{acronym}

\end{document}