%% BioMed_Central_Tex_Template_v1.06
%%                                      %
%  bmc_article.tex            ver: 1.06 %
%                                       %

%%IMPORTANT: do not delete the first line of this template
%%It must be present to enable the BMC Submission system to
%%recognise this template!!

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                     %%
%%  LaTeX template for BioMed Central  %%
%%     journal article submissions     %%
%%                                     %%
%%          <8 June 2012>              %%
%%                                     %%
%%                                     %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                                                 %%
%% For instructions on how to fill out this Tex template           %%
%% document please refer to Readme.html and the instructions for   %%
%% authors page on the biomed central website                      %%
%% http://www.biomedcentral.com/info/authors/                      %%
%%                                                                 %%
%% Please do not use \input{...} to include other tex files.       %%
%% Submit your LaTeX manuscript as one .tex document.              %%
%%                                                                 %%
%% All additional figures and files should be attached             %%
%% separately and not embedded in the \TeX\ document itself.       %%
%%                                                                 %%
%% BioMed Central currently use the MikTex distribution of         %%
%% TeX for Windows) of TeX and LaTeX.  This is available from      %%
%% http://www.miktex.org                                           %%
%%                                                                 %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%% additional documentclass options:
%  [doublespacing]
%  [linenumbers]   - put the line numbers on margins

%%% loading packages, author definitions

%\documentclass[twocolumn]{bmcart}% uncomment this for twocolumn layout and comment line below
\documentclass[doublespacing]{bmcart}

\usepackage{etoolbox}
\makeatletter
\patchcmd\set@numberlines@box{\rlap}{\@gobble}{}{}
\makeatother

%%% Load packages
%\usepackage{amsthm,amsmath}
%\RequirePackage{natbib}
%\RequirePackage[authoryear]{natbib}% uncomment this for author-year bibliography
%\RequirePackage{hyperref}
\usepackage[utf8]{inputenc} %unicode support
\usepackage{amsmath}
\usepackage{url}

\usepackage{lineno}
\usepackage{blindtext}
\linenumbers

%\usepackage[applemac]{inputenc} %applemac support if unicode package fails
%\usepackage[latin1]{inputenc} %UNIX support if unicode package fails


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                             %%
%%  If you wish to display your graphics for   %%
%%  your own use using includegraphic or       %%
%%  includegraphics, then comment out the      %%
%%  following two lines of code.               %%
%%  NB: These line *must* be included when     %%
%%  submitting to BMC.                         %%
%%  All figure files must be submitted as      %%
%%  separate graphics through the BMC          %%
%%  submission process, not included in the    %%
%%  submitted article.                         %%
%%                                             %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



\def\includegraphic{}
\def\includegraphics{}



%%% Put your definitions there:
\startlocaldefs
\endlocaldefs


%%% Begin ...
\begin{document}

%%% Start of article front matter
\begin{frontmatter}

\begin{fmbox}
\dochead{Technical Advance}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                          %%
%% Enter the title of your article here     %%
%%                                          %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\title{Machine learning approach for the binary classification of biomedical literature}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                          %%
%% Enter the authors here                   %%
%%                                          %%
%% Specify information, if available,       %%
%% in the form:                             %%
%%   <key>={<id1>,<id2>}                    %%
%%   <key>=                                 %%
%% Comment or delete the keys which are     %%
%% not used. Repeat \author command as much %%
%% as required.                             %%
%%                                          %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\author[
   addressref={aff1},                   % id's of addresses, e.g. {aff1,aff2}
   corref={aff1},                       % id of corresponding address, if any
   %noteref={n1},                        % id's of article notes, if any
   email={PriceA35@cardiff.ac.uk}   % email address
]{\inits{AP}\fnm{Anna} \snm{Price}}
\author[
   addressref={aff2},
   email={MortM@cardiff.ac.uk}
]{\inits{MM}\fnm{Matthew} \snm{Mort}}
\author[
   addressref={aff2},
   email={CooperDN@cardiff.ac.uk}
]{\inits{KEA}\fnm{David N.} \snm{Cooper}}
\author[
   addressref={aff3},
   email={AshelfordKE@cardiff.ac.uk}
]{\inits{KEA}\fnm{Kevin E.} \snm{Ashelford}}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                          %%
%% Enter the authors' addresses here        %%
%%                                          %%
%% Repeat \address commands as much as      %%
%% required.                                %%
%%                                          %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\address[id=aff1]{%                           % unique id
  \orgname{School of Biosciences}, % university, etc
  \street{Cardiff University},                     %
  %\postcode{}                                % post or zip code
  \city{Cardiff},                              % city
  \cny{UK}                                    % country
}
\address[id=aff2]{%
  \orgname{School of Medicine},
  \street{Institute of Medical Genetics},
  \street{Cardiff University},
  %\postcode{24105}
  \city{Cardiff},
  \cny{UK}
}
\address[id=aff3]{%
  \orgname{School of Medicine},
  \street{Wales Gene Park},
  \street{Cardiff University},
  %\postcode{24105}
  \city{Cardiff},
  \cny{UK}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                          %%
%% Enter short notes here                   %%
%%                                          %%
%% Short notes will be after addresses      %%
%% on first page.                           %%
%%                                          %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{artnotes}
%\note{Sample of title note}     % note to the article
%\note[id=n1]{Equal contributor} % note, connected to author
\end{artnotes}

\end{fmbox}% comment this for two column layout

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                          %%
%% The Abstract begins here                 %%
%%                                          %%
%% Please refer to the Instructions for     %%
%% authors on http://www.biomedcentral.com  %%
%% and include the section headings         %%
%% accordingly for your article type.       %%
%%                                          %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\nolinenumbers

\begin{abstractbox}

\begin{abstract} % abstract

\linenumbers

\parttitle{Background} %if any
We have applied machine learning techniques to automate the screening of biomedical literature prior to the manual curation of clinical databases such as performed by the Human Gene Mutation Database (HGMD). 
\parttitle{Methods} %if any
We have developed two machine learning models, one based on title and abstract data only, the other on the full text of the article. The models were built using a Natural Language Processing (NLP) pipeline and a logistic regression classifier. Our pipelines are implemented in Python and can be run using Docker.  They are made available to the wider community via GitHub (https://github.com/annacprice/nlp-bio-tools) and Docker Hub. 
\parttitle{Results}
During testing, both models performed well, correctly predicting HGMD relevant articles more than 93\% of the time and correctly discarding irrelevant articles more than 96\% of the time, with Matthews Correlation Coefficients (MCC's) of over 0.89. Evaluation of the finalised model using an unseen validation dataset demonstrated that the full text model correctly predicted HGMD-relevant articles more than 97\% of the time, an accuracy 9.5\% higher than that obtained with the title/abstract model.
\parttitle{Conclusions}
Through this work we have demonstrated that machine learning models can act as an effective pre-screen of biomedical literature, with the results indicating that a full text approach to screening biomedical literature is preferable to using just the title/abstract data.

\end{abstract}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                          %%
%% The keywords begin here                  %%
%%                                          %%
%% Put each keyword in separate \kwd{}.     %%
%%                                          %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{keyword}
\kwd{machine learning}
\kwd{text mining}
\kwd{natural language processing}
\end{keyword}

% MSC classifications codes, if any
%\begin{keyword}[class=AMS]
%\kwd[Primary ]{}
%\kwd{}
%\kwd[; secondary ]{}
%\end{keyword}

\end{abstractbox}
%
%\end{fmbox}% uncomment this for twcolumn layout

\end{frontmatter}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                          %%
%% The Main Body begins here                %%
%%                                          %%
%% Please refer to the instructions for     %%
%% authors on:                              %%
%% http://www.biomedcentral.com/info/authors%%
%% and include the section headings         %%
%% accordingly for your article type.       %%
%%                                          %%
%% See the Results and Discussion section   %%
%% for details on how to create sub-sections%%
%%                                          %%
%% use \cite{...} to cite references        %%
%%  \cite{koon} and                         %%
%%  \cite{oreg,khar,zvai,xjon,schn,pond}    %%
%%  \nocite{smith,marg,hunn,advi,koha,mouse}%%
%%                                          %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%% start of article main body
% <put your article body there>

%%%%%%%%%%%%%%%%
%% Background %%
%%
\section*{Background}
The ability to classify research articles at scale is an important pre-processing step for many databases seeking to curate and collate clinical articles to add value for research and healthcare. For example, the Human Gene Mutation Database (HGMD) \cite{stenson2017human} must screen many tens of thousands of clinical research articles each year in order to collate all known gene lesions responsible for human inherited disease, along with disease-associated variants and functional polymorphisms and thereby providing a unique and important resource for diagnostic healthcare and clinical research. Trained curators undertake the time-consuming and laborious task of classifying the articles themselves. Until now, the unstructured nature of clinical articles has prevented an initial automated screen of the literature. However, the task of determining HGMD-relevant articles provides an ideal subject for machine learning. 

Machine learning is a subfield of artificial intelligence. It uses algorithms to build models with the ability to automatically learn from data without explicit programming. Once built, these models are used to classify new data. Machine learning algorithms classify by inferring a function to map the input data to their discrete classes. The classification algorithms learn from training data and can be split into three main types: supervised \cite{kotsiantis2007supervised}, unsupervised \cite{hofmann2001unsupervised} and reinforcement \cite{sutton2018reinforcement}. Supervised algorithms are trained using a labelled training set (i.e. the classes of the data are already known). Unsupervised algorithms are trained using an unlabelled training set and try to infer a function that describes the underlying structure of the training dataset. Reinforcement machine learning algorithms use a trial and error approach to building models; they learn from feedback from interactions with an external environment.

Machine learning is applicable to many domains e.g. from facial recognition on phones  \cite{tang2013deep} to identifying disease-causing mutations from Next Generation Sequencing Data \cite{capriotti2011new}. Natural language processing (NLP) represents a field within machine learning that involves the analysis of natural language. One common use of NLP is in the classification of textual data such as required during spam email detection \cite{ruadulescu2014identification}. Early work in text classification started in the 1960s \cite{maron1961automatic}, with machine learning approaches gaining popularity in the 1990s \cite{joachims1998text, sebastiani2002machine}. NLP use has increased in recent years due to the growth in digital documents and computing power. 

Text documents can be classified by methods based on Naive Bayes \cite{yang1999re, rennie2003tackling, li1998classification}, k-nearest neighbours \cite{yang1999re, li1998classification}, decision trees \cite{li1998classification}, support vector machines  \cite{joachims1998text, tong2001support, zhang2001text}, neural networks \cite{lai2015recurrent, zhang2015character} and regression models \cite{zhang2001text, ifrim2008fast}. Comparisons between these different classifiers can be widely found in literature \cite{li1998classification, singh2016review}. A detailed review of text classification is provided by Sebastiani, 2002 \cite{sebastiani2002machine}. 

In recent years, there has been a marked increase in the text mining of biomedical literature. NLP has the potential to aid this research by extracting information from texts. For example, Wei, 2013  \cite{wei2013tmvar} uses text mining to extract sequence variants from biomedical literature. In another example, Li, 2015 \cite{li2015annotating} approaches the problem of using named entity recognition (NER) for recognising diseases and on how to extract information on chemical-induced disease relations. 

Increasingly, biomedical text mining has been applied to precision medicine, which aims to tailor medical care to the individual \cite{collins2015new}. For example, Liu, 2015 \cite{liu2015polysearch2} demonstrates an online text mining system for identifying relationships between different biomedical entities such as genes, drugs and toxins. Singhal, 2016 \cite{singhal2016text} describes a tool for extracting relationships among disease-related mutations relationships from literature to support work in precision medicine. A review of past work in biomedical text mining, and discussions on its future can be found in Huang, 2015 \cite{huang2015community}  and Gonzalez, 2015 \cite{gonzalez2015recent} respectively. 
 
In this study, we outline the development of machine learning classification models, to assist the HGMD in the curation of their database. We have built two supervised machine learning models that act to pre-screen clinical literature from PubMed, to predict whether articles contain reports of human disease-causing mutations which would therefore be relevant to the HGMD. This is an example of binary classification, where a positive class represents scientific articles that are relevant to the HGMD whilst a negative class represents papers that are not of interest. A logistic regression \cite{hosmer2013applied} algorithm is used to build our models. Through the use of these machine learning models we aim to improve the coverage of the database and save on curator time. Currently, HGMD curators manually screen PubMed articles based on the title and abstract text alone. We investigate two models, one based on title and abstract data only, the other on the full text of the article. In doing so we determine whether there is benefit in using a full text approach that provides more data. 


Although our models are built for HGMD use, our methods can be used to build models for the binary classification of any text based document. Our work is therefore relevant far beyond the immediate requirements of the HGMD. The ability to classify unstructured text documents at scale is increasingly important for healthcare, not least because it enables the incorporation of a much wider range of patient data into anonymised databases that can then serve as a resource for precision medicine research and healthcare. For example, much useful information is embedded within clinician letters, electronic health records, and clinical reports, that combined with other data types such as molecular data, bioimaging data, socio-economic data, and routine clinical data, would be useful for stratifying patients for more targeted healthcare. The use of machine learning techniques to classify clinical text, indeed any unstructured text has long been recognised, but with many algorithmic choices available, it is important to ensure that the technique used is appropriate and optimised for the task in hand. Furthermore, the unstructured nature of clinical records in general, and the clinical literature in particular, means that any machine learning application must be guided by relevant background clinical knowledge.
 

Our pipelines are freely available under an open-source licence and can be found on GitHub (https://github.com/annacprice/nlp-bio-tools). Our programs allow users to select different machine learning algorithms and vectorisers in order to build their own machine learning models. In addition, we have built ready-made docker images for each program, which can be found at Docker Hub (https://hub.docker.com/u/annacprice). The programs are provided in the hope that they will be beneficial to other researchers and act as a useful machine learning toolkit to benefit future research and healthcare delivery. 



\section*{Methods}
We prepared training, testing and validation datasets of research paper pdfs to train and evaluate our machine learning model. Each dataset contained papers categorised as either positive or negative. The positive papers, called the HGMD class, contained articles already present in the HGMD. The negative papers were subdivided into two further classes called COSMIC and RANDOM. The COSMIC class of papers are present in the Catalogue Of Somatic Mutations In Cancer (COSMIC) database \cite{forbes2014cosmic} and contain somatic and mitrochondrial mutations that are irrelevant to the remit of the HGMD. Papers labelled RANDOM class, were a random collection of papers from PubMed, known to be irrelevant to both the HGMD and COSMIC. 

To avoid training bias, \cite{provost2000machine} the training set was balanced with an equal number of positive and negative articles, with the negative articles further split equally between papers belonging to the COSMIC and RANDOM classes. 

The testing and validation datasets were separately prepared to assess how well our model performed on unseen data. The testing dataset acted as an early evaluation of our model in the latter stages of training. The validation dataset serves as our final test data containing completely `unseen' papers not used during the training process. The validation dataset was used on our finalised model to give an unbiased evaluation of the fit of our model. 

The size of the datasets for training, testing and the final validation set are given in Table 1. The articles for the training dataset were randomly selected from the total available dataset using the Unix command shuf. The remaining articles were then used as a testing dataset. A training dataset of 11,200 articles was used, equally spilt between 5,600 positive and negative examples of articles. We chose this size to avoid oversampling the smaller COSMIC dataset composed of 4,940 and 2,981 articles for the full text and title/abstract datasets respectively. 
 
It should be noted that the same articles were present in both the full text training set and the title/abstract training set. 

Our machine learning model was built in Python 2.7 and uses the Natural Language Toolkit, NLTK \cite{bird2004nltk}, scikit-learn \cite{pedregosa2011scikit} and pdfminer \cite{shinyama2014pdfminer} libraries. We produced three programs. The first is pdf2nlp which uses pdfminer to convert the pdf to utf-8 plain text and then passes the plain text through the NLP pipeline which uses NLTK. The second is mlpipe which uses scikit-learn to build the machine learning model. The final program is loadmodel which loads the saved machine learning model from mlpipe and uses it to evaluate new data. The scope of each program is detailed in Fig. 1. The full codebase can be found on GitHub at https://github.com/annacprice/nlp-bio-tools. Each program also has its own docker image, which can be used to spin up a docker container to run the program. 


The workflow for building and using our machine learning model is given in Fig 1. There are two paths to our workflow: one for training and one for prediction. The first two steps of the workflow are the same for both training and prediction. We first extracted the text from the pdf articles. The text was then passed through our NLP Pipeline. In training our classifier, we first selected the features that we wanted the classifier to fit to. Once extracted from the text, the features were used to build a term-document matrix. This was then passed to the classifier, along with the class labels for each document, for training. For the prediction workflow, we fit to the features selected during training, and the resulting term-document matrix was passed to the trained model to output the predicted class for each document. 


\subsection*{Natural Language Processing (NLP) Pipeline}
For text classification problems, the main issue is how we choose to represent our document, as the model's accuracy depends greatly on the quality of the input data. The aim of the NLP pipeline in pdf2nlp is to produce a clean and concise representation of each document. Each stage of the NLP pipeline is given in Fig. 2. 

The final output of the NLP pipeline is a collection of stemmed words, or tokens, for each document. Stemming reduces words to their base form. The use of stemming in information retrieval tasks solves the problem of vocabulary mismatch, ensuring variant words which have the same stem (and therefore similar meanings) are counted as the same token. This improves the precision and recall of a machine learning system \cite{aphinyanaphongs2014comprehensive}. It also has the added benefit of shrinking the size of the corpus, reducing both storage space and the size of the term-document matrix which needs to be calculated. 

We stemmed our tokens using the snowball stemmer \cite{porter2001snowball}, part of the NLTK  \cite{bird2004nltk}. We then used feature selection to select which tokens the classifier would fit to, and built a term-document matrix, which acted as a numerical representation of the abundance of features in each document. 


\subsection*{Feature Selection}

Feature selection defines which tokens our algorithm fits to. The selected tokens need to represent our data well and contain enough information for the model to predict the output accurately. Feature selection allows the number of tokens for text classification problems to be reduced by the tens of thousands. This is important because when the input feature vectors are too large this can result in the model struggling to fit to the data because of high variance. 

Many methods for feature selection exist  \cite{forman2003extensive}, including: bag-of-words (i.e. basic term frequency counting) \cite{harris1954distributional, bekkerman2004using}, tf-idf (term frequency-inverse document frequency) \cite{robertson2004understanding, ramos2003using}, information gain \cite{lee2006information}  and chi-square  \cite{chen2011using}. A study of the effectiveness of different feature selection methods can be found in Forman, 2003  \cite{forman2003extensive}. 
 
We used tf-idf to select our features. From the results of the tf-idf weighting, we selected the top 600 features. In our implementation of tf-idf, we included binary terms alongside singular tokens when selecting the top features for fitting. The tf-idf weight is comprised of two terms: the term-frequency (TF) and the inverse document frequency (IDF). The term frequency is simply the number of times a term appears in an individual document. This is weighted by the inverse document frequency which weighs down highly frequent terms (such terms are often insignificant words such as auxiliary and modal verbs). The tf-idf weighting was implemented using scikit-learn. To reduce document-length bias each term's tf-idf score is proportionally scaled by dividing by the squared average of the document. 

Note that, because features were selected by applying tf-idf to the training sets for full text and title/abstract separately, the full text and title/abstract corpuses were inevitably fitted to different features. 

\subsection*{Term-Document Matrix}
Hashing creates a numerical representation of the features to pass to the classifier. As we reduced our features to only 600, we simply used a direct mapping to build a term-document matrix (or hash table), that represented the weight of each feature in each document. For example, for a matrix $X$, the matrix element $x_{ij}$ represents the weight of the term/feature $j$ in the document $i$. For larger corpuses it is often necessary to implement methods such as the hashing trick to scale-up machine learning algorithms \cite{weinberger2009feature}. Note that the same hash table is used for training and prediction, i.e. we fit to the same features. 

\subsection*{Building a machine learning model}
Machine learning algorithms generalise from training data to make accurate observations on new, unseen, data. Algorithm choice is therefore crucial. Several models should be tested in order to find which works best. For text classification, algorithms such as Naive Bayes, logistic regression and nearest-neighbour models have been shown to perform well \cite{sebastiani2002machine}. In the preliminary stages of building our model, we investigated all three algorithms and found logistic regression to be most effective (Fig. 3). However, the difference between the classifiers is marginal, suggesting that in our case feature selection is more important than choice of model.  

Formally, our training set is given by $\mathcal{D}=\{(\mathbf{x}_1, y_1), (\mathbf{x}_2, y_2), ... , (\mathbf{x}_n, y_n)\}$ where $n$ is the total number of documents. Each document is represented by a feature vector $\mathbf{x}_i=(x_{i1},...,x_{if})^T$, where $f$ is the total number of features and $i=1,..., n$. Each feature vector belongs to a binary class $y_i\ \epsilon \ \{0,1\}$, where 0 is the negative class (not HGMD) and 1 is the positive class (HGMD). If $X$ is the set of all documents x, then the goal of the machine learning algorithm is to infer this mapping $f : X  \rightarrow \{0,1\}$.


Logistic regression estimates the log odds of an event with a binary outcome. Defining the parameter vector $\boldsymbol{\beta}=(\beta_1,\beta_2, ...., \beta_f)$, then the probability of a document belonging to the positive class (1) is 
 
	\begin{equation}
	p(y=1 | \mathbf{x}_i ;\ \boldsymbol{\beta})=\frac{1}{1+ exp(-\boldsymbol{\beta}^{T} .\ \mathbf{x}_i)}
	\end{equation}
	
For logistic regression, learning the inferred mapping function is equivalent to finding the parameter vector which maximises the log-likelihood function for the training set. The log-likelihood function is given by \cite{albert1984existence, ifrim2008fast}
 
 \begin{equation}
 l(\boldsymbol{\beta})=\sum^n_{i=1}[y_i .\ \boldsymbol{\beta}^T .\ \mathbf{x}_i-log(1+exp(\boldsymbol{\beta}^T .\ \mathbf{x}_i))]
 \end{equation}
 
Once the algorithm has determined the best parameter vector, the model can be saved and used to classify unseen documents.

\section*{Results}
We compare the results from the title/abstract and full text classifiers by analysing receiver operator characteristic (ROC) curves, percentage accuracy, Matthews correlation coefficients (MCC) \cite{matthews1975comparison}, precision and recall, and F1-score. The ROC curves for the title/abstract model are given in Fig. 4 for both the (a) training and (b) testing datasets. The ROC is a plot of the true positive rate (TPR) vs the false positive rate (FPR) for various predictions of the model at different thresholds between 0 and 1. The TPR, or sensitivity, is defined as the number of true positives (TP) divided by the sum of the number of true positives and the number of false negatives (FN), hence $TP/(TP +FN)$. The false positive rate (FPR) is defined as the number of false positives (FP) divided by the sum of the number of false positives and the number of true negatives (TN), $FP/(FP + TN)$. FPR is also sometimes referred to as $1 - specificity$. On average, a skilled model will assign a higher probability to a random true positive occurrence than a true negative (curves bow up to the top left of the plot). The area under the ROC curve (AUC) indicates how likely it is that the model will predict a higher probability for the true positive cases than the true negative cases. The larger the AUC, the greater the skill of the model, with an AUC of 1.0 indicating a perfect model. 

Fig. 4a shows the k-fold cross validation of the training set for the title/abstract dataset. In the early stages of training, we used k-fold cross-validation as a resampling method to estimate the skill of the model on new data, using just the training dataset. The dataset was randomly divided into k groups. The classifier then used (k-1)/k of the set for training holding back 1/k for testing, iterating through the entire dataset so each fraction of the dataset was used as a testing set once and was used to train the model (k-1) times. The k groups are constant for the entire process. We chose a value of k=10, as it has been shown to give test error-estimates that have neither an excessively high bias nor a high variance \cite{kohavi1995study}. 

After the initial stages of training, we used a testing dataset to evaluate the performance of the model. Fig. 4b shows the ROC for the testing dataset. Note that for Fig. 4b we have randomly selected an equal number of papers from the positive and negative datasets (with the negative dataset further equally split between COSMIC and RANDOM) to produce a balanced dataset required for ROC curves. 

The corresponding ROC curves for the full text dataset are not shown as they produce almost identical plots to those shown in Fig. 4. For the training set, both our title/abstract and full text models had AUC values of over 0.98 for each fold, indicating that the models are overfitting to the training set. However, despite this the models still performed well on evaluation of the testing set, with both models again achieving an AUC of over 0.98. The percentage accuracy for each class of the testing set is given in Table 2. The title/abstract and full text models show similar performance, with the COSMIC datasets showing around 97\% accuracy and the RANDOM datasets showing around 96\% accuracy. The title/abstract model showed slight improvements for the HGMD class, with a 94.5\% accuracy, a 1.4\% improvement over the full text model. However, bias is possible as the datasets were built by screening only the title/abstract data and not the full text of the article. Hence, one might expect to see improvements in the title/abstract model versus the full text model. 

The predicted probability for a binary classification problem can be interpreted with a threshold defining whether an article will be classified as negative or positive. The default threshold in machine learning problems is 0.5, with probability in the range of $[0, 0.5)$ indicating that the article belongs to the negative class (i.e. not HGMD), and a probability in the range $[0.5, 1.0]$ indicating that the article belongs to the positive class (belongs to the HGMD). By changing the threshold, we can tune the model for different rates of false positives vs. false negatives. 

Varying the threshold is important in problems where either false positives or false negatives are more important than the other, or in models where there is disproportionately one type of error over the other. We calculated the MCC for various thresholds between 0.4 and 0.6 and the results are presented in Table 3. The MCC has a range from -1 to 1, where a value of -1 would indicate a classifier which is completely wrong and a value of 1 would indicate a completely correct classifier. The MCC acts as a balanced measure of the fit of the model and can be used for datasets were the classes are different sizes. For both models, the training and testing datasets all show an MCC of at least 0.88 at each threshold, indicating that the models are fitting well to both the training and the testing datasets. As the difference between the MCC for different thresholds is negligible we kept the default threshold of 0.5.  

Table 4 shows the precision, recall, and the F1 score for the training and testing datasets. For the positive class, the precision is the ratio of true positives to the total predicted positives $TP/(TP +FP)$. The recall (or sensitivity) is then the ratio of true positives to the total of true positives and false negatives $TP/(TP +FN)$. A high precision and low recall for the positive class would indicate a greater number of false negatives than false positives. A low precision and high recall for the negative class would also indicate the same. The F1-score acts as a weighted average of the precision and recall $(2R + 2P)/(R + P)$. The precision and recall act as useful evaluations of imbalanced datasets (i.e. datasets with different class sizes) such as our testing dataset.

For the training dataset, the precision and recall are around the same for both models. In particular, the precision and recall for each model (and hence the F1-score) are almost equal, indicating an equal number of false positives and false negatives. This is confirmed by the confusion matrices (a) and (b) in Fig. 5. The high precision and recall of greater than 0.95 indicates that the model has few false results. 

For the testing datasets, both the title/abstract and full text model show a difference in the values for precision and recall. For the positive class, the precision is greater than the recall, and for the negative set the recall is greater than the precision. This indicates that, for the testing set, there are a greater number of false negatives than false positives. This is confirmed by the confusion matrices (c) and (d) in Fig. 5. For the full text dataset, the number of false negatives is almost 3 times greater than the number of false positives, and for the title/abstract dataset the number of false negatives is almost 2 times greater. 

In terms of percentage accuracy, the title/abstract and full text models showed very similar performance. However despite this, there were differences in the features the models fitted to. In total, of the 600 features that each model fitted to, the models shared 477 features in common. Fig. 6 shows the top 20 features as ranked by the logistic regression classifier for both positive and negative class for the two models. For the top 20 features of the positive class, the two models share 15 features. For the top 20 features of the negative class, the two models share 14 features

Overall, the full text model did show improved feature selection over the title/abstract model i.e. fitting to more significant tokens such as intron and splice, whereas the title/abstract model was more prone to fitting to less significant verbs such as cause and detect. This indicates the possible advantages of the full text approach for building more complex models and implies that the full text screening approach is more likely to produce accurate models than the screening of the title/abstract alone. 

\subsection*{Validation Dataset}
Once we had finished training and testing the two models, we created a finalised model for each by passing all of the available training and testing data to the classifier for training, fitting to the same 600 features as before. We then evaluated the performance of these models using a final unseen validation dataset, containing papers which were not used for either training or testing. The validation dataset consists of papers from the HGMD and RANDOM negative papers from PubMed. 

The percentage accuracy of classification for the validation dataset is given in Table 5. For the RANDOM class, the title/abstract and full text model are both around 94\% accurate. However, for the HGMD class the percentage accuracy is 97.3\% for the full text model, but only 87.8\% for the title/abstract model. This indicates that the use of full text data as opposed to just the title/abstract of a paper, is advantageous and provides better features for the machine learning algorithm to fit to. 

The precision, recall and f1-score for the validation dataset are given in Table 6. The precision and recall for both models are high, indicating there are few false results. For the title/abstract model, the precision and recall are almost equal in both the HGMD and RANDOM classes. This indicates a similar number of false positives (FP) and false negatives (FN). This result is confirmed by the confusion matrix in Fig.7a which shows the number of FP and FN are 12 and 11 respectively. The precision and recall are greater for the RANDOM class, indicating the higher accuracy for the RANDOM class. The full text model exhibits a significant difference between the precision and recall for the HGMD class, with a precision of 0.84 and a recall of 0.97. This indicates a far greater number of FP than FN, which is confirmed by the confusion matrix in Fig. 7b. The number of FP is 21, but the number of FN is only 3. 

\section*{Discussion}
Aside from solving our immediate challenge of how to automate the pre-selection of research articles for further investigation by HGMD curators, we have outlined the general methods for building a supervised machine learning model for any binary classification of biomedical research articles. Indeed, our approach can be used to classify many types of text document and this makes it a potentially powerful tool for rapidly screening all sorts of healthcare data. 

The ability to classify unstructured text documents at scale enables routinely collected patient data to be incorporated into databases that can serve as a resource for precision medicine research and healthcare. Binary classification methods of the kind we showcase here will therefore play an important role in helping to create clinical data resources of the future. 

Our pipelines have been uploaded to GitHub (https://github.com/annacprice/nlp-bio-tools) and Docker Hub (https://hub.docker.com/u/annacprice) with the hope they provide useful to other researchers. A short wiki is also provided on GitHub (https://github.com/annacprice/nlp-bio-tools/wiki) to assist researchers in using the software to build their own machine learning models. 


\section*{Conclusions}
We produced two models for classifying papers into the HGMD, both based on logistic regression: one for title/abstract data and one for full text data. During testing, both models performed well, correctly predicting HGMD-relevant articles more than 93\% of the time and discarding irrelevant articles more than 96\% of the time, with MCC's of over 0.89. However, on analysis of features that both models were fitted to it was found that the full text model was fitting to more significant features, indicating that using the full text data should provide greater accuracy. This was confirmed on evaluation of the final model using a validation dataset, the full text model showed significant improvement over the title/abstract model, correctly predicting HGMD-relevant articles more than 97\% of the time, showing a 9.5\% improvement on the title/abstract model. This along with the more significant features, indicates that a full text approach would be preferable when building more sophisticated models.


\section*{List of abbreviations}
HGMD = Human Gene Mutation Database\\
NLP = Natural Language Processing\\
COSMIC = Catalogue Of Somatic Mutations In Cancer\\
NLTK = Natural Language ToolKit\\
TF-IDF = Term Frequency-Inverse Document Frequency\\
ROC = Receiver Operator Characteristic\\
MCC = Matthews Correlation Coefficients\\
TP = True Positive\\
FP = False Positive\\
TN = True Negative\\
FN = False Negative\\
TPR = True Positive Rate\\
FPR = False Positive Rate\\

\nolinenumbers

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                          %%
%% Backmatter begins here                   %%
%%                                          %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section*{Declarations}
\begin{backmatter}

\section*{Ethics approval and consent to participate}
Not applicable

\section*{Consent for publication}
Not applicable

\section*{Availability of data and materials}
The software used to build the machine learning models and the finalised machine learning models can be found at \url{https://github.com/annacprice/nlp-bio-tools}.

\section*{Competing interests}
 The authors declare that they have no competing interests.
  
\section*{Funding}
AP is funded by the Supercomputing Wales project, which is part-funded by the European Regional Development Fund (ERDF) via Welsh Government. DNC's and MM's work with the HGMD is financially supported by the QIAGEN plc through a License Agreement with Cardiff University.  KEA is funded by Welsh Government through Health and Care Research Wales via Wales Gene Park, a research infrastructure support group embedded within Cardiff University.

\section*{Authors' contributions}
MM, DNC and KEA conceived and planned the project. MM prepared the datasets used to build the machine learning models. AP wrote the software, built the machine learning models and performed the analysis on the models' performance. AP, KEA and MM wrote the manuscript. All authors have read and approved the manuscript.

\section*{Acknowledgements}
Not applicable

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                  The Bibliography                       %%
%%                                                         %%
%%  Bmc_mathpys.bst  will be used to                       %%
%%  create a .BBL file for submission.                     %%
%%  After submission of the .TEX file,                     %%
%%  you will be prompted to submit your .BBL file.         %%
%%                                                         %%
%%                                                         %%
%%  Note that the displayed Bibliography will not          %%
%%  necessarily be rendered by Latex exactly as specified  %%
%%  in the online Instructions for Authors.                %%
%%                                                         %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% if your bibliography is in bibtex format, use those commands:
\bibliographystyle{bmc-mathphys} % Style BST file (bmc-mathphys, vancouver, spbasic).
\bibliography{bmc_article}      % Bibliography file (usually '*.bib' )
% for author-year bibliography (bmc-mathphys or spbasic)
% a) write to bib file (bmc-mathphys only)
% @settings{label, options="nameyear"}
% b) uncomment next line
%\nocite{label}

% or include bibliography directly:
% \begin{thebibliography}
% \bibitem{b1}
% \end{thebibliography}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                               %%
%% Figures                       %%
%%                               %%
%% NB: this is for captions and  %%
%% Titles. All graphics must be  %%
%% submitted separately and NOT  %%
%% included in the Tex document  %%
%%                               %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%
%% Do not use \listoffigures as most will included as separate files

\section*{Figure legends}
  \begin{figure}[h!]
  \caption{\csentence{}
      Workflow summarising the three programs used to run our analyses, showing the relationships between the training and prediction workflows. Each dotted box represents a separate program.}
      \end{figure}


\begin{figure}[h!]
  \caption{\csentence{}
      Workflow showing the natural language processing pipeline implemented in pdf2nlp, using NLTK. Text is separated into individual tokens and passed through a US-English to British-English dictionary. We want our model to fit only to English words, so tokens such as punctuation, numbers, and gene symbols are removed. Common stopwords ( e.g. a, and, or, the) are also removed as they have no semantic value. Words that are common to all academic articles (e.g. department, school, abstract, references) are also removed, as well as journal names and common names. Finally, the remaining tokens are stemmed.}
      \end{figure}
      
  
\begin{figure}[h!]
  \caption{\csentence{}
      ROC curves for the testing dataset for: logistic regression, 5-nearest-neighbours, Multinomial Naive Bayes and Bernoulli Naive Bayes classifiers.}
      \end{figure}
      
\begin{figure}[h!]
  \caption{\csentence{}
      ROC curves for the title/abstract dataset for (a) 10-fold cross-validation of the training dataset and (b) testing dataset.}
      \end{figure}

\begin{figure}[h!]
  \caption{\csentence{}
      Confusion matrices showing the number of classified articles which were True Negatives (TN), False Positives (FP), False Negatives (FN) and True Positives (TP) for the (a) full text training dataset, (b) title/abstract training dataset, (c) full text testing dataset and (d) title/abstract testing dataset}
      \end{figure}

\begin{figure}[h!]
  \caption{\csentence{}
      Top 20 positive and negative features as ranked by the logistic regression classifier for the (a) title/abstract and (b) full text model. Blue indicates the positive class and red indicates the negative class.}
      \end{figure}
      
 \begin{figure}[h!]
  \caption{\csentence{}
      Confusion matrices for the (a) title/abstract validation dataset and (b) full text validation dataset.}
      \end{figure}
      
      
      
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                               %%
%% Tables                        %%
%%                               %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%% Use of \listoftables is discouraged.
%%

\section*{Tables}

\begin{table}[!h]
\caption {\label{tab:table1} Size of datasets available for training, testing and final validation} 
\centering
	\begin{tabular}{lcccccc}
 		& \multicolumn{2}{c}{TRAINING} & \multicolumn{2}{c}{TESTING}   & \multicolumn{2}{c}{VALIDATION}  \\ 
		\cline{2-7}   
       		 & Title/Abstract   & Full Text  & Title/Abstract & Full Text & Title/Abstract & Full Text \\ \hline            
 		HGMD (+)   &  5600   &  5600  &  15697  &  26934 &  90  &  112\\
 		Cosmic (-)  &  2800   &    2800 &  181  &  2140 &  -  &  -\\
 		Random (-) &  2800   &    2800  & 11714  &  18016  &  224  &  328      
	\end{tabular}
\end{table}

\begin{table}[!h]
\caption {\label{tab:table2} Percentage accuracy of classification of the testing datasets for each class for a threshold of 0.5} 
\centering
	\begin{tabular}{lcc}
	\hline
	\multicolumn{3}{l}{\bf{TESTING}}\\
	\hline
	& Title/Abstract & Full Text \\ \hline            
	HGMD (+)    &  94.5 &  93.1\\
	COSMIC (-) &  97.2 &  97.2\\
	RANDOM (-) & 96.1  &  96.7      
	\end{tabular}
\end{table}


\begin{table}[!h]
\caption {\label{tab:table3} Matthews correlation coefficient at different thresholds for the training and testing datasets} 
\centering
	\begin{tabular}{lcccc}
 	&  \multicolumn{2}{c}{TRAINING}   & \multicolumn{2}{c}{TESTING}  \\ 
	\cline{2-5}   
       	THRESHOLD & Title/Abstract & Full Text & Title/Abstract & Full Text\\ \hline            
 	0.4 &  0.918  &  0.931 &  0.909  &  0.898\\     
	 0.45 &  0.922  &  0.933 &  0.905  &  0.897\\
	0.5 &  0.921  &  0.933 &  0.903  &  0.893\\
	0.55 &  0.918  &  0.928 &  0.898 &  0.887\\
	 0.6 &  0.909  &  0.922 &  0.909  &  0.881\\
	\end{tabular}
\end{table}


\begin{table}[!h]
\caption {\label{tab:table4} Precision, recall and f1-score for the training and testing datasets for a threshold of 0.5} 
\centering
	\begin{tabular}{lcccccc}
	\hline
	\multicolumn{7}{l}{\bf{TRAINING}}\\
	\hline
 	&  \multicolumn{3}{c}{Title/Abstract} & \multicolumn{3}{c}{Full Text}  \\ 
	\cline{2-4}    \cline{5-7}
       	CLASS & Precision & Recall & F1-score  & Precision & Recall & F1-score \\ \hline            
 	Positive (HGMD) &  0.97  &  0.95 &  0.96  &  0.97  &  0.97 &  0.97\\     
	Negative (Not HGMD) &  0.96  &  0.97 & 0.96 &  0.97  &  0.97 &  0.97\\
	 \\
	\hline
	 \multicolumn{7}{l}{\bf{TESTING}}\\
	\hline
 	&  \multicolumn{3}{c}{Title/Abstract} & \multicolumn{3}{c}{Full Text}  \\ 
	\cline{2-4}    \cline{5-7}
       	 CLASS & Precision & Recall & F1-score  & Precision & Recall & F1-score \\ \hline            
 	Positive (HGMD) & 0.97 &  0.94 &  0.96  & 0.97  &  0.93 &  0.95\\     
	Negative (Not HGMD) &  0.93  &  0.96 &  0.95 &  0.91  &  0.97 &  0.94\\
	\end{tabular}
\end{table}

\begin{table}[!h]
\caption {\label{tab:table5} Percentage accuracy of classification for the validation dataset for each class for a threshold of 0.5} 
\centering
	\begin{tabular}{lcc}
	\hline
	\multicolumn{3}{l}{\bf{VALIDATION}}\\
	\hline
	& Title/Abstract & Full Text \\ \hline            
	HGMD (+)    &  87.8\% &  97.3\%\\
	RANDOM (-) &  94.6\%  &  93.6\%
	\end{tabular}
\end{table}

\begin{table}[!h]
\caption {\label{tab:table6} Precision, recall and f1-score for the validation datasets for a threshold of 0.5} 
\centering
	\begin{tabular}{lcccccc}
	\hline
	\multicolumn{7}{l}{\bf{VALIDATION}}\\
	\hline
 	&  \multicolumn{3}{c}{Title/Abstract} & \multicolumn{3}{c}{Full Text}  \\ 
	\cline{2-4}    \cline{5-7}
       	CLASS & Precision & Recall & F1-score  & Precision & Recall & F1-score \\ \hline        
	Positive (HGMD) & 0.87  &  0.88 &  0.87  &  0.84  &  0.97 &  0.90\\         
	Negative (Not HGMD) &  0.95  &  0.95 & 0.95 &  0.99  &  0.94 &  0.96\\
	\end{tabular}
\end{table}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                               %%
%% Additional Files              %%
%%                               %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%\section*{Additional Files}
%  \subsection*{Additional file 1 --- Sample additional file title}
%    Additional file descriptions text (including details of how to
%    view the file, if it is in a non-standard format or the file extension).  This might
%    refer to a multi-page table or a figure.
%
%  \subsection*{Additional file 2 --- Sample additional file title}
%    Additional file descriptions text.


\end{backmatter}

\end{document}
