%% BioMed_Central_Tex_Template_v1.06
%%                                      %
%  bmc_article.tex            ver: 1.06 %
%                                       %

%%IMPORTANT: do not delete the first line of this template
%%It must be present to enable the BMC Submission system to
%%recognise this template!!

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                     %%
%%  LaTeX template for BioMed Central  %%
%%     journal article submissions     %%
%%                                     %%
%%          <8 June 2012>              %%
%%                                     %%
%%                                     %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                                                 %%
%% For instructions on how to fill out this Tex template           %%
%% document please refer to Readme.html and the instructions for   %%
%% authors page on the biomed central website                      %%
%% http://www.biomedcentral.com/info/authors/                      %%
%%                                                                 %%
%% Please do not use \input{...} to include other tex files.       %%
%% Submit your LaTeX manuscript as one .tex document.              %%
%%                                                                 %%
%% All additional figures and files should be attached             %%
%% separately and not embedded in the \TeX\ document itself.       %%
%%                                                                 %%
%% BioMed Central currently use the MikTex distribution of         %%
%% TeX for Windows) of TeX and LaTeX.  This is available from      %%
%% http://www.miktex.org                                           %%
%%                                                                 %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%% additional documentclass options:
%  [doublespacing]
%  [linenumbers]   - put the line numbers on margins

%%% loading packages, author definitions

\documentclass[twocolumn]{bmcart}% uncomment this for twocolumn layout and comment line below
%\documentclass{bmcart}

%%% Load packages
\usepackage{amsthm,amsmath}
%\RequirePackage{natbib}
%\RequirePackage[authoryear]{natbib}% uncomment this for author-year bibliography
%\RequirePackage{hyperref}
\usepackage[utf8]{inputenc} %unicode support
%\usepackage[applemac]{inputenc} %applemac support if unicode package fails
%\usepackage[latin1]{inputenc} %UNIX support if unicode package fails
\usepackage{algorithm,algorithmic}
\usepackage{multirow}
\usepackage{flushend}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                             %%
%%  If you wish to display your graphics for   %%
%%  your own use using includegraphic or       %%
%%  includegraphics, then comment out the      %%
%%  following two lines of code.               %%
%%  NB: These line *must* be included when     %%
%%  submitting to BMC.                         %%
%%  All figure files must be submitted as      %%
%%  separate graphics through the BMC          %%
%%  submission process, not included in the    %%
%%  submitted article.                         %%
%%                                             %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


\def\includegraphic{}
\def\includegraphics{}



%%% Put your definitions there:
\startlocaldefs
\endlocaldefs


%%% Begin ...
\begin{document}

%%% Start of article front matter
\begin{frontmatter}

\begin{fmbox}
\dochead{Research}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                          %%
%% Enter the title of your article here     %%
%%                                          %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\title{BRS$_c$S: A Hybrid Recommendation Model Fusing Multi-source Heterogeneous Data}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                          %%
%% Enter the authors here                   %%
%%                                          %%
%% Specify information, if available,       %%
%% in the form:                             %%
%%   <key>={<id1>,<id2>}                    %%
%%   <key>=                                 %%
%% Comment or delete the keys which are     %%
%% not used. Repeat \author command as much %%
%% as required.                             %%
%%                                          %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\author[
   addressref={aff1},                   % id's of addresses, e.g. {aff1,aff2}
   corref={aff1,aff2,aff3,aff4},                       % id of corresponding address, if any
   %noteref={n1},
   email={zhyji@bjtu.edu.cn}   % email address
]{\inits{zj}\fnm{ZHENYAN} \snm{JI}}
\author[
   addressref={aff1},
]{\inits{cy}\fnm{CHUN} \snm{YANG}}
\author[
   addressref={aff2},
]{\inits{hs}\fnm{HUIHUI} \snm{WANG}}
\author[
   addressref={aff3},
]{\inits{JE}\fnm{JOS{\'E} ENRIQUE  \snm{ARMEND{\'A}RIZ-I{\~N}IGO}}}
\author[
   addressref={aff4},
]{\inits{MA}\fnm{MARTA} \snm{ARCE-URRIZA}}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                          %%
%% Enter the authors' addresses here        %%
%%                                          %%
%% Repeat \address commands as much as      %%
%% required.                                %%
%%                                          %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\address[id=aff1]{School of Software Engineering, Beijing Jiaotong University, Beijing 100044, China}
\address[id=aff2]{Department of Engineering, Jacksonville University, Jacksonville, FL 32211, USA
}
\address[id=aff3]{Department of Statistics, Computer Science and Mathematics, Public University of Navarre, 31006 Pamplona, Spain}
\address[id=aff4]{Department of Business Management, Public University of Navarre, 31006 Pamplona, Spain}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                          %%
%% Enter short notes here                   %%
%%                                          %%
%% Short notes will be after addresses      %%
%% on first page.                           %%
%%                                          %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{artnotes}
%\note{Sample of title note}     % note to the article
%\note[id=n1]{This work is supported by the National Key Research and Development Program of China under Grant No. 2018YFC0831903 and  Major Project of National Natural Science Foundation of China  under Grant No. 51935002.} % note, connected to author
\end{artnotes}

%\end{fmbox}% comment this for two column layout

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                          %%
%% The Abstract begins here                 %%
%%                                          %%
%% Please refer to the Instructions for     %%
%% authors on http://www.biomedcentral.com  %%
%% and include the section headings         %%
%% accordingly for your article type.       %%
%%                                          %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{abstractbox}

\begin{abstract} % abstract
Recommendation systems are often used to solve the problem of information overload on the Internet. Many types of data can be used for recommendation, and fusing different types of data can make recommendation more accurate. Most existing fusion recommendation models simply combine the recommendation results from different data instead of fully fusing multi-source heterogeneous data to make recommendations. Furthermore, users' choices are usually affected by their direct and even indirect friends' preferences. This paper proposes a hybrid recommendation model BRS$_c$S (an acronym for BPR-Review-Score-Social). It fully fuses social data, score, and review together, uses improved BPR model to optimize the ranking, and trains them in a joint representation learning framework to get the top-N recommendations. User trust model is used to introduce social relationships into the rating and review data, PV-DBOW model is used to process the review data, and fully connected neural network is used to process the rating data. Experiments on Yelp public dataset show that the BRS$_c$S algorithm proposed outperforms other recommendation algorithms such as BRS$_c$, UserCF, HRS$_c$. BRS$_c$S model is also scalable and can fuse new type of data easily.
\end{abstract}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                          %%
%% The keywords begin here                  %%
%%                                          %%
%% Put each keyword in separate \kwd{}.     %%
%%                                          %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{keyword}
\kwd{Multi-source heterogeneous data}
\kwd{recommendation model}
\kwd{social network}
\end{keyword}

% MSC classifications codes, if any
%\begin{keyword}[class=AMS]
%\kwd[Primary ]{}
%\kwd{}
%\kwd[; secondary ]{}
%\end{keyword}

\end{abstractbox}
%
\end{fmbox}% uncomment this for twcolumn layout

\end{frontmatter}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                          %%
%% The Main Body begins here                %%
%%                                          %%
%% Please refer to the instructions for     %%
%% authors on:                              %%
%% http://www.biomedcentral.com/info/authors%%
%% and include the section headings         %%
%% accordingly for your article type.       %%
%%                                          %%
%% See the Results and Discussion section   %%
%% for details on how to create sub-sections%%
%%                                          %%
%% use \cite{...} to cite references        %%
%%  \cite{koon} and                         %%
%%  \cite{oreg,khar,zvai,xjon,schn,pond}    %%
%%  \nocite{smith,marg,hunn,advi,koha,mouse}%%
%%                                          %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%% start of article main body
% <put your article body there>

%%%%%%%%%%%%%%%%
%% Background %%
%%
\section{Introduction}
With the development of information technology, how to efficiently and quickly find valuable information from massive data has become a major challenge for users. In order to solve the problem of Internet information overload and enable users to quickly obtain interested information, recommendation system came into being. The recommendation system essentially abstracts user's interest characteristics from a bunch of disorganized raw data, and mines user's preferences to recommend different items or services\cite{1}. Currently, the recommendation system has been successfully applied into many fields, including social networks (Face-book, Twitter), e-commerce (Amazon, Alibaba, Netflix), information retrieval (Google, Baidu, Yahoo)\cite{2,3,4}.

In recent years, deep learning has been widely used in the engineering field\cite{5}. It has achieved better results than traditional machine learning in the fields of image recognition, speech recognition, and natural language processing\cite{6}. Compared with traditional shallow machine learning, deep learning has excellent ability to automatically extract abstract features, avoid complex feature engineering, and overcome the heterogeneity of multiple data. It has achieved ideal results in the representation learning of specific data such as images and text\cite{7}. At the same time, Internet has accumulated a wealth of information sources, such as ratings, reviews, images, which can reflect users' preferences from different aspects. It has become a trend to fuse heterogeneous data sources for recommendation.

Fusion of multi-source heterogeneous data recommendation models can be divided into models based on recommendation algorithm fusion and ones based on data feature fusion. The fusion models based on recommendation algorithm just select appropriate recommendation algorithms according to the data type, and combine them by weighting, cascading, mixing, etc. When adding a new heterogeneous data, the entire model needs to be redesigned, so the scalability of the model is poor. Recently, due to the excellent effects of deep learning in representation learning\cite{8}, research based on data feature fusion has received extensive attention. The fusion model based on data feature refers to merging features of heterogeneous data by means of averaging or concatenation\cite{9}. When adding a new heterogeneous data to the model, there is no need to redesign and train the original model so that the model is flexible and scalable. 

Based on these, considering the impact of users' direct and indirect friends on users' decision, this paper proposes a hybrid model based on social relationships that fuses multi-source heterogeneous data. The main contributions are as follows:

\begin{itemize}
\item This paper introduces social network into the recommendation algorithm. Not only the direct friends' influence on users' decision is considered, but also the indirect friends' influence.
\item A joint representation learning model that fuse scores, reviews, social relations is proposed. It fuses different types of data from the data source perspective rather than combining the recommendation results from different recommendation models.
\item Experiments are performed to compare the proposed model BRS$_c$S with other recommendation algorithms such as BS$_c$, UserCF, BRS$_c$, HRS$_c$. BRS$_c$S performs better than other recommendation models in terms of Precision, Recall and HT.
\end{itemize}

In this paper, section 2 introduces the related work of recommendation algorithms. Section 3 illustrates our hybrid recommendation model based on deep learning algorithm. The algorithms of different types of data are analyzed, and the objective function of fusing comments, scores and social information is derived. Section 4 proves the effectiveness of adding social network information on the recommendation results and compares the proposed model with other recommendation algorithms by experiments. Section 5 concludes the paper and discusses the future work.

\section{Related works}

\subsection{Content-based Recommendation}

A content-based recommendation algorithm analyzes users' preferences for the items and recommends items, which has similar features to the items that the users like, to them. Based on the deep structured semantic model(DSSM), Elkahky\cite{10} proposed a multi-view deep neural network to learn the features of users and items separately. The recommended items were determined by calculating the similarity between users and items. Zheng\cite{11} proposed a deep collaborative neural network (Deep CoNN) model that uses two parallel neural networks to learn the features of comments, and then construct an interaction layer on the two neural networks to predict the user's score. Based on DSSM, Xu\cite{12} proposed label-based item recommendation, which inputs user information and item information related to the label into two deep neural networks respectively, makes recommendation by calculating the similarities of abstract features between the user and the item. Seo\cite{13} presented an attention-based Convolutional Neural Network(CNN) model that combines review and ratings for product recommendation.

Deep learning can effectively alleviate the cold start problem of new projects. At the same time, it can integrate the feature extraction and recommendation process into a unified framework, but it cannot find new interesting items for users, it only can find similar items to users. 

\subsection{Collaborative Filtering Algorithm}

Collaborative filtering algorithm is the most widely used algorithm in recommendation system. The main idea is to find a certain similarity between users or items, and use this similarity to make recommendations for users\cite{14}. Collaborative filtering algorithms can be divided into user-based algorithms, item-based algorithms and model-based algorithms. The user-based collaborative filtering algorithm is the earliest recommendation algorithm. This algorithm first calculates the similarity between users, selects the similar user with highest similarity to the target user. Then, it recommends items selected by the similar user to the target user\cite{15}. Currently, the item-based collaborative filtering algorithm is used broadly in industry, which recommends items similar to the items liked by users to them\cite{16}. The model-based collaborative filtering algorithms mainly recommend items to users through machine learning and data mining models\cite{17}.

Salakhutdinov\cite{18} applied deep learning to solve the recommendation problem for the first time, and proposed collaborative filtering recommendation model based on restricted Boltzmann machine(RBM). Sedhain \cite{19} proposed a self-encoder-based collaborative filtering method, which utilizes an encoding process and a decoding process to produce an output and optimizes the model parameters by minimizing the reconstruction error. Wu\cite{20} used the noise reduction self-encoder to solve the top-N recommendation problem, and proposed a collaborative noise reduction self-encoder model. It makes recommendation by taking the user's rating vector as an input and learning the user's low-dimensional vector representation. Covington\cite{21} proposed a deep collaborative filtering model, first using the depth candidate video generation model to retrieve the candidate set, and then using the depth ordering model to sort the candidate videos, which is superior to the matrix-based decomposition model. 

The biggest advantage of collaborative filtering algorithm based on deep learning is to introduce nonlinear feature transformation into the process of learning the implicit representations of user and item\cite{22}. Compared with the traditional collaborative filtering method, it has better performance, but still can't solve the data sparse problem and the cold start problem.

\subsection{Hybrid Model}

A hybrid model combines different recommendation models to take advantage of different models' merit and avoid their disadvantages\cite{23}. Commonly used hybrid recommendation algorithms include weighted hybrid recommendation algorithm, cross-harmonic recommendation algorithm, and meta-model mixed recommendation algorithm\cite{24}. For example, Lee\cite{25} learned semantic representation from the context of user conversations by combining recurrent neural networks and convolutional neural networks. Dai\cite{26} proposed a dynamic recommendation algorithm that combines convolutional neural network and multivariate point process by learning the co-evolutionary model of user-commodity implied features.

To sum up, although the recommendation of single or dual source data based on deep learning has achieved good results, the recommendation accuracy is still poor\cite{27,28,29,30}. With the development of the Internet, more and more data can be obtained. Using deep learning to fuse multiple heterogeneous data in the data source layer to improve the accuracy of the recommendation results is still worth studying\cite{31,32,33,34}.

\section{Methods}

\subsection{Overview}

This paper proposes a recommendation model based on deep learning, which can process multi-source heterogeneous data: score, review and social information.

For score, the traditional matrix decomposition method suffers problems of sparse data and low accuracy. This paper adopts neural network to transform scores into the user/item representations. For the reviews, the traditional topic model can't accurately represent the characteristics of the text. This paper utilizes the Distributed Bag of Words version of Paragraph Vector (PV-DBOW) algorithm to learn the feature representations of reviews. PV-DBOW assumes that the words in the document are independent and unordered, uses document vector representation to predict the words with higher accuracy. For social network data, this paper takes into account the impact of users' friends on users' selection, introduces the user trust model and integrates the social relationship information into the pairwise learning method, which improves the accuracy of the recommendation results. 

\subsection{Recommendation process}

Due to the heterogeneity of different data, the traditional hybrid recommendation model usually fuses data at the algorithm level\cite{30}, i.e., makes the final recommendation by combining recommendation results from algorithms based on different data.  With the development of deep learning, multi-source heterogeneous data such as scores and reviews can be accurately represented through deep networks, which makes it possible to fuse multi-source heterogeneous data fully at the data source level\cite{30}. The multi-source heterogeneous data recommendation model proposed in this paper combines ratings, reviews, and social network information to make a more accurate recommendation. It has the advantages of high accuracy and strong scalability. 

The recommended process is shown in the Figure 1. A score is a user's overall evaluation for an item, which reflects the user's satisfaction with the item. Multi-layer fully connected neural network is used to directly learn the feature vector representations of the user and the item. Reviews can reflect users' evaluations for items in detail and contain rich information about users and items. PV-DBOW algorithm is used to learn the feature representation of the paragraph, thus obtains the feature vector representations of the user and the item. The social network reflects friendships between users. The preferences of users' friends will indirectly affect the users' choices. The social network can be used to improve the prediction accuracy of a user's potential purchase behavior. Bayesian Personalized Ranking(BPR) model is used to rank the nonlinear characteristics of users and items, and further improves the accuracy of the recommendation results.

\subsection{Recommendation Model}

The recommendation model of multi-source heterogeneous data consists of four steps. Firstly, construct the user and item triplet optimization model. Secondly, extract social relation from social network, and fuse social relation data, review data and scores together. Thirdly, obtain the feature representations of users and items through deep learning. Finally, a top-N recommendation list is acquired from the feature representations of users and items. The model is described in detail as below. 

\subsubsection{User Trust Model}

Social network can reflect the friendship between users. In real life, users are more likely to choose items that their friends buy or like. Thus, a user's behavior and preferences can be more precisely predicted based on the user's direct and indirect friend relationship. 

The trust-based recommendation model assumes that users have similar preferences to their trusted users. In general, direct and indirect friends can affect a user's decisions in different levels, and indirect friends have less impact on the user's decisions than direct friends. According to Kevin Bacon's 6 degrees of separation concept\cite{35,36}, the similarity between users can be defined in (\ref{eq:1}):
\begin{equation}
\label{eq:1}
s(a, b)=\left\{\begin{array}{cc}
0.2 \times\left(6-l_{a b}\right) & if~  l_{a b}<6 \\
{0.1} &  otherwise 
\end{array}\right.
\end{equation}

Among them, $a$ and $b$ represent any two users. $l_{ab}$ represents the distance between user $a$ and user $b$, and the distance of direct friend is 1, the distance of indirect friend is $2,3,4,\cdots,s(a,b)$ represents the similarity between two users. Figure 2 shows the distance values between users. 

The similarity between users can be calculated based on the distances between users. We name the direct friend as first-degree friend, the indirect friends with distance 2 as the second-degree friend, and so forth. We consider an indirect friend with distance 6 at most so that we name the model as 6 Degree Model. Algorithm 1 gives the model's pseudo codes and shows how to calculate similarities between users. After obtaining the similarity between users from social networks, the influence of different friends on user's selection can be get, then it can be input to a unified joint representation learning framework together with the other types of data. 

\begin{algorithm}
	\caption{6 Degree Model}
	\label{alg:1}
	\begin{algorithmic}
	\STATE {\bf Input:}  dataset \textit{User}, social relation data \textit{Relation};
	\STATE {\bf Output:}  similarity set $S$;
	\STATE {\bf repeat:}
	\STATE \hspace*{0.02in} 1: Construct a graph by using all users and relations in  
	\STATE \hspace*{0.15in} \textit{User} and  \textit{Relation};
	\STATE \hspace*{0.02in} 2: Calculate $l_{ab}$ using igraph;
	\STATE \hspace*{0.02in} 3: If $l_{ab}<6$
	\STATE \hspace*{0.02in} 4: $s(a,b)=0.2\times(6-l_{ab})$;
	\STATE \hspace*{0.02in} 5: else 
	\STATE \hspace*{0.02in} 6: $s(a,b)=0.1$;
	\STATE \hspace*{0.02in} 7: {\bf until} calculate all users' relations; 
    \STATE \hspace*{0.02in} 8: {\bf return}  $S$.
	\end{algorithmic}
\end{algorithm}

\subsubsection{Improved BPR Model}

BPR is a pairwise learning model\cite{37}. A triplet $(u,i,j)$ is constructed based on user preferences. A triplet can represent three cases:

\begin{itemize}
\item User $u$ purchases item $i$ but does not purchase item $j$. It means user $u$ has a preference on item $i$ than item $j$.
\item User $u$ purchases neither item $i$ nor item $j$. It means the user's preferences cannot be determined.
\item User $u$ purchases both item $i$ and item $j$. It means the user's preferences cannot be distinguished.
\end{itemize}

Comparing with point-wise learning, BPR model has two advantages. The first is considering both the items purchased by the user and the items not-purchased during learning, and the items not-purchased are the ones to be sorted in the future. The second advantage is that this model can reach good results when a small amount of data is selected for recommendation. 

BPR is a ranking algorithm based on matrix decomposition. Comparing with algorithms such as funkSVD, it is not a global scoring optimization but a sort optimization for each user's own commodity preferences. Its result is more accurate. Figure 3 shows the triplet generation process. Plus($+$) means user $u$ prefer item $i$ over item $j$. Minus($-$) means user prefer item $j$ over item $i$. Question mark (?) means that the user's preference cannot be determined.

However, the triplets constructed based on the standard BPR model are randomly sampled\cite{38}, and the effect of social relationships on the sampling process is not considered. In real life, users prefer the items that their friends have selected. So the similarity between users and friends can be applied to the sampling of the BPR model. By considering friends' influences on the user and adding social relation constraints to the sampling process, the triplet can more precisely reflect the user's preferences and thereby the recommendation accuracy can be improved.

According to the user's purchase records and the friendships reflected by the social network. For each user $u$, the item purchased by the user is defined as $i$, the item that the user has not purchased is defined as $j$, the item purchased by the user's direct or indirect friend is defined as $p$. All the items set in the system are defined as $D$. The set of items purchased by the user $u$ is defined as $D_u$. The items purchased by the user's direct and indirect friends are defined as $D_p$. The item set representing the user's strong preference is firstly $D_u$ and secondly $D_p\backslash D_u$. The reason is that the user is likely to purchase the item $D_p\backslash D_u$ purchased by the direct or indirect friends but not by the user according to the influences of the friends on the user's preference. Finally, the item that the user is least likely to purchase is $D\backslash(D_u\cup D_p)$. Constructing a triplet of users and items as a training set based on social network information, the train set $T$ can be expressed as follows. Where user-item triplet $(u,i,j)$ represents that the user $u$ has a greater preference for the item $i$ than the item $j$. Item $i$ is purchased by the user or by the direct or indirect friends of the user. Item $j$ means the item not purchased by the user or his/her friends. In this way, a user-item triplet based on social relations is constructed.
\begin{equation}
\label{eq:2}
{T}:=\left\{({u}, {i}, j) | i \in\left({D}_{u} \cup {D}_{p}\right), j \in {D} \backslash\left({D}_{u} \cup {D}_{p}\right)\right\}
\end{equation}

According to the Bayesian formula, it is necessary to maximize the following posterior probabilities for finding a list of items recommended. In (\ref{eq:3}), $(u,i,j)$ represents a constructed triplet with user's preference, and $\theta$ represents the parameters of the model. To make the triplet $(u,i,j)$ has the highest probability of occurrence, adjusting the model parameters. 
\begin{equation}
\label{eq:3}
{p}({\theta} |({u}, {i}, {j})) \propto {p}(({u}, {i}, {j}) | {\theta}) {p}({\theta})
\end{equation}

To simplify the aforementioned formula, we assume that the item pairs $(i,j)$ are independent of each other. Then we have:
\begin{equation}
\label{eq:4}
\begin{split}
{\prod}p\left(\left({u}, {i}, j\right) | {\theta}\right)=&{\prod} _{u \in(u, i, j)}{p}\left({i}>j | {\theta}\right)\\
&\cdot{\prod} _{u \notin(u, i, j)} (1-p({i}>{j} | {\theta}) )
\end{split}
\end{equation}

According to the integrity and anti-symmetry of pairwise learning, the above formula can be further simplified to:
\begin{equation}
\label{eq:5}
{\prod}p\left(\left({u}, {i}, j\right) | {\theta}\right)={\prod} _{u \in(u, i, j)}{p}\left({i}>j | {\theta}\right)
\end{equation}

To obtain the final sorting, a model needs to be constructed to calculate the probability of recommendation for each item. The sigmoid function is used to construct the model in which the probability of the user's purchasing item $i$ is greater than the one of purchasing item $j$. Where $x_{uij} (\theta)$ is an arbitrary parametric model that describes the potential relationship between the user and the item. In other words, any model that describes the relationship between the user and the item can be used.
\begin{equation}
\label{eq:6}
{p}(({u}, {i}, {j}) | {\theta}):={\sigma}\left({x}_{u i j}({\theta})\right)
\end{equation}

Improved BPR model is used to directly optimize the recommendation result based on the item recommendation ranking.

\subsubsection{PV-DBOW Model}

PV-DBOW model is used to learn review data to obtain feature representations of corresponding users and items. As Figure 4 shows, the model samples a text window, then samples a random word from the text window and forms a classification task given the Paragraph Vector\cite{39}. PV-DBOW assumes that the words in a sentence are independent of each other and requires only a small amount of data to be stored.

In our model, paragraph vectors are used to predict words. Each review will be mapped into a semantic space and then trained to predict words. The probability that the word $w$ appears in sentence $d$ can be calculated by calculating the softmax function.
\begin{equation}
\label{eq:7}
P\left(w | d_{u m}\right)=\frac{e^{w^{T} d_{u m}}}{\sum{w' \in V e^{w^{' T} d_{u m}}}}
\end{equation}

Where $d_{um}$ denotes the review given by user $u$ to item $m$, $w$ denotes the word, $V$ denotes the vocabulary. In order to reduce the cost of computation and improve the calculation efficiency, the negative sampling example strategy is adopted in this model. Therefore, the following objective function can be constructed.
\begin{equation}
\label{eq:8}
\begin{split}
&{L}_{1}=\sum_{w \in V} \sum_{(u, m) \in R} {f}_{w, d_{u m}} \log \sigma\left({w}^{T} {d}_{u m}\right)+\\
&\sum_{w \in V} \sum_{(u, m) \in R} f_{w, d_{u m}}\left(t \cdot E_{w_{N} \sim P_{V}} \log \sigma\left(-w_{N}^{T} d_{u m}\right)\right)
\end{split}
\end{equation}

Where $f_{w,d_{um} }$ represents the frequency of word and review pairs. $E_{w_N\sim P_V }$ represents the expected value on the noise distribution $P_V$, $t$ represents the negative sample numbers. According to (8), the comment representation $d_{um}$ can be obtained and $d_{um}$ corresponds to the user $u$ and the item $m$. The feature representations of $u$ and $m$ can be obtained from the comment data. 

\subsubsection{Fully Connected Neural Network}

For the reason that neural networks have the ability to quickly find optimal solutions, fully connected neural network is used to process the scoring data\cite{30}. The feature representations of user and item can be obtained from the score data. In this experiment, two fully connected layers are used to fit the nonlinear correlation:
\begin{equation}
\label{eq:9}
\hat{r}_{u m}=\phi\left(U_{2} \phi\left(U_{1}\left(r_{u} \odot r_{m}\right)+c_{1}\right)+c_{2}\right)
\end{equation}

Where $\phi(\cdot)$ is the ELU activation function, $U_1,U_2,c_1,c_2$ are parameters to be learned. Then, the following objective function can be get. 
\begin{equation}
\label{eq:10}
L_{2}=\sum\nolimits_{(u, m) \epsilon R}\left(\hat{r}_{u m}-r_{u m}\right)^{2}
\end{equation}

The goal of the scoring model is to make the difference between the predicted score and the true score as small as possible.

\subsubsection{BRS$_c$S Model}

To fuse multi-source heterogeneous data to make a recommendation, we propose a model named BRS$_c$S (an acronym for BPR-Review-Score-Social). In the model, improved BPR model is used to optimize the ranking, user trust model is used to introduce social relationships into the rating and review data, PV-DBOW model is used to process the review data, and fully connected neural network is used to process the rating data. finally, an integrated objective function is given to optimize.

The unified objective function for model optimization is given as (\ref{eq:11}). $u$ represents the fusion feature representation of the user, $i$ and $j$ represent the fusion feature representation of the items. According to the previous definition, it is known that the user $u$ has a greater preference for the item $i$ than the item $j\cdot g(\cdot)$ is a loss function that combines user and item features, this paper defines $g(\cdot)$ as a sigmoid function to calculate the user's different preferences for different items. Here $g(u,i,j)=\sigma(u^T i-u^T j)$. $L_1$ is the objective function of the review data and $L_2$ is the objective function of the score data.
\begin{equation}
\label{eq:11}
\begin{split}
&\max _{W, {\theta}} L=\sum_{u, i, j} {g}({u}, {i}, {j})+\lambda_{1} {L}_{1}-\lambda_{2} {L}_{2}\\
&=\sum_{u, i, j}\left\{{\sigma}({u}^{T} {i}-{u}^{T} {j})\right.\\
&+\lambda_{1}(\sum_{{w} \in V} \sum_{({u}, {m}) \in R} {f}_{{w}, {d}_{{u m}}} \log \sigma\left({w}^{T} {d}_{{u m}}\right) \\
&+\sum_{{w} \in V} \sum_{({u}, {m}) \in R} {f}_{{w}, {d}_{{u m}}}(t \cdot {E}_{{w}_{N} \sim P_{V}} \sigma(-{w}_{N}^{T} {d}_{{u} m}))- \\
&\left.\lambda_{2}({\phi}({U}_{2} \cdot {\phi}\left({U}_{\mathbf{1}}\left({r}_{{u}} \odot {r}_{{m}}\right)+{c}_{1}\right)+{c}_{2})-r_{u m})^{2}\right\}
\end{split}
\end{equation}

$W=\{W_1,W_2\}$ denotes the weight parameters of each model. In the review representation learning model, the weight parameter $W_1$ is different for different user's different review and need to be learned. In score representation learning model, the features of users and items are directly obtained, that is, the weight parameter $W_2$ can be set to 1. It is unnecessary to update $W_2$ by the optimization objective function. $\theta $ represents other parameters to be learned, $\theta =\{\theta _1,\theta _2 \}=\{\{w,d_{um} \},\{U_1,U_2,c_1,c_2,r_u,r_m\}\}$. $\lambda $ is the penalty parameter for each model, and its value is in the interval $[0,1]$. The objective function $L_2$ of the score model is preceded by a negative sign because the objective function of the score model should be minimized, while the objective function of the overall model should be maximized. The stochastic gradient descent (SGD) method\cite{40} can be used to optimize (\ref{eq:11}).

In the end, a recommendation list can be obtained by multiplying the user feature representations and the item feature representations:
\begin{equation}
\label{eq:12}
s=u^Tm
\end{equation}

The larger the $s$, the higher possibility for the user to select the item. A user's top-N recommendation list can be obtained by sorting the scores obtained from (\ref{eq:12}) in descending order.

\begin{algorithm}
	\caption{BRS$_c$S model}
	\label{alg:2}
	\begin{algorithmic}
	\STATE {\bf Input:} score dataset \textit{Score}, social relation dataset \textit{Relation}, 
	\STATE \hspace*{0.35in} review dataset \textit{Review}, vocabulary $V$;
	\STATE {\bf Output:} user representation $U$, item representation $M$, 
	\STATE \hspace*{0.35in} recommendation list $L$;
	\STATE \hspace*{0.02in} 1: Initialize $\theta$, embedding size$=300$, batch size$=64$, negative 
	\STATE \hspace*{0.15in} sample $= 5$;  
	\STATE \hspace*{0.02in} 2: {\bf for} epoch$=1, 2, \cdots, n$ {\bf do}
	\STATE \hspace*{0.02in} 3: split the dataset \textit{Score}, \textit{Relation} and \textit{Review} into 
	\STATE \hspace*{0.15in} training datasets (70\%) and testing datasets (30\%);
	\STATE \hspace*{0.02in} 4: construct positive and negative sample triplets $g(u,i,j)$ 
	\STATE \hspace*{0.15in} based on BPR;
	\STATE \hspace*{0.02in} 5: learn the frequency of word-review pair $f_{w,d_{um}}$ and the 
	\STATE \hspace*{0.15in} expected value $E_{w_N \sim P_V }$; 
	\STATE \hspace*{0.02in} 6: get review representation $d_{um}$;
	\STATE \hspace*{0.02in} 7: learn $U_1,U_2, c_1, c_2$ from score data; 
    \STATE \hspace*{0.02in} 8: get score representation $r_u , r_m$;
    \STATE \hspace*{0.02in} 9: get distance $l_{ab}$ between users; 
    \STATE \hspace*{0.02in} 10: calculate $\sum_{u, i, j} g(u, i, j)+\lambda_{1} L_{1}-\lambda_{2} L_{2}$;
    \STATE \hspace*{0.02in} 11: update $\theta=\{\theta_1,\theta_2 \}$ with back propagation;
    \STATE \hspace*{0.02in} 12: get corresponding user and item representations $U, M$ 
	\STATE \hspace*{0.2in} according to (\ref{eq:11});
    \STATE \hspace*{0.02in} 13: {\bf end for}
    \STATE \hspace*{0.02in} 14: compute $s$ according to (\ref{eq:12});
    \STATE \hspace*{0.02in} 15: {\bf return} recommendation list $L$.
	\end{algorithmic}
\end{algorithm}

\section{Results and discussion}

Two groups of experiments are performed on a single-core GPU GeForce GTX 1080 Ti with Ubuntu 16.04 operating system. The programming environment consists of python3.6, igraph, tensorflow1.4 and IntelliJ IDEA.

\subsection{Dataset}

Yelp is a business directory service and crowd-sourced review forum in America. It covers businesses in restaurants, shopping centers, hotels, and tourism. Users can rate, submit comments, and exchange experiences on Yelp website. This paper uses Yelp public dataset which can be obtained from the Yelp official website for experiments. Yelp dataset is in JSON format and contains details of users and businesses. Data contains the IDs of users and businesses, users' comments and ratings on businesses, and the friendship between the users. Social relationship between users is transformed into user-friend relationship pairs. Comments and ratings are used to analyze users' preferences.

Since the dataset is too sparse, it is necessary to filter out some users with few comments to verify the validity of the proposed model. Data with more than 20 comments are extracted from the Yelp dataset, and the new dataset is named as New-Yelp. Table 1 shows the detailed statistics of New-Yelp dataset. 

\subsection{Comparable Experiment}

\noindent Four indicators are used to measure the experimental results:

\noindent {\bf Recall}: the ratio of items purchased by users on the recommendation list to all items purchased by users.

\noindent  {\bf Precision}: the ratio of the number of recommended items purchased by users to the total recommended items.

\noindent  {\bf NDCG}: normalized breakage cumulative gain, it is used to calculate the ranking quality of recommended items.

\noindent  {\bf HT}: the hit rate refers to whether the user has purchased the recommended item. If user has purchased the recommended item, it means hits; Otherwise, it means misses.

\subsubsection{Experiment I}

To select a model to deal with texts and prove the positive effect of fusing social data on recommendation, Experiment I compares six models as below.  

\begin{itemize}
\item BR (BPR$+$Review) model is based on the BPR framework and uses the reviews for recommendation. 
\item BRS (BPR$+$Review$+$Social) model is based on the BPR framework, uses the reviews and social network information for recommendation. The social relation here contains both direct friend relation but also indirect friend relation.
\item HRS (HDC$+$Review$+$Social) model is a regularized embedding-based language model that combines reviews and social network information for recommendation. 
\item SRS (SEL$+$Review$+$Social) model is based on a simplified embedding-based language model. It uses the reviews and social network information for recommendations.
\item BRS$_c$ (BPR$+$Review$+$Score) model is based on the BPR framework. It uses reviews and scores for recommendation. 
\item BRS$_c$S (BPR$+$Review$+$Score$+$Social) model is based on the BPR framework. It combines reviews, scores and social network information for recommendation. 
\end{itemize}

In the experiment, 70\% of the data is used for training and 30\% of the data for testing. The batch size is 64, max train epoch is 40, negative sample is 5, and the feature dimension is 300.

BRS$_c$S is the hybrid model proposed by us. The top-5 recommendation experimental results are shown in Table 2 and Figure 5. The top-10 recommendation experimental results are shown in Table 3 and Figure 6. The best results are shown in bold.

In the experiment, we compare BRS, HRS and SRS, which are 3 different recommendation models based on reviews and social information. The results present that BRS model outstrips HRS model and SRS model significantly and proves that utilizing BPR framework in our model is a wise decision. Furthermore, experiments show that BRS model surpasses BR model and BRS$_c$S model surpasses BRS$_c$ model, which prove that adding social networks can improve the recommendation accuracy. Comparing all the models, we can see that the BRS$_c$S model proposed by us performs best in terms of Precision, Recall, NDCG and HT.

\subsubsection{Experiment II}

Experiment II compares our model with broadly-used classic recommendation models. 

BS$_c$ model is based on the BPR framework and uses scores to make a recommendation. UserCF (User-based collaborative filtering algorithm) is one of the most popular recommendation algorithms and uses scores as input. BRS$_c$is a model combining reviews and scores for recommendation. BRS$_c$S is the model proposed by us. The experimental results are shown in Table 4 and Figure 7. The best indicator results are shown in bold. 

The experimental results show that BRS$_c$ surpasses BSc and UserCF in terms of three indicators (Recall, Precision and HT). That proves the positive effect of fusing reviews together with scores to make a recommendation. The model proposed by us, BRS$_c$S, performs best in terms of all the indicators. It proves that the fusion model based on deep learning outperforms the traditional collaborative filtering algorithm due to the merits of feature representation by deep learning. Our model can fully fuse multi-source heterogeneous data such as scores, reviews and social network information from data source level by deep learning so as to make more accurate recommendation. Introducing social network information can also solve cold start and data sparse problems because direct and indirect friends' data can be used for making a recommendation. Our model has also good scalability. When adding new type of data to the framework, some experimental parameters do not need to be changed. It's easy to add another new type of heterogeneous data by using deep learning to represent features of new data. 


\section{Conclusions}

To utilize heterogeneous data to improve recommendation accuracy and solve cold start and data sparse problem, we propose a hybrid recommendation model that can fuse multi-source heterogeneous data such as scores, reviews and social network information.  The model is named as BRS$_c$S based on deep learning. Experiments are performed to compare our model with other recommendation models. The results show that our model can outperform the other models in Recall, Precision, NDCG and HT. Introducing user trust model to fuse social data together with scores and reviews can solve cold start problem and data sparse problem effectively because friends' data can be used to make a recommendation. The model is scalable and can fuse more types of heterogeneous data easily.

In the future, we plan to introduce image data into our recommendation model because images contain rich semantic information.




\begin{backmatter}

\section*{Abbreviations}

BPR: Bayesian Personalized Ranking; 

DSSM: Deep Structured Semantic Model; 

CNN: Convolutional Neural Network; 

CF: Collaborative Filtering; 

RBM: Restricted Boltzmann Machine; 

PV-DBOW: Paragraph Vector-Distributed Bag of Words; 

Deep CoNN: Deep Collaborative Neural Network.

\section*{Availability of data and material}

The dataset used in this paper are available from Yelp website [http://www.yelp.com/dataset], more details are available from the corresponding author.

\section*{Competing interests}

The authors declare that they have no competing interests. 

\section*{Funding}

This work is supported by the National Key Research and Development Program of China under Grant No. 2018YFC0831903 and Major Project of National Natural Science Foundation of China under Grant No. 51935002.

\section*{Authors' contributions}

All authors contribute to this work.

\section*{Acknowledgements}

This work was supported by the Research Center of Software Engineering in School of Software Engineering, Beijing Jiaotong University.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                  The Bibliography                       %%
%%                                                         %%
%%  Bmc_mathpys.bst  will be used to                       %%
%%  create a .BBL file for submission.                     %%
%%  After submission of the .TEX file,                     %%
%%  you will be prompted to submit your .BBL file.         %%
%%                                                         %%
%%                                                         %%
%%  Note that the displayed Bibliography will not          %%
%%  necessarily be rendered by Latex exactly as specified  %%
%%  in the online Instructions for Authors.                %%
%%                                                         %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% if your bibliography is in bibtex format, use those commands:
\bibliographystyle{bmc-mathphys} % Style BST file (bmc-mathphys, vancouver, spbasic).
\bibliography{Manuscript}      % Bibliography file (usually '*.bib' )
% for author-year bibliography (bmc-mathphys or spbasic)
% a) write to bib file (bmc-mathphys only)
% @settings{label, options="nameyear"}
% b) uncomment next line
%\nocite{label}

% or include bibliography directly:
% \begin{thebibliography}
% \bibitem{b1}
% \end{thebibliography}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                               %%
%% Figures                       %%
%%                               %%
%% NB: this is for captions and  %%
%% Titles. All graphics must be  %%
%% submitted separately and NOT  %%
%% included in the Tex document  %%
%%                               %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%
%% Do not use \listoffigures as most will included as separate files

\section*{Figures}
\begin{figure}[h!]
\caption{Recommendation process.}
\end{figure}


\begin{figure}[h!]
\caption{Social relations between users.}
\end{figure}

\begin{figure}[h!]
\caption{Triplet generation process.}
\end{figure}


\begin{figure}[h!]
\caption{Distributed Bag of Words of Paragraph Vectors.}
\end{figure}

\begin{figure}[h!]
\caption{Indicators of top-5 Recommendation.}
\end{figure}

\begin{figure}[h!]
\caption{Indicators of top-10 Recommendation.}
\end{figure}

\begin{figure}[h!]
\caption{Indicators of Experiment II}
\end{figure}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                               %%
%% Tables                        %%
%%                               %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%% Use of \listoftables is discouraged.
%%
\section*{Tables}
\begin{table}[h!]
\caption{Detailed statistics of {\bf New-Yelp}.}
      \begin{tabular}{cc}
        \hline
Dataset & New-Yelp\\\hline
User & 16371\\
Business & 22581\\
User-friend relation & 6111137\\
Review & 356990\\
Sparsity & 99.90\%\\\hline
      \end{tabular}
\end{table}

\begin{table}[h!]
\caption{Experiment I of top-5 Recommendation.}
      \begin{tabular}{ccccc}
        \hline
\multirow{2}*{{\textbf{Method/100\%}}} & \multicolumn{4}{c}{{\textbf{top-5 Recommendation}}}\\\cline{2-5}
 & Recall@5 & Pre@5 & NDCG@5 & HT@5\\\hline
BR & 3.422 & 1.119 & 2.37 & 6.334\\
BRS & 4.175 & 1.308 & 2.967 & 7.425\\
HRS & 1.12 & 0.326 & 0.751 & 1.93\\
SRS & 1.853 & 0.608 & 1.321 & 3.492\\
BRS$_c$ & 3.65 & 1.208 & 2.541 & 6.794\\
BRS$_c$S & {\bf 4.363} & {\bf 1.352} & {\bf 3.084} & {\bf 7.71}\\\hline
      \end{tabular}
\end{table}

\begin{table}[h!]
\caption{Experiment I of top-10 Recommendation.}
      \begin{tabular}{ccccc}
        \hline
\multirow{2}*{\textbf{Method/100\%}} & \multicolumn{4}{c}{\textbf{top-10 Recommendation}}\\\cline{2-5}
 & Recall@10 & Pre@10 & NDCG@10 & HT@10\\\hline
BR & 5.56 & 1.004 & 3.103 & 10.049\\
BRS & 6.617 & 1.125 & 3.784 & 11.483\\
HRS & 1.855 & 0.304 & 1.003 & 3.256\\
SRS & 2.843 & 0.517 & 1.651 & 5.268\\
BRS$_c$ & 5.787 & 1.057 & 3.264 & 10.48\\
BRS$_c$S & {\bf 6.85} & {\bf 1.168} & {\bf 3.915} & {\bf 11.83}\\\hline
      \end{tabular}
\end{table}

\begin{table}[h!]
\caption{Experiment II of Recommendation.}
      \begin{tabular}{ccccc}
        \hline
\textbf{Measure/100\%} & 
BS$_c$ & UserCF & BRS$_c$ & BRS$_c$S\\\hline
Recall@5 & 1.532 & 1.93 & 3.65 & {\bf 4.363}\\
Recall@10 & 2.26 & 2.78 & 5.787 & {\bf 6.85}\\
Precision@5 & 0.671 & 0.25 & 1.208 & {\bf 1.352}\\
Precision@10 & 0.227 & 0.18 & 1.057 & {\bf 1.168}\\
HT@5 & 5.784 & 6.38 & 6.794 & {\bf 7.71}\\
HT@10 & 8.628 & 9.15 & 10.48 & {\bf 11.83}\\\hline
      \end{tabular}
\end{table}


\end{backmatter}
\end{document}
