%% BioMed_Central_Tex_Template_v1.06
%%                                      %
%  bmc_article.tex            ver: 1.06 %
%                                       %

%%IMPORTANT: do not delete the first line of this template
%%It must be present to enable the BMC Submission system to
%%recognise this template!!

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                     %%
%%  LaTeX template for BioMed Central  %%
%%     journal article submissions     %%
%%                                     %%
%%          <8 June 2012>              %%
%%                                     %%
%%                                     %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                                                 %%
%% For instructions on how to fill out this Tex template           %%
%% document please refer to Readme.html and the instructions for   %%
%% authors page on the biomed central website                      %%
%% http://www.biomedcentral.com/info/authors/                      %%
%%                                                                 %%
%% Please do not use \input{...} to include other tex files.       %%
%% Submit your LaTeX manuscript as one .tex document.              %%
%%                                                                 %%
%% All additional figures and files should be attached             %%
%% separately and not embedded in the \TeX\ document itself.       %%
%%                                                                 %%
%% BioMed Central currently use the MikTex distribution of         %%
%% TeX for Windows) of TeX and LaTeX.  This is available from      %%
%% http://www.miktex.org                                           %%
%%                                                                 %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%% additional documentclass options:
%  [doublespacing]
%  [linenumbers]   - put the line numbers on margins

%%% loading packages, author definitions

% \documentclass[twocolumn]{bmcart}% uncomment this for twocolumn layout and comment line below % FINAL
\documentclass[linenumbers, doublespacing]{bmcart} % MANUSCRIPT
% \documentclass[doublespacing]{bmcart}

%%% Load packages
\usepackage{amsthm,amsmath}
%\RequirePackage{natbib}
%\RequirePackage[authoryear]{natbib}% uncomment this for author-year bibliography
\RequirePackage{hyperref}
\usepackage[utf8]{inputenc} %unicode support
%\usepackage[applemac]{inputenc} %applemac support if unicode package fails
%\usepackage[latin1]{inputenc} %UNIX support if unicode package fails
\usepackage[pdftex]{graphicx}
\graphicspath{{./figures/}{./pdf/}{./jpeg/}}
\DeclareGraphicsExtensions{.pdf,.jpeg,.png,.jpg}
\usepackage{todonotes}
\usepackage{microtype}
\usepackage{url}
\usepackage{dblfloatfix}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                             %%
%%  If you wish to display your graphics for   %%
%%  your own use using includegraphic or       %%
%%  includegraphics, then comment out the      %%
%%  following two lines of code.               %%
%%  NB: These line *must* be included when     %%
%%  submitting to BMC.                         %%
%%  All figure files must be submitted as      %%
%%  separate graphics through the BMC          %%
%%  submission process, not included in the    %%
%%  submitted article.                         %%
%%                                             %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%\def\includegraphic{}
%\def\includegraphics{}

\begin{document}

%%% Start of article front matter
\begin{frontmatter}
\begin{fmbox}
\dochead{Research}

\title{Generative Enhancement of 3D~Image Classifiers} 
\author[
   addressref={kkui},                   % id's of addresses, e.g. {aff1,aff2}
   corref={kkui},                       % id of corresponding address, if any
%    noteref={n1},                        % id's of article notes, if any
   email={michal.varga@tuke.sk}        % email address
]{\inits{MV}\fnm{Michal} \snm{Varga}}
\author[
   addressref={kkui},                   % id's of addresses, e.g. {aff1,aff2}
   corref={kkui},                       % id of corresponding address, if any
%    noteref={n1},                        % id's of article notes, if any
   email={jan.jadlovsky@tuke.sk}        % email address
]{\inits{JJ}\fnm{Ján} \snm{Jadlovský}}
\author[
   addressref={kkui},                   % id's of addresses, e.g. {aff1,aff2}
   corref={kkui},                       % id of corresponding address, if any
%    noteref={n1},                        % id's of article notes, if any
   email={slavka.jadlovska@tuke.sk}        % email address
]{\inits{JJ}\fnm{Slávka} \snm{Jadlovská}}

\address[id=kkui]{%                           % unique id
  \orgname{Dept. of Cybernetics and Artificial Intelligence, FEI TUKE}, % university, etc
%   \street{Waterloo Road},                     %
  %\postcode{}                                % post or zip code
  \city{Košice},                              % city
  \cny{Slovak Republic}                                    % country
}

% \author{\textit{\begin{Large} $^{1}$J\'an JADLOVSK\'Y, $^{2}$Michal~VARGA \end{Large}}\\
% \bigskip
% $^{1,2}$Dept. of Cybernetics and Artificial Intelligence, FEI TU of Ko\v{s}ice, Slovak Republic\\
% \bigskip
% $^{1}$jan.jadlovsky@tuke.sk, $^{2}$michal.varga@tuke.sk}
% \maketitle

\begin{artnotes}
    %\note{Sample of title note}     % note to the article
    % \note[id=n1]{Equal contributor} % note, connected to author
\end{artnotes}

\begin{abstractbox}

\begin{abstract} % abstract
    In this paper, we propose a methodology for generative enhancement of existing 3D image classifiers.
    This methodology is based on combining the strengths of both non-generative classifiers and generative modeling.
    Its purpose is to streamline the creation of new classifiers by embedding existing compatible classifiers in a generative network architecture.
    The demonstration of this process and evaluation of its effects is performed using a 3D convolutional classifier and its generative equivalent - a conditional generative adversarial network classifier.
    The results show that the generative model achieves greater classification performance, gaining a relative classification accuracy improvement of 7.43\%.
    Improvement of accuracy is also present when compared to a plain convolutional classifier trained on a dataset augmented with examples produced by a trained generator. 
    This suggests there is a desirable knowledge sharing within the hybrid discriminator-classifier network.

% By using class labels during the supervised or semi-supervised training of a conditional GAN model, its discriminator is trained to not only detect real and fake data but also to classify it.
% A discriminator trained in this manner achieves greater classification accuracy, especially when trained on a smaller volume of data.
% A conditional GAN model presented in this article is trained in this manner and evaluated on ModelNet10, a dataset of 3D objects categorized into 10 classes.
% Comparing a classifier trained only on the real data, and a conditional GAN discriminator with label output, a relative classification accuracy improvement of 7.43\% has been achieved.
% Improvement of accuracy is also present when compared to a plain convolutional classifier trained on a dataset augmented with examples produced by a trained generator. 
% This suggests there is a desirable knowledge sharing within the hybrid discriminator-classifier network.

\end{abstract}

\begin{keyword}
    \kwd{Generative modeling}
    \kwd{Image classification}
    \kwd{Convolutional neural network}
    \kwd{Deep learning}
    \kwd{3D imaging}
\end{keyword}
\end{abstractbox}

\end{fmbox}% comment this for two column layout

\end{frontmatter}


\section{Introduction} % WIP

Generative modeling has been successfully applied to solve a wide range of tasks including realistic data generation (numeric, text, image, audio, video, etc.), extrapolation, image super-resolution, inpainting, and many others.
Data classification is also one of those, even though it is usually not the main goal of designing a generative network.

Deep generative models have been proven to learn a salient representation of the data they had been trained on, which is ideal for the classification task.
This latent representation is perceptually significant on the generated output which has been shown by various latent vector arithmetic experiments.
\cite{tasseShape2VecSemanticbasedDescriptors2016, wuLearningProbabilisticLatent2016}


Lately, one of the most popular varieties of generative models is the Generative Adversarial Network (GAN)
\cite{goodfellowGenerativeAdversarialNets2014}.
From the classification perspective, a GAN network's ability to generate highly realistic data indicates a potential for high performance in classification tasks, especially where large training datasets are not available, which is not uncommon in the domain of 3D imaging.
While it was not originally designed for classification, simple modifications exist to enable this functionality.
One of those is to use the discriminator itself as a classifier.
This can be achieved by extending the existing discriminator output with additional units, one per class, and training the model (semi)supervisedly.~\cite{odenaSemisupervisedLearningGenerative2016}
While mostly used with 2D image data, a simple modification of the network (using 3D (de)convolution instead of 2D) enables the usage of 3D data in the form of voxel grids or point clouds.


State of the art generative and non-generative classifiers compete very closely in terms of classification accuracy \cite{wu3dShapenetsDeep2015}.
While models utilizing the two approaches differ significantly in structure, we see the potential for their combination by enhancing the non-generative classifiers with elements of generative modeling.
Aside from improving the classification accuracy, this could likely help with the 3D domain-specific problems, such as small datasets, too generic and synthetic training examples, etc.

This idea is lightly supported in \cite{salimansImprovedTechniquesTraining2016}, where authors observe an occurrence of beneficial interaction within the model when combining supervised and unsupervised loss when training a feature matching GAN classifier.
This hints towards a potential for such interactions within a generative classifier which allows for further improvement of non-generative state-of-the-art models by embedding them within a generative network architecture.

% Therefore, we see some room for improvement of existing 3D image classifiers which are not currently using any form of generative modeling.
For these reasons, we propose a general methodology of generative enhancement, which aims to enable any compatible classifier to utilize a generative modeling approach to improve its classification accuracy. 
We evaluate the methodology by applying it to a simple 3D voxel convolutional classifier, although it is not theoretically limited to this type of data.
We investigate the reproducibility and measurability of the benefits of this transformation by comparing the classification performance of the resulting conditional GAN network to the original 3D Convolutional Neural Network (CNN) classifier, identical to the GAN discriminator.
Experiments are also performed to investigate possible explanations such as the introduction of noise or data augmentation by the generator.

\section{Related Work}

As the training of a GAN network is very difficult in general, a lot of research focuses on the training process itself and examines various tweaks and hacks that improve the convergence of the model during the training. 
Many of these were utilized in our research.
\cite{salimansImprovedTechniquesTraining2016, chintalaHowTrainGAN2019}

Generative networks have been used very successfully as a tool for data augmentation, improving existing (vanilla) classifiers' performance.
The benefit of GAN-based data augmentation was shown to even exceed that of standard augmentation methods.
This augmentation has proven helpful especially with small and/or unbalanced datasets.
\cite{antoniouDataAugmentationGenerative2018, wangEffectivenessDataAugmentation2017, tanakaDataAugmentationUsing2019}

Conditional generation combined with a classifying discriminator (auxiliary classifier) has been shown to improve the quality of generated samples when trained on a labeled dataset using a model called Auxiliary Classifier GAN (ACGAN) \cite{odenaConditionalImageSynthesis2017}.
This network includes separate discrimination and classification terms in the objective function.
Another approach, also used in our models, is to combine the terms in a way, where a sample cannot be classified as both fake and of a particular class.
This is achieved by creating a 'fake' class and discarding the fake-real discrimination function.
Such an approach is also used in BAGAN \cite{marianiBAGANDataAugmentation2018}, a generative network designed to restore the balance of an unbalanced dataset by generating samples of the minority classes.
Additionally, this network tackles the mode collapse issue by initializing the parameters using an autoencoder.

% Classification
Data classification is also a domain, where generative models have found their application.
Many deep generative models, using both supervised and semi-supervised training, have been used for various learning and classification tasks. 
Unsupervised training of generative models is also viable and various techniques exist to perform this task, such as Output Distribution Matching cost function \cite{sutskeverPrincipledUnsupervisedLearning2015} or CatGAN \cite{springenbergUnsupervisedSemisupervisedLearning2017}.

A combination of GAN architecture and variational autoencoders (VAE) have found their use in classification and retrieval of 3D models, also being able to convert 2D images to 3D models.
\cite{wuLearningProbabilisticLatent2016}
Similarly, these models are also useful for tasks concerning other types of data, such as spoken language identification systems.~\cite{shenConditionalGenerativeAdversarial2017}

% 3D
In our research, we focus mostly on the classification of 3D images.
ModelNet (a labeled dataset of 3D objects) website \cite{wu3dShapenetsDeep2015} publishes a benchmark leaderboard comparing the results of different 3D object classifiers trained on the dataset, which shows that generative models can achieve state-of-the-art performance and sometimes also exceed it.
VRN Ensemble \cite{brockGenerativeDiscriminativeVoxel2016}, a generative model based on voxel variational autoencoders and convolutional architecture, achieves one of the best performances in the benchmark.

The standard deconvolutional layer is not the only building block which the generator can be constructed with.
A novel factorized generative model in \cite{khanUnsupervisedPrimitiveDiscovery2019} generates the 3D models by first building it with generic primitives and then refining the resulting shape.
The resulting representation was evaluated using a classifier which achieved state-of-the-art results on ModelNet10 among the unsupervised models.

\section{Dataset}              

% ModelNet10
The models presented in this paper have all been trained on and evaluated on a 3D model dataset called ModelNet \cite{wu3dShapenetsDeep2015}. 
Two subsets of the dataset exist, ModelNet10 and ModelNet40, containing 10 and 40 classes of objects respectively.
The orientation of the examples is normalized so that every model's vector of gravity is aligned to the common vertical axis. Therefore, no manual adjustments of the mesh objects were necessary.

\subsection{Preprocessing and Transformation}

% Preprocessing
The ModelNet dataset contains examples represented as 3D CAD meshes. To be usable as an input for 3D convolution-based networks it has to be converted to a voxel grid format. For this purpose our data transformation pipeline has been used \cite{vargaEvaluationDepthModality2018}.

% Voxelization-BinVox
% filling, scaling
This pipeline is utilizing a voxelization tool called \texttt{binvox} by Patrick Min \cite{minBinvox2004, nooruddinSimplificationRepairPolygonal2003}. 
The pipeline then fills the hollow voxel model and scales it down to reduce the amount of data due to performance considerations.

Filling the model is done by labeling the connected regions in the voxel grid and joining everything except the largest connected region, which is usually the free space around the model. 
This is not fault-proof, as some models are either not watertight or not aligned perfectly to one face of the volume.
While faulty results are rare, this is one of the possible areas of improvement in the processing pipeline.

The scaling is performed by a combination of max pooling and interpolated subsampling.
This process removes small artifacts and holes introduced by the voxelization step while preserving most of the information by using floating-point values in the subsampling pass.
The resulting size of one sample is $32\times32\times32$ floating-point value voxels. 
The value of a single voxel after pre-processing is the approximation of the fraction of the voxel volume occupied by the 3D object. 
The voxel values, originally from $[0,1]$ range, are finally normalized in $[-1,1]$ range.
There is no color information available in the dataset.

% Visualization
For debugging and visualization of the dataset and generated examples, the voxel grids are drawn using our custom rendering pipeline \cite{vargaContribution3DVoxel2019}.
The main parts of the pipeline are surface normal estimation using Sobel operators and texture mapping of an equirectangular lightmap to the voxels.
The resulting RGBA-valued voxel grid is rendered using the PyQtGraph library~\cite{campagnolaPyQtGraph2012}.


\section{Generative enhancement methodology}

We propose a methodology for constructing generatively enhanced classifiers by embedding existing well-performing neural network classifiers in a compatible generative network architecture. 
By that, the classification performance of the original classifier is improved along with its tolerance for small training datasets.
The methodology is designed to formalize this transformation process and its creation is in line with one of the goals established in \cite{vargaKlasifikcia3DObrazov2017}.

The process of modification consists of several steps.
First, a conditional generator network is designed with output identical to the classified data type (same number of dimensions, size, number of channels, etc.) and from the desired class.
Then, the classifier is extended to perform the discrimination task (for example, by adding a 'fake' class to the output value set).
Finally, the training algorithm is modified to include adversarial optimization of the generator.

Therefore, this can be conceptualized as a neural network transformation, which turns a traditional convolutional neural network classifier into a conditional GAN classifier.
By applying this transformation to different classifiers, entirely new architectures can be produced with potentially improved properties.

The main constraint for this methodology is the classifier must be fully differentiable from one end to the other so that the gradients can be calculated for all layers, including the input layer.
This is required to optimize the generator, which is attached to the classifier via the input layer itself.

\section{Models and Methods}

\begin{figure*} % high level 
    \centering
      \includegraphics[width=.9\textwidth]{models}
      \caption{ High-level architecture of the tree models designed for the proposed experiment. The first model (a) is a plain 3D CNN trained directly on ModelNet10 training examples. The other CNN (b) is trained on an augmented dataset created by combining ModelNet10 examples and the conditional generator (G) output. The generator output is labeled according to its input condition vector. The last model (c) is a 3D conditional GAN network where the discriminator (D) is used as the classifier during the testing phase.}
      \label{f:models}
\end{figure*}

To verify the viability of the proposed methodology, we evaluate its simplest application case -- the transformation of a convolutional classifier into a conditional GAN network.
%We designed an experiment to compare the classification accuracy of a standalone classifier compared to a classifier trained by or working alongside a generative model. 
For this purpose, three separate models have been designed.
The first is a control model represented by a plain 3D convolutional neural network (plain CNN) trained only on the training dataset examples. 
The second model is another classifier, which shares the same structure as the control, however, it is also trained on the output produced by the generator from a GAN network (augmented CNN). 
The third model is the aforementioned 3D conditional GAN network.
A high-level diagram of the designed models is shown in Figure \ref{f:models}.



\subsection{3D Convolutional Neural Network Classifiers}

The first two models are identical deep 3D convolutional networks consisting of 3 convolutional layers. 
Stride has been used instead of pooling layers for downsampling. 
Batch normalization and Leaky ReLU activation function have been used following each convolutional layer. 
The output is produced by a single fully-connected layer with a unit count of $C$, equal to the number of classes. 
A softmax function is used to convert the output logits to class probabilities.

The loss function used to train these classifiers is a categorical cross-entropy shown in (\ref{eq:cnnloss}). 

% todo Consider moving the eq to the training section
\begin{equation}
    \label{eq:cnnloss}
    L_C = - \sum_{i=1}^M \sum_{j=1}^C \mathbf{y}_j^{(i)} \log(\mathbf{\hat{y}}_j^{(i)})
\end{equation}

In the equation, $M$ stands for the number of examples (in a minibatch or total), $C$ is the number of classes in the dataset, $\mathbf{y}$ is a vector of one-hot encoded training data labels and $\mathbf{\hat{y}}$ is the classifier prediction of the class vector.
The $\mathbf{y}_j^{(i)}$ notation refers to the $j$\textsuperscript{th} element of the vector belonging to the $i$\textsuperscript{th} sample within the minibatch.
The optimizer used to train these classifiers was \hbox{RMSprop}.

%todo Add CNN structure figure here

\subsection{3D Conditional Generative Adversarial Network}

The final model is a 3D conditional generative adversarial network (3D CGAN), consisting of a generator-discriminator pair connected in sequence and trained in opposition to each other. 
The role of the generator is to transform a latent representation (vector) of an object to its visual representation (voxel grid), while the discriminator tries to determine whether the sample is real (from the training dataset) or fake (produced by the generator).
The goal of the adversarial training is to achieve the equilibrium, where the generator produces samples that are realistic enough to confuse the discriminator \cite{goodfellowGenerativeAdversarialNets2014}.
A conditional GAN introduces a condition for the generator, specifying the desired output by providing a one-hot encoded class vector or another form of specification (eg. keywords or description text) \cite{mirzaConditionalGenerativeAdversarial2014}.
The starting point for this architecture has been the deep convolutional 3D GAN model published in~\cite{wuLearningProbabilisticLatent2016}. 

% Discriminator
The main difference from the standard GAN architecture in our model is the discriminator, which is mostly identical to the plain and augmented CNN models mentioned above.
The only difference between the discriminator and the two previously described convolutional classifiers is in the output layer, which, in addition to the class label, contains one extra output indicating whether the input is considered fake or not. 
Essentially, it distinguishes one additional "fake" class.
The layers of discriminator use the Leaky ReLU activation function, along with batch normalization.

\begin{figure*} % G structure
    \centering
      \includegraphics[width=.9\textwidth]{generator}
      \caption{Architecture of the generator network. Input vector $z$ consisting of a gaussian noise vector and a condition (class) vector is expanded in a fully connected stage and then is deconvoluted three times with batch normalization and ReLU activation function in between.}
      \label{f:generator}
\end{figure*}

% Generator
The generative part of the CGAN network (generator), shown in Figure \ref{f:generator}, consists of 3 deconvolutional (or transposed convolutional) layers with a descending amount of volumetric filters.
The upsampling of the activation volumes is achieved by kernel stride, which is equal to 2 in every layer.
ReLU activation function is used after each layer, followed by a batch normalization function.
The output layer uses $tanh$ activation function instead to fit the output into the dataset value range of $[-1,1]$.

The generator is conditioned by appending the desired one-hot encoded class vector to the latent feature vector, containing 200 feature values sampled from a Gaussian distribution $\mathcal{N}(0,1)$ forming an input vector $\mathbf{z}$.
The input is first fed through a fully connected layer expanding it to the shape required by the first deconvolutional layer.
The dense layer is also followed by ReLU and batch normalization functions.

% todo: reference cnn model as D

% D Loss
The loss function chosen to train the discriminator, shown in (\ref{eq:loss_d}), is a standard categorical cross-entropy.

\begin{equation}
    \label{eq:loss_d}
    L_D = - \sum_{i=1}^M \sum_{j=1}^{C+1} \mathbf{y}_j^{*(i)} \log(\mathbf{\hat{y}}_j^{*(i)})
\end{equation}

To include both fake-real discrimination error and classification error, an extended target vector $\mathbf{y}^*$ is constructed, as shown in (\ref{eq:extendedy}). 

\begin{equation}
    \label{eq:extendedy}
    \mathbf{y}^{*(i)} = 
    \begin{cases}
        [0, \mathbf{y}^{(i)}] & real\\
        [1, 0, 0, \dots, 0] & fake
    \end{cases}
\end{equation}

For real samples, a single zero is prepended to the class label from the dataset. 
In the case of generated samples, the target is a $C+1$ long vector, with the first element (the "fake" class) set to one and the rest to zero.


% G Loss
For the generator, the loss function is chosen to minimize the "fake" output of the discriminator as well as achieve the class output according to the conditioning vector provided to generate the sample.
The generator loss function formula is essentially identical to the discriminator loss with a real example.
The only difference is the true class vector, which is created randomly, instead of sampling it from the dataset, and used both as generator input (concatenated with noise vector) and as the target vector compared with the discriminator output.

% TODO Move loss functions to the training section, perhaps?

An alternative loss function we experimented with is a modified Wasserstein loss function \cite{arjovskyWassersteinGAN2017} shown in (\ref{eq:loss_dw}) and (\ref{eq:loss_gw}).

\begin{equation}
    \label{eq:loss_dw}
    \begin{aligned}
        L_D^w = &\frac{1}{M} \sum_{i=1}^M D(\mathbf{x}^{(i)}) - \frac{1}{M} \sum_{i=1}^M D(G(\mathbf{z}^{(i)})) - \\
        &- \gamma \sum_{i=1}^M \sum_{j=1}^C \mathbf{y}_j^{(i)} \log(\mathbf{\hat{y}}_j^{(i)})
    \end{aligned}
\end{equation}

\begin{equation}
    \label{eq:loss_gw}
    L_G^w = \frac{1}{M} \sum_{i=1}^M D(G(\mathbf{z}^{(i)})) - \gamma \sum_{i=1}^M \sum_{j=1}^C \mathbf{y}_j^{(i)} \log(\mathbf{\hat{y}}_j^{(i)})
\end{equation}

In these formulas, $M$ stands for the number of samples (in a minibatch or total), $C$ is the number of classes in the dataset, $\mathbf{x}$ is a vector of training data samples, $\mathbf{y}$ is a vector of one-hot encoded training data labels, $\mathbf{\hat{y}}$ is the classifier prediction and $\gamma$ is a hyperparameter that is used to tune the weight of the classification loss term during training.
$D$ and $G$ are the discriminator and the generator functions respectively.
The output of the discriminator function referred to as $D$ is the logit value of the "fake" class output of the discriminator.
This formula further decouples the discrimination and classification objectives and allows for more competition which can be tweaked by the $\gamma$ parameter.
ADAM optimizer \cite{kingmaAdamMethodStochastic2015} has been used to train both generator and discriminator.


\subsection{Training}
%data is from ModelNet10
% how is plain GAN trained
% from gan to conditional gan
% how to train inverse discriminator
% global fake label vs one fake per class

The training is performed for all three models concurrently.
In each training epoch, the training data from the ModelNet10 dataset containing all 10 classes is shuffled and split into minibatches of 16 samples.
Each minibatch is then fed into the plain and augmented CNN classifiers, as well as the discriminator.
The discriminator is trained in two stages, first with the training data and then with generated 'fake' data.
This fake-real training separation of gradient calculation has been shown to improve training stability \cite{chintalaHowTrainGAN2019}.
Experiments using both approaches (separating or mixing real and fake data) have been conducted, however, no significant difference has been observed.
One-sided label smoothing is also utilized to improve overfitting resistance.

\section{Results and Discussion}

% TODO Add filter comparison at all 3 layers between the 3 models

The main evaluation criterion of the designed models is the classification accuracy on the ModelNet10 dataset.
We also examine the generative performance of the GAN model to verify whether the performance benefit is not caused by some other effect unrelated to the learned latent representation.

\subsection{Classification}

All three models have been evaluated using the test data from the ModelNet10 dataset.
Classification accuracy, as well as mean average precision, have been calculated.
The results are shown in Table \ref{t:accuracy}. 
Several other modern models utilizing generative modeling with published ModelNet10 results are also included in the table.
The results show that generative enhancement of a very simple convolutional classifier (plain CNN) brings its average precision close to the state-of-the-art classifiers.

\begin{table}[h]
    \centering
    \caption{Comparison of classification accuracy and mean average precision score of generative modelling-based classifiers evaluated on the ModelNet10 dataset. \cite{wu3dShapenetsDeep2015}}
    \label{t:accuracy}
    \begin{tabular}{|l|r|r|}
    \hline & \multicolumn{1}{l|}{Accuracy {[}\%{]}} & \multicolumn{1}{l|}{Mean AP {[}\%{]}} \\ 
    \hline

    Primitive-GAN \cite{khanUnsupervisedPrimitiveDiscovery2019}   & 92.20                                  &                                  \\ \hline
    VIPGAN \cite{hanViewInterPredictionGAN2019}          & 94.05                                  & 90.69                                  \\ \hline
    Achlioptas et al. \cite{achlioptasLearningRepresentationsGenerative2018}      & 95.40                                  &                                  \\ \hline
    VRN Ensemble \cite{brockGenerativeDiscriminativeVoxel2016}    & 97.14                                  &                                  \\ \hline
    3D-GAN \cite{wuLearningProbabilisticLatent2016}    & 91.00                                  &                                  \\ \hline
    Plain CNN (ours)      & 83.04                                  & 83.75                                 \\ \hline
    Augmented CNN (ours)  & 84.58                                  & 86.31                                 \\ \hline
    GAN Classifier (ours) & 89.21                                  & 89.18                                 \\ \hline
    \end{tabular}
\end{table}

Another comparison of classification performance of our three models is shown in the ROC plot in Figure~\ref{f:roc} and
some additional insight into classification performance is provided by the confusion matrix in Figure~\ref{f:cm}.

%FIGURE - ROC
\begin{figure}
    \centering
    \includegraphics[width=.9\columnwidth]{roc}
    \caption{ROC curves for each of the three designed models. These curves are calculated by micro-averaging the curves for each class.}
    \label{f:roc}
\end{figure}

%FIGURE - confusion matrix
\begin{figure}
    \centering
    \includegraphics[width=1\columnwidth]{cm}
    \caption{Confusion matrix of the GAN classifier evaluated on the ModelNet10 testing set. The values are normalized in [0,1] range per each true label. It shows the most misclassified classes are desks, tables, and nightstands. This can most likely be attributed to low resolution and non-identical scale of the voxel models caused by the voxelization tool scaling the mesh models to fit the voxel grid volume.}
    \label{f:cm}
\end{figure}

While the accuracy of our models does not exceed the state of the art, the results show that training an identical classifier inside a GAN model instead of conventionally, using just the training dataset, can produce a significant classification accuracy improvement. 
In our case, the relative improvement from plain CNN to the GAN-trained classifier was $7.43\%$.

Comparing the augmented CNN to the GAN classifier, the difference is smaller ($5.47\%$ relative improvement), however still significant. 
This suggests, that the improvement is caused not only by introducing noise to the dataset. 
It also means the generator is not just a data augmentation mechanism, because in that case, the augmented CNN and GAN classifier should achieve similar performance.
The results suggest there exists some form of interaction between the generative and classifying part of a conditional GAN network that improves the classification accuracy of the model.

The interactions likely improve the generative performance as well, as other researchers recommend using class labels, if available when training a GAN network in general. \cite{chintalaHowTrainGAN2019}

\subsection{Object Generation}

We also examined the output of the generator to verify the model is working as expected.
The quality of the generated examples was not our primary goal, however, it can be an indicator of errors within the architecture design or implementation.
Several generated examples are shown in Figure \ref{f:generated}.
Mode collapse, a common pitfall for generative models, has been a major problem during our experimentation. 
Several solutions have been tested, such as using the Wasserstein loss function, however, the most significant improvement of both mode collapse and general convergence of the network was caused by initializing the ADAM optimizer with a very low learning rate $\alpha \approx 10^{-4}$.
As can be seen, there is a visible intra-class variance in the output of the generator. 
Some noise and artifacts are also produced, however, apparently, it does not cause any adverse effects during classification aside from the lower visual quality of the output.

% \subsection{Retrieval} todo ??????????????

%FIGURE - generator output
\begin{figure}
    \centering
      \includegraphics[width=.9\columnwidth]{generated}
      \caption{Output of the conditional generator using random input noise and random condition class vector. This generator has been trained on the ModelNet10 dataset.}
      \label{f:generated}
\end{figure}


\subsection{Performance and Timing}

We also investigated the computational performance of the proposed models.
As training has been performed concurrently, we can compare the accuracy evolution over training steps or epochs instead of time.
This reveals the data efficiency of the models instead of computing efficiency.
On our hardware (Intel i7 6700K CPU, 16GB RAM, Nvidia GeForce GTX 1080), the plain CNN classifier achieved its peak performance around step 15k. 
The augmented CNN classifier took 40k steps to reach its peak. 
The GAN classifier reached its maximum around step 88k.
This would not translate proportionately to training time if the models were trained separately, however it shows the complexity gained by generative enhancement incurs a significant hit to the training speed.

Inference speed has been measured separately and the results confirm the expected outcome: the generative enhancement does not significantly modify the classifier and its complexity does not increase enough to measurably reduce the inference computational performance.
The testing set containing 908 samples has been classified in 830ms (plain CNN), 826ms (augmented CNN) and 786ms (GAN). 
This translates to 1092, 1098 and 1153 samples per second respectively.

\section{Conclusion and Future Work}

In this paper, we proposed a methodology for generative enhancement of 3D image classifiers and experimentally verified its viability and evaluated its performance in a basic application scenario.

The results of the experiment support the initial idea, that a supervisedly trained conditional GAN network (i.e. generatively enhanced convolutional classifier) enables a knowledge transfer between the generator and a classifying discriminator that significantly improves its classification accuracy. 
The mechanism of these interactions is not clear and it could be a subject of further theoretical research.

%arguments for: improvement caused not just by adding noise or data augmentation
One of the possible explanations for the classification performance improvement is that the generator introduces noise to the training process of the discriminator/classifier and therefore makes it more robust.
This explanation is not completely sufficient as the same effect occurs in the augmented CNN model which does not indicate the same improvement.
The same applies to another explanation, which points to the data augmentation caused by the generator producing realistic but not identical training examples and there improving the robustness of the classifier.
This should also manifest in the augmented CNN model which, however, falls short of the CGAN classifier performance.

% Future work
The results of this study suggest a potential for the generative enhancement methodology, enhancing other existing well-performing classifiers by embedding them within a generative architecture.
Our next efforts will focus on exploring other candidates for this transformation and evaluating the resulting architectures.
Additionally, we plan to perform a physical experiment to verify the real-life application potential of the newly created models.
For this purpose, a custom rotary table 3D scanner is being designed to acquire scans of real-life objects which will be used for the model evaluation.
These efforts, including the existing hardware and software solutions, are part of the Technicom project~\cite{jadlovskyResearchActivitiesCenter2016}.

\begin{backmatter}

\section*{Competing interests}
The authors declare that they have no competing interests.

\section*{Author's contributions}
J.J. is responsible for the general research supervision, S.J. for research project management and funding acquisition.
J.J. and S.J. guided the overall direction of the article and methodology, provided technical guidance and simulation equipment.
M.V. was a major contributor in writing the manuscript.
All authors read and approved the final manuscript.

\section*{Acknowledgements}
Authors are thankful for the support from project KEGA ``Implementation of research results in the area of modeling
and simulation of cyber-physical systems into the teaching process - development of modern university textbooks'' – 072TUKE-4/2018.

\section*{Funding}
This work has been funded by grant KEGA ``Implementation of research results in the area of modeling
and simulation of cyber-physical systems into the teaching process - development of modern university textbooks'' – 072TUKE-4/2018.

\section*{Availability of data and materials}
The datasets analyzed during the current study are available on the public ModelNet website, \url{https://modelnet.cs.princeton.edu/}.

\bibliographystyle{bmc-mathphys}
\bibliography{new.bib}

\end{backmatter}

\end{document}