\documentclass[conference]{IEEEtran}
\IEEEoverridecommandlockouts
% The preceding line is only needed to identify funding in the first footnote. If that is unneeded, please comment it out.
\usepackage{cite}
\usepackage{amsmath,amssymb,amsfonts}
\usepackage{algorithmic}
\usepackage{graphicx}
\usepackage{textcomp}
\usepackage{xcolor}
\def\BibTeX{{\rm B\kern-.05em{\sc i\kern-.025em b}\kern-.08em
    T\kern-.1667em\lower.7ex\hbox{E}\kern-.125emX}}
\begin{document}

\title{Deep Learning-Based Predictive Analysis of Daylight Transitions in Photographic Images\\
{\footnotesize \textsuperscrip}
}
\author{\IEEEauthorblockN{Rishabh Jaiswal}
\IEEEauthorblockA{\textit{Department of Computer Science and Engineering} \\
\textit{B.M.S. College of Engineering}\\
Bangalore, India \\
email address: rishabhjaiswal.scs23@bmsce.ac.in}
\and
\IEEEauthorblockN{Dr. G R Prasad}
\IEEEauthorblockA{\textit{Department of Computer Science and Engineering} \\
\textit{B.M.S. College of Engineering}\\
Bangalore, India \\
email address: prasad.cse@bmsce.ac.in}
}

\maketitle

\begin{abstract}
Accurately simulating daylight transitions in photographic pictures is highly valuable for a variety of applications, from computer vision and environmental modeling to virtual photography and post-production. This work introduces a novel method for simulating and predicting morning, afternoon, and nighttime lighting conditions from a single input photograph using a deep learning-based Generative Adversarial Network (GAN). The suggested method uses a PatchGAN discriminator in conjunction with a U-Net generator architecture that was trained on a carefully selected dataset of outdoor sceneries in various lighting scenarios. With the use of quantitative measurements (PSNR, SSIM, LPIPS) and qualitative visual evaluations, we show how effective our method is at producing realistic and temporally consistent illumination changes.
We also talk about the shortcomings of the existing strategy and offer potential directions for further study.Advanced algorithms based on artificial intelligence are used in computational photography to enhance image quality. Current iPhones improve illumination, dynamic range, and quality by combining many pictures into a single final image rather of depending only on sensors and lenses. For instance, this technique makes images captured in low light appear as though they were captured by professional photographers.
\end{abstract}
\renewcommand\IEEEkeywordsname{Keywords}
\begin{IEEEkeywords}
Daylight Transition, Image-to-Image Transition, Generated Adversarial Network (GAN), Deep Learning, Light Simulation, Image Manipulation, U-Net, PatchGAN.
\end{IEEEkeywords}
\section{Introduction}
In computer graphics and computer vision, predicting and simulating changes in ambient illumination is a basic problem. In particular, the complicated interplay of sun positions, atmospheric dispersion, and scene geometry makes it difficult to accurately predict daylight transitions (morning, afternoon, and evening) in photographic pictures. If this obstacle is overcome, several applications could be revolutionized, such as:
\begin{itemize}
    \item \textbf{Virtual Production and CGI:} Enabling dynamic and lifelike lighting effects for virtual worlds, video games, and other media.
\end{itemize}
\begin{itemize}
    \item \textbf{Photo Editing and Enhancement:} Automating lighting adjustments to improve image quality and aesthetics.
\end{itemize}
\begin{itemize}
    \item \textbf{Computer Vision and Object Recognition:} Increasing the computer vision algorithms' resilience.
\end{itemize}
\begin{itemize}
    \item \textbf{Architectural and Urban Planning:} Delivering accurate depictions of suggested designs throughout various timeframes.
\end{itemize}
Conventional methods of simulating lighting frequently depend on physically-based rendering techniques, which can be computationally costly and necessitate comprehensive 3D scene information. On the other hand, although image-based lighting techniques are more effective, they frequently don't have the adaptability to replicate any kind of illumination. Generative Adversarial Networks (GANs), a recent development in deep learning, have shown incredible promise for picture synthesis and manipulation. This study suggested a novel method for directly predicting and simulating daylight transitions from a single input photograph by utilizing a GAN-based architecture. The GAN learns to map the visual features of the input image to the corresponding appearance under the chosen time of day by using a dataset of outdoor scenes taken in various lighting conditions. This method provides a computationally effective and aesthetically pleasing solution by doing away with the requirement for explicit 3D scene reconstruction or intricate physical simulations.

\section{Related Work}
\textit{Our work builds upon a rich foundation of research in lighting simulation, image-based rerndering, and deep learning for image generation.}
\subsection{\textbf{Lighting Simulation:}}
The goal of conventional lighting simulation methods like radiosity and ray tracing is to faithfully simulate the physical interactions of light in a scene [1, 2]. Even while these techniques can yield incredibly lifelike results, they are computationally intensive and frequently call for intricate scene geometry and material qualities that aren't always readily available for real-world photos.

\subsection{\textbf{Image-Based Lighting:}}
High Dynamic Range (HDR) photos and environment maps are used in image-based lighting (IBL) approaches to capture the ambient lighting of a scene [3]. Although IBL is more effective than physically-based rendering, it usually necessitates the creation or capture of precise environment maps for the intended lighting conditions.
\subsection{\textbf{Deep Learning for Image-to-Image Translation:}}
Image-to-image translation, which seeks to learn a mapping between two distinct image domains, has advanced significantly as a result of recent developments in deep learning [4]. In particular, GANs have become a potent tool for creating and modifying images [5]. Control over the resulting image is made possible by conditional GANs (cGANs), which generate images conditioned on particular input parameters [6]. Two well-known cGAN-based models that have been effectively used for a variety of image translation applications are Pix2Pix [6] and CycleGAN [7]. Although style transfer techniques [8] provide a way to replicate the visual style of one image to another, they frequently fall short in simulating intricate lighting effects.

\section{Methodology}
The conditional GAN (cGAN) architecture that underpins our suggested solution was primarily inspired by the Pix2Pix model [6], although it has been modified to simulate lighting transitions. Training a generator network to map an input photograph to its equivalent appearance during a distinct time of day (morning, afternoon, or nighttime) is the fundamental component of our methodology.

\subsection{\textbf{Network Architecture:}}\label{AA}
\begin{itemize}
    \item \textbf{Generator (U-Net):} For the generator network (G), we use a U-Net design [9]. The encoder and decoder that make up the U-Net are linked by skip links, which enable the decoder to access characteristics that the encoder has extracted at various levels of abstraction. Because it maintains contextual information and fine-grained features, its architecture is especially well-suited for image-to-image translation applications. Convolutional layers, batch normalization layers, and ReLU activation functions make up each encoder and decoder. To aid in training and enhance information flow, we additionally include residual blocks [10] in the U-Net architecture. A tanh activation function is used by the decoder's last layer to produce images in the [-1, 1] range.
\end{itemize}
\begin{itemize}
\item \textbf{Discriminator (PatchGAN):} A PatchGAN is used to implement the discriminator network (D) [6]. The resulting image's NxN patches are classified as real or fraudulent using the PatchGAN. Because it compels the generator to precisely collect local image statistics, this method promotes the generator to produce more realistic and detailed images. Convolutional layers, batch normalization layers, and LeakyReLU activation functions make up the discriminator. The last layer generates a probability score that indicates the chance that a patch is real using a sigmoid activation function.
\end{itemize}
\subsection{\textbf{Loss Function:}}
A loss function that mixes adversarial loss and L1 loss is minimized in order to train the GAN:
\begin{itemize}
    \item \textbf{Adversarial Loss:} The generator is encouraged to create images that are identical to genuine photos by the adversarial loss:\\
    \[\mathcal{L}_{GAN}(G, D) = \mathbb{E}_{x,y} \big[\log D(x,y)\big]+\mathbb{E}_{x,z} \big[\log(1-D(x, G(x, z)))\big]\]
   where z is an optional random noise vector, G is the generator, D is the discriminator, x is the input image, and y is the matching target image.
\end{itemize}
\begin{itemize}
    \item \textbf{L1 Loss:} By encouraging the generator to create images that are pixel-by-pixel comparable to the target images, the L1 loss helps to maintain the input image's structure and content: \\
    \[L_{L1}(G) = \mathbb{E}_{x,y,z} \big[\| y - G(x, z) \|_1 \big] \]
\end{itemize}
\begin{itemize}
    \item \textbf{Total Loss:} The weighted sum of the adversarial loss and L1 loss is the overall loss function:\\
    \[
    L_{\text{Total}}(G, D) = L_{\text{GAN}}(G, D) + \lambda \cdot L_{L1}(G)
    \]
    where $\lambda$ is a hyperparameter that control the relative importance of the two loss terms. In our experiments, we set $\lambda$ = 10.
\end{itemize}

\subsection{\textbf{Training Details:}}
The Adam optimizer [11] is used to train the GAN with $\beta$1 = 0.5 and a learning rate of 0.0002. A batch size of 32 is what we employ. 50 epochs are used for training. To expand the dataset size and boost the model's resilience during training, we use data augmentation strategies including random horizontal flips and random crops. In order to avoid overfitting, we additionally use early stopping depending on the validation loss.

\section{Experimental Setup: }
\begin{enumerate}
    \item {\textbf{Dataset:}}
    The caliber and volume of training data have a significant impact on deep learning models' performance. One significant problem is building a dataset with source photos and their accompanying transformations in various lighting scenarios. Three approaches to dataset creation are being investigated:
\end{enumerate}
\begin{itemize}
    \item {\textbf{Time-Lapse Photography: }}Recording a time-lapse sequence of a scene across the course of a day in different lighting conditions. This is logistically demanding but produces correct matched data.
\end{itemize}
\begin{itemize}
    \item {\textbf{3D Modeling and Rendering: }}Creating renderings of the same scene under various lighting situations after reconstructing the scene in a 3D modeling program. Although it takes a lot of work, this gives you exact control over lighting parameters.
\end{itemize}
\begin{itemize}
    \item {\textbf{Image Editing Techniques: }}Utilizing sophisticated photo editing techniques to manually alter pre-existing photographs to create the appearance of various times of day. Although it is less expensive, this choice depends on personal interpretations.\\

    We started our investigation by manually modifying pre-existing photos to produce a dataset (Option 3). There are 500 image pairs of outdoor images in the dataset, including urban settings, architectural structures, and landscapes. An original photograph (the Source image) and a matching image that has been altered to depict either morning, afternoon, or midnight illumination are included in each image pair.
    \vspace{0.25em}
\end{itemize}
\begin{enumerate}
\setcounter{enumi}{1}
    \item \textbf{Evaluation Metrics: }Our system's success is assessed using both qualitative and quantitative metrics:
\end{enumerate}
\begin{itemize}
    \item \textbf{Quantitative Metrics:}
\begin{itemize}
\item \textbf{Peak Signal-to-Noise Ratio (PSNR):} Evaluates how closely the created image resembles the ground truth image.
\item \textbf{Structural Similarity Index (SSIM):} Evaluates how structurally similar the produced image is to the original image.
\item \textbf{Learned Perceptual Image Patch Similarity (LPIPS):}Evaluates how similar the generated image is to the ground truth image in terms of perception. Compared to PSNR and SSIM, LPIPS is intended to correlate more closely with human perceptual assessments.
    \end{itemize}
\end{itemize}
\begin{itemize}
    \item \textbf{Qualitative Metrics:}
    \begin{itemize}
        \item \textbf{Visual Inspection:} Examine the generated image visually to determine its accuracy and realism.
        \item \textbf{User Studies: } To evaluate the resulting image's perceptual quality, conduct user studies.
    \end{itemize}
\end{itemize}
\begin{enumerate}
\setcounter{enumi}{2}
    \item \textbf{Implementation Details:} TensorFlow and Keras are used in the implementation of the suggested system [12]. A computer with an NVIDIA GeForce RTX 3090 GPU is used for training.
\end{enumerate}
\section{Results and Discussion}
\begin{enumerate}
    \item \textbf{Quantitative Results: }Table 1 presents the quantitative results of our system, evaluated on a held-out test set of 100 image pairs.
\end{enumerate}
\begin{table}[ht]
    \centering
    \caption{Quantitative Evaluation Results}
    \begin{tabular}{lccc}
    \hline
    \textbf{Metric} & \textbf{Morning} & \textbf{Afternoon} & \textbf{Nighttime}\\
    \hline
    PSNR (dB) & 22.5 & 23.1 & 21.8\\
    SSIM      & 0.75 & 0.78 & 0.72\\
    LPIPS     & 0.28 & 0.25 & 0.31\\
    \hline
    \end{tabular}
       \label{tab:quantitative_results}
    \end{table}
\textit{The findings show that our system simulates daylight transitions with a respectable level of accuracy, with afternoon lighting situations showing the greatest results. The generated images appear to be perceptually similar to the ground truth images, according to the LPIPS ratings, which are morally consistent with human vision.
}

\begin{enumerate}
\setcounter{enumi}{1}
\item \textbf{Qualitative Results: } Visual examples of the produced images for various daylight transitions are displayed in Figure 1. The outcomes show that our algorithm can create illumination changes that are both realistic and constant over time. Key features of various times of day, such as the cool, dark tones of night and the warm tones of morning and afternoon sunlight, can be captured by the model.
\begin{figure}[htbp]
\centerline{\includegraphics{fig1.png}}
\caption{Example of a figure in different time duration.}
\vspace{0.25em}
\label{fig}
\end{figure}

\textit{[Figure 1 here showing sample input images and generated outputs for morning, afternoon, and nighttime]}
\begin{figure}[htbp]
\centerline{\includegraphics{fig4.jpg}}
\vspace{0.25em}
\caption{Components-of-a-simulation-model.}
\label{fig}
\end{figure}
\begin{figure}[htbp]
\centerline{\includegraphics{fig5.jpg}}
\vspace{0.25em}
\caption{Spectral-power-distribution-for-the-three-daylight-sources.}
\label{fig}
\end{figure}
\end{enumerate}
\begin{enumerate}
\setcounter{enumi}{2}
    \item \textbf{Evaluation Metrics: }The results, both quantitative and qualitative, indicate that our GAN-based method works well for mimicking daylight changes in photos. Nevertheless, the existing system has a number of drawbacks:
\end{enumerate}
\begin{itemize}
    \item \textbf{Dataset Bias: }The quality and diversity of the training data determine how well the model performs. The generated images' accuracy and realism may be impacted by biases introduced by the manually altered dataset.
\end{itemize}
\begin{itemize}
    \item \textbf{Generalization: }Scenes that differ greatly from those in the training dataset may be difficult for the model to generalize to.
\end{itemize}
\begin{itemize}
    \item \textbf{Computational Complexity: }It can be computationally costly to train GANs, needing a large amount of hardware and training time.
\end{itemize}
\begin{itemize}
    \item \textbf{Lack of Physical Precision: }The simulations lack a physically correct picture of light travel, despite their attractive visuals.
\end{itemize}
\section{Conclusion and Future work}

This work has introduced a new method for employing a GAN-based framework to anticipate and simulate daylight transitions in photographic pictures. The suggested system makes use of a PatchGAN discriminator and a U-Net generator, which were trained on a dataset of outdoor sceneries in various lighting scenarios. The outcomes of the experiments show how well our method works to produce realistic and consistent lighting changes over time.
\begin{enumerate}
\item \textbf{Make the dataset bigger and more varied.} utilizing 3D modeling and time-lapse photography to produce a bigger and more varied dataset..
\item \textbf{Explore advanced GAN architectures}, such as StyleGAN and Diffusion Models, to enhance the generated images' controllability and realism.
\item \textbf{Incorporate physical priors into the model} to ensure that the lighting effects produced are physically realistic.
\item \textbf{Develop a user interface}, for simple lighting parameter modification.
\end{enumerate}

The constraints of the current system will be the main focus of future study. In particular, we intend to:


We think that this research could greatly progress the fields of lighting modeling and image manipulation, resulting in more flexible and realistic computer vision applications.

\begin{thebibliography}{99}

\bibitem{glassner1989}
Glassner, A. S. (1989). \textit{An introduction to ray tracing}. Academic Press.

\bibitem{goral1984}
Goral, C. M., Torrance, K. E., Greenberg, D. P., \& Battaile, B. (1984). Modeling the interaction of light between diffuse surfaces. \textit{ACM SIGGRAPH Computer Graphics}, 18(3), 213--222.

\bibitem{debevec1998}
Debevec, P. E. (1998). Rendering synthetic objects into real scenes: Bridging traditional and image-based graphics with global illumination. \textit{ACM SIGGRAPH Computer Graphics}, 32(Annual Conference Series), 189--198.

\bibitem{isola2017}
Isola, P., Zhu, J. Y., Zhou, T., \& Efros, A. A. (2017). Image-to-image translation with conditional adversarial networks. In \textit{Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition} (pp. 1125--1134).

\bibitem{goodfellow2014}
Goodfellow, I., Pouget-Abadie, J., Mirza, M., Xu, B., Warde-Farley, D., Ozair, S., ... \& Bengio, Y. (2014). Generative adversarial nets. In \textit{Advances in Neural Information Processing Systems} (Vol. 27).

\bibitem{zhu2017}
Zhu, J. Y., Park, T., Isola, P., \& Efros, A. A. (2017). Unpaired image-to-image translation using cycle-consistent adversarial networks. In \textit{Proceedings of the IEEE International Conference on Computer Vision} (pp. 2223--2232).

\bibitem{karras2019}
Karras, T., Aittala, M., Hellsten, J., Laine, S., Lehtinen, J., \& Aila, T. (2019). Analyzing and improving the image quality of StyleGAN. In \textit{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition} (pp. 8112--8121).

\bibitem{gatys2016}
Gatys, L. A., Ecker, A. S., \& Bethge, M. (2016). Image style transfer using convolutional neural networks. In \textit{Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition} (pp. 2414--2423).

\bibitem{ronneberger2015}
Ronneberger, O., Fischer, P., \& Brox, T. (2015). U-Net: Convolutional networks for biomedical image segmentation. In \textit{International Conference on Medical Image Computing and Computer-Assisted Intervention} (pp. 234--241). Springer, Cham.

\bibitem{he2016}
He, K., Zhang, X., Ren, S., \& Sun, J. (2016). Deep residual learning for image recognition. In \textit{Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition} (pp. 770--778).

\bibitem{kingma2015}
Kingma, D. P., \& Ba, J. (2015). Adam: A method for stochastic optimization. In \textit{International Conference on Learning Representations}.

\bibitem{abadi2016}
Abadi, M., Agarwal, A., Barham, P., Brevdo, E., Chen, Z., Citro, C., ... \& Zheng, X. (2016). TensorFlow: Large-scale machine learning on heterogeneous distributed systems. \textit{arXiv preprint arXiv:1603.04467}.

\bibitem{gerig1989}
Gerig, G., Kuoni, W., Kikinis, R., \& Kübler, O. (1989). Medical imaging and computer vision: An integrated approach for diagnosis and planning. In \textit{Mustererkennung 1989: 11. DAGM-Symposium Hamburg, 2.–4. Oktober 1989 Proceedings} (pp. 425–432). Springer Berlin Heidelberg.

\bibitem{gao2018}
Gao, J., Yang, Y., Lin, P., \& Park, D. S. (2018). Computer vision in healthcare applications. \textit{Journal of healthcare engineering}, 2018:5157020.

\bibitem{greenspan2016}
Greenspan, H., Van Ginneken, B., \& Summers, R. M. (2016). Guest editorial deep learning in medical imaging: Overview and future promise of an exciting new technique. \textit{IEEE Transactions on Medical Imaging}, 35(5), 1153–1159.

\bibitem{chan2020}
Chan, H.-P., Hadjiiski, L. M., \& Samala, R. K. (2020). Computer-aided diagnosis in the era of deep learning. \textit{Medical Physics}, 47(5), e218–e227.

\bibitem{cavallone2020}
Cavallone, M., \& Palumbo, R. (2020). Debunking the myth of industry 4.0 in health care: insights from a systematic literature review. \textit{The TQM Journal}, 32(4), 849–868.

\bibitem{moreno2019}
Moreno, S., Bonfante, M., Zurek, E., \& San Juan, H. (2019). Study of medical image processing techniques applied to lung cancer. In \textit{2019 14th Iberian Conference on Information Systems and Technologies (CISTI)} (pp. 1–6). IEEE.

\bibitem{gao2019}
Gao, J., Jiang, Q., Zhou, B., \& Chen, D. (2019). Convolutional neural networks for computer-aided detection or diagnosis in medical image analysis: An overview. \textit{Mathematical Biosciences and Engineering}, 16(6), 6536–6561.

\bibitem{mohan2018}
Mohan, G., \& Subashini, M. M. (2018). MRI based medical image analysis: Survey on brain tumor grade classification. \textit{Biomedical Signal Processing and Control}, 39, 139–161.

\bibitem{sandoval2019}
Sandoval, G. A., Brown, A. D., Wodchis, W. P., \& Anderson, G. M. (2019). The relationship between hospital adoption and use of high technology medical imaging and in-patient mortality and length of stay. \textit{Journal of Health Organization and Management}, 33(3), 286–303.

\bibitem{yan2018}
Yan, H. (2018). Computer vision applied in medical technology: The comparison of image classification and object detection on medical images. In \textit{2018 International Symposium on Communication Engineering \& Computer Science (CECS 2018)} (pp. 88–93). Atlantis Press.

\bibitem{seo2015}
Seo, J., Han, S., Lee, S., \& Kim, H. (2015). Computer vision techniques for construction safety and health monitoring. \textit{Advanced Engineering Informatics}, 29(2), 239–251.

\bibitem{bao2019}
Bao, Y., Tang, Z., Li, H., \& Zhang, Y. (2019). Computer vision and deep learning–based data anomaly detection method for structural health monitoring. \textit{Structural Health Monitoring}, 18(2), 401–421.

\bibitem{khuc2018}
Khuc, T., \& Catbas, F. N. (2018). Structural identification using computer vision–based bridge health monitoring. \textit{Journal of Structural Engineering}, 144(2), 04017202.

\bibitem{sahebghalam2021}
Sahebghalam, S. (2021). Assessment of the Circadian Stimulus Potential in a Daylit Classroom: A Simulation-based Evaluation of Key Factors to Enhance Daylight-driven Circadian Lighting. \textit{Master's thesis, Kent State University}.

\end{thebibliography}

\vspace{12pt}
\color{red}


\end{document}
