% Updated in February 2016 by Hwann-Tzong Chen
% Updated in May 2014 by Hideo Saito
% Updated in March 2012 by Yasuyuki Matsushita
% Updated in April 2002 by Antje Endemann, ...., and in March 2010 by Reinhard Klette
% Based on CVPR 07 and LNCS style, with modifications by DAF, AZ and elle 2008, AA 2010, ACCV 2010

\documentclass[runningheads]{llncs}
\usepackage{graphicx}
\usepackage{amsmath,amssymb} % define this before the line numbering.
% \usepackage{ruler}
\usepackage{color}
\usepackage{threeparttable}
\newcommand{\tabincell}[2]{\begin{tabular}{@{}#1@{}}#2\end{tabular}}
\newcommand{\etal}{\textit{et al.}}
\renewcommand\thetable{\Alph{table}}
%===========================================================
\begin{document}
\pagestyle{headings}
\mainmatter

\def\ACCV20SubNumber{887}  % Insert your submission number here

%===========================================================
\title{Supplementary Material for "Low-light Color Imaging via Dual Camera Acquisition"} % Replace with your title
\titlerunning{Supplementary Material}
% \authorrunning{ACCV-20 submission ID \ACCV20SubNumber}

\author{Peiyao Guo\inst{1}\orcidID{0000-0003-2887-3463} \and
Zhan Ma\inst{1}\orcidID{0000-0003-3686-4057}}
%
\authorrunning{P. Guo et al.}
% First names are abbreviated in the running head.
% If there are more than two authors, 'et al.' is used.
%
\institute{Nanjing University, Nanjing, China \\
\email{peiyao@smail.nju.edu.cn,~mazhan@nju.edu.cn}}
\maketitle


In this supplementary material, we provide specific technical details and extra results visualization for the main paper.
\section{Network Architecture }  
We summarize each module of our pipeline in Table~\ref{tab:network_configuration}. 
More details about PWCNet module could be found in~\cite{Sun2018PWCNetCF}\footnote{https://github.com/NVlabs/PWC-Net}.  
Besides, the slicing layer interpolates the reshaped color coefficients (layer 30) with the guidance map (layer 32) in a
 bilateral-grid upsampling way~\cite{gharbi2017deep}.
Afterwards, the color coefficients at high resolution are applied to the input monochromic image  $I_{v_2,s_H,l_H}^{\text{Y}}$ 
in the form of affine combination via the applying coefficients layer for chrominance reconstruction.


\renewcommand{\arraystretch}{1.2}
\begin{table}
  \centering
  \begin{threeparttable}
    \caption{Network configuration for our workflow. }
    \label{tab:network_configuration}
    \begin{tabular}{|c|c|c| }
      \hline
       ~\qquad&Layer Description& Output Tensor Dim.\\
      \hline
       & Input color image  $I_{v_1,s_L,l_L}^{\text{YUV}}$ & $3\times\frac{h}{4}\times\frac{w}{4}$ \\
      \hline
      & Input monochrome image  $I_{v_2,s_H,l_H}^{\text{Y}}$ & $ h \times w$ \\
      \hline
      \multicolumn{3}{|c|}{\textbf{RefEC net}} \\
      \hline
      1& $9\times 9$ conv & $64\times\frac{h}{4}\times\frac{w}{4} $\\
      2-6& ($3\times 3$ conv, ReLU) $\times$ 5 &$ 64\times\frac{h}{4}\times\frac{w}{4}$ \\
      7& $3\times 3$ conv, Sigmoid, extra bias & $3\times\frac{h}{4}\times\frac{w}{4}$ \\
      \hline
      \multicolumn{3}{|c|}{\textbf{RefColor net}} \\
      \hline
      8  & PWCNet module & $2\times\frac{h}{16}\times\frac{w}{16}$\\
      9  & upsampling \& warping layer &  $3\times\frac{h}{4}\times\frac{w}{4}$\\
      10  &$ 9\times9 $ conv&$64\times\frac{h}{4}\times\frac{w}{4}$ \\
      11 & $ 3\times3 $ conv, Batch Normalization , ReLU & $64\times\frac{h}{4}\times\frac{w}{4}$ \\
      12  & $ 3\times3 $ conv, Batch Normalizatoin  &  $64\times\frac{h}{4}\times\frac{w}{4}$ \\
      13  & skip connection between layer 10 and 12, ReLU &   $64\times\frac{h}{4}\times\frac{w}{4}$  \\
      24 & repeat (layer 11,12,13) $\times$ 4 & $64\times\frac{h}{4}\times\frac{w}{4}$ \\
      25 & $3\times 3$ conv, Sigmoid, extra bias & $3\times\frac{h}{4}\times\frac{w}{4}$ \\
      \hline
      \multicolumn{3}{|c|}{\textbf{RefSR net}} \\
      \hline
      26  & $3\times 3$ conv (stride 2), ReLU& $8\times\frac{h}{8}\times\frac{w}{8}$ \\
      27 & $3\times 3$ conv (stride 2), ReLU& $16\times\frac{h}{16}\times\frac{w}{16}$ \\
      28  & $3\times 3$ conv (stride 2), ReLU& $32\times\frac{h}{32}\times\frac{w}{32}$ \\
      29  & $3\times 3$ conv (stride 2), ReLU& $64\times\frac{h}{64}\times\frac{w}{64}$ \\
      30 & global \& local stream fusion with $1\times 1$ conv, ReLU & $96\times\frac{h}{64}\times\frac{w}{64}$\\
      \cline{1-3}
      31  & $3\times 3$ conv, ReLU& $16\times h\times w$ \\
      32  & $1\times 1$ conv, Tanh& $1\times h\times w$ \\
      \cline{1-3}
      33  & slicing layer & $12\times  h \times w$ \\
      34  & applying coefficients layer & $3\times  h \times w$ \\
      35  & $3\times 3$ conv, ReLU& $16\times h\times w$ \\
      36  & $1\times 1$ conv, Tanh& $1\times h\times w$ \\
      37 & skip connection between layer 25 and 36 &   $3\times h \times w$\\             
      \hline
      \hline
      \multicolumn{3}{|c|}{Local stream in layer 30} \\  
      \hline 
      &   $3\times 3$ conv, ReLU &$64\times\frac{h}{64}\times\frac{w}{64}$ \\
      &   $3\times 3$ conv, w/o ReLU &$64\times\frac{h}{64}\times\frac{w}{64}$ \\
      \hline
      \multicolumn{3}{|c|}{Global stream in layer 30} \\  
      \hline 
      &   $3\times 3$ conv (stride 2), ReLU &$64\times\frac{h}{128}\times\frac{w}{128}$ \\
      &   $3\times 3$ conv (stride 2), ReLU &$64\times\frac{h}{256}\times\frac{w}{256}$ \\
      &  FC(1024,256)&256\\
      &  FC(256,128)&128\\
      &  FC(128,64)&64\\
      \hline
    \end{tabular}
  \end{threeparttable}
\end{table}
% Please follow the steps outlined below when submitting your manuscript.
%Do not use any additional Latex macros.

%------------------------------------------------------------------------- 

%===========================================================
\section{Training Details}
Details on training process are listed in Table~\ref{tab:training_datasets_introduction} as the supplementary of  Table 1 and section 4.1 in the body paragraphs.    
Losses in Table~\ref{tab:training_datasets_introduction} are in the form as, 
\begin{equation*}
  \begin{aligned}
  \text{L1 loss:~\quad}~\ell( y,\hat{y}) &= \frac{\sum_{i=1}^n\mid y_i - \hat{y}_i\mid}{n}\\
  \text{MSE loss:~\quad}~\ell( y,\hat{y}) &= \frac{\sum_{i=1}^n(y_i - \hat{y}_i)^2}{n}\\
  \text{Cosine similarity loss:~\quad}~\text{similarity}(y,\hat{y}) &= \frac{y \cdot \hat{y}}{\max(\Vert y \Vert _2 \cdot \Vert \hat{y} \Vert _2, \epsilon)},\\
   ~\ell(y,\hat{y}) &=mean(\frac{1-\text{similarity}(y,\hat{y})}{2})
  \end{aligned}
\end{equation*}
\begin{table}
  \centering
  \begin{threeparttable}
    \caption{Training details}
    \label{tab:training_datasets_introduction}
    \begin{tabular}{l |cccc}
      \hline
      Module & \tabincell{c}{Input-color}&\tabincell{c}{Input-monochrome}&\tabincell{c}{GT-color}  & Loss \\
      \hline
      RefEC& ($v_1, s_L , l_L$)& ($v_2, s_L , l_H$)&($v_1, s_L , l_H$)& L1+ CosineSimlarity loss \\
      RefColor& ($v_1, s_L , l_H$)& ($v_2, s_L , l_H$)&($v_2, s_L , l_H$)& L1+ CosineSimlarity loss \\
      RefSR&($v_2, s_L , l_H$)& ($v_2, s_H , l_H$)&($v_2, s_H , l_H$)&MSE+ CosineSimlarity loss\\
      Overall&($v_1, s_L , l_L$)& ($v_2, s_H , l_H$)&($v_2, s_H , l_H$)& L1+ CosineSimlarity loss \\
      \hline
      \tabincell{c}{descrip-\\tion}& \multicolumn{4}{c}{\tabincell{l}{$v,s,l$ represent the viewpoint, spatial resolution and 
      light engergy specification \\
      of the image while detailed introduction is in the Table 1 of the manuscript.\\
      The loss in each task is calculated between the GT and corresponding predici-\\
      tion.          For faster convergence, we use MSE loss in the RefSR.}}\\
      \hline    
    \end{tabular}
  \end{threeparttable}
\end{table}

%===========================================================
\section{Ablation Study}
We have divided the overall problem defined in this paper using three consecutive operational subtasks, i.e., RefEC for illumination compensation, RefColor for image alignment and color transfer, and RefSR for final spatial resolution enhancement. 
Visualizations with partial subtasks disabled are shown in Fig.~\ref{fig:ab_study} while Table~\ref{tab:ablation_study} demonstrates the quantitative evaluation.

\begin{table}
  \centering
  \begin{threeparttable}
    \caption{Ablation study}
    \label{tab:ablation_study}
    \begin{tabular}{l |c|c|c}
      \hline
       &  $~\text{PSNR}_{YUV}/\text{dB}~$&$~\text{PSNR}_{UV}/\text{dB}~$&~MS-SSIM~\\
      \hline
      w/o RefEC& 37.22& 35.51&0.9773\\
      w/o RefColor& 31.94& 30.20&0.9552\\
      Overall &\textbf{38.60}&\textbf{38.91}&\textbf{0.9804}\\         
      \hline             
    \end{tabular}
  \end{threeparttable}
\end{table}
\begin{figure}[h]
  \centering
  \includegraphics[scale=0.072]{./Figs/ab_study.pdf}
  \caption{Visual performance with partial subtasks disabled}
  \label{fig:ab_study}
\end{figure}
%===========================================================
\section{User Study}
We randomly pick up 5 cases for qualitative performance validation with 25 subjects. 
% We randomly pick up 5 cases with different imaging content for user study.
First, we investigate the user preference of reconstrcuted image quality  at each intermediate step by asking the subjects to pickup their favorites. 
We enforce the random display order of algorithms used in each case. Results are shown in~\ref{fig:us_3ref}, revealing that our proposed algorithms are preferred in each subtask.
\begin{figure}[h]
  \centering
  \includegraphics[scale=0.4]{./Figs/us_3Ref.eps}
  \caption{Subjective preference distribution of each sub-task}
  \label{fig:us_3ref}
\end{figure}

We further invite subjects to rate the final reconstructed images from 1 to 5 scales (e.g., representing the Bad, Poor, Fair, Good, Excellent levels) as suggested in ITU-R BT.500-11. 
As depicted in Fig.~\ref{fig:average_mos}, the average score of reconstructed images reaches at 4.56, revealing the perceptual quality that is close to the “Excellent” scale. This validates the efficiency of our algorithm as well. 
\begin{figure}[h]
  \centering
  \includegraphics[scale=0.4]{./Figs/us_Overall.png}
  \caption{Subjective score distribution of 5 test cases}
  \label{fig:average_mos}
\end{figure}
%===========================================================
\section{Results Visualization}
\begin{figure}[t]
  \centering
    \includegraphics[scale=0.10]{./Figs/example/overall_example.pdf} 
   %\subfigure[]{\includegraphics[scale=0.8]{./Figs/giga/giga_descr5.pdf}\label{sfig:fov_navi_stat}}
  \caption{Overall simulation on Middlebury2014: (a) is the input color image with poor illumination at the low resolution $I_{v_1,s_L,l_L}^{\text{YUV}}$. ($\times$4 for illusration)
  (b) represents the monochrome input at the high resolution $I_{v_2,s_H,l_H}^{\text{Y}}$.  (c) and (d) show the prediction and the ground truth of the color image $I_{v_2,s_H,l_H}^{\text{YUV}}$. }
  \label{fig:overall_mb14}
\end{figure}

\begin{figure}[h]
  \centering
    \includegraphics[scale=0.12]{./Figs/example/test_result.pdf} 
   %\subfigure[]{\includegraphics[scale=0.8]{./Figs/giga/giga_descr5.pdf}\label{sfig:fov_navi_stat}}
  \caption{Overall performance on captured scenes: (a) is the input color image with poor illumination at the low resolution $I_{v_1,s_L,l_L}^{\text{YUV}}$. ($\times$4 for illusration)  
  (b) represents the monochrome input at the high resolution $I_{v_2,s_H,l_H}^{\text{Y}}$.  (c) presents the final reconstruction of the color image $I_{v_2,s_H,l_H}^{\text{YUV}}$.}
  \label{fig:overall_capture}
\end{figure}

In this section, we present more details of the overall test performance of our proposal.   
Fig.~\ref{fig:overall_mb14} illustrates the qualitative results on the simulated dataset (Middlebury2014~\cite{Scharstein2014High}).
And Fig.~\ref{fig:overall_capture} shows the complete performance on the captured scene via industrial cameras and Huawei P20, 
which supplements Fig. 8 in the main paper.
These visualization envidences our workflow's efficiency  in different low-light condition.
As referred in the last paragraph of Section 4, the only supervision of the re-colored HSR  
monochromic image influences the separative performance on  refEC, refColor and refSR,
inducing the slightly over-saturated color which is shown in $4^{th} $ row of Fig.~\ref{fig:overall_mb14}.
Besides, the color bleeding caused via the ill-measured gray channel correlation affects the final reconstruction's quality as shown in the third row of Fig.~\ref{fig:overall_mb14}.


% \textbf{RefEC.} Fig.~\ref{fig:example_ref_exposure} supplements the test scenes of refEC task and evidences our model's efficiency 
% in different poor light condition.
% \begin{figure}[b]
%   \centering
%   \includegraphics[scale=0.21]{./Figs/example/exposure_example.pdf} 
%   \caption{Qualitative results of different image enhancement algorithms:(a-b) are the input color image with 
%   low luminance and the reference monochrome image.
%    (c-g) respectively denotes the results of Histogram equalization, RetinexNet~\cite{Wei2018DeepRD}, 
%    SSIENet~\cite{Zhang2020SelfsupervisedIE}, Zero-DCE~\cite{Guo2020Zero-DCE} and our model. (h) is the ground truth with normal luminance.}
%   \label{fig:example_ref_exposure}
% \end{figure}

% \textbf{RefColor.} The colorization performance on various objects are provided in Fig.~\ref{fig:example_ref_colorization}.

% \begin{figure}[h]
%   \centering
%   \includegraphics[scale=0.25]{./Figs/example/colorization_example.pdf} 
%    %\subfigure[]{\includegraphics[scale=0.8]{./Figs/giga/giga_descr5.pdf}\label{sfig:fov_navi_stat}}
%   \caption{Colorization results of different proposals. (a-b) illustrates the reference color image and the input monochrome image.
%   (c-f) are the qualitative performance of Patch Match~\cite{welsh2002transferring}, {IAIC}~\cite{Su-CVPR-2020}, {DEC}~\cite{He2018DeepEC} and our proposal.
%   (g) denotes the ground truth of the colorized image.}
%   \label{fig:example_ref_colorization}
% \end{figure}

% \begin{figure}[h]
%   \centering
%     \includegraphics[scale=0.10]{./Figs/example/sr_example.pdf} 
%    %\subfigure[]{\includegraphics[scale=0.8]{./Figs/giga/giga_descr5.pdf}\label{sfig:fov_navi_stat}}
%   \caption{Qualitative results of super resolution proposals:(a) represents the input monochrome image at the high resolution. And another input color image is at the $1/4\times1/4$ scale of it. (b) illustrates 
% the details of corresponding green and yellow patches in  respectively, monochrome image, color image($\times$4 for display), RCAN~\cite{zhang2018rcan},  our method,  TTSR~\cite{yang2020learning} and ground truth.
%  (c) and (d) give the global presentation of our proposal and ground truth. }
%   \label{fig:example_ref_sr}
% \end{figure}

% \textbf{RefSR.} We give some examples for iphone-captured and nikon-captured subsets from City100, respectively illustrated in the $1^{st},2^{nd}$ and $3^{rd},4^{th}$ row 
% of Fig.~\ref{fig:example_ref_sr}.

%===========================================================

%References are listed in alphabetic order by the surname of the first author, or the identifying word (e.g., in case of a website). Have
%all anonymized references at the beginning of the list.

%here would be your acknowledgement (if any) in the final accepted paper

%===========================================================
\bibliographystyle{splncs}
\bibliography{egbib,0887}

%this would normally be the end of your paper, but you may also have an appendix
%within the given limit of number of pages
\end{document}
