%Version 3 October 2023
% See section 11 of the User Manual for version history
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                                                 %%
%% Please do not use \input{...} to include other tex files.       %%
%% Submit your LaTeX manuscript as one .tex document.              %%
%%                                                                 %%
%% All additional figures and files should be attached             %%
%% separately and not embedded in the \TeX\ document itself.       %%
%%                                                                 %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%\documentclass[referee,sn-basic]{sn-jnl}% referee option is meant for double line spacing

%%=======================================================%%
%% to print line numbers in the margin use lineno option %%
%%=======================================================%%

%%\documentclass[lineno,sn-basic]{sn-jnl}% Basic Springer Nature Reference Style/Chemistry Reference Style

%%======================================================%%
%% to compile with pdflatex/xelatex use pdflatex option %%
%%======================================================%%

%%\documentclass[pdflatex,sn-basic]{sn-jnl}% Basic Springer Nature Reference Style/Chemistry Reference Style


%%Note: the following reference styles support Namedate and Numbered referencing. By default the style follows the most common style. To switch between the options you can add or remove Numbered in the optional parenthesis. 
%%The option is available for: sn-basic.bst, sn-vancouver.bst, sn-chicago.bst%  
 
%%\documentclass[sn-nature]{sn-jnl}% Style for submissions to Nature Portfolio journals
%%\documentclass[sn-basic]{sn-jnl}% Basic Springer Nature Reference Style/Chemistry Reference Style
\documentclass[sn-mathphys-num]{sn-jnl}% Math and Physical Sciences Numbered Reference Style 
%%\documentclass[sn-mathphys-ay]{sn-jnl}% Math and Physical Sciences Author Year Reference Style
%%\documentclass[sn-aps]{sn-jnl}% American Physical Society (APS) Reference Style
%%\documentclass[sn-vancouver,Numbered]{sn-jnl}% Vancouver Reference Style
%%\documentclass[sn-apa]{sn-jnl}% APA Reference Style 
%%\documentclass[sn-chicago]{sn-jnl}% Chicago-based Humanities Reference Style

%%%% Standard Packages
%%<additional latex packages if required can be included here>

\usepackage{graphicx}%
\usepackage{booktabs} % 可选，用于更美观的表格线（例如 \hline）
\usepackage{multirow}%
\usepackage{amsmath,amssymb,amsfonts}%
\usepackage{amsthm}%
\usepackage{mathrsfs}%
\usepackage[title]{appendix}%
\usepackage{xcolor}%
\usepackage{textcomp}%
\usepackage{manyfoot}%
\usepackage{booktabs}%
\usepackage{algorithm}%
\usepackage{algorithmicx}%
\usepackage{algpseudocode}%
\usepackage{listings}%
\newtheorem{prop}{Proposition}
%%%%

%%%%%=============================================================================%%%%
%%%%  Remarks: This template is provided to aid authors with the preparation
%%%%  of original research articles intended for submission to journals published 
%%%%  by Springer Nature. The guidance has been prepared in partnership with 
%%%%  production teams to conform to Springer Nature technical requirements. 
%%%%  Editorial and presentation requirements differ among journal portfolios and 
%%%%  research disciplines. You may find sections in this template are irrelevant 
%%%%  to your work and are empowered to omit any such section if allowed by the 
%%%%  journal you intend to submit to. The submission guidelines and policies 
%%%%  of the journal take precedence. A detailed User Manual is available in the 
%%%%  template package for technical guidance.
%%%%%=============================================================================%%%%

%% as per the requirement new theorem styles can be included as shown below
\theoremstyle{thmstyleone}%
\newtheorem{theorem}{Theorem}%  meant for continuous numbers
%%\newtheorem{theorem}{Theorem}[section]% meant for sectionwise numbers
%% optional argument [theorem] produces theorem numbering sequence instead of independent numbers for Proposition
\newtheorem{proposition}[theorem]{Proposition}% 
%%\newtheorem{proposition}{Proposition}% to get separate numbers for theorem and proposition etc.

\theoremstyle{thmstyletwo}%
\newtheorem{example}{Example}%
\newtheorem{remark}{Remark}%

\theoremstyle{thmstylethree}%
\newtheorem{definition}{Definition}%

\raggedbottom
%%\unnumbered% uncomment this for unnumbered level heads

\begin{document}

\title[Article Title]{HIAT: Human-in-the-Loop Reinforcement Learning with Auxiliary Task}

%%=============================================================%%
%% GivenName	-> \fnm{Joergen W.}
%% Particle	-> \spfx{van der} -> surname prefix
%% FamilyName	-> \sur{Ploeg}
%% Suffix	-> \sfx{IV}
%% \author*[1,2]{\fnm{Joergen W.} \spfx{van der} \sur{Ploeg} 
%%  \sfx{IV}}\email{iauthor@gmail.com}
%%=============================================================%%
\author[1]{\fnm{Bo} \sur{Niu}}\email{niuboboo@csu.edu.cn}

\author*[1]{\fnm{Biao} \sur{Luo}}\email{biao.luo@hotmail.com}


\author[1]{\fnm{Yongzheng} \sur{Cui}}\email{yongzhengcui@csu.edu.cn}

\author[1]{\fnm{Xiaodong} \sur{Xu}}\email{xx1@ualberta.ca}

\author[1]{\fnm{Yuqian} \sur{Zhao}}\email{zyq@csu.edu.cn}
\author[2]{\fnm{Yu} \sur{Feng}}\email{yfeng@zjut.edu.cn}


\affil[1]{\orgdiv{School of Automation}, \orgname{Central South University}, \orgaddress{\city{Changsha}, \postcode{410083}, \state{Hunan}, \country{China}}}
\affil[2]{\orgdiv{Information engineering College}, \orgname{ Zhejiang  University of Technology}, \orgaddress{\city{Hangzhou}, \postcode{310014}, \state{Zhejiang}, \country{China}}}


%%==================================%%
%% Sample for unstructured abstract %%
%%==================================%%

\abstract{Human-in-the-loop reinforcement learning (HIRL) tackles the challenges of inefficient exploration and low sample efficiency by incorporating human feedback into the learning process to guide exploration and speed up learning.
Nevertheless, most recent approaches overlook the problem of sample imbalance caused by varying behavioral policy distributions in the early stages of training, \textcolor{red}{which results in policy performance degradation due to biased learning toward dominant low-quality data.} In this paper, we propose a human-in-the-loop reinforcement learning method by constructing an auxiliary task, namely Human-in-the-Loop Reinforcement Learning with Auxiliary Task (HIAT). Specifically, state-action pairs are utilized to create an auxiliary task that assesses the similarity between different policies, \textcolor{red}{forcing the model to focus on the differences in data sources.} The HIAT method guides the learning process of new policies during the policy evaluation phase by implicitly influencing the assessment of different actions, thereby compensating for sample imbalance. Subsequently, we introduce an adaptive weighting version, namely adaptive weighting based HIAT (AWHIAT), which adjusts the impact of the auxiliary task on the primary task based on their task-relevance and mitigates conflicts between different tasks caused by cognitive load. Through empirical evaluations on six continuous control benchmarks, the results demonstrate that AWHIAT significantly improves mean episode reward and sample efficiency compared to four representative HIRL methods.}

\keywords{Reinforcement learning, Human-in-the-loop, Deep learning, Auxiliary task}

%%\pacs[JEL Classification]{D8, H51}

%%\pacs[MSC Classification]{35A01, 65L10, 65L12, 65L20, 65L70}

\maketitle

\section{Introduction}\label{sec1}
Reinforcement learning (RL) uses exploration and trial-and-error mechanisms to learn the optimal policy, solving sequential decision-making problems autonomously \cite{sutton1998reinforcement,Hu2023MO-MIX}. In recent years, deep reinforcement learning (DRL) has emerged by combining RL’s credit assignment capabilities with the function approximation potential of deep neural networks. This approach has achieved significant success in various fields. For instance, it excels in industrial process control \cite{IFGONGYEDENG2023221,IFgongyeZHENG2023119684}, recommendation systems \cite{resys,Zhou2024SMONAC}, multi-agent systems \cite{zhouyuhao, luo2025survey} and playing Go and various video games \cite{alphago,DQN}.
However, RL is often criticized for its low sample efficiency due to the trial-and-error mechanism and excessive ineffective exploration \cite{sample-eff,yangbenxiaolv1}. 
%However, reinforcement learning is often criticized for its low sample efficiency and the data cold start issues  due to trial-and-error mechanism and excessive ineffective explorations \cite{sample-eff} \cite{yangbenxiaolv1}.
%These limitations hinder its application in many real-world problems, especially in high-dimensional observations, such as complex continuous space.

Incorporating human prior knowledge has been proven to be an effective solution to problems in existing RL algorithms \cite{IFhumanPERRUSQUIA2021358}. 
Human intervention skillfully guides the agent towards  deeper, more insightful exploration, thus significantly boosting the efficiency of data sampling \cite{yangbenxiaolv2}. 
This approach is commonly known as human-in-the-loop reinforcement learning (HIRL) \cite{Hirl1,hirl3}. 
The core idea of HIRL is to establish a feedback model that interacts with human experts to facilitate the learning process of intelligent agents. Through the feedback mechanism, human experts can express preferences for the current actions of the  agent or directly provide demonstration actions for the agent to learn from. The training of the agent relies on the interaction data between the agent and the environment, as well as the demonstration data from human experts.
Regrettably, the data obtained from the interaction of various policies with environments are characterized by class imbalance, which adversely affects the performance of HIRL. 
In HIRL interactions, actions are specifically generated by either human or agent policies. Relative to agent actions, those originating from human policies are superior in quality and thus carry significant instructional value. However, the quantity of human actions is limited and scarce due to the high costs associated with human labour.
The volume of data generated by the agent is higher; due to the trial-and-error nature of RL, the learning value of the data is diminished.
 %人在回路提高的学习效率有限，问题分类不平衡
The issue of class imbalance leads to the trained model severely overfitting to the more abundant categories, neglecting the higher-quality data, thereby resulting in a decline in the policy performance of HIRL.
Therefore, balancing and utilizing data effectively is crucial for enhancing the performance of HIRL.

\textcolor{red}{
Furthermore, human policies in HIRL are constrained by cognitive load, which significantly affects the quality of demonstrations \cite{hirl2}. As training duration increases, humans face a growing cognitive burden, resulting in reduced attention and suboptimal decision-making. This cognitive fatigue leads to a decline in human policy performance and conflicts in HIRL’s optimization objectives. However, current HIRL approaches lack effective methods to address this issue.}

One approach involves specifically having humans offer binary preferences for the data derived from interactions. Subsequently, a reward function that incorporates these preference details is trained, avoiding direct intervention with the agent policy. This strategy effectively mitigates the issue of class imbalance \cite{HumanPreferences,WeakHumanPreference}. However, this method struggles in more complex application scenarios where clear evaluations are unattainable.
It is difficult to quantify the information of different data trajectories only in terms of good or bad in a complex environment, which can easily lead to misjudgments by the agent.
Additionally, there are supervised learning methods, including directly modeling offline human demonstrations \cite{bc1,bc2,bc3}. This approach requires a large volume of high-quality human demonstrations for effective fitting. However, in most complex control domains, obtaining data built by the optimal policy
is challenging. Low-quality data is always inevitable, leading to compound errors in models and resulting in poor generalization.
%事实上，class imbalance的数据也可以用来提高
Compared to the methods mentioned above, constructing fitting auxiliary tasks for HIRL has more potential. In deep learning, auxiliary tasks are utilized to tackle the challenge of class imbalance \cite{AT1.11,biaozheng_at} and many studies have proven that transferring knowledge from other auxiliary tasks can enhance the efficiency of RL \cite{at2,at3}. 
Due to conflicting optimization objectives, auxiliary tasks may not always be beneficial to the primary task \cite{at1,at6}. Additionally, there are no effective auxiliary task construction methods for addressing the class imbalance issue in HIRL.

Inspired by the above discussion, we propose a new method for constructing human-in-the-loop reinforcement learning with auxiliary task (HIAT) to balance and utilize data effectively, thereby enhancing the performance of HIRL.
%这里重新凝练一下
Specifically, we utilize the auxiliary task to learn the distinction between different policies, and then share the learned representation with the primary task.
Subsequently, to mitigate potential conflicts between tasks and address the variable human policy quality caused by cognitive load, we introduce an adaptive weighting version, namely adaptive weighting based HIAT (AWHIAT). This approach dynamically adjusts the impact of the auxiliary task on the primary task based on their task relevance, and can effectively compensate for conflicts in auxiliary task objectives caused by cognitive fatigue. When human policy show inconsistencies typical of increased cognitive burden, AWHIAT recalibrates the weighting mechanism to maintain effective learning even under non-ideal human guidance conditions.
% Subsequently, to mitigate potential conflicts between tasks, we introduce an adaptive weighting version, namely adaptive weighting based HIAT (AWHIAT), which adjusts the impact of the auxiliary task on the primary task based on their task relevance.
%To balance the losses between the two tasks, we further introduce an adaptive weighting version, namely adaptive weighting based HIAT (AWHIAT). The impact of the auxiliary task on the primary task is adjusted by AWHIAT based on their task-relevance.
Finally, we conduct comparative evaluations with some HIRL algorithms in complex control environments. Our method achieved significant improvements in terms of final performance, sample efficiency, and stability in the MuJoCo experiments compared to existing state-of-the-art baseline methods. For instance, HIAT demonstrated a 34.3\% improvement in sample efficiency and a 7.02\% increase in average reward over 1000k timesteps compared to the best baseline algorithm in the Ant environment, we also obtained competitive experimental results in several other environments.


\section{Related Works} \label{Sec_2}

The current research on HIRL focuses on developing reward engineering to enhance learning efficiency. Specifically, interaction data is stored as trajectories, with humans offering feedback categorizing them as good, bad, or neutral according to their preferences \cite{HumanPreferences,WeakHumanPreference}. 
Agents utilize segments of these trajectories and feedback data to train reward engineering. Others have built upon this foundation to design frameworks based on offline reinforcement learning \cite{DT}. 
This method replaces direct human intervention with additional human preference labels in the buffer.
However, the direct reward engineering framework by humans only obtains human preferences after interactions and cannot acquire explicit human guidance in real-time to enhance exploration efficiency. 
%Additionally, it requires human involvement throughout the entire training process. 
In contrast, our approach encourages agents to autonomously explore the environment after acquiring certain capabilities, aiming to develop policy that surpass human experts. 
%Direct human control and guidance of the agent in the early stages of training can more effectively achieve this vision.

Unlike reward engineering, constructing auxiliary tasks for actors and critics is also considered a method to solve current problems \cite{at1,at2}. Some related work involves constructing behavior cloning supervised learning auxiliary tasks in the actor network using data from the human experience pool to achieve better performance \cite{hirl2,IA_RL}. However, this has not yet been widely applied in the HIRL domain. Unlike direct impacts on behavioral policy, critics' auxiliary tasks improve the Q-value estimation ability of the Q function, thereby indirectly influencing outcomes. Some research also constructs auxiliary tasks for the critic network \cite{lossreward,Luo2024Human}. Specifically, QDP is built from the perspective of the advantage function as an auxiliary task for the critic network. QDP captures the relative effectiveness of human expert Q-value actions and agent Q-value actions, timely correcting the network's update direction.

Our work is related to QDP in constructing auxiliary tasks for the critic network. However, our method not only obtains better network representation capabilities through shared network layers, but also adjusts task weights according to the policy learning process. In contrast, QDP relies entirely on the optimization gradient of auxiliary tasks throughout, which may be affected by the optimization objective of the auxiliary task in the later stages of training.

\section{ Preliminaries} \label{Sec_3}
\subsection{Markov Decision Process}
The RL problem is usually formulated as a Markov decision process (MDP) \cite{MarkovDecisionProcesses}. 
It is defined by a tuple $\mathcal{M}=(\mathcal{S}, \mathcal{A}, \mathcal{R}, \mathcal{P}, \gamma)$, where $\mathcal{S}$ represents the state space describing all possible states in the agent's environment, $\mathcal{A}$ is the action space representing all possible actions, $\mathcal{P}$ indicates the state-transition probabilities, $\mathcal{R}$ is the reward function for state-action pairs, and $\gamma$ is the discount factor.
When the environment is complex or not fully observable, trying to achieve $\mathcal{P}$ is difficult; thus, agents interact with the environment by directly learning a policy $\pi$. This policy is typically learned through trial-and-error, where the agent chooses an action $ a \in \mathcal{A}$ according to its policy function $ a \sim \pi(s)$, leading the environment to update its state to $s^{\prime} \in \mathcal{S}$ and the agent receives a reward $ r \sim \mathcal{R}(s, a)$.
These empirical data are crucial to the agent's learning process as they capture the essential information, allowing the agent to infer state transition probabilities accurately.
Instead of explicitly modeling the environment, the agent prioritizes continuous optimization of expected returns. It interacts with the environment, gathering data as tuples $\left( s, a, r, s' \right)$. 
Considering the state-action pair $(s, a)$ at a specific time step $t$, the return is characterized by a discounted sum of future rewards
\begin{equation}\label{abstraction function}
% \resizebox{0.6\linewidth}{!}{$
\mathbf{U}_t= \sum_{t^{\prime}=t}^{\infty} \gamma^{t'-t} r \left( s_{t'},a_{t'}\right).
% $}
\end{equation}
Using these data samples, the agent enhances its policy $\pi$ to approximate the optimal policy $\pi^*$.


\subsection{Human-in-the-loop Reinforcement Learning}\label{HILRL}
Unlike in standard actor-critic RL setups, humans can be involved in the training process in HIRL \cite{nanda_YUYANG}.
The interactive framework of HIRL is illustrated in Fig. \ref{renzaihuilu}.
The agent receives observations of the current state $s$ from the environment,   then a human policy, denoted as $\pi_{h}$, is established to guide the agent's behavior. The agent automatically switches between the human policy $\pi_{h}$ and the agent policy $\pi$. If a human intervenes, the current action $a$ will be generated by the human policy instead of the current agent policy \cite{IA_RL,hirl2}.
Interaction data under human policy guidance is stored alongside agent interaction data in an experience replay pool, enabling the agent to sample from it during training.
\begin{figure}[h]
    \centering
    \includegraphics[width=0.7\columnwidth]{picture/HIRL.pdf}
    \caption{Framework for HIRL. Similar to the interaction process in model-free reinforcement learning, the agents in HIRL acquire current observations from the environment and take actions based on these observations. The difference lies in the fact that human experts can guide the interaction between the agents and the environment through specific criteria judgments. }
    \label{renzaihuilu}
\end{figure}
During the initial training phase, the human policy is often considered superior to the current policy, which can be regarded as an approximation of the optimal policy \cite{Luo2024Human}. To incorporate human intervention at the appropriate time, a triggering condition $I(s)$ is formulated based on the current state $s_{t}$.


This current policy is represented as follows:
\begin{equation}\label{switch}
% \resizebox{0.7\linewidth}{!}{$
    \pi'(\cdot|s)=\begin{cases}
    \pi_h(\cdot|s), &I(s) = 1 \\
    \pi(\cdot|s), & I(s) = 0
    % \pi^h(\cdot|s) & \scriptstyle\text{if } \boldsymbol{I}(s_{t}) \text{ is True}. \\
    %\pi(\cdot|s) & \scriptstyle\text{otherwise}.
    \end{cases}
    % $}
\end{equation}
where $\pi'(\cdot|s)$ represents the current behavioral policy. $I(s)= 0/1$ denotes without/with human intervention. The triggering condition $I(s)$ is based on the action-value function $Q^{\pi}(s, a)$, which is defined as:  
\begin{equation}
    % \resizebox{0.7\linewidth}{!}{$
    Q^{\pi}(s,a)=\mathbb{E}_{\pi}\left[ \mathbf{U}_t \mid s_{t}=s, a_{t}=a\right].
    % $$}
\end{equation}%
%$Q^{\pi}$ is approximated by the Q-network \cite{biaozheng2023_CVPR}. 
\textcolor{red}{In some representative DRL methods such as \cite{TD3,SAC}, multiple Q-networks are employed to enhance the accuracy of Q-value estimation. If the deviation between Q-value estimates from different Q-networks exceeds a predefined threshold, it can be inferred that the agent has poor cognition of the current situation, prompting a request for human intervention. 
To quantify such deviations and dynamically monitor the agent's performance, QDP \cite{Luo2024Human}  define the critic network discrepancy \( T(s) \) for the current state-action pair as:
\begin{equation}
    T(s) = Q_{1}(s, a) - Q_{2}(s, a),
\end{equation}
where \( Q_{i=1,2}(s, a) \) represents the expected return estimated by different critic networks, and \( T(s) \) captures the difference between these estimates. To track the variations of \( T(s) \) over time, we introduce a fixed-length FIFO queue \( \mathcal{Q} \), which stores the most recent values of \( T(s) \): 
\begin{equation}
\mathcal{Q} \leftarrow \mathcal{Q} \cup \{ T(s) \}.
\end{equation}
When the queue length exceeds a predefined capacity \( n \), the oldest element is removed to accommodate new entries.
Based on the values stored in \( \mathcal{Q} \), the decision to trigger human intervention is determined by the following rule:
\begin{equation}
    I(s) = 
    \begin{cases} 
        1, & \text{if } T(s) = \max(\mathcal{Q}) \\
        0, & \text{otherwise}
    \end{cases}
\end{equation}
where \( T(s) \) denotes an evaluation metric for the current state \( s \). Human intervention is triggered (i.e., \(I(s) = 1 \)) when \( T(s) \) reaches the maximum value in \( \mathcal{Q} \); otherwise, the agent continues to operate autonomously (i.e., \( I(s) = 0 \)).
The criterion will be employed in HIAT and subsequent experiments to reduce manual costs.}

%动作-值函数是在给定状态下采取某个动作的质量度量。

%In the experiments of section \ref{Experiments}, we utilize this method as the criterion for human intervention requests.

\subsection{Reinforcement Learning with Auxiliary Tasks}\label{RLAT}
DRL algorithm must extract effective feature representations from a multitude of samples when confronting complex tasks \cite{biaozheng2023_CVPR}. Knowledge transfer across tasks significantly enhances learning efficiency \cite{at3,IFMTYIN2024120362}. 
Shared representations of primary and auxiliary tasks are used in RL to introduce additional structure and information, thereby optimizing the learning process. 
In the actor-critic framework, the primary task is generally considered to be the actor's decision-making or the estimation of the Q-value,  denoted as \(\mathcal{T}_{pri}\), alongside a set of auxiliary tasks labeled as a vector \(\mathbf{T}_{{aux}}\). \(\mathcal{L}_{{pri}}\) and \(\mathbf{L}_{{aux}}\) are the optimization goals of primary task and auxiliary task, respectively.  
%These tasks are each aligned with distinct optimization goals: \(\mathcal{L}_{{pri}}\) for the primary task and \(\mathbf{L}_{{aux}}\) for the auxiliary tasks. 
Within the RL architecture, \(\mathcal{L}_{{pri}}\) can either represent the expected return loss which is used for calculating policy gradient, or the temporal difference error for Q-learning.
%Specifically, we define a primary task $\mathbf{T}_{{pri}}$ and a series of auxiliary tasks $T_{{aux},i}$, each with their respective optimization objectives, $L_{{pri}}$ and $L_{{aux}}$. Within the actor-critic architecture, $L_{{pri}}$ can be the loss $L_{\pi}$ for computing policy gradients in the actor network or the temporal difference (TD) error $L_{Q}$ for calculating Q-values in the critic network.

Our goal is to leverage the gradients from auxiliary tasks to more efficiently enable the primary task to learn the features of the input, such as state $s$. By jointly training both tasks, we aspire to achieve more efficient and effective learning outcomes.
%我们考虑两个个由参数M(Θ,\phi}_{pri})和 N(Θ,\phi}_{aux})表示的神经网络来求解主任务与辅助任务，他们的共享参数用Θ，任务独享参数分别用 ϕ primary 和 ϕ aux表示，RLAT的更新最小化以下的损失：
We utilize two distinct neural networks for handling tasks: the first network, represented by parameters \(M({\theta}, \phi_{{pri}})\), is dedicated to the primary task, while the second network, denoted as \(N({\theta}, \phi_{{aux}})\) focuses on the auxiliary task. These networks share the parameters \({\theta}\), while their task-specific parameters are denoted as \(\phi_{{pri}}\) and \(\mathbf{\phi}_{{aux}}\). The following optimization objective is aimed to be minimized:

% \begin{equation}
% \resizebox{0.8\linewidth}{!}{$
% \mathcal{L}\left(\theta_{t}\right)=\mathcal{L}_{\text {primary }}\left(\theta_{t}\right)+\mathcal{W} \sum_{i=1}^{K} \mathcal{L}_{a u x, i}\left(\theta_{t}\right)
% $}
% \end{equation}%

\begin{equation}
% \resizebox{0.8\linewidth}{!}{$
\underset{{\theta},\phi_{pri},\phi_{aux}}{\arg\min}\mathcal{L}_{pri}({\theta},\boldsymbol{\phi}_{pri})+ \omega^{\mathrm{T}} \mathbf{L}_{aux}({\theta},\boldsymbol{\phi}_{aux}),
% $}
\end{equation}%
where the vector $\omega^{\mathrm{T}}$ represents the weights of different auxiliary tasks. During the optimization process, the weighted loss of various tasks might positively influence early feature learning but may diminish or become counterproductive with advancing training.
If weights are improperly allocated, excessive emphasis on certain auxiliary tasks will decelerate the performance of the primary task. 
%Therefore, adaptive weighting $\omega^{\mathrm{T}}$ is therefore essential to ensure that the assistive task continues to effectively facilitate learning throughout the training process.%分配不当过度重视
Hence, adaptive adjustment of weights $\omega^{\mathrm{T}}$ is crucial for the auxiliary task throughout the training process.

\section{Method}\label{Method}
In this section, we present our method HIAT, which adopts a joint training approach, using auxiliary tasks to learn a representation containing human policy information for the primary task. In Section \ref{Method2}, to ensure that the primary task is not adversely affected by the auxiliary task, we propose AWHIAT, which determines the weights of the auxiliary task by considering the similarity between the gradients of the primary and auxiliary tasks.

\subsection{HIAT: Human-in-the-Loop Reinforcement Learning with Auxiliary Task}\label{Method1}

%正如在section3.1中描述的，将定义为Dh和D。我们希望直接从。
%受3.2节中
%将Q网络分成两部分

% \begin{equation}
% \begin{split}
% L(\theta_\mu) = & -\mathbb{E}_{s_t \sim D}[Q(s_t, \mu(s_t|\theta_\mu)|\heta_Q)]  + \lambda \mathbb{E}_{s_t \sim D}[(\mu(s_t|\theta_\mu) - a_\text{h})^2]
% \end{split}
% \end{equation}

As described in Section \ref{HILRL}, to implement the HIRL algorithm, previous research has improved the actor-critic framework by directly extracting historical tuples from the replay buffer $\mathcal{D}$ during training. Based on the value of $I(s)$, $\mathcal{D}$ can be categorized into the human buffer $\mathcal{D}_{h}$ and agent buffer $\mathcal{D}_{a}$. We aim to focus on the sparse data of $\mathcal{D}_{h}$ to enhance the performance of HIRL.

Inspired by the objective of RL with an auxiliary task in Section \ref{RLAT}, we propose the adoption of an auxiliary discriminative task in policy evaluation. Assuming that $\mathbf{E}_{\theta}$ is a learnable encoder in the HIRL setting, the encoder aims to learn the features of both the human policy distribution and agent policy distribution, outputting an encoded representation $\Phi$. This representation is then shared between the primary and auxiliary tasks, enabling a synergistic use of the encoded knowledge in both tasks.
The auxiliary discriminative task is denoted as:
\begin{equation}
G(s,a ; \mathbf{\theta},\phi_{aux}): \mathcal{X}_{\mathcal{D}} \rightarrow [0, 1],
\end{equation}
where \( \mathcal{X}_\mathcal{D} \) represents the  space of $\mathcal{D}$. $G(s,a ; \mathbf{\theta},\phi_{aux})$ outputs a value close to 1 $(G(s, a_{h}) \approx 1) $ for the $\mathcal{D}_{h}$ of the human policy distribution, and a value close to 0 $(G(s, a) \approx 0) $ for the remaining values of $\mathcal{D}_{a}$.
To improve  capability of $G( \cdot )$ in discerning the human policy distribution, we have formulated the following optimization objectives:
\begin{equation}\label{5}
% \resizebox{.91\linewidth}{!}{$
\underset{\boldsymbol{G},{\theta},\phi_{aux}}{\arg\min} \left[ -\mathbb{E}_{ a \sim \pi_h( \cdot|s)}\left[ \log G(s, a) \right]   \right],
% $}
\end{equation}
where $\theta$ is the parameter of the encoder, and $\phi_{aux}$ is the exclusive parameter for this task.
Similarly, to enhance the proficiency of $G(\cdot) $ in discerning agent policy distribution, we have formulated the following optimization objectives:
\begin{equation}\label{6}
% \resizebox{.91\linewidth}{!}{$
\underset{\boldsymbol{G},{\theta},\phi_{aux}}{\arg\min} \left[ -\mathbb{E}_{ a \sim \pi( \cdot|s)}\left[ \log (1 -G(s, a)) \right]   \right].
% $}
\end{equation}
The encoder \(\mathbf{E}_{\theta}\) employs a alternating training strategy, wherein the learning of the auxiliary task facilitates the encoder to acquire discriminative features that effectively distinguish between human and agent policy distributions. This strategy ensures that each task maintains its operational independence within its designated network layer, thereby eliminating potential cross-task interference. Gradient signals from the auxiliary discriminator \(G(\cdot)\) are leveraged to enhance the encoder's ability to differentiate between state-action pairs originating from human demonstrations and those from the agent's policy.

\begin{figure*}[!htbp]
    \centering
    \includegraphics[width=0.7\textwidth]{picture/loss.pdf}
    \caption{The method to optimize HIAT. State-action pairs$(s,a)$ are encoded to generate representation $\Phi$, which is then used as inputs for different downstream tasks. By differentiating the representations of $(s, a)$ and $ (s,a_{h})$, as well as jointly optimizing with $\mathcal{L}_{pri}$ during the policy evaluation phase, $\lambda$ and $\mu$, which are weights for different tasks, are adaptively controlled to assist learning.}
    \label{overall framework}
\end{figure*}

More significantly, the rich latent representation \(\Phi\) learned through this optimization process serves as a crucial knowledge bridge between the auxiliary and primary tasks. By incorporating this shared representation, the Q-value estimation leads to more accurate value assessments. In essence, HIAT creates a synergistic learning framework where the shared encoder layers are incentivized to extract and utilize information that is beneficial for both the discriminative auxiliary task and the primary RL objective.

%In this section, we present our method HIAT (Human-in-the-Loop Reinforcement Learning with Auxiliary Task), which adopts a joint training approach, using auxiliary tasks to learn a representation containing human policy information for the primary task. In Section \ref{Method2}, to ensure that the primary task is not adversely affected by the auxiliary task, we propose an adaptive weighting version of HIAT (AWHIAT). This adaptive version determines the weights of the auxiliary task by considering the similarity between the gradients of the primary and auxiliary task.

% \subsection{HIAT: Human-in-the-Loop Reinforcement Learning with Auxiliary Task}\label{Method1}


% The auxiliary discriminative task is denoted as:
% \begin{equation}
% G(s,a ; \mathbf{\theta},\phi_{aux}): \mathcal{X}_{\mathcal{D}} \rightarrow [0, 1]
% \end{equation}
% where \( \mathcal{X}_\mathcal{D} \) represents the space of $\mathcal{D}$. $G(s,a ; \mathbf{\theta},\phi_{aux})$ outputs a value close to 1 $(G(s, a_{h}) \approx 1) $ for the $\mathcal{D}_{h}$ of the human policy distribution, and a value close to 0 $(G(s, a) \approx 0) $ for the remaining values of $\mathcal{D}_{a}$.

% To improve the capability of $G( \cdot )$ in discerning the human policy distribution, we have formulated the following optimization objectives:
% \begin{equation}\label{5}
% \underset{\boldsymbol{G},{\theta},\phi_{aux}}{\arg\min} \left[ -\mathbb{E}_{ a \sim \pi_h( \cdot|s)}\left[ \log G(s, a) \right] \right]
% \end{equation}
% where $\theta$ is the parameter of the encoder, and $\phi_{aux}$ is the exclusive parameter for this task.

% Similarly, to enhance the proficiency of $G(\cdot) $ in discerning agent policy distribution, we have formulated the following optimization objectives:
% \begin{equation}\label{6}
% \underset{\boldsymbol{G},{\theta},\phi_{aux}}{\arg\min} \left[ -\mathbb{E}_{ a \sim \pi( \cdot|s)}\left[ \log (1 -G(s, a)) \right] \right]
% \end{equation}

% The encoder \(\mathbf{E}_{\theta}\) employs a sophisticated alternating training strategy, wherein the learning of the auxiliary task facilitates the encoder to acquire discriminative features that effectively distinguish between human and agent policy distributions. This strategy ensures that each task maintains its operational independence within its designated network layer, thereby eliminating potential cross-task interference. The gradient signals from the auxiliary discriminator \(G(\cdot)\) are strategically leveraged to enhance the encoder's ability to differentiate between state-action pairs originating from human demonstrations \((s,a_h)\) and those from the agent's policy \((s,a)\).

% More significantly, the rich latent representation \(\Phi\) learned through this multi-objective optimization process serves as a crucial knowledge bridge between the auxiliary and primary tasks. This representation encapsulates essential characteristics of both human and agent policy distributions, providing a robust foundation for the policy evaluation component of the model. By incorporating this shared representation, the Q-value estimation benefits from the contextual understanding derived from the auxiliary task, leading to more accurate value assessments. In essence, HIAT creates a synergistic learning framework where the shared encoder layers are incentivized to extract, preserve, and utilize information that is mutually beneficial for both the discriminative auxiliary task and the primary reinforcement learning objective.

%\mathbb{E}_{\pi}\left[ \log (1 - D(s, a)) \right]
%最小化它区分真实数据和生成器生成的假数据之间差异

% 缓冲区怎么表述？？
%
%我们假设一个函数E,用于对当前状态动作提取信息，
%HIAT的主要关注点是学习高维度的z空间并在这样的空间内评估策略，这个高维度的z空间捕捉到了更多与任务相关的信息，使其非常适合于策略优化的目的



\subsection{AWHIAT: Adaptive Weighting for HIAT}\label{Method2}

% In Section \ref{Method1}, the primary and auxiliary tasks are alternatively trained with equal weights of 1, implying that both tasks are of equal importance. 
% In reality, our primary focus is on $\mathcal{T}_{{ pri}}$. 
% Moreover, HIAT does not take into account the impact of conflicts between tasks. 
% Specifically, when the auxiliary discriminative task struggles to differentiate between human and agent policy, it evidently affects the learning.
% In Section \ref{HILRL},
% we initially assume the human policy to approximate optimal policy.
% The discrepancies between the current policy and an optimal policy is identified through $\mathcal{T}_{{ aux}}$.
% However, as training progresses, the deviation of human policies from the actual optimal policies becomes increasingly pronounced.
% Such discrepancies may impede the learning of the primary task $ \mathcal{T}_{{ pri}} $, that is, as agent policy improves, the performance difference between agent policy and human policy will gradually diminish, and may even surpass the human policy.
% % 导致辅助任务学到更差的表征
% It becomes challenging to determine whether $\mathcal{T}_{{ aux}}$ continues to advance the learning of $\mathcal{T}_{{ pri}}$. 
% Therefore, it is necessary to balance the optimization of the  $\mathcal{T}_{{ pri}}$ and  $\mathcal{T}_{{ aux}}$ by adjusting weights, ensuring that the agent's learning is not adversely affected by the auxiliary tasks. 

% To address the above optimization problem, it is noted that both tasks have high relevance when utilizing similar standards to evaluate $(s, a)$.
% Besides, the learning of $\mathcal{T}_{ pri}$ is more challenging than $\mathcal{T}_{ aux}$. 
% The auxiliary discriminative task is a coarse-grained task with relatively lower learning difficulty, requiring binary classification of inputs. 
% Unlike the binary classification of $\mathcal{T}_{ aux}$, $\mathcal{T}_{ pri}$ involves processing representations to generate estimations of Q-values.
%  %相反，与 $\mathcal{T}_{ aux}}$ 相比，需要对 Q 值进行更复杂的估计。
% Therefore, we propose a framework that dynamically adjusts the learning weights $\lambda$ and $\mu$ based on the learning difficulty. 
% In the initial stage of training, this approach prioritizes the coarse-grained discriminative task to capture useful features effectively.
% Subsequently, the focus is shifted towards policy evaluation in the later stages of the process.
% We define the task weighting $\lambda$ for $\mathcal{T}_{{ pri}}$ as:

% \begin{equation}\label{lamda}
% \lambda (t)=\frac{\, 1\, }{\,1 + \text{exp}(-\kappa (t-t_{\tau}))\,}
% \end{equation}
% \textcolor{red}{where $t$ represents the training steps, $t_{\tau}$ denotes the task threshold adjusted based on different training environments, typically being larger for more complex environments. $\kappa$ is the temperature coefficient, which is dependent on task relevance.}

% A method based on the cosine similarity between the gradient updates of different tasks is utilized to determine the appropriateness of the auxiliary task's updates.  Specifically, when  the cosine similarity is less than 0, it indicates a conflict between two tasks.
% In such case, only the primary task's loss should be updated. 
% Conversely, when no conflict exists, such as indicated by a non-negative cosine similarity, the primary and auxiliary losses should be updated simultaneously. We have summarized the above discussion summarised as Proposition \ref{prop}, proved in the appendix and verified in the experiment \ref{Abla}.
% \textcolor{red}{
% Here, we introduce an intuitive Proposition \ref{prop} : if the cosine similarity of the gradients consistently exceeds 0, the convergence properties of $\mathcal{T}_{{ pri}}$ will not be compromised by $\mathcal{T}_{{ aux}}$ \cite{at6}.
% \begin{prop} \label{prop}
% Given respectively the gradient vector  of the shared layer for primary and auxiliary tasks as \( M_{share}(\theta) = \nabla_{\theta_{pri}} \mathcal{L}_{pri}(\theta) \) , \( N_{share}(\theta) = \nabla_{\theta_{{aux}}} \mathcal{L}_{{aux}}(\theta) \), the following update rule:
% \begin{align}
%     \theta^{(t+1)} := \: &\theta^{(t)} - \alpha \Bigl( M_{share}(\theta) + \notag
%     \\ 
%     &    N_{share}(\theta)H( \cos(M_{share}(\theta), N_{share}(\theta)))\Bigr)
% \end{align}
% ensures that, provided the learning rate $\alpha$ is sufficiently small, the algorithm will converge to a local minimum of the loss function $ \mathcal{\mathcal{L}}_{pri}$.
% \end{prop}
% }
% %Proof is provided in Appendix .   
% Let's explain the characteristics of the hypothesis mentioned earlier:
% Proposition \ref{prop} guarantees the avoidance of divergence but does not ensure an enhancement in convergence. It serves as a precautionary measure against worst-case scenarios.
% This ensures that the auxiliary task will not affect the convergence of the  primary task.
% In summary, define the task weighting $\mu$ for the auxiliary task as:
% \textcolor{red}{
% \begin{equation}
% \renewcommand{\arraystretch}{1.5} % Adjust the 1.5 to whatever factor suits your needs
% \mu(t) = 
% \begin{cases}
% \displaystyle H(\cos(M_{\text{share}}(\theta), N_{\text{share}}(\theta))) &  \lambda(t) \geq \lambda(t_{\tau}) \\
% 1 &  \lambda(t) < \lambda(t_{\tau})
% \end{cases}
% \end{equation}}
% where \(H(\cdot)\) denotes the Heaviside step function, which is defined as 0 when the cosine similarity is less than zero. In the early stages of training, the primary focus is on learning from the auxiliary task. Hence, \( \mu(t) \) remains unaffected by the cosine similarity until time \( t_{\tau} \).

In Section \ref{Method1}, the primary and auxiliary tasks are alternatively trained with equal weights of 1, implying that both tasks are of equal importance. 
In reality, our primary focus is on $\mathcal{T}_{{ pri}}$. 
Moreover, HIAT does not take into account the impact of conflicts between tasks. 
Specifically, when the auxiliary discriminative task struggles to differentiate between human and agent policy, it evidently affects the learning.
In Section \ref{HILRL}, we initially assume the human policy to approximate optimal policy.
The discrepancies between the current policy and an optimal policy is identified through $\mathcal{T}_{{ aux}}$.

However, due to cognitive load constraints, human policies often deviate from theoretical optimality, especially in extended or complex tasks. As cognitive resources deplete, human decision quality fluctuates due to fatigue, divided attention, and decreased motivation, making human demonstrations inconsistent. These cognitive limitations, combined with the agent's improving performance, result in human policies that increasingly diverge from the true optimal policies as training progresses.
Such discrepancies may impede the learning of the primary task $ \mathcal{T}_{{ pri}} $, that is, as agent policy improves, the performance difference between agent policy and human policy will gradually diminish, and may even surpass the human policy.
It becomes challenging to determine whether $\mathcal{T}_{{ aux}}$ continues to advance the learning of $\mathcal{T}_{{ pri}}$. 
Therefore, it is necessary to balance the optimization of the $\mathcal{T}_{{ pri}}$ and  $\mathcal{T}_{{ aux}}$ by adjusting weights, ensuring that the agent's learning is not adversely affected by the auxiliary tasks. 

To address the above optimization problem, it is noted that both tasks have high relevance when utilizing similar standards to evaluate state-action pairs.
Besides, the learning of $\mathcal{T}_{ pri}$ is more challenging than $\mathcal{T}_{ aux}$. 
The auxiliary discriminative task is a coarse-grained task with relatively lower learning difficulty, requiring binary classification of inputs. 
Unlike the binary classification of $\mathcal{T}_{ aux}$, $\mathcal{T}_{ pri}$ involves processing representations to generate estimations of Q-values.
Therefore, we propose a framework that dynamically adjusts the learning weights $\lambda$ and $\mu$ based on the learning difficulty. 
This adaptive approach also helps mitigate the negative effects of cognitive load-induced inconsistencies in human demonstrations. By dynamically adjusting the influence of human policy data based on its quality and consistency, we can compensate for the varying reliability of human guidance throughout the training process.
In the initial stage of training, this approach prioritizes the coarse-grained discriminative task to capture useful features effectively.
Subsequently, the focus is shifted towards policy evaluation in the later stages of the process.
We define the task weighting $\lambda$ for $\mathcal{T}_{{ pri}}$ as:

\begin{equation}\label{lamda}
\lambda (t)=\frac{\, 1\, }{\,1 + \text{exp}(-\kappa (t-t_{\tau}))\,},
\end{equation}
\textcolor{red}{where $t$ represents the training steps, $t_{\tau}$ denotes the task threshold adjusted based on different training environments, typically being larger for more complex environments. $\kappa$ is the temperature coefficient, which is dependent on task relevance.}

A method based on the cosine similarity between the gradient updates of different tasks is utilized to determine the appropriateness of the auxiliary task's updates. This mechanism is particularly valuable when human demonstrations become less reliable due to cognitive fatigue. Specifically, when the cosine similarity is less than 0, it indicates a conflict between two tasks.
In such case, only the primary task's loss should be updated. 
Conversely, when no conflict exists, such as indicated by a non-negative cosine similarity, the primary and auxiliary losses should be updated simultaneously. We have summarized the above discussion summarised as Proposition \ref{prop}, proved in the appendix and verified in the experiment \ref{Abla}.
\textcolor{red}{
Namely, if the cosine similarity of the gradients consistently exceeds 0, the convergence properties of $\mathcal{T}_{{ pri}}$ will not be compromised by $\mathcal{T}_{{ aux}}$ \cite{at6}.
\begin{prop} \label{prop}
Given respectively the gradient vector  of the shared layer for primary and auxiliary tasks as \( M_{share}(\theta) = \nabla_{\theta_{pri}} \mathcal{L}_{pri}(\theta) \) , \( N_{share}(\theta) = \nabla_{\theta_{{aux}}} \mathcal{L}_{{aux}}(\theta) \), the following update rule:
\begin{align}
    \theta^{(t+1)} := \: &\theta^{(t)} - \alpha \Bigl( M_{share}(\theta) + \notag
    \\ 
    &    N_{share}(\theta)H( \cos(M_{share}(\theta), N_{share}(\theta)))\Bigr)
\end{align}
ensures that, provided the learning rate $\alpha$ is sufficiently small, the algorithm will converge to a local minimum of the loss function $ \mathcal{\mathcal{L}}_{pri}$.
\end{prop}
}
%Proof is provided in Appendix .   
Let's explain the characteristics of the hypothesis mentioned earlier:
Proposition \ref{prop} guarantees the avoidance of divergence but does not ensure an enhancement in convergence. It serves as a precautionary measure against worst-case scenarios.
This ensures that the auxiliary task will not affect the convergence of the primary task.
This safeguard is especially important in scenarios where cognitive load affects human policy quality, as it protects the primary learning objective from potential degradation due to inconsistent human guidance.
In summary, define the task weighting $\mu$ for the auxiliary task as:

\textcolor{red}{
\begin{equation}
\renewcommand{\arraystretch}{1.5} % Adjust the 1.5 to whatever factor suits your needs
\mu(t) = 
\begin{cases}
\displaystyle H(\cos(M_{\text{share}}(\theta), N_{\text{share}}(\theta))) &  \lambda(t) \geq \lambda(t_{\tau}) \\
1 &  \lambda(t) < \lambda(t_{\tau})
\end{cases}
\end{equation}}
where \(H(\cdot)\) denotes the Heaviside step function, which is defined as 0 when the cosine similarity is less than zero. In the early stages of training, the primary focus is on learning from the auxiliary task. Hence, \( \mu(t) \) remains unaffected by the cosine similarity until time \( t_{\tau} \). As training progresses and human guidance potentially deteriorates due to cognitive fatigue, the adaptive weighting mechanism increasingly relies on gradient alignment to determine the utility of the auxiliary task.

\subsection{ Training Objectives \& Implementation Details}

To construct the auxiliary task, the Q-network represented by a set of parameters ${\theta}$ , is split into a linear layer $\phi_{pri}$ and the shared encoder $\mathbf{E}_{\theta}$. 
%The representation $\Phi$ is defined as the output of the encoder. 
%Consider a  neural network , where ${\theta}$ denotes the parameters of the sharing layer also csnsidered the encoder,  represents the parameters of the other layer.
Within this framework, the feature representation of the Q-network, denoted as $\Phi$, is defined as:
\begin{equation}
% \resizebox{0.6\linewidth}{!}{$
Q(s, a; {\theta})
= \langle \Phi(s, a; {\theta}), \phi_{pri} \rangle.
%$}
\end{equation}
To optimize the Q-network, our objective is to minimize the difference between the Q-value projected by the network and the target Q-value, typically obtained from the Bellman equation. The target, which takes into account the anticipated future rewards, is expressed as:
\begin{equation}
%\resizebox{0.8\linewidth}{!}{$
y = R(s, a) + \gamma \max_{a'}Q(s', a'; {\theta} ' ),
%$}
\end{equation}
where ${\theta} '$ represents the parameters of a target network, which is a delayed or fixed copy of the original network parameters used to stabilize the learning process. The optimization objective is defined as the mean squared error between the predicted Q-values and the target Q-values:
\begin{equation}
\mathcal{L}_{pri}(\theta, \phi_{pri}) = \mathbb{E}\left[\left(y - Q(s, a; {\theta})\right)^2\right],
\end{equation}
where the parameters $\theta$ are adjusted in the direction that minimizes the discrepancy between the predicted Q-values and the target Q-values.
We conceptualize the combined optimization objective in Eq (\ref{5}) and Eq (\ref{6}) as a cross-entropy loss, denoted as:
\begin{equation}
\begin{aligned}
\mathcal{L}_{aux}(\theta, \phi_{aux}) = &-\mathbb{E}_{\pi_h}\left[ \log G(s, a) \right]
-\mathbb{E}_{\pi}\left[ \log (1 - G(s, a)) \right].
\end{aligned}
\end{equation}
Thereby, alter the optimization objective of the encoder as follows:
\begin{equation}
% \resizebox{.71\linewidth}{!}{$
\mathcal{L}(\theta)= \lambda \mathcal{L}_{pri}(\theta) + \mu \mathcal{L}_{aux}(\theta),
% $}
\end{equation}
where the parameters $\lambda$  and $\mu$ are the weights of $\mathcal{T}_{{ pri}}$ and $\mathcal{T}_{{ aux}}$, respectively. The training process of AWHIAT is illustrated in Fig. \ref{overall framework} , and the algorithm pseudocode is depicted in Section \ref{awhiat}.




\begin{algorithm}\label{awhiat}
\caption{algorithm framework of AWHIAT}
\begin{algorithmic}[1]
\State \textbf{Input:}

\State \textbf{Given} temperature coefficient $\kappa$
\State \textbf{Given} task correlation coefficient $t_{\tau}$
\State \textbf{Given}  two tasks $\mathcal{T}_{{ pri}}$ and $\mathcal{T}_{{ aux}}$
\State \textbf{Initialize} parameters of shared encoder $\theta$ , task-specific parameters \(\phi_{{pri}}\) , \(\mathbf{\phi}_{{aux}}\)  the primary task weighting $\lambda$ , the auxiliary task weighting $ \mu $ and parameters $ \alpha $
\For{ $t\leftarrow 1$ to $T$}
        \State Sample a mini-batch data from $\mathcal{D}$
        \State Collect the loss of primary task $\mathcal{L}_{{ pri}}$ , the loss of auxiliary task $\mathcal{L}_{{ pri}}$
        \State Based on Eq \ref{lamda} , collect the the auxiliary task weight $\lambda$  
        \If{step $t \leq t_r$}
            \State Set the auxiliary task weight $\mu = 1$
        \Else
            \If{$\cos(\nabla_{\theta_{pri}} \mathcal{L}_{pri}(\theta), \nabla_{\theta_{{aux}}} \mathcal{L}_{{aux}}(\theta)) > 0 $}
            \State Set the auxiliary task weight $\mu = H(\cos(\nabla_{\theta_{pri}} \mathcal{L}_{pri}(\theta), \nabla_{\theta_{{aux}}} \mathcal{L}_{{aux}}(\theta)))$
            \Else
            \State Set the auxiliary task weight $\mu = 0$
            \EndIf
            
        \EndIf
        
        \State \(\phi_{{pri}}^{(t+1)} \leftarrow : \phi_{{pri}}^{(t)} - \alpha \nabla_{\phi_{pri}} \mathcal{L}_{pri}(\phi)\)
        \State \(\phi_{{aux}}^{(t+1)} \leftarrow : \phi_{{aux}}^{(t)} - \alpha \nabla_{\phi_{aux}} \mathcal{L}_{aux}(\phi)\)
        \State Collect the loss of shared encoder $\theta$ $\mathcal{L}(\theta)= \lambda \mathcal{L}_{pri}(\theta) + \mu \mathcal{L}_{aux}(\theta)$
        \State $\theta^{(t+1)} \leftarrow : \theta^{(t)} - \alpha \Bigl( \lambda \nabla_{\theta_{pri}} \mathcal{L}_{pri}(\theta) + \notag
            \mu \nabla_{\theta_{{aux}}} \mathcal{L}_{{aux}}(\theta)\Bigr)$
        
    \EndFor

\end{algorithmic}
\end{algorithm}



\section{Experiments}\label{Experiments}
% \subsection{Offline Abstraction for Visual Gridworlds}

In this section, we conducted extensive empirical and comparative experiments. Final performance was assessed by measuring the average reward per episode, while sample efficiency was evaluated based on the number of steps required to achieve optimal performance. To ensure a fair assessment, all algorithms used the same set of hyperparameters across all experiments. All experimental results are presented as the mean and standard deviation of performance across different random seeds. All experiments were performed on an Intel Xeon(R) Silver 4210R CPU and an NVIDIA Tesla V100 GPU.

\begin{table*}[ht]
  \centering
  \caption{Hyperparameter Configuration for MuJoCo Experiments} \label{tab1}
  \resizebox{0.8\textwidth}{!}{%
  \begin{tabular}{lc}
    \hline
    Hyperparameters & Values \\
    \hline
    Discount $\gamma$ & 0.99 \\
    Buffer $\mathcal{D}$ size & $10^6$ \\
    Optimizer of actor network and critic network & Adam  \\
    Learning rate  & $1 \times 10^{-4}$ \\
    Number of hidden layers  & 2 \\
    Number of hidden units & 256 \\
    Activation function & ReLU \\
    Mini-batch size & 256  \\
    Delayed policy update frequency & 2 \\
    Target smoothing coefficient ($\eta$) & 0.005 \\
    Target update interval  & 2 \\
    Exploration noise & Gaussian \\
    Variance of target policy smoothing & 0.2 \\
    Noise clip range & $[-0.5, 0.5]$ \\
    Gradient Clipping & False \\
    AWHIAT coefficient ($t_{\tau}$) & $2 \times 10^{4}$ \\
    \hline

  \end{tabular}%
  } % Closing \resizebox
\end{table*}


 Our framework was benchmarked in an online environment through a series of challenging and diverse continuous control tasks in MuJoCo \cite{mujoco}. The experiments were carried out in six different MuJoCo environments: \textit{Inverted Double Pendulum, Humanoid, HalfCheetah, Walker2d, Hopper, and Ant}, aiming to assess sample efficiency and performance. 
 \begin{figure*}[htbp]
    \centering
    \includegraphics[width=\textwidth]{picture/env.pdf}
    \caption{The images from the MuJoCo suite used in our experiments. The states in this suite are represented as vectors.}
    \label{fig:enter-label}
\end{figure*}
When training the agent on the MuJoCo suite, the agent initially collects 20,000 states and actions randomly to enhance exploration capabilities.
The mean reward across three episodes serves as a primary evaluation criterion.
During the evaluation phase, exploration techniques, such as injecting noise into actions in TD3, are not employed to maintain fairness in algorithm assessment. This precaution is taken because such exploration strategies could potentially influence the performance of the testing algorithm.
% We pretrain a policy as an human policy, which is equivalent to $60 \% $ of the convergent performance.
\textcolor{red}{
We trained the model using the TD3 algorithm for 3 million steps, saving a checkpoint every 100,000 steps. We then selected the checkpoint that achieved 60\% of the convergent performance to serve as the human policy.}

 The representative RL algorithm Twin-Delayed Deep Deterministic (TD3) \cite{TD3}, HIRL state-of-the-art baseline methods, including QDP, IA-RL \cite{IA_RL}, Hug-RL \cite{hirl2} and QDP+\textit{rwf} \cite{at2}, were selected to compare with HIAT because they have outstanding performance. Below is a detailed introduction to the algorithm:
 
{\color{red}
\textbf{IA-RL}: A HIRL algorithm applied to Atari games, for which we implemented a QDP version following the original paper. IA-RL trains a supervised learning head of a critic network to predict whether the agent's actions will lead the model into danger.
This predictive capability functions as an auxiliary learning task, enhancing the model's safety awareness and decision-making capabilities in complex environments.

\textbf{Hug-RL}: A HIRL algorithm applied to autonomous driving. The algorithm operates through a dual-learning paradigm, enabling the agent to maximize expected return while assimilating expert behavioral patterns.This synergistic approach facilitates accelerated learning through expert imitation while maintaining the agent's capacity for independent decision-making and reward optimization.We implemented Hug-RL by incorporating the active learning mechanism of QDP.

\textbf{QDP}: QDP represents a HIRL algorithm specifically engineered for continuous space tasks. The algorithm's fundamental premise lies in its dual-objective optimization: accelerating the agent's learning process through strategic human expert intervention while simultaneously minimizing the expert's monitoring burden. This is achieved through a novel Q-value-based advisory mechanism, where the agent autonomously solicits human expertise during the initial training phase based on Q-value discrepancies, rather than depending on continuous expert supervision. This approach effectively balances learning efficiency with human resource optimization.

\textbf{QDP}+\textbf{\textit{rwf}} \cite{at2}: \textit{rwf} constructs an auxiliary task for the Q-network to predict the next step reward, aiding in the learning of Q-value estimation. Therefore, we combine \textit{rwf} with QDP as a standard, referred to as QDP+\textit{rwf}.
}


\begin{table*}[ht]
\centering
\resizebox{1\linewidth}{!}{
\renewcommand{\arraystretch}{2}
\begin{tabular}{c|cccccc}
\hline
Task & Walker2d & Ant & HalfCheetah & Hopper & Humanoid & IDP \\ \hline
TD3  &      $3811.20 \scriptstyle{\pm 619.21}$           &    $3919.32 \scriptstyle{\pm 360.32}$     &    $9065.10 \scriptstyle{\pm 195.77}$     &    $3071.46 \scriptstyle{\pm 274.85}$    &    $4917.46 \scriptstyle{\pm 548.20}$     &    $9315.37 \scriptstyle{\pm 6.14}$    \\

IA-RL  &     $\underline{5109.08} \scriptstyle{\pm 366.19}$       &    $5016.54 \scriptstyle{\pm 130.52}$     &    $9535.20 \scriptstyle{\pm 353.74}$     &    $3336.79 \scriptstyle{\pm 173.30}$    &    $5407.48 \scriptstyle{\pm 390.10}$     &    $9315.95 \scriptstyle{\pm 4.05}$    \\

Hug-RL   &     $4741.41 \scriptstyle{\pm 339.91}$          &    $4901.79 \scriptstyle{\pm 464.30}$     &    $9378.23 \scriptstyle{\pm 265.70}$     &    $\underline{3518.22} \scriptstyle{\pm 48.71}$    &    $ \underline{5835.72} \scriptstyle{\pm 840.29}$     &    $9315.15 \scriptstyle{\pm 8.45}$    \\

QDP   &     $4948.97 \scriptstyle{\pm 198.42}$         &    $4943.37 \scriptstyle{\pm 206.96}$     &    $9725.66 \scriptstyle{\pm 119.37}$     &    $3121.03 \scriptstyle{\pm 749.38}$    &    $5581.95 \scriptstyle{\pm 731.89}$     &    $\textbf{9323.39} \scriptstyle{\pm 10.01}$    \\

QDP+rwf   &     $4737.89 \scriptstyle{\pm 532.92}$         &    $\underline{5198.09} \scriptstyle{\pm 157.45}$     &    $\underline{10432.66} \scriptstyle{\pm 433.82}$     &   $3433.27 \scriptstyle{\pm 220.06}$    &     $5485.18 \scriptstyle{\pm 168.29}$    &   $\underline{9320.13} \scriptstyle{\pm 6.43}$    \\
\hline
\framework (Ours)   &     $\textbf{5561.68} \scriptstyle{\pm 168.18}          &   $\textbf{5563.96} \scriptstyle{\pm 114.21}$      &    $\textbf{10667.49} \scriptstyle{\pm 711.11}$     &    $\textbf{3535.95} \scriptstyle{\pm 28.92}$    &    $ \textbf{6075.77} \scriptstyle{\pm 668.30}$     &    $ 9318.42 \scriptstyle{\pm 7.45}$    \\
\hline
% TD3  &      $3811.20 \pm 619.21$           &    $3919.32 \pm 360.32 $     &    $9065.10 \pm 195.77$     &    $3071.46 \pm 274.85$    &    $4917.46 \pm 548.20$     &    $9315.37 \pm 6.14$    \\

% IA-RL  &     $\underline{5109.08} \pm 366.19$       &    $5016.54 \pm 130.52$     &    $9535.20 \pm 353.74$     &    $3336.79 \pm 173.30$    &    $5407.48 \pm 390.10$     &    $9315.95 \pm 4.05$    \\

% Hug-RL   &     $4741.41 \pm 339.91$          &    $4901.79 \pm 464.30$     &    $9378.23 \pm 265.70$     &    $\underline{3518.22} \pm48.71$    &    $ \underline{5835.72} \pm 840.29$     &    $9315.15 \pm 8.45$    \\

% QDP   &     $4948.97 \pm 198.42$         &    $4943.37 \pm 206.96$     &    $9725.66 \pm 119.37$     &    $3121.03 \pm 749.38$    &    $5581.95 \pm 731.89$     &    $\textbf{9323.39} \pm 10.01$    \\

% QDP+rwf   &     $4737.89 \pm 532.92$         &    $\underline{5198.09} \pm 157.45$     &    $\underline{10432.66} \pm 433.82$     &   $3433.27 \pm 220.06$    &     $5485.18 \pm 168.29$    &   $\underline{9320.13} \pm 6.43$    \\
% \hline
% \framework (Ours)   &     $\textbf{5561.68} \scriptstyle{\pm 168.18}          &   $\textbf{5563.96} \pm 114.21$      &    $\textbf{10667.49} \pm 711.11$     &    $\textbf{3535.95} \pm 28.92$    &    $ \textbf{6075.77} \pm 668.30$     &    $ 9318.42 \pm 7.45$    \\
% \hline
% \multicolumn{7}{c}{Improvements}                          \\ \hline
%Abs.($\%$) Avg. $\uparrow$     &       $8.32(0.89)$        &    $18.98(2.26)$     &     $4.14(0.52)$    &   $0.54(0.06)$     &    $4.81(0.52)$     &    $3.20(0.35)$    \\
%\hline

\end{tabular}
}
\caption{In the past five simulations, each simulation included one million time steps. Below is a summary of the average episode rewards obtained from training across some tasks in MujoCo: "mean value ± standard deviation". \textbf{Bold}: the best performance under each category, \underline{underline}: the second-best performance.}

\label{table}
\end{table*}

\begin{figure*}[!htbp]
    \centering
    \includegraphics[width=\textwidth]{picture/reward.pdf}
    \caption{ The learning curve graphs shows the mean episode rewards for six algorithms on six MuJoCo environments. The corresponding algorithm for each color can be identified by referring to the subplots to determine which curve algorithm corresponds to each color.  }
    %在子图ant中给出了不同颜色代表什么算法
    \label{reward}
\end{figure*}

\subsection{Evaluations for Continuous Control}

We selected a duration of 1,000k time steps to evaluate all comparative methods across various environments,  with the mean and variance of the average reward per episode summarized in Table \ref{table}. It can be observed that AWHIAT has improved the average episodic rewards across each MuJoCo environment. Specifically, AWHIAT achieved the best performance under experimental conditions across all five tasks. In terms of stability, AWHIAT exhibited the lowest standard deviations in three environments, namely Hopper (28.92), Ant (114.21), and Walker2d (168.18).
The reason lies in the adaptive weight adjustment, which prevents loss conflicts caused by auxiliary tasks without the need for manual assistance, thus ensuring its stability.
Regarding these results, a reasonable explanation for the nearly similar performance in the InvertedDoublePendulum of all algorithms is that this test was simple for every method to solve.
\begin{figure}[h]
    \centering
    \includegraphics[width=0.9\columnwidth]{picture/ybxl.pdf}
    \caption{The results of the MuJoCo experiment's sample efficiency are shown, with the unit on the vertical axis being the percentage of the number of steps TD3 takes to reach the SE80 benchmark compared to the two algorithms being analyzed}
    \label{ybxl}
\end{figure}
Fig.\ref{reward} displays the learning curves of AWHIAT and five other algorithms in the continuous control environment.
Meanwhile, the results of sample efficiency in the MuJoCo experiments are depicted in Fig. \ref{ybxl}. For each experiment, the average reward target is set at $80\%$ of the convergence performance of TD3, following the setup of the SE80 experiment \cite{at2}. This lower threshold allows for a more noticeable capture of increases in sample efficiency. 
TD3 is utilized as the benchmark for SE80, and it is compared against the algorithm with the best sample efficiency among the other four algorithms. Fewer steps are required by AWHIAT to achieve the average per-episode target compared to other algorithms, indicating an enhancement in sample efficiency. Specifically, in the Ant task, a $34.3\%$ improvement in sample efficiency is observed for AWHIAT when compared to the other algorithms.


%在本小节中，我们探索了AWHIAT是否能够在由编码器学习到的隐空间Z中学到预想的对于不同质量动作的区分。首先，我们从缓冲区中随机采样轨迹，并使用学习到的编码器进行编码，利用t-SNE ()作为手段来可视化编码后的z，点的颜色代表了对应的归一化reward。如图1、2所示。我们分别使用300k和1000k训练步数的TD3、QDP以及AWHIAT进行比较实验。
%可以看出，同样训练步数情况下，AWHIAT能对数据进行明显的区分，数据点明显聚类在图的各个区域，不同奖励数值的分布明显，有清楚可分的边界不同簇间分界分明，这证明在embedding space中学到了对不同质量数据的区分，同样在300k实验中更为明显，TD3没有明显聚类痕迹，而AWHIAT有清晰的奖励过渡带
\begin{figure*}[t]
    \centering
    \includegraphics[width=\textwidth]{picture/tsne1000.pdf}
    \caption{We used the t-SNE technique to visualize the representation $\Phi$ trained with 1000k in the Walker2d environment. In this visualization, each point represents the embedding of random data in buffer D, and the color of the point indicates the normalized reward of the corresponding data.}
    \label{tsne1000}
\end{figure*}

\subsection{Evaluations in Representation }
In this section, we explore whether AWHIAT can discern different quality $(s,a)$ within the representation $\Phi$, as learned by the encoder. 
We have designed the following experiments to verify that our algorithm, which trains with auxiliary tasks, learns a representation that can distinctly differentiate data of varying quality in contrast to other algorithms.

% Initially, data are randomly sampled from the buffer, and the learned encoder is utilized for encoding. The encoded $\Phi$ is visualized using t-SNE \cite{tsne}, with the colors of the points representing the corresponding normalized reward, as illustrated in Fig.s \ref{tsne1000} and \ref{tsne500}.
% Comparative experiments were conducted using TD3, QDP, and AWHIAT, each trained with 300k and 1000k training steps, respectively.

% Analyzing  Fig. \ref{tsne1000} shows that AWHIAT can distinctly differentiate data under the same training step conditions. The data points are clearly clustered in various regions of the graph, with distinct distributions for different reward values and well-defined boundaries between clusters. This demonstrates that AWHIAT has learned to differentiate data of varying quality within the representation. This phenomenon is more pronounced in the 300k-step experiment in Fig. \ref{tsne500}. While TD3 shows no clear signs of clustering, AWHIAT exhibits a clear transition band of rewards.

For the experimental setup, we begin by randomly sampling data from the replay buffer. The encoder, trained by each algorithm, is then applied to generate the representation $\Phi$. To visualize the structure of these encoded representations, we employ t-SNE \cite{tsne}, a widely-used dimensionality reduction technique. In the resulting visualizations, data points are colored according to their corresponding normalized reward values, as depicted in Fig. \ref{tsne1000} and \ref{tsne500}. To provide a comprehensive comparison, we evaluate three algorithms: TD3, QDP and AWHIAT trained for 300k and 1000k steps, respectively, under identical conditions.

Analysis of Fig. \ref{tsne1000} reveals that AWHIAT demonstrates a remarkable ability to differentiate data quality after 1000k training steps. The t-SNE visualization exhibits distinct clustering of data points, with each cluster occupying a unique region of the plot. These clusters correspond closely to different reward levels, featuring well-defined boundaries and minimal overlap. This clear separation underscores AWHIAT’s capacity to encode a representation that effectively captures and distinguishes variations in data quality. In contrast, the representations learned by TD3 and QDP exhibit less pronounced clustering, with data points appearing more diffusely distributed and lacking the sharp delineation observed with AWHIAT.

This trend becomes even more evident in the 300k-step experiment, as shown in Fig. \ref{tsne500}. At this earlier training stage, TD3 produces a representation with little to no discernible clustering, suggesting an inability to meaningfully separate data by quality. QDP fares slightly better but still lacks the clarity of AWHIAT. AWHIAT, however, displays a well-defined transition band across reward values, with data points organized into coherent groups that reflect their quality. This consistent separation, even with fewer training steps, highlights the robustness of AWHIAT’s auxiliary task-driven training and its superior ability to learn a structured and interpretable representation compared to the baseline methods.

\begin{figure*}[t]
    \centering
    \includegraphics[width=\textwidth]{picture/tsne500.pdf}
    \caption{t-SNE visualization of the representation learned in the Walker2d environment after 300k training steps.}
    \label{tsne500}
\end{figure*}

%我们在“Walker2d”任务中进行了消融研究，以了解自适应权重设计对任务学习的影响。我们设计了平均奖励约为4000的策略对两个算法进行指导，在前期两个算法比较接近，但越接近平均奖励，HIAT越受到影响，而AWHIAT则并没有被影响。如图8所示，AWHIAT在性能、采样效率和稳定性方面优于HIAT，这表明自适应权重在HIAT中都非常重要。

\subsection{Ablation Studies}\label{Abla}

\begin{figure}[h]
    \centering
    \includegraphics[width=0.8\columnwidth]{picture/datasb.png}
    \caption{: Mean episodic rewards of ablation experiments in the Walker2d environment.}
    \label{xrsy}
\end{figure}

% We conducted an ablation study in the Walker2d task to explore the influence of adaptive weight design on task learning.
% Policies with an average reward of approximately 4000 were designed to guide two algorithms. As shown in Fig. \ref{xrsy},  Initially, these algorithms demonstrated similar performance levels. However, as they neared the average reward benchmark, HIAT was increasingly impacted, whereas AWHIAT remained unaffected.
% AWHIAT outperforms HIAT in terms of performance, sampling efficiency, and stability, indicating the significant importance of adaptive weights in HIAT.
We conducted an ablation study in the Walker2d task to specifically evaluate how the adaptive weight design of AWHIAT mitigates the negative effects of cognitive load on human policy guidance. For this experiment, we designed policies with an average reward of approximately 4000 to guide both algorithms, simulating a scenario where human demonstrations exhibit varying quality due to cognitive fatigue.

As illustrated in Fig. \ref{xrsy}, both algorithms initially demonstrated comparable performance levels when benefiting from high-quality human guidance. However, as training progressed and the algorithms approached the average reward benchmark, a significant performance divergence emerged. HIAT exhibited increasing performance instability and reduced learning efficiency, which aligns with our hypothesis regarding the detrimental effects of cognitive load on human policy quality. In contrast, AWHIAT maintained consistent performance improvement throughout the training process, effectively compensating for the degradation in human guidance quality.

The superior performance of AWHIAT over HIAT in terms of final reward achievement, sampling efficiency, and stability clearly demonstrates the crucial role of adaptive weighting in mitigating cognitive load effects. By dynamically adjusting the influence of the auxiliary task based on gradient alignment and learning progress, AWHIAT effectively filters out inconsistent or suboptimal human guidance that naturally occurs as cognitive resources deplete during extended training sessions. This adaptive mechanism ensures that the agent continues to benefit from human knowledge while being protected from the negative impacts of cognitive fatigue-induced demonstration inconsistencies.

\section{Conclusions} \label{Sec_6}
In this article, we introduce auxiliary tasks into HIRL and propose a method called HIAT, which mitigates the class-balance issue, effectively improving the algorithm’s performance and sample efficiency. In the HIAT framework, the agent utilizes auxiliary tasks to distinguish between human and current agent policies. By sharing encoders, the agent integrates representations learned from auxiliary tasks into policy evaluation, thereby facilitating the learning process of the policy.
Then, we introduce an adaptive weighting variant called AWHIAT, which adjusts the impact of the auxiliary task on the primary task based on their task-relevance. By adjusting weighting, the agent focuses on the auxiliary task during the early stages of training, while ensuring the auxiliary task does not compromise the policy's convergence. In MuJoCo experiments assessing complex control tasks, our method significantly enhances final performance and sample efficiency compared to existing state-of-the-art methods. In the Ant environment, HIAT showed a 34.3\% improvement in sample efficiency  and a 7.02\% increase in average reward over 1000k compared to the best baseline algorithm. It also achieved competitive results in other environments.
However, this approach is currently limited to the actor-critic framework and requires real-time guidance.
Efforts are underway to address these limitations in the future.
Additionally, an interesting future direction is how to minimize the need for expert guidance while preserving AWHIAT performance. 
% \textcolor{red}{$^3$Finally, although the experiments are performed with a mobile robot on 2D spaces, it is straightforward to deploy the pH-DRL framework to 3D navigation tasks with moving target, 
% with a simple adjustment of the state and action space of the navigation model from the x-y coordinate system to the x-y-z coordinate system. }
%This work enhanced the ability of DRL-based navigation models,and can provide a solution for service robots or transport robots that aim to a moving target. Although the pH-DRL architecture is well-performed in the closed environments (indoor environments or outdoors with bounding walls), challenges may arise when there are large uncertainties in the environment. For instance, due to having neither the access to the map nor the memory of previous observations, pH-DRL cannot stably choose the right direction in advance to avoid the dead ends in the environment. In this regard, using LSTM \cite{devo2020GeneralizationTargetDriven} or auxiliary tasks like loop closeure prediction \cite{mirowski2017LearningNavigate} will be possible solutions. Additionally, if without the boundaries restricting the robot, it might be misguided to far locations by the predicted long-term trajectories of the target.DRL may also suffer in learning a effective strategy in cases where the static obstacles can also move at any moment. Those topics are not trivial and left for our further investigation. 
%future work the development of more robust learning approaches towards more open and dynamic environments.

\section*{Acknowledgements}
This study was supported in part by the National Natural Science Foundation
of China under Grants 62373375, U2341216 and 62022094.

% \appendix




\begin{appendices}

% \section{Section title of first appendix}\label{secA1}

\section{PROOF} \label{appendix_1}
Given respectively the gradient vectors of the shared layer for primary and auxiliary tasks as \( M_{share}(\theta) = \nabla_{\theta_{pri}} \mathcal{L}_{pri}(\theta) \) and \( N_{share}(\theta) = \nabla_{\theta_{{aux}}} \mathcal{L}_{{aux}}(\theta) \), the following update rule:
\begin{align}\label{tt}
    \theta^{(t+1)} := \: &\theta^{(t)} - \alpha \Bigl( M_{share}(\theta) + \notag
    \\ 
    &    N_{share}(\theta)H( \cos(M_{share}(\theta), N_{share}(\theta)))\Bigr)
\end{align}
converges to the local minimum of $\mathcal{L}$ given a small enough $\alpha $.

\emph{Proof}:
Let us denote
$$
\nabla \mathcal{L}^{(t)} := \nabla_\theta \mathcal{L}(\theta^{(t)})
$$
%
$$
\Delta\theta^{(t)} :=  M_{share}(\theta) + N_{share}(\theta) H( \cos(M_{share}(\theta), N_{share}(\theta))).
$$

We know that the convergence of the update rule 
%
$\theta^{(t+1)} := \theta^{(t)} - \alpha^{(t)} \Delta\theta^{(t)}$
%
is established. thus we have
\begin{align}
\langle \Delta\theta^{(t)}, \nabla \mathcal{L}^{(t)} \rangle &= 
\langle M_{share}(\theta) + N_{share}(\theta) H( \cos(M_{share}(\theta), N_{share}(\theta))),  \nabla \mathcal{L}^{(t)}  \rangle \nonumber\\
&= \langle M_{share}(\theta), \nabla \mathcal{L}^{(t)} \rangle + \langle N_{share}(\theta) H( \cos(M_{share}(\theta), N_{share}(\theta))),  \nabla \mathcal{L}^{(t)}  \rangle \nonumber\\
&= \| \nabla \mathcal{L}^{(t)} \|^2 + \tfrac{1}{\| N_{share}(\theta) \| \| \nabla \mathcal{L}^{(t)} \|} H(\langle \nabla \mathcal{L}^{(t)} , N_{share}(\theta) \rangle) \langle N_{share}(\theta) ,  \nabla \mathcal{L}^{(t)}\rangle  \nonumber\\
& \geq 0.
\end{align}
It equals 0 solely when the $\mathcal{L}$ reaches the critical point of $\mathcal{L}$. because it can attain the value of 0 exclusively when $|\nabla \mathcal{L}^{(t)}| = 0$ (the sum of two non-negative terms equals zero only when both are 0, and \ref{tt} is feasible only if this condition is not satisfied).



\end{appendices}

%%===========================================================================================%%
%% If you are submitting to one of the Nature Portfolio journals, using the eJP submission   %%
%% system, please include the references within the manuscript file itself. You may do this  %%
%% by copying the reference list from your .bbl file, paste it into the main manuscript .tex %%
%% file, and delete the associated \verb+\bibliography+ commands.                            %%
%%===========================================================================================%%
\bibliographystyle{sn-mathphys-num}
\bibliography{sn-bibliography}% common bib file
%% if required, the content of .bbl file can be included here once bbl is generated
%%\input sn-article.bbl
% \bibliographystyle{elsart-num-sort}
% \bibliography{Moving-Navigation-Refs}


\end{document}
