\documentclass[10pt,twocolumn]{article}
\usepackage[margin=0.72in]{geometry}
\usepackage{amsmath,amssymb,amsfonts}
\usepackage{booktabs}
\usepackage{graphicx}
\usepackage{caption}
\usepackage{subcaption}
\usepackage{hyperref}
\usepackage{array}
\usepackage{xcolor}
\graphicspath{{../figures/}}

\title{\vspace{-1.2em}Nonnormality in Reinforcement Learning Closed Loops:\\
From Action Smoothing to Source-Amplifier Diagnostics}
\author{Bilingual independent draft package}
\date{}

\begin{document}
\maketitle

\begin{abstract}
Continuous-control reinforcement learning often treats action jitter as a smoothness problem. This paper reframes the issue as a source-amplifier problem: residual actions, exploration noise, execution noise, and actuator disturbances become consequential when they enter a nonnormal closed-loop surrogate that can directionally amplify them into state covariance, safety-margin loss, projection burden, and posterior Bellman-Lyapunov residuals. We develop a minimal local model, a diagnostic table for source and amplifier effects, and a certificate-weighted response based on the action metric $M=\Phi_u^\top P\Phi_u$. The current evidence supports local-surrogate analysis and numerical simulation, not a global nonlinear safety guarantee. Controlled interventions show that, under fixed eigenvalues and source covariance, increasing eigenvector nonorthogonality can raise state covariance trace from about $0.37$ to $15.66$; under a fixed nonnormal amplifier, source filtering reduces action variance from $0.03992$ to $0.00317$ and state covariance from $15.849$ to $12.393$. A matched two-dimensional constrained-RL ablation further shows why scalar action variance and action-difference penalties are insufficient by themselves.
\end{abstract}

\textbf{Keywords:} reinforcement learning, nonnormality, transient amplification, action smoothing, Koopman surrogate, Lyapunov certificate, safe control

\section{Introduction}

High-return continuous-control policies can still produce jittery commands, dense constraint-margin losses, and frequent safety-layer corrections. The usual response is to penalize action variance, action differences, or actuator roughness. These penalties are useful, but they do not explain why the same magnitude of residual action noise can be benign in one closed loop and dangerous in another.

This paper argues that the central issue is not whether actions are smooth in Euclidean norm. The central issue is whether residual action covariance is injected into directions that a nonnormal closed-loop surrogate strongly amplifies. The action residual is a source; the closed-loop dynamics form an amplifier; state covariance, safety-margin loss, posterior Bellman-Lyapunov residuals, and projection burden are downstream consequences.

The goal is therefore a problem-first diagnostic/control framework rather than a broad safe-RL benchmark or a global nonlinear safety theorem. The concrete response framework, certificate-guided residual variance control (CG-RVC), is used only as one implementation of this source-amplifier view. It combines local surrogate fitting, Lyapunov weighting, residual covariance control, smoothing, rate limiting, and safety projection.

\section{Nonnormal Closed-Loop Amplifier}

Consider a local lifted surrogate
\begin{equation}
z_{t+1}=Kz_t+\Phi_u r_t+w_t ,
\label{eq:local}
\end{equation}
where $z_t=\psi(x_t)$ is a lifted state, $r_t$ is a residual action or command fluctuation, $\Phi_u$ is the input channel, and $w_t$ captures disturbances and model error. For white source covariance $\Sigma_r$ and disturbance covariance $W$, the local stationary covariance satisfies
\begin{equation}
\Sigma_z=\sum_{i=0}^{\infty}K^i(\Phi_u\Sigma_r\Phi_u^\top+W)(K^i)^\top .
\label{eq:covprop}
\end{equation}
Equation~\eqref{eq:covprop} is the basic separation: action variance is not state variance itself; it becomes state variance only after propagation through the closed-loop amplifier.

Spectral-radius stability, $\rho(K)<1$, is not enough to rule out short-horizon energy growth. If $K=V\Lambda V^{-1}$ is highly nonnormal, eigenvectors can be ill-conditioned even when eigenvalues lie inside the unit disk. Diagnostics such as $\kappa(V)$, peak transient gain $G_{\mathrm{peak}}=\max_{t\le H}\|K^t\|_2$, resolvent norms, and Kreiss-like proxies are therefore relevant to reinforcement-learning evaluation.

\begin{figure}[t]
\centering
\includegraphics[width=\linewidth]{source_amplifier_layers.png}
\caption{Source-amplifier view. Residual actions, exploration noise, execution noise, and disturbances enter the input channel $\Phi_u$ and are propagated by a nonnormal closed-loop amplifier $K$. The downstream consequences are state covariance, safety-margin loss, posterior Bellman-Lyapunov residuals, and projection burden.}
\label{fig:source_amp}
\end{figure}

\section{Why Ordinary Smoothing Is Not Enough}

A scalar action-variance penalty,
\begin{equation}
R_{\mathrm{var}}=\mathrm{tr}(\Sigma_u),
\end{equation}
measures total source energy but ignores input direction. A temporal roughness penalty,
\begin{equation}
R_{\Delta u}=\mathbb{E}\|u_t-u_{t-1}\|^2,
\end{equation}
suppresses fast changes but does not determine whether the remaining residual covariance aligns with high-amplification directions.

Given a local Lyapunov metric $P$ with measured contraction factor $\gamma_P$,
\begin{equation}
K^\top P K \preceq \gamma_P^2P,
\end{equation}
the residual input directions can be weighted by
\begin{equation}
M=\Phi_u^\top P\Phi_u .
\end{equation}
The corresponding P-weighted action Lyapunov energy (P-ALE) is
\begin{equation}
R_{\mathrm{P\text{-}ALE}}=
\frac{\mathrm{tr}(M\Sigma_r)}{1-\gamma_P^2}.
\label{eq:pale}
\end{equation}
Smoothness is temporal regularity. P-ALE is amplifier-aware residual covariance control. They are related, but not equivalent.

\begin{table}[t]
\centering
\small
\caption{Diagnostic quantities for a source-amplifier paper.}
\label{tab:diagnostics}
\begin{tabular}{p{0.25\linewidth}p{0.31\linewidth}p{0.34\linewidth}}
\toprule
Layer & Quantity & Interpretation \\
\midrule
Source & $\mathrm{tr}(\Sigma_r)$, applied variance, diff MSE & How much residual or executed action jitter is injected. \\
Amplifier & $\rho(K)$, $\kappa(V)$, $G_{\mathrm{peak}}$, resolvent norm & Whether stable-looking dynamics can transiently amplify sources. \\
Certificate & $P$, $\gamma_P$, $M=\Phi_u^\top P\Phi_u$ & Which action directions are high-energy under the local certificate. \\
Consequence & state covariance trace, dense violation, B-L residual & Whether source jitter becomes dispersion or safety-margin loss. \\
Execution & projection trigger rate, normalized correction & How much repair the safety layer must perform. \\
\bottomrule
\end{tabular}
\end{table}

\section{Minimal Certificate-Guided Response}

The response framework has five steps. First, fit a local surrogate
$z_{t+1}=K_0z_t+\Phi_u u_t+w_t$ using local linearization, EDMDc-style regression, or a replay-buffer local model. Second, construct a nominal or projected closed-loop matrix, for instance $K_{\mathrm{raw}}=K_0-\Phi_u L$ followed by $K_{\mathrm{proj}}=\Pi_\beta(K_{\mathrm{raw}})$. Third, solve a Lyapunov equation or inequality to obtain $P$. Fourth, add the P-ALE penalty in Eq.~\eqref{eq:pale} to the actor objective. Fifth, apply execution-layer shaping through smoothing, rate limiting, and safety projection.

The spectral target $\beta$ is not a P-metric contraction guarantee. Bounds and diagnostics should use the measured $\gamma_P$. Moreover, smoothing and projection make the execution layer nonlinear; the certificate is therefore a local design and diagnostic object rather than a global nonlinear plant certificate.

If the residual policy is state-dependent, the residual is not purely exogenous. A local diagnostic should estimate
\begin{equation}
K_\theta=K+\Phi_u J_r ,
\label{eq:ktheta}
\end{equation}
where $J_r$ is a finite-difference actor-Jacobian proxy. When $J_r$ is large, the feedback component must be folded into the amplifier.

\section{Evidence}

The evidence should separate source, amplifier, and safety repair. Figure~\ref{fig:ci} combines two controlled interventions. CI-1 fixes eigenvalues, spectral radius, input channel, and source covariance while changing eigenvector geometry. In the archived result, $\kappa(V)$ grows from about $1.0$ to $20.05$, state covariance trace grows from about $0.37$ to $15.66$, and peak transient gain grows from about $1.0$ to $5.55$. This supports the statement that nonnormal geometry can independently amplify a fixed source in a controlled matrix family.

CI-2 fixes the strongly nonnormal amplifier and changes the source. In the archived result, the filtered source lowers action variance from $0.03992$ to $0.00317$ and state covariance trace from $15.849$ to $12.393$, while structural peak gain stays fixed near $5.55$. This is source-side evidence, but the filtered input is colored; it should be interpreted through an augmented colored-input model.

\begin{figure}[t]
\centering
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{fig1_ci1_amplifier_isolation.png}
\caption{CI-1}
\end{subfigure}
\hfill
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{fig2_ci2_source_intervention.png}
\caption{CI-2}
\end{subfigure}
\caption{Controlled interventions. CI-1 manipulates nonnormal amplifier geometry while fixing source statistics. CI-2 manipulates source statistics while fixing the amplifier.}
\label{fig:ci}
\end{figure}

\begin{table}[t]
\centering
\small
\caption{Compressed high-noise 2D constrained-RL evidence. Values are mean summaries from the archived matched ablation.}
\label{tab:ablation}
\begin{tabular}{p{0.34\linewidth}rrrr}
\toprule
Method & RMSE & Var. & Viol. & B-L max \\
\midrule
Direct TD3 & 0.537 & 0.1138 & 0.9389 & 92.4 \\
TD3 + Var$(u)$ & 0.612 & 0.1026 & 0.9278 & 161.9 \\
TD3 + action diff & 0.565 & 0.1118 & 0.9389 & 109.1 \\
Projection only & 0.505 & 0.1033 & 0.1222 & 82.3 \\
Full response & 0.284 & 0.0060 & 0.0000 & 5.24 \\
\bottomrule
\end{tabular}
\end{table}

Table~\ref{tab:ablation} shows why ordinary smoothing is incomplete in the high-noise constrained-RL task. Scalar action variance and action-difference penalties do not remove dense violations or posterior residuals. Projection alone lowers violations but leaves high applied variance and residuals. The full response combines source shaping and execution repair.

Projection burden gives an operational downstream diagnostic. In the archived 2D high-noise run, normalized correction is about $0.09362$ for projection-only, $0.05426$ for P-ALE plus projection, and $0.01093$ for the full method. A residual-feedback diagnostic gives another caveat: the finite-difference actor-Jacobian proxy is about $16.23$ for Direct TD3 and $0.274$ for the full method. This supports, but does not prove, the source-side interpretation.

\section{Limitations}

The claim level is deliberately narrow. First, all certificate claims are local-surrogate-level statements, not global nonlinear safety guarantees. Second, P-ALE reduces certificate-weighted residual covariance entering the amplifier; it does not directly reduce nonnormality of $K$. Third, if residual feedback is strong, the source-only interpretation becomes diagnostic unless the feedback is folded into Eq.~\eqref{eq:ktheta}. Fourth, projection depends on local feasibility and cannot create feasibility. Fifth, Full3D quadrotor and ControlGym results are simulation-level relevance checks, not hardware validation or broad safe-RL benchmark dominance.

\section{Conclusion}

Low-variance safe reinforcement learning should report both the source and the amplifier. The relevant object is not only $\mathrm{Var}(u)$ or $\|u_t-u_{t-1}\|^2$, but the full chain from $\Sigma_r$ through $\Phi_u$, $K$, $P$, and $M$ into state covariance, safety-margin loss, posterior Bellman-Lyapunov residuals, and projection burden. This source-amplifier framing explains why ordinary smoothing can be insufficient and why certificate-weighted residual covariance is a natural diagnostic/control response for nonnormal RL closed loops.

\end{document}
