% Options for packages loaded elsewhere
\PassOptionsToPackage{unicode}{hyperref}
\PassOptionsToPackage{hyphens}{url}
\PassOptionsToPackage{dvipsnames,svgnames,x11names}{xcolor}
%
\documentclass[
  letterpaper,
  DIV=11,
  numbers=noendperiod]{scrartcl}

\usepackage{amsmath,amssymb}
\usepackage{iftex}
\ifPDFTeX
  \usepackage[T1]{fontenc}
  \usepackage[utf8]{inputenc}
  \usepackage{textcomp} % provide euro and other symbols
\else % if luatex or xetex
  \usepackage{unicode-math}
  \defaultfontfeatures{Scale=MatchLowercase}
  \defaultfontfeatures[\rmfamily]{Ligatures=TeX,Scale=1}
\fi
\usepackage{lmodern}
\ifPDFTeX\else  
    % xetex/luatex font selection
\fi
% Use upquote if available, for straight quotes in verbatim environments
\IfFileExists{upquote.sty}{\usepackage{upquote}}{}
\IfFileExists{microtype.sty}{% use microtype if available
  \usepackage[]{microtype}
  \UseMicrotypeSet[protrusion]{basicmath} % disable protrusion for tt fonts
}{}
\makeatletter
\@ifundefined{KOMAClassName}{% if non-KOMA class
  \IfFileExists{parskip.sty}{%
    \usepackage{parskip}
  }{% else
    \setlength{\parindent}{0pt}
    \setlength{\parskip}{6pt plus 2pt minus 1pt}}
}{% if KOMA class
  \KOMAoptions{parskip=half}}
\makeatother
\usepackage{xcolor}
\usepackage[lmargin=40mm,rmargin=40mm]{geometry}
\setlength{\emergencystretch}{3em} % prevent overfull lines
\setcounter{secnumdepth}{5}
% Make \paragraph and \subparagraph free-standing
\ifx\paragraph\undefined\else
  \let\oldparagraph\paragraph
  \renewcommand{\paragraph}[1]{\oldparagraph{#1}\mbox{}}
\fi
\ifx\subparagraph\undefined\else
  \let\oldsubparagraph\subparagraph
  \renewcommand{\subparagraph}[1]{\oldsubparagraph{#1}\mbox{}}
\fi


\providecommand{\tightlist}{%
  \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}\usepackage{longtable,booktabs,array}
\usepackage{calc} % for calculating minipage widths
% Correct order of tables after \paragraph or \subparagraph
\usepackage{etoolbox}
\makeatletter
\patchcmd\longtable{\par}{\if@noskipsec\mbox{}\fi\par}{}{}
\makeatother
% Allow footnotes in longtable head/foot
\IfFileExists{footnotehyper.sty}{\usepackage{footnotehyper}}{\usepackage{footnote}}
\makesavenoteenv{longtable}
\usepackage{graphicx}
\makeatletter
\def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth\else\Gin@nat@width\fi}
\def\maxheight{\ifdim\Gin@nat@height>\textheight\textheight\else\Gin@nat@height\fi}
\makeatother
% Scale images if necessary, so that they will not overflow the page
% margins by default, and it is still possible to overwrite the defaults
% using explicit options in \includegraphics[width, height, ...]{}
\setkeys{Gin}{width=\maxwidth,height=\maxheight,keepaspectratio}
% Set default figure placement to htbp
\makeatletter
\def\fps@figure{htbp}
\makeatother

\usepackage{booktabs}
\usepackage{longtable}
\usepackage{array}
\usepackage{multirow}
\usepackage{wrapfig}
\usepackage{float}
\usepackage{colortbl}
\usepackage{pdflscape}
\usepackage{tabu}
\usepackage{threeparttable}
\usepackage{threeparttablex}
\usepackage[normalem]{ulem}
\usepackage{makecell}
\usepackage{xcolor}
\usepackage{float}
\usepackage{amsmath}
\floatplacement{figure}{H}
\usepackage{hyperref}
\usepackage{nameref}
\usepackage{titling}
\usepackage{setspace}
%\usepackage{fontspec} had to remove as bmc system doesn't use lualatex
\usepackage{url}
\usepackage[export]{adjustbox}
\usepackage[square,numbers]{natbib}
\usepackage[table,xcdraw]{xcolor}
\usepackage{lscape}
\usepackage{amsmath}

\addtokomafont{disposition}{\rmfamily}
\usepackage[right]{lineno}
\newcommand{\keywords}[1]{\footnotesize \textbf{Keywords: } #1}


\usepackage{xr}
\externaldocument{6__evaluating-gender-bias}
\KOMAoption{captions}{tableheading}
\makeatletter
\@ifpackageloaded{caption}{}{\usepackage{caption}}
\AtBeginDocument{%
\ifdefined\contentsname
  \renewcommand*\contentsname{Table of contents}
\else
  \newcommand\contentsname{Table of contents}
\fi
\ifdefined\listfigurename
  \renewcommand*\listfigurename{List of Figures}
\else
  \newcommand\listfigurename{List of Figures}
\fi
\ifdefined\listtablename
  \renewcommand*\listtablename{List of Tables}
\else
  \newcommand\listtablename{List of Tables}
\fi
\ifdefined\figurename
  \renewcommand*\figurename{Figure}
\else
  \newcommand\figurename{Figure}
\fi
\ifdefined\tablename
  \renewcommand*\tablename{Table}
\else
  \newcommand\tablename{Table}
\fi
}
\@ifpackageloaded{float}{}{\usepackage{float}}
\floatstyle{ruled}
\@ifundefined{c@chapter}{\newfloat{codelisting}{h}{lop}}{\newfloat{codelisting}{h}{lop}[chapter]}
\floatname{codelisting}{Listing}
\newcommand*\listoflistings{\listof{codelisting}{List of Listings}}
\makeatother
\makeatletter
\makeatother
\makeatletter
\@ifpackageloaded{caption}{}{\usepackage{caption}}
\@ifpackageloaded{subcaption}{}{\usepackage{subcaption}}
\makeatother
\ifLuaTeX
  \usepackage{selnolig}  % disable illegal ligatures
\fi
\usepackage[]{natbib}
\bibliographystyle{unsrtnat}
\usepackage{bookmark}

\IfFileExists{xurl.sty}{\usepackage{xurl}}{} % add URL line breaks if available
\urlstyle{same} % disable monospaced font for URLs
\hypersetup{
  pdftitle={Evaluating gender bias in Large Language Models in long-term care: supplementary information},
  pdfauthor={Sam Rickman},
  colorlinks=true,
  linkcolor={blue},
  filecolor={Maroon},
  citecolor={Blue},
  urlcolor={Blue},
  pdfcreator={LaTeX via pandoc}}

\title{Evaluating gender bias in Large Language Models in long-term
care: supplementary information}
\author{Sam Rickman}
\date{}

\begin{document}
\maketitle

\section{Supplementary Information}\label{sec-paper3appendix}

Three appendices are included:

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\tightlist
\item
  Evaluation of sentiment metrics: establishing which sentiment metrics
  are appropriate for conducting this analysis.
\item
  Model diagnostics and robustness checks: verifying the robustness of
  conclusions using several other methods.
\item
  Evaluation of themes: full lists of words counted in the frequency of
  the words appearing in each theme.
\end{enumerate}

The code to reproduce this analysis is available with synthetic data in
the GitHub repository \citep{egbrepo}.

\subsection{Appendix 1 - Evaluation of appropriateness of sentiment
metrics}\label{appendix-1---evaluation-of-appropriateness-of-sentiment-metrics}

It was important to establish that any differences in sentiment output
were due to bias in the summaries, rather than bias in the sentiment
metrics used. To this end, prior to summarising the texts, the three
sentiment metrics were evaluated on the male and female versions of each
of the original documents. This was done to determine whether any of the
sentiment analysis metrics identified significant differences in
sentiment in texts that were identical apart from gender. Such
differences would indicate that the sentiment metrics, rather than the
summaries, were responsible for any observed disparities in sentiment.
Regard and SiEBERT did not show significant differences based on gender.
However, the DistilBERT-based model did, and as a result, it was not
used to analyse differences in sentiment in the summaries.

\subsubsection{Paired t-test}\label{paired-t-test}

A \(t\)-test was used to compare the scores between the continuous
metrics, the DistilBERT-based measure, and Regard. For the binary
SiEBERT model, McNemar's \(\chi^2\) test for symmetry was used. As these
documents are identical except for gender, the paired implementation of
these tests was applied, using the \texttt{t.test} function for the
continuous measure and \texttt{mcnemar.test} for the binary measure,
both in the \texttt{stats} package in R \citep{rcore}. The results
comparing sentiment between genders for the original sentences are set
out in Table~\ref{tbl-tmcnemar}. The null hypothesis is that there are
no differences in sentiment. As the needs and circumstances described in
the male and female versions of the documents are identical, it was
expected that this hypothesis would not be rejected. Indeed, the null
hypothesis was not rejected for SiEBERT and Regard. However, the
DistilBERT-based model showed a larger effect size, and the \(p\)-value
indicated that the null hypothesis should be rejected, meaning
gender-based differences in how sentiment is measured by this model were
observed.

\begin{table}

\caption{\label{tbl-tmcnemar}t-test and Mcnemar test results}

\centering{

\centering
\begin{threeparttable}
\begin{tabular}[t]{lrll}
\toprule
Direction & Effect size & Pr(>|t|) & signif\\
\midrule
\addlinespace[0.3em]
\multicolumn{4}{l}{\textbf{siebert}}\\
\hspace{1em}fm & 0.00409 & 0.627 & \\
\hspace{1em}mf & -0.00680 & 0.804 & \\
\addlinespace[0.3em]
\multicolumn{4}{l}{\textbf{regard}}\\
\hspace{1em}fm & 0.01610 & 0.228 & \\
\hspace{1em}mf & 0.00799 & 0.613 & \\
\addlinespace[0.3em]
\multicolumn{4}{l}{\textbf{distilbert}}\\
\hspace{1em}fm & -0.39400 & 1.03e-177 & ***\\
\hspace{1em}mf & -0.32700 & 5.2e-91 & ***\\
\bottomrule
\end{tabular}
\begin{tablenotes}
\item \textit{Note: } 
\item t-test is used for the continuous metrics and the McNemar's test for the binary SiEBERT metric
\end{tablenotes}
\end{threeparttable}

}

\end{table}%

\subsubsection{Mixed effects model: sentence
level}\label{mixed-effects-model-sentence-level}

The sentiment metrics were also examined using a mixed effects model. A
random intercept was introduced at the sentence level, as the sentiment
of each sentence is known to depend on what it describes. Gender and a
variable called \texttt{gender\_direction}, indicating whether the
original text was written about a male and the generated text about a
female (or vice versa), were also included in the model. This was done
to control for any differences in the content typically written about
men and women. The mixed-effects model was specified as follows:

\begin{align}
\begin{split}
\text{sentiment}_{ij} &= \beta_0 + \beta_1 \cdot \text{gender}_i + \beta_2 \cdot \text{gender\_direction}_i \\
&\quad + u_{0j} + \epsilon_{ij}
\label{eq:originalsmixed}
\end{split}
\end{align}

Where:

\begin{itemize}
\tightlist
\item
  \(sentiment\) is a continuous indicator of the proportion of the text
  which contains non-negative sentiment
\item
  \(gender\) is a binary indicator of whether a text is about a man or a
  woman.
\item
  \(gender\_direction\) is a binary indicator of whether the original
  text was written about a male and the generated text about a female,
  or vice versa.
\item
  \(u_{0j}\) is a random intercept for the \(j\)-th group (Sentence ID),
  accounting for the variability in sentiment across different
  sentences.
\item
  \(\epsilon_{ij}\): Residual error term for the \(j\)-th observation
  within the \(j\)-th group.
\end{itemize}

The covariance of the random intercept was allowed to be unstructured.
It was assumed that the random intercepts \(u_{0j}\) follow a normal
distribution with mean 0 and variance \(\sigma_{u0}^2\), the residuals
\(\epsilon_{ij}\) are independently and normally distributed with mean 0
and variance \(\sigma^2\), and the random intercepts \(u_{0j}\) are
independent of the residuals \(\epsilon_{ij}\).

Since the final activation layer of SiEBERT is softmax, producing binary
predictions of sentiment (i.e., positive or negative), a generalised
linear model with a logistic link function was used for the
sentence-level SiEBERT predictions. In this case,
\(\text{logit}(P(sentiment=1))\) was estimated, where sentiment can take
the values 0 (negative) or 1 (positive). The right-hand side of the
equation remained the same.

The results of the mixed model, as specified in Equation
\eqref{eq:originalsmixed}, are shown in Table~\ref{tbl-originalsmixed}.
These results are consistent with the \(t\)-test findings, indicating
that Regard and SiEBERT do not find systematic differences in the
sentiment of the original documents based on gender, but the
DistilBERT-based model does.

\begin{table}

\caption{\label{tbl-originalsmixed}Sentiment output: mixed model
(sentence level)}

\centering{

\centering
\begin{threeparttable}
\begin{tabular}[t]{lrrrll}
\toprule
Coef & Estimate & Std. Error & t value & Pr(>|t|) & Signif\\
\midrule
\addlinespace[0.3em]
\multicolumn{6}{l}{\textbf{regard}}\\
\hspace{1em}(Intercept) & 0.320000 & 0.003650 & 87.700 & <0.001 & ***\\
\hspace{1em}Gender: Male & 0.000561 & 0.000435 & 1.290 & 0.197 & \\
\hspace{1em}Gender direction: mf & 0.003790 & 0.005660 & 0.669 & 0.504 & \\
\addlinespace[0.3em]
\multicolumn{6}{l}{\textbf{siebert}}\\
\hspace{1em}(Intercept) & 0.400000 & 0.009360 & 42.700 & 3.53e-187 & ***\\
\hspace{1em}Gender: Male & 0.000569 & 0.001280 & 0.443 & 0.658 & \\
\hspace{1em}Gender direction: mf & -0.018000 & 0.014500 & -1.240 & 0.214 & \\
\addlinespace[0.3em]
\multicolumn{6}{l}{\textbf{distilbert}}\\
\hspace{1em}(Intercept) & 0.665000 & 0.003450 & 193.000 & <0.001 & ***\\
\hspace{1em}Gender: Male & -0.007110 & 0.000241 & -29.500 & 3.12e-120 & ***\\
\hspace{1em}Gender direction: mf & 0.004730 & 0.005350 & 0.883 & 0.378 & \\
\bottomrule
\end{tabular}
\begin{tablenotes}
\item \textit{Note: } 
\item The SiEBERT binomial produces a z-value rather than t-value. For the purpose of presentation, this is included in the t-value column.
\end{tablenotes}
\end{threeparttable}

}

\end{table}%

\subsubsection{Mixed effects model: document
level}\label{mixed-effects-model-document-level}

It is reassuring that the mixed model results at sentence level are
consistent with the \(t\)-test results. However, summaries do not
necessarily have the same number of sentences (and if they do the
sentences may not correspond). This means that sentiment for the male
and female versions of each summary will need to be aggregated and
compared at document level. To confirm that the metrics are appropriate,
the sentiment results were aggregated for the original texts at document
level, taking the mean of sentence-level sentiment. This is the same
model as Equation \eqref{eq:originalsmixed}, though clustering at
Document ID rather than Sentence ID level, i.e.

\begin{align}
\begin{split}
\text{sentiment}_{ij} &= \beta_0 + \beta_1 \cdot \text{gender}_i + \beta_2 \cdot \text{gender\_direction}_i \\
&\quad + u_{0j} + \epsilon_{ij}
\label{eq:originalsmixedmean}
\end{split}
\end{align}

Where:

\begin{itemize}
\tightlist
\item
  \(sentiment\) is a continuous indicator of the proportion of the text
  which contains non-negative sentiment (mean of each sentence across
  documents)
\item
  \(gender\) is a binary indicator of whether a text is about a man or a
  woman.
\item
  \(gender\_direction\) is a binary indicator of whether the original
  text was written about a male and the generated text about a female,
  or vice versa.
\item
  \(u_{0j}\) is a random intercept for the \(j\)-th group (Document ID),
  accounting for the variability in sentiment across different
  sentences.
\item
  \(\epsilon_{ij}\): Residual error term for the \(j\)-th observation
  within the \(j\)-th group.
\end{itemize}

Once again, the assumptions are the same. The covariance of the random
intercept is unstructured. The model assumes that the random intercepts
\(u_{0j}\) follow a normal distribution with mean 0 and variance
\(\sigma_{u0}^2\), the residuals \(\epsilon_{ij}\) are independently and
normally distributed with mean 0 and variance \(\sigma^2\) and the
random intercepts \(u_{0j}\) are independent of the residuals
\(\epsilon_{ij}\). A linear model is used for SiEBERT here too, as the
per-document average of binary sentence classifications is continuous.
Table~\ref{tbl-originalsmixedmeans} shows the results aggregated at
document level.

\begin{table}

\caption{\label{tbl-originalsmixedmeans}Sentiment output: mixed model
(document level)}

\centering{

\centering
\begin{tabular}[t]{lrrrll}
\toprule
Coef & Estimate & Std. Error & t value & Pr(>|t|) & Signif\\
\midrule
\addlinespace[0.3em]
\multicolumn{6}{l}{\textbf{regard}}\\
\hspace{1em}(Intercept) & 0.320000 & 0.003650 & 87.700 & <0.001 & ***\\
\hspace{1em}Gender: Male & 0.000561 & 0.000435 & 1.290 & 0.197 & \\
\hspace{1em}Gender direction: mf & 0.003790 & 0.005660 & 0.669 & 0.504 & \\
\addlinespace[0.3em]
\multicolumn{6}{l}{\textbf{siebert}}\\
\hspace{1em}(Intercept) & 0.400000 & 0.009360 & 42.700 & 3.53e-187 & ***\\
\hspace{1em}Gender: Male & 0.000569 & 0.001280 & 0.443 & 0.658 & \\
\hspace{1em}Gender direction: mf & -0.018000 & 0.014500 & -1.240 & 0.214 & \\
\addlinespace[0.3em]
\multicolumn{6}{l}{\textbf{distilbert}}\\
\hspace{1em}(Intercept) & 0.665000 & 0.003450 & 193.000 & <0.001 & ***\\
\hspace{1em}Gender: Male & -0.007110 & 0.000241 & -29.500 & 3.12e-120 & ***\\
\hspace{1em}Gender direction: mf & 0.004730 & 0.005350 & 0.883 & 0.378 & \\
\bottomrule
\end{tabular}

}

\end{table}%

Across all three measures, the DistilBERT-based model finds significant
differences in sentiment once gender is changed. This means it is not an
appropriate measure of sentiment for our analysis. This is why it is not
used in the paper to measure differences in sentiment of the summaries.
However, there are no significant differences using Regard or SiEBERT,
which is why these metrics are used to evaluate the output of the
summarisation models.

\newpage

\subsection{Appendix 2 - Model diagnostics and robustness
checks}\label{appendix-2---model-diagnostics-and-robustness-checks}

Table \ref{tab:varcorrdt} contains the covariance matrix for the random
effects in the model specified in Equation 2, with the results for the
main effects in Table 2. Table \ref{tab:varcorrdt} includes the
variances of individual variables and the covariances between variables.

\begin{table}[H]
\centering
\caption{\label{tab:varcorrdt}Covariance Matrix of Random Effects}
\centering
\resizebox{\ifdim\width>\linewidth\linewidth\else\width\fi}{!}{
\begin{tabular}[t]{llrrrr}
\toprule
\multicolumn{2}{c}{ } & \multicolumn{2}{c}{Regard} & \multicolumn{2}{c}{SiEBERT} \\
\cmidrule(l{3pt}r{3pt}){3-4} \cmidrule(l{3pt}r{3pt}){5-6}
Group & Variable & Variance-Covariance & Standard Deviation & Variance-Covariance & Standard Deviation\\
\midrule
\cellcolor{gray!10}{Residual} & \cellcolor{gray!10}{} & \cellcolor{gray!10}{0.006} & \cellcolor{gray!10}{0.078} & \cellcolor{gray!10}{0.035} & \cellcolor{gray!10}{0.187}\\
doc\_id & (Intercept) & 0.011 & 0.103 & 0.074 & 0.272\\
\cellcolor{gray!10}{doc\_id} & \cellcolor{gray!10}{(Intercept) - modelgemma} & \cellcolor{gray!10}{-0.008} & \cellcolor{gray!10}{-0.835} & \cellcolor{gray!10}{-0.050} & \cellcolor{gray!10}{-0.809}\\
doc\_id & (Intercept) - modelllama3 & -0.007 & -0.791 & -0.044 & -0.704\\
\cellcolor{gray!10}{doc\_id} & \cellcolor{gray!10}{(Intercept) - modelt5} & \cellcolor{gray!10}{-0.007} & \cellcolor{gray!10}{-0.678} & \cellcolor{gray!10}{-0.042} & \cellcolor{gray!10}{-0.660}\\
\addlinespace
doc\_id & modelgemma & 0.008 & 0.090 & 0.051 & 0.226\\
\cellcolor{gray!10}{doc\_id} & \cellcolor{gray!10}{modelgemma - modelllama3} & \cellcolor{gray!10}{0.007} & \cellcolor{gray!10}{0.895} & \cellcolor{gray!10}{0.046} & \cellcolor{gray!10}{0.888}\\
doc\_id & modelgemma - modelt5 & 0.006 & 0.691 & 0.038 & 0.723\\
\cellcolor{gray!10}{doc\_id} & \cellcolor{gray!10}{modelllama3} & \cellcolor{gray!10}{0.008} & \cellcolor{gray!10}{0.090} & \cellcolor{gray!10}{0.053} & \cellcolor{gray!10}{0.229}\\
doc\_id & modelllama3 - modelt5 & 0.006 & 0.685 & 0.038 & 0.701\\
\addlinespace
\cellcolor{gray!10}{doc\_id} & \cellcolor{gray!10}{modelt5} & \cellcolor{gray!10}{0.009} & \cellcolor{gray!10}{0.097} & \cellcolor{gray!10}{0.055} & \cellcolor{gray!10}{0.234}\\
\bottomrule
\end{tabular}}
\end{table}

The distribution of the linear mixed model's random effects is presented
in Figure~\ref{fig-re}, and a Q-Q plot of observed and expected values
for residuals is shown in Figure~\ref{fig-qqplot}. The random effects
are generally normally distributed, with the notable exception of the
intercept for the SiEBERT model, which demonstrates clear asymmetry at
the tails. The Q-Q plot reveals the presence of some outliers and
heteroscedasticity, particularly with the SiEBERT predictions, which
deviate more from normality at the tails. The Regard predictions fit
more closely to the normal distribution, although the residuals do not
perfectly follow the expected distribution at the tails. Despite these
deviations, the bootstrapping results and robustness checks ensure the
conclusions remain reliable.

\begin{figure}[H]

\centering{

\includegraphics{./papers-files/paper3/plots/random_slopes.pdf}

}

\caption{\label{fig-re}Distribution of random effects}

\end{figure}%

\begin{figure}[H]

\centering{

\includegraphics{./papers-files/paper3/plots/qqplot.pdf}

}

\caption{\label{fig-qqplot}Normal QQ plot}

\end{figure}%

The linear mixed model assumes normality of random effects and
homoscedasticity. Simulations show that violations of these assumptions
often have little or no effect on parameter estimates, although they do
affect the interpretation of the significance of the variance of the
random effects \citep{maas2004influence}. The primary focus is on the
fixed effects rather than document-level random effects, which are
mainly included to account for the clustering of sentiment within
documents. However, as the assumptions of the model are not always
satisfied, other approaches were explored to assess the sensitivity of
the conclusions to these assumptions. Given the presence of some
non-linearities, interaction terms, such as the interaction between
gender and the maximum number of tokens, were tested to account for
possible non-linear relationships. However, analysis of variance (ANOVA)
and likelihood-ratio tests indicated that the interaction term did not
significantly improve model fit (p \textgreater{} 0.05). Consequently,
the interaction was removed to simplify the model without affecting the
overall conclusions regarding gender bias in the summaries. The model
equation was retained without interactions, and other methods were used
to assess the robustness of the findings:

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\tightlist
\item
  \textbf{Bootstrapping:} To test sensitivity to outliers, 1,000
  bootstrap samples were generated by resampling the original data with
  replacement, and the model was refitted on each sample. This approach
  provided an estimate of the distribution of the parameter estimates
  and allowed an assessment of the stability of the findings across
  different datasets.
\item
  \textbf{Relaxing the variance structure:} To test sensitivity to the
  assumption of homoscedasticity, a mixed-effects model was fitted,
  allowing for different residual variances across each language model.
\item
  \textbf{Robust linear mixed model:} To test sensitivity to outliers, a
  robust linear mixed model was used.
\item
  \textbf{Generalised Estimating Equations (GEE) model:} To test
  sensitivity to the correlation structure of the data and the
  assumption of normally distributed random effects, a GEE model was
  fitted. This model used robust sandwich estimators to adjust standard
  errors, allowing for heteroscedasticity in the residuals.
\item
  \textbf{Linear models:} To test sensitivity to the inclusion of random
  effects at the model level, each language model's results were split
  into its own dataset, and a linear model was run with Document ID as a
  main effect.
\end{enumerate}

The results of each of these models were generally consistent with the
findings of the mixed model. None of the models identified gender-based
differences caused by Llama 3. Some variation was observed across the
models regarding the significance of the differences in sentiment for
the BART and T5 models. However, all models agreed that there were
significant gender-based differences in the summaries generated by the
Gemma model.

\subsubsection{Bootstrapped model output and estimated marginal
means}\label{bootstrapped-model-output-and-estimated-marginal-means}

Bootstrapped datasets were generated by creating 1,000 new datasets,
each the same size as the original data, through non-parametric sampling
of the original data with replacement. Samples were taken at the
Document ID level to preserve the correlation of sentiment within
documents \citep{carpenter2000bootstrap}. The original linear mixed
model was then run for each bootstrapped dataset. The bootstrapped
estimates represent the mean of all 1,000 estimates. The results for the
SiEBERT model are shown in Table
\ref{tab:tbl-final-sentiment-models-siebert-bootstrap}, and the results
for Regard are shown in Table
\ref{tab:tbl-final-sentiment-models-regard-bootstrap}. The additional
columns in Table \ref{tab:tbl-final-sentiment-models-siebert-bootstrap}
were calculated as follows:

\begin{align*}
\text{Absolute Bias} &= \text{Bootstrapped Estimate} - \text{Original Estimate} \\
\text{Relative Bias} &= \frac{\text{Absolute Bias}}{\text{Original Estimate}} \\
\text{Standardised Bias} &= \frac{\text{Absolute Bias}}{\text{Standard Error}}
\end{align*}

Bootstrapped estimated marginal means are presented in Table
\ref{tab:tbl-emmeansbootstrap}. The table also includes the number of
times the \(p\)-values for the estimated marginal means were less than
0.05 and 0.01. The differences in gender in the Gemma model are larger
using SiEBERT, with a larger \(t\)-value and a \(p\)-value of less than
0.01 in all 1,000 bootstrapped datasets. The difference is somewhat
smaller in the case of Regard, though \(p < 0.05\) in 962 of the 1,000
simulated datasets. The BART models show this effect in approximately
30-40\% of cases, and T5 shows it in 40-60\% of cases, suggesting that
there is an effect of gender bias greater than random chance, although
not as strong as the disparities observed in the Gemma model. There is
no indication of a systematic effect of gender on sentiment in Llama 3,
with slightly under 5\% of estimated marginal mean differences resulting
in \(p < 0.05\). Overall, the bootstrapping results confirm that while
some observable gender-based differences exist in BART and T5, the
largest differences are in the Gemma model.

\begin{table}[H]
\centering
\caption{\label{tab:tbl-final-sentiment-models-siebert-bootstrap}Bootstrapped model output (SiEBERT)}
\centering
\resizebox{\ifdim\width>\linewidth\linewidth\else\width\fi}{!}{
\begin{tabular}[t]{lrrrlrllr}
\toprule
\multicolumn{1}{c}{ } & \multicolumn{4}{c}{Original model} & \multicolumn{4}{c}{Bootstrapped model} \\
\cmidrule(l{3pt}r{3pt}){2-5} \cmidrule(l{3pt}r{3pt}){6-9}
\multicolumn{1}{c}{ } & \multicolumn{5}{c}{ } & \multicolumn{3}{c}{Bias} \\
\cmidrule(l{3pt}r{3pt}){7-9}
  & Estimate & Std. Error & t value & Pr(>|t|) & Estimate & Absolute & Relative & Standardised\\
\midrule
\cellcolor{gray!10}{(Intercept)} & \cellcolor{gray!10}{0.579} & \cellcolor{gray!10}{0.012} & \cellcolor{gray!10}{49.749} & \cellcolor{gray!10}{<0.001} & \cellcolor{gray!10}{0.579} & \cellcolor{gray!10}{<0.001} & \cellcolor{gray!10}{<0.001} & \cellcolor{gray!10}{-0.008}\\
modelgemma & 0.147 & 0.010 & 14.558 & <0.001 & 0.147 & <0.001 & 0.001 & 0.015\\
\cellcolor{gray!10}{modelllama3} & \cellcolor{gray!10}{0.052} & \cellcolor{gray!10}{0.010} & \cellcolor{gray!10}{5.132} & \cellcolor{gray!10}{<0.001} & \cellcolor{gray!10}{0.052} & \cellcolor{gray!10}{<0.001} & \cellcolor{gray!10}{0.001} & \cellcolor{gray!10}{0.004}\\
modelt5 & 0.103 & 0.010 & 9.904 & <0.001 & 0.103 & <0.001 & <0.001 & 0.002\\
\cellcolor{gray!10}{gendermale} & \cellcolor{gray!10}{-0.009} & \cellcolor{gray!10}{0.004} & \cellcolor{gray!10}{-2.161} & \cellcolor{gray!10}{0.031} & \cellcolor{gray!10}{-0.009} & \cellcolor{gray!10}{<0.001} & \cellcolor{gray!10}{0.001} & \cellcolor{gray!10}{-0.001}\\
\addlinespace
max\_tokens75 & -0.024 & 0.004 & -6.431 & <0.001 & -0.024 & <0.001 & -0.001 & 0.009\\
\cellcolor{gray!10}{max\_tokens100} & \cellcolor{gray!10}{-0.039} & \cellcolor{gray!10}{0.004} & \cellcolor{gray!10}{-10.304} & \cellcolor{gray!10}{<0.001} & \cellcolor{gray!10}{-0.039} & \cellcolor{gray!10}{<0.001} & \cellcolor{gray!10}{-0.002} & \cellcolor{gray!10}{0.021}\\
max\_tokens150 & -0.050 & 0.004 & -13.299 & <0.001 & -0.050 & <0.001 & -0.001 & 0.008\\
\cellcolor{gray!10}{max\_tokens300} & \cellcolor{gray!10}{-0.054} & \cellcolor{gray!10}{0.004} & \cellcolor{gray!10}{-14.419} & \cellcolor{gray!10}{<0.001} & \cellcolor{gray!10}{-0.054} & \cellcolor{gray!10}{<0.001} & \cellcolor{gray!10}{-0.002} & \cellcolor{gray!10}{0.026}\\
max\_tokensNone & -0.084 & 0.004 & -22.262 & <0.001 & -0.084 & <0.001 & -0.001 & 0.027\\
\addlinespace
\cellcolor{gray!10}{modelgemma:gendermale} & \cellcolor{gray!10}{-0.033} & \cellcolor{gray!10}{0.006} & \cellcolor{gray!10}{-5.318} & \cellcolor{gray!10}{<0.001} & \cellcolor{gray!10}{-0.033} & \cellcolor{gray!10}{<0.001} & \cellcolor{gray!10}{0.006} & \cellcolor{gray!10}{-0.030}\\
modelllama3:gendermale & 0.015 & 0.006 & 2.426 & 0.015 & 0.015 & <0.001 & -0.001 & -0.002\\
\cellcolor{gray!10}{modelt5:gendermale} & \cellcolor{gray!10}{0.020} & \cellcolor{gray!10}{0.006} & \cellcolor{gray!10}{3.185} & \cellcolor{gray!10}{0.001} & \cellcolor{gray!10}{0.019} & \cellcolor{gray!10}{<0.001} & \cellcolor{gray!10}{-0.012} & \cellcolor{gray!10}{-0.038}\\
\bottomrule
\end{tabular}}
\end{table}

\begin{table}[H]
\centering
\caption{\label{tab:tbl-final-sentiment-models-regard-bootstrap}Bootstrapped model output (Regard)}
\centering
\resizebox{\ifdim\width>\linewidth\linewidth\else\width\fi}{!}{
\begin{tabular}[t]{lrrrlrllr}
\toprule
\multicolumn{1}{c}{ } & \multicolumn{4}{c}{Original model} & \multicolumn{4}{c}{Bootstrapped model} \\
\cmidrule(l{3pt}r{3pt}){2-5} \cmidrule(l{3pt}r{3pt}){6-9}
\multicolumn{1}{c}{ } & \multicolumn{5}{c}{ } & \multicolumn{3}{c}{Bias} \\
\cmidrule(l{3pt}r{3pt}){7-9}
  & Estimate & Std. Error & t value & Pr(>|t|) & Estimate & Absolute & Relative & Standardised\\
\midrule
\cellcolor{gray!10}{(Intercept)} & \cellcolor{gray!10}{0.278} & \cellcolor{gray!10}{0.004} & \cellcolor{gray!10}{61.965} & \cellcolor{gray!10}{<0.001} & \cellcolor{gray!10}{0.278} & \cellcolor{gray!10}{<0.001} & \cellcolor{gray!10}{<0.001} & \cellcolor{gray!10}{0.019}\\
modelgemma & 0.025 & 0.004 & 6.109 & <0.001 & 0.025 & <0.001 & -0.004 & -0.027\\
\cellcolor{gray!10}{modelllama3} & \cellcolor{gray!10}{0.029} & \cellcolor{gray!10}{0.004} & \cellcolor{gray!10}{7.061} & \cellcolor{gray!10}{<0.001} & \cellcolor{gray!10}{0.029} & \cellcolor{gray!10}{<0.001} & \cellcolor{gray!10}{0.001} & \cellcolor{gray!10}{0.010}\\
modelt5 & -0.033 & 0.004 & -7.712 & <0.001 & -0.033 & <0.001 & 0.003 & -0.023\\
\cellcolor{gray!10}{gendermale} & \cellcolor{gray!10}{0.004} & \cellcolor{gray!10}{0.002} & \cellcolor{gray!10}{1.954} & \cellcolor{gray!10}{0.051} & \cellcolor{gray!10}{0.004} & \cellcolor{gray!10}{<0.001} & \cellcolor{gray!10}{0.009} & \cellcolor{gray!10}{0.018}\\
\addlinespace
max\_tokens75 & 0.019 & 0.002 & 11.865 & <0.001 & 0.019 & <0.001 & <0.001 & -0.004\\
\cellcolor{gray!10}{max\_tokens100} & \cellcolor{gray!10}{0.027} & \cellcolor{gray!10}{0.002} & \cellcolor{gray!10}{17.076} & \cellcolor{gray!10}{<0.001} & \cellcolor{gray!10}{0.027} & \cellcolor{gray!10}{<0.001} & \cellcolor{gray!10}{0.001} & \cellcolor{gray!10}{0.020}\\
max\_tokens150 & 0.032 & 0.002 & 20.246 & <0.001 & 0.032 & <0.001 & -0.001 & -0.026\\
\cellcolor{gray!10}{max\_tokens300} & \cellcolor{gray!10}{0.039} & \cellcolor{gray!10}{0.002} & \cellcolor{gray!10}{25.052} & \cellcolor{gray!10}{<0.001} & \cellcolor{gray!10}{0.040} & \cellcolor{gray!10}{<0.001} & \cellcolor{gray!10}{0.001} & \cellcolor{gray!10}{0.022}\\
max\_tokensNone & 0.045 & 0.002 & 28.303 & <0.001 & 0.045 & <0.001 & <0.001 & -0.001\\
\addlinespace
\cellcolor{gray!10}{modelgemma:gendermale} & \cellcolor{gray!10}{-0.011} & \cellcolor{gray!10}{0.003} & \cellcolor{gray!10}{-4.082} & \cellcolor{gray!10}{<0.001} & \cellcolor{gray!10}{-0.011} & \cellcolor{gray!10}{<0.001} & \cellcolor{gray!10}{0.003} & \cellcolor{gray!10}{-0.012}\\
modelllama3:gendermale & -0.001 & 0.003 & -0.561 & 0.575 & -0.001 & <0.001 & 0.038 & -0.021\\
\cellcolor{gray!10}{modelt5:gendermale} & \cellcolor{gray!10}{0.001} & \cellcolor{gray!10}{0.003} & \cellcolor{gray!10}{0.521} & \cellcolor{gray!10}{0.603} & \cellcolor{gray!10}{0.001} & \cellcolor{gray!10}{<0.001} & \cellcolor{gray!10}{0.033} & \cellcolor{gray!10}{0.017}\\
\bottomrule
\end{tabular}}
\end{table}

\begin{table}[H]
\centering
\caption{\label{tab:tbl-emmeansbootstrap}Mixed effects model: bootstrapped estimated marginal means (female - male)}
\centering
\resizebox{\ifdim\width>\linewidth\linewidth\else\width\fi}{!}{
\begin{tabular}[t]{lrrrrrrrrrr}
\toprule
\multicolumn{1}{c}{ } & \multicolumn{5}{c}{Regard} & \multicolumn{5}{c}{SiEBERT} \\
\cmidrule(l{3pt}r{3pt}){2-6} \cmidrule(l{3pt}r{3pt}){7-11}
Model & Estimate & t & N & N Pr(|t|)<0.01 & N Pr(|t|)<0.05 & Estimate & t & N & N Pr(|t|)<0.01 & N Pr(|t|)<0.05\\
\midrule
\cellcolor{gray!10}{bart} & \cellcolor{gray!10}{-0.0036} & \cellcolor{gray!10}{-1.60} & \cellcolor{gray!10}{1000} & \cellcolor{gray!10}{146} & \cellcolor{gray!10}{331} & \cellcolor{gray!10}{0.0094} & \cellcolor{gray!10}{1.80} & \cellcolor{gray!10}{1000} & \cellcolor{gray!10}{235} & \cellcolor{gray!10}{430}\\
gemma & 0.0069 & 3.00 & 1000 & 764 & 962 & 0.0420 & 7.70 & 1000 & 1000 & 1000\\
\cellcolor{gray!10}{llama3} & \cellcolor{gray!10}{-0.0021} & \cellcolor{gray!10}{-0.91} & \cellcolor{gray!10}{1000} & \cellcolor{gray!10}{1} & \cellcolor{gray!10}{32} & \cellcolor{gray!10}{-0.0055} & \cellcolor{gray!10}{-0.99} & \cellcolor{gray!10}{1000} & \cellcolor{gray!10}{3} & \cellcolor{gray!10}{40}\\
t5 & -0.0050 & -2.20 & 1000 & 275 & 651 & -0.0099 & -1.80 & 1000 & 107 & 421\\
\bottomrule
\end{tabular}}
\end{table}

\subsubsection{Variance-structured mixed effects
model}\label{variance-structured-mixed-effects-model}

The Q-Q plots demonstrated deviations from normality, especially in the
tails, which differ by model. To account for this heteroscedasticity and
deviation from normality, the R \texttt{nlme} package \citep{nlme} was
used to employ a linear mixed-effects model which allowed the variance
to differ by model, i.e.

\begin{equation}
\text{Var}(\epsilon_{ij}) = \sigma^2_{\text{model}_i}
\end{equation}

This model would not converge with a random intercept and slope and this
variance specification, so the random slope was removed. The model was
therefore specified as follows:

\begin{align}
\begin{split}
\text{sentiment}_{ij} &= \beta_0 + \beta_1 \cdot \text{model}_i + \beta_2 \cdot \text{gender}_j \\
&\quad + \beta_3 \cdot (\text{model}_i \times \text{gender}_j) + \beta_4 \cdot \text{max\_tokens}_i \\
&\quad + u_{0j} + \epsilon_{ij}
\label{eq:robustlmm}
\end{split}
\end{align}

Where \(\beta_0\) is the intercept, \(\beta_1\), \(\beta_2\), and
\(\beta_3\) are the coefficients for model, gender, and their
interaction, \(\beta_4\) is the coefficient for maximum tokens,
\(u_{0j}\) is the random intercept for document \(j\) and
\(\epsilon_{ij}\) is the residual error term. The results are set out in
Table \ref{tab:tbl-final-sentiment-models-nlme} and the estimated
marginal means in Table \ref{tab:tbl-emmeansnlme}. The estimates are
very close to the output from the linear mixed model, though the
\(p\)-values are slightly larger. The BART and T5 models are on the
boundary of significance, but now the \(p\)-values are slightly larger
than 0.05. Llama 3 has no significant differences in sentiment between
men and women, and Gemma has the largest standardised estimates and
smallest \(p\)-values. This model reduces the risk of Type 1 error,
which is seen in the larger \(p\)-values, so it is reassuring that the
main findings about Llama 3 and Gemma remain consistent.

\begin{table}[H]
\centering
\caption{\label{tab:tbl-final-sentiment-models-nlme}Variance-structured mixed effects model output}
\centering
\resizebox{\ifdim\width>\linewidth\linewidth\else\width\fi}{!}{
\begin{tabular}[t]{lrlrrrrlrrr}
\toprule
\multicolumn{1}{c}{ } & \multicolumn{3}{c}{Regard} & \multicolumn{3}{c}{SiEBERT} \\
\cmidrule(l{3pt}r{3pt}){2-4} \cmidrule(l{3pt}r{3pt}){5-7}
Coef & Estimate &  & Std. Error & t & p & Estimate &  & Std. Error & t & p\\
\midrule
\cellcolor{gray!10}{(Intercept)} & \cellcolor{gray!10}{0.3100} & \cellcolor{gray!10}{***} & \cellcolor{gray!10}{0.0030} & \cellcolor{gray!10}{100.5866481} & \cellcolor{gray!10}{0.0e+00} & \cellcolor{gray!10}{0.5400} & \cellcolor{gray!10}{***} & \cellcolor{gray!10}{0.0083} & \cellcolor{gray!10}{64.278954} & \cellcolor{gray!10}{0.0e+00}\\
Model gemma & 0.0250 & *** & 0.0019 & 13.0451295 & 0.0e+00 & 0.1500 & *** & 0.0048 & 30.836185 & 0.0e+00\\
\cellcolor{gray!10}{Model llama3} & \cellcolor{gray!10}{0.0290} & \cellcolor{gray!10}{***} & \cellcolor{gray!10}{0.0019} & \cellcolor{gray!10}{14.7169102} & \cellcolor{gray!10}{0.0e+00} & \cellcolor{gray!10}{0.0520} & \cellcolor{gray!10}{***} & \cellcolor{gray!10}{0.0049} & \cellcolor{gray!10}{10.750380} & \cellcolor{gray!10}{0.0e+00}\\
Model t5 & -0.0330 & *** & 0.0024 & -13.7578664 & 0.0e+00 & 0.1000 & *** & 0.0059 & 17.502350 & 0.0e+00\\
\cellcolor{gray!10}{gendermale} & \cellcolor{gray!10}{0.0036} & \cellcolor{gray!10}{.} & \cellcolor{gray!10}{0.0020} & \cellcolor{gray!10}{1.7997477} & \cellcolor{gray!10}{7.2e-02} & \cellcolor{gray!10}{-0.0094} & \cellcolor{gray!10}{.} & \cellcolor{gray!10}{0.0052} & \cellcolor{gray!10}{-1.809674} & \cellcolor{gray!10}{7.0e-02}\\
\addlinespace
Max tokens 150 & 0.0046 & ** & 0.0018 & 2.5900877 & 9.6e-03 & -0.0069 &  & 0.0043 & -1.603439 & 1.1e-01\\
\cellcolor{gray!10}{Max tokens 300} & \cellcolor{gray!10}{0.0120} & \cellcolor{gray!10}{***} & \cellcolor{gray!10}{0.0018} & \cellcolor{gray!10}{6.6848292} & \cellcolor{gray!10}{0.0e+00} & \cellcolor{gray!10}{-0.0110} & \cellcolor{gray!10}{**} & \cellcolor{gray!10}{0.0043} & \cellcolor{gray!10}{-2.682003} & \cellcolor{gray!10}{7.3e-03}\\
Max tokens 50 & -0.0270 & *** & 0.0018 & -15.2606004 & 0.0e+00 & 0.0360 & *** & 0.0043 & 8.430521 & 0.0e+00\\
\cellcolor{gray!10}{Max tokens 75} & \cellcolor{gray!10}{-0.0083} & \cellcolor{gray!10}{***} & \cellcolor{gray!10}{0.0018} & \cellcolor{gray!10}{-4.6806345} & \cellcolor{gray!10}{2.9e-06} & \cellcolor{gray!10}{0.0130} & \cellcolor{gray!10}{**} & \cellcolor{gray!10}{0.0043} & \cellcolor{gray!10}{3.089684} & \cellcolor{gray!10}{2.0e-03}\\
Max tokens None & 0.0150 & *** & 0.0018 & 8.4733104 & 0.0e+00 & -0.0290 & *** & 0.0043 & -6.866574 & 0.0e+00\\
\addlinespace
\cellcolor{gray!10}{Model gemma : Male} & \cellcolor{gray!10}{-0.0110} & \cellcolor{gray!10}{***} & \cellcolor{gray!10}{0.0027} & \cellcolor{gray!10}{-3.9068436} & \cellcolor{gray!10}{9.4e-05} & \cellcolor{gray!10}{-0.0330} & \cellcolor{gray!10}{***} & \cellcolor{gray!10}{0.0067} & \cellcolor{gray!10}{-4.851052} & \cellcolor{gray!10}{1.2e-06}\\
Model llama3 : Male & -0.0014 &  & 0.0028 & -0.5237945 & 6.0e-01 & 0.0150 & * & 0.0069 & 2.159936 & 3.1e-02\\
\cellcolor{gray!10}{Model t5 : Male} & \cellcolor{gray!10}{0.0013} & \cellcolor{gray!10}{} & \cellcolor{gray!10}{0.0034} & \cellcolor{gray!10}{0.3931827} & \cellcolor{gray!10}{6.9e-01} & \cellcolor{gray!10}{0.0200} & \cellcolor{gray!10}{*} & \cellcolor{gray!10}{0.0083} & \cellcolor{gray!10}{2.351745} & \cellcolor{gray!10}{1.9e-02}\\
\bottomrule
\end{tabular}}
\end{table}

\begin{table}[H]
\centering
\caption{\label{tab:tbl-emmeansnlme}Variance-structured mixed effects: estimated marginal means (female - male)}
\centering
\begin{tabular}[t]{lrlrrrlrr}
\toprule
\multicolumn{1}{c}{ } & \multicolumn{4}{c}{Regard} & \multicolumn{4}{c}{SiEBERT} \\
\cmidrule(l{3pt}r{3pt}){2-5} \cmidrule(l{3pt}r{3pt}){6-9}
Model & Estimate &  & t & p & Estimate &  & t & p\\
\midrule
\cellcolor{gray!10}{bart} & \cellcolor{gray!10}{-0.0036} & \cellcolor{gray!10}{.} & \cellcolor{gray!10}{-1.8} & \cellcolor{gray!10}{0.07200} & \cellcolor{gray!10}{0.0094} & \cellcolor{gray!10}{.} & \cellcolor{gray!10}{1.8} & \cellcolor{gray!10}{0.07}\\
gemma & 0.0069 & *** & 3.8 & 0.00014 & 0.0420 & *** & 9.8 & 0.00\\
\cellcolor{gray!10}{llama3} & \cellcolor{gray!10}{-0.0021} & \cellcolor{gray!10}{} & \cellcolor{gray!10}{-1.1} & \cellcolor{gray!10}{0.27000} & \cellcolor{gray!10}{-0.0055} & \cellcolor{gray!10}{} & \cellcolor{gray!10}{-1.2} & \cellcolor{gray!10}{0.23}\\
t5 & -0.0049 & . & -1.8 & 0.07800 & -0.0100 &  & -1.6 & 0.12\\
\bottomrule
\end{tabular}
\end{table}

\subsubsection{Robust linear mixed
model}\label{robust-linear-mixed-model}

The results of the bootstrapping were reassuring in the case of the
Gemma model. However, significant differences were not always observed
in the BART and T5 models. From the Q-Q plots, it is known that
deviations from normality exist in the tails. To test the sensitivity of
the results to outliers, a robust linear mixed model was used. This
model follows the same structure as the standard linear mixed model,
given in Equation (1) in the main body of the paper:

\begin{align}
\begin{split}
\text{sentiment}_{ij} &= \beta_0 + \beta_1 \cdot \text{model}_i + \beta_2 \cdot \text{gender}_j \\
&\quad + \beta_3 \cdot (\text{model}_i \times \text{gender}_j) + \beta_4 \cdot \text{max\_tokens}_i \\
&\quad + u_{0j} + u_{1j} \cdot \text{model}_i + \epsilon_{ij}
\label{eq:summarieslmmrobust}
\end{split}
\end{align}

Where \(\beta_0\) is the intercept, \(\beta_1\), \(\beta_2\), and
\(\beta_3\) are the coefficients for model, gender, and their
interaction, \(\beta_4\) is the coefficient for maximum tokens,
\(u_{0j}\) is the random intercept for document \(j\), \(u_{1j}\) is the
random slope for model within document \(j\), and \(\epsilon_{ij}\) is
the residual error term.

The difference from the standard mixed effects model is that a robust
loss function was incorporated to reduce the impact of outliers in the
residuals. This was implemented using the \texttt{robustlmm} R package
\citep{robustlmm}. The results are shown in Table
\ref{tab:tbl-final-sentiment-models-robust}. The estimates obtained from
both the mixed-effects and robust mixed-effects models were extremely
close. The package does not produce \(p\)-values; however, marginal
means were estimated \citep{emmeans} and are presented in Table
\ref{tab:tbl-emmeansrobust}. The estimates and \(p\)-values are very
close to the output from the standard linear mixed model. Once again,
The BART and T5 models show p-values hovering around conventional
significance thresholds, with some disagreement in the direction of the
gender effect in the BART model between Regard and SiEBERT. For these
models, p-values range between 0.04 and 0.08, suggesting borderline
statistical significance that should be interpreted cautiously. The
Gemma model exhibits the largest standardised estimates and the smallest
\(p\)-values, with both sentiment metrics indicating that male sentiment
is more negative than female sentiment.

\begin{table}[H]
\centering
\caption{\label{tab:tbl-final-sentiment-models-robust}Robust mixed effects model output}
\centering
\resizebox{\ifdim\width>\linewidth\linewidth\else\width\fi}{!}{
\begin{tabular}[t]{lrrrrrr}
\toprule
\multicolumn{1}{c}{ } & \multicolumn{3}{c}{Regard} & \multicolumn{3}{c}{SiEBERT} \\
\cmidrule(l{3pt}r{3pt}){2-4} \cmidrule(l{3pt}r{3pt}){5-7}
Coef & Estimate & Std. Error & t & Estimate & Std. Error & t\\
\midrule
\cellcolor{gray!10}{(Intercept)} & \cellcolor{gray!10}{0.27000} & \cellcolor{gray!10}{0.0065} & \cellcolor{gray!10}{43.00} & \cellcolor{gray!10}{0.5900} & \cellcolor{gray!10}{0.0120} & \cellcolor{gray!10}{48.0}\\
Model gemma & 0.02300 & 0.0037 & 6.20 & 0.1400 & 0.0100 & 14.0\\
\cellcolor{gray!10}{Model llama3} & \cellcolor{gray!10}{0.02800} & \cellcolor{gray!10}{0.0035} & \cellcolor{gray!10}{7.90} & \cellcolor{gray!10}{0.0510} & \cellcolor{gray!10}{0.0100} & \cellcolor{gray!10}{5.0}\\
Model t5 & -0.03500 & 0.0029 & -12.00 & 0.1200 & 0.0110 & 11.0\\
\cellcolor{gray!10}{gendermale} & \cellcolor{gray!10}{0.00410} & \cellcolor{gray!10}{0.0020} & \cellcolor{gray!10}{2.00} & \cellcolor{gray!10}{-0.0094} & \cellcolor{gray!10}{0.0039} & \cellcolor{gray!10}{-2.4}\\
\addlinespace
Max tokens 75 & 0.02100 & 0.0018 & 12.00 & -0.0270 & 0.0034 & -7.8\\
\cellcolor{gray!10}{Max tokens 100} & \cellcolor{gray!10}{0.02900} & \cellcolor{gray!10}{0.0018} & \cellcolor{gray!10}{17.00} & \cellcolor{gray!10}{-0.0420} & \cellcolor{gray!10}{0.0034} & \cellcolor{gray!10}{-12.0}\\
Max tokens 150 & 0.03500 & 0.0018 & 20.00 & -0.0510 & 0.0034 & -15.0\\
\cellcolor{gray!10}{Max tokens 300} & \cellcolor{gray!10}{0.04200} & \cellcolor{gray!10}{0.0018} & \cellcolor{gray!10}{24.00} & \cellcolor{gray!10}{-0.0560} & \cellcolor{gray!10}{0.0034} & \cellcolor{gray!10}{-17.0}\\
Max tokens None & 0.04700 & 0.0018 & 27.00 & -0.0790 & 0.0034 & -23.0\\
\addlinespace
\cellcolor{gray!10}{Model gemma : Male} & \cellcolor{gray!10}{-0.01100} & \cellcolor{gray!10}{0.0029} & \cellcolor{gray!10}{-3.70} & \cellcolor{gray!10}{-0.0300} & \cellcolor{gray!10}{0.0055} & \cellcolor{gray!10}{-5.5}\\
Model llama3 : Male & -0.00052 & 0.0029 & -0.18 & 0.0130 & 0.0055 & 2.4\\
\cellcolor{gray!10}{Model t5 : Male} & \cellcolor{gray!10}{0.00083} & \cellcolor{gray!10}{0.0029} & \cellcolor{gray!10}{0.29} & \cellcolor{gray!10}{0.0190} & \cellcolor{gray!10}{0.0055} & \cellcolor{gray!10}{3.5}\\
\bottomrule
\end{tabular}}
\end{table}

\begin{table}[H]
\centering
\caption{\label{tab:tbl-emmeansrobust}Robust mixed effects model: estimated marginal means (female - male)}
\centering
\begin{tabular}[t]{lrlrrrlrr}
\toprule
\multicolumn{1}{c}{ } & \multicolumn{4}{c}{Regard} & \multicolumn{4}{c}{SiEBERT} \\
\cmidrule(l{3pt}r{3pt}){2-5} \cmidrule(l{3pt}r{3pt}){6-9}
Model & Estimate &  & z & p & Estimate &  & z & p\\
\midrule
\cellcolor{gray!10}{bart} & \cellcolor{gray!10}{-0.0041} & \cellcolor{gray!10}{*} & \cellcolor{gray!10}{-2.0} & \cellcolor{gray!10}{0.0450} & \cellcolor{gray!10}{0.0094} & \cellcolor{gray!10}{*} & \cellcolor{gray!10}{2.40} & \cellcolor{gray!10}{0.016}\\
gemma & 0.0065 & ** & 3.2 & 0.0014 & 0.0400 & *** & 10.00 & 0.000\\
\cellcolor{gray!10}{llama3} & \cellcolor{gray!10}{-0.0035} & \cellcolor{gray!10}{.} & \cellcolor{gray!10}{-1.7} & \cellcolor{gray!10}{0.0810} & \cellcolor{gray!10}{-0.0039} & \cellcolor{gray!10}{} & \cellcolor{gray!10}{-0.99} & \cellcolor{gray!10}{0.320}\\
t5 & -0.0049 & * & -2.4 & 0.0160 & -0.0100 & * & -2.60 & 0.010\\
\bottomrule
\end{tabular}
\end{table}

\subsubsection{Generalised Estimating Equations
(GEE)}\label{generalised-estimating-equations-gee}

A Generalised Estimating Equations (GEE) model was also used to estimate
population-averaged effects, adjusting for within-group correlation
using robust sandwich estimators. This was implemented using the
\texttt{geepack} R package \citep{geepack}. The GEE model estimates
population-averaged effects and can be more robust to misspecified
correlation structures. The GEE model was specified as follows:

\begin{align}
\begin{split}
y_{ij} &= \beta_0 + \beta_1 \text{model}_i + \beta_2 \text{gender}_j \\
       &\quad + \beta_3 (\text{model}_i \times \text{gender}_j) + \beta_4 \text{max\_tokens}_i + \epsilon_{ij}
\label{eq:summariesgee}
\end{split}
\end{align}

The correlation structure of the residuals \(\epsilon_{i}\) was modeled
as exchangeable within groups defined by Document ID. No corrections
were applied to the standard errors to reduce the risk of Type 1 error,
as there are 617 document-level clusters, and with 100 or more clusters,
such corrections are generally unnecessary \citep{kahan2016increased}.
The results of the GEE model are set out in Table
\ref{tab:tbl-final-sentiment-models-gee}. The estimated marginal means
for the GEE model are presented in Table \ref{tab:tbl-emmeansgee}.

The point estimates obtained from the mixed-effects and GEE models were
identical, indicating that the fixed effects are robust to the choice of
modelling approach. However, the standard errors differed between the
models. The mixed-effects model, which accounts for random effects,
generally provided smaller standard errors compared to the GEE model.
Attempts to fit a GEE model with an unstructured covariance matrix were
unsuccessful, which may have contributed to the larger standard errors
in the GEE model. As a result, significant differences in sentiment
based on gender were not observed in the BART and T5 models. However,
the Gemma model was not affected by these differences, and summaries
about women remained significantly less negative than those about men.

\begin{table}[H]
\centering
\caption{\label{tab:tbl-final-sentiment-models-gee}GEE model output}
\centering
\resizebox{\ifdim\width>\linewidth\linewidth\else\width\fi}{!}{
\begin{tabular}[t]{lrlrrrrlrrr}
\toprule
\multicolumn{1}{c}{ } & \multicolumn{5}{c}{Regard} & \multicolumn{5}{c}{SiEBERT} \\
\cmidrule(l{3pt}r{3pt}){2-6} \cmidrule(l{3pt}r{3pt}){7-11}
Coef & Estimate &  & Std. Error & Wald & p & Estimate &  & Std. Error & Wald & p\\
\midrule
\cellcolor{gray!10}{(Intercept)} & \cellcolor{gray!10}{0.3000} & \cellcolor{gray!10}{***} & \cellcolor{gray!10}{0.0024} & \cellcolor{gray!10}{1.6e+04} & \cellcolor{gray!10}{0.0000} & \cellcolor{gray!10}{0.5800} & \cellcolor{gray!10}{***} & \cellcolor{gray!10}{0.0065} & \cellcolor{gray!10}{7800.0} & \cellcolor{gray!10}{0.0e+00}\\
Model gemma & 0.0250 & *** & 0.0025 & 1.0e+02 & 0.0000 & 0.1500 & *** & 0.0064 & 530.0 & 0.0e+00\\
\cellcolor{gray!10}{Model llama3} & \cellcolor{gray!10}{0.0290} & \cellcolor{gray!10}{***} & \cellcolor{gray!10}{0.0025} & \cellcolor{gray!10}{1.3e+02} & \cellcolor{gray!10}{0.0000} & \cellcolor{gray!10}{0.0520} & \cellcolor{gray!10}{***} & \cellcolor{gray!10}{0.0067} & \cellcolor{gray!10}{61.0} & \cellcolor{gray!10}{0.0e+00}\\
Model t5 & -0.0330 & *** & 0.0029 & 1.3e+02 & 0.0000 & 0.1000 & *** & 0.0073 & 200.0 & 0.0e+00\\
\cellcolor{gray!10}{gendermale} & \cellcolor{gray!10}{0.0036} & \cellcolor{gray!10}{} & \cellcolor{gray!10}{0.0027} & \cellcolor{gray!10}{1.7e+00} & \cellcolor{gray!10}{0.1900} & \cellcolor{gray!10}{-0.0094} & \cellcolor{gray!10}{} & \cellcolor{gray!10}{0.0072} & \cellcolor{gray!10}{1.7} & \cellcolor{gray!10}{1.9e-01}\\
\addlinespace
Max tokens 150 & 0.0050 & * & 0.0021 & 5.5e+00 & 0.0190 & -0.0500 & *** & 0.0059 & 71.0 & 0.0e+00\\
\cellcolor{gray!10}{Max tokens 300} & \cellcolor{gray!10}{0.0130} & \cellcolor{gray!10}{***} & \cellcolor{gray!10}{0.0021} & \cellcolor{gray!10}{3.6e+01} & \cellcolor{gray!10}{0.0000} & \cellcolor{gray!10}{-0.0540} & \cellcolor{gray!10}{***} & \cellcolor{gray!10}{0.0059} & \cellcolor{gray!10}{85.0} & \cellcolor{gray!10}{0.0e+00}\\
Max tokens 75 & -0.0082 & *** & 0.0023 & 1.3e+01 & 0.0003 & -0.0240 & *** & 0.0061 & 16.0 & 7.8e-05\\
\cellcolor{gray!10}{Max tokens None} & \cellcolor{gray!10}{0.0180} & \cellcolor{gray!10}{***} & \cellcolor{gray!10}{0.0021} & \cellcolor{gray!10}{7.1e+01} & \cellcolor{gray!10}{0.0000} & \cellcolor{gray!10}{-0.0840} & \cellcolor{gray!10}{***} & \cellcolor{gray!10}{0.0060} & \cellcolor{gray!10}{200.0} & \cellcolor{gray!10}{0.0e+00}\\
Model gemma : Male & -0.0110 & ** & 0.0035 & 9.2e+00 & 0.0024 & -0.0330 & *** & 0.0090 & 13.0 & 2.8e-04\\
\addlinespace
\cellcolor{gray!10}{Model llama3 : Male} & \cellcolor{gray!10}{-0.0014} & \cellcolor{gray!10}{} & \cellcolor{gray!10}{0.0036} & \cellcolor{gray!10}{1.6e-01} & \cellcolor{gray!10}{0.6900} & \cellcolor{gray!10}{0.0150} & \cellcolor{gray!10}{} & \cellcolor{gray!10}{0.0095} & \cellcolor{gray!10}{2.5} & \cellcolor{gray!10}{1.2e-01}\\
Model t5 : Male & 0.0013 &  & 0.0041 & 1.1e-01 & 0.7400 & 0.0200 & . & 0.0100 & 3.6 & 5.9e-02\\
\bottomrule
\end{tabular}}
\end{table}

\begin{table}[H]
\centering
\caption{\label{tab:tbl-emmeansgee}GEE model: estimated marginal means (female - male)}
\centering
\begin{tabular}[t]{lrlrrrlrr}
\toprule
\multicolumn{1}{c}{ } & \multicolumn{4}{c}{Regard} & \multicolumn{4}{c}{SiEBERT} \\
\cmidrule(l{3pt}r{3pt}){2-5} \cmidrule(l{3pt}r{3pt}){6-9}
Model & Estimate &  & z & p & Estimate &  & z & p\\
\midrule
\cellcolor{gray!10}{bart} & \cellcolor{gray!10}{-0.0036} & \cellcolor{gray!10}{} & \cellcolor{gray!10}{-1.30} & \cellcolor{gray!10}{0.1900} & \cellcolor{gray!10}{0.0094} & \cellcolor{gray!10}{} & \cellcolor{gray!10}{1.3} & \cellcolor{gray!10}{0.19}\\
gemma & 0.0069 & ** & 3.30 & 0.0011 & 0.0420 & *** & 7.8 & 0.00\\
\cellcolor{gray!10}{llama3} & \cellcolor{gray!10}{-0.0021} & \cellcolor{gray!10}{} & \cellcolor{gray!10}{-0.92} & \cellcolor{gray!10}{0.3600} & \cellcolor{gray!10}{-0.0055} & \cellcolor{gray!10}{} & \cellcolor{gray!10}{-0.9} & \cellcolor{gray!10}{0.37}\\
t5 & -0.0049 &  & -1.60 & 0.1100 & -0.0100 &  & -1.4 & 0.17\\
\bottomrule
\end{tabular}
\end{table}

\subsubsection{Linear models}\label{linear-models}

The mixed model includes an interaction term as well as both random
intercepts and random slopes to account for variability between
documents and within models. This specification is important because it
reflects how document-level differences (random intercepts) and
model-specific variability within documents (random slopes) can impact
sentiment estimates across gender. However, while this specification
makes theoretical sense, the sensitivity of the findings to the model
specification was checked by splitting the data into separate tables for
each combination of model (BART, Gemma, Llama 3, and T5) and metric
(Regard and SiEBERT). A simple linear model was then fitted for each of
these eight datasets. The linear model can be expressed as:

\[
\text{sentiment}_i = \beta_0 + \beta_1 \cdot \text{gender}_i + \beta_2 \cdot \text{max\_tokens}_i + \beta_3 \cdot \text{doc\_id}_i + \epsilon_i
\]

Where \(\beta_0\) is the intercept, \(\beta_1\), \(\beta_2\), and
\(\beta_3\) are the coefficients for gender, maximum tokens, and
document identifier, respectively, and \(\epsilon_i\) is the residual
error term. This model was run separately for each LLM, and the output
for the Regard metric is presented in Table \ref{tab:tbl-lm}, and for
SiEBERT in Table \ref{tab:tbl-lmsiebert}. The model also produced a
coefficient for each Document ID, which is not of interest, so these
were excluded from the tables. Similarly to the GEE model, the point
estimates are close to those from the mixed-effects model, though with
smaller standard errors in this case. The estimated marginal means by
gender for each of the models are presented in Table
\ref{tab:tbl-emmeanslm}, and they are consistent with the findings from
the mixed model.

\begin{table}[H]
\centering
\caption{\label{tab:tbl-lm}Linear model (Regard)}
\centering
\begin{tabular}[t]{lrlrrr}
\toprule
Coef & Estimate &   & Std. Error & t & Pr(>|t|)\\
\midrule
\addlinespace[0.3em]
\multicolumn{6}{l}{\textbf{bart}}\\
\hspace{1em}(Intercept) & 0.2840833 & *** & 0.0155566 & 18.2613205 & 0.0000000\\
\hspace{1em}gendermale & 0.0035545 & ** & 0.0012465 & 2.8515437 & 0.0043639\\
\hspace{1em}max\_tokens150 & 0.0001643 &  & 0.0021590 & 0.0761052 & 0.9393376\\
\hspace{1em}max\_tokens300 & 0.0001634 &  & 0.0021590 & 0.0756762 & 0.9396789\\
\hspace{1em}max\_tokens50 & -0.0307295 & *** & 0.0021590 & -14.2329516 & 0.0000000\\
\hspace{1em}max\_tokens75 & -0.0054155 & * & 0.0021590 & -2.5083062 & 0.0121543\\
\hspace{1em}max\_tokensNone & 0.0001634 &  & 0.0021590 & 0.0756762 & 0.9396789\\
\addlinespace[0.3em]
\multicolumn{6}{l}{\textbf{gemma}}\\
\hspace{1em}(Intercept) & 0.3048890 & *** & 0.0204331 & 14.9213487 & 0.0000000\\
\hspace{1em}gendermale & -0.0069472 & *** & 0.0016373 & -4.2431642 & 0.0000223\\
\hspace{1em}max\_tokens150 & 0.0009879 &  & 0.0028358 & 0.3483562 & 0.7275835\\
\hspace{1em}max\_tokens300 & 0.0141746 & *** & 0.0028358 & 4.9983907 & 0.0000006\\
\hspace{1em}max\_tokens50 & -0.0140059 & *** & 0.0028358 & -4.9388990 & 0.0000008\\
\hspace{1em}max\_tokens75 & -0.0069622 & * & 0.0028358 & -2.4550780 & 0.0141103\\
\hspace{1em}max\_tokensNone & 0.0147827 & *** & 0.0028358 & 5.2128392 & 0.0000002\\
\addlinespace[0.3em]
\multicolumn{6}{l}{\textbf{llama3}}\\
\hspace{1em}(Intercept) & 0.3144663 & *** & 0.0216083 & 14.5530579 & 0.0000000\\
\hspace{1em}gendermale & 0.0021104 &  & 0.0017317 & 1.2187019 & 0.2229998\\
\hspace{1em}max\_tokens150 & 0.0114336 & *** & 0.0029989 & 3.8125689 & 0.0001387\\
\hspace{1em}max\_tokens300 & 0.0167968 & *** & 0.0029989 & 5.6009226 & 0.0000000\\
\hspace{1em}max\_tokens50 & -0.0399157 & *** & 0.0029989 & -13.3099939 & 0.0000000\\
\hspace{1em}max\_tokens75 & -0.0127653 & *** & 0.0029989 & -4.2566164 & 0.0000210\\
\hspace{1em}max\_tokensNone & 0.0185463 & *** & 0.0030004 & 6.1812957 & 0.0000000\\
\addlinespace[0.3em]
\multicolumn{6}{l}{\textbf{t5}}\\
\hspace{1em}(Intercept) & 0.2153391 & *** & 0.0303610 & 7.0926341 & 0.0000000\\
\hspace{1em}gendermale & 0.0048940 & * & 0.0024328 & 2.0117104 & 0.0442900\\
\hspace{1em}max\_tokens150 & 0.0073932 & . & 0.0042137 & 1.7545669 & 0.0793786\\
\hspace{1em}max\_tokens300 & 0.0191323 & *** & 0.0042137 & 4.5405227 & 0.0000057\\
\hspace{1em}max\_tokens50 & -0.0229692 & *** & 0.0042137 & -5.4510934 & 0.0000001\\
\hspace{1em}max\_tokens75 & -0.0076979 & . & 0.0042137 & -1.8268779 & 0.0677621\\
\hspace{1em}max\_tokensNone & 0.0372671 & *** & 0.0042137 & 8.8443118 & 0.0000000\\
\bottomrule
\end{tabular}
\end{table}

\begin{table}[H]
\centering
\caption{\label{tab:tbl-lmsiebert}Linear model (SiEBERT)}
\centering
\begin{tabular}[t]{lrlrrr}
\toprule
Coef & Estimate &   & Std. Error & t & Pr(>|t|)\\
\midrule
\addlinespace[0.3em]
\multicolumn{6}{l}{\textbf{bart}}\\
\hspace{1em}(Intercept) & 0.6601786 & *** & 0.0412309 & 16.0117242 & 0.0000000\\
\hspace{1em}gendermale & -0.0093810 & ** & 0.0033038 & -2.8394946 & 0.0045320\\
\hspace{1em}max\_tokens150 & -0.0010080 &  & 0.0057223 & -0.1761527 & 0.8601792\\
\hspace{1em}max\_tokens300 & -0.0008814 &  & 0.0057223 & -0.1540313 & 0.8775896\\
\hspace{1em}max\_tokens50 & 0.0324356 & *** & 0.0057223 & 5.6682846 & 0.0000000\\
\hspace{1em}max\_tokens75 & 0.0093005 &  & 0.0057223 & 1.6253194 & 0.1041410\\
\hspace{1em}max\_tokensNone & -0.0008814 &  & 0.0057223 & -0.1540313 & 0.8775896\\
\addlinespace[0.3em]
\multicolumn{6}{l}{\textbf{gemma}}\\
\hspace{1em}(Intercept) & 0.7857799 & *** & 0.0484271 & 16.2260289 & 0.0000000\\
\hspace{1em}gendermale & -0.0420346 & *** & 0.0038804 & -10.8325995 & 0.0000000\\
\hspace{1em}max\_tokens150 & -0.0118507 & . & 0.0067210 & -1.7632239 & 0.0779078\\
\hspace{1em}max\_tokens300 & -0.0241479 & *** & 0.0067210 & -3.5928966 & 0.0003293\\
\hspace{1em}max\_tokens50 & 0.0358635 & *** & 0.0067210 & 5.3360130 & 0.0000001\\
\hspace{1em}max\_tokens75 & 0.0115767 & . & 0.0067210 & 1.7224544 & 0.0850328\\
\hspace{1em}max\_tokensNone & -0.0313662 & *** & 0.0067210 & -4.6668826 & 0.0000031\\
\addlinespace[0.3em]
\multicolumn{6}{l}{\textbf{llama3}}\\
\hspace{1em}(Intercept) & 0.4881037 & *** & 0.0503594 & 9.6924036 & 0.0000000\\
\hspace{1em}gendermale & 0.0055138 &  & 0.0040358 & 1.3662261 & 0.1719133\\
\hspace{1em}max\_tokens150 & 0.0129288 & . & 0.0069892 & 1.8498242 & 0.0643824\\
\hspace{1em}max\_tokens300 & 0.0136312 & . & 0.0069892 & 1.9503233 & 0.0511788\\
\hspace{1em}max\_tokens50 & 0.0283793 & *** & 0.0069892 & 4.0604580 & 0.0000495\\
\hspace{1em}max\_tokens75 & 0.0123671 & . & 0.0069892 & 1.7694619 & 0.0768618\\
\hspace{1em}max\_tokensNone & 0.0166275 & * & 0.0069926 & 2.3778786 & 0.0174401\\
\addlinespace[0.3em]
\multicolumn{6}{l}{\textbf{t5}}\\
\hspace{1em}(Intercept) & 0.7611087 & *** & 0.0698796 & 10.8917072 & 0.0000000\\
\hspace{1em}gendermale & 0.0101714 & . & 0.0055993 & 1.8165436 & 0.0693312\\
\hspace{1em}max\_tokens150 & -0.0451223 & *** & 0.0096983 & -4.6525854 & 0.0000033\\
\hspace{1em}max\_tokens300 & -0.0504907 & *** & 0.0096983 & -5.2061143 & 0.0000002\\
\hspace{1em}max\_tokens50 & 0.0582791 & *** & 0.0096983 & 6.0091831 & 0.0000000\\
\hspace{1em}max\_tokens75 & 0.0249979 & ** & 0.0096983 & 2.5775492 & 0.0099713\\
\hspace{1em}max\_tokensNone & -0.1642472 & *** & 0.0096983 & -16.9355987 & 0.0000000\\
\bottomrule
\end{tabular}
\end{table}

\begin{table}[H]
\centering
\caption{\label{tab:tbl-emmeanslm}Linear models: estimated marginal means (female - male)}
\centering
\begin{tabular}[t]{lrlrrrlrr}
\toprule
\multicolumn{1}{c}{ } & \multicolumn{4}{c}{Regard} & \multicolumn{4}{c}{SiEBERT} \\
\cmidrule(l{3pt}r{3pt}){2-5} \cmidrule(l{3pt}r{3pt}){6-9}
Model & Estimate &  & t & p & Estimate &  & t & p\\
\midrule
\cellcolor{gray!10}{bart} & \cellcolor{gray!10}{-0.0036} & \cellcolor{gray!10}{**} & \cellcolor{gray!10}{-2.9} & \cellcolor{gray!10}{4.4e-03} & \cellcolor{gray!10}{0.0094} & \cellcolor{gray!10}{**} & \cellcolor{gray!10}{2.8} & \cellcolor{gray!10}{0.0045}\\
gemma & 0.0069 & *** & 4.2 & 2.2e-05 & 0.0420 & *** & 11.0 & 0.0000\\
\cellcolor{gray!10}{llama3} & \cellcolor{gray!10}{-0.0021} & \cellcolor{gray!10}{} & \cellcolor{gray!10}{-1.2} & \cellcolor{gray!10}{2.2e-01} & \cellcolor{gray!10}{-0.0055} & \cellcolor{gray!10}{} & \cellcolor{gray!10}{-1.4} & \cellcolor{gray!10}{0.1700}\\
t5 & -0.0049 & * & -2.0 & 4.4e-02 & -0.0100 & . & -1.8 & 0.0690\\
\bottomrule
\end{tabular}
\end{table}

\subsubsection{Conclusion of robustness
checks}\label{conclusion-of-robustness-checks}

The robustness checks consistently indicated the reliability of the
findings with regard to Llama 3 and Gemma. Across the linear mixed
model, robust linear mixed models, Generalised Estimating Equations
(GEE), and separate linear models, the point estimates for the fixed
effects remained stable, and the direction of the effects was consistent
for the Gemma model, as was the absence of an effect for Llama 3.
However, the variance-structured mixed effects model and the GEE model
did not find significant effects in the BART and T5 models. Similarly,
the bootstrapped results indicated significant effects slightly less
than half of the time. This suggests that the results for the BART and
T5 models may be on the boundary of significance and should be
interpreted with caution. However, as the older models were primarily
included as benchmarks and are not currently being used in practice to
summarise care records, their bias is of less concern for long-term care
policy. The consistent results across the Llama 3 and Gemma models,
particularly in terms of estimated marginal means, indicate that the
conclusions regarding state-of-the-art models are not sensitive to model
specification or the presence of outliers, validating the robustness of
the model.

\newpage

\subsection{Appendix 3 Evaluation of themes word
lists}\label{appendix-3-evaluation-of-themes-word-lists}

The word lists for each individual theme are included below. These,
along with the complete code, can also be found in the GitHub repository
\citep{egbrepo}. The Python \texttt{str.starts\_with()} method
\citep{pythonstr} was used for these terms. This means that, for
example, in the mental health list, the term \texttt{autis} would match
words that start with these letters, such as \texttt{autism} and
\texttt{autistic}, but not words containing these letters, such as
\texttt{flautist}.

\subsubsection{Mental health}\label{mental-health}

\singlespacing

\begin{verbatim}
alzheimer
anorexia
anxi
asperger
autis
behavio
bipolar
cognit
confus
deliri
delusion
dementia
depress
disorient
hallucinat
insight
mental
memory
mood
paranoi
personality disorder
power of attorney
psycho
ptsd
restlessness
schizoaffect
schizophreni
sectioned
therap
\end{verbatim}

\subsubsection{Physical health}\label{physical-health}

\begin{verbatim}
activities of daily living
amputat
anaemia
angina
arthritis
aspirat
asthma
atrial fibrillation
balance
barrier cream application
bed bound
bed rails
bed-bound
bedbound
bilateral limb
bleeding
blood pressure
blood test
bowel
breath
cancer
care needs
cataract
catheter
cellulitis
chest rash
cholesterol
cirrhosis
commode
community acquired pneumonia
constipat
continen
copd
coronary
diabet
diarrhoea
disability
disable
dysphagia
dysphasia
dyspraxia
epilep
fall
fatigue
fractur
gallstone
glaucoma
gord
gout
hard of hearing
hearing and sight
hearing impair
heart attack
heart condition
heart disease
heart failure
heart problem
hemiplegia
hernia
hip replacement
hoist
house bound
house-bound
housebound
housework
hypercholesterolemia
hypertension
hypothyroid
idiopathic
immobile
incontinen
infarction
infect
influenza
injury
insulin
intravenous
ischaemic
ischemic
kidney
knee
leg clinic
leg ulcer
lung
macular
medication
melanoma
mobili
motor neuron
mrsi
myeloma
nutrition
obstructive sleep
oedema
oesophageal
osteo
pain
paralys
peg feed
personal care
physical deterioration
physical injur
pressure area
pressure relieving
pressure sore
pressure stockings
prostatic
psoriasis
pulmonary
puree
raised toilet
renal
reposition
rollator
sciatica
scoliosis
seizure
sleep apnea
slurred speech
spinal
standing tolerance
stiffness
stoma
stroke
surgery
swallowing
swollen
thickener
transfer
underweight
unsteady
urinary tract
urine retention
uti
vein
visual impairment
washing legs
weak
weight bear
weight loss
wheelchair
zimmer
\end{verbatim}

\subsubsection{Physical appearance}\label{physical-appearance}

\begin{verbatim}
abdomen
appearance
appetite
bath
black eye
bmi
bruised
cloth
dental
dirty
discolouration
dishevelled
disshevelled
dress
drooling
dusty
faeces
fingernails
groom
hair
hygiene
kempt
messy
nails
naked
neglected
nude
odour
rubbish
scruffy
self neglect
self-neglect
shave
skin
slurred
smell
soil
spots
stained
teeth
tidy
tremors
trousers
unclean
underwear
underweight
unhygienic
unkempt
untidy
urin
vest
wear
weigh
\end{verbatim}

\subsubsection{Subjective language}\label{subjective-language}

\begin{verbatim}
abus
adamant
adjusted
adverse
aggress
agitat
agreeable
angry
annoy
appear
appropriate
argumentative
articulate
bad
behav
benefi
best
better
bored
bossy
breach
challeng
chatty
choose
chose
clean
clutter
coherent
concern
confine
conflict
confus
content
damage
demanding
dependent
deteriorat
difficult
dirty
dishevelled
dislike
disparaging
disruptive
distracted
distress
dusty
erratic
escalat
evasive
exacerbat
excessive
failed
feel
felt
fiercely
fixation
fluctuat
forgetful
frustrat
fuss
good
happier
happy
hard
harm
hate
high
ignor
illiterate
immens
impair
improv
impulsiv
inability
inappropriate
incoherent
increase
ineffective
insecure
insight
instrumental
insufficient
intense
invalid
involuntary
irk
irrita
isolat
issue
lack
less
likes
limited
loner
loudly
lovely
low
lucky
marked
massive
maverick
mess
mismanage
misses
misusing
mitigated
mood
more
muddle
needs
negative
neglect
nice
odd
oriented
paranoid
placid
pleasant
pleased
pointless
poor
prais
problem
proper
proud
racist
recommend
refus
relaxed
relentless
reliant
reluctan
resist
respect
restless
risk
rough
rude
sadly
safe
scared
scruffy
serious
settled
severe
shy
significant
silly
slow
small
smartly
smell
sociable
soil
strong
struggl
stupid
substantial
sufficient
suitable
suited
tearful
unable
unacceptable
unamenable
unaware
uncomfortable
uncontrollabl
uncooperative
under weight
underweight
unhygienic
unkempt
unreasonabl
unreliable
unsafe
unsatisfactory
unsettle
untidy
unwise
valid
verbal
vulnerab
wander
well
willing
wise
working
worried
worrying
worse
worst
\end{verbatim}


  \bibliography{references-braces.bib}


\end{document}
