<#
.SYNOPSIS
  Corrected and extended analysis with corrected cohort filtering, separated location and dispersion, notice-level clustering, regime definitions and stratified co-coding.
.DESCRIPTION
  Changes from the first version:
   1. Restricts the analysis to RetractionNature = 'Retraction'. Expressions of concern, corrections,
      reinstatements and blank record types are excluded and reported in a selection flow table.
   2. Runs the Brown-Forsythe dispersion test on EVERY sensitivity subset, so location and dispersion
      claims are never conflated.
   3. Adds cluster-robust inference: a cluster bootstrap resampling whole retraction notices, plus a
      notice-level reanalysis using one record per notice.
   4. Adds four progressively stricter post-LLM definitions and a text-based definition, plus a
      falsification cut-point that precedes the release of the model it nominally marks.
   5. Adds year-stratified and publisher-stratified comparisons.
.EXAMPLE
  .\Analyze-Retractions2.ps1 -Csv ..\data\retractionwatch.csv -OutDir ..\runs\ai-retractions-v2 -Bootstrap 400
#>
[CmdletBinding()]
param(
    [Parameter(Mandatory = $true)][string]$Csv,
    [Parameter(Mandatory = $true)][string]$OutDir,
    [string]$ReasonPattern = 'Computer-Aided Content|Computer-Generated Content',
    [int]$Bootstrap = 400,
    [int]$Seed = 20260731
)
$ErrorActionPreference = 'Stop'
New-Item -ItemType Directory -Force -Path (Join-Path $OutDir 'tables') | Out-Null
$inv = [Globalization.CultureInfo]::InvariantCulture

# ---------------- primitives ----------------
function Get-NormalCdf { param([double]$Z)
    if ($Z -lt -8.5) { return 0.0 }; if ($Z -gt 8.5) { return 1.0 }
    $t = 1.0 / (1.0 + 0.2316419 * [Math]::Abs($Z))
    $d = 0.3989422804014327 * [Math]::Exp(-$Z * $Z / 2.0)
    $p = $d * $t * (0.319381530 + $t * (-0.356563782 + $t * (1.781477937 + $t * (-1.821255978 + $t * 1.330274429))))
    if ($Z -ge 0) { return 1.0 - $p }
    return $p
}
function Get-TwoSidedP { param([double]$Z)
    $az = [Math]::Abs($Z); if ($az -gt 8.2) { return 0.0 }
    return 2.0 * (1.0 - (Get-NormalCdf $az))
}
function Format-P { param([double]$P)
    if ($P -le 0) { return '< 1e-15' }
    if ($P -lt 0.001) { return ('{0:0.0e+0}' -f $P) }
    return ('{0:0.000}' -f $P)
}
function Get-Q { param([double[]]$S, [double]$P)
    if ($S.Length -eq 0) { return 0.0 }
    $i = ($S.Length - 1) * $P
    $lo = [int][Math]::Floor($i); $hi = [int][Math]::Ceiling($i)
    if ($lo -eq $hi) { return $S[$lo] }
    return $S[$lo] + ($i - $lo) * ($S[$hi] - $S[$lo])
}
function Get-MedianFast { param([double[]]$V)
    $a = $V.Clone(); [Array]::Sort($a); return (Get-Q $a 0.5)
}
function Get-MadFast { param([double[]]$V)
    $m = Get-MedianFast $V; $s = 0.0
    foreach ($x in $V) { $s += [Math]::Abs($x - $m) }
    return $s / $V.Length
}
function Get-Desc { param([double[]]$V, [string]$Label)
    $s = $V.Clone(); [Array]::Sort($s)
    $n = $s.Length; if ($n -eq 0) { return $null }
    $sum = 0.0; foreach ($x in $s) { $sum += $x }
    $mean = $sum / $n
    $med = Get-Q $s 0.5; $q1 = Get-Q $s 0.25; $q3 = Get-Q $s 0.75
    $ss = 0.0; foreach ($x in $s) { $ss += ($x - $mean) * ($x - $mean) }
    $sd = 0.0; if ($n -gt 1) { $sd = [Math]::Sqrt($ss / ($n - 1)) }
    $half = 1.959963985 * [Math]::Sqrt([double]$n) / 2.0
    $kLo = [int][Math]::Max(0, [Math]::Floor($n / 2.0 - $half) - 1)
    $kHi = [int][Math]::Min($n - 1, [Math]::Ceiling($n / 2.0 + $half) - 1)
    $mad = 0.0; foreach ($x in $s) { $mad += [Math]::Abs($x - $med) }
    return [pscustomobject]@{
        Cohort = $Label; N = $n
        Median = [Math]::Round($med, 1); MedCiLo = [Math]::Round($s[$kLo], 1); MedCiHi = [Math]::Round($s[$kHi], 1)
        Q1 = [Math]::Round($q1, 1); Q3 = [Math]::Round($q3, 1); IQR = [Math]::Round($q3 - $q1, 1)
        Mean = [Math]::Round($mean, 1); SD = [Math]::Round($sd, 1)
        MeanAbsDev = [Math]::Round($mad / $n, 1)
        Max = [Math]::Round($s[$n - 1], 1)
    }
}
function Test-MW { param([double[]]$A, [double[]]$B)
    $n1 = $A.Length; $n2 = $B.Length
    $N = $n1 + $n2
    $v = New-Object 'double[]' $N
    $g = New-Object 'int[]' $N
    [Array]::Copy($A, 0, $v, 0, $n1); [Array]::Copy($B, 0, $v, $n1, $n2)
    for ($i = 0; $i -lt $n1; $i++) { $g[$i] = 1 }
    for ($i = $n1; $i -lt $N; $i++) { $g[$i] = 2 }
    [Array]::Sort($v, $g)
    $r1 = 0.0; $tie = 0.0; $i = 0
    while ($i -lt $N) {
        $j = $i
        while (($j + 1) -lt $N -and $v[$j + 1] -eq $v[$i]) { $j++ }
        $t = $j - $i + 1
        $avg = ($i + 1 + $j + 1) / 2.0
        if ($t -gt 1) { $tie += ([double]$t * $t * $t - $t) }
        for ($k = $i; $k -le $j; $k++) { if ($g[$k] -eq 1) { $r1 += $avg } }
        $i = $j + 1
    }
    $u1 = $r1 - ($n1 * ($n1 + 1.0) / 2.0)
    $mu = $n1 * $n2 / 2.0
    $sd = [Math]::Sqrt(($n1 * $n2 / 12.0) * (($N + 1.0) - $tie / ([double]$N * ($N - 1.0))))
    $z = ($u1 - $mu) / $sd
    $p = Get-TwoSidedP $z
    $delta = 2.0 * $u1 / ($n1 * $n2) - 1.0
    $se = 2.0 * $sd / ($n1 * $n2)
    return [pscustomobject]@{ U = [Math]::Round($u1, 0); Z = [Math]::Round($z, 3); P = $p; Pstr = Format-P $p
        Delta = [Math]::Round($delta, 4); DeltaLo = [Math]::Round($delta - 1.959963985 * $se, 4); DeltaHi = [Math]::Round($delta + 1.959963985 * $se, 4) }
}
function Test-BF { param([double[]]$A, [double[]]$B)
    $ma = Get-MedianFast $A; $mb = Get-MedianFast $B
    $n1 = $A.Length; $n2 = $B.Length
    $s1 = 0.0; foreach ($x in $A) { $s1 += [Math]::Abs($x - $ma) }; $m1 = $s1 / $n1
    $s2 = 0.0; foreach ($x in $B) { $s2 += [Math]::Abs($x - $mb) }; $m2 = $s2 / $n2
    $v1 = 0.0; foreach ($x in $A) { $d = [Math]::Abs($x - $ma) - $m1; $v1 += $d * $d }; $v1 = $v1 / ($n1 - 1)
    $v2 = 0.0; foreach ($x in $B) { $d = [Math]::Abs($x - $mb) - $m2; $v2 += $d * $d }; $v2 = $v2 / ($n2 - 1)
    $se = [Math]::Sqrt($v1 / $n1 + $v2 / $n2)
    $t = ($m1 - $m2) / $se
    $dfN = [Math]::Pow($v1 / $n1 + $v2 / $n2, 2)
    $dfD = [Math]::Pow($v1 / $n1, 2) / ($n1 - 1) + [Math]::Pow($v2 / $n2, 2) / ($n2 - 1)
    $p = Get-TwoSidedP $t
    return [pscustomobject]@{ Mad1 = [Math]::Round($m1, 1); Mad2 = [Math]::Round($m2, 1)
        Diff = [Math]::Round($m1 - $m2, 1); DiffLo = [Math]::Round(($m1 - $m2) - 1.959963985 * $se, 1); DiffHi = [Math]::Round(($m1 - $m2) + 1.959963985 * $se, 1)
        T = [Math]::Round($t, 3); DF = [Math]::Round($dfN / $dfD, 0); P = $p; Pstr = Format-P $p
        Ratio = [Math]::Round($m1 / $m2, 3) }
}
function Get-Wilson { param([int]$K, [int]$N)
    if ($N -eq 0) { return [pscustomobject]@{ P = 0; Lo = 0; Hi = 0 } }
    $z = 1.959963985; $p = [double]$K / $N; $den = 1.0 + $z * $z / $N
    $c = ($p + $z * $z / (2.0 * $N)) / $den
    $h = ($z * [Math]::Sqrt($p * (1 - $p) / $N + $z * $z / (4.0 * $N * $N))) / $den
    return [pscustomobject]@{ P = [Math]::Round(100 * $p, 2); Lo = [Math]::Round(100 * ($c - $h), 2); Hi = [Math]::Round(100 * ($c + $h), 2) }
}
function Test-Prop { param([int]$K1, [int]$N1, [int]$K2, [int]$N2, [string]$Label)
    $p1 = [double]$K1 / $N1; $p2 = [double]$K2 / $N2
    $pp = [double]($K1 + $K2) / ($N1 + $N2)
    $z = ($p1 - $p2) / [Math]::Sqrt($pp * (1 - $pp) * (1.0 / $N1 + 1.0 / $N2))
    $seD = [Math]::Sqrt($p1 * (1 - $p1) / $N1 + $p2 * (1 - $p2) / $N2)
    $w1 = Get-Wilson $K1 $N1; $w2 = Get-Wilson $K2 $N2
    $p = Get-TwoSidedP $z
    $rr = 0; if ($p2 -gt 0) { $rr = [Math]::Round($p1 / $p2, 2) }
    return [pscustomobject]@{ Measure = $Label; K1 = $K1; N1 = $N1; Pct1 = $w1.P; Lo1 = $w1.Lo; Hi1 = $w1.Hi
        K2 = $K2; N2 = $N2; Pct2 = $w2.P; Lo2 = $w2.Lo; Hi2 = $w2.Hi
        DiffPP = [Math]::Round(100 * ($p1 - $p2), 2); DiffLo = [Math]::Round(100 * (($p1 - $p2) - 1.959963985 * $seD), 2)
        DiffHi = [Math]::Round(100 * (($p1 - $p2) + 1.959963985 * $seD), 2); RR = $rr; Z = [Math]::Round($z, 2); P = $p; Pstr = Format-P $p }
}
function ConvertTo-Date { param([string]$S)
    if ([string]::IsNullOrWhiteSpace($S)) { return $null }
    try { return [datetime]::Parse($S, $inv) } catch { return $null }
}

# ---------------- data and selection flow ----------------
Write-Host "Loading $Csv ..." -ForegroundColor Cyan
$raw = @(Import-Csv $Csv)
$nSnapshot = $raw.Count
$nature = @{}
foreach ($r in $raw) { $k = $r.RetractionNature; if (-not $k) { $k = '(blank)' }; if (-not $nature.ContainsKey($k)) { $nature[$k] = 0 }; $nature[$k]++ }
$ret = @($raw | Where-Object { $_.RetractionNature -eq 'Retraction' })
$nRetraction = $ret.Count

foreach ($r in $ret) {
    $od = ConvertTo-Date $r.OriginalPaperDate
    $rd = ConvertTo-Date $r.RetractionDate
    $lag = $null
    if ($od -and $rd -and $rd -ge $od) { $lag = ($rd - $od).TotalDays }
    $ry = $null; if ($rd) { $ry = $rd.Year }
    $notice = $r.RetractionDOI
    if (-not $notice -or $notice -match '^(unavailable|Unavailable)$') { $notice = 'NA-' + $r.'Record ID' }
    Add-Member -InputObject $r -NotePropertyName OrigDate -NotePropertyValue $od -Force
    Add-Member -InputObject $r -NotePropertyName RetYear -NotePropertyValue $ry -Force
    Add-Member -InputObject $r -NotePropertyName LagDays -NotePropertyValue $lag -Force
    Add-Member -InputObject $r -NotePropertyName Notice -NotePropertyValue $notice -Force
    Add-Member -InputObject $r -NotePropertyName IsMachine -NotePropertyValue ($r.Reason -match $ReasonPattern) -Force
    Add-Member -InputObject $r -NotePropertyName IsMill -NotePropertyValue ($r.Reason -match 'Paper Mill') -Force
    Add-Member -InputObject $r -NotePropertyName IsPeer -NotePropertyValue ($r.Reason -match 'Peer Review') -Force
    Add-Member -InputObject $r -NotePropertyName IsPlag -NotePropertyValue ($r.Reason -match 'Plagiarism') -Force
    Add-Member -InputObject $r -NotePropertyName IsDup -NotePropertyValue ($r.Reason -match 'Duplication') -Force
    Add-Member -InputObject $r -NotePropertyName LlmText -NotePropertyValue ((($r.Title + ' ' + $r.Notes) -match '(?i)chatgpt|large language model|\bLLM\b|generative a\.?i|\bGPT-?[0-9]?\b')) -Force
}
$ai = @($ret | Where-Object { $_.IsMachine })
$other = @($ret | Where-Object { -not $_.IsMachine })
$aiLagOk = @($ai | Where-Object { $null -ne $_.LagDays })
$otLagOk = @($other | Where-Object { $null -ne $_.LagDays })
Write-Host ("  snapshot {0} | retractions {1} | machine {2} | comparison {3}" -f $nSnapshot, $nRetraction, $ai.Count, $other.Count)

$flow = @(
    [pscustomobject]@{ Step = '1 Complete database snapshot (all record types)'; Records = $nSnapshot }
    [pscustomobject]@{ Step = '2 Excluded: expressions of concern'; Records = -1 * $(if ($nature.ContainsKey('Expression of concern')) { $nature['Expression of concern'] } else { 0 }) }
    [pscustomobject]@{ Step = '3 Excluded: corrections'; Records = -1 * $(if ($nature.ContainsKey('Correction')) { $nature['Correction'] } else { 0 }) }
    [pscustomobject]@{ Step = '4 Excluded: reinstatements'; Records = -1 * $(if ($nature.ContainsKey('Reinstatement')) { $nature['Reinstatement'] } else { 0 }) }
    [pscustomobject]@{ Step = '5 Excluded: blank record type'; Records = -1 * $(if ($nature.ContainsKey('(blank)')) { $nature['(blank)'] } else { 0 }) }
    [pscustomobject]@{ Step = '6 Analysed: RetractionNature = Retraction'; Records = $nRetraction }
    [pscustomobject]@{ Step = '7 Machine-attributed cohort'; Records = $ai.Count }
    [pscustomobject]@{ Step = '8 Comparison cohort (all other reasons)'; Records = $other.Count }
    [pscustomobject]@{ Step = '9 Machine cohort with usable date pair'; Records = $aiLagOk.Count }
    [pscustomobject]@{ Step = '10 Comparison cohort with usable date pair'; Records = $otLagOk.Count }
)
$flow | Export-Csv (Join-Path $OutDir 'tables\t1_selection_flow.csv') -NoTypeInformation -Encoding UTF8

function Get-LagArray { param($Set) 
    $l = New-Object 'System.Collections.Generic.List[double]'
    foreach ($r in $Set) { if ($null -ne $r.LagDays) { $l.Add([double]$r.LagDays) } }
    return $l.ToArray()
}
$lagAi = Get-LagArray $ai
$lagOt = Get-LagArray $other

# ---------------- sensitivity subsets, each with BOTH tests ----------------
$topPub = (@($ai | Where-Object { $_.Publisher } | Group-Object Publisher | Sort-Object Count -Descending)[0]).Name
$noticeGroups = @($ai | Group-Object Notice)
$bigNotices = @(($noticeGroups | Sort-Object Count -Descending | Select-Object -First 10) | ForEach-Object { $_.Name })
$noticeLevel = @($noticeGroups | ForEach-Object { $_.Group[0] })

$subsets = @(
    @{ Label = 'Full machine cohort'; Set = $ai }
    @{ Label = "Excluding $topPub"; Set = @($ai | Where-Object { $_.Publisher -ne $topPub }) }
    @{ Label = 'Excluding 10 largest notices'; Set = @($ai | Where-Object { $bigNotices -notcontains $_.Notice }) }
    @{ Label = 'Notice level (one record per notice)'; Set = $noticeLevel }
)
$sens = New-Object System.Collections.ArrayList
foreach ($s in $subsets) {
    $lg = Get-LagArray $s.Set
    if ($lg.Length -lt 30) { continue }
    $d = Get-Desc $lg $s.Label
    $mw = Test-MW -A $lg -B $lagOt
    $bf = Test-BF -A $lg -B $lagOt
    $mill = @($s.Set | Where-Object { $_.IsMill }).Count
    $w = Get-Wilson $mill $s.Set.Count
    [void]$sens.Add([pscustomobject]@{
            Analysis = $s.Label; N = $s.Set.Count; NLag = $d.N
            Median = $d.Median; IQR = $d.IQR; MeanAbsDev = $d.MeanAbsDev; SD = $d.SD
            LocDelta = $mw.Delta; LocZ = $mw.Z; LocP = $mw.Pstr
            DispMadCohort = $bf.Mad1; DispMadComparison = $bf.Mad2; DispRatio = $bf.Ratio; DispT = $bf.T; DispP = $bf.Pstr
            PaperMillPct = $w.P; PaperMillLo = $w.Lo; PaperMillHi = $w.Hi
        })
    Write-Host ("  {0,-38} n={1,-6} med={2,-7} MAD={3,-7} dispP={4}" -f $s.Label, $s.Set.Count, $d.Median, $d.MeanAbsDev, $bf.Pstr)
}
$sens | Export-Csv (Join-Path $OutDir 'tables\t5_sensitivity_location_and_dispersion.csv') -NoTypeInformation -Encoding UTF8

# ---------------- cluster bootstrap by retraction notice ----------------
Write-Host "Cluster bootstrap by retraction notice (B=$Bootstrap)..." -ForegroundColor Cyan
function Build-Clusters { param($Set)
    $h = @{}
    foreach ($r in $Set) {
        if ($null -eq $r.LagDays) { continue }
        $k = $r.Notice
        if (-not $h.ContainsKey($k)) { $h[$k] = New-Object 'System.Collections.Generic.List[double]' }
        $h[$k].Add([double]$r.LagDays)
    }
    $arr = New-Object 'System.Collections.Generic.List[double[]]'
    foreach ($k in $h.Keys) { $arr.Add($h[$k].ToArray()) }
    return $arr
}
function Build-MillClusters { param($Set)
    $h = @{}
    foreach ($r in $Set) {
        $k = $r.Notice
        if (-not $h.ContainsKey($k)) { $h[$k] = New-Object 'System.Collections.Generic.List[int]' }
        $v = 0; if ($r.IsMill) { $v = 1 }
        $h[$k].Add($v)
    }
    $arr = New-Object 'System.Collections.Generic.List[int[]]'
    foreach ($k in $h.Keys) { $arr.Add($h[$k].ToArray()) }
    return $arr
}
$clA = Build-Clusters $ai
$clB = Build-Clusters $other
$mlA = Build-MillClusters $ai
$mlB = Build-MillClusters $other
$rand = New-Object System.Random $Seed
function Resample-Lag { param($Clusters)
    $buf = New-Object 'System.Collections.Generic.List[double]'
    $n = $Clusters.Count
    for ($i = 0; $i -lt $n; $i++) { $buf.AddRange($Clusters[$rand.Next(0, $n)]) }
    return $buf.ToArray()
}
function Resample-Mill { param($Clusters)
    $n = $Clusters.Count; $k = 0; $tot = 0
    for ($i = 0; $i -lt $n; $i++) { foreach ($v in $Clusters[$rand.Next(0, $n)]) { $tot++; $k += $v } }
    return @($k, $tot)
}
$bMedDiff = New-Object 'System.Collections.Generic.List[double]'
$bMadDiff = New-Object 'System.Collections.Generic.List[double]'
$bMillDiff = New-Object 'System.Collections.Generic.List[double]'
for ($b = 1; $b -le $Bootstrap; $b++) {
    $ra = Resample-Lag $clA; $rb = Resample-Lag $clB
    $bMedDiff.Add((Get-MedianFast $ra) - (Get-MedianFast $rb))
    $bMadDiff.Add((Get-MadFast $ra) - (Get-MadFast $rb))
    $ma = Resample-Mill $mlA; $mb = Resample-Mill $mlB
    $bMillDiff.Add(100.0 * ($ma[0] / [double]$ma[1] - $mb[0] / [double]$mb[1]))
    if ($b % 100 -eq 0) { Write-Host ("    replicate {0}/{1}" -f $b, $Bootstrap) }
}
function Get-BootCi { param($List, [string]$Label, [double]$Point)
    $a = $List.ToArray(); [Array]::Sort($a)
    return [pscustomobject]@{ Estimate = $Label; PointEstimate = [Math]::Round($Point, 2)
        ClusterCiLo = [Math]::Round((Get-Q $a 0.025), 2); ClusterCiHi = [Math]::Round((Get-Q $a 0.975), 2)
        Replicates = $a.Length }
}
$pointMed = (Get-MedianFast $lagAi) - (Get-MedianFast $lagOt)
$pointMad = (Get-MadFast $lagAi) - (Get-MadFast $lagOt)
$millA = @($ai | Where-Object { $_.IsMill }).Count; $millB = @($other | Where-Object { $_.IsMill }).Count
$pointMill = 100.0 * ($millA / [double]$ai.Count - $millB / [double]$other.Count)
$boot = @(
    (Get-BootCi $bMedDiff  'Median lag difference (days)' $pointMed)
    (Get-BootCi $bMadDiff  'Mean-absolute-deviation difference (days)' $pointMad)
    (Get-BootCi $bMillDiff 'Paper-mill flag difference (percentage points)' $pointMill)
)
$boot | Export-Csv (Join-Path $OutDir 'tables\t6_cluster_bootstrap.csv') -NoTypeInformation -Encoding UTF8

# ---------------- integrity flag co-occurrence ----------------
$props = @(
    (Test-Prop -K1 $millA -N1 $ai.Count -K2 $millB -N2 $other.Count -Label 'Paper mill'),
    (Test-Prop -K1 @($ai | Where-Object { $_.IsPeer }).Count -N1 $ai.Count -K2 @($other | Where-Object { $_.IsPeer }).Count -N2 $other.Count -Label 'Any peer-review concern'),
    (Test-Prop -K1 @($ai | Where-Object { $_.IsPlag }).Count -N1 $ai.Count -K2 @($other | Where-Object { $_.IsPlag }).Count -N2 $other.Count -Label 'Plagiarism'),
    (Test-Prop -K1 @($ai | Where-Object { $_.IsDup }).Count -N1 $ai.Count -K2 @($other | Where-Object { $_.IsDup }).Count -N2 $other.Count -Label 'Duplication')
)
$props | Export-Csv (Join-Path $OutDir 'tables\t4_flag_cooccurrence.csv') -NoTypeInformation -Encoding UTF8

# ---------------- year-stratified and publisher-stratified paper-mill comparison ----------------
$yearStrat = New-Object System.Collections.ArrayList
foreach ($y in @(2021, 2022, 2023, 2024, 2025)) {
    $a = @($ai | Where-Object { $_.RetYear -eq $y }); $o = @($other | Where-Object { $_.RetYear -eq $y })
    if ($a.Count -lt 20 -or $o.Count -lt 20) { continue }
    $t = Test-Prop -K1 @($a | Where-Object { $_.IsMill }).Count -N1 $a.Count -K2 @($o | Where-Object { $_.IsMill }).Count -N2 $o.Count -Label "Retraction year $y"
    [void]$yearStrat.Add($t)
}
$yearStrat | Export-Csv (Join-Path $OutDir 'tables\t7_year_stratified_papermill.csv') -NoTypeInformation -Encoding UTF8

$pubStrat = New-Object System.Collections.ArrayList
foreach ($pg in (@($ai | Where-Object { $_.Publisher } | Group-Object Publisher | Sort-Object Count -Descending | Select-Object -First 6))) {
    $p = $pg.Name
    $a = @($ai | Where-Object { $_.Publisher -eq $p }); $o = @($other | Where-Object { $_.Publisher -eq $p })
    if ($a.Count -lt 20 -or $o.Count -lt 20) {
        [void]$pubStrat.Add([pscustomobject]@{ Measure = $p; K1 = @($a | Where-Object { $_.IsMill }).Count; N1 = $a.Count; Pct1 = (Get-Wilson @($a | Where-Object { $_.IsMill }).Count $a.Count).P; K2 = 'n/a'; N2 = $o.Count; Pct2 = 'n/a'; DiffPP = 'n/a'; RR = 'n/a'; Pstr = 'too few comparison records' })
        continue
    }
    $t = Test-Prop -K1 @($a | Where-Object { $_.IsMill }).Count -N1 $a.Count -K2 @($o | Where-Object { $_.IsMill }).Count -N2 $o.Count -Label $p
    [void]$pubStrat.Add($t)
}
$pubStrat | Export-Csv (Join-Path $OutDir 'tables\t8_publisher_stratified_papermill.csv') -NoTypeInformation -Encoding UTF8

# ---------------- regime definitions ----------------
$cuts = @(
    @{ Label = '2022-06-01 (falsification: precedes public LLM release)'; Date = '2022-06-01' }
    @{ Label = '2022-11-30 (ChatGPT public release)'; Date = '2022-11-30' }
    @{ Label = '2023-07-01 (stricter)'; Date = '2023-07-01' }
    @{ Label = '2024-01-01 (stricter)'; Date = '2024-01-01' }
    @{ Label = '2024-07-01 (strictest)'; Date = '2024-07-01' }
)
$regime = New-Object System.Collections.ArrayList
foreach ($c in $cuts) {
    $d = [datetime]::Parse($c.Date, $inv)
    $pre = @($ai | Where-Object { $_.OrigDate -and $_.OrigDate -lt $d })
    $post = @($ai | Where-Object { $_.OrigDate -and $_.OrigDate -ge $d })
    if ($post.Count -lt 10) { continue }
    $mPre = Get-MedianFast (Get-LagArray $pre); $mPost = Get-MedianFast (Get-LagArray $post)
    $wPre = Get-Wilson @($pre | Where-Object { $_.IsMill }).Count $pre.Count
    $wPost = Get-Wilson @($post | Where-Object { $_.IsMill }).Count $post.Count
    $t = Test-Prop -K1 @($post | Where-Object { $_.IsMill }).Count -N1 $post.Count -K2 @($pre | Where-Object { $_.IsMill }).Count -N2 $pre.Count -Label $c.Label
    [void]$regime.Add([pscustomobject]@{
            Definition = $c.Label; NPre = $pre.Count; NPost = $post.Count
            MedianLagPre = [Math]::Round($mPre, 1); MedianLagPost = [Math]::Round($mPost, 1)
            MillPctPre = $wPre.P; MillPreLo = $wPre.Lo; MillPreHi = $wPre.Hi
            MillPctPost = $wPost.P; MillPostLo = $wPost.Lo; MillPostHi = $wPost.Hi
            DiffPP = $t.DiffPP; DiffLo = $t.DiffLo; DiffHi = $t.DiffHi; P = $t.Pstr
        })
}
# text-based definition
$llmAll = @($ret | Where-Object { $_.LlmText })
$llmAi = @($ai | Where-Object { $_.LlmText })
$textDef = [pscustomobject]@{
    RetractionsMentioningLlmInTitleOrNotes = $llmAll.Count
    OfWhichMachineAttributed               = $llmAi.Count
    ShareOfMachineCohort                   = [Math]::Round(100 * $llmAi.Count / $ai.Count, 3)
    MillPctAmongLlmMentioning              = $(if ($llmAi.Count -gt 0) { (Get-Wilson @($llmAi | Where-Object { $_.IsMill }).Count $llmAi.Count).P } else { 'n/a' })
}
$regime | Export-Csv (Join-Path $OutDir 'tables\t9_regime_definitions.csv') -NoTypeInformation -Encoding UTF8
$textDef | Export-Csv (Join-Path $OutDir 'tables\t10_text_based_llm_definition.csv') -NoTypeInformation -Encoding UTF8

# ---------------- descriptives, concentration, yearly share ----------------
$dAi = Get-Desc $lagAi 'Machine-attributed'
$dOt = Get-Desc $lagOt 'All other retraction reasons'
@($dAi, $dOt) | Export-Csv (Join-Path $OutDir 'tables\t3_lag_descriptives.csv') -NoTypeInformation -Encoding UTF8
$mwMain = Test-MW -A $lagAi -B $lagOt
$bfMain = Test-BF -A $lagAi -B $lagOt
$mwMain | Export-Csv (Join-Path $OutDir 'tables\t3b_mannwhitney.csv') -NoTypeInformation -Encoding UTF8
$bfMain | Export-Csv (Join-Path $OutDir 'tables\t3c_brownforsythe.csv') -NoTypeInformation -Encoding UTF8

$byYear = New-Object System.Collections.ArrayList
foreach ($y in (@($ret | Where-Object { $_.RetYear -and [int]$_.RetYear -ge 2010 } | Group-Object RetYear | ForEach-Object { [int]$_.Name } | Sort-Object))) {
    $tot = @($ret | Where-Object { $_.RetYear -eq $y }).Count
    $k = @($ai | Where-Object { $_.RetYear -eq $y }).Count
    $w = Get-Wilson $k $tot
    [void]$byYear.Add([pscustomobject]@{ Year = $y; MachineAttributed = $k; AllRetractions = $tot; SharePct = $w.P; CiLo = $w.Lo; CiHi = $w.Hi })
}
$byYear | Export-Csv (Join-Path $OutDir 'tables\t2_by_year.csv') -NoTypeInformation -Encoding UTF8

foreach ($spec in @(@{F = 'Publisher'; O = 't11_publishers.csv' }, @{F = 'Journal'; O = 't12_journals.csv' }, @{F = 'Country'; O = 't13_countries.csv' })) {
    @($ai | Where-Object { $_.($spec.F) } | Group-Object $spec.F | Sort-Object Count -Descending | Select-Object -First 15 |
        ForEach-Object { [pscustomobject]@{ Value = $_.Name; N = $_.Count; Pct = [Math]::Round(100 * $_.Count / $ai.Count, 2) } }) |
        Export-Csv (Join-Path $OutDir "tables\$($spec.O)") -NoTypeInformation -Encoding UTF8
}
$noticeStats = [pscustomobject]@{
    Records = $ai.Count
    DistinctNotices = $noticeGroups.Count
    RecordsInMultiArticleNotices = (($noticeGroups | Where-Object { $_.Count -gt 1 } | Measure-Object Count -Sum).Sum)
    LargestNotice = (($noticeGroups | Sort-Object Count -Descending)[0].Count)
    Top10NoticesCover = (($noticeGroups | Sort-Object Count -Descending | Select-Object -First 10 | Measure-Object Count -Sum).Sum)
}
$noticeStats | Export-Csv (Join-Path $OutDir 'tables\t14_notice_structure.csv') -NoTypeInformation -Encoding UTF8

# ---------------- report ----------------
$sb = New-Object System.Text.StringBuilder
[void]$sb.AppendLine("# Corrected analysis (v2)")
[void]$sb.AppendLine("")
[void]$sb.AppendLine("Generated $(Get-Date -Format 'yyyy-MM-dd HH:mm') by tools\Analyze-Retractions2.ps1. Bootstrap replicates: $Bootstrap, seed $Seed.")
[void]$sb.AppendLine("")
[void]$sb.AppendLine("## Record selection")
[void]$sb.AppendLine("| Step | Records |"); [void]$sb.AppendLine("|---|---|")
foreach ($f in $flow) { [void]$sb.AppendLine("| $($f.Step) | $($f.Records) |") }
[void]$sb.AppendLine("")
[void]$sb.AppendLine("Record types in the snapshot: " + (($nature.GetEnumerator() | Sort-Object Value -Descending | ForEach-Object { "$($_.Key) = $($_.Value)" }) -join '; '))
[void]$sb.AppendLine("")
[void]$sb.AppendLine("## Notice structure (clustering)")
[void]$sb.AppendLine("Records $($noticeStats.Records); distinct notices $($noticeStats.DistinctNotices); records in multi-article notices $($noticeStats.RecordsInMultiArticleNotices); largest notice $($noticeStats.LargestNotice); ten largest notices cover $($noticeStats.Top10NoticesCover).")
[void]$sb.AppendLine("")
[void]$sb.AppendLine("## Time to retraction")
[void]$sb.AppendLine("| Cohort | n | Median (95% CI) | Q1 | Q3 | IQR | Mean | SD | Mean abs dev | Max |")
[void]$sb.AppendLine("|---|---|---|---|---|---|---|---|---|---|")
foreach ($d in @($dAi, $dOt)) { [void]$sb.AppendLine("| $($d.Cohort) | $($d.N) | $($d.Median) ($($d.MedCiLo)-$($d.MedCiHi)) | $($d.Q1) | $($d.Q3) | $($d.IQR) | $($d.Mean) | $($d.SD) | $($d.MeanAbsDev) | $($d.Max) |") }
[void]$sb.AppendLine("")
[void]$sb.AppendLine("Location: U = $($mwMain.U), z = $($mwMain.Z), p $($mwMain.Pstr); Cliff's delta $($mwMain.Delta) (95% CI $($mwMain.DeltaLo) to $($mwMain.DeltaHi)).")
[void]$sb.AppendLine("Dispersion: mean absolute deviation $($bfMain.Mad1) vs $($bfMain.Mad2) d, ratio $($bfMain.Ratio); difference $($bfMain.Diff) (95% CI $($bfMain.DiffLo) to $($bfMain.DiffHi)); t = $($bfMain.T), df = $($bfMain.DF), p $($bfMain.Pstr).")
[void]$sb.AppendLine("")
[void]$sb.AppendLine("## Sensitivity: location and dispersion reported separately")
[void]$sb.AppendLine("| Analysis | n | Median | IQR | Mean abs dev | Location delta | Location p | Cohort MAD | Comparison MAD | MAD ratio | Dispersion p | Paper mill % (95% CI) |")
[void]$sb.AppendLine("|---|---|---|---|---|---|---|---|---|---|---|---|")
foreach ($s in $sens) {
    [void]$sb.AppendLine("| $($s.Analysis) | $($s.N) | $($s.Median) | $($s.IQR) | $($s.MeanAbsDev) | $($s.LocDelta) | $($s.LocP) | $($s.DispMadCohort) | $($s.DispMadComparison) | $($s.DispRatio) | $($s.DispP) | $($s.PaperMillPct) ($($s.PaperMillLo)-$($s.PaperMillHi)) |")
}
[void]$sb.AppendLine("")
[void]$sb.AppendLine("## Cluster bootstrap by retraction notice")
[void]$sb.AppendLine("| Estimate | Point | Cluster-robust 95% CI | Replicates |")
[void]$sb.AppendLine("|---|---|---|---|")
foreach ($b in $boot) { [void]$sb.AppendLine("| $($b.Estimate) | $($b.PointEstimate) | $($b.ClusterCiLo) to $($b.ClusterCiHi) | $($b.Replicates) |") }
[void]$sb.AppendLine("")
[void]$sb.AppendLine("## Integrity flag co-occurrence")
[void]$sb.AppendLine("| Flag | Machine % (95% CI) | Comparison % (95% CI) | Diff pp (95% CI) | RR | z | p |")
[void]$sb.AppendLine("|---|---|---|---|---|---|---|")
foreach ($p in $props) { [void]$sb.AppendLine("| $($p.Measure) | $($p.Pct1) ($($p.Lo1)-$($p.Hi1)) | $($p.Pct2) ($($p.Lo2)-$($p.Hi2)) | $($p.DiffPP) ($($p.DiffLo) to $($p.DiffHi)) | $($p.RR) | $($p.Z) | $($p.Pstr) |") }
[void]$sb.AppendLine("")
[void]$sb.AppendLine("## Year-stratified paper-mill comparison")
[void]$sb.AppendLine("| Stratum | Machine % | Comparison % | Diff pp (95% CI) | RR | p |")
[void]$sb.AppendLine("|---|---|---|---|---|---|")
foreach ($t in $yearStrat) { [void]$sb.AppendLine("| $($t.Measure) | $($t.Pct1) | $($t.Pct2) | $($t.DiffPP) ($($t.DiffLo) to $($t.DiffHi)) | $($t.RR) | $($t.Pstr) |") }
[void]$sb.AppendLine("")
[void]$sb.AppendLine("## Publisher-stratified paper-mill comparison")
[void]$sb.AppendLine("| Publisher | Machine % (n) | Comparison % (n) | Diff pp | RR | p |")
[void]$sb.AppendLine("|---|---|---|---|---|---|")
foreach ($t in $pubStrat) { [void]$sb.AppendLine("| $($t.Measure) | $($t.Pct1) ($($t.N1)) | $($t.Pct2) ($($t.N2)) | $($t.DiffPP) | $($t.RR) | $($t.Pstr) |") }
[void]$sb.AppendLine("")
[void]$sb.AppendLine("## Regime definitions")
[void]$sb.AppendLine("| Definition | n pre | n post | Median lag pre | Median lag post | Mill % pre (95% CI) | Mill % post (95% CI) | Diff pp (95% CI) | p |")
[void]$sb.AppendLine("|---|---|---|---|---|---|---|---|---|")
foreach ($r in $regime) { [void]$sb.AppendLine("| $($r.Definition) | $($r.NPre) | $($r.NPost) | $($r.MedianLagPre) | $($r.MedianLagPost) | $($r.MillPctPre) ($($r.MillPreLo)-$($r.MillPreHi)) | $($r.MillPctPost) ($($r.MillPostLo)-$($r.MillPostHi)) | $($r.DiffPP) ($($r.DiffLo) to $($r.DiffHi)) | $($r.P) |") }
[void]$sb.AppendLine("")
[void]$sb.AppendLine("## Text-based LLM definition")
[void]$sb.AppendLine("Retractions whose title or notes mention ChatGPT, LLM, GPT or generative AI: $($textDef.RetractionsMentioningLlmInTitleOrNotes) across all retractions, of which $($textDef.OfWhichMachineAttributed) are machine-attributed ($($textDef.ShareOfMachineCohort)% of the cohort). Paper-mill flag among those: $($textDef.MillPctAmongLlmMentioning)%.")
[void]$sb.AppendLine("")
[void]$sb.AppendLine("## Share of all retractions by year")
[void]$sb.AppendLine("| Year | Machine-attributed | All retractions | Share % (95% CI) |")
[void]$sb.AppendLine("|---|---|---|---|")
foreach ($b in $byYear) { [void]$sb.AppendLine("| $($b.Year) | $($b.MachineAttributed) | $($b.AllRetractions) | $($b.SharePct) ($($b.CiLo)-$($b.CiHi)) |") }
Set-Content -Path (Join-Path $OutDir 'stats-v2.md') -Value $sb.ToString() -Encoding UTF8

Write-Host "`nWrote stats-v2.md and tables\t*.csv in $OutDir" -ForegroundColor Green
$sens | Format-Table Analysis, N, Median, MeanAbsDev, LocDelta, LocP, DispRatio, DispP, PaperMillPct -AutoSize
$boot | Format-Table -AutoSize
