<#
.SYNOPSIS
  Fourth-round corrections: exact citation attrition, paired analyses, consistent denominators,
  early-year paper-mill proportions and notice-identifier quality.
.EXAMPLE
  .\Analyze-Paired.ps1 -Csv ..\data\retraction_watch_gitlab.csv -RunDir ..\runs\ai-retractions-v3
#>
[CmdletBinding()]
param(
    [Parameter(Mandatory = $true)][string]$Csv,
    [Parameter(Mandatory = $true)][string]$RunDir,
    [string]$ReasonPattern = 'Computer-Aided Content|Computer-Generated Content',
    [int]$Bootstrap = 2000,
    [int]$Seed = 20260731
)
$ErrorActionPreference = 'Stop'
$inv = [Globalization.CultureInfo]::InvariantCulture
$tab = Join-Path $RunDir 'tables'

function Get-Q { param([double[]]$S, [double]$P)
    if ($S.Length -eq 0) { return 0.0 }
    $i = ($S.Length - 1) * $P
    $lo = [int][Math]::Floor($i); $hi = [int][Math]::Ceiling($i)
    if ($lo -eq $hi) { return $S[$lo] }
    return $S[$lo] + ($i - $lo) * ($S[$hi] - $S[$lo])
}
function Get-NormalCdf { param([double]$Z)
    if ($Z -lt -8.5) { return 0.0 }; if ($Z -gt 8.5) { return 1.0 }
    $t = 1.0 / (1.0 + 0.2316419 * [Math]::Abs($Z))
    $d = 0.3989422804014327 * [Math]::Exp(-$Z * $Z / 2.0)
    $p = $d * $t * (0.319381530 + $t * (-0.356563782 + $t * (1.781477937 + $t * (-1.821255978 + $t * 1.330274429))))
    if ($Z -ge 0) { return 1.0 - $p }
    return $p
}
function Format-P { param([double]$P)
    if ($P -le 0) { return '< 1e-15' }
    if ($P -lt 0.001) { return ('{0:0.0e+0}' -f $P) }
    return ('{0:0.000}' -f $P)
}
function Get-PZ { param([double]$Z) $a = [Math]::Abs($Z); if ($a -gt 8.2) { return 0.0 }; return 2.0 * (1.0 - (Get-NormalCdf $a)) }

# ---------- load retrieved citation records and pairs ----------
$raw = @(Import-Csv (Join-Path $tab 't19_citations_full.csv'))
$pairs = @(Import-Csv (Join-Path $tab 't18_matched_pairs.csv'))
$byDoi = @{}; $dupDoi = @{}
foreach ($r in $raw) {
    $d = $r.Doi.ToLower()
    if ($byDoi.ContainsKey($d)) { $dupDoi[$d] = $true; continue }
    $byDoi[$d] = $r
}
$nDupRecords = $raw.Count - $byDoi.Count

# exact attrition accounting
$lostMachine = 0; $lostControl = 0; $lostBoth = 0; $kept = 0
$complete = New-Object System.Collections.ArrayList
foreach ($p in $pairs) {
    $m = $p.MachineDoi.ToLower(); $c = $p.MatchDoi.ToLower()
    $hm = $byDoi.ContainsKey($m); $hc = $byDoi.ContainsKey($c)
    if ($hm -and $hc) { $kept++; [void]$complete.Add([pscustomobject]@{ M = $byDoi[$m]; C = $byDoi[$c]; Tier = $p.MatchTier }) }
    elseif (-not $hm -and -not $hc) { $lostBoth++ }
    elseif (-not $hm) { $lostMachine++ }
    else { $lostControl++ }
}
$attr = @(
    [pscustomobject]@{ Step = 'Pairs formed 1:1 without replacement'; Pairs = $pairs.Count }
    [pscustomobject]@{ Step = 'Pairs lost: machine member not returned by OpenAlex'; Pairs = -$lostMachine }
    [pscustomobject]@{ Step = 'Pairs lost: control member not returned by OpenAlex'; Pairs = -$lostControl }
    [pscustomobject]@{ Step = 'Pairs lost: neither member returned'; Pairs = -$lostBoth }
    [pscustomobject]@{ Step = 'Complete pairs analysed'; Pairs = $kept }
    [pscustomobject]@{ Step = 'Duplicate retrieved records collapsed (records, not pairs)'; Pairs = $nDupRecords }
    [pscustomobject]@{ Step = 'Distinct DOIs affected by duplication'; Pairs = $dupDoi.Count }
)
$attr | Export-Csv (Join-Path $tab 't32_citation_attrition.csv') -NoTypeInformation -Encoding UTF8

# ---------- paired analyses ----------
$rand = New-Object System.Random $Seed
function Test-McNemar { param($Pairs, [scriptblock]$Outcome, [string]$Label)
    $b = 0; $c = 0; $both = 0; $neither = 0
    foreach ($p in $Pairs) {
        $om = & $Outcome $p.M; $oc = & $Outcome $p.C
        if ($om -and -not $oc) { $b++ } elseif (-not $om -and $oc) { $c++ } elseif ($om -and $oc) { $both++ } else { $neither++ }
    }
    $n = $Pairs.Count
    $chi = 0.0; $p = 1.0
    if (($b + $c) -gt 0) {
        $chi = [Math]::Pow([Math]::Abs($b - $c) - 1, 2) / ($b + $c)
        $p = Get-PZ ([Math]::Sqrt($chi))
    }
    $diff = ($b - $c) / [double]$n
    $se = [Math]::Sqrt(($b + $c) - ($b - $c) * ($b - $c) / [double]$n) / [double]$n
    return [pscustomobject]@{ Outcome = $Label; Pairs = $n
        MachineOnly = $b; ControlOnly = $c; Both = $both; Neither = $neither
        PctMachine = [Math]::Round(100 * ($b + $both) / [double]$n, 2)
        PctControl = [Math]::Round(100 * ($c + $both) / [double]$n, 2)
        PairedDiffPP = [Math]::Round(100 * $diff, 2)
        CiLo = [Math]::Round(100 * ($diff - 1.959963985 * $se), 2)
        CiHi = [Math]::Round(100 * ($diff + 1.959963985 * $se), 2)
        McNemarChiSq = [Math]::Round($chi, 2); P = Format-P $p }
}
$exp1 = @($complete | Where-Object { [int]$_.M.ExposureYears -ge 1 -and [int]$_.C.ExposureYears -ge 1 })
$exp2 = @($complete | Where-Object { [int]$_.M.ExposureYears -ge 2 -and [int]$_.C.ExposureYears -ge 2 })
$exp3 = @($complete | Where-Object { [int]$_.M.ExposureYears -ge 3 -and [int]$_.C.ExposureYears -ge 3 })
$mc = @(
    (Test-McNemar $exp1 { param($r) [int]$r.AfterRetraction -gt 0 } 'Cited in any post-retraction calendar year')
    (Test-McNemar $exp1 { param($r) [int]$r.CitesY1 -gt 0 } 'Cited in the first post-retraction calendar year')
    (Test-McNemar $exp2 { param($r) ([int]$r.CitesY1 + [int]$r.CitesY2) -gt 0 } 'Cited in the first two post-retraction calendar years')
    (Test-McNemar $exp3 { param($r) ([int]$r.CitesY1 + [int]$r.CitesY2 + [int]$r.CitesY3) -gt 0 } 'Cited in the first three post-retraction calendar years')
    (Test-McNemar $complete { param($r) ($r.FlaggedRetOA -eq 'True' -or $r.FlaggedRetOA -eq $true) } 'Flagged as retracted in OpenAlex')
)
$mc | Export-Csv (Join-Path $tab 't33_mcnemar_paired.csv') -NoTypeInformation -Encoding UTF8

# Wilcoxon signed-rank on within-pair rate differences
$d = New-Object 'System.Collections.Generic.List[double]'
foreach ($p in $exp1) {
    $rm = [double]$p.M.AfterRetraction / [double]$p.M.ExposureYears
    $rc = [double]$p.C.AfterRetraction / [double]$p.C.ExposureYears
    $d.Add($rm - $rc)
}
$nz = @($d | Where-Object { $_ -ne 0 })
$abs = [double[]]@($nz | ForEach-Object { [Math]::Abs($_) })
$idx = 0..($abs.Length - 1)
$sorted = @($idx | Sort-Object { $abs[$_] })
$ranks = New-Object 'double[]' $abs.Length
$i = 0
while ($i -lt $sorted.Count) {
    $j = $i
    while (($j + 1) -lt $sorted.Count -and $abs[$sorted[$j + 1]] -eq $abs[$sorted[$i]]) { $j++ }
    $avg = ($i + 1 + $j + 1) / 2.0
    for ($k = $i; $k -le $j; $k++) { $ranks[$sorted[$k]] = $avg }
    $i = $j + 1
}
$wPlus = 0.0
for ($k = 0; $k -lt $nz.Count; $k++) { if ($nz[$k] -gt 0) { $wPlus += $ranks[$k] } }
$nW = $nz.Count
$muW = $nW * ($nW + 1) / 4.0
$sdW = [Math]::Sqrt($nW * ($nW + 1) * (2 * $nW + 1) / 24.0)
$zW = ($wPlus - $muW) / $sdW
# paired bootstrap for the median within-pair difference
$dArr = $d.ToArray()
$bootMed = New-Object 'System.Collections.Generic.List[double]'
for ($b = 1; $b -le $Bootstrap; $b++) {
    $buf = New-Object 'double[]' $dArr.Length
    for ($k = 0; $k -lt $dArr.Length; $k++) { $buf[$k] = $dArr[$rand.Next(0, $dArr.Length)] }
    [Array]::Sort($buf)
    $bootMed.Add((Get-Q $buf 0.5))
}
$bm = $bootMed.ToArray(); [Array]::Sort($bm)
$dSorted = $dArr.Clone(); [Array]::Sort($dSorted)
$wil = [pscustomobject]@{
    PairsWithExposure = $exp1.Count
    NonZeroDifferences = $nW
    MedianWithinPairDifference = [Math]::Round((Get-Q $dSorted 0.5), 4)
    BootstrapCiLo = [Math]::Round((Get-Q $bm 0.025), 4)
    BootstrapCiHi = [Math]::Round((Get-Q $bm 0.975), 4)
    MeanWithinPairDifference = [Math]::Round((($dArr | Measure-Object -Average).Average), 4)
    WilcoxonZ = [Math]::Round($zW, 3)
    WilcoxonP = Format-P (Get-PZ $zW)
    BootstrapReplicates = $Bootstrap
}
$wil | Export-Csv (Join-Path $tab 't34_wilcoxon_paired_rates.csv') -NoTypeInformation -Encoding UTF8

# ---------- early-year paper-mill proportions and notice identifier quality ----------
$ret = @(Import-Csv $Csv | Where-Object { $_.RetractionNature -eq 'Retraction' })
$early = New-Object System.Collections.ArrayList
foreach ($grpDef in @(@{L = '2010-2019'; Lo = 2010; Hi = 2019 }, @{L = '2020'; Lo = 2020; Hi = 2020 }, @{L = '2021'; Lo = 2021; Hi = 2021 }, @{L = '2022'; Lo = 2022; Hi = 2022 })) {
    $sub = @($ret | Where-Object {
            $y = $null; try { $y = [datetime]::Parse($_.RetractionDate, $inv).Year } catch { }
            $y -ne $null -and $y -ge $grpDef.Lo -and $y -le $grpDef.Hi -and ($_.Reason -match $ReasonPattern)
        })
    $mill = @($sub | Where-Object { $_.Reason -match 'Paper Mill' }).Count
    $pct = 'n/a'; if ($sub.Count -gt 0) { $pct = [Math]::Round(100 * $mill / $sub.Count, 2) }
    [void]$early.Add([pscustomobject]@{ Period = $grpDef.L; MachineRecords = $sub.Count; WithPaperMillFlag = $mill; PaperMillPct = $pct })
}
$early | Export-Csv (Join-Path $tab 't35_early_year_papermill.csv') -NoTypeInformation -Encoding UTF8

$ai = @($ret | Where-Object { $_.Reason -match $ReasonPattern })
$valid = @($ai | Where-Object { $_.RetractionDOI -match '^10\.' }).Count
$blank = @($ai | Where-Object { -not $_.RetractionDOI -or $_.RetractionDOI.Trim() -eq '' }).Count
$unav = @($ai | Where-Object { $_.RetractionDOI -match '^(unavailable|Unavailable)$' }).Count
$malformed = $ai.Count - $valid - $blank - $unav
$noticeQ = [pscustomobject]@{ MachineRecords = $ai.Count; ValidRetractionDoi = $valid; BlankRetractionDoi = $blank
    LiteralUnavailable = $unav; OtherMalformed = $malformed
    TreatedAsSingletonClusters = ($blank + $unav + $malformed) }
$noticeQ | Export-Csv (Join-Path $tab 't36_notice_identifier_quality.csv') -NoTypeInformation -Encoding UTF8

# ---------- report ----------
$sb = New-Object System.Text.StringBuilder
[void]$sb.AppendLine("# Fourth-round corrections")
[void]$sb.AppendLine("")
[void]$sb.AppendLine("Generated $(Get-Date -Format 'yyyy-MM-dd HH:mm').")
[void]$sb.AppendLine("")
[void]$sb.AppendLine("## Exact citation attrition")
[void]$sb.AppendLine("| Step | Pairs |"); [void]$sb.AppendLine("|---|---|")
foreach ($a in $attr) { [void]$sb.AppendLine("| $($a.Step) | $($a.Pairs) |") }
[void]$sb.AppendLine("")
[void]$sb.AppendLine("## Paired binary outcomes (McNemar)")
[void]$sb.AppendLine("| Outcome | Pairs | Machine % | Control % | Discordant M-only | Discordant C-only | Paired difference pp (95% CI) | McNemar chi-square | p |")
[void]$sb.AppendLine("|---|---|---|---|---|---|---|---|---|")
foreach ($m in $mc) { [void]$sb.AppendLine("| $($m.Outcome) | $($m.Pairs) | $($m.PctMachine) | $($m.PctControl) | $($m.MachineOnly) | $($m.ControlOnly) | $($m.PairedDiffPP) ($($m.CiLo) to $($m.CiHi)) | $($m.McNemarChiSq) | $($m.P) |") }
[void]$sb.AppendLine("")
[void]$sb.AppendLine("## Paired citation rates (Wilcoxon signed-rank and paired bootstrap)")
[void]$sb.AppendLine("Pairs with exposure in both members: $($wil.PairsWithExposure); non-zero within-pair differences: $($wil.NonZeroDifferences).")
[void]$sb.AppendLine("Median within-pair difference $($wil.MedianWithinPairDifference) citations per article-year (paired bootstrap 95% CI $($wil.BootstrapCiLo) to $($wil.BootstrapCiHi)); mean $($wil.MeanWithinPairDifference).")
[void]$sb.AppendLine("Wilcoxon signed-rank z = $($wil.WilcoxonZ), p $($wil.WilcoxonP).")
[void]$sb.AppendLine("")
[void]$sb.AppendLine("## Paper-mill flag in the machine-attributed cohort before 2023")
[void]$sb.AppendLine("| Retraction period | Machine records | With paper-mill flag | Paper mill % |"); [void]$sb.AppendLine("|---|---|---|---|")
foreach ($e in $early) { [void]$sb.AppendLine("| $($e.Period) | $($e.MachineRecords) | $($e.WithPaperMillFlag) | $($e.PaperMillPct) |") }
[void]$sb.AppendLine("")
[void]$sb.AppendLine("## Notice identifier quality")
[void]$sb.AppendLine("Machine records $($noticeQ.MachineRecords); valid retraction DOI $($noticeQ.ValidRetractionDoi); blank $($noticeQ.BlankRetractionDoi); literal 'unavailable' $($noticeQ.LiteralUnavailable); other malformed $($noticeQ.OtherMalformed); treated as singleton clusters $($noticeQ.TreatedAsSingletonClusters).")
Set-Content -Path (Join-Path $RunDir 'paired-v5.md') -Value $sb.ToString() -Encoding UTF8
Write-Host "Wrote paired-v5.md and tables t32-t36" -ForegroundColor Green
$attr | Format-Table -AutoSize
$mc | Format-Table Outcome, Pairs, PctMachine, PctControl, PairedDiffPP, CiLo, CiHi, P -AutoSize
$wil | Format-List
$early | Format-Table -AutoSize
$noticeQ | Format-List
