<#
.SYNOPSIS
  Complete citation analysis with a matched comparison cohort.
.DESCRIPTION
  Queries OpenAlex for EVERY machine-attributed retracted article with a usable DOI, plus a 1:1 matched
  cohort of other retracted articles. Matching is exact on publication year, retraction year and
  publisher, falling back to publication year and retraction year where no publisher match remains;
  the match tier is recorded for every pair. Pre-retraction citation counts are reported as a balance
  check rather than used for matching, since they are not observable before retrieval.
  Requires an OpenAlex API key (mandatory since February 2026). Keys are rotated across requests.
.EXAMPLE
  .\Get-CitationMatched.ps1 -Csv ..\data\retraction_watch_gitlab.csv -OutDir ..\runs\ai-retractions-v3 -ApiKeys key1,key2
#>
[CmdletBinding()]
param(
    [Parameter(Mandatory = $true)][string]$Csv,
    [Parameter(Mandatory = $true)][string]$OutDir,
    [Parameter(Mandatory = $true)][string[]]$ApiKeys,
    [string]$ReasonPattern = 'Computer-Aided Content|Computer-Generated Content',
    [int]$Seed = 20260731,
    [int]$SnapshotYear = 2026,
    [string]$Mailto = 'jonas.mandalunes@deped.gov.ph'
)
$ErrorActionPreference = 'Stop'
New-Item -ItemType Directory -Force -Path (Join-Path $OutDir 'tables') | Out-Null
$inv = [Globalization.CultureInfo]::InvariantCulture
$ua = "analysis/1.0 (mailto:$Mailto)"

function Get-Q { param([double[]]$S, [double]$P)
    if ($S.Length -eq 0) { return 0.0 }
    $i = ($S.Length - 1) * $P
    $lo = [int][Math]::Floor($i); $hi = [int][Math]::Ceiling($i)
    if ($lo -eq $hi) { return $S[$lo] }
    return $S[$lo] + ($i - $lo) * ($S[$hi] - $S[$lo])
}
function Get-Wilson { param([int]$K, [int]$N)
    if ($N -eq 0) { return [pscustomobject]@{ P = 0; Lo = 0; Hi = 0 } }
    $z = 1.959963985; $p = [double]$K / $N; $den = 1.0 + $z * $z / $N
    $c = ($p + $z * $z / (2.0 * $N)) / $den
    $h = ($z * [Math]::Sqrt($p * (1 - $p) / $N + $z * $z / (4.0 * $N * $N))) / $den
    return [pscustomobject]@{ P = [Math]::Round(100 * $p, 2); Lo = [Math]::Round(100 * ($c - $h), 2); Hi = [Math]::Round(100 * ($c + $h), 2) }
}
function Get-NormalCdf { param([double]$Z)
    if ($Z -lt -8.5) { return 0.0 }; if ($Z -gt 8.5) { return 1.0 }
    $t = 1.0 / (1.0 + 0.2316419 * [Math]::Abs($Z))
    $d = 0.3989422804014327 * [Math]::Exp(-$Z * $Z / 2.0)
    $p = $d * $t * (0.319381530 + $t * (-0.356563782 + $t * (1.781477937 + $t * (-1.821255978 + $t * 1.330274429))))
    if ($Z -ge 0) { return 1.0 - $p }
    return $p
}
function Format-P { param([double]$P)
    if ($P -le 0) { return '< 1e-15' }
    if ($P -lt 0.001) { return ('{0:0.0e+0}' -f $P) }
    return ('{0:0.000}' -f $P)
}
function Test-MW { param([double[]]$A, [double[]]$B)
    $n1 = $A.Length; $n2 = $B.Length; $N = $n1 + $n2
    $v = New-Object 'double[]' $N; $g = New-Object 'int[]' $N
    [Array]::Copy($A, 0, $v, 0, $n1); [Array]::Copy($B, 0, $v, $n1, $n2)
    for ($i = 0; $i -lt $n1; $i++) { $g[$i] = 1 }
    for ($i = $n1; $i -lt $N; $i++) { $g[$i] = 2 }
    [Array]::Sort($v, $g)
    $r1 = 0.0; $tie = 0.0; $i = 0
    while ($i -lt $N) {
        $j = $i
        while (($j + 1) -lt $N -and $v[$j + 1] -eq $v[$i]) { $j++ }
        $t = $j - $i + 1; $avg = ($i + 1 + $j + 1) / 2.0
        if ($t -gt 1) { $tie += ([double]$t * $t * $t - $t) }
        for ($k = $i; $k -le $j; $k++) { if ($g[$k] -eq 1) { $r1 += $avg } }
        $i = $j + 1
    }
    $u1 = $r1 - ($n1 * ($n1 + 1.0) / 2.0)
    $sd = [Math]::Sqrt(($n1 * $n2 / 12.0) * (($N + 1.0) - $tie / ([double]$N * ($N - 1.0))))
    $z = ($u1 - $n1 * $n2 / 2.0) / $sd
    $az = [Math]::Abs($z); $p = 0.0
    if ($az -le 8.2) { $p = 2.0 * (1.0 - (Get-NormalCdf $az)) }
    return [pscustomobject]@{ Z = [Math]::Round($z, 3); Pstr = Format-P $p; Delta = [Math]::Round(2.0 * $u1 / ($n1 * $n2) - 1.0, 4) }
}
function Test-Prop { param([int]$K1, [int]$N1, [int]$K2, [int]$N2)
    $p1 = [double]$K1 / $N1; $p2 = [double]$K2 / $N2
    $pp = [double]($K1 + $K2) / ($N1 + $N2)
    $z = ($p1 - $p2) / [Math]::Sqrt($pp * (1 - $pp) * (1.0 / $N1 + 1.0 / $N2))
    $seD = [Math]::Sqrt($p1 * (1 - $p1) / $N1 + $p2 * (1 - $p2) / $N2)
    $az = [Math]::Abs($z); $p = 0.0
    if ($az -le 8.2) { $p = 2.0 * (1.0 - (Get-NormalCdf $az)) }
    return [pscustomobject]@{ DiffPP = [Math]::Round(100 * ($p1 - $p2), 2)
        DiffLo = [Math]::Round(100 * (($p1 - $p2) - 1.959963985 * $seD), 2)
        DiffHi = [Math]::Round(100 * (($p1 - $p2) + 1.959963985 * $seD), 2)
        Z = [Math]::Round($z, 2); Pstr = Format-P $p }
}

# ---------------- validate keys before doing any work ----------------
$ApiKeys = @($ApiKeys | ForEach-Object { $_ -split ',' } | ForEach-Object { $_.Trim() } | Where-Object { $_ })
Write-Host ("Validating {0} API key(s) ..." -f $ApiKeys.Count) -ForegroundColor Cyan
$goodKeys = New-Object 'System.Collections.Generic.List[string]'
foreach ($k in $ApiKeys) {
    $probe = "https://api.openalex.org/works?filter=doi:10.1371/journal.pone.0068397&per-page=1&select=doi&api_key=$k"
    try {
        $wc = New-Object System.Net.WebClient
        $wc.Encoding = [Text.Encoding]::UTF8
        $null = $wc.DownloadString($probe); $wc.Dispose()
        $goodKeys.Add($k)
        Write-Host ("  key {0}... OK" -f $k.Substring(0, 4)) -ForegroundColor Green
    }
    catch {
        Write-Host ("  key {0}... REJECTED: {1}" -f $k.Substring(0, [Math]::Min(4, $k.Length)), $_.Exception.Message) -ForegroundColor Red
    }
}
if ($goodKeys.Count -eq 0) { throw "No usable OpenAlex API key. Aborting before wasting requests." }
$ApiKeys = $goodKeys.ToArray()

# ---------------- build cohorts and matched pairs ----------------
Write-Host "Loading and selecting records ..." -ForegroundColor Cyan
$ret = @(Import-Csv $Csv | Where-Object { $_.RetractionNature -eq 'Retraction' })
$mach = New-Object System.Collections.ArrayList
$pool = @{}
foreach ($r in $ret) {
    if ($r.OriginalPaperDOI -notmatch '^10\.') { continue }
    $doi = $r.OriginalPaperDOI.Trim().ToLower()
    $py = $null; $ry = $null
    try { $py = [datetime]::Parse($r.OriginalPaperDate, $inv).Year } catch { continue }
    try { $ry = [datetime]::Parse($r.RetractionDate, $inv).Year } catch { continue }
    $rec = [pscustomobject]@{ Doi = $doi; PubYear = $py; RetYear = $ry; Publisher = $r.Publisher }
    if ($r.Reason -match $ReasonPattern) { [void]$mach.Add($rec) }
    else {
        $k1 = "$py|$ry|$($r.Publisher)"; $k2 = "$py|$ry"
        foreach ($k in @($k1, $k2)) {
            if (-not $pool.ContainsKey($k)) { $pool[$k] = New-Object 'System.Collections.Generic.List[string]' }
            $pool[$k].Add($doi)
        }
    }
}
Write-Host ("  machine-attributed with usable DOI and dates: {0}" -f $mach.Count)

$used = @{}
$pairs = New-Object System.Collections.ArrayList
$tier1 = 0; $tier2 = 0; $unmatched = 0
foreach ($m in $mach) {
    $pick = $null; $tier = 0
    foreach ($spec in @(@{K = "$($m.PubYear)|$($m.RetYear)|$($m.Publisher)"; T = 1 }, @{K = "$($m.PubYear)|$($m.RetYear)"; T = 2 })) {
        if ($pick) { break }
        if (-not $pool.ContainsKey($spec.K)) { continue }
        foreach ($cand in $pool[$spec.K]) {
            if ($used.ContainsKey($cand)) { continue }
            $pick = $cand; $tier = $spec.T; break
        }
    }
    if ($pick) {
        $used[$pick] = $true
        if ($tier -eq 1) { $tier1++ } else { $tier2++ }
        [void]$pairs.Add([pscustomobject]@{ MachineDoi = $m.Doi; MatchDoi = $pick; PubYear = $m.PubYear; RetYear = $m.RetYear; Publisher = $m.Publisher; MatchTier = $tier })
    }
    else { $unmatched++ }
}
Write-Host ("  matched pairs: {0} (exact on publisher {1}, year-only fallback {2}); unmatched {3}" -f $pairs.Count, $tier1, $tier2, $unmatched)
$pairs | Export-Csv (Join-Path $OutDir 'tables\t18_matched_pairs.csv') -NoTypeInformation -Encoding UTF8

$index = @{}
foreach ($m in $mach) { $index[$m.Doi] = @($m.RetYear, 'machine') }
foreach ($p in $pairs) { if (-not $index.ContainsKey($p.MatchDoi)) { $index[$p.MatchDoi] = @($p.RetYear, 'matched') } }
$queue = @($index.Keys)
Write-Host ("  querying {0} DOIs" -f $queue.Count)

# ---------------- retrieve ----------------
$rows = New-Object System.Collections.ArrayList
$ki = 0; $failed = 0
for ($i = 0; $i -lt $queue.Count; $i += 40) {
    $hi = [Math]::Min($i + 39, $queue.Count - 1)
    $batch = @($queue[$i..$hi])
    $key = $ApiKeys[$ki % $ApiKeys.Count]; $ki++
    $u = "https://api.openalex.org/works?filter=doi:" + ($batch -join '|') + "&per-page=40&select=doi,publication_year,cited_by_count,counts_by_year,is_retracted&api_key=$key"
    $txt = $null
    for ($try = 1; $try -le 4; $try++) {
        try {
            $wc = New-Object System.Net.WebClient
            $wc.Encoding = [Text.Encoding]::UTF8
            $txt = $wc.DownloadString($u); $wc.Dispose()
            break
        }
        catch { Start-Sleep -Milliseconds (400 * $try) }
    }
    if (-not $txt) {
        $failed += $batch.Count
        if ($failed -ge 200 -and $rows.Count -eq 0) { throw "First 200 DOIs all failed to retrieve. Aborting rather than continuing silently." }
        continue
    }
    $resp = $null
    try { $resp = $txt | ConvertFrom-Json } catch { $failed += $batch.Count; continue }
    foreach ($w in @($resp.results)) {
        $doi = ($w.doi -replace 'https://doi.org/', '').ToLower()
        if (-not $index.ContainsKey($doi)) { continue }
        $ry = [int]$index[$doi][0]; $coh = [string]$index[$doi][1]
        $pre = 0; $post = 0; $y1 = 0; $y2 = 0; $y3 = 0
        foreach ($c in @($w.counts_by_year)) {
            $n = [int]$c.PSObject.Properties['cited_by_count'].Value
            $yr = [int]$c.year
            if ($yr -gt $ry) { $post += $n } else { $pre += $n }
            if ($yr -eq $ry + 1) { $y1 = $n }
            if ($yr -eq $ry + 2) { $y2 = $n }
            if ($yr -eq $ry + 3) { $y3 = $n }
        }
        [void]$rows.Add([pscustomobject]@{
                Doi = $doi; Cohort = $coh; PubYear = $w.publication_year; RetractionYear = $ry
                ExposureYears = $SnapshotYear - $ry; TotalCitations = [int]$w.cited_by_count
                UpToRetraction = $pre; AfterRetraction = $post; CitesY1 = $y1; CitesY2 = $y2; CitesY3 = $y3
                FlaggedRetOA = $w.is_retracted
            })
    }
    Start-Sleep -Milliseconds 110
    if ((($i / 40) % 50) -eq 0) { Write-Host ("    {0}/{1}" -f ($hi + 1), $queue.Count) }
}
$rows | Export-Csv (Join-Path $OutDir 'tables\t19_citations_full.csv') -NoTypeInformation -Encoding UTF8
Write-Host ("  matched in OpenAlex: {0} of {1}; failed batches covering {2} DOIs" -f $rows.Count, $queue.Count, $failed) -ForegroundColor Green

function Get-Stats { param($Set, [string]$Label)
    $n = $Set.Count
    $we = @($Set | Where-Object { [int]$_.ExposureYears -ge 1 })
    $rate = [double[]]@($we | ForEach-Object { [double]$_.AfterRetraction / [double]$_.ExposureYears }); [Array]::Sort($rate)
    $tot = [double[]]@($Set | ForEach-Object { [double]$_.TotalCitations }); [Array]::Sort($tot)
    $pre = [double[]]@($Set | ForEach-Object { [double]$_.UpToRetraction }); [Array]::Sort($pre)
    $anyPost = @($Set | Where-Object { [int]$_.AfterRetraction -gt 0 }).Count
    $e1 = @($Set | Where-Object { [int]$_.ExposureYears -ge 1 }); $c1 = @($e1 | Where-Object { [int]$_.CitesY1 -gt 0 }).Count
    $e2 = @($Set | Where-Object { [int]$_.ExposureYears -ge 2 }); $c2 = @($e2 | Where-Object { ([int]$_.CitesY1 + [int]$_.CitesY2) -gt 0 }).Count
    $e3 = @($Set | Where-Object { [int]$_.ExposureYears -ge 3 }); $c3 = @($e3 | Where-Object { ([int]$_.CitesY1 + [int]$_.CitesY2 + [int]$_.CitesY3) -gt 0 }).Count
    $flag = @($Set | Where-Object { $_.FlaggedRetOA -eq 'True' -or $_.FlaggedRetOA -eq $true }).Count
    return [pscustomobject]@{
        Cohort = $Label; N = $n
        PreCitMedian = [Math]::Round((Get-Q $pre 0.5), 1)
        TotalMedian = [Math]::Round((Get-Q $tot 0.5), 1); TotalQ1 = [Math]::Round((Get-Q $tot 0.25), 1); TotalQ3 = [Math]::Round((Get-Q $tot 0.75), 1)
        RateMedian = [Math]::Round((Get-Q $rate 0.5), 3); RateQ1 = [Math]::Round((Get-Q $rate 0.25), 3); RateQ3 = [Math]::Round((Get-Q $rate 0.75), 3)
        RateMean = [Math]::Round((($rate | Measure-Object -Average).Average), 3)
        AnyPostK = $anyPost; AnyPostPct = (Get-Wilson $anyPost $n).P; AnyPostLo = (Get-Wilson $anyPost $n).Lo; AnyPostHi = (Get-Wilson $anyPost $n).Hi
        W1K = $c1; W1N = $e1.Count; W1Pct = (Get-Wilson $c1 $e1.Count).P
        W2K = $c2; W2N = $e2.Count; W2Pct = (Get-Wilson $c2 $e2.Count).P
        W3K = $c3; W3N = $e3.Count; W3Pct = (Get-Wilson $c3 $e3.Count).P
        FlagK = $flag; FlagPct = (Get-Wilson $flag $n).P; FlagLo = (Get-Wilson $flag $n).Lo; FlagHi = (Get-Wilson $flag $n).Hi
        RateArray = $rate
    }
}
$mSet = @($rows | Where-Object { $_.Cohort -eq 'machine' })
$cSet = @($rows | Where-Object { $_.Cohort -eq 'matched' })
$sM = Get-Stats $mSet 'Machine-attributed'
$sC = Get-Stats $cSet 'Matched other retracted articles'
$mwRate = Test-MW -A $sM.RateArray -B $sC.RateArray
$pAny = Test-Prop -K1 $sM.AnyPostK -N1 $sM.N -K2 $sC.AnyPostK -N2 $sC.N
$pFlag = Test-Prop -K1 $sM.FlagK -N1 $sM.N -K2 $sC.FlagK -N2 $sC.N
$p1 = Test-Prop -K1 $sM.W1K -N1 $sM.W1N -K2 $sC.W1K -N2 $sC.W1N
$p2 = Test-Prop -K1 $sM.W2K -N1 $sM.W2N -K2 $sC.W2K -N2 $sC.W2N
$p3 = Test-Prop -K1 $sM.W3K -N1 $sM.W3N -K2 $sC.W3K -N2 $sC.W3N
@($sM, $sC) | Select-Object * -ExcludeProperty RateArray | Export-Csv (Join-Path $OutDir 'tables\t20_citation_matched_summary.csv') -NoTypeInformation -Encoding UTF8

$sb = New-Object System.Text.StringBuilder
[void]$sb.AppendLine("# Citation analysis with matched comparison cohort (v3)")
[void]$sb.AppendLine("")
[void]$sb.AppendLine("Generated $(Get-Date -Format 'yyyy-MM-dd HH:mm'). Source: authoritative Crossref GitLab CSV, RetractionNature = 'Retraction'.")
[void]$sb.AppendLine("All $($mach.Count) machine-attributed articles with a usable DOI and date pair were queried, together with")
[void]$sb.AppendLine("$($pairs.Count) matched comparison articles (exact on publisher $tier1, year-only fallback $tier2; $unmatched unmatched).")
[void]$sb.AppendLine("Matched in OpenAlex: $($rows.Count) of $($queue.Count). Exposure = $SnapshotYear minus retraction year.")
[void]$sb.AppendLine("")
[void]$sb.AppendLine("| Measure | Machine-attributed | Matched comparison | Difference (95% CI) | Test |")
[void]$sb.AppendLine("|---|---|---|---|---|")
[void]$sb.AppendLine("| Articles matched in OpenAlex | $($sM.N) | $($sC.N) | | |")
[void]$sb.AppendLine("| Pre-retraction citations, median (balance check) | $($sM.PreCitMedian) | $($sC.PreCitMedian) | | |")
[void]$sb.AppendLine("| Total citations, median (IQR) | $($sM.TotalMedian) ($($sM.TotalQ1)-$($sM.TotalQ3)) | $($sC.TotalMedian) ($($sC.TotalQ1)-$($sC.TotalQ3)) | | |")
[void]$sb.AppendLine("| Post-retraction citations per article-year, median (IQR) | $($sM.RateMedian) ($($sM.RateQ1)-$($sM.RateQ3)) | $($sC.RateMedian) ($($sC.RateQ1)-$($sC.RateQ3)) | | z = $($mwRate.Z), p $($mwRate.Pstr), delta $($mwRate.Delta) |")
[void]$sb.AppendLine("| Post-retraction citations per article-year, mean | $($sM.RateMean) | $($sC.RateMean) | | |")
[void]$sb.AppendLine("| Cited at least once after retraction year | $($sM.AnyPostPct)% ($($sM.AnyPostLo)-$($sM.AnyPostHi)) | $($sC.AnyPostPct)% ($($sC.AnyPostLo)-$($sC.AnyPostHi)) | $($pAny.DiffPP) pp ($($pAny.DiffLo) to $($pAny.DiffHi)) | z = $($pAny.Z), p $($pAny.Pstr) |")
[void]$sb.AppendLine("| Cited within 1 year of retraction | $($sM.W1Pct)% (n = $($sM.W1N)) | $($sC.W1Pct)% (n = $($sC.W1N)) | $($p1.DiffPP) pp ($($p1.DiffLo) to $($p1.DiffHi)) | p $($p1.Pstr) |")
[void]$sb.AppendLine("| Cited within 2 years | $($sM.W2Pct)% (n = $($sM.W2N)) | $($sC.W2Pct)% (n = $($sC.W2N)) | $($p2.DiffPP) pp ($($p2.DiffLo) to $($p2.DiffHi)) | p $($p2.Pstr) |")
[void]$sb.AppendLine("| Cited within 3 years | $($sM.W3Pct)% (n = $($sM.W3N)) | $($sC.W3Pct)% (n = $($sC.W3N)) | $($p3.DiffPP) pp ($($p3.DiffLo) to $($p3.DiffHi)) | p $($p3.Pstr) |")
[void]$sb.AppendLine("| Flagged as retracted in OpenAlex | $($sM.FlagPct)% ($($sM.FlagLo)-$($sM.FlagHi)) | $($sC.FlagPct)% ($($sC.FlagLo)-$($sC.FlagHi)) | $($pFlag.DiffPP) pp ($($pFlag.DiffLo) to $($pFlag.DiffHi)) | z = $($pFlag.Z), p $($pFlag.Pstr) |")
Set-Content -Path (Join-Path $OutDir 'citations-v3.md') -Value $sb.ToString() -Encoding UTF8
Write-Host "`nWrote citations-v3.md and tables t18-t20" -ForegroundColor Green
