Files
blance 76dc830580 Support sftp:// and ftps:// URL schemes in host values; fix TLS-flag bug
CAMBS's partner turned out to receive insurance files over SFTP - Melissa
pasted "SFTP://sft.polluxsystems.com/" as ftps.host, which the engine fed to
WinSCP as a literal FTP hostname. Host values are now parsed: an sftp://
scheme switches the upload to the SFTP protocol (default port 22), ftps://
forces explicit TLS, a bare hostname behaves as before per the tls flag, and
the scheme/trailing slash are stripped from the hostname either way.

Also fixes a real bug this exposed: GiveUpSecurityAndAcceptAnyTlsHostCertificate
was set unconditionally, and WinSCP refuses that combination when FtpSecure
is None - exactly the "TlsHostCertificateFingerprint ... is set, but neither
FtpSecure nor Secure is enabled" error from her run. The flag is now only set
when TLS is actually on.
2026-07-05 20:33:01 -05:00

431 lines
20 KiB
PowerShell

# Core upload engine - called by run.ps1 with an entity directory path
param(
[Parameter(Mandatory=$true)]
[string]$EntityDir
)
$scriptRoot = Split-Path -Parent $PSCommandPath
$winscpDll = Join-Path $scriptRoot "WinSCP\WinSCPnet.dll"
$today = Get-Date -Format "yyyyMMdd"
# -- Logging setup ------------------------------------------------------------
$logsDir = Join-Path $scriptRoot "Logs"
if (-not (Test-Path $logsDir)) { New-Item -ItemType Directory -Path $logsDir | Out-Null }
$logFile = Join-Path $logsDir "$today.log"
$settingsFile = Join-Path $scriptRoot "settings.json"
$logLevel = "normal"
if (Test-Path $settingsFile) {
$settings = Get-Content $settingsFile -Raw | ConvertFrom-Json
if ($settings.log_level) { $logLevel = $settings.log_level.ToLower() }
}
function Write-Log($level, $message) {
$ts = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
$line = "[$ts] [$($level.ToUpper())] $message"
Add-Content -Path $logFile -Value $line
if ($level -eq "error") {
Write-Host $line -ForegroundColor Red
} elseif ($logLevel -eq "debug") {
Write-Host $line -ForegroundColor DarkGray
}
}
function Write-Info($message) {
$ts = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
$line = "[$ts] [INFO ] $message"
Add-Content -Path $logFile -Value $line
}
function Write-Debug($message) {
if ($logLevel -eq "debug") {
$ts = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
$line = "[$ts] [DEBUG] $message"
Add-Content -Path $logFile -Value $line
}
}
# -- WinSCP -------------------------------------------------------------------
if (-not (Test-Path $winscpDll)) {
Write-Host "ERROR: WinSCP not found. Run setup.bat first." -ForegroundColor Red
Write-Log "error" "WinSCP DLL not found at $winscpDll"
exit 1
}
# Unblock every run, regardless of how the WinSCP folder got here - Windows
# tags files as untrusted ("Mark of the Web") if they were ever downloaded or
# copied from a network share/zip, which blocks Add-Type from loading them.
Get-ChildItem -Path (Join-Path $scriptRoot "WinSCP") -Recurse -File |
Unblock-File -ErrorAction SilentlyContinue
try {
Add-Type -Path $winscpDll
} catch {
Write-Host "ERROR: Could not load WinSCPnet.dll - $($_.Exception.Message)" -ForegroundColor Red
Write-Host "Try deleting the WinSCP\ folder and running setup.bat again." -ForegroundColor Yellow
Write-Log "error" "Add-Type failed for $winscpDll : $($_.Exception.Message)"
exit 1
}
# -- Entity config -------------------------------------------------------------
$entityConfigFile = Join-Path $EntityDir "entity.json"
if (-not (Test-Path $entityConfigFile)) {
Write-Host "ERROR: entity.json not found in $EntityDir" -ForegroundColor Red
Write-Log "error" "entity.json not found in $EntityDir"
exit 1
}
$entityConfig = Get-Content $entityConfigFile -Raw | ConvertFrom-Json
# Add any missing keys to entity.json as null so they never have to be typed
# by hand (a null value means "not used - ignore"). Writes the file back only
# when something was actually added.
$configTemplate = [ordered]@{
workflow = $null
insurance_file_type = $null
ftps = [ordered]@{ host = $null; port = $null; tls = $null; username = $null; password = $null; tiff_path = $null; pdf_path = $null }
sftp = [ordered]@{ host = $null; port = $null; username = $null; password = $null; patient_path = $null }
}
$addedKeys = @()
foreach ($topKey in $configTemplate.Keys) {
if ($null -eq $entityConfig.PSObject.Properties[$topKey]) {
$value = if ($configTemplate[$topKey] -is [System.Collections.Specialized.OrderedDictionary]) {
[PSCustomObject]$configTemplate[$topKey]
} else { $configTemplate[$topKey] }
$entityConfig | Add-Member -NotePropertyName $topKey -NotePropertyValue $value
$addedKeys += $topKey
} elseif ($configTemplate[$topKey] -is [System.Collections.Specialized.OrderedDictionary]) {
foreach ($subKey in $configTemplate[$topKey].Keys) {
if ($null -eq $entityConfig.$topKey.PSObject.Properties[$subKey]) {
$entityConfig.$topKey | Add-Member -NotePropertyName $subKey -NotePropertyValue $null
$addedKeys += "$topKey.$subKey"
}
}
}
}
if ($addedKeys.Count -gt 0) {
$entityConfig | ConvertTo-Json -Depth 10 | Set-Content -Path $entityConfigFile -Encoding UTF8
Write-Host "Added missing key(s) to entity.json (as null = ignored): $($addedKeys -join ', ')" -ForegroundColor Yellow
Write-Info "CONFIG added missing keys to entity.json: $($addedKeys -join ',')"
}
$workflow = $entityConfig.workflow
$insuranceExt = if ($entityConfig.insurance_file_type) { $entityConfig.insurance_file_type.TrimStart(".") } else { "pdf" }
$export1 = Join-Path $EntityDir "Export1"
$entityName = Split-Path -Leaf $EntityDir
if (-not (Test-Path $export1)) {
Write-Host "ERROR: Export1 folder not found in $EntityDir" -ForegroundColor Red
Write-Log "error" "Export1 not found in $EntityDir"
exit 1
}
Write-Host ""
Write-Host "Entity: $entityName | Workflow: $workflow | File type: .$insuranceExt | Date: $today | Log: $logLevel" -ForegroundColor Cyan
Write-Host ("=" * 60)
Write-Info "=== START entity=$entityName workflow=$workflow filetype=$insuranceExt ==="
# -- Helpers ------------------------------------------------------------------
function Merge-Config($base, $override) {
if (-not $override) { return $base }
$merged = $base | ConvertTo-Json -Depth 10 | ConvertFrom-Json
foreach ($section in $override.PSObject.Properties) {
if ($section.Name -like "_*") { continue }
$baseSection = $merged.PSObject.Properties[$section.Name]
if ($null -eq $baseSection) {
$merged | Add-Member -NotePropertyName $section.Name -NotePropertyValue $section.Value
} elseif ($section.Value -is [System.Management.Automation.PSCustomObject]) {
foreach ($prop in $section.Value.PSObject.Properties) {
$merged.$($section.Name) | Add-Member -NotePropertyName $prop.Name -NotePropertyValue $prop.Value -Force
}
} else {
$merged.$($section.Name) = $section.Value
}
}
return $merged
}
function Expand-Path($template, $practice) {
return $template -replace '\{practice\}', $practice
}
function Get-MissingFields($section, $sectionName, $fieldNames) {
$missing = @()
foreach ($f in $fieldNames) {
if (-not $section -or [string]::IsNullOrWhiteSpace([string]$section.$f)) {
$missing += "$sectionName.$f"
}
}
return $missing
}
function Get-ArchiveDir($parent, $name) {
$found = Get-ChildItem -Path $parent -Directory |
Where-Object { $_.Name -ieq $name } | Select-Object -First 1
if ($found) { return $found.FullName }
$path = Join-Path $parent $name
New-Item -ItemType Directory -Path $path | Out-Null
return $path
}
# Host values may be pasted as URLs, e.g. "SFTP://server.com/" - pull out the
# scheme (decides the protocol) and the bare hostname.
function Get-HostInfo($rawHost) {
$h = ([string]$rawHost).Trim()
$scheme = $null
if ($h -imatch '^(\w+)://') {
$scheme = $Matches[1].ToLower()
$h = $h -replace '^\w+://', ''
}
$h = $h.Split('/')[0]
return @{ Scheme = $scheme; HostName = $h }
}
# -- Upload via FTPS -----------------------------------------------------------
function Invoke-FTPSUpload($ftpConfig, $files, $remotePath, $entity, $practice) {
$hostInfo = Get-HostInfo $ftpConfig.host
$opts = New-Object WinSCP.SessionOptions
if ($hostInfo.Scheme -eq "sftp") {
$opts.Protocol = [WinSCP.Protocol]::Sftp
$opts.PortNumber = if ($ftpConfig.port) { [int]$ftpConfig.port } else { 22 }
$opts.GiveUpSecurityAndAcceptAnySshHostKey = $true
} else {
$useTls = ($ftpConfig.tls -eq $true) -or ($hostInfo.Scheme -eq "ftps")
$opts.Protocol = [WinSCP.Protocol]::Ftp
$opts.FtpSecure = if ($useTls) { [WinSCP.FtpSecure]::Explicit } else { [WinSCP.FtpSecure]::None }
$opts.PortNumber = if ($ftpConfig.port) { [int]$ftpConfig.port } else { 21 }
if ($useTls) { $opts.GiveUpSecurityAndAcceptAnyTlsHostCertificate = $true }
}
$opts.HostName = $hostInfo.HostName
$opts.UserName = $ftpConfig.username
$opts.Password = $ftpConfig.password
$session = New-Object WinSCP.Session
if ($logLevel -eq "debug") {
$session.SessionLogPath = Join-Path $logsDir "${today}_winscp_debug.log"
}
$ok = 0; $fail = 0
try {
Write-Debug "Connect: $($opts.Protocol) $($hostInfo.HostName):$($opts.PortNumber) user=$($ftpConfig.username)"
$session.Open($opts)
foreach ($file in $files) {
$remote = $remotePath.TrimEnd("/") + "/" + $file.Name
try {
$result = $session.PutFiles($file.FullName, $remote)
$result.Check()
Write-Host " $($file.Name) ... ok" -ForegroundColor Green
Write-Info "FTPS OK entity=$entity practice=$practice file=$($file.Name) remote=$remote"
$ok++
} catch {
$err = $_.Exception.Message
Write-Host " $($file.Name) ... FAILED" -ForegroundColor Red
Write-Log "error" "FTPS FAIL entity=$entity practice=$practice file=$($file.Name) remote=$remote error=$err"
$fail++
}
}
} catch {
$err = $_.Exception.Message
Write-Log "error" "FTPS connect FAILED entity=$entity practice=$practice host=$($ftpConfig.host) error=$err"
Write-Host " Connection failed: $err" -ForegroundColor Red
$fail += $files.Count
} finally {
$session.Dispose()
}
return @{ Ok = $ok; Fail = $fail }
}
# -- Upload via SFTP -----------------------------------------------------------
function Invoke-SFTPUpload($sftpConfig, $files, $remotePath, $entity, $practice) {
$hostInfo = Get-HostInfo $sftpConfig.host
$opts = New-Object WinSCP.SessionOptions
$opts.Protocol = [WinSCP.Protocol]::Sftp
$opts.HostName = $hostInfo.HostName
$opts.PortNumber = if ($sftpConfig.port) { [int]$sftpConfig.port } else { 22 }
$opts.UserName = $sftpConfig.username
$opts.Password = $sftpConfig.password
$opts.GiveUpSecurityAndAcceptAnySshHostKey = $true
$session = New-Object WinSCP.Session
if ($logLevel -eq "debug") {
$session.SessionLogPath = Join-Path $logsDir "${today}_winscp_debug.log"
}
$ok = 0; $fail = 0
try {
Write-Debug "SFTP connect: $($sftpConfig.host) user=$($sftpConfig.username)"
$session.Open($opts)
foreach ($file in $files) {
$remote = $remotePath.TrimEnd("/") + "/" + $file.Name
try {
$result = $session.PutFiles($file.FullName, $remote)
$result.Check()
Write-Host " $($file.Name) ... ok" -ForegroundColor Green
Write-Info "SFTP OK entity=$entity practice=$practice file=$($file.Name) remote=$remote"
$ok++
} catch {
$err = $_.Exception.Message
Write-Host " $($file.Name) ... FAILED" -ForegroundColor Red
Write-Log "error" "SFTP FAIL entity=$entity practice=$practice file=$($file.Name) remote=$remote error=$err"
$fail++
}
}
} catch {
$err = $_.Exception.Message
Write-Log "error" "SFTP connect FAILED entity=$entity practice=$practice host=$($sftpConfig.host) error=$err"
Write-Host " Connection failed: $err" -ForegroundColor Red
$fail += $files.Count
} finally {
$session.Dispose()
}
return @{ Ok = $ok; Fail = $fail }
}
# -- Main loop -----------------------------------------------------------------
$totalOk = 0; $totalFail = 0
$practices = Get-ChildItem -Path $export1 -Directory
if ($practices.Count -eq 0) {
Write-Host "No practice folders found in Export1." -ForegroundColor Yellow
Write-Info "No practice folders found in $export1"
exit 0
}
foreach ($practice in $practices) {
$pracName = $practice.Name
Write-Host ""
Write-Host " [ $pracName ]" -ForegroundColor Cyan
Write-Info "--- practice=$pracName ---"
$overrideFile = Join-Path $practice.FullName "practice.json"
$override = if (Test-Path $overrideFile) { Get-Content $overrideFile -Raw | ConvertFrom-Json } else { $null }
$config = Merge-Config $entityConfig $override
if ($override) { Write-Debug "practice.json override loaded for $pracName" }
# tiff_path is the current name; insurance_path is accepted as a fallback
# so older config files keep working.
if ($config.ftps -and -not $config.ftps.PSObject.Properties["tiff_path"] -and $config.ftps.PSObject.Properties["insurance_path"]) {
$config.ftps | Add-Member -NotePropertyName "tiff_path" -NotePropertyValue $config.ftps.insurance_path
}
# -- Insurance + PT STMT -> FTPS -------------------------------------------
$dateDir = Join-Path $practice.FullName $today
if (-not (Test-Path $dateDir)) {
Write-Host " No $today\ folder - skipping insurance." -ForegroundColor Gray
Write-Info "SKIP entity=$entityName practice=$pracName reason=no date folder"
} else {
$allInsuranceFiles = Get-ChildItem -Path $dateDir -Filter "*.$insuranceExt" -File
$ptStmts = $allInsuranceFiles | Where-Object { $_.Name -imatch 'pt[\s_]*stmt' }
$insFiles = $allInsuranceFiles | Where-Object { $_.Name -notmatch 'pt[\s_]*stmt' }
$insPath = Expand-Path $config.ftps.tiff_path $pracName
$pdfPath = Expand-Path $config.ftps.pdf_path $pracName
$insResult = @{ Ok = 0; Fail = 0 }
$connMissing = Get-MissingFields $config.ftps "ftps" @("host", "username", "password")
if (($insFiles.Count -gt 0 -or $ptStmts.Count -gt 0) -and $connMissing.Count -gt 0) {
Write-Host " FTPS not configured ($($connMissing -join ', ') is null/blank) - $($insFiles.Count + $ptStmts.Count) file(s) ignored, left in place." -ForegroundColor Yellow
Write-Info "IGNORE entity=$entityName practice=$pracName reason=unconfigured fields=$($connMissing -join ',')"
} else {
$ignored = 0
if ($insFiles.Count -gt 0) {
if ([string]::IsNullOrWhiteSpace([string]$config.ftps.tiff_path)) {
Write-Host " ftps.tiff_path is null/blank - $($insFiles.Count) insurance file(s) ignored, left in place." -ForegroundColor Yellow
Write-Info "IGNORE entity=$entityName practice=$pracName reason=tiff_path null count=$($insFiles.Count)"
$ignored += $insFiles.Count
} else {
Write-Host " Insurance ($($insFiles.Count)) -> $insPath" -ForegroundColor White
$r = Invoke-FTPSUpload $config.ftps $insFiles $insPath $entityName $pracName
$insResult.Ok += $r.Ok; $insResult.Fail += $r.Fail
}
}
if ($ptStmts.Count -gt 0) {
if ([string]::IsNullOrWhiteSpace([string]$config.ftps.pdf_path)) {
Write-Host " ftps.pdf_path is null/blank - $($ptStmts.Count) PT STMT file(s) ignored, left in place." -ForegroundColor Yellow
Write-Info "IGNORE entity=$entityName practice=$pracName reason=pdf_path null count=$($ptStmts.Count)"
$ignored += $ptStmts.Count
} else {
Write-Host " PT STMT ($($ptStmts.Count)) -> $pdfPath" -ForegroundColor White
$r = Invoke-FTPSUpload $config.ftps $ptStmts $pdfPath $entityName $pracName
$insResult.Ok += $r.Ok; $insResult.Fail += $r.Fail
}
}
$totalUploaded = $insFiles.Count + $ptStmts.Count - $ignored
if ($insResult.Fail -eq 0 -and $totalUploaded -gt 0 -and $ignored -eq 0) {
$archive = Get-ArchiveDir $practice.FullName "Archive sent to FTP"
Move-Item -Path $dateDir -Destination (Join-Path $archive $today) -Force
Write-Host " $today\ archived." -ForegroundColor Green
Write-Info "ARCHIVE entity=$entityName practice=$pracName folder=$today"
} elseif ($insResult.Fail -gt 0) {
Write-Host " $($insResult.Fail) failed - $today\ left for retry." -ForegroundColor Yellow
} elseif ($totalUploaded -gt 0 -and $ignored -gt 0) {
Write-Host " $today\ NOT archived - $ignored file(s) were ignored (unconfigured path)." -ForegroundColor Yellow
Write-Info "NOARCHIVE entity=$entityName practice=$pracName reason=$ignored ignored files"
} elseif ($insFiles.Count + $ptStmts.Count -eq 0) {
Write-Host " No .$insuranceExt files in $today\." -ForegroundColor Gray
Write-Info "SKIP entity=$entityName practice=$pracName reason=no .$insuranceExt files in date folder"
}
$totalOk += $insResult.Ok; $totalFail += $insResult.Fail
}
}
# -- Patient files -> SFTP -------------------------------------------------
if ($workflow -eq "insurance+patient") {
$patientExport = Join-Path $practice.FullName "Patient\Export"
if (-not (Test-Path $patientExport)) {
Write-Host " No Patient\Export folder - skipping patient." -ForegroundColor Gray
Write-Info "SKIP entity=$entityName practice=$pracName reason=no Patient\Export"
} else {
$ptFolder = Get-ChildItem -Path $patientExport -Directory |
Where-Object { $_.Name -imatch "PT STMT_$today" } |
Select-Object -First 1
if (-not $ptFolder) {
Write-Host " No PT STMT_$today folder - skipping patient." -ForegroundColor Gray
Write-Info "SKIP entity=$entityName practice=$pracName reason=no PT STMT_$today folder"
} else {
$patFiles = Get-ChildItem -Path $ptFolder.FullName -File |
Where-Object { $_.Extension -imatch '\.(txt|tif|tiff)$' }
if ($patFiles.Count -eq 0) {
Write-Host " No txt/tif files in $($ptFolder.Name)\ - skipping." -ForegroundColor Gray
Write-Info "SKIP entity=$entityName practice=$pracName reason=no txt/tif files"
} elseif (($sftpMissing = Get-MissingFields $config.sftp "sftp" @("host", "username", "password", "patient_path")).Count -gt 0) {
Write-Host " Patient upload not configured ($($sftpMissing -join ', ') is null/blank) - $($patFiles.Count) file(s) ignored, left in place." -ForegroundColor Yellow
Write-Info "IGNORE entity=$entityName practice=$pracName reason=unconfigured fields=$($sftpMissing -join ',')"
} else {
$patPath = Expand-Path $config.sftp.patient_path $pracName
Write-Host " Patient ($($patFiles.Count)) -> $patPath" -ForegroundColor White
$r = Invoke-SFTPUpload $config.sftp $patFiles $patPath $entityName $pracName
if ($r.Fail -eq 0) {
$patArchive = Get-ArchiveDir $patientExport "Archive Sent to FTP"
Move-Item -Path $ptFolder.FullName -Destination (Join-Path $patArchive $ptFolder.Name) -Force
Write-Host " $($ptFolder.Name)\ archived." -ForegroundColor Green
Write-Info "ARCHIVE entity=$entityName practice=$pracName folder=$($ptFolder.Name)"
} else {
Write-Host " $($r.Fail) failed - left for retry." -ForegroundColor Yellow
}
$totalOk += $r.Ok; $totalFail += $r.Fail
}
}
}
}
}
Write-Host ""
Write-Host ("=" * 60)
Write-Info "=== END entity=$entityName ok=$totalOk fail=$totalFail ==="
if ($totalFail -eq 0) {
Write-Host "Done: $totalOk file(s) uploaded for $entityName." -ForegroundColor Green
} else {
Write-Host "Done: $totalOk uploaded, $totalFail failed. See Logs\$today.log" -ForegroundColor Yellow
}
exit $(if ($totalFail -gt 0) { 1 } else { 0 })