Add central logging with normal/debug levels in Logs\YYYYMMDD.log

engine.ps1

@@ -8,16 +8,56 @@ $scriptRoot = Split-Path -Parent $PSCommandPath
 $winscpDll  = Join-Path $scriptRoot "WinSCP\WinSCPnet.dll"
 $today      = Get-Date -Format "yyyyMMdd"
 
+# ── Logging setup ────────────────────────────────────────────────────────────
+$logsDir    = Join-Path $scriptRoot "Logs"
+if (-not (Test-Path $logsDir)) { New-Item -ItemType Directory -Path $logsDir | Out-Null }
+$logFile    = Join-Path $logsDir "$today.log"
+
+$settingsFile = Join-Path $scriptRoot "settings.json"
+$logLevel   = "normal"
+if (Test-Path $settingsFile) {
+    $settings = Get-Content $settingsFile -Raw | ConvertFrom-Json
+    if ($settings.log_level) { $logLevel = $settings.log_level.ToLower() }
+}
+
+function Write-Log($level, $message) {
+    $ts   = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
+    $line = "[$ts] [$($level.ToUpper())] $message"
+    Add-Content -Path $logFile -Value $line
+    if ($level -eq "error") {
+        Write-Host $line -ForegroundColor Red
+    } elseif ($logLevel -eq "debug") {
+        Write-Host $line -ForegroundColor DarkGray
+    }
+}
+
+function Write-Info($message) {
+    $ts   = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
+    $line = "[$ts] [INFO ] $message"
+    Add-Content -Path $logFile -Value $line
+}
+
+function Write-Debug($message) {
+    if ($logLevel -eq "debug") {
+        $ts   = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
+        $line = "[$ts] [DEBUG] $message"
+        Add-Content -Path $logFile -Value $line
+    }
+}
+
+# ── WinSCP ───────────────────────────────────────────────────────────────────
 if (-not (Test-Path $winscpDll)) {
-    Write-Host ""
     Write-Host "ERROR: WinSCP not found. Run setup.bat first." -ForegroundColor Red
+    Write-Log "error" "WinSCP DLL not found at $winscpDll"
     exit 1
 }
 Add-Type -Path $winscpDll
 
+# ── Entity config ─────────────────────────────────────────────────────────────
 $entityConfigFile = Join-Path $EntityDir "entity.json"
 if (-not (Test-Path $entityConfigFile)) {
     Write-Host "ERROR: entity.json not found in $EntityDir" -ForegroundColor Red
+    Write-Log "error" "entity.json not found in $EntityDir"
     exit 1
 }
 
@@ -28,18 +68,19 @@ $entityName   = Split-Path -Leaf $EntityDir
 
 if (-not (Test-Path $export1)) {
     Write-Host "ERROR: Export1 folder not found in $EntityDir" -ForegroundColor Red
+    Write-Log "error" "Export1 not found in $EntityDir"
     exit 1
 }
 
 Write-Host ""
-Write-Host "Entity: $entityName | Workflow: $workflow | Date: $today" -ForegroundColor Cyan
+Write-Host "Entity: $entityName | Workflow: $workflow | Date: $today | Log: $logLevel" -ForegroundColor Cyan
 Write-Host ("=" * 60)
+Write-Info "=== START entity=$entityName workflow=$workflow ==="
 
-# Merge a practice-level override on top of the entity config
+# ── Helpers ──────────────────────────────────────────────────────────────────
 function Merge-Config($base, $override) {
     if (-not $override) { return $base }
-    $json   = $base | ConvertTo-Json -Depth 10
+    $merged = $base | ConvertTo-Json -Depth 10 | ConvertFrom-Json
-    $merged = $json | ConvertFrom-Json
     foreach ($section in $override.PSObject.Properties) {
         if ($null -ne $merged.PSObject.Properties[$section.Name]) {
             foreach ($prop in $section.Value.PSObject.Properties) {
@@ -50,24 +91,21 @@ function Merge-Config($base, $override) {
     return $merged
 }
 
-# Replace {practice} token in a remote path template
 function Expand-Path($template, $practice) {
     return $template -replace '\{practice\}', $practice
 }
 
-# Find archive subfolder by name (case-insensitive), create if missing
 function Get-ArchiveDir($parent, $name) {
     $found = Get-ChildItem -Path $parent -Directory |
-             Where-Object { $_.Name -ieq $name } |
+             Where-Object { $_.Name -ieq $name } | Select-Object -First 1
-             Select-Object -First 1
     if ($found) { return $found.FullName }
     $path = Join-Path $parent $name
     New-Item -ItemType Directory -Path $path | Out-Null
     return $path
 }
 
-# Upload a list of files to a remote path via FTPS/FTP using WinSCP
+# ── Upload via FTPS ───────────────────────────────────────────────────────────
-function Invoke-FTPSUpload($ftpConfig, $files, $remotePath) {
+function Invoke-FTPSUpload($ftpConfig, $files, $remotePath, $entity, $practice) {
     $opts = New-Object WinSCP.SessionOptions
     $opts.Protocol    = [WinSCP.Protocol]::Ftp
     $opts.FtpSecure   = if ($ftpConfig.tls) { [WinSCP.FtpSecure]::Explicit } else { [WinSCP.FtpSecure]::None }
@@ -78,8 +116,13 @@ function Invoke-FTPSUpload($ftpConfig, $files, $remotePath) {
     $opts.GiveUpSecurityAndAcceptAnyTlsHostCertificate = $true
 
     $session = New-Object WinSCP.Session
+    if ($logLevel -eq "debug") {
+        $session.SessionLogPath = Join-Path $logsDir "${today}_winscp_debug.log"
+    }
+
     $ok = 0; $fail = 0
     try {
+        Write-Debug "FTPS connect: $($ftpConfig.host) user=$($ftpConfig.username) tls=$($ftpConfig.tls)"
         $session.Open($opts)
         foreach ($file in $files) {
             $remote = $remotePath.TrimEnd("/") + "/" + $file.Name
@@ -87,20 +130,28 @@ function Invoke-FTPSUpload($ftpConfig, $files, $remotePath) {
                 $result = $session.PutFiles($file.FullName, $remote)
                 $result.Check()
                 Write-Host "      $($file.Name) ... ok" -ForegroundColor Green
+                Write-Info "FTPS OK   entity=$entity practice=$practice file=$($file.Name) remote=$remote"
                 $ok++
             } catch {
-                Write-Host "      $($file.Name) ... FAILED: $($_.Exception.Message)" -ForegroundColor Red
+                $err = $_.Exception.Message
+                Write-Host "      $($file.Name) ... FAILED" -ForegroundColor Red
+                Write-Log "error" "FTPS FAIL entity=$entity practice=$practice file=$($file.Name) remote=$remote error=$err"
                 $fail++
             }
         }
+    } catch {
+        $err = $_.Exception.Message
+        Write-Log "error" "FTPS connect FAILED entity=$entity practice=$practice host=$($ftpConfig.host) error=$err"
+        Write-Host "    Connection failed: $err" -ForegroundColor Red
+        $fail += $files.Count
     } finally {
         $session.Dispose()
     }
     return @{ Ok = $ok; Fail = $fail }
 }
 
-# Upload a list of files to a remote path via SFTP using WinSCP
+# ── Upload via SFTP ───────────────────────────────────────────────────────────
-function Invoke-SFTPUpload($sftpConfig, $files, $remotePath) {
+function Invoke-SFTPUpload($sftpConfig, $files, $remotePath, $entity, $practice) {
     $opts = New-Object WinSCP.SessionOptions
     $opts.Protocol   = [WinSCP.Protocol]::Sftp
     $opts.HostName   = $sftpConfig.host
@@ -110,8 +161,13 @@ function Invoke-SFTPUpload($sftpConfig, $files, $remotePath) {
     $opts.GiveUpSecurityAndAcceptAnySshHostKey = $true
 
     $session = New-Object WinSCP.Session
+    if ($logLevel -eq "debug") {
+        $session.SessionLogPath = Join-Path $logsDir "${today}_winscp_debug.log"
+    }
+
     $ok = 0; $fail = 0
     try {
+        Write-Debug "SFTP connect: $($sftpConfig.host) user=$($sftpConfig.username)"
         $session.Open($opts)
         foreach ($file in $files) {
             $remote = $remotePath.TrimEnd("/") + "/" + $file.Name
@@ -119,23 +175,33 @@ function Invoke-SFTPUpload($sftpConfig, $files, $remotePath) {
                 $result = $session.PutFiles($file.FullName, $remote)
                 $result.Check()
                 Write-Host "      $($file.Name) ... ok" -ForegroundColor Green
+                Write-Info "SFTP OK   entity=$entity practice=$practice file=$($file.Name) remote=$remote"
                 $ok++
             } catch {
-                Write-Host "      $($file.Name) ... FAILED: $($_.Exception.Message)" -ForegroundColor Red
+                $err = $_.Exception.Message
+                Write-Host "      $($file.Name) ... FAILED" -ForegroundColor Red
+                Write-Log "error" "SFTP FAIL entity=$entity practice=$practice file=$($file.Name) remote=$remote error=$err"
                 $fail++
             }
         }
+    } catch {
+        $err = $_.Exception.Message
+        Write-Log "error" "SFTP connect FAILED entity=$entity practice=$practice host=$($sftpConfig.host) error=$err"
+        Write-Host "    Connection failed: $err" -ForegroundColor Red
+        $fail += $files.Count
     } finally {
         $session.Dispose()
     }
     return @{ Ok = $ok; Fail = $fail }
 }
 
+# ── Main loop ─────────────────────────────────────────────────────────────────
 $totalOk = 0; $totalFail = 0
 $practices = Get-ChildItem -Path $export1 -Directory
 
 if ($practices.Count -eq 0) {
     Write-Host "No practice folders found in Export1." -ForegroundColor Yellow
+    Write-Info "No practice folders found in $export1"
     exit 0
 }
 
@@ -143,16 +209,18 @@ foreach ($practice in $practices) {
     $pracName = $practice.Name
     Write-Host ""
     Write-Host "  [ $pracName ]" -ForegroundColor Cyan
+    Write-Info "--- practice=$pracName ---"
 
-    # Load optional practice-level credential override
     $overrideFile = Join-Path $practice.FullName "practice.json"
     $override     = if (Test-Path $overrideFile) { Get-Content $overrideFile -Raw | ConvertFrom-Json } else { $null }
     $config       = Merge-Config $entityConfig $override
+    if ($override) { Write-Debug "practice.json override loaded for $pracName" }
 
-    # ── INSURANCE + PT STMT → FTPS ──────────────────────────────────────────
+    # ── Insurance + PT STMT → FTPS ───────────────────────────────────────────
     $dateDir = Join-Path $practice.FullName $today
     if (-not (Test-Path $dateDir)) {
-        Write-Host "    No $today\ folder — skipping insurance upload." -ForegroundColor Gray
+        Write-Host "    No $today\ folder — skipping insurance." -ForegroundColor Gray
+        Write-Info "SKIP entity=$entityName practice=$pracName reason=no date folder"
     } else {
         $allPdfs  = Get-ChildItem -Path $dateDir -Filter "*.pdf" -File
         $ptStmts  = $allPdfs | Where-Object { $_.Name -imatch 'pt[\s_]*stmt' }
@@ -164,12 +232,12 @@ foreach ($practice in $practices) {
 
         if ($insFiles.Count -gt 0) {
             Write-Host "    Insurance ($($insFiles.Count)) → $insPath" -ForegroundColor White
-            $r = Invoke-FTPSUpload $config.ftps $insFiles $insPath
+            $r = Invoke-FTPSUpload $config.ftps $insFiles $insPath $entityName $pracName
             $insResult.Ok += $r.Ok; $insResult.Fail += $r.Fail
         }
         if ($ptStmts.Count -gt 0) {
             Write-Host "    PT STMT PDF ($($ptStmts.Count)) → $pdfPath" -ForegroundColor White
-            $r = Invoke-FTPSUpload $config.ftps $ptStmts $pdfPath
+            $r = Invoke-FTPSUpload $config.ftps $ptStmts $pdfPath $entityName $pracName
             $insResult.Ok += $r.Ok; $insResult.Fail += $r.Fail
         }
 
@@ -178,44 +246,50 @@ foreach ($practice in $practices) {
             $archive = Get-ArchiveDir $practice.FullName "Archive sent to FTP"
             Move-Item -Path $dateDir -Destination (Join-Path $archive $today) -Force
             Write-Host "    $today\ archived." -ForegroundColor Green
+            Write-Info "ARCHIVE entity=$entityName practice=$pracName folder=$today"
         } elseif ($insResult.Fail -gt 0) {
-            Write-Host "    $($insResult.Fail) failed — $today\ left in place for retry." -ForegroundColor Yellow
+            Write-Host "    $($insResult.Fail) failed — $today\ left for retry." -ForegroundColor Yellow
         } elseif ($totalUploaded -eq 0) {
-            Write-Host "    No PDFs found in $today\." -ForegroundColor Gray
+            Write-Host "    No PDFs in $today\." -ForegroundColor Gray
+            Write-Info "SKIP entity=$entityName practice=$pracName reason=no PDFs in date folder"
         }
 
         $totalOk += $insResult.Ok; $totalFail += $insResult.Fail
     }
 
-    # ── PATIENT FILES → SFTP (only if workflow = insurance+patient) ──────────
+    # ── Patient files → SFTP ─────────────────────────────────────────────────
     if ($workflow -eq "insurance+patient") {
         $patientExport = Join-Path $practice.FullName "Patient\Export"
         if (-not (Test-Path $patientExport)) {
-            Write-Host "    No Patient\Export folder — skipping patient upload." -ForegroundColor Gray
+            Write-Host "    No Patient\Export folder — skipping patient." -ForegroundColor Gray
+            Write-Info "SKIP entity=$entityName practice=$pracName reason=no Patient\Export"
         } else {
             $ptFolder = Get-ChildItem -Path $patientExport -Directory |
                         Where-Object { $_.Name -imatch "PT STMT_$today" } |
                         Select-Object -First 1
 
             if (-not $ptFolder) {
-                Write-Host "    No PT STMT_$today folder — skipping patient upload." -ForegroundColor Gray
+                Write-Host "    No PT STMT_$today folder — skipping patient." -ForegroundColor Gray
+                Write-Info "SKIP entity=$entityName practice=$pracName reason=no PT STMT_$today folder"
             } else {
                 $patFiles = Get-ChildItem -Path $ptFolder.FullName -File |
                             Where-Object { $_.Extension -imatch '\.(txt|tif|tiff)$' }
 
                 if ($patFiles.Count -eq 0) {
                     Write-Host "    No txt/tif files in $($ptFolder.Name)\ — skipping." -ForegroundColor Gray
+                    Write-Info "SKIP entity=$entityName practice=$pracName reason=no txt/tif files"
                 } else {
                     $patPath = Expand-Path $config.sftp.patient_path $pracName
                     Write-Host "    Patient ($($patFiles.Count)) → $patPath" -ForegroundColor White
-                    $r = Invoke-SFTPUpload $config.sftp $patFiles $patPath
+                    $r = Invoke-SFTPUpload $config.sftp $patFiles $patPath $entityName $pracName
 
                     if ($r.Fail -eq 0) {
                         $patArchive = Get-ArchiveDir $patientExport "Archive Sent to FTP"
                         Move-Item -Path $ptFolder.FullName -Destination (Join-Path $patArchive $ptFolder.Name) -Force
                         Write-Host "    $($ptFolder.Name)\ archived." -ForegroundColor Green
+                        Write-Info "ARCHIVE entity=$entityName practice=$pracName folder=$($ptFolder.Name)"
                     } else {
-                        Write-Host "    $($r.Fail) failed — left in place for retry." -ForegroundColor Yellow
+                        Write-Host "    $($r.Fail) failed — left for retry." -ForegroundColor Yellow
                     }
 
                     $totalOk += $r.Ok; $totalFail += $r.Fail
@@ -227,10 +301,12 @@ foreach ($practice in $practices) {
 
 Write-Host ""
 Write-Host ("=" * 60)
+Write-Info "=== END entity=$entityName ok=$totalOk fail=$totalFail ==="
+
 if ($totalFail -eq 0) {
     Write-Host "Done: $totalOk file(s) uploaded for $entityName." -ForegroundColor Green
 } else {
-    Write-Host "Done: $totalOk uploaded, $totalFail failed." -ForegroundColor Yellow
+    Write-Host "Done: $totalOk uploaded, $totalFail failed. See Logs\$today.log" -ForegroundColor Yellow
 }
 
 exit $(if ($totalFail -gt 0) { 1 } else { 0 })

settings.json

@@ -0,0 +1,3 @@
+{
+  "log_level": "normal"
+}