Extended FileDataProcessing pipeline with Metadata

- fixed problem with output for large files
- added WebClient to better handle large files
- variable cleanup for better memory usage

Author: alan-null

Slack.Backup/Private/backup/Invoke-FilesBackup.ps1

@@ -29,8 +29,9 @@ function Invoke-FilesBackup {
                 $ext = $f.filetype
                 $path = "$backupLoc/$($f.id)"
                 $slackFile = Get-SlackFile -Token $Token -Uri $f.url_private
-                $slackFile = Invoke-FileDataProcessingPipeline -SlackFile $slackFile
+                $slackFile = Invoke-FileDataProcessingPipeline -SlackFile $slackFile -Metadata $f
                 [System.IO.File]::WriteAllBytes("$path.$ext", $slackFile)
+                Remove-Variable -Name slackFile
                 $f | Invoke-FileProcessingPipeline | ConvertTo-Json -Depth 10 | Set-Content -Path "$path.json"
             }
         }

Slack.Backup/Private/pipelines/Invoke-FileDataProcessingPipeline.ps1

@@ -1,12 +1,14 @@
 function Invoke-FileDataProcessingPipeline {
     param (
         [Parameter(Mandatory = $true, Position = 0, ValueFromPipeline = $true )]
-        [byte[]]$SlackFile
+        [byte[]]$SlackFile,
+        [Parameter(Mandatory = $true, Position = 1 )]
+        [System.Object]$Metadata
     )
     process {
         Get-PipelineProcessor -Name Invoke-FileDataProcessor* -Parameters @( @{Name = "SlackFile"; Type = [byte[]] } ) | % {
             Write-Verbose "Invoking processor: $($_.Name)"
-            $SlackFile = &($_.Name) -SlackFile $SlackFile
+            $SlackFile = &($_.Name) -SlackFile $SlackFile -Metadata $Metadata
         }
         $SlackFile
     }

Slack.Backup/Public/Get-SlackFile.ps1

@@ -1,3 +1,16 @@
+class Net {
+    hidden static [Net] $_instance = [Net]::new()
+    static [Net] $Instance = [Net]::GetInstance()
+
+
+    [Net.WebClient]$WebClient = [Net.WebClient]::new()
+
+    hidden Net() {}
+
+    hidden static [Net] GetInstance() {
+        return [Net]::_instance
+    }
+}
 function Get-SlackFile {
     [CmdletBinding()]
     param (
@@ -18,8 +31,14 @@ function Get-SlackFile {
         $headers = Get-RequestHeader $Token
         if ($OutFile) {
             Invoke-WebRequest -Method Get -Uri $Uri -Headers $headers -OutFile $OutFile
-        }else{
-            Invoke-WebRequest -Method Get -Uri $Uri -Headers $headers | Select-Object -ExpandProperty Content
+        }
+        else {
+            [Net]::Instance.WebClient.Headers.Clear()
+            $headers.Keys | % {
+                [Net]::Instance.WebClient.Headers.Add($_, $headers[$_])
+            }
+            [byte[]]$response = [Net]::Instance.WebClient.DownloadData($Uri)
+            Write-Output $response -NoEnumerate
         }
     }
 

Tests/Slack.Backup.Tests.ps1

@@ -10,7 +10,9 @@ $script:processorInvoked = $false
 function Invoke-FileDataProcessorMock {
     param (
         [Parameter(Mandatory = $true, Position = 0, ValueFromPipeline = $true )]
-        [byte[]]$SlackFile
+        [byte[]]$SlackFile,
+        [Parameter(Mandatory = $true, Position = 1 )]
+        [System.Object]$Metadata
     )
     $script:processorInvoked = $true
     $SlackFile