1: param
2: (
3: [string] $SaveAsPath = $(throw "`$SaveAsPath is a required Parameter. Please specify a path to a folder where your files will be saved!") ,
4: [string] $ContentManifestPath = $(throw "`$ContentManifestPath is a required Parameter. Please specify a path to the PDC2008ContentManifest.csv file!") ,
5: [string] $URLMappingPath = $(throw "`$URLMappingPath is a required Parameter. Please specify a path to the PDC2008URLMapping.xml file!") ,
6: [string[]] $SessionIds ,
7: [string[]] $ContentFilter ,
8: [string[]] $TrackFilter ,
9: [switch] $Force ,
10: [switch] $WhatIf
11: )
12:
13:
14:
15: #Variables in script scope
16: $SessionList = New-Object System.Collections.ArrayList # List of explicitly provided sessions ids via SessionIds parameter
17: $HasExlicitSessionIDs = $false
18: $ContentManifest = $null # Stores FileInfo objects of all downloadable content files
19: $ContentDictionary = @{} # Hash table built from the original content manifest 20: $DownloadQueue = @{} # This is a hash table mapping a file name to its FileInfo Object. 21: # These represent the files that need to be actually downloaded after applying all filters.
22: $URLMappings = @{} # This is a hash table that maps file extensions to their download URLs 23: [string[]] $ContentTracks = $null # Array of strings with distinct track ids e.g. "PS", "TL", "BB", etc.
24: [string[]] $ContentTypes = $null # Array of strings with distinct content types e.g. "PPTX", "ZIP", "WMV" etc.
25: $RemoveList = $null # Array of objects that need to be removed form the DownloadQueue
26:
27:
28: function Validate-Parameters
29: { 30: if ($(Test-Path $SaveAsPath) -eq $false)
31: { 32: throw "Save as path: `"{0}`" not found" -f $SaveAsPath 33: }
34:
35: if ($SessionIds -ne $null)
36: { 37: Write "Sessions have been explicitly specified!"
38: $script:HasExlicitSessionIDs = $true
39: foreach ($s in $SessionIds) { [void] ($script:SessionList.Add($s))} 40: }
41: else
42: { 43: Write "No explicit sessions specified!"
44: $script:SessionList.Clear()
45: $script:HasExlicitSessionIDs = $false
46: }
47:
48: # Get a list of tracks and extensions
49: # This is needed here to validate the ContentFilter and TrackFilter parameters
50: Analyze-ContentManifest $script:ContentManifest
51: ValidateTrackFilter -ValidTracks $Script:ContentTracks -AskedForTracks $TrackFilter
52: ValidateContentFilter -ValidTypes $Script:ContentTypes -AskedForTypes $ContentFilter
53:
54: }
55:
56: function ValidateTrackFilter([string[]] $ValidTracks=$Script:ContentTracks, [string[]] $AskedForTracks=$TrackFilter)
57: { 58: if ($ValidTracks -eq $null)
59: { 60: throw "ValidateTrackFilter failed, because `$ValidTracks is null!"
61: }
62:
63: if ($AskedForTracks -ne $null)
64: { 65: VerifyAllStringsAreInCollection -MasterArray $ValidTracks -ContainedArray $AskedForTracks
66: }
67: else
68: { 69: Write-Warning "No TrackFilter specified"
70: }
71: }
72:
73: function ValidateContentFilter([string[]] $ValidTypes=$Script:ContentTypes, [string[]] $AskedForTypes=$ContentFilter)
74: { 75: if ($ValidTypes -eq $null)
76: { 77: throw "ValidateContentFilter failed, because `$ValidTypes is null!"
78: }
79:
80: if ($AskedForTypes -ne $null)
81: { 82: VerifyAllStringsAreInCollection -MasterArray $ValidTypes -ContainedArray $AskedForTypes
83: }
84: else
85: { 86: Write-Warning "No ContentFilter specified"
87: }
88: }
89:
90: function VerifyAllStringsAreInCollection( [object[]] $MasterArray, [object[]] $ContainedArray)
91: { 92: $ContainedArray | foreach-object {if ($MasterArray -inotcontains $_) {throw "$_ is not a valid selection. These selections are valid: $MasterArray"}} 93: }
94:
95:
96: function Load-ContentManifestFromCSV ([string] $CSVPath = $ContentManifestPath)
97: { 98: if (Test-Path $CSVPath)
99: { 100: $script:ContentManifest = Import-Csv -Path $CSVPath
101: }
102: else
103: { 104: throw "CSV file: `"{0}`" not found" -f $CSVPath 105: }
106: BuildContentDictionary -Dictionary $Script:ContentDictionary -Manifest $Script:ContentManifest
107: }
108:
109: function BuildContentDictionary ([Object] $Dictionary=$Script:ContentDictionary , [Object[]] $Manifest=$script:ContentManifest)
110: { 111: if($Manifest -eq $null)
112: { 113: throw "Invalid Manifest!"
114: }
115: # Building hashtable from array
116: $Manifest | ForEach-Object { $Dictionary[ $_.Name ] = $_ } 117:
118: Write "Building ContentDictionary hashtable from ContentManifest array"
119: $Dictionary
120: }
121:
122:
123: function Load-URLMappingFromXML ([string] $XMLPath = $URLMappingPath)
124: { 125: if (Test-Path $XMLPath)
126: { 127: $script:URLMappings = Import-CliXML -Path $XMLPath
128: if ($script:URLMappings -eq $null)
129: { 130: throw "`$URLMappings is not initialized! Check the contents of the file: {0}" -f $XMLPath 131: }
132: }
133: else
134: { 135: throw "XML file: `"{0}`" not found" -f $XMLPath 136: }
137: }
138:
139:
140:
141:
142: function Download-File ([string] $URL, [string] $SaveAsFile, [int] $Length)
143: { 144: $WebDownloader = new-object System.Net.Webclient
145: $StartTime = Get-Date
146: Write ("Download started at {0}" -f $StartTime) 147: Write ("Downloading {0} bytes from {1} and saving file as {2}" -f $Length, $URL, $SaveAsFile) 148:
149: trap { $WebDownloader.Dispose(); throw "$_" } $WebDownloader.DownloadFile( $URL, $SaveAsFile); 150:
151: $EndTime = Get-Date
152: $Duration = $EndTime - $StartTime
153: Write ("Download ended at {0}" -f $EndTime) 154: Write ("Download took {0:F} seconds, which is {1:F} minutes" -f $Duration.TotalSeconds, $Duration.TotalMinutes ) 155: $DownloadedFile = Get-ChildItem $SaveAsFile
156: $Speed = $DownloadedFile.Length / ($Duration.TotalSeconds + 1)
157: $KBRate = $Speed / 1MB
158: Write ("Download rate is {0:F} MB/s" -f $KBRate) 159:
160: if ((Verify-DownloadedFile $SaveAsFile $Length) -eq $false)
161: { 162: Write-Warning ("Download of file {0} is not complete" -f $SaveAsFile) 163: }
164: else
165: { 166: Write "Download completed successfully!"
167: }
168:
169: Write ""
170: Write ""
171: $WebDownloader.Dispose()
172: }
173:
174: function Process-DownloadQueue([Object] $Queue)
175: { 176: if ($Queue -eq $null)
177: { 178: Write "No file in download list. Try again!"
179: }
180: else
181: { 182: $Queue.GetEnumerator() | foreach-object `
183: { ` 184: $Ext = $_.Value.Extension.ToLower(); $Name = $_.Value.Name; $Length = $_.Value.Length ;`
185: trap { write-warning "$_`n`n" ; continue } Download-File -URL $($script:URLMappings[$Ext] + $Name) -SaveAsFile $($SaveAsPath + $Name) -Length $Length; ` 186: }
187: }
188: }
189:
190: function Compile-DownloadQueue
191: { 192: Write "Compiling DownloadQueue"
193: if ($script:HasExlicitSessionIDs -eq $true)
194: { 195: $script:ContentDictionary.GetEnumerator() | ForEach-Object -Process{ foreach( $s in $script:SessionList){ if ($_.Key.Substring(0, $_.Key.IndexOf(".")).ToLower() -eq $s.ToLower()) { $script:DownloadQueue[$_.Key] = $_.Value} }} 196: }
197: else
198: { 199: $script:DownloadQueue = $script:ContentDictionary
200: }
201:
202: if($Force -eq $false)
203: { 204: Remove-DownloadedFilesFromQueue -Queue $script:DownloadQueue -DownloadPath $SaveAsPath
205: }
206:
207: Select-MatchingFilesFromQueue -Queue $script:DownloadQueue -Types $ContentFilter -Tracks $TrackFilter
208: }
209:
210: function Select-MatchingFilesFromQueue([Object] $Queue=$script:DownloadQueue, [String[]] $Types=$ContentTypes , [String[]] $Tracks=$ContentTracks)
211: { 212: Write "Select-MatchingFilesFromQueue"
213:
214: $TrackFilter = ""
215: if ($Tracks -eq $null -or $Tracks.Length -eq 0)
216: { 217: $TrackFilter = ".*?"
218: }
219: elseif ($Tracks.Length -gt 0)
220: { 221: for($i=0; $i -lt $Tracks.Length; $i++)
222: { 223: $TrackFilter += "({0})" -f $Tracks[$i] 224: if( $i -lt ($Tracks.Length - 1))
225: { 226: $TrackFilter += "|"
227: }
228: }
229: }
230:
231: $TypeFilter = ""
232: if ($Types -eq $null -or $Types.Length -eq 0)
233: { 234: $TypeFilter = "\..*"
235: }
236: elseif ($Types.Length -gt 0)
237: { 238: for($i=0; $i -lt $Types.Length; $i++)
239: { 240: $TypeFilter += "(\{0})" -f $Types[$i] 241: if( $i -lt ($Types.Length - 1))
242: { 243: $TypeFilter += "|"
244: }
245: }
246: }
247: Write "TrackFilter Regex part"
248: $TrackFilter
249: Write "TypeFilter Regex part"
250: $TypeFilter
251:
252: $RegexString = "^(?<TRACK>({0}))(?<INDEX>\d{2})(?<EXT>({1}))$" ` 253: -f $TrackFilter, $TypeFilter, "{2}" 254:
255: Write "Queue before filter selection"
256: $Queue
257: $ResultQueue = @{} 258:
259: $Queue.GetEnumerator() | Foreach-Object {if($_.Key -match $RegexString) { $ResultQueue[$_.Key] = $_.Value}} 260:
261: Write "ResultQueue after filter selection"
262: $script:DownloadQueue = $ResultQueue
263: $script:DownloadQueue.GetType().FullName
264: $script:DownloadQueue
265:
266: Write "Types"
267: $Types
268:
269: Write "Tracks"
270: $Tracks
271: }
272:
273: function Remove-DownloadedFilesFromQueue ([Object] $Queue , [string] $DownloadPath)
274: { 275: Write "Remove-DownloadedFilesFromQueue"
276: if(( $Queue -ne $null) -and (Test-Path $SaveAsPath))
277: { 278: Compare-FileList -Reference $Queue -FilePath $DownloadPath
279: Remove-FilesFromQueue -Current $Queue -Delta $Script:RemoveList
280: }
281: }
282:
283: function Compare-FileList ([Object] $Reference , [string] $FilePath)
284: { 285: Write "Comparing Files Lists"
286: if (Test-Path $FilePath )
287: { 288: Set-Location $FilePath
289:
290: $LocalFiles = dir | Select-Object Name, Length | Sort-Object Name
291: $QueuedFiles = $Reference.Values | Select-Object Name, Length | Sort-Object Name
292:
293: Write "Files in Queue"
294: $QueuedFiles
295:
296: Write "Local Files"
297: $LocalFiles
298:
299: if ($LocalFiles -ne $null)
300: { 301: # Files and their length that have matches in both lists
302: $SyncRange = 100
303: if ($Script:ContentDictionary.Count -gt 100 )
304: { 305: $SyncRange = $Script:ContentDictionary.Count
306: }
307: Write ("SyncWindow {0}" -f $SyncRange) 308: $DownloadedFiles = Compare-Object -ReferenceObject $QueuedFiles -DifferenceObject $LocalFiles -Property Name, Length -IncludeEqual -ExcludeDifferent -PassThru -SyncWindow $SyncRange
309: $Script:RemoveList = $DownloadedFiles
310: Write "Files that don't need to be downloaded anymore, because they are already downloaded!"
311: $Script:RemoveList
312: }
313: else
314: { 315: $Script:RemoveList = $null
316: }
317: }
318: else
319: { 320: throw "DifferenceFilePath: `"{0}`" not found" -f $DifferenceFilePath 321: }
322: }
323:
324: function Verify-DownloadedFile ([string] $FileName, [int] $ExpectedFileLength )
325: { 326: if(Test-Path $FileName) `
327: { ` 328: $File = dir $FileName; `
329: if( $File -ne $Null) `
330: { ` 331: if ($File.Length -eq $ExpectedFileLength) `
332: { ` 333: return $true `
334: } `
335: else `
336: { ` 337: return $false `
338: } `
339: } `
340: else `
341: { ` 342: return $false `
343: } `
344: } `
345: else `
346: { ` 347: return $false `
348: } `
349: }
350:
351: function Analyze-ContentManifest([Object[]] $Manifest = $script:ContentManifest)
352: { 353: if($Manifest -ne $null)
354: { 355: $Manifest | foreach-object `
356: -Begin {"Analyzing Content Manifest"} ` 357: -Process { [Void] ($_.Name -match '^(?<TRACK>.*?)(?<INDEX>\d{2})(?<EXT>\..*$)'); $script:ContentTracks += $matches.TRACK.ToUpper(); $script:ContentTypes += $matches.EXT.ToUpper(); } ` 358: -End { $script:ContentTracks = $script:ContentTracks | sort-object -unique; $script:ContentTypes = $script:ContentTypes | sort-object -unique; } 359:
360: Write "ContentTracks"
361: $script:ContentTracks;
362: Write "ContentTypes"
363: $script:ContentTypes;
364: }
365: }
366:
367: function Remove-FilesFromQueue ([Object] $Current=$Script:DownloadQueue, [Object[]] $Delta )
368: { 369: if( $Delta -eq $null -or $Delta.count -eq 0)
370: { 371: Write-Warning "Removing files from Download Queue. No files to remove!"
372: return
373: }
374: if( $Current -eq $null)
375: { 376: throw "Removing files from Download Queue. Queue does not exist!"
377: }
378:
379: if( $Current.count -eq 0)
380: { 381: Write-Warning "Removing files from Download Queue. Queue is empty!"
382: return
383: }
384:
385: Write "Remove-FilesFromQueue"
386: Write "Files to remove"
387: $Delta
388:
389: Write "Queue before removal"
390: $Current
391:
392: $Delta | ForEach-Object -Process{ $Current.Remove( $_.Name )} 393:
394: Write "Queue after removal"
395: $Current
396: }
397:
398: Load-ContentManifestFromCSV
399: Load-URLMappingFromXML
400: Validate-Parameters
401: Compile-DownloadQueue
402: if ($WhatIf -eq $false)
403: { 404: Process-DownloadQueue -Queue $script:DownloadQueue
405: }
Subscribe
