<# Purpose: Deploys PDQ Packs Requirements: 1. Expects master copy directory to look like this: \job_files \repository changelog-vX.Y.X-updated-yyyy-mm-dd.txt Instructions -- read this.txt Roughly what it should look like.png 2. Expects master changelog file to have accurate "PACK_DATE=yyyy-mm-dd" and "PACK_VERSION=x.y.z" strings because these are parsed and used to name everything correctly 3. Expects seed server directory structure to look like this: \resiliosync\pdq_pack \pdq_pack \job_files \repository changelog-vX.Y.Z-updated-YYYY-MM-DD.txt Instructions -- read this.txt Roughly what it should look like.png \integrity_verification checksums.txt checksums.txt.asc vocatus-public-key.asc Author: reddit.com/user/vocatus ( vocatus.gate@gmail.com ) // PGP key: 0x07d1490f82a211a2 Version: 1.0.2 + Add command to nuke any existing "UPLOADING" file in the temp directory. Was causing upload script to upload the wrong file - Remove -speed=350 command because Cox stopped being stupid 1.0.1 + Add -speed=350 (KB) command to WinSCP FTP upload script because Cox is stupid and auto-kills any FTP upload that goes above a certain rate + add -resume command to WinSCP FTP upload script for the new binary exe only 1.0.0 . Initial write, fork of deploy_tron.ps1 Behavior/steps: 1. Delete content from seed server 2. Calculate sha256 hashes of all files in \pdq_pack\pdq_pack 3. PGP-sign checksums.txt 3. Make .torrent file, save locally and upload to the autoloader folder 4. Create binary pack 5. PGP-sign the binary pack 6. Background upload the binary pack to the seed server 7. Fetch sha256sums.txt from repo and update it with sha256sum of the new binary pack 8. Delete current version from repo server; upload .exe pack and sha256sums.txt to repo server; cleans up residual temp files; notifies of completion and advises to restart BT Sync 9. Build FTP upload script 10. Upload via FTP 11. Clean up (deletes temp files used) #> ############# # VARIABLES # ---------------------- Set these to match your environment ------------------------ # ############# # Rules for variables: # * NO quotes! (bad: "c:\directory\path" ) # * NO trailing slashes on paths! (bad: c:\directory\ ) # * Spaces are okay (okay: c:\my folder\with spaces ) # * Network paths are okay (okay: \\server\share name ) param ( # Logging information [string]$logpath = "$env:systemdrive\Logs", [string]$logfile = "deploy_pdq_packs.log", # Path to 7z.exe [string]$SevenZip = "C:\Program Files\7-Zip\7z.exe", # Path to WinSCP.com [string]$WinSCP = "R:\applications\WinSCP\WinSCP.com", # Path to hashdeep.exe [string]$HashDeep = "R:\utilities\cli_utils\hashdeep.exe", # e.g. "$env:SystemRoot\system32\hashdeep.exe" # Path to gpg.exe (for signing) [string]$gpg = "${env:ProgramFiles(x86)}\GnuPG\bin\gpg.exe", # e.g. "$env:ProgramFiles\gpg4win\bin\gpg.exe" # Path to mktorrent.exe (for .torrent generation) [string]$mktorrent = "R:\applications\mktorrent\mktorrent.exe", # e.g. "R:\applications\mktorrent\mktorrent.exe" # List of torrent trackers to pack into the .torrent [string]$TorrentTracker1 = "udp://tracker.coppersurfer.tk:6969/announce", [string]$TorrentTracker2 = "udp://tracker.ccc.de:80/announce", [string]$TorrentTracker3 = "udp://tracker.publicbt.com:80", [string]$TorrentTracker4 = "udp://tracker.istole.it:80", [string]$TorrentTracker5 = "http://tracker.openbittorrent.com:80/announce", [string]$TorrentTracker6 = "http://tracker.ipv6tracker.org:80/announce", [string]$TorrentTracker7 = "http://9.rarbg.com:2710/announce", # Master copy. Directory path, not file path. This directory should contain \pdq_pack and \integrity_verification subfolders # Cygwin format version is required for mktorrent.exe. This directory should contain \pdq_pack and \integrity_verification subfolders [string]$MasterCopy = "R:\utilities\pdq_pack", # e.g. "r:\utilities\security\cleanup-repair\pdq_pack" [string]$MasterCopyCygwinFormat = "/cygdrive/r/utilities/pdq_pack", # e.g. "/cygdrive/r/utilities/security/cleanup-repair/pdq_pack" # .torrent output folder, e.g. where to spit out the generated .torrent file. Supply path # in both standard Windows format and in cygwin format. No trailing slash [string]$TorrentSaveLocation = "R:\documents\logs\torrents", # e.g. "R:\documents\logs\torrents" [string]$TorrentSaveLocationCygwinFormat = "/cygdrive/r/documents/logs/torrents", # e.g. "/cygdrive/r/unsorted" # Server holding the seed directories [string]$SeedServer = "\\thebrain", # e.g. "\\thebrain" # Torrent upload directory. e.g. where to copy the .torrent file so it can be loaded by the Bittorrent software # Relative path to $SeedServer [string]$TorrentAutoloaderLocation = "downloads\torrent_files\autoloader", # e.g. "downloads\torrent_files\autoloader" # Seeding subdirectories containing \repository and \job_files directories (relative paths). No leading or trailing slashes # RELEASE seeds [string]$SeedFolderRS = "downloads\seeders\resiliosync\pdq_pack", # e.g. "downloads\seeders\resiliosync\pdq_pack" [string]$SeedFolderTorrent = "downloads\seeders\torrent", # e.g. "downloads\seeders\torrent" # Static pack storage location. RELATIVE path from root on the # local deployment server. Where we stash the compiled .exe # after uploading to the repo server. # No leading or trailing slashes [string]$StaticPackStorageLocation = "downloads\seeders\static_packs", # e.g. "downloads\seeders\static packs" # Repository server where we'll fetch sha256sums.txt from [string]$Repo_URL = "https://bmrf.org/repos/pdq_packs", # e.g. "http://bmrf.org/repos/pdq_packs" # FTP information for where we'll upload the final sha256sums.txt and "PDQ Pack vX.Y.Z (yyyy-mm-dd).exe" file to [string]$Repo_FTP_Host = "bmrf.org", # e.g. "bmrf.org" [string]$Repo_FTP_Username = "xxx", [string]$Repo_FTP_Password = "xxx", [string]$Repo_FTP_DepositPath = "/public_html/repos/pdq_packs/", # e.g. "/public_html/repos/pdq_packs/" # PGP key authentication information [string]$gpgUsername = "xxx", [string]$gpgPassphrase = "xxx" ) # ----------------------------- Don't edit anything below this line ----------------------------- # ################### # PREP AND CHECKS # ################### $SCRIPT_VERSION = "1.0.2" $SCRIPT_UPDATED = "2018-12-13" $CUR_DATE=get-date -f "yyyy-MM-dd" # The "split" command/method is similar to variable cutting in batch (e.g. %myVar:~3,0%) # Extract version number of current version from the seed server changelog and stash it in $OldVersion $OldVersion = gc $SeedServer\$SeedFolderRS\pdq_pack\changelog* -ea SilentlyContinue | Select-String -pattern "PACK_VERSION" $OldVersion = "$OldVersion".Split("=")[1] # Extract release date of current version from the seed server changelog and stash it in $OldDate $OldDate = gc $SeedServer\$SeedFolderRS\pdq_pack\changelog* -ea SilentlyContinue | Select-String -pattern "PACK_DATE" $OldDate = "$OldDate".Split("=")[1] # Extract version number from master changelog and stash it in $NewVersion, then calculate and store the full .exe name for the binary we'll be building $NewVersion = gc $MasterCopy\pdq_pack\changelog* -ea SilentlyContinue | Select-String -pattern "PACK_VERSION" $NewVersion = "$NewVersion".Split("=")[1] $NewBinary = "PDQ Pack v$NewVersion ($CUR_DATE).exe" ################# # SANITY CHECKS # ################# # List of items to make sure they exist before running the script $pathsToCheck = @( # Local machine: 7z.exe "$SevenZip", # Local machine: WinSCP.com "$WinSCP", # Local machine: hashdeep "$HashDeep", # Master copy: \job_files subfolder "$MasterCopy\pdq_pack\job_files", # Master copy: \repository "$MasterCopy\pdq_pack\repository", # Master copy: changelog "$MasterCopy\pdq_pack\changelog-v$NewVersion-updated-$CUR_DATE.txt", # Master copy: the Instructions file "$MasterCopy\pdq_pack\Instructions -- read this.txt", # Seed server: top level folder (Resilio Sync) "$SeedServer\$SeedFolderRS", # Seed server: \pdq_pack\integrity_verification sub-folder (Resilio Sync) "$SeedServer\$SeedFolderRS\integrity_verification", # Seed server: the public key (BT Sync) "$SeedServer\$SeedFolderRS\integrity_verification\vocatus-public-key.asc" ) # Run the check foreach ($i in $pathstoCheck) { if ( -not (test-path -LiteralPath $i)) { "" write-host -n " ["; write-host -n "ERROR" -f red; write-host -n "]"; write-host " Couldn't find the following required item:" "" write-host " $i" "" write-host " Check paths and permissions and make sure it exists" $null = $Host.UI.RawUI.ReadKey('NoEcho,IncludeKeyDown'); return } } # Are you sure? "" write-host " Build passed sanity checks." -f green "" write-host " About to replace PDQ Pack v$OldVersion ($OldDate) with v$NewVersion ($CUR_DATE)" "" write-host " Are you sure?" -f red "" write-host " Don't forget to input GPG pin at the prompt!" -f red "" pause clear ########### # EXECUTE # ########### # The rest of the script is wrapped in the "main" function. This is literally just so we can put the logging function at the bottom of the script instead of at the top function main() { "" log " PDQ Pack deployment script v$SCRIPT_VERSION" blue "" log " Replacing v$OldVersion ($OldDate) with v$NewVersion ($CUR_DATE)" green # JOB: Clear target area log " Clearing RELEASE targets on local seed server..." green remove-item $SeedServer\$SeedFolderRS\pdq_pack\* -force -recurse -ea SilentlyContinue | out-null remove-item $SeedServer\$SeedFolderRS\integrity_verification\*txt* -force -recurse -ea SilentlyContinue | out-null log " Done" darkgreen # JOB: Calculate hashes of every file in the directory structure log " Calculating individual hashes of all included files, please wait..." green pushd $MasterCopy del $env:temp\checksum* -force -recurse | out-null & $HashDeep -s -e -c sha256 -l -r .\ | Out-File $env:temp\checksums.txt -encoding ascii mv $env:temp\checksums.txt $MasterCopy\integrity_verification\checksums.txt -force log " Done" darkgreen # JOB: PGP sign the resulting checksums.txt then upload the signature file to the seed locations log " PGP signing checksums.txt..." green "" remove-item $MasterCopy\integrity_verification\checksums.txt.asc -force -recurse -ea SilentlyContinue | out-null & $gpg --batch --yes --local-user $gpgUsername --passphrase $gpgPassphrase --armor --verbose --detach-sign $MasterCopy\integrity_verification\checksums.txt while (1 -eq 1) { if (test-path $MasterCopy\integrity_verification\checksums.txt.asc) { "" log " Done" darkgreen break } # sleep before looking again start-sleep -s 2 } # JOB: Verify PGP signature before FTP upload log " Verifying PGP signature of checksums.txt..." green "" & $gpg --batch --yes --verbose --verify $MasterCopy\integrity_verification\checksums.txt.asc $MasterCopy\integrity_verification\checksums.txt if ($? -eq "True") { "" log " Done" darkgreen } else { log " ! There was a problem verifying the signature!" red pause } # JOB: Create .torrent file for the release log " Generating .torrent file and saving to $TorrentSaveLocationCygwinFormat..." green & $mktorrent -n "PDQ Pack v$NewVersion ($CUR_DATE)" -c "Instructions at https://www.reddit.com/r/sysadmin" -a $TorrentTracker1,$TorrentTracker2,$TorrentTracker3,$TorrentTracker4,$TorrentTracker5,$TorrentTracker6,$TorrentTracker7 -o "$TorrentSaveLocationCygwinFormat/PDQ Pack v$NewVersion ($CUR_DATE).torrent" $MasterCopyCygwinFormat/pdq_pack if ($? -eq "True") { log " Done" darkgreen } else { log " ! There was a problem creating the .torrent" red } # JOB: Upload from master copy to seed server directories log " Master copy is stamped and good to go. Copying from master to local seeds..." green log " Loading Resilio Sync seed..." green cp $MasterCopy\* $SeedServer\$SeedFolderRS\ -recurse -force log " Done" darkgreen log " Loading Torrent seed..." green mkdir "$SeedServer\$SeedFolderTorrent\PDQ Pack v$NewVersion ($CUR_DATE)" -ea silentlycontinue | out-null cp $MasterCopy\pdq_pack\* "$SeedServer\$SeedFolderTorrent\PDQ Pack v$NewVersion ($CUR_DATE)" -recurse -force log " Done" darkgreen log " Uploading .torrent file to $TorrentAutoloaderLocation..." green cp "$TorrentSaveLocation\PDQ Pack v$NewVersion ($CUR_DATE).torrent" $SeedServer\$TorrentAutoloaderLocation if ($? -eq "True") { log " Done" darkgreen } else { log " ! There was a problem copying the .torrent to the autoloader folder" red } log " Done, seed server loaded." darkgreen # Notify that we're done loading the seed server and are starting deployment to the master repo log " Updating master repo (remote)..." green # JOB: Pack into a binary pack (.exe archive) using 7z and stash it in the TEMP directory. # Create the file name using the new version number extracted from the changelog and exclude any # files with "sync" in the title (these are Resilio Sync hidden files, we don't need to pack them) log " Building binary pack, please wait..." green & "$SevenZip" a -sfx "$env:temp\$NewBinary" ".\*" -x!*sync* -x!*ini* >> $LOGPATH\$LOGFILE log " Done" darkgreen # JOB: Background upload the binary pack to the static pack folder on the local seed server log " Background uploading $NewBinary to $SeedServer\$StaticPackStorageLocation..." green start-job -name pdq_pack_copy_to_seed_server -scriptblock {cp "$env:temp\$($args[0])" "$($args[1])\$($args[2])" -force} -ArgumentList $NewBinary, $SeedServer, $StaticPackStorageLocation "" # JOB: Fetch sha256sums.txt from the repo for updating log " Fetching repo copy of sha256sums.txt to update..." green Invoke-WebRequest $Repo_URL/sha256sums.txt -outfile $env:temp\sha256sums.txt log " Done" darkgreen # JOB: Calculate SHA256 hash of newly-created binary pack and append it to sha256sums.txt log " Calculating SHA256 hash for binary pack and appending it to sha256sums.txt..." green pushd $env:temp # First hash the file & $HashDeep -s -e -l -c sha256 "PDQ Pack v$NewVersion ($CUR_DATE).exe" | Out-File .\sha256sums_TEMP.txt -Encoding utf8 # Strip out the annoying hashdeep header gc .\sha256sums_TEMP.txt | Where-Object {$_ -notmatch '#'} | where-object {$_ -notmatch '%'} | sc .\sha256sums_TEMP2.txt # Strip out blank lines and trailing spaces (not needed?) #(gc .\sha256sums_TEMP2.txt) | ? {$_.trim() -ne "" } | sc .\sha256sums_TEMP2.txt # Append the result to the sha256sums.txt we pulled from the repo gc .\sha256sums_TEMP2.txt | out-file .\sha256sums.txt -encoding utf8 -append # Sleep for a few seconds to make sure the pack has had time to finish uploading to the local seed server static pack location start-sleep -s 30 # Nuke any old version that might be lingering in temp remove-item "$env:temp\UPLOADING" -force -recurse -ea SilentlyContinue | out-null # Rename the file to prepare it for uploading ren "$env:temp\$NewBinary" "$env:temp\UPLOADING" popd log " Done" darkgreen # JOB: PGP sign sha256sums.txt log " PGP signing sha256sums.txt..." green "" remove-item $env:temp\sha256sums.txt.asc -force -recurse -ea SilentlyContinue | out-null & $gpg --batch --yes --local-user $gpgUsername --passphrase $gpgPassphrase --armor --verbose --detach-sign $env:temp\sha256sums.txt while (1 -eq 1) { if (test-path $env:temp\sha256sums.txt.asc) { "" log " Done" darkgreen break } # sleep before looking again start-sleep -s 2 } # JOB: Verify PGP signature of sha256sums.txt log " Verifying PGP signature of sha256sums.txt..." green & $gpg --batch --yes --verbose --verify $env:temp\sha256sums.txt.asc $env:temp\sha256sums.txt if ($? -eq "True") { log " Done" darkgreen } else { log " ! There was a problem verifying the signature!" red } # JOB: Build FTP upload script # PDQ Pack exe will have "UPLOADING" appended to its name until upload is complete log " Building FTP deployment script..." green "option batch abort" | Out-File $env:temp\deploy_pdq_pack_ftp_script.txt -encoding ascii "option confirm off" | Out-File $env:temp\deploy_pdq_pack_ftp_script.txt -append -encoding ascii "open ftp://$Repo_FTP_Username`:$Repo_FTP_Password@$Repo_FTP_Host" | Out-File $env:temp\deploy_pdq_pack_ftp_script.txt -append -encoding ascii "cd $Repo_FTP_DepositPath" | Out-File $env:temp\deploy_pdq_pack_ftp_script.txt -append -encoding ascii "rm *.exe" | Out-File $env:temp\deploy_pdq_pack_ftp_script.txt -append -encoding ascii "rm *.torrent" | Out-File $env:temp\deploy_pdq_pack_ftp_script.txt -append -encoding ascii "rm sha256sums*" | Out-File $env:temp\deploy_pdq_pack_ftp_script.txt -append -encoding ascii add-content -path $env:temp\deploy_pdq_pack_ftp_script.txt -value "put -transfer=binary `"$TorrentSaveLocation\PDQ Pack v$NewVersion ($CUR_DATE).torrent`"" add-content -path $env:temp\deploy_pdq_pack_ftp_script.txt -value "put -transfer=binary -resume `"$env:temp\UPLOADING`"" add-content -path $env:temp\deploy_pdq_pack_ftp_script.txt -value "mv `"UPLOADING`" `"$NewBinary`"" add-content -path $env:temp\deploy_pdq_pack_ftp_script.txt -value "put -transfer=binary `"$env:temp\sha256sums.txt`"" add-content -path $env:temp\deploy_pdq_pack_ftp_script.txt -value "put -transfer=ascii `"$env:temp\sha256sums.txt.asc`"" #write-output "mv "UPLOADING_$NewBinary" "$NewBinary"" | Out-File $env:temp\deploy_pdq_pack_ftp_script.txt -append -encoding ascii "exit" | Out-File $env:temp\deploy_pdq_pack_ftp_script.txt -append -encoding ascii log " Done" darkgreen # JOB: Upload binary pack and hash files to FTP repo server # Get in TEMP directory and call WinSCP to run the script we just created log " Uploading $NewBinary to $Repo_FTP_Host..." green "" pushd $env:temp & $WinSCP /script=.\deploy_pdq_pack_ftp_script.txt popd "" log " Done" darkgreen # JOB: Clean up after ourselves log " Cleaning up..." green remove-item $env:temp\sha256sums* -force -recurse -ea SilentlyContinue | out-null remove-item $env:temp\$NewBinary* -force -recurse -ea SilentlyContinue | out-null remove-item $env:temp\deploy_pdq_pack_ftp_script.txt -force -recurse -ea SilentlyContinue | out-null # Remove our background upload job from the job list get-job | remove-job log " Done" darkgreen ############ # Finished # ############ log " Deployment done " green log " Version deployed: v$NewVersion ($CUR_DATE)" log " Version replaced: v$OldVersion ($OldDate)" log " Local seed server: $SeedServer" log " Local seed directories:" log " Resilio Sync (RELEASE): $SeedFolderRS" log " Local torrent autoloader location: $TorrentAutoloaderLocation" log " Local torrent save location: $TorrentSaveLocation" log " Local static pack storage: $StaticPackStorageLocation" log " Remote repo host: $Repo_FTP_Host" log " Remote repo upload path: $Repo_FTP_Host/$Repo_FTP_DepositPath" log " Log file: $LOGPATH\$LOGFILE" log " Start the .torrent file" blue write-output "Press any key to continue..."; $HOST.UI.RawUI.ReadKey("NoEcho,IncludeKeyDown") | out-null # Close the main() function. End of the script } ############# # FUNCTIONS # ############# function log($message, $color) { if ($color -eq $null) {$color = "gray"} #console write-host (get-date -f "yyyy-MM-dd hh:mm:ss") -n -f darkgray; write-host "$message" -f $color #log #(get-date -f "yyyy-mm-dd hh:mm:ss") +"$message" | out-file -Filepath "$logpath\$logfile" -append (get-date -f "yyyy-MM-dd hh:mm:ss") +"$message" | out-file -Filepath "C:\logs\deploy_pdq_packs.log" -append } # call the main script main