Offload hashcat processing to Linux (GPU) host via SSH

HelperFuncs.ps1

@@ -1,7 +1,8 @@
 #  Requirements
 # *  DSInternals module v4.4.1,  PSGallery or https://github.com/MichaelGrafnetter/DSInternals
-# *  Hashcat installation
+# *  Hashcat https://hashcat.net/hashcat/
 # *  Haveibeenpwned.com  sorted-by-NTLM-hash list
+# *  Install-Module -Name Posh-SSH   # if using separate hashcat server
 
 # Minimum Permissions to pull password hashes from a DC (online)
 # - The “DS-Replication-Get-Changes” extended right
@@ -23,13 +24,14 @@ Function Format-NTHash {
 Function Get-ADHashesAsTestSet {
     [CmdletBinding()]
     param(
-        [Parameter(Mandatory)] [ScriptBlock] $Filter
+          [Parameter(Mandatory)] [ScriptBlock] $Filter
+        , [Parameter(Mandatory)] [string] $DC
     )
 
     $rootdse = New-Object System.DirectoryServices.DirectoryEntry("LDAP://RootDSE",$null,$null,[System.DirectoryServices.AuthenticationTypes]::Anonymous) -ErrorAction Stop
-    $NBName = (Get-ADDomain).NetBIOSName
+    $NBName = (Get-ADDomain -Server $DC).NetBIOSName
 
-    if(!($users = Get-ADUser -Filter $filter -Properties PasswordLastSet))
+    if(!($users = @(Get-ADUser -Server $DC -Filter $filter -Properties PasswordLastSet)))
     {    
         Write-Warning "No users matched filter"
     }
@@ -42,7 +44,7 @@ Function Get-ADHashesAsTestSet {
     $retrievedUsers = @{}
     foreach($u in $users)
     {
-        $repl = Get-ADReplAccount -SamAccountName $u.SamAccountName  -Server $rootdse.Properties["dnsHostName"].Value  -Domain $NBName
+        $repl = Get-ADReplAccount  -Server $DC  -SamAccountName $u.SamAccountName  -Domain $NBName
 
         $retrievedUsers[$u.samaccountname] = [pscustomobject] @{
             Replica = $repl
@@ -59,15 +61,20 @@ Function Get-ADHashesAsTestSet {
 function Test-HashesWithHashcat{
     [CmdletBinding()]
     param(
-          [Parameter(Mandatory)] $TestSet   # from Get-ADHashes
-        , [Parameter(Mandatory)] [string] $HashcatDir
-        ,                        [switch] $ShowOutput
+          [Parameter(Mandatory = $true)] $TestSet   # from Get-ADHashes
+        , [Parameter(Mandatory = $false)] [string] $HashcatHost
+        , [Parameter(Mandatory = $false)] [pscredential] $HashcatHostCred
+        , [Parameter(Mandatory = $true)] [string] $HashcatDir
+        , [Parameter(Mandatory = $true)] [string] $WordList 
+        , [Parameter(Mandatory = $true)] [string] $Rules                        
+        , [Parameter(Mandatory = $false)] [switch] $ShowOutput
+        , [Parameter(Mandatory = $false)] [int] $TimeoutHours = 6
     )
 
-
     # build a hashtable of username keyed by password hashes 
     $hashesToTest = @{}
 
+    # Test anything that hasn't already been confirmed to be weak,  "Condition -eq $null"
     foreach($v in ($TestSet.Values | ?{$_.Condition -eq $null} ) )
     {
         $hash = Format-NTHash -NTHash $v.Replica.NTHash
@@ -87,24 +94,82 @@ function Test-HashesWithHashcat{
     }
 
     $jobName = "hcu-{0}" -f (Get-Random -Minimum 1000 -Maximum 9999)
-    $scratchFile = [IO.Path]::Combine( $env:TMP, ("{0}-i.txt" -f $jobName))
-    $outputFile = [IO.Path]::Combine( $env:TMP, ("{0}-o.txt" -f  $jobName))
+    $scratchFile = [IO.FileInfo] [IO.Path]::Combine( $env:TMP, ("{0}.input" -f $jobName))
+    $outputFile = [IO.FileInfo] [IO.Path]::Combine( $env:TMP, ("{0}.output" -f  $jobName))
+    $logFile = [IO.FileInfo] [IO.Path]::Combine( $env:TMP, ("{0}.log" -f  $jobName))
 
     # output hashes for processing by hashcat
     $hashesToTest.Keys | Out-File -Encoding ascii -FilePath $scratchFile
     $null | Out-File -Encoding ascii -FilePath $outputFile
 
-    Push-Location $hashcatDir
+    $hashcatOutput = "NO HASHCAT OUTPUT"
+
+    $stopwatch =  [system.diagnostics.stopwatch]::StartNew()
+
+    if(![string]::IsNullOrWhiteSpace($HashcatHost) -and $HashcatHostCred -ne $null)
+    {   
+        # remote hashcat (e.g.,  Linux host with GPU)
+
+        $session = New-SSHSession -ComputerName $HashcatHost -Credential $HashcatHostCred
+
+        # Transfer the hashes to crack
+        Set-SCPItem -ComputerName $HashcatHost -Credential $HashcatHostCred -Path $scratchFile.FullName -Destination "~" -NewName $scratchFile.Name
+
+        $session = New-SSHSession -ComputerName $HashcatHost -Credential $HashcatHostCred
+
+		# crack hashes and add to potfile
+        $cmd = "{0}hashcat  -m 1000 -O --session {1} {2} --rules-file {3} {4} 2>&1 1> {1}" -f $HashcatDir,$logFile.Name,$scratchFile.Name,$($HashcatDir + $Rules),$($HashcatDir + $WordList)    
+        $result = Invoke-SSHCommand -SSHSession $session -Command $cmd  -TimeOut (60*60*$TimeoutHours)
+
+		# export results
+        $cmd = "{0}hashcat  -m 1000 --show --outfile {1} {2}  " -f $HashcatDir,$outputFile.Name,$scratchFile.Name,$logFile.Name
+        $result = Invoke-SSHCommand -SSHSession $session -Command $cmd  -TimeOut (60*60*$TimeoutHours)
+
+
+        # Retrieve the results
+        Get-SCPItem -ComputerName $HashcatHost -Credential $HashcatHostCred -Path $outputFile.Name -PathType File -Destination $outputFile.Directory.FullName -Force
+        Get-SCPItem -ComputerName $HashcatHost -Credential $HashcatHostCred -Path $logFile.Name -PathType File -Destination $logFile.Directory.FullName -Force
+
+        # Clean up temp files
+        $result = Invoke-SSHCommand -SSHSession $session -Command ("rm {0}*" -f $jobName)
+
+        Remove-SSHSession $session | Out-Null
+
+        $hashcatOutput = Get-Content $logFile.FullName
+    }
+    else
+    {
+        # local hashcat   #### NEEDS REVIEW.  MAY BE BROKEN AFTER CHANGES TO ALLOW SSH REMOTING  #####
+
+		# crack hashes and add to potfile
+        $cmd = "{0}hashcat  -m 1000 -O --session {1} {2} --rules-file {3} {4} 2>&1 1> {1}.log" -f $HashcatDir,$jobName,$scratchFile.Name,$($HashcatDir + $Rules),$($HashcatDir + $WordList)    
+
+		# export results
+        $cmd = "{0}hashcat  -m 1000 --show --outfile {1} {2}" -f $HashcatDir,$outputFile.Name,$scratchFile.Name
+
+#        $cmd = "{0}hashcat  -m 1000 -O --session {1}  --potfile-disable --outfile {2} {3} --rules-file {4} {5}" -f $HashcatDir,$jobName,$outputFile.FullName,$scratchFile.FullName,$($HashcatDir + $WordList),$($HashcatDir + $Rules)
+        $cmd = "{0}hashcat  -m 1000 -O --session {1}  --show --outfile {2} {3} --rules-file {4} {5}" -f $HashcatDir,$jobName,$outputFile.FullName,$scratchFile.FullName,$($HashcatDir + $WordList),$($HashcatDir + $Rules)
+        $hashcatOutput = Invoke-Expression -Command $cmd 
+    }
+
+    $stopwatch.Stop()
+
+    Write-Host ("Hashcat processing time: {0:n0} minutes" -f $stopwatch.Elapsed.TotalMinutes)
 
-    $hashcatOutput = .\hashcat.exe  -m 1000 -O --session $jobName --outfile $outputfile $scratchFile wordlists\Top353Million-probable-v2.txt -r rules\best64.rule --potfile-disable
     if($ShowOutput)
     {
-        $hashcatOutput
+        $hashcatOutput | Write-Host
     }
 
     # hashcat-ing complete,  import the cracked results
     foreach($crack in (Import-Csv $outputFile -Delimiter ":" -Header "hash","result"))
     {
+        if(-not $hashesToTest[$crack.hash])
+        {
+            Write-Verbose ("Skipping over unmatched hash (probably from potfile cache): {0}" -f $crack.hash)
+            continue  
+        }
+
         foreach($user in $hashesToTest[$crack.hash].Users)
         {
             $TestSet[$user].Condition = "weak"
@@ -114,8 +179,7 @@ function Test-HashesWithHashcat{
 
     Remove-Item -Force $scratchFile
     Remove-Item -Force $outputFile
-
-    Pop-Location
+    Remove-Item -Force $logFile
 }
 
 
@@ -127,9 +191,10 @@ function Test-HashesAgainstList {
     )
 
     # Get only the values we haven't already marked
-    $testSubset = $TestSet.Values | ?{$_.Condition -eq $null} 
-
-    $testResults = $testSubset.Replica | Test-PasswordQuality -WeakPasswordHashesSortedFile $BadHashesSortedFile.FullName
+    if($testSubset = $TestSet.Values | ?{$_.Condition -eq $null} )
+    {
+        $testResults = $testSubset.Replica | Test-PasswordQuality -WeakPasswordHashesSortedFile $BadHashesSortedFile.FullName
+    }
 
     foreach($failedUser in $testResults.WeakPassword)
     {
@@ -186,20 +251,21 @@ function Test-HashesForPasswordReuse {
 function Test-HashesForPasswordSharing {
     [CmdletBinding()]
     param(
-        [Parameter(Mandatory)] $TestSet   # from Get-ADHashes - logic below assumes both the user and the user's manager are in this replica.  Don't use this function if you are only analyzing recent password changes.
+          [Parameter(Mandatory)] $TestSet   # from Get-ADHashes - logic below assumes both the user and the user's manager are in this replica.  Don't use this function if you are only analyzing recent password changes.
+        , [Parameter(Mandatory)] $DC
     )
 
     foreach($t in $TestSet.Values)
     {
         $userHash =   Format-NTHash -NTHash $t.Replica.NTHash
 
         # find the manager of this user
-        $aduser =  Get-ADUser -Properties manager -LDAPFilter ("(&(objectCategory=person)(sAMAccountName={0}))" -f $t.Replica.sAMAccountName)
+        $aduser =  Get-ADUser -Server $DC -Properties manager -LDAPFilter ("(&(objectCategory=person)(sAMAccountName={0}))" -f $t.Replica.sAMAccountName)
 
         if($adUser -eq $null -or $aduser.Enabled -ne $true)
         {    continue      }
 
-        if($aduser.Manager -gt "" -and $aduser.Manager -ne $aduser.DistinguishedName -and ($adManager = Get-ADUser $aduser.Manager -Properties displayName,mail) -and $TestSet.ContainsKey($adManager.SamAccountName))
+        if($aduser.Manager -gt "" -and $aduser.Manager -ne $aduser.DistinguishedName -and ($adManager = Get-ADUser -Server $DC $aduser.Manager -Properties displayName,mail) -and $TestSet.ContainsKey($adManager.SamAccountName))
         {
             $managerHash =  Format-NTHash -NTHash $TestSet[$adManager.SamAccountName].Replica.NTHash
 

hashcat-setup-notes-for-Ubuntu-with-Windows-remoting.txt

@@ -0,0 +1,179 @@
+
+# Hashcat setup adapted from:
+https://arminreiter.com/2020/11/using-azure-vm-to-crack-passwords/
+
+https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html
+
+# PSremoting steps adapted from:
+https://adamtheautomator.com/psremoting-linux/
+
+lsb_release -a
+
+
+
+Check correct urls at: http://developer.download.nvidia.com/compute/cuda/repos/ for
+- cuda-repo-ubuntu****_amd64.deb
+- keys  (exact URL will be provided by this command:
+        sudo dpkg -i /tmp/${CUDA_REPO_PKG}
+
+http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/cuda-repo-ubuntu1604_10.2.89-1_amd64.deb
+
+
+CUDA_REPO_PKG=cuda-repo-ubuntu1604_10.2.89-1_amd64.deb
+wget -O /tmp/${CUDA_REPO_PKG} http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/${CUDA_REPO_PKG}
+
+
+sudo dpkg -i /tmp/${CUDA_REPO_PKG}
+sudo apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/7fa2af80.pub
+rm -f /tmp/${CUDA_REPO_PKG}
+sudo apt-get update
+
+sudo apt-get install cuda   # HCK
+
+sudo apt-get install cuda-drivers
+sudo apt-get install nvidia-cuda-toolkit
+sudo apt-get install p7zip-full -y
+
+sudo reboot
+
+nvidia-smi
+
+# Output from nvidia-smi:
+Tue Sep 14 14:32:38 2021
++-----------------------------------------------------------------------------+
+| NVIDIA-SMI 465.19.01    Driver Version: 465.19.01    CUDA Version: 11.3     |
+|-------------------------------+----------------------+----------------------+
+| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
+| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
+|                               |                      |               MIG M. |
+|===============================+======================+======================|
+|   0  NVIDIA Tesla K80    Off  | 00000001:00:00.0 Off |                    0 |
+| N/A   72C    P0    65W / 149W |      0MiB / 11441MiB |      1%      Default |
+|                               |                      |                  N/A |
++-------------------------------+----------------------+----------------------+
+
++-----------------------------------------------------------------------------+
+| Processes:                                                                  |
+|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
+|        ID   ID                                                   Usage      |
+|=============================================================================|
+|  No running processes found                                                 |
++-----------------------------------------------------------------------------+
+
+# Install latest version of Hashcat (check https://hashcat.net/hashcat/ to get latest version number)
+
+
+cd /opt
+
+HASHCAT_VER=hashcat-6.2.4
+sudo wget https://hashcat.net/files/${HASHCAT_VER}.7z
+
+
+sudo 7z x ${HASHCAT_VER}.7z
+sudo rm ${HASHCAT_VER}.7z
+
+
+cd ${HASHCAT_VER}
+
+# make a link using short name 
+ln -s hashcat.bin hashcat
+
+# grant write perms so that the .pid, .induct and other temp files can be written without sudo
+sudo chmod -R 757 /opt/hashcat-6.2.4/
+
+
+
+# Chec for GPU recognition
+sudo ./hashcat.bin -I
+
+
+hashcat (v6.2.4) starting in backend information mode
+
+CUDA Info:
+==========
+
+CUDA.Version.: 11.3
+
+Backend Device ID #1
+  Name...........: NVIDIA Tesla K80
+  Processor(s)...: 13
+  Clock..........: 823
+  Memory.Total...: 11441 MB
+  Memory.Free....: 11382 MB
+  PCI.Addr.BDFe..: 0001:00:00.0
+
+OpenCL Info:
+============
+
+OpenCL Platform ID #1
+  Vendor..: NVIDIA Corporation
+  Name....: NVIDIA CUDA
+  Version.: OpenCL 3.0 CUDA 11.3.55
+
+  Backend Device ID #2
+    Type...........: GPU
+    Vendor.ID......: 32
+    Vendor.........: NVIDIA Corporation
+    Name...........: NVIDIA Tesla K80
+    Version........: OpenCL 3.0 CUDA
+    Processor(s)...: 13
+    Clock..........: 823
+    Memory.Total...: 11441 MB (limited to 2860 MB allocatable in one block)
+    Memory.Free....: 11328 MB
+    OpenCL.Version.: OpenCL C 1.2
+    Driver.Version.: 465.19.01
+    PCI.Addr.BDF...: 00:00.0
+
+
+
+
+mkdir wordlists
+cd wordlists
+
+
+sudo wget http://thehackerplaybook.com/get.php?type=THP-password
+sudo 7z x 
+rm get
+
+cd ../rules
+sudo wget https://raw.githubusercontent.com/NotSoSecure/password_cracking_rules/master/OneRuleToRuleThemAll.rule
+
+
+# Finicky syntax
+sudo ./hashcat.bin  -m 1000 -O --outfile x.txt --potfile-disable  -r rules/OneRuleToRuleThemAll.rule hash-to-crack.txt   wordlists/40GB_CleanUpFile.txt
+
+
+
+
+# Remoting setup
+
+# On Windows client
+# Install OpenSSH
+Get-WindowsCapability -Online | Where-Object {$_.Name -like 'OpenSSH.Client*'} | Add-WindowsCapability -Online
+
+- OR - 
+See DISM batch file in this folder
+
+Make sure ssh.exe is in your path.
+c:\Windows\System32\OpenSSH\ssh.exe
+
+
+# on Ubuntu server (using direct download method)
+https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-core-on-linux?view=powershell-7.1#installation-via-direct-download---ubuntu-1604
+
+sudo wget https://github.com/PowerShell/PowerShell/releases/download/v7.1.4/powershell_7.1.4-1.ubuntu.16.04_amd64.deb
+
+sudo dpkg -i powershell_7.1.4-1.ubuntu.16.04_amd64.deb
+# Ignore dependency errors from above ^^.  Next command resolves
+sudo apt-get install -f
+
+# Add the following (commented)  line to the sshd config (near SFTP line)
+# Subsystem powershell /usr/bin/pwsh -sshs -NoLogo
+sudo vi /etc/ssh/sshd_config
+
+
+sudo systemctl restart sshd.
+
+
+# Test from client
+$SessionParams = @{SSHTransport = $true; UserName = "$env:USERNAME@$env:USERDNSDOMAIN"; HostName = "SRV1";  }