Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement FQDN Beaconing using TLS SNI and HTTP Host #739

Merged
merged 25 commits into from
Aug 1, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
c69b328
initial commit for parsing in data needed for TLS and HTTP beaconing
Jun 29, 2022
153d386
First pass at laying out the equivalent of unique connection package …
Jun 29, 2022
aa48e12
Add total durations to SNI analysis
Jun 29, 2022
de87b48
beacon pkg comment updates
Jun 30, 2022
d692a3e
SNIconn updates
Jun 30, 2022
3262152
beaconSNI implementation
Jun 30, 2022
5d0a23d
Add SNI Beaconing reporting command
Jun 30, 2022
0d1ef15
Add html report for sni beaconing
Jul 1, 2022
bb12bba
Add BeaconSNI config section. Fix DefaultConnectionThreshold for prox…
Jul 1, 2022
19c729c
Create host summary entries for dat.mbsni
Jul 1, 2022
7f8b017
Ensure top level document is rolled out in SNIconn and that the remov…
Jul 1, 2022
8122d78
readme updates in beaconproxy and uconn
Jul 5, 2022
7ba22c3
SNIconn readme
Jul 5, 2022
f85c442
strobe updates for SNIconn readme
Jul 5, 2022
5540832
more strobe updates for SNIconn readme
Jul 5, 2022
e5f46db
beaconSNI readme
Jul 5, 2022
c8d6d51
Record whether the SNI beacon has an invalid certificate associated w…
Jul 6, 2022
3d74703
Add responding_ips to beaconSNI collection documents to make search easy
Jul 7, 2022
10dcc30
Requested documentation and comment changes
Jul 28, 2022
5e205b0
Add dat.beacon.strobe to beaconsni dissector
Jul 28, 2022
0576f5c
more uconn -> sniconn fixes
Jul 28, 2022
6293335
again...
Jul 28, 2022
5167c13
change dat.beacon.strobe to dat.merged.strobe
Jul 28, 2022
146e0c0
Move cid fields in tls, beacon, and merged subdocs in sniconn up a le…
Jul 28, 2022
e54627d
Merge branch 'master' into sni-beaconing
caffeinatedpixel Aug 1, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
156 changes: 156 additions & 0 deletions commands/show-beacons-sni.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
package commands

import (
"fmt"
"os"
"strings"

"github.com/activecm/rita/pkg/beaconsni"
"github.com/activecm/rita/resources"
"github.com/olekukonko/tablewriter"
"github.com/urfave/cli"
)

func init() {
command := cli.Command{
Name: "show-beacons-sni",
Usage: "Print hosts which show signs of C2 software (SNI Analysis)",
ArgsUsage: "<database>",
Flags: []cli.Flag{
ConfigFlag,
humanFlag,
delimFlag,
netNamesFlag,
},
Action: showBeaconsSNI,
}

bootstrapCommands(command)
}

func showBeaconsSNI(c *cli.Context) error {
db := c.Args().Get(0)
if db == "" {
return cli.NewExitError("Specify a database", -1)
}
res := resources.InitResources(getConfigFilePath(c))
res.DB.SelectDB(db)

data, err := beaconsni.Results(res, 0)

if err != nil {
res.Log.Error(err)
return cli.NewExitError(err, -1)
}

if !(len(data) > 0) {
return cli.NewExitError("No results were found for "+db, -1)
}

showNetNames := c.Bool("network-names")

if c.Bool("human-readable") {
err := showBeaconsSNIHuman(data, showNetNames)
if err != nil {
return cli.NewExitError(err.Error(), -1)
}
return nil
}

err = showBeaconsSNIDelim(data, c.String("delimiter"), showNetNames)
if err != nil {
return cli.NewExitError(err.Error(), -1)
}
return nil
}

func showBeaconsSNIHuman(data []beaconsni.Result, showNetNames bool) error {
table := tablewriter.NewWriter(os.Stdout)
var headerFields []string
if showNetNames {
headerFields = []string{
"Score", "Source Network", "Source IP", "SNI",
"Connections", "Avg. Bytes", "Intvl Range", "Size Range", "Top Intvl",
"Top Size", "Top Intvl Count", "Top Size Count", "Intvl Skew",
"Size Skew", "Intvl Dispersion", "Size Dispersion",
}
} else {
headerFields = []string{
"Score", "Source IP", "SNI",
"Connections", "Avg. Bytes", "Intvl Range", "Size Range", "Top Intvl",
"Top Size", "Top Intvl Count", "Top Size Count", "Intvl Skew",
"Size Skew", "Intvl Dispersion", "Size Dispersion",
}
}

table.SetHeader(headerFields)

for _, d := range data {
var row []string

if showNetNames {
row = []string{
f(d.Score), d.SrcNetworkName,
d.SrcIP, d.FQDN, i(d.Connections), f(d.AvgBytes),
i(d.Ts.Range), i(d.Ds.Range), i(d.Ts.Mode), i(d.Ds.Mode),
i(d.Ts.ModeCount), i(d.Ds.ModeCount), f(d.Ts.Skew), f(d.Ds.Skew),
i(d.Ts.Dispersion), i(d.Ds.Dispersion),
}
} else {
row = []string{
f(d.Score), d.SrcIP, d.FQDN, i(d.Connections), f(d.AvgBytes),
i(d.Ts.Range), i(d.Ds.Range), i(d.Ts.Mode), i(d.Ds.Mode),
i(d.Ts.ModeCount), i(d.Ds.ModeCount), f(d.Ts.Skew), f(d.Ds.Skew),
i(d.Ts.Dispersion), i(d.Ds.Dispersion),
}
}
table.Append(row)
}
table.Render()
return nil
}

func showBeaconsSNIDelim(data []beaconsni.Result, delim string, showNetNames bool) error {
var headerFields []string
if showNetNames {
headerFields = []string{
"Score", "Source Network", "Source IP", "SNI",
"Connections", "Avg. Bytes", "Intvl Range", "Size Range", "Top Intvl",
"Top Size", "Top Intvl Count", "Top Size Count", "Intvl Skew",
"Size Skew", "Intvl Dispersion", "Size Dispersion",
}
} else {
headerFields = []string{
"Score", "Source IP", "SNI",
"Connections", "Avg. Bytes", "Intvl Range", "Size Range", "Top Intvl",
"Top Size", "Top Intvl Count", "Top Size Count", "Intvl Skew",
"Size Skew", "Intvl Dispersion", "Size Dispersion",
}
}

// Print the headers and analytic values, separated by a delimiter
fmt.Println(strings.Join(headerFields, delim))
for _, d := range data {

var row []string
if showNetNames {
row = []string{
f(d.Score), d.SrcNetworkName,
d.SrcIP, d.FQDN, i(d.Connections), f(d.AvgBytes),
i(d.Ts.Range), i(d.Ds.Range), i(d.Ts.Mode), i(d.Ds.Mode),
i(d.Ts.ModeCount), i(d.Ds.ModeCount), f(d.Ts.Skew), f(d.Ds.Skew),
i(d.Ts.Dispersion), i(d.Ds.Dispersion),
}
} else {
row = []string{
f(d.Score), d.SrcIP, d.FQDN, i(d.Connections), f(d.AvgBytes),
i(d.Ts.Range), i(d.Ds.Range), i(d.Ts.Mode), i(d.Ds.Mode),
i(d.Ts.ModeCount), i(d.Ds.ModeCount), f(d.Ts.Skew), f(d.Ds.Skew),
i(d.Ts.Dispersion), i(d.Ds.Dispersion),
}
}

fmt.Println(strings.Join(row, delim))
}
return nil
}
7 changes: 7 additions & 0 deletions config/static.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ type (
Beacon BeaconStaticCfg `yaml:"Beacon"`
BeaconFQDN BeaconFQDNStaticCfg `yaml:"BeaconFQDN"`
BeaconProxy BeaconProxyStaticCfg `yaml:"BeaconProxy"`
BeaconSNI BeaconSNIStaticCfg `yaml:"BeaconSNI"`
DNS DNSStaticCfg `yaml:"DNS"`
UserAgent UserAgentStaticCfg `yaml:"UserAgent"`
Bro BroStaticCfg `yaml:"Bro"` // kept in for MetaDB backwards compatibility
Expand Down Expand Up @@ -104,6 +105,12 @@ type (
DefaultConnectionThresh int `yaml:"DefaultConnectionThresh" default:"20"`
}

//BeaconSNIStaticCfg is used to control the SNI beaconing analysis module
BeaconSNIStaticCfg struct {
Enabled bool `yaml:"Enabled" default:"true"`
DefaultConnectionThresh int `yaml:"DefaultConnectionThresh" default:"20"`
}

//DNSStaticCfg is used to control the DNS analysis module
DNSStaticCfg struct {
Enabled bool `yaml:"Enabled" default:"true"`
Expand Down
7 changes: 7 additions & 0 deletions config/tables.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ type (
DNS DNSTableCfg
Structure StructureTableCfg
Beacon BeaconTableCfg
BeaconSNI BeaconSNITableCfg
BeaconFQDN BeaconFQDNTableCfg
BeaconProxy BeaconProxyTableCfg
UserAgent UserAgentTableCfg
Expand All @@ -29,6 +30,7 @@ type (
SSLTable string `default:"ssl"`
UniqueConnTable string `default:"uconn"`
UniqueConnProxyTable string `default:"uconnProxy"`
SNIConnTable string `default:"SNIconn"`
}

//DNSTableCfg is used to control the dns analysis module
Expand All @@ -42,6 +44,11 @@ type (
BeaconTable string `default:"beacon"`
}

//BeaconSNITableCfg is used to control the SNI beaconing analysis module
BeaconSNITableCfg struct {
BeaconSNITable string `default:"beaconSNI"`
}

//BeaconFQDNTableCfg is used to control the beaconing analysis module
BeaconFQDNTableCfg struct {
BeaconFQDNTable string `default:"beaconFQDN"`
Expand Down
13 changes: 13 additions & 0 deletions etc/rita.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,19 @@ BeaconFQDN:
# about slow beacons.
DefaultConnectionThresh: 20

BeaconSNI:
Enabled: true
# The default minimum number of connections used for beacons SNI analysis.
# Any two hosts connecting fewer than this number will not be analyzed.
# 20 was chosen as it is a little bit less than once per hour in a day,
# and allows for any packet loss that could occur.

# If you choose a lower value, this will significantly increase both
# the analysis time and the number of false positives. You can safely
# increase this value to improve performance if you are not concerned
# about slow beacons.
DefaultConnectionThresh: 20

BeaconProxy:
Enabled: true
# The default minimum number of connections used for beacons proxy analysis.
Expand Down
21 changes: 21 additions & 0 deletions parser/conn.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@ func parseConnEntry(parseConn *parsetypes.Conn, filter filter, retVals ParseResu
)

updateCertificatesByConn(dstKey, tuple, retVals)

updateZeekUIDRecordsByConn(parseConn.UID, parseConn.OrigIPBytes, parseConn.RespBytes, roundedDuration, retVals)
}

func updateUniqueConnectionsByConn(srcIP, dstIP net.IP, srcDstPair data.UniqueIPPair, srcDstKey string,
Expand Down Expand Up @@ -229,3 +231,22 @@ func updateCertificatesByConn(dstKey string, tuple string, retVals ParseResults)
retVals.CertificateMap[dstKey].Tuples.Insert(tuple)
}
}

func updateZeekUIDRecordsByConn(uid string, origIPBytes int64, respIPBytes int64, duration float64, retVals ParseResults) {
// Don't do any work if the UID is missing
if len(uid) == 0 {
return
}

retVals.ZeekUIDLock.Lock()
defer retVals.ZeekUIDLock.Unlock()

// ///// CREATE UID RECORD IN MAP IF IT DOES NOT EXIST /////
if _, ok := retVals.ZeekUIDMap[uid]; !ok {
retVals.ZeekUIDMap[uid] = &data.ZeekUIDRecord{}
}

retVals.ZeekUIDMap[uid].Conn.OrigBytes = origIPBytes
retVals.ZeekUIDMap[uid].Conn.RespBytes = respIPBytes
retVals.ZeekUIDMap[uid].Conn.Duration = duration
}
43 changes: 43 additions & 0 deletions parser/fsimporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,15 @@ import (
"github.com/activecm/rita/pkg/beacon"
"github.com/activecm/rita/pkg/beaconfqdn"
"github.com/activecm/rita/pkg/beaconproxy"
"github.com/activecm/rita/pkg/beaconsni"
"github.com/activecm/rita/pkg/blacklist"
"github.com/activecm/rita/pkg/certificate"
"github.com/activecm/rita/pkg/data"
"github.com/activecm/rita/pkg/explodeddns"
"github.com/activecm/rita/pkg/host"
"github.com/activecm/rita/pkg/hostname"
"github.com/activecm/rita/pkg/remover"
"github.com/activecm/rita/pkg/sniconn"
"github.com/activecm/rita/pkg/uconn"
"github.com/activecm/rita/pkg/uconnproxy"
"github.com/activecm/rita/pkg/useragent"
Expand Down Expand Up @@ -179,6 +182,9 @@ func (fs *FSImporter) Run(indexedFiles []*files.IndexedFile, threads int) {
// build uconnsProxy table. Must go before proxy beacons
fs.buildUconnsProxy(retVals.ProxyUniqueConnMap)

// build SNIconns table. Must go before SNI beacons
fs.buildSNIConns(retVals.TLSConnMap, retVals.HTTPConnMap, retVals.ZeekUIDMap, retVals.HostMap)

// update ts range for dataset (needs to be run before beacons)
minTimestamp, maxTimestamp := fs.updateTimestampRange()

Expand All @@ -197,6 +203,9 @@ func (fs *FSImporter) Run(indexedFiles []*files.IndexedFile, threads int) {
// build or update the Proxy Beacons Table
fs.buildProxyBeacons(retVals.ProxyUniqueConnMap, retVals.HostMap, minTimestamp, maxTimestamp)

// build or update SNI Beacons Table
fs.buildSNIBeacons(retVals.TLSConnMap, retVals.HTTPConnMap, retVals.HostMap, minTimestamp, maxTimestamp)

// build or update UserAgent table
fs.buildUserAgent(retVals.UseragentMap)

Expand Down Expand Up @@ -473,6 +482,22 @@ func (fs *FSImporter) buildHostnames(hostnameMap map[string]*hostname.Input) {

}

func (fs *FSImporter) buildSNIConns(tlsMap map[string]*sniconn.TLSInput, httpMap map[string]*sniconn.HTTPInput,
zeekUIDMap map[string]*data.ZeekUIDRecord, hostMap map[string]*host.Input) {
if len(tlsMap) != 0 || len(httpMap) != 0 {
sniconnRepo := sniconn.NewMongoRepository(fs.database, fs.config, fs.log)

err := sniconnRepo.CreateIndexes()
if err != nil {
fs.log.Error(err)
}

sniconnRepo.Upsert(tlsMap, httpMap, zeekUIDMap, hostMap)
} else {
fmt.Println("\t[!] No TLS or HTTP connections to analyze")
}
}

func (fs *FSImporter) buildUconnsProxy(uconnProxyMap map[string]*uconnproxy.Input) {
// non-optional module
if len(uconnProxyMap) > 0 {
Expand Down Expand Up @@ -604,6 +629,24 @@ func (fs *FSImporter) buildProxyBeacons(uconnProxyMap map[string]*uconnproxy.Inp

}

func (fs *FSImporter) buildSNIBeacons(tlsMap map[string]*sniconn.TLSInput, httpMap map[string]*sniconn.HTTPInput, hostMap map[string]*host.Input, minTimestamp, maxTimestamp int64) {
if fs.config.S.BeaconSNI.Enabled {
if len(tlsMap) > 0 || len(httpMap) > 0 {
beaconSNIRepo := beaconsni.NewMongoRepository(fs.database, fs.config, fs.log)

err := beaconSNIRepo.CreateIndexes()
if err != nil {
fs.log.Error(err)
}

// send SNI conns to beacon analysis
beaconSNIRepo.Upsert(tlsMap, httpMap, hostMap, minTimestamp, maxTimestamp)
} else {
fmt.Println("\t[!] No TLS or HTTP Beacon data to analyze")
}
}
}

//buildUserAgent .....
func (fs *FSImporter) buildUserAgent(useragentMap map[string]*useragent.Input) {

Expand Down
Loading