From a4726f1139c7f317eff4410ddc95c6137cf255f2 Mon Sep 17 00:00:00 2001
From: lisa <lisa.s.woody@gmail.com>
Date: Wed, 28 Apr 2021 14:31:35 -0600
Subject: [PATCH] fixed missing chunk removal for invalid certificate table,
 fixed logic for adding cert records to uconns (#634)

Co-authored-by: Logan Lembke <logo2462@gmail.com>
---
 parser/fsimporter.go   | 36 ++++++++++++++++++++----------------
 pkg/remover/mongodb.go |  1 +
 2 files changed, 21 insertions(+), 16 deletions(-)

diff --git a/parser/fsimporter.go b/parser/fsimporter.go
index 90896cbd..0748e92c 100644
--- a/parser/fsimporter.go
+++ b/parser/fsimporter.go
@@ -763,7 +763,7 @@ func (fs *FSImporter) parseFiles(indexedFiles []*fpt.IndexedFile, parsingThreads
 							// Safely store ja3 information
 							mutex.Lock()
 
-							// create record if it doesn't exist
+							// create useragent record if it doesn't exist
 							if _, ok := useragentMap[ja3Hash]; !ok {
 								useragentMap[ja3Hash] = &useragent.Input{
 									Name:     ja3Hash,
@@ -785,22 +785,25 @@ func (fs *FSImporter) parseFiles(indexedFiles []*fpt.IndexedFile, parsingThreads
 								}
 							}
 
-							//if there's any problem in the certificate, mark it invalid
-							if certStatus != "ok" && certStatus != "-" && certStatus != "" && certStatus != " " {
-								// Run conn pair through filter to filter out certain connections
-								ignore := fs.filterConnPair(srcIP, dstIP)
-								if !ignore {
+							// create uconn and cert records
+							// Run conn pair through filter to filter out certain connections
+							ignore := fs.filterConnPair(srcIP, dstIP)
+							if !ignore {
 
-									// Check if uconn map value is set, because this record could
-									// come before a relevant uconns record
-									if _, ok := uconnMap[srcDstKey]; !ok {
-										// create new uconn record if it does not exist
-										uconnMap[srcDstKey] = &uconn.Input{
-											Hosts:      srcDstPair,
-											IsLocalSrc: util.ContainsIP(fs.GetInternalSubnets(), srcIP),
-											IsLocalDst: util.ContainsIP(fs.GetInternalSubnets(), dstIP),
-										}
+								// Check if uconn map value is set, because this record could
+								// come before a relevant uconns record (or may be the only source
+								// for the uconns record)
+								if _, ok := uconnMap[srcDstKey]; !ok {
+									// create new uconn record if it does not exist
+									uconnMap[srcDstKey] = &uconn.Input{
+										Hosts:      srcDstPair,
+										IsLocalSrc: util.ContainsIP(fs.GetInternalSubnets(), srcIP),
+										IsLocalDst: util.ContainsIP(fs.GetInternalSubnets(), dstIP),
 									}
+								}
+
+								//if there's any problem in the certificate, mark it invalid
+								if certStatus != "ok" && certStatus != "-" && certStatus != "" && certStatus != " " {
 									// mark as having invalid cert
 									uconnMap[srcDstKey].InvalidCertFlag = true
 
@@ -829,6 +832,7 @@ func (fs *FSImporter) parseFiles(indexedFiles []*fpt.IndexedFile, parsingThreads
 									certMap[dstKey].OrigIps.Insert(srcUniqIP)
 								}
 							}
+
 							mutex.Unlock()
 						}
 					}
@@ -877,7 +881,7 @@ func (fs *FSImporter) buildCertificates(certMap map[string]*certificate.Input) {
 		}
 		certificateRepo.Upsert(certMap)
 	} else {
-		fmt.Println("\t[!] No certificate data to analyze")
+		fmt.Println("\t[!] No invalid certificate data to analyze")
 	}
 
 }
diff --git a/pkg/remover/mongodb.go b/pkg/remover/mongodb.go
index d15e0eb9..39d435ba 100644
--- a/pkg/remover/mongodb.go
+++ b/pkg/remover/mongodb.go
@@ -98,6 +98,7 @@ func (r *remover) removeOutdatedCIDs(cid int) error {
 		r.res.Config.T.Structure.UniqueConnTable,
 		r.res.Config.T.DNS.ExplodedDNSTable,
 		r.res.Config.T.DNS.HostnamesTable,
+		r.res.Config.T.Cert.CertificateTable,
 		r.res.Config.T.UserAgent.UserAgentTable,
 	}