From 2c6966d7db104d8ff87148a578a07c506b9a8330 Mon Sep 17 00:00:00 2001 From: Manabu Niseki Date: Sat, 20 Jan 2024 17:24:38 +0900 Subject: [PATCH] refactor: renew enrichers --- lib/mihari/enrichers/base.rb | 66 +++- lib/mihari/enrichers/google_public_dns.rb | 33 +- lib/mihari/enrichers/mmdb.rb | 32 +- lib/mihari/enrichers/shodan.rb | 39 ++- lib/mihari/enrichers/whois.rb | 26 +- lib/mihari/models/alert.rb | 12 + lib/mihari/models/artifact.rb | 285 +++++++----------- lib/mihari/models/rule.rb | 21 ++ lib/mihari/rule.rb | 11 +- lib/mihari/schemas/macros.rb | 4 +- lib/mihari/services/builders.rb | 153 ---------- lib/mihari/services/enrichers.rb | 2 +- lib/mihari/services/getters.rb | 2 +- spec/commands/search_spec.rb | 4 +- spec/enrichers/google_public_dns_spec.rb | 6 +- spec/enrichers/mmdb_spec.rb | 8 +- spec/enrichers/shodan_spec.rb | 6 +- spec/enrichers/whois_spec.rb | 10 +- spec/models/artifact_spec.rb | 171 +---------- .../autonomous_system_builder_spec.rb | 12 - spec/services/cpe_builder_spec.rb | 12 - spec/services/dns_record_builder_spec.rb | 15 - spec/services/geolocation_builder_spec.rb | 13 - spec/services/port_builder_spec.rb | 12 - spec/services/reverse_dns_builder_spec.rb | 12 - spec/services/vulnerability_builder_spec.rb | 12 - spec/services/whois_builder_spec.rb | 22 -- 27 files changed, 328 insertions(+), 673 deletions(-) delete mode 100644 spec/services/autonomous_system_builder_spec.rb delete mode 100644 spec/services/cpe_builder_spec.rb delete mode 100644 spec/services/dns_record_builder_spec.rb delete mode 100644 spec/services/geolocation_builder_spec.rb delete mode 100644 spec/services/port_builder_spec.rb delete mode 100644 spec/services/reverse_dns_builder_spec.rb delete mode 100644 spec/services/vulnerability_builder_spec.rb delete mode 100644 spec/services/whois_builder_spec.rb diff --git a/lib/mihari/enrichers/base.rb b/lib/mihari/enrichers/base.rb index 2aef9eaef..9e52583bf 100644 --- a/lib/mihari/enrichers/base.rb +++ b/lib/mihari/enrichers/base.rb @@ -6,46 +6,88 @@ module Enrichers # Base class for enrichers # class Base < Actor - prepend MemoWise - + # + # @param [Hash, nil] options + # def initialize(options: nil) super(options: options) end # - # @param [String] value + # Enrich an artifact + # + # @param [Mihari::Models::Artifact] artifact # - def call(value) + # @return [Mihari::Models::Artifact] + # + def call(artifact) raise NotImplementedError, "You must implement #{self.class}##{__method__}" end # - # @param [Mihari::Models::Artifact] value + # @param [Mihari::Models::Artifact] artifact # # @return [Dry::Monads::Result::Success, Dry::Monads::Result::Failure] # - def result(value) + def result(artifact) + return unless callable?(artifact) + result = Try[StandardError] do - retry_on_error( - times: retry_times, - interval: retry_interval, - exponential_backoff: retry_exponential_backoff - ) { call value } + retry_on_error(times: retry_times, interval: retry_interval, + exponential_backoff: retry_exponential_backoff) do + call artifact + end end.to_result if result.failure? - Mihari.logger.warn("Enricher:#{self.class.key} for #{value.truncate(32)} failed: #{result.failure}") + Mihari.logger.warn("Enricher:#{self.class.key} for #{artifact.data.truncate(32)} failed: #{result.failure}") end result end + # + # @param [Mihari::Models::Artifact] artifact + # + # @return [Boolean] + # + def callable?(artifact) + callable_data_type?(artifact) && callable_relationships?(artifact) + end + class << self def inherited(child) super Mihari.enrichers << child end end + + private + + # + # @param [Mihari::Models::Artifact] artifact + # + # @return [Boolean] + # + def callable_data_type?(artifact) + supported_data_types.include? artifact.data_type + end + + # + # @param [Mihari::Models::Artifact] artifact + # + # @return [Boolean] + # + def callable_relationships?(artifact) + raise NotImplementedError, "You must implement #{self.class}##{__method__}" + end + + # + # @return [Array] + # + def supported_data_types + [] + end end end end diff --git a/lib/mihari/enrichers/google_public_dns.rb b/lib/mihari/enrichers/google_public_dns.rb index 5ad4128d1..9a1f54357 100644 --- a/lib/mihari/enrichers/google_public_dns.rb +++ b/lib/mihari/enrichers/google_public_dns.rb @@ -7,14 +7,20 @@ module Enrichers # class GooglePublicDNS < Base # - # Query Google Public DNS + # @param [Mihari::Models::Artifact] artifact # - # @param [String] name + # @return [Mihari::Models::Artifact] # - # @return [Mihari::Structs::GooglePublicDNS::Response] - # - def call(name) - client.query_all name + def call(artifact) + res = client.query_all(artifact.domain) + + artifact.tap do |tapped| + if tapped.dns_records.empty? + tapped.dns_records = res.answers.map do |answer| + Models::DnsRecord.new(resource: answer.resource_type, value: answer.data) + end + end + end end class << self @@ -28,8 +34,21 @@ def key private + # + # @param [Mihari::Models::Artifact] artifact + # + # @return [Boolean] + # + def callable_relationships?(artifact) + artifact.dns_records.empty? + end + + def supported_data_types + %w[url domain] + end + def client - Clients::GooglePublicDNS.new(timeout: timeout) + @client ||= Clients::GooglePublicDNS.new(timeout: timeout) end end end diff --git a/lib/mihari/enrichers/mmdb.rb b/lib/mihari/enrichers/mmdb.rb index e66ec266a..9737a1171 100644 --- a/lib/mihari/enrichers/mmdb.rb +++ b/lib/mihari/enrichers/mmdb.rb @@ -7,18 +7,36 @@ module Enrichers # class MMDB < Base # - # Query MMDB + # @param [Mihari::Models::Artifact] artifact # - # @param [String] ip + def call(artifact) + res = client.query(artifact.data) + + artifact.tap do |tapped| + tapped.autonomous_system ||= Models::AutonomousSystem.new(number: res.asn) if res.asn + if res.country_code + tapped.geolocation ||= Models::Geolocation.new( + country: NormalizeCountry(res.country_code, to: :short), + country_code: res.country_code + ) + end + end + end + + private + + # + # @param [Mihari::Models::Artifact] artifact # - # @return [Mihari::Structs::MMDB::Response] + # @return [Boolean] # - def call(ip) - client.query ip + def callable_relationships?(artifact) + artifact.geolocation.nil? || artifact.autonomous_system.nil? end - memo_wise :call - private + def supported_data_types + %w[ip] + end def client @client ||= Clients::MMDB.new(timeout: timeout) diff --git a/lib/mihari/enrichers/shodan.rb b/lib/mihari/enrichers/shodan.rb index 01a54ae16..5c3cfabfd 100644 --- a/lib/mihari/enrichers/shodan.rb +++ b/lib/mihari/enrichers/shodan.rb @@ -9,17 +9,48 @@ class Shodan < Base # # Query Shodan Internet DB # - # @param [String] ip + # @param [Mihari::Models::Artifact] artifact # # @return [Mihari::Structs::Shodan::InternetDBResponse, nil] # - def call(ip) - client.query ip + def call(artifact) + res = client.query(artifact.data) + + artifact.tap do |tapped| + tapped.cpes = (res&.cpes || []).map { |cpe| Models::CPE.new(name: cpe) } if tapped.cpes.empty? + tapped.ports = (res&.ports || []).map { |port| Models::Port.new(number: port) } if tapped.ports.empty? + if tapped.reverse_dns_names.empty? + tapped.reverse_dns_names = (res&.hostnames || []).map do |name| + Models::ReverseDnsName.new(name: name) + end + end + end + end + + # + # @param [Mihari::Models::Artifact] artifact + # + # @return [Boolean] + # + def callable?(artifact) + false unless supported_data_types.include?(artifact.data_type) end - memo_wise :call private + # + # @param [Mihari::Models::Artifact] artifact + # + # @return [Boolean] + # + def callable_relationships?(artifact) + artifact.cpes.empty? || artifact.ports.empty? || artifact.reverse_dns_names.empty? + end + + def supported_data_types + %w[ip] + end + def client @client ||= Clients::ShodanInternetDB.new(timeout: timeout) end diff --git a/lib/mihari/enrichers/whois.rb b/lib/mihari/enrichers/whois.rb index 1f9350217..63cff39ac 100644 --- a/lib/mihari/enrichers/whois.rb +++ b/lib/mihari/enrichers/whois.rb @@ -8,26 +8,34 @@ module Enrichers # Whois enricher # class Whois < Base - # - # @param [Hash, nil] options - # - def initialize(options: nil) - super(options: options) - end + prepend MemoWise # # Query IAIA Whois API # - # @param [String] domain + # @param [Mihari::Models::Artifact] artifact # # @return [Mihari::Models::WhoisRecord, nil] # - def call(domain) - memoized_call PublicSuffix.domain(domain) + def call(artifact) + artifact.whois_record ||= memoized_call(PublicSuffix.domain(artifact.domain)) end private + # + # @param [Mihari::Models::Artifact] artifact + # + # @return [Boolean] + # + def callable_relationships?(artifact) + artifact.whois_record.nil? + end + + def supported_data_types + %w[url domain] + end + # # @param [String] domain # diff --git a/lib/mihari/models/alert.rb b/lib/mihari/models/alert.rb index a0380162f..a6704f28e 100644 --- a/lib/mihari/models/alert.rb +++ b/lib/mihari/models/alert.rb @@ -6,6 +6,18 @@ module Models # Alert model # class Alert < ActiveRecord::Base + # @!attribute [r] id + # @return [Integer, nil] + + # @!attribute [rw] created_at + # @return [DateTime] + + # @!attribute [r] rule + # @return [Mihari::Models::Rule] + + # @!attribute [r] artifacts + # @return [Array] + belongs_to :rule has_many :artifacts, dependent: :destroy diff --git a/lib/mihari/models/artifact.rb b/lib/mihari/models/artifact.rb index 25085d4c4..3b190e8dc 100644 --- a/lib/mihari/models/artifact.rb +++ b/lib/mihari/models/artifact.rb @@ -17,6 +17,90 @@ def validate(record) # Artifact model # class Artifact < ActiveRecord::Base + # @!attribute [r] id + # @return [Integer, nil] + + # @!attribute [rw] data + # @return [String] + + # @!attribute [rw] data_type + # @return [String] + + # @!attribute [rw] source + # @return [String, nil] + + # @!attribute [rw] query + # @return [String, nil] + + # @!attribute [rw] metadata + # @return [Hash, nil] + + # @!attribute [rw] created_at + # @return [DateTime] + + # @!attribute [r] alert + # @return [Mihari::Models::Alert] + + # @!attribute [r] rule + # @return [Mihari::Models::Rule] + + # @!attribute [rw] autonomous_system + # @return [Mihari::Models::AutonomousSystem, nil] + + # @!attribute [rw] geolocation + # @return [Mihari::Models::Geolocation, nil] + + # @!attribute [rw] whois_record + # @return [Mihari::Models::WhoisRecord, nil] + + # @!attribute [rw] cpes + # @return [Array] + + # @!attribute [rw] dns_records + # @return [Array] + + # @!attribute [rw] ports + # @return [Array] + + # @!attribute [rw] reverse_dns_names + # @return [Array] + + # @!attribute [rw] vulnerabilities + # @return [Array] + + # @!attribute [r] alert + # @return [Mihari::Models::Alert] + + # @!attribute [r] rule + # @return [Mihari::Models::Rule] + + # @!attribute [rw] autonomous_system + # @return [Mihari::Models::AutonomousSystem, nil] + + # @!attribute [rw] geolocation + # @return [Mihari::Models::Geolocation, nil] + + # @!attribute [rw] whois_record + # @return [Mihari::Models::WhoisRecord, nil] + + # @!attribute [rw] cpes + # @return [Array] + + # @!attribute [rw] dns_records + # @return [Array] + + # @!attribute [rw] ports + # @return [Array] + + # @!attribute [rw] reverse_dns_names + # @return [Array] + + # @!attribute [rw] vulnerabilities + # @return [Array] + + # @!attribute [rw] tags + # @return [Array] + belongs_to :alert has_one :autonomous_system, dependent: :destroy @@ -90,173 +174,23 @@ def unique?(base_time: nil, artifact_ttl: nil) artifact.created_at < decayed_at end - # - # Enrich whois record - # - # @param [Mihari::Enrichers::Whois] enricher - # - def enrich_whois(enricher = Enrichers::Whois.new) - return unless can_enrich_whois? - - self.whois_record = Services::WhoisRecordBuilder.call(domain, enricher: enricher) - end - - # - # Enrich DNS records - # - # @param [Mihari::Enrichers::GooglePublicDNS] enricher - # - def enrich_dns(enricher = Enrichers::GooglePublicDNS.new) - return unless can_enrich_dns? - - self.dns_records = Services::DnsRecordBuilder.call(domain, enricher: enricher) - end - - # - # Enrich reverse DNS names - # - # @param [Mihari::Enrichers::Shodan] enricher - # - def enrich_reverse_dns(enricher = Enrichers::Shodan.new) - return unless can_enrich_reverse_dns? - - self.reverse_dns_names = Services::ReverseDnsNameBuilder.call(data, enricher: enricher) - end - - # - # Enrich geolocation - # - # @param [Mihari::Enrichers::IPInfo] enricher - # - def enrich_geolocation(enricher = Enrichers::MMDB.new) - return unless can_enrich_geolocation? - - self.geolocation = Services::GeolocationBuilder.call(data, enricher: enricher) - end - - # - # Enrich AS - # - # @param [Mihari::Enrichers::IPInfo] enricher - # - def enrich_autonomous_system(enricher = Enrichers::MMDB.new) - return unless can_enrich_autonomous_system? - - self.autonomous_system = Services::AutonomousSystemBuilder.call(data, enricher: enricher) - end - - # - # Enrich ports - # - # @param [Mihari::Enrichers::Shodan] enricher - # - def enrich_ports(enricher = Enrichers::Shodan.new) - return unless can_enrich_ports? - - self.ports = Services::PortBuilder.call(data, enricher: enricher) - end - - # - # Enrich CPEs - # - # @param [Mihari::Enrichers::Shodan] enricher - # - def enrich_cpes(enricher = Enrichers::Shodan.new) - return unless can_enrich_cpes? - - self.cpes = Services::CPEBuilder.call(data, enricher: enricher) - end - - # - # Enrich vulnerabilities - # - # @param [Mihari::Enrichers::Shodan] enricher - # - def enrich_vulnerabilities(enricher = Enrichers::Shodan.new) - return unless can_enrich_vulnerabilities? - - self.vulnerabilities = Services::VulnerabilityBuilder.call(data, enricher: enricher) + def enrichable? + !callable_enrichers.empty? end - # - # Enrich all the enrichable relationships of the artifact - # - def enrich_all - enrich_autonomous_system mmdb - enrich_dns - enrich_geolocation mmdb - enrich_reverse_dns shodan - enrich_whois - enrich_ports shodan - enrich_cpes shodan - enrich_vulnerabilities shodan + def enrich + callable_enrichers.each { |enricher| enricher.result self } end - ENRICH_METHODS_BY_ENRICHER = { - Enrichers::Whois => %i[ - enrich_whois - ], - Enrichers::MMDB => %i[ - enrich_autonomous_system - enrich_geolocation - ], - Enrichers::Shodan => %i[ - enrich_ports - enrich_cpes - enrich_reverse_dns - enrich_vulnerabilities - ], - Enrichers::GooglePublicDNS => %i[ - enrich_dns - ] - }.freeze - - # - # Enrich by name of enricher # - # @param [Mihari::Enrichers::Base] enricher + # @return [String, nil] # - def enrich_by_enricher(enricher) - methods = ENRICH_METHODS_BY_ENRICHER[enricher.class] || [] - methods.each { |method| send(method, enricher) if respond_to?(method) } - end - - def can_enrich_whois? - %w[domain url].include?(data_type) && whois_record.nil? - end - - def can_enrich_dns? - %w[domain url].include?(data_type) && dns_records.empty? - end - - def can_enrich_reverse_dns? - data_type == "ip" && reverse_dns_names.empty? - end - - def can_enrich_geolocation? - data_type == "ip" && geolocation.nil? - end - - def can_enrich_autonomous_system? - data_type == "ip" && autonomous_system.nil? - end - - def can_enrich_ports? - data_type == "ip" && ports.empty? - end - - def can_enrich_cpes? - data_type == "ip" && cpes.empty? - end - - def can_enrich_vulnerabilities? - data_type == "ip" && vulnerabilities.empty? - end - - def enrichable? - enrich_methods = methods.map(&:to_s).select { |method| method.start_with?("can_enrich_") } - enrich_methods.map(&:to_sym).any? do |method| - send(method) if respond_to?(method) + def domain + case data_type + when "domain" + data + when "url" + Addressable::URI.parse(data).host end end @@ -272,6 +206,15 @@ class << self private + # + # @return [Array] + # + def callable_enrichers + @callable_enrichers ||= Mihari.enrichers.map(&:new).select do |enricher| + enricher.callable?(self) + end + end + def set_data_type self.data_type = DataType.type(data) end @@ -279,26 +222,6 @@ def set_data_type def set_rule_id @set_rule_id ||= nil end - - def mmdb - @mmdb ||= Enrichers::MMDB.new - end - - def shodan - @shodan ||= Enrichers::Shodan.new - end - - # - # @return [String, nil] - # - def domain - case data_type - when "domain" - data - when "url" - Addressable::URI.parse(data).host - end - end end end end diff --git a/lib/mihari/models/rule.rb b/lib/mihari/models/rule.rb index 7044a0963..ebc2de62c 100644 --- a/lib/mihari/models/rule.rb +++ b/lib/mihari/models/rule.rb @@ -6,6 +6,27 @@ module Models # Rule model # class Rule < ActiveRecord::Base + # @!attribute [rw] id + # @return [String] + + # @!attribute [rw] title + # @return [String] + + # @!attribute [rw] description + # @return [String] + + # @!attribute [rw] data + # @return [Hash] + + # @!attribute [rw] created_at + # @return [DateTime] + + # @!attribute [rw] updated_at + # @return [DateTime] + + # @!attribute [r] alerts + # @return [Array] + has_many :alerts, dependent: :destroy has_many :taggings, dependent: :destroy has_many :tags, through: :taggings diff --git a/lib/mihari/rule.rb b/lib/mihari/rule.rb index cafb5c2c2..31a11984c 100644 --- a/lib/mihari/rule.rb +++ b/lib/mihari/rule.rb @@ -174,8 +174,11 @@ def unique_artifacts # @return [Array] # def enriched_artifacts + # TODO: same whois query can be issued multiple times @enriched_artifacts ||= Parallel.map(unique_artifacts) do |artifact| - enrichers.each { |enricher| artifact.enrich_by_enricher enricher } + enrichers.each do |enricher| + enricher.result(artifact) if enricher.callable?(artifact) + end artifact end end @@ -289,11 +292,11 @@ def diff_comparable_data # # @return [Boolean] # - def falsepositive?(value) - return true if falsepositives.include?(value) + def falsepositive?(artifact) + return true if falsepositives.include?(artifact) regexps = falsepositives.select { |fp| fp.is_a?(Regexp) } - regexps.any? { |fp| fp.match?(value) } + regexps.any? { |fp| fp.match?(artifact) } end # diff --git a/lib/mihari/schemas/macros.rb b/lib/mihari/schemas/macros.rb index eba2be35d..545686523 100644 --- a/lib/mihari/schemas/macros.rb +++ b/lib/mihari/schemas/macros.rb @@ -8,9 +8,9 @@ module Macros # (see https://github.com/dry-rb/dry-schema/issues/70) # class DSL - def default(value) + def default(artifact) schema_dsl.before(:rule_applier) do |result| - result.update(name => value) if result.output && !result[name] + result.update(name => artifact) if result.output && !result[name] end end end diff --git a/lib/mihari/services/builders.rb b/lib/mihari/services/builders.rb index 057be64a0..42c4cab43 100644 --- a/lib/mihari/services/builders.rb +++ b/lib/mihari/services/builders.rb @@ -20,158 +20,5 @@ def call(path_or_id) Rule.from_yaml ERB.new(File.read(path_or_id)).result end end - - # - # Autonomous system builder - # - class AutonomousSystemBuilder < Service - # - # @param [String] ip - # @param [Mihari::Enrichers::MMDB] enricher - # - # @return [Mihari::Models::AutonomousSystem, nil] - # - def call(ip, enricher: Enrichers::MMDB.new) - enricher.result(ip).fmap do |res| - Models::AutonomousSystem.new(number: res.asn) if res.asn - end.value_or nil - end - end - - # - # CPE builder - # - class CPEBuilder < Service - # - # Build CPEs - # - # @param [String] ip - # @param [Mihari::Enrichers::Shodan] enricher - # - # @return [Array] - # - def call(ip, enricher: Enrichers::Shodan.new) - enricher.result(ip).fmap do |res| - (res&.cpes || []).map { |cpe| Models::CPE.new(name: cpe) } - end.value_or [] - end - end - - # - # DNS record builder - # - class DnsRecordBuilder < Service - # - # Build DNS records - # - # @param [String] domain - # @param [Mihari::Enrichers::Shodan] enricher - # - # @return [Array] - # - def call(domain, enricher: Enrichers::GooglePublicDNS.new) - enricher.result(domain).fmap do |res| - res.answers.map { |answer| Models::DnsRecord.new(resource: answer.resource_type, value: answer.data) } - end.value_or [] - end - end - - # - # Geolocation builder - # - class GeolocationBuilder < Service - # - # Build Geolocation - # - # @param [String] ip - # @param [Mihari::Enrichers::MMDB] enricher - # - # @return [Mihari::Models::Geolocation, nil] - # - def call(ip, enricher: Enrichers::MMDB.new) - enricher.result(ip).fmap do |res| - if res.country_code - Models::Geolocation.new( - country: NormalizeCountry(res.country_code, to: :short), - country_code: res.country_code - ) - end - end.value_or nil - end - end - - # - # Port builder - # - class PortBuilder < Service - # - # Build ports - # - # @param [String] ip - # @param [Mihari::Enrichers::Shodan] enricher - # - # @return [Array] - # - def call(ip, enricher: Enrichers::Shodan.new) - enricher.result(ip).fmap do |res| - (res&.ports || []).map { |port| Models::Port.new(number: port) } - end.value_or [] - end - end - - # - # Reverse DNS name builder - # - class ReverseDnsNameBuilder < Service - # - # Build reverse DNS names - # - # @param [String] ip - # @param [Mihari::Enrichers::Shodan] enricher - # - # @return [Array] - # - def call(ip, enricher: Enrichers::Shodan.new) - enricher.result(ip).fmap do |res| - (res&.hostnames || []).map { |name| Models::ReverseDnsName.new(name: name) } - end.value_or [] - end - end - - # - # Vulnerability builder - # - class VulnerabilityBuilder < Service - # - # Build vulnerabilities - # - # @param [String] ip - # @param [Mihari::Enrichers::Shodan] enricher - # - # @return [Array] - # - def call(ip, enricher: Enrichers::Shodan.new) - enricher.result(ip).fmap do |res| - (res&.vulns || []).map { |name| Models::Vulnerability.new(name: name) } - end.value_or [] - end - end - - # - # Whois record builder - # - class WhoisRecordBuilder < Service - # - # Build whois record - # - # @param [String] domain - # @param [Mihari::Enrichers::Whois] enricher - # - # @return [Mihari::Models::WhoisRecord, nil] - # - def call(domain, enricher: Enrichers::Whois.new) - enricher.result(domain).value_or nil - end - end end end diff --git a/lib/mihari/services/enrichers.rb b/lib/mihari/services/enrichers.rb index b2c15738f..4c676a5a0 100644 --- a/lib/mihari/services/enrichers.rb +++ b/lib/mihari/services/enrichers.rb @@ -19,7 +19,7 @@ def call(id) raise UnenrichableError.new, "#{artifact.id} is already enriched or unenrichable" unless artifact.enrichable? - artifact.enrich_all + artifact.enrich artifact.save end end diff --git a/lib/mihari/services/getters.rb b/lib/mihari/services/getters.rb index 18ffd0d23..0635ca606 100644 --- a/lib/mihari/services/getters.rb +++ b/lib/mihari/services/getters.rb @@ -51,7 +51,7 @@ class IPGetter < Service # @return [Mihari::Structs::MMDB::Response] # def call(ip) - Mihari::Enrichers::MMDB.new.call ip + Clients::MMDB.new.query ip end end end diff --git a/spec/commands/search_spec.rb b/spec/commands/search_spec.rb index 6962e4925..a296781f4 100644 --- a/spec/commands/search_spec.rb +++ b/spec/commands/search_spec.rb @@ -12,9 +12,7 @@ class SearchCLI < Mihari::CLI::Base let!(:rule) { Mihari::Rule.from_yaml yaml } before do - double = class_double(Mihari::Services::WhoisRecordBuilder) - allow(double).to receive(:call).and_return(nil) - + allow(rule).to receive(:enrichers).and_return([]) allow(Parallel).to receive(:processor_count).and_return(0) end diff --git a/spec/enrichers/google_public_dns_spec.rb b/spec/enrichers/google_public_dns_spec.rb index bb32026c3..cdc0d2c3e 100644 --- a/spec/enrichers/google_public_dns_spec.rb +++ b/spec/enrichers/google_public_dns_spec.rb @@ -4,11 +4,11 @@ describe ".query_by_type" do subject(:enricher) { described_class.new } - let!(:name) { "example.com" } + let!(:artifact) { Mihari::Models::Artifact.new(data: "example.com") } it do - res = enricher.call(name) - expect(res.answers.first.data).to eq("93.184.216.34") + enricher.call artifact + expect(artifact.dns_records.first.value).to eq("93.184.216.34") end end end diff --git a/spec/enrichers/mmdb_spec.rb b/spec/enrichers/mmdb_spec.rb index afc92e6c0..145850c76 100644 --- a/spec/enrichers/mmdb_spec.rb +++ b/spec/enrichers/mmdb_spec.rb @@ -4,12 +4,12 @@ subject(:enricher) { described_class.new } describe ".call" do - let!(:ip) { "1.1.1.1" } + let!(:artifact) { Mihari::Models::Artifact.new(data: "1.1.1.1") } it do - res = enricher.call(ip) - expect(res.asn).to eq(13_335) - expect(res.country_code).to eq("US") + enricher.call artifact + expect(artifact.autonomous_system.number).to eq(13_335) + expect(artifact.geolocation.country_code).to eq("US") end end end diff --git a/spec/enrichers/shodan_spec.rb b/spec/enrichers/shodan_spec.rb index e90e1fe80..e522de289 100644 --- a/spec/enrichers/shodan_spec.rb +++ b/spec/enrichers/shodan_spec.rb @@ -4,11 +4,11 @@ subject(:enricher) { described_class.new } describe ".call" do - let!(:ip) { "1.1.1.1" } + let!(:artifact) { Mihari::Models::Artifact.new(data: "1.1.1.1") } it do - res = enricher.call(ip) - expect(res.hostnames).to eq(["one.one.one.one"]) + enricher.call artifact + expect(artifact.reverse_dns_names.map(&:name)).to eq(["one.one.one.one"]) end end end diff --git a/spec/enrichers/whois_spec.rb b/spec/enrichers/whois_spec.rb index f35843c7c..edda005b8 100644 --- a/spec/enrichers/whois_spec.rb +++ b/spec/enrichers/whois_spec.rb @@ -4,13 +4,13 @@ subject(:enricher) { described_class.new } describe "#call" do - let!(:domain) { "example.com" } + let!(:artifact) { Mihari::Models::Artifact.new(data: "example.com") } it do - whois_record = enricher.call(domain) - expect(whois_record.created_on&.iso8601).to eq("1992-01-01") - expect(whois_record.registrar).to be_a(Hash).or be_a(NilClass) - expect(whois_record.contacts).to be_a(Array) + enricher.call artifact + expect(artifact.whois_record.created_on&.iso8601).to eq("1992-01-01") + expect(artifact.whois_record.registrar).to be_a(Hash).or be_a(NilClass) + expect(artifact.whois_record.contacts).to be_a(Array) end end end diff --git a/spec/models/artifact_spec.rb b/spec/models/artifact_spec.rb index 789ddc1a7..560cddbf2 100644 --- a/spec/models/artifact_spec.rb +++ b/spec/models/artifact_spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true -RSpec.describe Mihari::Models::Artifact, :vcr do - let_it_be(:artifact) { FactoryBot.create(:artifact) } +RSpec.describe Mihari::Models::Artifact do + let_it_be(:artifact) { FactoryBot.create(:artifact, :ip) } let_it_be(:tag) { artifact.tags.first } let_it_be(:alert) { artifact.alert } let_it_be(:rule) { artifact.rule } @@ -63,173 +63,16 @@ end end - describe "#enrich_whois" do - let!(:enricher) do - enricher = instance_double("whois_enricher") - allow(enricher).to receive(:result).and_return( - Dry::Monads::Result::Success.new( - Mihari::Models::WhoisRecord.new(domain: "example.com") - ) - ) - enricher - end - - context "with domain" do - let(:data) { "example.com" } - let(:artifact) { described_class.new(data: data, alert_id: alert.id) } - - it do - expect(artifact.whois_record).to be_nil - end - - it do - artifact.enrich_whois enricher - expect(artifact.whois_record).not_to be_nil - end - end - - context "with URL" do - let(:data) { "https://example.com" } - let(:artifact) { described_class.new(data: data, alert_id: alert.id) } - - it do - expect(artifact.whois_record).to be_nil - end - - it do - artifact.enrich_whois enricher - expect(artifact.whois_record).not_to be_nil - end - end - end - - describe "#enrich_dns" do - context "with domain" do - let(:data) { "example.com" } - let(:artifact) { described_class.new(data: data, alert_id: alert.id) } - - it do - expect(artifact.dns_records.length).to eq(0) - end - - it do - artifact.enrich_dns - expect(artifact.dns_records.length).to be > 0 - end - end - - context "with URL" do - let(:data) { "https://example.com" } - let(:artifact) { described_class.new(data: data, alert_id: alert.id) } - - it do - artifact = described_class.new(data: data, alert_id: alert.id) - expect(artifact.dns_records.length).to eq(0) - end - - it do - artifact.enrich_dns - expect(artifact.dns_records.length).to be > 0 - end - end - end - - describe "#enrich_geolocation" do - let(:data) { "1.1.1.1" } - let(:artifact) { described_class.new(data: data, alert_id: alert.id) } - + describe "#enrichable?" do it do - expect(artifact.geolocation).to eq(nil) + expect(artifact.enrichable?).to eq(true) end - it do - artifact.enrich_geolocation - expect(artifact.geolocation.country_code).to eq("US") - end - end - - describe "#enrich_autonomous_system" do - let(:data) { "1.1.1.1" } - let(:artifact) { described_class.new(data: data, alert_id: alert.id) } - - it do - expect(artifact.autonomous_system).to eq(nil) - end - - it do - artifact.enrich_autonomous_system - expect(artifact.autonomous_system.number).to eq(13_335) - end - end - - describe "#enrich_by_enricher" do - context "with IPInfo" do - let(:data) { "1.1.1.1" } - let(:artifact) { described_class.new(data: data, alert_id: alert.id) } - - it do - expect(artifact.autonomous_system).to eq(nil) - expect(artifact.geolocation).to eq(nil) - end - - it do - artifact.enrich_by_enricher Mihari::Enrichers::MMDB.new - expect(artifact.autonomous_system).not_to eq(nil) - expect(artifact.geolocation).not_to eq(nil) - end - end - - context "with Shodan" do - let(:data) { "1.1.1.1" } - let(:artifact) { described_class.new(data: data, alert_id: alert.id) } - - it do - expect(artifact.reverse_dns_names.empty?).to be true - expect(artifact.ports.empty?).to be true - end - - it do - artifact.enrich_by_enricher Mihari::Enrichers::Shodan.new - expect(artifact.reverse_dns_names.empty?).to be false - expect(artifact.ports.empty?).to be false - end - end - - context "with Google Public DNS" do - let(:data) { "example.com" } - let(:artifact) { described_class.new(data: data, alert_id: alert.id) } - - it do - expect(artifact.dns_records.empty?).to be true - end - - it do - artifact.enrich_by_enricher(Mihari::Enrichers::GooglePublicDNS.new) - expect(artifact.dns_records.empty?).to be false - end - end - - context "with Whois" do - let(:data) { "example.com" } - let(:artifact) { described_class.new(data: data, alert_id: alert.id) } - let!(:enricher) do - enricher = instance_double("whois_enricher") - allow(enricher).to receive(:result).and_return( - Dry::Monads::Result::Success.new( - Mihari::Models::WhoisRecord.new(domain: data) - ) - ) - allow(enricher).to receive(:class).and_return(Mihari::Enrichers::Whois) - enricher - end - - it do - expect(artifact.whois_record).to be_nil - end + context "with unenrichable artifact" do + let(:artifact) { FactoryBot.build(:artifact, :unenrichable) } it do - artifact.enrich_by_enricher(enricher) - expect(artifact.whois_record).not_to be_nil + expect(artifact.enrichable?).to eq(false) end end end diff --git a/spec/services/autonomous_system_builder_spec.rb b/spec/services/autonomous_system_builder_spec.rb deleted file mode 100644 index bbc2ae907..000000000 --- a/spec/services/autonomous_system_builder_spec.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -RSpec.describe Mihari::Services::AutonomousSystemBuilder, vcr: "Mihari_Enrichers_MMDB/ip:1.1.1.1" do - describe ".call" do - let!(:ip) { "1.1.1.1" } - - it do - as = described_class.call(ip) - expect(as.number).to eq(13_335) - end - end -end diff --git a/spec/services/cpe_builder_spec.rb b/spec/services/cpe_builder_spec.rb deleted file mode 100644 index 684818620..000000000 --- a/spec/services/cpe_builder_spec.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -RSpec.describe Mihari::Services::CPEBuilder, vcr: "Mihari_Enrichers_Shodan/ip:1.1.1.1" do - describe ".call" do - let!(:ip) { "1.1.1.1" } - - it do - names = described_class.call(ip) - expect(names).to be_an(Array) - end - end -end diff --git a/spec/services/dns_record_builder_spec.rb b/spec/services/dns_record_builder_spec.rb deleted file mode 100644 index 6d51b3b8b..000000000 --- a/spec/services/dns_record_builder_spec.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -RSpec.describe Mihari::Services::DnsRecordBuilder, :vcr do - describe ".call" do - let!(:domain) { "example.com" } - - it do - dns_records = described_class.call(domain) - - # example.com has A records and does not have CNAME records - expect(dns_records.any? { |record| record.resource == "A" }).to be true - expect(dns_records.any? { |record| record.resource == "CNAME" }).to be false - end - end -end diff --git a/spec/services/geolocation_builder_spec.rb b/spec/services/geolocation_builder_spec.rb deleted file mode 100644 index f43239c8f..000000000 --- a/spec/services/geolocation_builder_spec.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -RSpec.describe Mihari::Services::GeolocationBuilder, vcr: "Mihari_Enrichers_MMDB/ip:1.1.1.1" do - describe ".call" do - let!(:ip) { "1.1.1.1" } - - it do - geolocation = described_class.call(ip) - expect(geolocation.country_code).to eq("US") - expect(geolocation.country).to eq("United States") - end - end -end diff --git a/spec/services/port_builder_spec.rb b/spec/services/port_builder_spec.rb deleted file mode 100644 index 81384af25..000000000 --- a/spec/services/port_builder_spec.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -RSpec.describe Mihari::Services::PortBuilder, vcr: "Mihari_Enrichers_Shodan/ip:1.1.1.1" do - describe ".call" do - let!(:ip) { "1.1.1.1" } - - it do - names = described_class.call(ip) - expect(names).to be_an(Array) - end - end -end diff --git a/spec/services/reverse_dns_builder_spec.rb b/spec/services/reverse_dns_builder_spec.rb deleted file mode 100644 index a6f465da9..000000000 --- a/spec/services/reverse_dns_builder_spec.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -RSpec.describe Mihari::Services::ReverseDnsNameBuilder, vcr: "Mihari_Enrichers_Shodan/ip:1.1.1.1" do - describe ".call" do - let!(:ip) { "1.1.1.1" } - - it do - names = described_class.call(ip) - expect(names).to be_an(Array) - end - end -end diff --git a/spec/services/vulnerability_builder_spec.rb b/spec/services/vulnerability_builder_spec.rb deleted file mode 100644 index 784b2637b..000000000 --- a/spec/services/vulnerability_builder_spec.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -RSpec.describe Mihari::Services::VulnerabilityBuilder, vcr: "Mihari_Enrichers_Shodan/ip:120.78.59.202" do - describe ".call" do - let!(:ip) { "120.78.59.202" } - - it do - vulnerabilities = described_class.call(ip) - expect(vulnerabilities).to be_an(Array) - end - end -end diff --git a/spec/services/whois_builder_spec.rb b/spec/services/whois_builder_spec.rb deleted file mode 100644 index 72dfc8ed6..000000000 --- a/spec/services/whois_builder_spec.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -RSpec.describe Mihari::Services::WhoisRecordBuilder do - describe ".call" do - let!(:domain) { "example.com" } - - let!(:enricher) do - enricher = instance_double("whois_enricher") - allow(enricher).to receive(:result).and_return( - Dry::Monads::Result::Success.new( - Mihari::Models::WhoisRecord.new(domain: "example.com") - ) - ) - enricher - end - - it do - whois_record = described_class.call(domain, enricher: enricher) - expect(whois_record.domain).to eq(domain) - end - end -end