diff --git a/docs/diagrams.md b/docs/diagrams.md index 1e2ea5666..32d0c3f2a 100644 --- a/docs/diagrams.md +++ b/docs/diagrams.md @@ -37,6 +37,7 @@ classDiagram } Artifact --* Alert Artifact *-- CPE + Artifact *-- Vulnerability Artifact *-- DnsRecord Artifact *-- Port Artifact *-- ReverseDnsName @@ -48,7 +49,7 @@ classDiagram class AutonomousSystem { integer id - integer asn + integer number datetime created_at integer artifact_id } @@ -56,7 +57,7 @@ classDiagram class Port { integer id - integer port + integer number datetime created_at integer artifact_id } @@ -64,12 +65,20 @@ classDiagram class CPE { integer id - string cpe + string name datetime created_at integer artifact_id } CPE --* Artifact + class Vulnerability { + integer id + string name + datetime created_at + integer artifact_id + } + Vulnerability --* Artifact + class DnsRecord { integer id string resource @@ -131,96 +140,103 @@ classDiagram ```mermaid erDiagram alerts { - datetime6 created_at - INTEGER id PK + datetime created_at + integer id PK varchar rule_id FK } artifacts { - INTEGER alert_id FK - datetime6 created_at + integer alert_id FK + datetime created_at varchar data varchar data_type - INTEGER id PK + integer id PK json metadata varchar query varchar source } autonomous_systems { - INTEGER artifact_id FK - INTEGER asn - datetime6 created_at - INTEGER id PK + integer artifact_id FK + integer number + datetime created_at + integer id PK } cpes { - INTEGER artifact_id FK - varchar cpe - datetime6 created_at - INTEGER id PK + integer artifact_id FK + varchar name + datetime created_at + integer id PK + } + + vulnerabilities { + integer artifact_id FK + varchar name + datetime created_at + integer id PK } dns_records { - INTEGER artifact_id FK - datetime6 created_at - INTEGER id PK + integer artifact_id FK + datetime created_at + integer id PK varchar resource varchar value } geolocations { - INTEGER artifact_id FK + integer artifact_id FK varchar country varchar country_code - datetime6 created_at - INTEGER id PK + datetime created_at + integer id PK } ports { - INTEGER artifact_id FK - datetime6 created_at - INTEGER id PK - INTEGER port + integer artifact_id FK + datetime created_at + integer id PK + integer number } reverse_dns_names { - INTEGER artifact_id FK - datetime6 created_at - INTEGER id PK + integer artifact_id FK + datetime created_at + integer id PK varchar name } rules { - datetime6 created_at + datetime created_at json data varchar description varchar id PK varchar title - datetime6 updated_at + datetime updated_at } taggings { - datetime6 created_at - INTEGER id PK + datetime created_at + integer id PK varchar rule_id - INTEGER tag_id + integer tag_id } tags { - datetime6 created_at - INTEGER id PK + datetime created_at + integer id PK varchar name } whois_records { - INTEGER artifact_id FK + integer artifact_id FK json contacts - datetime6 created_at + datetime created_at date created_on varchar domain date expires_on - INTEGER id PK + integer id PK json registrar date updated_on } @@ -229,6 +245,7 @@ erDiagram artifacts }o--|| alerts : "alert_id" autonomous_systems }o--|| artifacts : "artifact_id" cpes }o--|| artifacts : "artifact_id" + vulnerabilities }o--|| artifacts : "artifact_id" dns_records }o--|| artifacts : "artifact_id" geolocations }o--|| artifacts : "artifact_id" ports }o--|| artifacts : "artifact_id" diff --git a/docs/enrichers/google_public_dns.md b/docs/enrichers/google_public_dns.md index 7b313f839..fab0e1c8c 100644 --- a/docs/enrichers/google_public_dns.md +++ b/docs/enrichers/google_public_dns.md @@ -13,6 +13,10 @@ This enricher uses Google Public DNS to enrich an URL and domain artifact. enricher: google_public_dns ``` +This enricher can add the following components: + +- DNS records + ## Supported Artifacts - URL diff --git a/docs/enrichers/mmdb.md b/docs/enrichers/mmdb.md index bb72e77f8..35a3eba85 100644 --- a/docs/enrichers/mmdb.md +++ b/docs/enrichers/mmdb.md @@ -14,6 +14,11 @@ This enricher uses public MMDB API to enrich an IP artifact. enricher: mmdb ``` +This enricher can add the following components: + +- Geolocation +- Autonomous System + ## Supported Artifacts - IP address diff --git a/docs/enrichers/shodan.md b/docs/enrichers/shodan.md index c1b137329..afc89dd66 100644 --- a/docs/enrichers/shodan.md +++ b/docs/enrichers/shodan.md @@ -3,6 +3,7 @@ tags: - Enrichment:Port - Enrichment:CPE - Enrichment:Reverse_DNS_Name + - Enrichment:Vulnerability --- # Shodan (The InternetDB API) @@ -17,6 +18,13 @@ This enricher uses Shodan InternetDB API to enrich an artifact. enricher: shodan ``` +This enricher can add the following components: + +- Ports +- CPEs +- Reverse DNS names +- Vulnerabilities + ## Supported Artifacts - IP address diff --git a/docs/enrichers/whois.md b/docs/enrichers/whois.md index 37b8b9fbb..6ffb2f412 100644 --- a/docs/enrichers/whois.md +++ b/docs/enrichers/whois.md @@ -1,6 +1,6 @@ --- tags: - - Enrichment:Whois + - Enrichment:Whois_Record --- # Whois @@ -11,6 +11,10 @@ This enricher uses “whois” command to enrich an artifact. enricher: whois ``` +This enricher can add the following components: + +- Whois record + ## Supported Artifacts - URL diff --git a/docs/usage.md b/docs/usage.md index b38468894..8cba9ccd1 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -173,12 +173,12 @@ Search query supports `AND`, `OR`, `:`, `=`, `!=`, `<`, `<=`, `>`, `>=`, `NOT` a Searchable fields are: -| Type | Searchable fields | -| ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `alert` | `id`, `tag`, `created_at`, `rule.id`, `rule.title`, `rule.description`, `artifact.data`, `artifact.data_type`, `artifact.source` and `artifact.query` | -| `artifact` | `id`, `data`, `data_type`, `source`, `query`, `tag`, `rule.id`, `rule.title`, `rule.description`, `tag`,`created_at`, `asn`, `country_code`, `dns_record.value`, `dns_record.resource`, `reverse_dns_name`, `cpe` and `port` | -| `rule` | `id`, `title`, `description`, `tag`, `created_at` and `updated_at` | -| `tag` | `id` and `name` | +| Type | Searchable fields | +| ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `alert` | `id`, `tag`, `created_at`, `rule.id`, `rule.title`, `rule.description`, `artifact.data`, `artifact.data_type`, `artifact.source` and `artifact.query` | +| `artifact` | `id`, `data`, `data_type`, `source`, `query`, `tag`, `rule.id`, `rule.title`, `rule.description`, `tag`,`created_at`, `asn`, `country_code`, `dns_record.value`, `dns_record.resource`, `reverse_dns_name`, `cpe`, `vuln` and `port` | +| `rule` | `id`, `title`, `description`, `tag`, `created_at` and `updated_at` | +| `tag` | `id` and `name` | **Examples** diff --git a/frontend/src/components/artifact/AS.vue b/frontend/src/components/artifact/AS.vue index 78198ba88..69a8a9d6e 100644 --- a/frontend/src/components/artifact/AS.vue +++ b/frontend/src/components/artifact/AS.vue @@ -1,7 +1,7 @@ diff --git a/frontend/src/components/artifact/ArtifactDetail.vue b/frontend/src/components/artifact/ArtifactDetail.vue index 8d23400e0..e68338817 100644 --- a/frontend/src/components/artifact/ArtifactDetail.vue +++ b/frontend/src/components/artifact/ArtifactDetail.vue @@ -83,6 +83,10 @@

CPEs

+
+

Vulnerabilities

+ +

Ports

@@ -111,6 +115,7 @@ import DnsRecords from "@/components/artifact/DnsRecords.vue" import Ports from "@/components/artifact/Ports.vue" import ReverseDnsNames from "@/components/artifact/ReverseDnsNames.vue" import Tags from "@/components/artifact/Tags.vue" +import Vulnerabilities from "@/components/artifact/Vulnerabilities.vue" import WhoisRecord from "@/components/artifact/WhoisRecord.vue" import ErrorMessage from "@/components/ErrorMessage.vue" import Links from "@/components/link/Links.vue" @@ -137,7 +142,8 @@ export default defineComponent({ CPEs, Ports, ErrorMessage, - Message + Message, + Vulnerabilities }, emits: ["refresh", "delete"], setup(props, context) { diff --git a/frontend/src/components/artifact/ArtifactsWrapper.vue b/frontend/src/components/artifact/ArtifactsWrapper.vue index 18e9a0a40..7dcb20242 100644 --- a/frontend/src/components/artifact/ArtifactsWrapper.vue +++ b/frontend/src/components/artifact/ArtifactsWrapper.vue @@ -34,8 +34,8 @@ query, tag, rule.id, rule.title, rule.description, tag,created_at, asn, country_code, dns_record.value, - dns_record.resource, reverse_dns_name, cpe and - port. + dns_record.resource, reverse_dns_name, cpe, + vuln and port.
diff --git a/frontend/src/components/artifact/CPEs.vue b/frontend/src/components/artifact/CPEs.vue index 37196df7e..886dd2676 100644 --- a/frontend/src/components/artifact/CPEs.vue +++ b/frontend/src/components/artifact/CPEs.vue @@ -1,7 +1,7 @@ diff --git a/frontend/src/components/artifact/Ports.vue b/frontend/src/components/artifact/Ports.vue index 11afaa3b6..a466883a7 100644 --- a/frontend/src/components/artifact/Ports.vue +++ b/frontend/src/components/artifact/Ports.vue @@ -1,7 +1,7 @@ diff --git a/frontend/src/components/artifact/Vulnerabilities.vue b/frontend/src/components/artifact/Vulnerabilities.vue new file mode 100644 index 000000000..d971752b3 --- /dev/null +++ b/frontend/src/components/artifact/Vulnerabilities.vue @@ -0,0 +1,23 @@ + + + diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 30cfe3fe0..d4c0eb969 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -49,7 +49,7 @@ export interface WhoisRecord { } export interface AutonomousSystem { - asn: number + number: number } export interface Geolocation { @@ -62,11 +62,15 @@ export interface ReverseDnsName { } export interface CPE { - cpe: string + name: string +} + +export interface Vulnerability { + name: string } export interface Port { - port: string + number: string } export interface Artifact { @@ -86,6 +90,7 @@ export interface Artifact { reverseDnsNames?: ReverseDnsName[] cpes?: CPE[] ports?: Port[] + vulnerabilities?: Vulnerability[] } export interface ArtifactWithTags extends Artifact { diff --git a/lib/mihari.rb b/lib/mihari.rb index 450800863..2655300a5 100644 --- a/lib/mihari.rb +++ b/lib/mihari.rb @@ -207,6 +207,7 @@ def initialize_sentry require "mihari/models/rule" require "mihari/models/tag" require "mihari/models/tagging" +require "mihari/models/vulnerability" require "mihari/models/whois" # Emitters @@ -323,6 +324,7 @@ def initialize_sentry require "mihari/entities/port" require "mihari/entities/reverse_dns" require "mihari/entities/tag" +require "mihari/entities/vulnerability" require "mihari/entities/whois" require "mihari/entities/artifact" diff --git a/lib/mihari/database.rb b/lib/mihari/database.rb index aa58e93d5..c5aa0bcb8 100644 --- a/lib/mihari/database.rb +++ b/lib/mihari/database.rb @@ -104,11 +104,28 @@ def change end end +class V72Schema < ActiveRecord::Migration[7.1] + def change + create_table :vulnerabilities, if_not_exists: true do |t| + t.string :name, null: false + t.datetime :created_at + + t.belongs_to :artifact, foreign_key: true, null: false + end + + rename_column :cpes, :cpe, :name if ActiveRecord::Base.connection.column_exists?(:cpes, :cpe) + rename_column :autonomous_systems, :asn, :number if ActiveRecord::Base.connection.column_exists?( + :autonomous_systems, :asn + ) + rename_column :ports, :port, :number if ActiveRecord::Base.connection.column_exists?(:ports, :port) + end +end + # # @return [Array] schemas # def schemas - [V7Schema] + [V7Schema, V72Schema] end module Mihari diff --git a/lib/mihari/entities/artifact.rb b/lib/mihari/entities/artifact.rb index f394aefb8..d2507eb62 100644 --- a/lib/mihari/entities/artifact.rb +++ b/lib/mihari/entities/artifact.rb @@ -36,6 +36,10 @@ class Artifact < BaseArtifact as: :ports do |status, _options| status.ports.empty? ? nil : status.ports end + expose :vulnerabilities, using: Vulnerability, documentation: { type: Vulnerability, is_array: true, required: false }, + as: :vulnerabilities do |status, _options| + status.vulnerabilities.empty? ? nil : status.vulnerabilities + end end class ArtifactsWithPagination < Pagination diff --git a/lib/mihari/entities/autonomous_system.rb b/lib/mihari/entities/autonomous_system.rb index 7a8f4f09b..27aaa430e 100644 --- a/lib/mihari/entities/autonomous_system.rb +++ b/lib/mihari/entities/autonomous_system.rb @@ -3,7 +3,7 @@ module Mihari module Entities class AutonomousSystem < Grape::Entity - expose :asn, documentation: { type: Integer, required: true } + expose :number, documentation: { type: Integer, required: true } end end end diff --git a/lib/mihari/entities/cpe.rb b/lib/mihari/entities/cpe.rb index 2dd447429..cf8a98a5a 100644 --- a/lib/mihari/entities/cpe.rb +++ b/lib/mihari/entities/cpe.rb @@ -3,7 +3,7 @@ module Mihari module Entities class CPE < Grape::Entity - expose :cpe, documentation: { type: String, required: true } + expose :name, documentation: { type: String, required: true } expose :created_at, documentation: { type: DateTime, required: true }, as: :createdAt end end diff --git a/lib/mihari/entities/port.rb b/lib/mihari/entities/port.rb index 1c9dd56e5..8f1e2bef3 100644 --- a/lib/mihari/entities/port.rb +++ b/lib/mihari/entities/port.rb @@ -3,7 +3,7 @@ module Mihari module Entities class Port < Grape::Entity - expose :port, documentation: { type: Integer, required: true } + expose :number, documentation: { type: Integer, required: true } expose :created_at, documentation: { type: DateTime, required: true }, as: :createdAt end end diff --git a/lib/mihari/entities/vulnerability.rb b/lib/mihari/entities/vulnerability.rb new file mode 100644 index 000000000..32a082fcf --- /dev/null +++ b/lib/mihari/entities/vulnerability.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Mihari + module Entities + class Vulnerability < Grape::Entity + expose :name, documentation: { type: String, required: true } + expose :created_at, documentation: { type: DateTime, required: true }, as: :createdAt + end + end +end diff --git a/lib/mihari/models/artifact.rb b/lib/mihari/models/artifact.rb index 3b62e2850..99c177d15 100644 --- a/lib/mihari/models/artifact.rb +++ b/lib/mihari/models/artifact.rb @@ -28,6 +28,8 @@ class Artifact < ActiveRecord::Base has_many :dns_records, dependent: :destroy has_many :ports, dependent: :destroy has_many :reverse_dns_names, dependent: :destroy + has_many :vulnerabilities, dependent: :destroy + has_many :tags, through: :alert include ActiveModel::Validations @@ -38,12 +40,13 @@ class Artifact < ActiveRecord::Base attributes :id, :data, :data_type, :source, :query, :created_at, "alert.id", "rule.id", "rule.title", "rule.description" attributes tag: "tags.name" - attributes asn: "autonomous_system.asn" + attributes asn: "autonomous_system.number" attributes country_code: "geolocation.country_code" attributes "dns_record.value": "dns_records.value" attributes "dns_record.resource": "dns_records.resource" attributes reverse_dns_name: "reverse_dns_names.name" attributes cpe: "cpes.name" + attributes vuln: "vulnerabilities.name" attributes port: "ports.port" end @@ -156,6 +159,17 @@ def enrich_cpes(enricher = Enrichers::Shodan.new) self.cpes = Services::CPEBuilder.call(data, enricher: enricher) end + # + # Enrich vulnerabilities + # + # @param [Mihari::Enrichers::Shodan] enricher + # + def enrich_vulnerabilities(enricher = Enrichers::Shodan.new) + return unless can_enrich_vulnerabilities? + + self.vulnerabilities = Services::VulnerabilityBuilder.call(data, enricher: enricher) + end + # # Enrich all the enrichable relationships of the artifact # @@ -167,6 +181,7 @@ def enrich_all enrich_whois enrich_ports shodan enrich_cpes shodan + enrich_vulnerabilities shodan end ENRICH_METHODS_BY_ENRICHER = { @@ -181,6 +196,7 @@ def enrich_all enrich_ports enrich_cpes enrich_reverse_dns + enrich_vulnerabilities ], Enrichers::GooglePublicDNS => %i[ enrich_dns @@ -264,6 +280,10 @@ def can_enrich_ports? def can_enrich_cpes? data_type == "ip" && cpes.empty? end + + def can_enrich_vulnerabilities? + data_type == "ip" && vulnerabilities.empty? + end end end end diff --git a/lib/mihari/models/vulnerability.rb b/lib/mihari/models/vulnerability.rb new file mode 100644 index 000000000..cf51a0596 --- /dev/null +++ b/lib/mihari/models/vulnerability.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Mihari + module Models + # + # Vulnerability model + # + class Vulnerability < ActiveRecord::Base + belongs_to :artifact + end + end +end diff --git a/lib/mihari/services/builders.rb b/lib/mihari/services/builders.rb index 76be80043..057be64a0 100644 --- a/lib/mihari/services/builders.rb +++ b/lib/mihari/services/builders.rb @@ -33,7 +33,7 @@ class AutonomousSystemBuilder < Service # def call(ip, enricher: Enrichers::MMDB.new) enricher.result(ip).fmap do |res| - Models::AutonomousSystem.new(asn: res.asn) if res.asn + Models::AutonomousSystem.new(number: res.asn) if res.asn end.value_or nil end end @@ -52,7 +52,7 @@ class CPEBuilder < Service # def call(ip, enricher: Enrichers::Shodan.new) enricher.result(ip).fmap do |res| - (res&.cpes || []).map { |cpe| Models::CPE.new(cpe: cpe) } + (res&.cpes || []).map { |cpe| Models::CPE.new(name: cpe) } end.value_or [] end end @@ -114,7 +114,7 @@ class PortBuilder < Service # def call(ip, enricher: Enrichers::Shodan.new) enricher.result(ip).fmap do |res| - (res&.ports || []).map { |port| Models::Port.new(port: port) } + (res&.ports || []).map { |port| Models::Port.new(number: port) } end.value_or [] end end @@ -138,6 +138,25 @@ def call(ip, enricher: Enrichers::Shodan.new) end end + # + # Vulnerability builder + # + class VulnerabilityBuilder < Service + # + # Build vulnerabilities + # + # @param [String] ip + # @param [Mihari::Enrichers::Shodan] enricher + # + # @return [Array] + # + def call(ip, enricher: Enrichers::Shodan.new) + enricher.result(ip).fmap do |res| + (res&.vulns || []).map { |name| Models::Vulnerability.new(name: name) } + end.value_or [] + end + end + # # Whois record builder # diff --git a/lib/mihari/structs/censys.rb b/lib/mihari/structs/censys.rb index c70039d69..b7ed4dab7 100644 --- a/lib/mihari/structs/censys.rb +++ b/lib/mihari/structs/censys.rb @@ -14,7 +14,7 @@ class AutonomousSystem < Dry::Struct # @return [Mihari::AutonomousSystem] # def as - Mihari::Models::AutonomousSystem.new(asn: normalize_asn(asn)) + Mihari::Models::AutonomousSystem.new(number: normalize_asn(asn)) end class << self @@ -76,7 +76,7 @@ class Service < Dry::Struct # @return [Mihari::Port] # def _port - Models::Port.new(port: port) + Models::Port.new(number: port) end class << self diff --git a/lib/mihari/structs/greynoise.rb b/lib/mihari/structs/greynoise.rb index a62917fe2..3dfb78370 100644 --- a/lib/mihari/structs/greynoise.rb +++ b/lib/mihari/structs/greynoise.rb @@ -22,7 +22,7 @@ class Metadata < Dry::Struct # @return [Mihari::AutonomousSystem] # def as - Mihari::Models::AutonomousSystem.new(asn: normalize_asn(asn)) + Mihari::Models::AutonomousSystem.new(number: normalize_asn(asn)) end # diff --git a/lib/mihari/structs/onyphe.rb b/lib/mihari/structs/onyphe.rb index c63e3230c..6cbbb7400 100644 --- a/lib/mihari/structs/onyphe.rb +++ b/lib/mihari/structs/onyphe.rb @@ -51,7 +51,7 @@ def geolocation # @return [Mihari::AutonomousSystem] # def as - Mihari::Models::AutonomousSystem.new(asn: normalize_asn(asn)) + Mihari::Models::AutonomousSystem.new(number: normalize_asn(asn)) end class << self diff --git a/lib/mihari/structs/shodan.rb b/lib/mihari/structs/shodan.rb index 2de66d9d6..40bf84767 100644 --- a/lib/mihari/structs/shodan.rb +++ b/lib/mihari/structs/shodan.rb @@ -67,17 +67,25 @@ class Match < Dry::Struct # @return [Integer] attribute :port, Types::Int + # @!attribute [r] cpe + # @return [Array] + attribute :cpe, Types::Array(Types::String) + + # @!attribute [r] vulns + # @return [Hash] + attribute :vulns, Types::Hash + # @!attribute [r] metadata # @return [Hash] attribute :metadata, Types::Hash # - # @return [Mihari::AutonomousSystem, nil] + # @return [Mihari::Models::AutonomousSystem, nil] # - def _asn + def autonomous_system return nil if asn.nil? - Mihari::Models::AutonomousSystem.new(asn: normalize_asn(asn)) + Models::AutonomousSystem.new(number: normalize_asn(asn)) end class << self @@ -103,6 +111,8 @@ def from_dynamic!(d) domains: d.fetch("domains"), ip_str: d.fetch("ip_str"), port: d.fetch("port"), + cpe: d["cpe"] || [], + vulns: d["vulns"] || {}, metadata: d ) end @@ -110,6 +120,8 @@ def from_dynamic!(d) end class Response < Dry::Struct + prepend MemoWise + # @!attribute [r] matches # @return [Array] attribute :matches, Types.Array(Match) @@ -118,6 +130,16 @@ class Response < Dry::Struct # @return [Integer] attribute :total, Types::Int + # + # @param [String] ip + # + # @return [Array] + # + def select_matches_by_ip(ip) + matches.select { |match| match.ip_str == ip } + end + memo_wise :select_matches_by_ip + # # Collect metadata from matches # @@ -126,7 +148,7 @@ class Response < Dry::Struct # @return [Array] # def collect_metadata_by_ip(ip) - matches.select { |match| match.ip_str == ip }.map(&:metadata) + select_matches_by_ip(ip).map(&:metadata) end # @@ -137,7 +159,7 @@ def collect_metadata_by_ip(ip) # @return [Array] # def collect_ports_by_ip(ip) - matches.select { |match| match.ip_str == ip }.map(&:port) + select_matches_by_ip(ip).map(&:port) end # @@ -148,7 +170,30 @@ def collect_ports_by_ip(ip) # @return [Array] # def collect_hostnames_by_ip(ip) - matches.select { |match| match.ip_str == ip }.map(&:hostnames).flatten.uniq + select_matches_by_ip(ip).map(&:hostnames).flatten.uniq + end + + # + # Collect CPE from matches + # + # @param [String] ip + # + # @return [Array] + # + def collect_cpes_by_ip(ip) + select_matches_by_ip(ip).map(&:cpe).flatten.uniq + end + + # + # Collect vulnerabilities from matches + # + # @param [String] ip + # + # @return [Array] + # + def collect_vulns_by_ip(ip) + # NOTE: vuln keys = CVE IDs + select_matches_by_ip(ip).map { |match| match.vulns.keys }.flatten.uniq end # @@ -158,20 +203,22 @@ def artifacts matches.map do |match| metadata = collect_metadata_by_ip(match.ip_str) - ports = collect_ports_by_ip(match.ip_str).map do |port| - Mihari::Models::Port.new(port: port) - end + ports = collect_ports_by_ip(match.ip_str).map { |port| Models::Port.new(number: port) } reverse_dns_names = collect_hostnames_by_ip(match.ip_str).map do |name| - Mihari::Models::ReverseDnsName.new(name: name) + Models::ReverseDnsName.new(name: name) end + cpes = collect_cpes_by_ip(match.ip_str).map { |name| Models::CPE.new(name: name) } + vulnerabilities = collect_vulns_by_ip(match.ip_str).map { |name| Models::Vulnerability.new(name: name) } Mihari::Models::Artifact.new( data: match.ip_str, metadata: metadata, - autonomous_system: match._asn, + autonomous_system: match.autonomous_system, geolocation: match.location.geolocation, ports: ports, - reverse_dns_names: reverse_dns_names + reverse_dns_names: reverse_dns_names, + cpes: cpes, + vulnerabilities: vulnerabilities ) end end @@ -232,15 +279,6 @@ def from_dynamic!(d) vulns: d.fetch("vulns") ) end - - # - # @param [String] json - # - # @return [InternetDBResponse] - # - def from_json!(json) - from_dynamic!(JSON.parse(json)) - end end end end diff --git a/spec/analyzers/censys_spec.rb b/spec/analyzers/censys_spec.rb index fed2111cf..6177e1be9 100644 --- a/spec/analyzers/censys_spec.rb +++ b/spec/analyzers/censys_spec.rb @@ -15,7 +15,7 @@ first = artifacts.first expect(first.data).to eq("1.1.1.1") - expect(first.autonomous_system.asn).to eq(13_335) + expect(first.autonomous_system.number).to eq(13_335) expect(first.ports.length).to be > 0 end end diff --git a/spec/analyzers/onyphe_spec.rb b/spec/analyzers/onyphe_spec.rb index b45f59483..3e4ab14c7 100644 --- a/spec/analyzers/onyphe_spec.rb +++ b/spec/analyzers/onyphe_spec.rb @@ -10,7 +10,7 @@ artifacts = analyzer.artifacts expect(artifacts).to be_an(Array) - expect(artifacts.first.autonomous_system.asn).to eq(3462) + expect(artifacts.first.autonomous_system.number).to eq(3462) expect(artifacts.first.geolocation.country).to eq("Taiwan") end end diff --git a/spec/analyzers/shodan_spec.rb b/spec/analyzers/shodan_spec.rb index 6eb59766e..23860b835 100644 --- a/spec/analyzers/shodan_spec.rb +++ b/spec/analyzers/shodan_spec.rb @@ -15,7 +15,7 @@ first = artifacts.first expect(first.data).to eq("1.1.1.1") - expect(first.autonomous_system.asn).to eq(13_335) + expect(first.autonomous_system.number).to eq(13_335) expect(first.geolocation.country_code).to eq("US") diff --git a/spec/factories/artifacts.rb b/spec/factories/artifacts.rb index 6077b24c1..868fb1bd8 100644 --- a/spec/factories/artifacts.rb +++ b/spec/factories/artifacts.rb @@ -4,7 +4,7 @@ factory :autonomous_system, class: "Mihari::Models::AutonomousSystem" do artifact - asn { Faker::Number.unique.number(digits: 4) } + number { Faker::Number.unique.number(digits: 4) } end factory :reverse_dns_name, class: "Mihari::Models::ReverseDnsName" do diff --git a/spec/fixtures/vcr_cassettes/Mihari_Enrichers_Shodan/ip_120_78_59_202.yml b/spec/fixtures/vcr_cassettes/Mihari_Enrichers_Shodan/ip_120_78_59_202.yml new file mode 100644 index 000000000..3166cc1ac --- /dev/null +++ b/spec/fixtures/vcr_cassettes/Mihari_Enrichers_Shodan/ip_120_78_59_202.yml @@ -0,0 +1,55 @@ +--- +http_interactions: +- request: + method: get + uri: https://internetdb.shodan.io/120.78.59.202 + body: + encoding: ASCII-8BIT + string: '' + headers: + User-Agent: + - mihari/7.1.3 + Connection: + - close + Host: + - internetdb.shodan.io + response: + status: + code: 200 + message: OK + headers: + Date: + - Sat, 13 Jan 2024 06:25:25 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - close + Cache-Control: + - public, max-age=432000 + Vary: + - Accept-Encoding + Last-Modified: + - Sat, 13 Jan 2024 06:12:59 GMT + Cf-Cache-Status: + - HIT + Age: + - '746' + Expires: + - Thu, 18 Jan 2024 06:25:25 GMT + Permissions-Policy: + - interest-cohort=() + Access-Control-Allow-Origin: + - "*" + Server: + - cloudflare + Cf-Ray: + - 844b8fd469e9e384-NRT + Alt-Svc: + - h3=":443"; ma=86400 + body: + encoding: UTF-8 + string: '{"cpes":["cpe:/a:mysql:mysql","cpe:/a:php:php:7.4.8","cpe:/a:apache:http_server:2.4.38","cpe:/a:php:php:7.0.19","cpe:/a:f5:nginx:1.11.13","cpe:/a:wordpress:wordpress","cpe:/a:openbsd:openssh:7.4","cpe:/a:jquery:jquery","cpe:/a:php:php"],"hostnames":["api02t.c.ylt100.cn"],"ip":"120.78.59.202","ports":[22,80,443,3306,8081,8090],"tags":["database","eol-product"],"vulns":["CVE-2019-9516","CVE-2017-11144","CVE-2017-16642","CVE-2019-9020","CVE-2018-17082","CVE-2019-9675","CVE-2019-10098","CVE-2016-1283","CVE-2017-9226","CVE-2017-9229","CVE-2021-26690","CVE-2019-6110","CVE-2023-25690","CVE-2017-12933","CVE-2019-17567","CVE-2018-14851","CVE-2017-7963","CVE-2022-22721","CVE-2020-7070","CVE-2023-27522","CVE-2018-20783","CVE-2019-20372","CVE-2023-38408","CVE-2022-31813","CVE-2020-9490","CVE-2021-44224","CVE-2021-36160","CVE-2020-7069","CVE-2018-10546","CVE-2018-15919","CVE-2017-7890","CVE-2019-6111","CVE-2015-9253","CVE-2017-9120","CVE-2022-28330","CVE-2019-9511","CVE-2019-0217","CVE-2021-40438","CVE-2019-9513","CVE-2020-1927","CVE-2021-39275","CVE-2018-14884","CVE-2022-37436","CVE-2022-31628","CVE-2021-26691","CVE-2021-33193","CVE-2022-37454","CVE-2017-12932","CVE-2018-14883","CVE-2020-11984","CVE-2016-20012","CVE-2019-10097","CVE-2022-26377","CVE-2017-20005","CVE-2019-0196","CVE-2017-9118","CVE-2017-12934","CVE-2022-31630","CVE-2021-44790","CVE-2022-31625","CVE-2020-11993","CVE-2018-16843","CVE-2018-19935","CVE-2022-28615","CVE-2020-7071","CVE-2021-21706","CVE-2022-31626","CVE-2018-7584","CVE-2020-15778","CVE-2019-10092","CVE-2017-9227","CVE-2021-21704","CVE-2021-21702","CVE-2021-21705","CVE-2017-11628","CVE-2019-9639","CVE-2019-10082","CVE-2015-8866","CVE-2019-9517","CVE-2018-19396","CVE-2022-22720","CVE-2022-30556","CVE-2017-11362","CVE-2020-14145","CVE-2022-23943","CVE-2017-9224","CVE-2019-6109","CVE-2021-3618","CVE-2019-0215","CVE-2018-10547","CVE-2019-10081","CVE-2017-9228","CVE-2021-21703","CVE-2018-20685","CVE-2022-31629","CVE-2018-15473","CVE-2019-9024","CVE-2017-7529","CVE-2019-9022","CVE-2019-9637","CVE-2021-34798","CVE-2022-28614","CVE-2019-9638","CVE-2021-36368","CVE-2018-10549","CVE-2021-21708","CVE-2019-9023","CVE-2022-29404","CVE-2018-16845","CVE-2017-11145","CVE-2021-23017","CVE-2020-1934","CVE-2022-36760","CVE-2019-9641","CVE-2018-19395","CVE-2020-35452","CVE-2006-20001","CVE-2019-0211","CVE-2019-9021","CVE-2019-0220","CVE-2020-13938","CVE-2018-10548","CVE-2021-21707","CVE-2017-7272","CVE-2021-41617","CVE-2023-44487","CVE-2018-15132","CVE-2020-7068","CVE-2018-10545","CVE-2019-0197","CVE-2022-22719","CVE-2017-15906","CVE-2018-16844","CVE-2018-19518"]}' + recorded_at: Sat, 13 Jan 2024 06:25:25 GMT +recorded_with: VCR 6.2.0 diff --git a/spec/models/artifact_spec.rb b/spec/models/artifact_spec.rb index 28d631f6b..47782c140 100644 --- a/spec/models/artifact_spec.rb +++ b/spec/models/artifact_spec.rb @@ -158,7 +158,7 @@ it do artifact.enrich_autonomous_system - expect(artifact.autonomous_system.asn).to eq(13_335) + expect(artifact.autonomous_system.number).to eq(13_335) end end diff --git a/spec/services/enrichers_spec.rb b/spec/services/artifact_enricher_spec.rb similarity index 100% rename from spec/services/enrichers_spec.rb rename to spec/services/artifact_enricher_spec.rb diff --git a/spec/services/autonomous_system_spec.rb b/spec/services/autonomous_system_builder_spec.rb similarity index 86% rename from spec/services/autonomous_system_spec.rb rename to spec/services/autonomous_system_builder_spec.rb index a7ac82200..bbc2ae907 100644 --- a/spec/services/autonomous_system_spec.rb +++ b/spec/services/autonomous_system_builder_spec.rb @@ -6,7 +6,7 @@ it do as = described_class.call(ip) - expect(as.asn).to eq(13_335) + expect(as.number).to eq(13_335) end end end diff --git a/spec/services/cpe_spec.rb b/spec/services/cpe_builder_spec.rb similarity index 100% rename from spec/services/cpe_spec.rb rename to spec/services/cpe_builder_spec.rb diff --git a/spec/services/dns_spec.rb b/spec/services/dns_record_builder_spec.rb similarity index 100% rename from spec/services/dns_spec.rb rename to spec/services/dns_record_builder_spec.rb diff --git a/spec/services/feed_spec.rb b/spec/services/feed_parser_spec.rb similarity index 52% rename from spec/services/feed_spec.rb rename to spec/services/feed_parser_spec.rb index 0c5168835..36975ca92 100644 --- a/spec/services/feed_spec.rb +++ b/spec/services/feed_parser_spec.rb @@ -26,25 +26,3 @@ end end end - -RSpec.describe Mihari::Services::FeedReader do - describe "#read" do - context "with CSV file" do - let!(:uri) { "file://#{File.expand_path("../fixtures/feed/test.csv", __dir__)}" } - - it do - data = described_class.call(uri) - expect(data).to be_an(Array) - end - end - - context "with JSON file" do - let!(:uri) { "file://#{File.expand_path("../fixtures/feed/test.json", __dir__)}" } - - it do - data = described_class.call(uri) - expect(data).to be_an(Hash) - end - end - end -end diff --git a/spec/services/feed_reader_spec.rb b/spec/services/feed_reader_spec.rb new file mode 100644 index 000000000..08f9a107e --- /dev/null +++ b/spec/services/feed_reader_spec.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +RSpec.describe Mihari::Services::FeedReader do + describe "#read" do + context "with CSV file" do + let!(:uri) { "file://#{File.expand_path("../fixtures/feed/test.csv", __dir__)}" } + + it do + data = described_class.call(uri) + expect(data).to be_an(Array) + end + end + + context "with JSON file" do + let!(:uri) { "file://#{File.expand_path("../fixtures/feed/test.json", __dir__)}" } + + it do + data = described_class.call(uri) + expect(data).to be_an(Hash) + end + end + end +end diff --git a/spec/services/geolocation_spec.rb b/spec/services/geolocation_builder_spec.rb similarity index 100% rename from spec/services/geolocation_spec.rb rename to spec/services/geolocation_builder_spec.rb diff --git a/spec/services/port_spec.rb b/spec/services/port_builder_spec.rb similarity index 100% rename from spec/services/port_spec.rb rename to spec/services/port_builder_spec.rb diff --git a/spec/services/reverse_dns_spec.rb b/spec/services/reverse_dns_builder_spec.rb similarity index 100% rename from spec/services/reverse_dns_spec.rb rename to spec/services/reverse_dns_builder_spec.rb diff --git a/spec/services/initializers_spec.rb b/spec/services/rule_initializers_spec.rb similarity index 100% rename from spec/services/initializers_spec.rb rename to spec/services/rule_initializers_spec.rb diff --git a/spec/services/vulnerability_builder_spec.rb b/spec/services/vulnerability_builder_spec.rb new file mode 100644 index 000000000..784b2637b --- /dev/null +++ b/spec/services/vulnerability_builder_spec.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +RSpec.describe Mihari::Services::VulnerabilityBuilder, vcr: "Mihari_Enrichers_Shodan/ip:120.78.59.202" do + describe ".call" do + let!(:ip) { "120.78.59.202" } + + it do + vulnerabilities = described_class.call(ip) + expect(vulnerabilities).to be_an(Array) + end + end +end diff --git a/spec/services/whois_spec.rb b/spec/services/whois_builder_spec.rb similarity index 100% rename from spec/services/whois_spec.rb rename to spec/services/whois_builder_spec.rb