From c5929efc992298422be1bc952e1de6f1d54b212f Mon Sep 17 00:00:00 2001 From: pedrooot Date: Tue, 1 Oct 2024 13:47:39 -0600 Subject: [PATCH 1/6] feat(inventory): add prowler inventory for aws --- docs/tutorials/quick-inventory.md | 2 +- poetry.lock | 24 +++- prowler/__main__.py | 12 +- prowler/lib/cli/parser.py | 13 ++ .../providers/aws/lib/arguments/arguments.py | 6 +- prowler/providers/aws/lib/service/service.py | 47 +++++++ prowler/providers/common/inventory.py | 118 ++++++++++++++++++ prowler/providers/common/quick_inventory.py | 26 ---- pyproject.toml | 1 + tests/lib/cli/parser_test.py | 2 +- 10 files changed, 212 insertions(+), 39 deletions(-) create mode 100644 prowler/providers/common/inventory.py delete mode 100644 prowler/providers/common/quick_inventory.py diff --git a/docs/tutorials/quick-inventory.md b/docs/tutorials/quick-inventory.md index 62214469671..89ba813e440 100644 --- a/docs/tutorials/quick-inventory.md +++ b/docs/tutorials/quick-inventory.md @@ -6,7 +6,7 @@ Prowler allows you to execute a quick inventory to extract the number of resourc Currently, it is only available for AWS provider. -- You can use option `-i`/`--quick-inventory` to execute it: +- You can use option `-i`/`--inventory` to execute it: ```sh prowler -i ``` diff --git a/poetry.lock b/poetry.lock index f9bee54f00f..a5d2ad9ebd3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "about-time" @@ -4643,6 +4643,26 @@ files = [ {file = "tomlkit-0.13.2.tar.gz", hash = "sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79"}, ] +[[package]] +name = "tqdm" +version = "4.66.5" +description = "Fast, Extensible Progress Meter" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tqdm-4.66.5-py3-none-any.whl", hash = "sha256:90279a3770753eafc9194a0364852159802111925aa30eb3f9d85b0e805ac7cd"}, + {file = "tqdm-4.66.5.tar.gz", hash = "sha256:e1020aef2e5096702d8a025ac7d16b1577279c9d63f8375b63083e9a5f0fcbad"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] + [[package]] name = "typer" version = "0.12.5" @@ -5060,4 +5080,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" -content-hash = "6ab3e45b4316275b0c26103a5dff489527d927bb4d9e4ff6a847849cb106d837" +content-hash = "68fb9f6bc687d8be8324a80f955119d9934750cfd3ee3ffd6745c109e5334483" diff --git a/prowler/__main__.py b/prowler/__main__.py index 7a229112f39..68ad0b800cd 100644 --- a/prowler/__main__.py +++ b/prowler/__main__.py @@ -71,8 +71,8 @@ from prowler.providers.aws.lib.security_hub.security_hub import SecurityHub from prowler.providers.aws.models import AWSOutputOptions from prowler.providers.azure.models import AzureOutputOptions +from prowler.providers.common.inventory import run_prowler_inventory from prowler.providers.common.provider import Provider -from prowler.providers.common.quick_inventory import run_provider_quick_inventory from prowler.providers.gcp.models import GCPOutputOptions from prowler.providers.kubernetes.models import KubernetesOutputOptions @@ -257,11 +257,6 @@ def prowler(): args, bulk_checks_metadata, global_provider.identity ) - # Run the quick inventory for the provider if available - if hasattr(args, "quick_inventory") and args.quick_inventory: - run_provider_quick_inventory(global_provider, args) - sys.exit() - # Execute checks findings = [] @@ -688,6 +683,11 @@ def prowler(): if checks_folder: remove_custom_checks_module(checks_folder, provider) + # Run the quick inventory for the provider if available + if hasattr(args, "inventory") and args.inventory: + run_prowler_inventory(checks_to_execute, args.provider) + sys.exit() + # If there are failed findings exit code 3, except if -z is input if ( not args.ignore_exit_code_3 diff --git a/prowler/lib/cli/parser.py b/prowler/lib/cli/parser.py index 2309d8742ca..9c62b6bbb89 100644 --- a/prowler/lib/cli/parser.py +++ b/prowler/lib/cli/parser.py @@ -378,3 +378,16 @@ def __init_third_party_integrations_parser__(self): action="store_true", help="Send a summary of the execution with a Slack APP in your channel. Environment variables SLACK_API_TOKEN and SLACK_CHANNEL_NAME are required (see more in https://docs.prowler.cloud/en/latest/tutorials/integrations/#slack).", ) + + def __init_inventory_parser__(self): + inventory_parser = self.common_providers_parser.add_argument_group("Inventory") + inventory_parser.add_argument( + "--inventory", + action="store_true", + help="Run Prowler in inventory mode", + ) + inventory_parser.add_argument( + "-i", + nargs="?", + help="Output folder path for the inventory", + ) diff --git a/prowler/providers/aws/lib/arguments/arguments.py b/prowler/providers/aws/lib/arguments/arguments.py index 8cac4b6a1a4..cd6014d0300 100644 --- a/prowler/providers/aws/lib/arguments/arguments.py +++ b/prowler/providers/aws/lib/arguments/arguments.py @@ -93,12 +93,12 @@ def init_parser(self): help="Send only Prowler failed findings to SecurityHub", ) # AWS Quick Inventory - aws_quick_inventory_subparser = aws_parser.add_argument_group("Quick Inventory") + aws_quick_inventory_subparser = aws_parser.add_argument_group("Inventory") aws_quick_inventory_subparser.add_argument( - "--quick-inventory", + "--inventory", "-i", action="store_true", - help="Run Prowler Quick Inventory. The inventory will be stored in an output csv by default", + help="Run Prowler Inventory. The inventory will be stored in an output json file.", ) # AWS Outputs aws_outputs_subparser = aws_parser.add_argument_group("AWS Outputs to S3") diff --git a/prowler/providers/aws/lib/service/service.py b/prowler/providers/aws/lib/service/service.py index 2aaca3e3941..46784f62cbb 100644 --- a/prowler/providers/aws/lib/service/service.py +++ b/prowler/providers/aws/lib/service/service.py @@ -1,4 +1,7 @@ +from collections import deque from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime +from typing import Any, Dict from prowler.lib.logger import logger from prowler.providers.aws.aws_provider import AwsProvider @@ -101,3 +104,47 @@ def __threading_call__(self, call, iterator=None): except Exception: # Handle exceptions if necessary pass # Replace 'pass' with any additional exception handling logic. Currently handled within the called function + + def __to_dict__(self, seen=None) -> Dict[str, Any]: + if seen is None: + seen = set() + + def convert_value(value): + if isinstance(value, (AwsProvider,)): + return {} + if isinstance(value, datetime): + return value.isoformat() # Convert datetime to ISO 8601 string + elif isinstance(value, deque): + return [convert_value(item) for item in value] + elif isinstance(value, list): + return [convert_value(item) for item in value] + elif isinstance(value, tuple): + return tuple(convert_value(item) for item in value) + elif isinstance(value, dict): + # Ensure keys are strings and values are processed + return { + convert_value(str(k)): convert_value(v) for k, v in value.items() + } + elif hasattr(value, "__dict__"): + obj_id = id(value) + if obj_id in seen: + return None # Avoid infinite recursion + seen.add(obj_id) + return {key: convert_value(val) for key, val in value.__dict__.items()} + else: + return value # Handle basic types and non-serializable objects + + return { + key: convert_value(value) + for key, value in self.__dict__.items() + if key + not in [ + "audit_config", + "provider", + "session", + "regional_clients", + "client", + "thread_pool", + "fixer_config", + ] + } diff --git a/prowler/providers/common/inventory.py b/prowler/providers/common/inventory.py new file mode 100644 index 00000000000..fac27319cd4 --- /dev/null +++ b/prowler/providers/common/inventory.py @@ -0,0 +1,118 @@ +import importlib +import json +import os +import shutil +from collections import deque +from datetime import datetime + +from pydantic import BaseModel +from tqdm import tqdm + + +def run_prowler_inventory(checks_to_execute, provider): + output_folder_path = f"./output/inventory/{provider}" + meta_json_file = {} + + os.makedirs(output_folder_path, exist_ok=True) + + # Recursive function to handle serialization + def class_to_dict(obj, seen=None): + if seen is None: + seen = set() + + if isinstance(obj, dict): + new_dict = {} + for key, value in obj.items(): + if isinstance(key, tuple): + key = str(key) # Convert tuple to string + new_dict[key] = class_to_dict(value) + return new_dict + if isinstance(obj, datetime): + return obj.isoformat() + elif isinstance(obj, deque): + return list(class_to_dict(item, seen) for item in obj) + elif isinstance(obj, BaseModel): + return obj.dict() + elif isinstance(obj, (list, tuple)): + return [class_to_dict(item, seen) for item in obj] + elif hasattr(obj, "__dict__") and id(obj) not in seen: + seen.add(id(obj)) + return { + key: class_to_dict(value, seen) for key, value in obj.__dict__.items() + } + else: + return obj + + service_set = set() + + for check_name in tqdm(checks_to_execute): + try: + service = check_name.split("_")[0] + + if service in service_set: + continue + + service_set.add(service) + + service_path = f"./prowler/providers/{provider}/services/{service}" + + # List to store all _client filenames + client_files = [] + + # Walk through the directory and find all files + for root, dirs, files in os.walk(service_path): + for file in files: + if file.endswith("_client.py"): + # Append only the filename to the list (not the full path) + client_files.append(file) + + service_output_folder = f"{output_folder_path}/{service}" + + os.makedirs(service_output_folder, exist_ok=True) + + for service_client in client_files: + + service_client = service_client.split(".py")[0] + check_module_path = ( + f"prowler.providers.{provider}.services.{service}.{service_client}" + ) + + try: + lib = importlib.import_module(f"{check_module_path}") + except ModuleNotFoundError: + print(f"Module not found: {check_module_path}") + break + except Exception as e: + print(f"Error while importing module {check_module_path}: {e}") + break + + client_path = getattr(lib, f"{service_client}") + + if not meta_json_file.get(f"{service}"): + meta_json_file[f"{service}"] = [] + + # Convert to JSON + output_file = service_client.split("_client")[0] + + meta_json_file[f"{service}"].append( + f"./{service}/{output_file}_output.json" + ) + + with open( + f"{service_output_folder}/{output_file}_output.json", "w+" + ) as fp: + output = client_path.__to_dict__() + json.dump(output, fp=fp, default=str, indent=4) + + except Exception as e: + print("Exception: ", e) + + with open(f"{output_folder_path}/output_metadata.json", "w+") as fp: + json.dump(meta_json_file, fp=fp, default=str, indent=4) + + # end of all things + folder_to_compress = f"{output_folder_path}" + output_zip_file = f"{output_folder_path}/rmfx-scan-compressed" # The output file (without extension) + + # Compress the folder into a zip file + shutil.make_archive(f"{output_zip_file}", "zip", folder_to_compress) diff --git a/prowler/providers/common/quick_inventory.py b/prowler/providers/common/quick_inventory.py deleted file mode 100644 index 1023656caa8..00000000000 --- a/prowler/providers/common/quick_inventory.py +++ /dev/null @@ -1,26 +0,0 @@ -import importlib -import sys - -from prowler.lib.logger import logger -from prowler.providers.aws.lib.quick_inventory.quick_inventory import quick_inventory - - -def run_provider_quick_inventory(provider, args): - """ - run_provider_quick_inventory executes the quick inventory for the provider - """ - try: - # Dynamically get the Provider quick inventory handler - provider_quick_inventory_function = f"{provider.type}_quick_inventory" - getattr(importlib.import_module(__name__), provider_quick_inventory_function)( - provider, args - ) - except Exception as error: - logger.critical( - f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" - ) - sys.exit(1) - - -def aws_quick_inventory(provider, args): - quick_inventory(provider, args) diff --git a/pyproject.toml b/pyproject.toml index 74a0a204d13..59c3195c0bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,7 @@ schema = "0.7.7" shodan = "1.31.0" slack-sdk = "3.33.1" tabulate = "0.9.0" +tqdm = "^4.66.5" tzlocal = "5.2" [tool.poetry.group.dev.dependencies] diff --git a/tests/lib/cli/parser_test.py b/tests/lib/cli/parser_test.py index f1d75b23745..f51be0c9970 100644 --- a/tests/lib/cli/parser_test.py +++ b/tests/lib/cli/parser_test.py @@ -974,7 +974,7 @@ def test_aws_parser_quick_inventory_short(self): assert parsed.quick_inventory def test_aws_parser_quick_inventory_long(self): - argument = "--quick-inventory" + argument = "--inventory" command = [prowler_command, argument] parsed = self.parser.parse(command) assert parsed.quick_inventory From 2b51cf8fca959b96a5bfeb229a823ea5db883139 Mon Sep 17 00:00:00 2001 From: pedrooot Date: Tue, 1 Oct 2024 13:57:52 -0600 Subject: [PATCH 2/6] feat(inventory): rename inventory to scan-inventory --- docs/tutorials/quick-inventory.md | 2 +- prowler/__main__.py | 2 +- prowler/lib/cli/parser.py | 6 ++++-- prowler/providers/aws/lib/arguments/arguments.py | 2 +- prowler/providers/common/inventory.py | 1 + tests/lib/cli/parser_test.py | 2 +- 6 files changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/tutorials/quick-inventory.md b/docs/tutorials/quick-inventory.md index 89ba813e440..0c5654d10f7 100644 --- a/docs/tutorials/quick-inventory.md +++ b/docs/tutorials/quick-inventory.md @@ -6,7 +6,7 @@ Prowler allows you to execute a quick inventory to extract the number of resourc Currently, it is only available for AWS provider. -- You can use option `-i`/`--inventory` to execute it: +- You can use option `-i`/`--scan-inventory` to execute it: ```sh prowler -i ``` diff --git a/prowler/__main__.py b/prowler/__main__.py index 68ad0b800cd..b75d11a33a1 100644 --- a/prowler/__main__.py +++ b/prowler/__main__.py @@ -684,7 +684,7 @@ def prowler(): remove_custom_checks_module(checks_folder, provider) # Run the quick inventory for the provider if available - if hasattr(args, "inventory") and args.inventory: + if hasattr(args, "scan_inventory") and args.scan_inventory: run_prowler_inventory(checks_to_execute, args.provider) sys.exit() diff --git a/prowler/lib/cli/parser.py b/prowler/lib/cli/parser.py index 9c62b6bbb89..809a6b447e5 100644 --- a/prowler/lib/cli/parser.py +++ b/prowler/lib/cli/parser.py @@ -380,9 +380,11 @@ def __init_third_party_integrations_parser__(self): ) def __init_inventory_parser__(self): - inventory_parser = self.common_providers_parser.add_argument_group("Inventory") + inventory_parser = self.common_providers_parser.add_argument_group( + "ScanInventory" + ) inventory_parser.add_argument( - "--inventory", + "--scan-inventory", action="store_true", help="Run Prowler in inventory mode", ) diff --git a/prowler/providers/aws/lib/arguments/arguments.py b/prowler/providers/aws/lib/arguments/arguments.py index cd6014d0300..471d3a16c70 100644 --- a/prowler/providers/aws/lib/arguments/arguments.py +++ b/prowler/providers/aws/lib/arguments/arguments.py @@ -95,7 +95,7 @@ def init_parser(self): # AWS Quick Inventory aws_quick_inventory_subparser = aws_parser.add_argument_group("Inventory") aws_quick_inventory_subparser.add_argument( - "--inventory", + "--scan-inventory", "-i", action="store_true", help="Run Prowler Inventory. The inventory will be stored in an output json file.", diff --git a/prowler/providers/common/inventory.py b/prowler/providers/common/inventory.py index fac27319cd4..134d5533bfe 100644 --- a/prowler/providers/common/inventory.py +++ b/prowler/providers/common/inventory.py @@ -10,6 +10,7 @@ def run_prowler_inventory(checks_to_execute, provider): + print(f"\nRunning scan inventory for {provider}") output_folder_path = f"./output/inventory/{provider}" meta_json_file = {} diff --git a/tests/lib/cli/parser_test.py b/tests/lib/cli/parser_test.py index f51be0c9970..07a31f482fb 100644 --- a/tests/lib/cli/parser_test.py +++ b/tests/lib/cli/parser_test.py @@ -974,7 +974,7 @@ def test_aws_parser_quick_inventory_short(self): assert parsed.quick_inventory def test_aws_parser_quick_inventory_long(self): - argument = "--inventory" + argument = "--scan-inventory" command = [prowler_command, argument] parsed = self.parser.parse(command) assert parsed.quick_inventory From e7d719e5141ec793918eb5be5a8f898b5d64b273 Mon Sep 17 00:00:00 2001 From: pedrooot Date: Tue, 1 Oct 2024 14:18:44 -0600 Subject: [PATCH 3/6] feat(inventory): change prints for inventory --- poetry.lock | 22 +--------------------- prowler/providers/common/inventory.py | 10 +++++++--- pyproject.toml | 1 - 3 files changed, 8 insertions(+), 25 deletions(-) diff --git a/poetry.lock b/poetry.lock index a5d2ad9ebd3..280f600a4c3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4643,26 +4643,6 @@ files = [ {file = "tomlkit-0.13.2.tar.gz", hash = "sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79"}, ] -[[package]] -name = "tqdm" -version = "4.66.5" -description = "Fast, Extensible Progress Meter" -optional = false -python-versions = ">=3.7" -files = [ - {file = "tqdm-4.66.5-py3-none-any.whl", hash = "sha256:90279a3770753eafc9194a0364852159802111925aa30eb3f9d85b0e805ac7cd"}, - {file = "tqdm-4.66.5.tar.gz", hash = "sha256:e1020aef2e5096702d8a025ac7d16b1577279c9d63f8375b63083e9a5f0fcbad"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[package.extras] -dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"] -notebook = ["ipywidgets (>=6)"] -slack = ["slack-sdk"] -telegram = ["requests"] - [[package]] name = "typer" version = "0.12.5" @@ -5080,4 +5060,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.13" -content-hash = "68fb9f6bc687d8be8324a80f955119d9934750cfd3ee3ffd6745c109e5334483" +content-hash = "6ab3e45b4316275b0c26103a5dff489527d927bb4d9e4ff6a847849cb106d837" diff --git a/prowler/providers/common/inventory.py b/prowler/providers/common/inventory.py index 134d5533bfe..1461298b014 100644 --- a/prowler/providers/common/inventory.py +++ b/prowler/providers/common/inventory.py @@ -5,12 +5,13 @@ from collections import deque from datetime import datetime +from colorama import Fore, Style from pydantic import BaseModel -from tqdm import tqdm + +from prowler.config.config import orange_color def run_prowler_inventory(checks_to_execute, provider): - print(f"\nRunning scan inventory for {provider}") output_folder_path = f"./output/inventory/{provider}" meta_json_file = {} @@ -46,7 +47,7 @@ def class_to_dict(obj, seen=None): service_set = set() - for check_name in tqdm(checks_to_execute): + for check_name in checks_to_execute: try: service = check_name.split("_")[0] @@ -117,3 +118,6 @@ def class_to_dict(obj, seen=None): # Compress the folder into a zip file shutil.make_archive(f"{output_zip_file}", "zip", folder_to_compress) + print( + f"\n{Style.BRIGHT}{Fore.GREEN}Scan inventory for {provider} results: {orange_color}{output_folder_path}" + ) diff --git a/pyproject.toml b/pyproject.toml index 59c3195c0bf..74a0a204d13 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,7 +71,6 @@ schema = "0.7.7" shodan = "1.31.0" slack-sdk = "3.33.1" tabulate = "0.9.0" -tqdm = "^4.66.5" tzlocal = "5.2" [tool.poetry.group.dev.dependencies] From 3ef1d416303c20ec243dc8853a27c92b7c9ff81e Mon Sep 17 00:00:00 2001 From: pedrooot Date: Wed, 2 Oct 2024 08:46:42 -0600 Subject: [PATCH 4/6] feat(inventory): name on output folder --- prowler/providers/common/inventory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prowler/providers/common/inventory.py b/prowler/providers/common/inventory.py index 1461298b014..d7c9aec7ff8 100644 --- a/prowler/providers/common/inventory.py +++ b/prowler/providers/common/inventory.py @@ -114,7 +114,7 @@ def class_to_dict(obj, seen=None): # end of all things folder_to_compress = f"{output_folder_path}" - output_zip_file = f"{output_folder_path}/rmfx-scan-compressed" # The output file (without extension) + output_zip_file = f"{output_folder_path}/prowler-scan-compressed" # The output file (without extension) # Compress the folder into a zip file shutil.make_archive(f"{output_zip_file}", "zip", folder_to_compress) From 2cf1f222350dc949d8910b4dc4d0f7579ed58d90 Mon Sep 17 00:00:00 2001 From: pedrooot Date: Wed, 2 Oct 2024 15:45:01 -0600 Subject: [PATCH 5/6] feat(scan-inventory): take back changes from quick inventory --- docs/tutorials/quick-inventory.md | 2 +- docs/tutorials/scan-inventory.md | 39 +++++++++++++++++++ mkdocs.yml | 1 + prowler/__main__.py | 6 +++ prowler/lib/cli/parser.py | 15 ------- .../providers/aws/lib/arguments/arguments.py | 13 +++++-- prowler/providers/common/inventory.py | 11 ------ prowler/providers/common/quick_inventory.py | 26 +++++++++++++ tests/lib/cli/parser_test.py | 8 +++- 9 files changed, 90 insertions(+), 31 deletions(-) create mode 100644 docs/tutorials/scan-inventory.md create mode 100644 prowler/providers/common/quick_inventory.py diff --git a/docs/tutorials/quick-inventory.md b/docs/tutorials/quick-inventory.md index 0c5654d10f7..62214469671 100644 --- a/docs/tutorials/quick-inventory.md +++ b/docs/tutorials/quick-inventory.md @@ -6,7 +6,7 @@ Prowler allows you to execute a quick inventory to extract the number of resourc Currently, it is only available for AWS provider. -- You can use option `-i`/`--scan-inventory` to execute it: +- You can use option `-i`/`--quick-inventory` to execute it: ```sh prowler -i ``` diff --git a/docs/tutorials/scan-inventory.md b/docs/tutorials/scan-inventory.md new file mode 100644 index 00000000000..6257844c5e2 --- /dev/null +++ b/docs/tutorials/scan-inventory.md @@ -0,0 +1,39 @@ +# Scan Inventory + +The scan-inventory feature is a tool that generates a JSON report within the `/output/inventory/` directory and the scanned service. This feature allows you to perform a inventory of the resources existing in your provider that are scanned by Prowler. + +## Usage + +To use the scan-inventory feature, run Prowler with the `--scan-inventory` option. For example: + +``` +prowler --scan-inventory +``` + +This will generate a JSON report within the `/output/inventory/` directory and the scanned service. + +## Output Directory Contents + +The contents of the `/output/` directory and the scanned service depend on the Prowler execution. This directory contains all the information gathered during scanning, including a JSON report containing all the gathered information. + +## Limitations + +The scan-inventory feature has some limitations. For example: + +* It is only available for the AWS provider. +* It only contains the information retrieved by Prowler during the execution. + +## Example + +Here's an example of how to use the scan-inventory feature and the contents of the `/output/inventory/` directory and the scanned service: + +`prowler aws -s ec2 --scan-inventory` + +``` +/output/inventory/aws directory + | + |-- ec2 + | | + | |-- ec2_output.json +``` +In this example, Prowler is run with the `-s ec2` and `--scan-inventory` options for the AWS provider. The `/output/inventory/aws` directory contains a JSON report showing all the information gathered during scanning. diff --git a/mkdocs.yml b/mkdocs.yml index d696191908e..10cd0dad990 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -55,6 +55,7 @@ nav: - Dashboard: tutorials/dashboard.md - Fixer (remediations): tutorials/fixer.md - Quick Inventory: tutorials/quick-inventory.md + - Scan Inventory: tutorials/scan-inventory.md - Slack Integration: tutorials/integrations.md - Configuration File: tutorials/configuration_file.md - Logging: tutorials/logging.md diff --git a/prowler/__main__.py b/prowler/__main__.py index b75d11a33a1..06bb700a0ff 100644 --- a/prowler/__main__.py +++ b/prowler/__main__.py @@ -73,6 +73,7 @@ from prowler.providers.azure.models import AzureOutputOptions from prowler.providers.common.inventory import run_prowler_inventory from prowler.providers.common.provider import Provider +from prowler.providers.common.quick_inventory import run_provider_quick_inventory from prowler.providers.gcp.models import GCPOutputOptions from prowler.providers.kubernetes.models import KubernetesOutputOptions @@ -257,6 +258,11 @@ def prowler(): args, bulk_checks_metadata, global_provider.identity ) + # Run the quick inventory for the provider if available + if hasattr(args, "quick_inventory") and args.quick_inventory: + run_provider_quick_inventory(global_provider, args) + sys.exit() + # Execute checks findings = [] diff --git a/prowler/lib/cli/parser.py b/prowler/lib/cli/parser.py index 809a6b447e5..2309d8742ca 100644 --- a/prowler/lib/cli/parser.py +++ b/prowler/lib/cli/parser.py @@ -378,18 +378,3 @@ def __init_third_party_integrations_parser__(self): action="store_true", help="Send a summary of the execution with a Slack APP in your channel. Environment variables SLACK_API_TOKEN and SLACK_CHANNEL_NAME are required (see more in https://docs.prowler.cloud/en/latest/tutorials/integrations/#slack).", ) - - def __init_inventory_parser__(self): - inventory_parser = self.common_providers_parser.add_argument_group( - "ScanInventory" - ) - inventory_parser.add_argument( - "--scan-inventory", - action="store_true", - help="Run Prowler in inventory mode", - ) - inventory_parser.add_argument( - "-i", - nargs="?", - help="Output folder path for the inventory", - ) diff --git a/prowler/providers/aws/lib/arguments/arguments.py b/prowler/providers/aws/lib/arguments/arguments.py index 471d3a16c70..c3df623117d 100644 --- a/prowler/providers/aws/lib/arguments/arguments.py +++ b/prowler/providers/aws/lib/arguments/arguments.py @@ -93,12 +93,19 @@ def init_parser(self): help="Send only Prowler failed findings to SecurityHub", ) # AWS Quick Inventory - aws_quick_inventory_subparser = aws_parser.add_argument_group("Inventory") + aws_quick_inventory_subparser = aws_parser.add_argument_group("Quick Inventory") aws_quick_inventory_subparser.add_argument( - "--scan-inventory", + "--quick-inventory", "-i", action="store_true", - help="Run Prowler Inventory. The inventory will be stored in an output json file.", + help="Run Prowler Quick Inventory. The inventory will be stored in an output csv by default", + ) + # AWS Scan Inventory + aws_scan_inventory_subparser = aws_parser.add_argument_group("Scan Inventory") + aws_scan_inventory_subparser.add_argument( + "--scan-inventory", + action="store_true", + help="Run Prowler Scan Inventory. The inventory will be stored in an output json file.", ) # AWS Outputs aws_outputs_subparser = aws_parser.add_argument_group("AWS Outputs to S3") diff --git a/prowler/providers/common/inventory.py b/prowler/providers/common/inventory.py index d7c9aec7ff8..42df5c1e69d 100644 --- a/prowler/providers/common/inventory.py +++ b/prowler/providers/common/inventory.py @@ -1,7 +1,6 @@ import importlib import json import os -import shutil from collections import deque from datetime import datetime @@ -108,16 +107,6 @@ def class_to_dict(obj, seen=None): except Exception as e: print("Exception: ", e) - - with open(f"{output_folder_path}/output_metadata.json", "w+") as fp: - json.dump(meta_json_file, fp=fp, default=str, indent=4) - - # end of all things - folder_to_compress = f"{output_folder_path}" - output_zip_file = f"{output_folder_path}/prowler-scan-compressed" # The output file (without extension) - - # Compress the folder into a zip file - shutil.make_archive(f"{output_zip_file}", "zip", folder_to_compress) print( f"\n{Style.BRIGHT}{Fore.GREEN}Scan inventory for {provider} results: {orange_color}{output_folder_path}" ) diff --git a/prowler/providers/common/quick_inventory.py b/prowler/providers/common/quick_inventory.py new file mode 100644 index 00000000000..1023656caa8 --- /dev/null +++ b/prowler/providers/common/quick_inventory.py @@ -0,0 +1,26 @@ +import importlib +import sys + +from prowler.lib.logger import logger +from prowler.providers.aws.lib.quick_inventory.quick_inventory import quick_inventory + + +def run_provider_quick_inventory(provider, args): + """ + run_provider_quick_inventory executes the quick inventory for the provider + """ + try: + # Dynamically get the Provider quick inventory handler + provider_quick_inventory_function = f"{provider.type}_quick_inventory" + getattr(importlib.import_module(__name__), provider_quick_inventory_function)( + provider, args + ) + except Exception as error: + logger.critical( + f"{error.__class__.__name__}[{error.__traceback__.tb_lineno}]: {error}" + ) + sys.exit(1) + + +def aws_quick_inventory(provider, args): + quick_inventory(provider, args) diff --git a/tests/lib/cli/parser_test.py b/tests/lib/cli/parser_test.py index 07a31f482fb..d726a381176 100644 --- a/tests/lib/cli/parser_test.py +++ b/tests/lib/cli/parser_test.py @@ -974,11 +974,17 @@ def test_aws_parser_quick_inventory_short(self): assert parsed.quick_inventory def test_aws_parser_quick_inventory_long(self): - argument = "--scan-inventory" + argument = "--quick-inventory" command = [prowler_command, argument] parsed = self.parser.parse(command) assert parsed.quick_inventory + def test_aws_parser_scan_inventory_long(self): + argument = "--scan-inventory" + command = [prowler_command, argument] + parsed = self.parser.parse(command) + assert parsed.scan_inventory + def test_aws_parser_output_bucket_short(self): argument = "-B" bucket = "test-bucket" From d437d1c4b8020da671de32eda1a04314bbbfcf55 Mon Sep 17 00:00:00 2001 From: pedrooot Date: Thu, 3 Oct 2024 11:57:17 +0200 Subject: [PATCH 6/6] feat(inventory): change name and behaviour --- prowler/__main__.py | 4 ++-- .../common/{inventory.py => scan_inventory.py} | 10 +--------- 2 files changed, 3 insertions(+), 11 deletions(-) rename prowler/providers/common/{inventory.py => scan_inventory.py} (91%) diff --git a/prowler/__main__.py b/prowler/__main__.py index 06bb700a0ff..c796677624a 100644 --- a/prowler/__main__.py +++ b/prowler/__main__.py @@ -71,9 +71,9 @@ from prowler.providers.aws.lib.security_hub.security_hub import SecurityHub from prowler.providers.aws.models import AWSOutputOptions from prowler.providers.azure.models import AzureOutputOptions -from prowler.providers.common.inventory import run_prowler_inventory from prowler.providers.common.provider import Provider from prowler.providers.common.quick_inventory import run_provider_quick_inventory +from prowler.providers.common.scan_inventory import run_prowler_scan_inventory from prowler.providers.gcp.models import GCPOutputOptions from prowler.providers.kubernetes.models import KubernetesOutputOptions @@ -691,7 +691,7 @@ def prowler(): # Run the quick inventory for the provider if available if hasattr(args, "scan_inventory") and args.scan_inventory: - run_prowler_inventory(checks_to_execute, args.provider) + run_prowler_scan_inventory(checks_to_execute, args.provider) sys.exit() # If there are failed findings exit code 3, except if -z is input diff --git a/prowler/providers/common/inventory.py b/prowler/providers/common/scan_inventory.py similarity index 91% rename from prowler/providers/common/inventory.py rename to prowler/providers/common/scan_inventory.py index 42df5c1e69d..37413b8f677 100644 --- a/prowler/providers/common/inventory.py +++ b/prowler/providers/common/scan_inventory.py @@ -10,9 +10,8 @@ from prowler.config.config import orange_color -def run_prowler_inventory(checks_to_execute, provider): +def run_prowler_scan_inventory(checks_to_execute, provider): output_folder_path = f"./output/inventory/{provider}" - meta_json_file = {} os.makedirs(output_folder_path, exist_ok=True) @@ -89,16 +88,9 @@ def class_to_dict(obj, seen=None): client_path = getattr(lib, f"{service_client}") - if not meta_json_file.get(f"{service}"): - meta_json_file[f"{service}"] = [] - # Convert to JSON output_file = service_client.split("_client")[0] - meta_json_file[f"{service}"].append( - f"./{service}/{output_file}_output.json" - ) - with open( f"{service_output_folder}/{output_file}_output.json", "w+" ) as fp: