diff --git a/README.md b/README.md index 1d6e3f4..8b7d985 100644 --- a/README.md +++ b/README.md @@ -77,14 +77,16 @@ Options: -l command -l, --location-name TEXT Location name in string format - -p, --pan-enhancement Enhance image with panchromatic band - --no-preview Skip downloading of preview image + -pan, --pan-enhancement Enhance image with panchromatic band + --no-preview Skip previewing of pre-processed low resolution RGB + satellite image. + -v, --vegetation Show Color Infrared image to highlight vegetation -V, --version Show the version number and quit + -p, --product TEXT Product name 'landsat'/'sentinel' --help Show this message and exit. - ``` Felicette can download and process Landsat images taking the location's input as `(lon, lat)` or the location name. They can be used in the following way. @@ -97,10 +99,17 @@ With coordinates: $ felicette -c 77.5385 8.0883 -`-p` option uses the panchromatic band to enhance image's resolution to 15 meters, contrary to resolution of RGB bands(30 meters). -To get a better image using felicette use: +`--product` / `-p` option is used to specify which data-product is used to generate images i.e Sentinel or Landsat. By default, Landsat-8 data will be used to generate images. + + $ felicette -l "Kanyakumari" -p "sentinel" + +**NB**: *To use sentinel data source, one has to set `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` (To generate a pair, go to AWS console -> My Security Credentials -> Access keys). This is because [Sentinel-2 data](https://registry.opendata.aws/sentinel-2/) is in a [Requester Pays](https://docs.aws.amazon.com/AmazonS3/latest/dev/RequesterPaysBuckets.html) bucket. + + +`-pan` option uses the panchromatic band to enhance image's resolution to 15 meters, contrary to resolution of RGB bands(30 meters) if Landsat product is being used. Felicette doesn't support any panchromatic enhancements for Sentinel-2 data which already have a resolution of 10m. +To get a better Landsat image using felicette use: - $ felicette -p -c 77.5385 8.0883 + $ felicette -pan -c 77.5385 8.0883 `--no-preview` option doesn't download image to preview, and directly downloads and processes original data. Please use this if you're sure of the location/quality of the images to be generated by felicette with the arguments provided to it. diff --git a/felicette/cli.py b/felicette/cli.py index 62d6666..49521e3 100644 --- a/felicette/cli.py +++ b/felicette/cli.py @@ -7,18 +7,19 @@ from felicette.utils.file_manager import check_sat_path, file_paths_wrt_id from felicette.sat_downloader import ( download_landsat_data, - search_landsat_data, - preview_landsat_image, + search_satellite_data, + preview_satellite_image, + download_data, ) from felicette.utils.sys_utils import exit_cli, remove_dir -from felicette.sat_processor import process_landsat_data +from felicette.sat_processor import process_data -def trigger_download_and_processing(landsat_item, bands): +def trigger_download_and_processing(item, bands): # download data - data_id = download_landsat_data(landsat_item, bands) + data_id = download_data(item, bands) # process data - process_landsat_data(data_id, bands) + process_data(data_id, bands) @click.command() @@ -31,7 +32,7 @@ def trigger_download_and_processing(landsat_item, bands): ) @click.option("-l", "--location-name", type=str, help="Location name in string format") @click.option( - "-p", + "-pan", "--pan-enhancement", default=False, is_flag=True, @@ -41,7 +42,7 @@ def trigger_download_and_processing(landsat_item, bands): "--no-preview", default=False, is_flag=True, - help="Preview pre-processed low resolution RGB satellite image.", + help="Skip previewing of pre-processed low resolution RGB satellite image.", ) @click.option( "-v", @@ -57,7 +58,8 @@ def trigger_download_and_processing(landsat_item, bands): is_flag=True, help="Show the version number and quit", ) -def main(coordinates, location_name, pan_enhancement, no_preview, vegetation, version): +@click.option("-p", "--product", type=str, default="landsat", help="Product name 'landsat'/'sentinel'") +def main(coordinates, location_name, pan_enhancement, no_preview, vegetation, version, product): """Satellite imagery for dummies.""" if version: version_no = pkg_resources.require("felicette")[0].version @@ -68,37 +70,38 @@ def main(coordinates, location_name, pan_enhancement, no_preview, vegetation, ve coordinates = geocoder_util(location_name) # unless specified, cloud_cover_lt is 10 - landsat_item = search_landsat_data(coordinates, 10) + item = search_satellite_data(coordinates, 10, product=product) # check if directory exists to save the data for this product id - check_sat_path(landsat_item._data["id"]) + check_sat_path(item._data["id"]) # if preview option is set, download and preview image if not no_preview: - preview_landsat_image(landsat_item) + preview_satellite_image(item) # set bands to process bands = [2, 3, 4] - if pan_enhancement: + if pan_enhancement and (product != "sentinel"): bands.append(8) if vegetation: bands = [3, 4, 5] # NB: can't enable pan-enhancement with vegetation + # NB: can't enable pan-enhancement with sentinel try: - trigger_download_and_processing(landsat_item, bands) + trigger_download_and_processing(item, bands) except RasterioIOError: response = input( "Local data for this location is corrupted, felicette will remove existing data to proceed, are you sure? [Y/n]" ) if response in ["y", "Y", ""]: # remove file dir - file_paths = file_paths_wrt_id(landsat_item._data["id"]) + file_paths = file_paths_wrt_id(item._data["id"]) remove_dir(file_paths["base"]) # retry downloading and processing image with a clean directory - trigger_download_and_processing(landsat_item, bands) + trigger_download_and_processing(item, bands) elif response in ["n", "N"]: exit_cli(print, "") diff --git a/felicette/sat_downloader.py b/felicette/sat_downloader.py index 07347b6..add2724 100644 --- a/felicette/sat_downloader.py +++ b/felicette/sat_downloader.py @@ -1,6 +1,8 @@ from satsearch import Search import sys from rich import print as rprint +import requests +import json from felicette.utils.geo_utils import get_tiny_bbox from felicette.utils.sys_utils import exit_cli @@ -9,6 +11,7 @@ save_to_file, data_file_exists, file_paths_wrt_id, + get_product_type_from_id, ) @@ -24,12 +27,22 @@ def handle_prompt_response(response): exit_cli(rprint, "[red]Sorry, invalid response. Exiting :([/red]") -def search_landsat_data(coordinates, cloud_cover_lt): +def search_satellite_data(coordinates, cloud_cover_lt, product="landsat"): + """ + coordinates: bounding box's coordinates + cloud_cover_lt: maximum cloud cover + product: landsat, sentinel + """ + if product == "landsat": + product = "landsat-8-l1" + elif product == "sentinel": + product = "sentinel-2-l1c" + search = Search( bbox=get_tiny_bbox(coordinates), query={ "eo:cloud_cover": {"lt": cloud_cover_lt}, - "collection": {"eq": "landsat-8-l1"}, + "collection": {"eq": product}, }, sort=[{"field": "eo:cloud_cover", "direction": "asc"}], ) @@ -39,18 +52,20 @@ def search_landsat_data(coordinates, cloud_cover_lt): search_items = search.items() if not len(search_items): exit_cli(print, "No data matched your search, please try different parameters.") - landsat_item = search_items[0] - return landsat_item + # return the first result + item = search_items[0] + return item -def preview_landsat_image(landsat_item): - paths = file_paths_wrt_id(landsat_item._data["id"]) + +def preview_satellite_image(item): + paths = file_paths_wrt_id(item._data["id"]) # download image and save it in directory if not data_file_exists(paths["preview"]): save_to_file( - landsat_item.assets["thumbnail"]["href"], + item.assets["thumbnail"]["href"], paths["preview"], - landsat_item._data["id"], + item._data["id"], "✗ preview data doesn't exist, downloading image", ) else: @@ -65,6 +80,44 @@ def preview_landsat_image(landsat_item): return handle_prompt_response(response) +def download_data(item, bands): + product_type = get_product_type_from_id(item._data["id"]) + if product_type == "sentinel": + return download_sentinel_data(item, bands) + else: + return download_landsat_data(item, bands) + + +def download_sentinel_data(item, bands): + # get paths w.r.t. id + paths = file_paths_wrt_id(item._data["id"]) + # get meta info on path, to be used by boto3 + info_response = requests.get(item.assets["info"]["href"]) + info_response_json = json.loads(info_response.text) + # save bands generically + for band in bands: + # pass band id in metadata + info_response_json["band_id"] = band + band_filename = paths["b%s" % band] + if not data_file_exists(band_filename): + save_to_file( + item.assets["B0{}".format(band)]["href"], + band_filename, + item._data["id"], + "✗ required data doesn't exist, downloading %s %s" + % (band_tag_map["b" + str(band)], "band"), + meta=info_response_json, + ) + else: + rprint( + "[green] ✓ ", + "required data exists for {} band".format( + band_tag_map["b" + str(band)] + ), + ) + return item._data["id"] + + def download_landsat_data(landsat_item, bands): # get paths w.r.t. id diff --git a/felicette/sat_processor.py b/felicette/sat_processor.py index f96784a..ef49515 100644 --- a/felicette/sat_processor.py +++ b/felicette/sat_processor.py @@ -7,7 +7,7 @@ from felicette.utils.color import color from felicette.utils.gdal_pansharpen import gdal_pansharpen -from felicette.utils.file_manager import file_paths_wrt_id +from felicette.utils.file_manager import file_paths_wrt_id, get_product_type_from_id from felicette.utils.image_processing_utils import process_sat_image from felicette.utils.sys_utils import display_file @@ -15,7 +15,7 @@ PIL.Image.MAX_IMAGE_PIXELS = 933120000 -def process_landsat_vegetation(id, bands): +def process_vegetation(id, bands, ops_string, angle_rotation=None): # get paths of files related to this id paths = file_paths_wrt_id(id) @@ -53,7 +53,6 @@ def process_landsat_vegetation(id, bands): rprint("Let's make our 🌍 imagery a bit more colorful for a human eye!") # apply rio-color correction - ops_string = "sigmoidal rgb 20 0.2" # refer to felicette.utils.color.py to see the parameters of this function # Bug: number of jobs if greater than 1, fails the job color( @@ -68,7 +67,7 @@ def process_landsat_vegetation(id, bands): # resize and save as jpeg image print("Generated 🌍 images!🎉") rprint("[yellow]Please wait while I resize and crop the image :) [/yellow]") - process_sat_image(paths["vegetation_path"], paths["vegetation_path_jpeg"]) + process_sat_image(paths["vegetation_path"], paths["vegetation_path_jpeg"], rotate=angle_rotation) rprint("[blue]GeoTIFF saved at:[/blue]") print(paths["vegetation_path"]) rprint("[blue]JPEG image saved at:[/blue]") @@ -77,7 +76,7 @@ def process_landsat_vegetation(id, bands): display_file(paths["vegetation_path_jpeg"]) -def process_landsat_rgb(id, bands): +def process_rgb(id, bands, ops_string, angle_rotation=None): # get paths of files related to this id paths = file_paths_wrt_id(id) @@ -124,7 +123,6 @@ def process_landsat_rgb(id, bands): rprint("Let's make our 🌍 imagery a bit more colorful for a human eye!") # apply rio-color correction - ops_string = "sigmoidal rgb 20 0.2" # refer to felicette.utils.color.py to see the parameters of this function # Bug: number of jobs if greater than 1, fails the job color( @@ -139,7 +137,7 @@ def process_landsat_rgb(id, bands): # resize and save as jpeg image print("Generated 🌍 images!🎉") rprint("[yellow]Please wait while I resize and crop the image :) [/yellow]") - process_sat_image(paths["output_path"], paths["output_path_jpeg"]) + process_sat_image(paths["output_path"], paths["output_path_jpeg"], rotate=angle_rotation) rprint("[blue]GeoTIFF saved at:[/blue]") print(paths["output_path"]) rprint("[blue]JPEG image saved at:[/blue]") @@ -149,8 +147,23 @@ def process_landsat_rgb(id, bands): def process_landsat_data(id, bands): - + ops_string = "sigmoidal rgb 20 0.2" if bands == [2, 3, 4] or bands == [2, 3, 4, 8]: - process_landsat_rgb(id, bands) + process_rgb(id, bands, ops_string) + elif bands == [3, 4, 5]: + process_vegetation(id, bands, ops_string) + +def process_sentinel_data(id, bands): + ops_string = "gamma G 1.85 gamma B 1.85 gamma R 1.85 sigmoidal RGB 35 0.13 saturation 1.15" + angle_rotation = 0 + if bands == [2, 3, 4]: + process_rgb(id, bands, ops_string, angle_rotation=angle_rotation) elif bands == [3, 4, 5]: - process_landsat_vegetation(id, bands) + process_vegetation(id, bands, ops_string, angle_rotation=angle_rotation) + +def process_data(id, bands): + product_type = get_product_type_from_id(id) + if product_type == "sentinel": + process_sentinel_data(id, bands) + else: + process_landsat_data(id, bands) diff --git a/felicette/utils/file_manager.py b/felicette/utils/file_manager.py index f74ee3e..d69218b 100644 --- a/felicette/utils/file_manager.py +++ b/felicette/utils/file_manager.py @@ -3,8 +3,10 @@ import requests from tqdm import tqdm from rich import print as rprint +import boto3 from felicette.constants import band_tag_map +from felicette.utils.sys_utils import exit_cli workdir = os.path.join(os.path.expanduser("~"), "felicette-data") @@ -16,44 +18,95 @@ def check_sat_path(id): os.makedirs(data_path, exist_ok=True) -def save_to_file(url, filename, id, info_message): +def hook(t): + def inner(bytes_amount): + t.update(bytes_amount) + + return inner + + +def save_to_file(url, filename, id, info_message, meta=None): + product_type = get_product_type_from_id(id) data_path = os.path.join(workdir, id) data_id = filename.split("/")[-1].split("-")[1].split(".")[0] rprint(info_message) file_path = os.path.join(data_path, filename) - response = requests.get(url, stream=True) - with tqdm.wrapattr( - open(file_path, "wb"), - "write", - miniters=1, - desc=data_id, - total=int(response.headers.get("content-length", 0)), - ) as fout: - for chunk in response.iter_content(chunk_size=4096): - fout.write(chunk) - fout.close() + + if product_type == "sentinel" and meta: + aws_access_key_id = os.environ.get("AWS_ACCESS_KEY_ID", None) + aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY", None) + # if access key or secret isn't defined, print error message and exit + if (not aws_access_key_id) or (not aws_secret_access_key): + exit_cli(rprint, "Error: [red]AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY[/red] must be set in environment variables to access Sentinel data.") + # prepare boto3 client + s3_client = boto3.Session().client( + "s3", + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + ) + band = os.path.join(meta["path"], "B0%s.jp2" % (meta["band_id"])) + filesize = s3_client.head_object( + Bucket="sentinel-s2-l1c", Key=band, RequestPayer="requester" + ).get("ContentLength") + with tqdm(total=filesize, unit="B", unit_scale=True, desc=data_id) as t: + response = s3_client.download_file( + Bucket="sentinel-s2-l1c", + Key=band, + Filename=file_path, + ExtraArgs={"RequestPayer": "requester"}, + Callback=hook(t), + ) + else: + # for landsat, and preview images - resources which can be downloaded via http + response = requests.get(url, stream=True) + with tqdm.wrapattr( + open(file_path, "wb"), + "write", + miniters=1, + desc=data_id, + total=int(response.headers.get("content-length", 0)), + ) as fout: + for chunk in response.iter_content(chunk_size=4096): + fout.write(chunk) + fout.close() def data_file_exists(filename): return os.path.exists(filename) +def get_product_type_from_id(id): + if "LC" in id: + return "landsat" + else: + return "sentinel" + + def file_paths_wrt_id(id): home_path_id = os.path.join(workdir, id) + extension = None + if get_product_type_from_id(id) == "landsat": + extension = "tiff" + else: + extension = "jp2" return { "base": home_path_id, "preview": os.path.join(home_path_id, "%s-preview.jpg" % (id)), - "b5": os.path.join(home_path_id, "%s-b5.tiff" % (id)), - "b4": os.path.join(home_path_id, "%s-b4.tiff" % (id)), - "b3": os.path.join(home_path_id, "%s-b3.tiff" % (id)), - "b2": os.path.join(home_path_id, "%s-b2.tiff" % (id)), - "b8": os.path.join(home_path_id, "%s-b8.tiff" % (id)), + "b5": os.path.join(home_path_id, "%s-b5.%s" % (id, extension)), + "b4": os.path.join(home_path_id, "%s-b4.%s" % (id, extension)), + "b3": os.path.join(home_path_id, "%s-b3.%s" % (id, extension)), + "b2": os.path.join(home_path_id, "%s-b2.%s" % (id, extension)), + "b8": os.path.join(home_path_id, "%s-b8.%s" % (id, extension)), "stack": os.path.join(home_path_id, "%s-stack.tiff" % (id)), "pan_sharpened": os.path.join(home_path_id, "%s-pan.tiff" % (id)), - "output_path": os.path.join(home_path_id, "%s-color-processed.tiff" % (id)), + "output_path": os.path.join( + home_path_id, "%s-color-processed.tiff" % (id) + ), "output_path_jpeg": os.path.join( home_path_id, "%s-color-processed.jpeg" % (id) ), - "vegetation_path": os.path.join(home_path_id, "%s-vegetation.tiff" % (id)), + "vegetation_path": os.path.join( + home_path_id, "%s-vegetation.tiff" % (id) + ), "vegetation_path_jpeg": os.path.join(home_path_id, "%s-vegetation.jpeg" % (id)), } diff --git a/felicette/utils/image_processing_utils.py b/felicette/utils/image_processing_utils.py index b9d405b..5258e43 100644 --- a/felicette/utils/image_processing_utils.py +++ b/felicette/utils/image_processing_utils.py @@ -8,7 +8,7 @@ def find_max_area_index(contours): return contour_areas.index(max(contour_areas)) -def straighten_image(image): +def straighten_image(image, rotate): gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) gray = cv2.bitwise_not(gray) # threshold the image, setting all foreground pixels to 255 and all background pixels to 0 @@ -16,15 +16,17 @@ def straighten_image(image): # flip data in threshold array thresh1 = thresh - 255 np.where(thresh1 == -255, 0, thresh1) - + angle = None # find angle of inclination - - coords = np.column_stack(np.where(thresh1 > 0)) - angle = cv2.minAreaRect(coords)[-1] - if angle < -45: - angle = -(90 + angle) + if rotate == None: + coords = np.column_stack(np.where(thresh1 > 0)) + angle = cv2.minAreaRect(coords)[-1] + if angle < -45: + angle = -(90 + angle) + else: + angle = -angle else: - angle = -angle + angle = 0.0 # rotate the image to deskew it (h, w) = image.shape[:2] @@ -54,10 +56,10 @@ def remove_margin(rotated): return crop -def process_sat_image(source_path, dest_path): +def process_sat_image(source_path, dest_path, rotate): # load the image from disk image = cv2.imread(source_path) - straightened_image = straighten_image(image) + straightened_image = straighten_image(image, rotate=rotate) cropped_image = remove_margin(straightened_image) # write jpeg to destination diff --git a/setup.py b/setup.py index 57478f9..0b2432a 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ name="felicette", version="0.1.10", url="https://github.com/plant99/felicette", - license="BSD", + license="MIT", author="Shivashis Padhi", author_email="shivashispadhi@gmail.com", description="Satellite imagery for dummies.", @@ -44,7 +44,7 @@ # 'Development Status :: 7 - Inactive', "Environment :: Console", "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License", + "License :: OSI Approved :: MIT License", "Operating System :: POSIX", "Operating System :: MacOS", "Operating System :: Unix",