Merge pull request #30 from plant99/sentinel-integration

Integrate felicette with Sentinel-2 data source

README.md

@@ -77,14 +77,16 @@ Options:
                               -l command
 
   -l, --location-name TEXT    Location name in string format
-  -p, --pan-enhancement       Enhance image with panchromatic band
-  --no-preview                Skip downloading of preview image
+  -pan, --pan-enhancement     Enhance image with panchromatic band
+  --no-preview                Skip previewing of pre-processed low resolution RGB
+                              satellite image.
+
   -v, --vegetation            Show Color Infrared image to highlight
                               vegetation
 
   -V, --version               Show the version number and quit
+  -p, --product TEXT          Product name 'landsat'/'sentinel'
   --help                      Show this message and exit.
-
 ```
 
 Felicette can download and process Landsat images taking the location's input as `(lon, lat)` or the location name. They can be used in the following way.
@@ -97,10 +99,17 @@ With coordinates:
 
     $ felicette -c 77.5385 8.0883
 
-`-p` option uses the panchromatic band to enhance image's resolution to 15 meters, contrary to resolution of RGB bands(30 meters). 
-To get a better image using felicette use:
+`--product` / `-p` option is used to specify which data-product is used to generate images i.e Sentinel or Landsat. By default, Landsat-8 data will be used to generate images.
+
+    $ felicette -l "Kanyakumari" -p "sentinel"
+
+**NB**: *To use sentinel data source, one has to set `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` (To generate a pair, go to AWS console -> My Security Credentials -> Access keys). This is because [Sentinel-2 data](https://registry.opendata.aws/sentinel-2/) is in a [Requester Pays](https://docs.aws.amazon.com/AmazonS3/latest/dev/RequesterPaysBuckets.html) bucket.
+
+
+`-pan` option uses the panchromatic band to enhance image's resolution to 15 meters, contrary to resolution of RGB bands(30 meters) if Landsat product is being used. Felicette doesn't support any panchromatic enhancements for Sentinel-2 data which already have a resolution of 10m.
+To get a better Landsat image using felicette use:
 
-    $ felicette -p -c 77.5385 8.0883
+    $ felicette -pan -c 77.5385 8.0883
 
 `--no-preview` option doesn't download image to preview, and directly downloads and processes original data. Please use this if you're sure of the location/quality of the images to be generated by felicette with the arguments provided to it.
 

felicette/cli.py

@@ -7,18 +7,19 @@
 from felicette.utils.file_manager import check_sat_path, file_paths_wrt_id
 from felicette.sat_downloader import (
     download_landsat_data,
-    search_landsat_data,
-    preview_landsat_image,
+    search_satellite_data,
+    preview_satellite_image,
+    download_data,
 )
 from felicette.utils.sys_utils import exit_cli, remove_dir
-from felicette.sat_processor import process_landsat_data
+from felicette.sat_processor import process_data
 
 
-def trigger_download_and_processing(landsat_item, bands):
+def trigger_download_and_processing(item, bands):
     # download data
-    data_id = download_landsat_data(landsat_item, bands)
+    data_id = download_data(item, bands)
     # process data
-    process_landsat_data(data_id, bands)
+    process_data(data_id, bands)
 
 
 @click.command()
@@ -31,7 +32,7 @@ def trigger_download_and_processing(landsat_item, bands):
 )
 @click.option("-l", "--location-name", type=str, help="Location name in string format")
 @click.option(
-    "-p",
+    "-pan",
     "--pan-enhancement",
     default=False,
     is_flag=True,
@@ -41,7 +42,7 @@ def trigger_download_and_processing(landsat_item, bands):
     "--no-preview",
     default=False,
     is_flag=True,
-    help="Preview pre-processed low resolution RGB satellite image.",
+    help="Skip previewing of pre-processed low resolution RGB satellite image.",
 )
 @click.option(
     "-v",
@@ -57,7 +58,8 @@ def trigger_download_and_processing(landsat_item, bands):
     is_flag=True,
     help="Show the version number and quit",
 )
-def main(coordinates, location_name, pan_enhancement, no_preview, vegetation, version):
+@click.option("-p", "--product", type=str, default="landsat", help="Product name 'landsat'/'sentinel'")
+def main(coordinates, location_name, pan_enhancement, no_preview, vegetation, version, product):
     """Satellite imagery for dummies."""
     if version:
         version_no = pkg_resources.require("felicette")[0].version
@@ -68,37 +70,38 @@ def main(coordinates, location_name, pan_enhancement, no_preview, vegetation, ve
         coordinates = geocoder_util(location_name)
 
     # unless specified, cloud_cover_lt is 10
-    landsat_item = search_landsat_data(coordinates, 10)
+    item = search_satellite_data(coordinates, 10, product=product)
 
     # check if directory exists to save the data for this product id
-    check_sat_path(landsat_item._data["id"])
+    check_sat_path(item._data["id"])
 
     # if preview option is set, download and preview image
     if not no_preview:
-        preview_landsat_image(landsat_item)
+        preview_satellite_image(item)
 
     # set bands to process
     bands = [2, 3, 4]
-    if pan_enhancement:
+    if pan_enhancement and (product != "sentinel"):
         bands.append(8)
 
     if vegetation:
         bands = [3, 4, 5]
 
     # NB: can't enable pan-enhancement with vegetation
+    # NB: can't enable pan-enhancement with sentinel
 
     try:
-        trigger_download_and_processing(landsat_item, bands)
+        trigger_download_and_processing(item, bands)
     except RasterioIOError:
         response = input(
             "Local data for this location is corrupted, felicette will remove existing data to proceed, are you sure? [Y/n]"
         )
         if response in ["y", "Y", ""]:
             # remove file dir
-            file_paths = file_paths_wrt_id(landsat_item._data["id"])
+            file_paths = file_paths_wrt_id(item._data["id"])
             remove_dir(file_paths["base"])
             # retry downloading and processing image with a clean directory
-            trigger_download_and_processing(landsat_item, bands)
+            trigger_download_and_processing(item, bands)
         elif response in ["n", "N"]:
             exit_cli(print, "")
 

felicette/sat_downloader.py

@@ -1,6 +1,8 @@
 from satsearch import Search
 import sys
 from rich import print as rprint
+import requests
+import json
 
 from felicette.utils.geo_utils import get_tiny_bbox
 from felicette.utils.sys_utils import exit_cli
@@ -9,6 +11,7 @@
     save_to_file,
     data_file_exists,
     file_paths_wrt_id,
+    get_product_type_from_id,
 )
 
 
@@ -24,12 +27,22 @@ def handle_prompt_response(response):
         exit_cli(rprint, "[red]Sorry, invalid response. Exiting :([/red]")
 
 
-def search_landsat_data(coordinates, cloud_cover_lt):
+def search_satellite_data(coordinates, cloud_cover_lt, product="landsat"):
+    """
+    coordinates: bounding box's coordinates
+    cloud_cover_lt: maximum cloud cover
+    product: landsat, sentinel
+    """
+    if product == "landsat":
+        product = "landsat-8-l1"
+    elif product == "sentinel":
+        product = "sentinel-2-l1c"
+
     search = Search(
         bbox=get_tiny_bbox(coordinates),
         query={
             "eo:cloud_cover": {"lt": cloud_cover_lt},
-            "collection": {"eq": "landsat-8-l1"},
+            "collection": {"eq": product},
         },
         sort=[{"field": "eo:cloud_cover", "direction": "asc"}],
     )
@@ -39,18 +52,20 @@ def search_landsat_data(coordinates, cloud_cover_lt):
     search_items = search.items()
     if not len(search_items):
         exit_cli(print, "No data matched your search, please try different parameters.")
-    landsat_item = search_items[0]
-    return landsat_item
 
+    # return the first result
+    item = search_items[0]
+    return item
 
-def preview_landsat_image(landsat_item):
-    paths = file_paths_wrt_id(landsat_item._data["id"])
+
+def preview_satellite_image(item):
+    paths = file_paths_wrt_id(item._data["id"])
     # download image and save it in directory
     if not data_file_exists(paths["preview"]):
         save_to_file(
-            landsat_item.assets["thumbnail"]["href"],
+            item.assets["thumbnail"]["href"],
             paths["preview"],
-            landsat_item._data["id"],
+            item._data["id"],
             "✗ preview data doesn't exist, downloading image",
         )
     else:
@@ -65,6 +80,44 @@ def preview_landsat_image(landsat_item):
     return handle_prompt_response(response)
 
 
+def download_data(item, bands):
+    product_type = get_product_type_from_id(item._data["id"])
+    if product_type == "sentinel":
+        return download_sentinel_data(item, bands)
+    else:
+        return download_landsat_data(item, bands)
+
+
+def download_sentinel_data(item, bands):
+    # get paths w.r.t. id
+    paths = file_paths_wrt_id(item._data["id"])
+    # get meta info on path, to be used by boto3
+    info_response = requests.get(item.assets["info"]["href"])
+    info_response_json = json.loads(info_response.text)
+    # save bands generically
+    for band in bands:
+        # pass band id in metadata
+        info_response_json["band_id"] = band
+        band_filename = paths["b%s" % band]
+        if not data_file_exists(band_filename):
+            save_to_file(
+                item.assets["B0{}".format(band)]["href"],
+                band_filename,
+                item._data["id"],
+                "✗ required data doesn't exist, downloading %s %s"
+                % (band_tag_map["b" + str(band)], "band"),
+                meta=info_response_json,
+            )
+        else:
+            rprint(
+                "[green] ✓ ",
+                "required data exists for {} band".format(
+                    band_tag_map["b" + str(band)]
+                ),
+            )
+    return item._data["id"]
+
+
 def download_landsat_data(landsat_item, bands):
 
     # get paths w.r.t. id

felicette/sat_processor.py

@@ -7,15 +7,15 @@
 
 from felicette.utils.color import color
 from felicette.utils.gdal_pansharpen import gdal_pansharpen
-from felicette.utils.file_manager import file_paths_wrt_id
+from felicette.utils.file_manager import file_paths_wrt_id, get_product_type_from_id
 from felicette.utils.image_processing_utils import process_sat_image
 from felicette.utils.sys_utils import display_file
 
 # increase PIL image processing pixels count limit
 PIL.Image.MAX_IMAGE_PIXELS = 933120000
 
 
-def process_landsat_vegetation(id, bands):
+def process_vegetation(id, bands, ops_string, angle_rotation=None):
 
     # get paths of files related to this id
     paths = file_paths_wrt_id(id)
@@ -53,7 +53,6 @@ def process_landsat_vegetation(id, bands):
 
     rprint("Let's make our 🌍 imagery a bit more colorful for a human eye!")
     # apply rio-color correction
-    ops_string = "sigmoidal rgb 20 0.2"
     # refer to felicette.utils.color.py to see the parameters of this function
     # Bug: number of jobs if greater than 1, fails the job
     color(
@@ -68,7 +67,7 @@ def process_landsat_vegetation(id, bands):
     # resize and save as jpeg image
     print("Generated 🌍 images!🎉")
     rprint("[yellow]Please wait while I resize and crop the image :) [/yellow]")
-    process_sat_image(paths["vegetation_path"], paths["vegetation_path_jpeg"])
+    process_sat_image(paths["vegetation_path"], paths["vegetation_path_jpeg"], rotate=angle_rotation)
     rprint("[blue]GeoTIFF saved at:[/blue]")
     print(paths["vegetation_path"])
     rprint("[blue]JPEG image saved at:[/blue]")
@@ -77,7 +76,7 @@ def process_landsat_vegetation(id, bands):
     display_file(paths["vegetation_path_jpeg"])
 
 
-def process_landsat_rgb(id, bands):
+def process_rgb(id, bands, ops_string, angle_rotation=None):
     # get paths of files related to this id
     paths = file_paths_wrt_id(id)
 
@@ -124,7 +123,6 @@ def process_landsat_rgb(id, bands):
 
     rprint("Let's make our 🌍 imagery a bit more colorful for a human eye!")
     # apply rio-color correction
-    ops_string = "sigmoidal rgb 20 0.2"
     # refer to felicette.utils.color.py to see the parameters of this function
     # Bug: number of jobs if greater than 1, fails the job
     color(
@@ -139,7 +137,7 @@ def process_landsat_rgb(id, bands):
     # resize and save as jpeg image
     print("Generated 🌍 images!🎉")
     rprint("[yellow]Please wait while I resize and crop the image :) [/yellow]")
-    process_sat_image(paths["output_path"], paths["output_path_jpeg"])
+    process_sat_image(paths["output_path"], paths["output_path_jpeg"], rotate=angle_rotation)
     rprint("[blue]GeoTIFF saved at:[/blue]")
     print(paths["output_path"])
     rprint("[blue]JPEG image saved at:[/blue]")
@@ -149,8 +147,23 @@ def process_landsat_rgb(id, bands):
 
 
 def process_landsat_data(id, bands):
-
+    ops_string = "sigmoidal rgb 20 0.2"
     if bands == [2, 3, 4] or bands == [2, 3, 4, 8]:
-        process_landsat_rgb(id, bands)
+        process_rgb(id, bands, ops_string)
+    elif bands == [3, 4, 5]:
+        process_vegetation(id, bands, ops_string)
+
+def process_sentinel_data(id, bands):
+    ops_string = "gamma G 1.85 gamma B 1.85 gamma R 1.85 sigmoidal RGB 35 0.13 saturation 1.15"
+    angle_rotation = 0
+    if bands == [2, 3, 4]:
+        process_rgb(id, bands, ops_string, angle_rotation=angle_rotation)
     elif bands == [3, 4, 5]:
-        process_landsat_vegetation(id, bands)
+        process_vegetation(id, bands, ops_string, angle_rotation=angle_rotation)
+
+def process_data(id, bands):
+    product_type = get_product_type_from_id(id)
+    if product_type == "sentinel":
+        process_sentinel_data(id, bands)
+    else:
+        process_landsat_data(id, bands)