46-ETL_lab.py

import sys
from awsglue.transforms import *
from awsglue.utils import getResolvedOptions
from pyspark.context import SparkContext
from awsglue.context import GlueContext
from awsglue.job import Job

glueContext = GlueContext(SparkContext.getOrCreate())

items = glueContext.create_dynamic_frame.from_catalog(
           database="testdb",
           table_name="sparkstreametl")

items = items.drop_fields(['description',
                        'identifiers']).rename_field(
                            'item', 'product').rename_field(
                               'price', 'product_price')

s_items = items.toDF().repartition(1)
s_items.write.parquet('s3://go-lambda-bucket/output-dir/items_single')



# https://kinesis.us-west-2.amazonaws.com