diff --git a/jaffle_shop/models/customers.sql b/jaffle_shop/models/customers.sql deleted file mode 100644 index 016a004fe..000000000 --- a/jaffle_shop/models/customers.sql +++ /dev/null @@ -1,69 +0,0 @@ -with customers as ( - - select * from {{ ref('stg_customers') }} - -), - -orders as ( - - select * from {{ ref('stg_orders') }} - -), - -payments as ( - - select * from {{ ref('stg_payments') }} - -), - -customer_orders as ( - - select - customer_id, - - min(order_date) as first_order, - max(order_date) as most_recent_order, - count(order_id) as number_of_orders - from orders - - group by customer_id - -), - -customer_payments as ( - - select - orders.customer_id, - sum(amount) as total_amount - - from payments - - left join orders on - payments.order_id = orders.order_id - - group by orders.customer_id - -), - -final as ( - - select - customers.customer_id, - customers.first_name, - customers.last_name, - customer_orders.first_order, - customer_orders.most_recent_order, - customer_orders.number_of_orders, - customer_payments.total_amount as customer_lifetime_value - - from customers - - left join customer_orders - on customers.customer_id = customer_orders.customer_id - - left join customer_payments - on customers.customer_id = customer_payments.customer_id - -) - -select * from final diff --git a/jaffle_shop/models/finance/_exposures.yml b/jaffle_shop/models/finance/_exposures.yml new file mode 100644 index 000000000..e69de29bb diff --git a/jaffle_shop/models/finance/_models.yml b/jaffle_shop/models/finance/_models.yml new file mode 100644 index 000000000..001b4c53b --- /dev/null +++ b/jaffle_shop/models/finance/_models.yml @@ -0,0 +1,16 @@ +version: 2 + +models: + - name: fnl_finance_order_returns + config: + tags: [access:public] + meta: + owner: 'example.person@octoenergy.com' + team_owner: 'example_owner@octoenergy.com' + description: Table containing the value of all returned orders, per customer. + columns: + - name: customer_id + description: Primary key of the table + tests: + - unique + - not_null diff --git a/jaffle_shop/models/finance/fnl_finance_order_returns.sql b/jaffle_shop/models/finance/fnl_finance_order_returns.sql new file mode 100644 index 000000000..ba758533c --- /dev/null +++ b/jaffle_shop/models/finance/fnl_finance_order_returns.sql @@ -0,0 +1,6 @@ +SELECT + customer_id + , SUM(amount) AS total_value_returned_per_customer +FROM {{ ref('wh_orders') }} +WHERE status = 'returned' +GROUP BY customer_id diff --git a/jaffle_shop/models/orders.sql b/jaffle_shop/models/orders.sql deleted file mode 100644 index cbb293491..000000000 --- a/jaffle_shop/models/orders.sql +++ /dev/null @@ -1,56 +0,0 @@ -{% set payment_methods = ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] %} - -with orders as ( - - select * from {{ ref('stg_orders') }} - -), - -payments as ( - - select * from {{ ref('stg_payments') }} - -), - -order_payments as ( - - select - order_id, - - {% for payment_method in payment_methods -%} - sum(case when payment_method = '{{ payment_method }}' then amount else 0 end) as {{ payment_method }}_amount, - {% endfor -%} - - sum(amount) as total_amount - - from payments - - group by order_id - -), - -final as ( - - select - orders.order_id, - orders.customer_id, - orders.order_date, - orders.status, - - {% for payment_method in payment_methods -%} - - order_payments.{{ payment_method }}_amount, - - {% endfor -%} - - order_payments.total_amount as amount - - from orders - - - left join order_payments - on orders.order_id = order_payments.order_id - -) - -select * from final diff --git a/jaffle_shop/models/sales/_exposures.yml b/jaffle_shop/models/sales/_exposures.yml new file mode 100644 index 000000000..9452f4cfe --- /dev/null +++ b/jaffle_shop/models/sales/_exposures.yml @@ -0,0 +1,15 @@ +version: 2 + +################################################################################ +# EXPOSURES https://docs.getdbt.com/reference/exposure-properties +################################################################################ + +exposures: + - name: fnl_new_customer_sales + description: Inksacio dashboard + type: dashboard + url: https://inksacio.eks.octopus.engineering/my_certification_dashboard/ + owner: + email: 'example.person@octoenergy.com' + depends_on: + - ref('fnl_new_customer_sales') diff --git a/jaffle_shop/models/sales/_models.yml b/jaffle_shop/models/sales/_models.yml new file mode 100644 index 000000000..f96f3c2e7 --- /dev/null +++ b/jaffle_shop/models/sales/_models.yml @@ -0,0 +1,16 @@ +version: 2 + +models: + - name: fnl_new_customer_sales + config: + tags: [access:public] + meta: + owner: 'example.person@octoenergy.com' + team_owner: 'example_owner@octoenergy.com' + description: Table on the number of customers who have made their first orders per month. + columns: + - name: first_order_month + description: Primary key of the table + tests: + - unique + - not_null diff --git a/jaffle_shop/models/sales/fnl_new_customer_sales.sql b/jaffle_shop/models/sales/fnl_new_customer_sales.sql new file mode 100644 index 000000000..7e33ea0aa --- /dev/null +++ b/jaffle_shop/models/sales/fnl_new_customer_sales.sql @@ -0,0 +1,6 @@ +SELECT + DATE_TRUNC('month', first_order) AS first_order_month + , COUNT(customer_id) AS customer_count +FROM {{ ref('wh_customers') }} +WHERE number_of_orders = 1 -- Only count customers with their first order +GROUP BY DATE_TRUNC('month', first_order) diff --git a/jaffle_shop/models/src_seed/_models.yml b/jaffle_shop/models/src_seed/_models.yml new file mode 100644 index 000000000..6ab551d1f --- /dev/null +++ b/jaffle_shop/models/src_seed/_models.yml @@ -0,0 +1,76 @@ +version: 2 + +models: + - name: stg_customers_pii + access: private + config: + group: example_group + meta: + owner: 'example.person@octoenergy.com' + team_owner: 'example_team@octoenergy.com' + sensitive: true + description: Table containing customer name information with PII included. + columns: + - name: customer_id + tests: + - unique + - not_null + - name: first_name + meta: + sensitive: true + - name: last_name + meta: + sensitive: true + - name: stg_customers + access: private + config: + group: example_group + meta: + owner: 'example.person@octoenergy.com' + team_owner: 'example_team@octoenergy.com' + description: Table containing customer name information with PII hashed + columns: + - name: customer_id + tests: + - unique + - not_null + - name: first_name_hash + tests: + - dbt_expectations.expect_column_to_exist + - name: last_name_hash + tests: + - dbt_expectations.expect_column_to_exist + - name: stg_orders + access: private + config: + group: example_group + meta: + owner: 'example.person@octoenergy.com' + team_owner: 'example_team@octoenergy.com' + description: Table containing customer orders. + columns: + - name: order_id + tests: + - unique + - not_null + - name: status + tests: + - accepted_values: + values: ['placed', 'shipped', 'completed', 'return_pending', 'returned'] + - name: stg_payments + access: private + config: + group: example_group + meta: + owner: 'example.person@octoenergy.com' + team_owner: 'example_team@octoenergy.com' + description: Table containing customer payments. + columns: + - name: payment_id + tests: + - unique + - not_null + - name: payment_method + tests: + - accepted_values: + values: ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] diff --git a/jaffle_shop/models/src_seed/stg_customers.sql b/jaffle_shop/models/src_seed/stg_customers.sql new file mode 100644 index 000000000..10ea1a0ef --- /dev/null +++ b/jaffle_shop/models/src_seed/stg_customers.sql @@ -0,0 +1,3 @@ +SELECT + {{ hash_sensitive_columns('stg_customers_pii') }} +FROM {{ ref('stg_customers_pii') }} diff --git a/jaffle_shop/models/src_seed/stg_customers_pii.sql b/jaffle_shop/models/src_seed/stg_customers_pii.sql new file mode 100644 index 000000000..de98487f7 --- /dev/null +++ b/jaffle_shop/models/src_seed/stg_customers_pii.sql @@ -0,0 +1,5 @@ +SELECT + id AS customer_id + , first_name + , last_name +FROM {{ ref('raw_customers') }} diff --git a/jaffle_shop/models/src_seed/stg_orders.sql b/jaffle_shop/models/src_seed/stg_orders.sql new file mode 100644 index 000000000..b378c7a29 --- /dev/null +++ b/jaffle_shop/models/src_seed/stg_orders.sql @@ -0,0 +1,6 @@ +SELECT + id AS order_id + , user_id AS customer_id + , order_date + , status +FROM {{ ref('raw_orders') }} diff --git a/jaffle_shop/models/src_seed/stg_payments.sql b/jaffle_shop/models/src_seed/stg_payments.sql new file mode 100644 index 000000000..9386b06e5 --- /dev/null +++ b/jaffle_shop/models/src_seed/stg_payments.sql @@ -0,0 +1,6 @@ +SELECT + id AS payment_id + , order_id + , payment_method + , amount / 100 AS amount +FROM {{ ref('raw_payments') }} diff --git a/jaffle_shop/models/staging/schema.yml b/jaffle_shop/models/staging/schema.yml deleted file mode 100644 index c207e4cf5..000000000 --- a/jaffle_shop/models/staging/schema.yml +++ /dev/null @@ -1,31 +0,0 @@ -version: 2 - -models: - - name: stg_customers - columns: - - name: customer_id - tests: - - unique - - not_null - - - name: stg_orders - columns: - - name: order_id - tests: - - unique - - not_null - - name: status - tests: - - accepted_values: - values: ['placed', 'shipped', 'completed', 'return_pending', 'returned'] - - - name: stg_payments - columns: - - name: payment_id - tests: - - unique - - not_null - - name: payment_method - tests: - - accepted_values: - values: ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] diff --git a/jaffle_shop/models/staging/stg_customers.sql b/jaffle_shop/models/staging/stg_customers.sql deleted file mode 100644 index cad047269..000000000 --- a/jaffle_shop/models/staging/stg_customers.sql +++ /dev/null @@ -1,22 +0,0 @@ -with source as ( - - {#- - Normally we would select from the table here, but we are using seeds to load - our data in this project - #} - select * from {{ ref('raw_customers') }} - -), - -renamed as ( - - select - id as customer_id, - first_name, - last_name - - from source - -) - -select * from renamed diff --git a/jaffle_shop/models/staging/stg_orders.sql b/jaffle_shop/models/staging/stg_orders.sql deleted file mode 100644 index a654dcb94..000000000 --- a/jaffle_shop/models/staging/stg_orders.sql +++ /dev/null @@ -1,23 +0,0 @@ -with source as ( - - {#- - Normally we would select from the table here, but we are using seeds to load - our data in this project - #} - select * from {{ ref('raw_orders') }} - -), - -renamed as ( - - select - id as order_id, - user_id as customer_id, - order_date, - status - - from source - -) - -select * from renamed diff --git a/jaffle_shop/models/staging/stg_payments.sql b/jaffle_shop/models/staging/stg_payments.sql deleted file mode 100644 index 700cf7f4f..000000000 --- a/jaffle_shop/models/staging/stg_payments.sql +++ /dev/null @@ -1,25 +0,0 @@ -with source as ( - - {#- - Normally we would select from the table here, but we are using seeds to load - our data in this project - #} - select * from {{ ref('raw_payments') }} - -), - -renamed as ( - - select - id as payment_id, - order_id, - payment_method, - - -- `amount` is currently stored in cents, so we convert it to dollars - amount / 100 as amount - - from source - -) - -select * from renamed diff --git a/jaffle_shop/models/schema.yml b/jaffle_shop/models/warehouse/_models.yml similarity index 62% rename from jaffle_shop/models/schema.yml rename to jaffle_shop/models/warehouse/_models.yml index 381349cfd..b2f51510f 100644 --- a/jaffle_shop/models/schema.yml +++ b/jaffle_shop/models/warehouse/_models.yml @@ -1,81 +1,78 @@ version: 2 -models: - - name: customers - description: This table has basic information about a customer, as well as some derived facts based on a customer's orders - +models: + - name: wh_customers + access: public + config: + group: example_group + meta: + owner: 'example.person@octoenergy.com' + team_owner: 'example_team@octoenergy.com' + description: This table has basic information about a customer, as well as some derived facts based on a customer's orders. + information. columns: - name: customer_id description: This is a unique identifier for a customer tests: - unique - not_null - - - name: first_name - description: Customer's first name. PII. - + - name: first_name + description: Hashed version of the customer's first name - name: last_name - description: Customer's last name. PII. - + description: Hashed version of the customer's last name - name: first_order description: Date (UTC) of a customer's first order - - name: most_recent_order description: Date (UTC) of a customer's most recent order - - name: number_of_orders description: Count of the number of orders a customer has placed - - - name: total_order_amount + - name: customer_lifetime_value description: Total value (AUD) of a customer's orders - - name: orders - description: This table has basic information about orders, as well as some derived facts based on payments - + - name: wh_orders + access: public + config: + group: example_group + meta: + owner: 'example.person@octoenergy.com' + team_owner: 'example_team@octoenergy.com' + description: This table has basic information about orders, as well as some derived facts based on payments. columns: - name: order_id + description:This is a unique identifier for an order tests: - unique - not_null - description: This is a unique identifier for an order - - - name: customer_id - description: Foreign key to the customers table + - name: customer_id + description: Foreign key to customers table- unique customer number tests: - - not_null + - not_null - relationships: - to: ref('customers') + to: ref('wh_customers') field: customer_id - - name: order_date description: Date (UTC) that the order was placed - - name: status - description: '{{ doc("orders_status") }}' + description: {{ docs("orders_status") }} tests: - accepted_values: values: ['placed', 'shipped', 'completed', 'return_pending', 'returned'] - - name: amount description: Total amount (AUD) of the order tests: - not_null - - name: credit_card_amount description: Amount of the order (AUD) paid for by credit card tests: - not_null - - - name: coupon_amount + - name: coupon_amount description: Amount of the order (AUD) paid for by coupon tests: - not_null - - name: bank_transfer_amount description: Amount of the order (AUD) paid for by bank transfer tests: - not_null - - name: gift_card_amount description: Amount of the order (AUD) paid for by gift card tests: diff --git a/jaffle_shop/models/docs.md b/jaffle_shop/models/warehouse/docs.md similarity index 100% rename from jaffle_shop/models/docs.md rename to jaffle_shop/models/warehouse/docs.md diff --git a/jaffle_shop/models/warehouse/wh_customers.sql b/jaffle_shop/models/warehouse/wh_customers.sql new file mode 100644 index 000000000..d3987709e --- /dev/null +++ b/jaffle_shop/models/warehouse/wh_customers.sql @@ -0,0 +1,38 @@ +WITH customer_orders AS ( + SELECT + customer_id + , MIN(order_date) AS first_order + , MAX(order_date) AS latest_order + , COUNT(order_id) AS number_of_orders + FROM {{ ref('stg_orders') }} + + GROUP BY customer_id +) + +, customer_payments AS ( + SELECT + customer_orders.customer_id + , SUM(payments.amount) AS total_amount + FROM {{ ref('stg_payments') }} AS payments + LEFT JOIN customer_orders + ON payments.order_id = customer_orders.order_id + + GROUP BY customer_orders.customer_id + +) + +SELECT + customers.customer_id + , customers.first_name + , customers.last_name + , customer_orders.first_order + , customer_orders.most_recent_order + , customer_orders.number_of_orders + , customer_payments.total_amount AS customer_lifetime_value +FROM {{ ref('stg_customers') }} AS customers +LEFT JOIN customer_orders + ON + customers.customer_id = customer_orders.customer_id +LEFT JOIN customer_payments + ON + customers.customer_id = customer_payments.customer_id diff --git a/jaffle_shop/models/warehouse/wh_orders.sql b/jaffle_shop/models/warehouse/wh_orders.sql new file mode 100644 index 000000000..cdeace0b6 --- /dev/null +++ b/jaffle_shop/models/warehouse/wh_orders.sql @@ -0,0 +1,34 @@ +{% set payment_methods = ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] %} + +WITH order_payments AS ( + + SELECT + order_id + + , {% for payment_method in payment_methods -%} + SUM(CASE WHEN payment_method = '{{ payment_method }}' THEN amount ELSE 0 END) AS {{ payment_method }}_amount + {% endfor -%} + + , SUM(amount) AS total_amount + + FROM {{ ref('stg_payments') }} + GROUP BY order_id + +) +SELECT + , orders.order_id + , orders.customer_id + , orders.order_date + , orders.status + + {% for payment_method in payment_methods -%} + + order_payments.{{ payment_method }}_amount + + {% endfor -%} + + , order_payments.total_amount as amount + +FROM {{ ref('stg_orders') }} AS orders +LEFT JOIN order_payments + ON orders.order_id = order_payments.order_id diff --git a/jaffle_shop/seeds/_seeds.yml b/jaffle_shop/seeds/_seeds.yml new file mode 100644 index 000000000..6d5f5e9dc --- /dev/null +++ b/jaffle_shop/seeds/_seeds.yml @@ -0,0 +1,26 @@ +version: 2 + +seeds: + - name: raw_customers + description: CSV file containing customer name information + config: + column_types: + id: int + first_name: string + last_name: string + - name: raw_orders + description: CSV file containing customer order information + config: + column_types: + id: int + user_id: int + order_date: date + status: string + - name: raw_payments + description: CSV file containing customer payment information + config: + column_types: + id: int + order_id: int + payment_method: string + amount: int diff --git a/jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv b/jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv new file mode 100644 index 000000000..b9f94e579 --- /dev/null +++ b/jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv @@ -0,0 +1,2 @@ +fct_name,column_name,id_to_exclude,comment +fct_staging_dependent_on_staging,parent,stg_customers_pii,Scrubbing pii permitted in staging layer. \ No newline at end of file