Fix repo for gatekeeper cert

jaffle_shop/models/final/finance/_models.yml

@@ -0,0 +1,26 @@
+version: 2
+
+models:
+  - name: fnl_finance_orders
+    description: This table has some basic information about the amount and value of orders made per customer split by order status
+    columns:
+      - name: customer_id
+        description: Primary key
+        tests:
+          - unique
+          - not_null
+      - name: total_orders
+        description: This is the total number of orders made by a customer
+      - name: total_amount
+        description: This is the total amount of money spent by a customer
+      - name: order_status
+        description: This is the status of the order
+exposures:
+  - name: fnl_finance_orders
+    description: Dashboard showing the number and value of orders made per customer split by order status
+    type: dashboard
+    url: https://inksacio.eks.octopus.engineering/finance-orders/
+    owner:
+      email: [email protected]
+    depends_on:
+      - ref('wh_orders')

jaffle_shop/models/final/finance/fnl_finance_orders.sql

@@ -0,0 +1,15 @@
+WITH orders AS (
+    SELECT
+        *
+    FROM {{ ref('wh_orders') }}
+)
+
+SELECT
+    customer_id
+    , SUM(amount) AS total_amount
+    , COUNT(order_id) AS total_orders
+    , status AS order_status
+FROM orders
+GROUP BY
+    customer_id
+    , order_status

jaffle_shop/models/final/sales/_models.yml

@@ -0,0 +1,22 @@
+version: 2
+
+models:
+  - name: fnl_sales_newcustomers
+    description: This table has some basic information about the number of first time customers per month
+    columns:
+      - name: first_order_month
+        description: Primary key
+        tests:
+          - unique
+          - not_null
+      - name: new_customer_count
+        description: New customer count per month
+exposures:
+  - name: fnl_sales_newcustomers
+    description: Dashboard showing the number and value of new customers per month
+    type: dashboard
+    url: https://inksacio.eks.octopus.engineering/sales-newcustomers/
+    owner:
+      email: [email protected]
+    depends_on:
+      - ref('wh_customers')

jaffle_shop/models/final/sales/fnl_sales_newcustomers.sql

@@ -0,0 +1,11 @@
+WITH first_orders AS (
+    SELECT
+        *
+    FROM {{ ref('wh_customers') }}
+)
+
+SELECT
+    TRUNC(first_order, "MM") AS first_order_month
+    , COUNT(DISTINCT customer_id) AS new_customer_count
+FROM first_orders
+GROUP BY first_order_month

jaffle_shop/models/staging/src_seed/_models.yml

@@ -0,0 +1,55 @@
+version: 2
+
+models:
+  - name: stg_customers_pii
+    description: This table has basic information about a customer and contains PII
+    columns:
+      - name: customer_id
+        description: Primary key
+        tests:
+          - unique
+          - not_null
+      - name: first_name
+        meta:
+          sensitive: true
+      - name: last_name
+        meta:
+          sensitive: true
+  - name: stg_customers
+    description: This table has basic information about a customer
+    columns:
+      - name: customer_id
+        description: Primary key
+        tests:
+          - unique
+          - not_null
+      - name: first_name_hash
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: last_name_hash
+        tests:
+          - dbt_expectations.expect_column_to_exist
+  - name: stg_orders
+    description: This table has basic information about orders
+    columns:
+      - name: order_id
+        description: Primary key
+        tests:
+          - unique
+          - not_null
+      - name: status
+        tests:
+          - accepted_values:
+              values: ['placed', 'shipped', 'completed', 'return_pending', 'returned']
+  - name: stg_payments
+    description: This table has basic information about payments
+    columns:
+      - name: payment_id
+        description: Primary key
+        tests:
+          - unique
+          - not_null
+      - name: payment_method
+        tests:
+          - accepted_values:
+              values: ['credit_card', 'coupon', 'bank_transfer', 'gift_card']

jaffle_shop/models/staging/src_seed/stg_customers.sql

@@ -0,0 +1,3 @@
+SELECT
+  {{ hash_sensitive_columns('stg_customers_pii') }}
+FROM {{ ref('stg_customers_pii') }}

jaffle_shop/models/staging/stg_customers.sql → jaffle_shop/models/staging/src_seed/stg_customers_pii.sql

@@ -11,9 +11,9 @@ with source as (
 renamed as (
 
     select
-        id as customer_id,
-        first_name,
-        last_name
+        CAST(id AS BIGINT) as customer_id,
+        NULLIF(first_name,"") as first_name,
+        NULLIF(last_name,"") as last_name
 
     from source
 

jaffle_shop/models/staging/stg_orders.sql → jaffle_shop/models/staging/src_seed/stg_orders.sql

@@ -11,10 +11,10 @@ with source as (
 renamed as (
 
     select
-        id as order_id,
-        user_id as customer_id,
-        order_date,
-        status
+        CAST(id AS BIGINT) AS order_id,
+        CAST(user_id AS BIGINT) AS customer_id,
+        CAST(order_date AS DATE) AS order_date,
+        NULLIF(status, "") AS order_status
 
     from source
 

jaffle_shop/models/staging/stg_payments.sql → jaffle_shop/models/staging/src_seed/stg_payments.sql

@@ -11,9 +11,9 @@ with source as (
 renamed as (
 
     select
-        id as payment_id,
-        order_id,
-        payment_method,
+        CAST(id AS BIGINT) AS payment_id,
+        CAST(order_id AS BIGINT) AS order_id,
+        NULLIF(payment_method,"") AS payment_method,
 
         -- `amount` is currently stored in cents, so we convert it to dollars
         amount / 100 as amount

jaffle_shop/models/schema.yml → jaffle_shop/models/warehouse/_models.yml

@@ -1,81 +1,64 @@
 version: 2
 
 models:
-  - name: customers
+  - name: wh_customers
     description: This table has basic information about a customer, as well as some derived facts based on a customer's orders
-
     columns:
       - name: customer_id
-        description: This is a unique identifier for a customer
+        description: Primary key
         tests:
           - unique
           - not_null
-
-      - name: first_name
-        description: Customer's first name. PII.
-
-      - name: last_name
-        description: Customer's last name. PII.
-
+      - name: first_name_hash
+        description: Customer's first name (hashed)
+      - name: last_name_hash
+        description: Customer's last name (hashed)
       - name: first_order
         description: Date (UTC) of a customer's first order
-
       - name: most_recent_order
         description: Date (UTC) of a customer's most recent order
-
       - name: number_of_orders
         description: Count of the number of orders a customer has placed
-
       - name: total_order_amount
         description: Total value (AUD) of a customer's orders
-
-  - name: orders
+  - name: wh_orders
     description: This table has basic information about orders, as well as some derived facts based on payments
-
     columns:
       - name: order_id
+        description: Primary key
         tests:
           - unique
           - not_null
-        description: This is a unique identifier for an order
-
       - name: customer_id
         description: Foreign key to the customers table
         tests:
           - not_null
           - relationships:
-              to: ref('customers')
+              to: ref('stg_customers')
               field: customer_id
-
       - name: order_date
         description: Date (UTC) that the order was placed
-
       - name: status
         description: '{{ doc("orders_status") }}'
         tests:
           - accepted_values:
               values: ['placed', 'shipped', 'completed', 'return_pending', 'returned']
-
       - name: amount
         description: Total amount (AUD) of the order
         tests:
           - not_null
-
       - name: credit_card_amount
         description: Amount of the order (AUD) paid for by credit card
         tests:
           - not_null
-
       - name: coupon_amount
         description: Amount of the order (AUD) paid for by coupon
         tests:
           - not_null
-
       - name: bank_transfer_amount
         description: Amount of the order (AUD) paid for by bank transfer
         tests:
           - not_null
-
       - name: gift_card_amount
         description: Amount of the order (AUD) paid for by gift card
         tests:

jaffle_shop/models/customers.sql → jaffle_shop/models/warehouse/wh_customers.sql

@@ -49,8 +49,8 @@ final as (
 
     select
         customers.customer_id,
-        customers.first_name,
-        customers.last_name,
+        customers.first_name_hash,
+        customers.last_name_hash,
         customer_orders.first_order,
         customer_orders.most_recent_order,
         customer_orders.number_of_orders,

jaffle_shop/models/orders.sql → jaffle_shop/models/warehouse/wh_orders.sql

@@ -35,7 +35,7 @@ final as (
         orders.order_id,
         orders.customer_id,
         orders.order_date,
-        orders.status,
+        orders.order_status,
 
         {% for payment_method in payment_methods -%}
 

jaffle_shop/seeds/_seeds.yml

@@ -0,0 +1,29 @@
+version: 2
+
+seeds:
+  - name: raw_customers
+    description: >
+      Raw customer data
+    config:
+      column_types:
+        id: int
+        first_name: string
+        last_name: string
+  - name: raw_orders
+    description: >
+      Raw orders data
+    config:
+      column_types:
+        id: int
+        user_id: int
+        order_date: string
+        status: string
+  - name: raw_payments
+    description: >
+      Raw payments data
+    config:
+      column_types:
+        id: int
+        user_id: int
+        payment_method: string
+        amount: int

jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv

@@ -0,0 +1,2 @@
+fct_name,column_name,id_to_exclude,comment
+fct_staging_dependent_on_staging,parent,stg_customers_pii,Scrubbing pii permitted in staging layer.