File size: 1,926 Bytes
1cd137c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
contract:
  name: phase1_validation
  version: 1.0.0
  description: Validate ingested retailer product data before downstream processing

input:
  dataset: raw_products
  required_columns:
    - external_id
    - source_retailer
    - title

output:
  dataset: validated_products
  grain: one record per retailer product
  columns:
    - external_id
    - source_retailer
    - title
    - brand
    - barcode
    - description
    - ingredients_text
    - nutrition_text
    - image_url
    - size_amount
    - size_unit
    - price
    - currency
    - scraped_at

keys:
  primary_key:
    - source_retailer
    - external_id

constraints:
  required:
    - external_id
    - source_retailer
    - title

  nullable:
    - brand
    - barcode
    - description
    - ingredients_text
    - nutrition_text
    - image_url
    - size_amount
    - size_unit
    - price
    - currency
    - scraped_at

  uniqueness:
    - source_retailer
    - external_id

quality:
  critical:
    - missing_external_id
    - missing_source_retailer
    - missing_title
    - duplicate_external_id_within_retailer
    - invalid_required_data_type

  warnings:
    - missing_brand
    - missing_barcode
    - missing_size
    - missing_optional_column
    - new_optional_column

failure_handling:
  statuses:
    PASS: continue
    WARNING: continue_and_report
    AMBIGUOUS: quarantine_for_review
    FAIL: stop

processing:
  does:
    - validate_required_columns
    - validate_data_types
    - detect_duplicates
    - detect_missing_required_values
    - detect_schema_changes
    - clean_useless_columns
    - preserve_valid_records
    - produce_quality_report

  does_not:
    - classify_products
    - assign_taxonomy
    - create_product_groups
    - create_variants
    - calculate_scores
    - invent_missing_values

versioning:
  schema_version: 1.0.0
  rules_version: 1.0.0

lineage:
  source: ingestion
  pipeline_version: 1.0.0