Fix dataset, price and slaProperties in JSON schema to allow correct types, update examples to adhere to JSON schema specification

Author: pflooky

CHANGELOG.md

@@ -1,6 +1,16 @@
 This document tracks the history and evolution of the **Open Data Contract Standard**.
 
-# v2.2.1 - 2023-10-03 - OPEN
+# v2.2.2 - 2024-01-05 - OPEN
+
+* Change `dataset.description` data type from `array` to `string`
+* Change `dataset.column.isPrimaryKey` data type from `string` to `boolean`
+* Change `price.priceAmount` data type from `string` to `number`
+* Change `slaProperties.value` data type from `string` to `oneOf[string, number]`
+* Change `slaProperties.valueExt` data type from `string` to `oneOf[string, number]`
+* Update [examples](examples) to adhere to JSON schema
+* Full example from README directs to [full-example.yaml](examples/all/full-example.yaml)
+
+# v2.2.1 - 2023-10-03 - APPROVED
 
 * Reformat Markdown tables
 * Reformat quality examples to be valid YAML

docs/README.md

@@ -136,9 +136,8 @@ dataset:
         clusterStatus: false
         clusterKeyPosition: -1
         criticalDataElementStatus: false
-        tags: null
-        classification: null
-        encryptedColumnName: null
+        tags: []
+        classification: public
         transformSourceTables:
           - table_name_1
           - table_name_2
@@ -161,9 +160,9 @@ dataset:
         clusterStatus: true
         clusterKeyPosition: 1
         criticalDataElementStatus: false
-        tags: null
-        classification: null
-        encryptedColumnName: null
+        tags: []
+        classification: restricted
+        encryptedColumnName: enc_rcvr_id
       - column: rcvr_cntry_code
         isPrimary: false # NEW in v2.1.0, Optional, default value is false, indicates whether the column is primary key in the table.
         primaryKeyPosition: -1
@@ -177,8 +176,8 @@ dataset:
         clusterStatus: false
         clusterKeyPosition: -1
         criticalDataElementStatus: false
-        tags: null
-        classification: null
+        tags: []
+        classification: public
         authoritativeDefinitions:
           - url: https://collibra.com/asset/742b358f-71a5-4ab1-bda4-dcdba9418c25
             type: businessDefinition
@@ -197,7 +196,7 @@ dataset:
 | dataset.table                                          |             | Yes      | Name of the table being cataloged; the value should only contain the table name. Do not include the project or dataset name in the value.                                                                                                             |
 | dataset.table.physicalName                             |             | No       | Physical name of the table, default value is table name + version separated by underscores, as `table_1_2_0`.                                                                                                                                         |
 | dataset.table.priorTableName                           |             | No       | Name of the previous version of the dataset, if applicable.                                                                                                                                                                                           |
-| dataset.table.description                              |             | No       | List of links to sources that provide more detail on column logic or values; examples would be URL to a GitHub repo, Collibra, on another tool.                                                                                                       |
+| dataset.table.description                              |             | No       | Description of the dataset.                                                                                                                                                                                                                           |
 | dataset.table.authoritativeDefinitions                 |             | No       | List of links to sources that provide more details on the table; examples would be a link to an external definition, a training video, a GitHub repo, Collibra, or another tool. Authoritative definitions follow the same structure in the standard. |
 | dataset.table.dataGranularity                          |             | No       | Granular level of the data in the table. Example would be `pmt_txn_id`.                                                                                                                                                                               |
 | dataset.table.columns                                  |             | Yes      | Array. A list of columns in the table.                                                                                                                                                                                                                |
@@ -375,14 +374,10 @@ stakeholders:
   - username: mhopper
     role: Data Scientist
     dateIn: 2022-10-01
-    dateOut: null
-    replacedByUsername: null
   - username: daustin
     role: Owner
     comment: Keeper of the grail
     dateIn: 2022-10-01
-    dateOut: null
-    replacedByUsername: null
 ```
 
 ### Definitions
@@ -518,243 +513,4 @@ contractCreatedTs: 2022-11-15 02:59:43
 
 ## Full example
 
-```yaml
-# What's this data contract about?
-datasetDomain: seller # Domain
-quantumName: my quantum # Data product name
-userConsumptionMode: Analytical
-version: 1.1.0 # Version (follows semantic versioning)
-status: current
-uuid: 53581432-6c55-4ba2-a65f-72344a91553a
-
-# Lots of information
-description:
-  purpose: Views built on top of the seller tables.
-  limitations: null
-  usage: null
-tenant: ClimateQuantumInc
-
-# Getting support
-productDl: product-dl@ClimateQuantum.org
-productSlackChannel: '#product-help'
-productFeedbackUrl: null
-
-# Physical parts / GCP / BigQuery specific
-sourcePlatform: googleCloudPlatform
-sourceSystem: bigQuery
-datasetProject: edw # BQ dataset
-datasetName: access_views # BQ dataset
-
-kind: DataContract
-apiVersion: 2.3.0 # Standard version (follows semantic versioning, previously known as templateVersion)
-
-type: tables
-
-# Physical access
-driver: null
-driverVersion: null
-server: null
-database: pypl-edw.pp_access_views
-username: '${env.username}'
-password: '${env.password}'
-schedulerAppName: name_coming_from_scheduler # NEW 2.1.0 Required if you want to schedule stuff, comes from DataALM.
-
-# Dataset, schema and quality
-dataset:
-  - table: tbl
-    physicalName: tbl_1 # NEW in v2.1.0, Optional, default value is table name + version separated by underscores, as table_1_2_0
-    priorTableName: null # if needed
-    description: Provides core payment metrics
-    authoritativeDefinitions: # NEW in v2.2.0, inspired by the column-level authoritative links
-      - url: https://catalog.data.gov/dataset/air-quality
-        type: businessDefinition
-      - url: https://youtu.be/jbY1BKFj9ec
-        type: videoTutorial
-    tags: null
-    dataGranularity: Aggregation on columns txn_ref_dt, pmt_txn_id
-    columns:
-      - column: txn_ref_dt
-        isPrimary: false # NEW in v2.1.0, Optional, default value is false, indicates whether the column is primary key in the table.
-        primaryKeyPosition: -1
-        businessName: transaction reference date
-        logicalType: date
-        physicalType: date
-        isNullable: false
-        description: null
-        partitionStatus: true
-        partitionKeyPosition: 1
-        clusterStatus: false
-        clusterKeyPosition: -1
-        criticalDataElementStatus: false
-        tags: null
-        classification: null
-        encryptedColumnName: null
-        transformSourceTables:
-          - table_name_1
-          - table_name_2
-          - table_name_3
-        transformLogic: sel t1.txn_dt as txn_ref_dt from table_name_1 as t1, table_name_2 as t2, table_name_3 as t3 where t1.txn_dt=date-3
-        transformDescription: defines the logic in business terms; logic for dummies
-        sampleValues:
-          - 2022-10-03
-          - 2020-01-28
-      - column: rcvr_id
-        isPrimary: true # NEW in v2.1.0, Optional, default value is false, indicates whether the column is primary key in the table.
-        primaryKeyPosition: 1
-        businessName: receiver id
-        logicalType: string
-        physicalType: varchar(18)
-        isNullable: false
-        description: A description for column rcvr_id.
-        partitionStatus: false
-        partitionKeyPosition: -1
-        clusterStatus: true
-        clusterKeyPosition: 1
-        criticalDataElementStatus: false
-        tags: null
-        classification: null
-        encryptedColumnName: null
-      - column: rcvr_cntry_code
-        isPrimary: false # NEW in v2.1.0, Optional, default value is false, indicates whether the column is primary key in the table.
-        primaryKeyPosition: -1
-        businessName: receiver country code
-        logicalType: string
-        physicalType: varchar(2)
-        isNullable: false
-        description: null
-        partitionStatus: false
-        partitionKeyPosition: -1
-        clusterStatus: false
-        clusterKeyPosition: -1
-        criticalDataElementStatus: false
-        tags: null
-        classification: null
-        authoritativeDefinitions:
-          - url: https://collibra.com/asset/742b358f-71a5-4ab1-bda4-dcdba9418c25
-            type: businessDefinition
-          - url: https://github.com/myorg/myrepo
-            type: transformationImplementation
-          - url: jdbc:postgresql://localhost:5432/adventureworks/tbl_1/rcvr_cntry_code
-            type: implementation
-        encryptedColumnName: rcvr_cntry_code_encrypted
-        quality:
-          - code: nullCheck
-            templateName: NullCheck
-            description: column should not contain null values
-            toolName: Elevate
-            toolRuleName: DQ.rw.tab1_2_0_0.rcvr_cntry_code.NullCheck
-            dimension: completeness # dropdown 7 values
-            type: dataQuality
-            severity: error
-            businessImpact: operational
-            scheduleCronExpression: 0 20 * * *
-            customProperties:
-              - property: FIELD_NAME
-                value:
-              - property: COMPARE_TO
-                value:
-              - property: COMPARISON_TYPE
-                value: Greater than
-    quality:
-      - code: countCheck # Required, name of the rule
-        templateName: CountCheck # NEW in v2.1.0 Required
-        description: Ensure row count is within expected volume range       # Optional
-        toolName: Elevate # Required
-        toolRuleName: DQ.rw.tab1.CountCheck # NEW in v2.1.0 Optional (Available only to the users who can change in source code edition)
-        dimension: completeness                                             # Optional
-        type: reconciliation                                                # Optional NEW in v2.1.0 default value for column level check - dataQuality and for table level reconciliation
-        severity: error                                                     # Optional NEW in v2.1.0, default value is error
-        businessImpact: operational                                         # Optional NEW in v2.1.0
-        scheduleCronExpression: 0 20 * * *                                  # Optional NEW in v2.1.0 default schedule - every day 10 a.m. UTC
-
-# Pricing
-price:
-  priceAmount: 9.95
-  priceCurrency: USD
-  priceUnit: megabyte
-
-# Stakeholders
-stakeholders:
-  - username: ceastwood
-    role: Data Scientist
-    dateIn: 2022-08-02
-    dateOut: 2022-10-01
-    replacedByUsername: mhopper
-  - username: mhopper
-    role: Data Scientist
-    dateIn: 2022-10-01
-    dateOut: null
-    replacedByUsername: null
-  - username: daustin
-    role: Owner
-    comment: Keeper of the grail
-    dateIn: 2022-10-01
-    dateOut: null
-    replacedByUsername: null
-
-# Roles
-roles:
-  - role: microstrategy_user_opr
-    access: read
-    firstLevelApprovers: Reporting Manager
-    secondLevelApprovers: 'mandolorian'
-  - role: bq_queryman_user_opr
-    access: read
-    firstLevelApprovers: Reporting Manager
-    secondLevelApprovers: na
-  - role: risk_data_access_opr
-    access: read
-    firstLevelApprovers: Reporting Manager
-    secondLevelApprovers: 'dathvador'
-  - role: bq_unica_user_opr
-    access: write
-    firstLevelApprovers: Reporting Manager
-    secondLevelApprovers: 'mickey'
-
-# SLA
-slaDefaultColumn: tab1.txn_ref_dt # Optional, default value is partitionColumn.
-slaProperties:
-  - property: latency # Property, see list of values in DP QoS
-    value: 4
-    unit: d # d, day, days for days; y, yr, years for years
-    column: tab1.txn_ref_dt # This would not be needed as it is the same table.column as the default one
-  - property: generalAvailability
-    value: 2022-05-12T09:30:10-08:00
-  - property: endOfSupport
-    value: 2032-05-12T09:30:10-08:00
-  - property: endOfLife
-    value: 2042-05-12T09:30:10-08:00
-  - property: retention
-    value: 3
-    unit: y
-    column: tab1.txn_ref_dt
-  - property: frequency
-    value: 1
-    valueExt: 1
-    unit: d
-    column: tab1.txn_ref_dt
-  - property: timeOfAvailability
-    value: 09:00-08:00
-    column: tab1.txn_ref_dt
-    driver: regulatory # Describes the importance of the SLA: [regulatory|analytics|operational|...]
-  - property: timeOfAvailability
-    value: 08:00-08:00
-    column: tab1.txn_ref_dt
-    driver: analytics
-
-# Tags
-tags:
-  - transactions
-
-# Custom properties
-customProperties:
-  - property: refRulesetName
-    value: gcsc.ruleset.name
-  - property: somePropertyName
-    value: property.value
-  - property: dataprocClusterName # Used for specific applications like Elevate
-    value: [cluster name]
-
-systemInstance: instance.ClimateQuantum.org
-contractCreatedTs: 2022-11-15 02:59:43
-```
+[Check full example here.](../examples/all/full-example.yaml)