diff --git a/OLIDS/Testing/data-quality/test_column_completeness.sql b/OLIDS/Testing/data-quality/test_column_completeness.sql index 6e5138b..907f50f 100644 --- a/OLIDS/Testing/data-quality/test_column_completeness.sql +++ b/OLIDS/Testing/data-quality/test_column_completeness.sql @@ -39,15 +39,22 @@ WITH checks AS ( UNION ALL SELECT 'PATIENT', 'birth_year', 1.0, COUNT(*), SUM(CASE WHEN birth_year IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT UNION ALL SELECT 'PATIENT', 'birth_month', 1.0, COUNT(*), SUM(CASE WHEN birth_month IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT UNION ALL SELECT 'PATIENT', 'lds_start_datetime', 1.0, COUNT(*), SUM(CASE WHEN lds_start_datetime IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT + UNION ALL SELECT 'PATIENT', 'NHS_NUMBER_HASH', 1.0, COUNT(*), SUM(CASE WHEN NHS_NUMBER_HASH IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT + UNION ALL SELECT 'PATIENT', 'BIRTH_YEAR', 1.0, COUNT(*), SUM(CASE WHEN BIRTH_YEAR IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT + UNION ALL SELECT 'PATIENT', 'BIRTH_MONTH', 1.0, COUNT(*), SUM(CASE WHEN BIRTH_MONTH IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT + UNION ALL SELECT 'PATIENT', 'DEATH_YEAR', 1.0, COUNT(*), SUM(CASE WHEN DEATH_YEAR IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT + UNION ALL SELECT 'PATIENT', 'DEATH_MONTH', 1.0, COUNT(*), SUM(CASE WHEN DEATH_MONTH IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT -- PERSON (OLIDS_MASKED) UNION ALL SELECT 'PERSON', 'id', 0.0, COUNT(*), SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PERSON UNION ALL SELECT 'PERSON', 'lds_start_datetime', 1.0, COUNT(*), SUM(CASE WHEN lds_start_datetime IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PERSON + UNION ALL SELECT 'PERSON', 'MATCHED_NHS_NUMBER_HASH', 1.0, COUNT(*), SUM(CASE WHEN MATCHED_NHS_NUMBER_HASH IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PERSON -- PATIENT_ADDRESS (OLIDS_MASKED) UNION ALL SELECT 'PATIENT_ADDRESS', 'id', 0.0, COUNT(*), SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT_ADDRESS UNION ALL SELECT 'PATIENT_ADDRESS', 'address_type_source_concept_id', 5.0, COUNT(*), SUM(CASE WHEN address_type_source_concept_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT_ADDRESS UNION ALL SELECT 'PATIENT_ADDRESS', 'lds_start_datetime', 1.0, COUNT(*), SUM(CASE WHEN lds_start_datetime IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT_ADDRESS + UNION ALL SELECT 'PATIENT_ADDRESS', 'POSTCODE_HASH', 1.0, COUNT(*), SUM(CASE WHEN POSTCODE_HASH IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT_ADDRESS -- PATIENT_CONTACT (OLIDS_MASKED) UNION ALL SELECT 'PATIENT_CONTACT', 'id', 0.0, COUNT(*), SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT_CONTACT @@ -57,10 +64,16 @@ WITH checks AS ( -- PATIENT_UPRN (OLIDS_MASKED) UNION ALL SELECT 'PATIENT_UPRN', 'id', 0.0, COUNT(*), SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT_UPRN UNION ALL SELECT 'PATIENT_UPRN', 'lds_start_datetime', 1.0, COUNT(*), SUM(CASE WHEN lds_start_datetime IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT_UPRN + UNION ALL SELECT 'PATIENT_UPRN', 'MASKED_UPRN', 1.0, COUNT(*), SUM(CASE WHEN MASKED_UPRN IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT_UPRN + UNION ALL SELECT 'PATIENT_UPRN', 'MASKED_USRN', 1.0, COUNT(*), SUM(CASE WHEN MASKED_USRN IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT_UPRN + UNION ALL SELECT 'PATIENT_UPRN', 'MASKED_POSTCODE', 1.0, COUNT(*), SUM(CASE WHEN MASKED_POSTCODE IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT_UPRN -- PATIENT_PERSON (OLIDS_COMMON) — no ID column in this table UNION ALL SELECT 'PATIENT_PERSON', 'patient_id', 0.5, COUNT(*), SUM(CASE WHEN patient_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.PATIENT_PERSON UNION ALL SELECT 'PATIENT_PERSON', 'person_id', 0.5, COUNT(*), SUM(CASE WHEN person_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.PATIENT_PERSON + UNION ALL SELECT 'PATIENT_PERSON', 'WD_NHS_NUMBER_HASH', 1.0, COUNT(*), SUM(CASE WHEN WD_NHS_NUMBER_HASH IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.PATIENT_PERSON + UNION ALL SELECT 'PATIENT_PERSON', 'WD_MONTH_OF_BIRTH', 1.0, COUNT(*), SUM(CASE WHEN WD_MONTH_OF_BIRTH IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.PATIENT_PERSON + UNION ALL SELECT 'PATIENT_PERSON', 'WD_MONTH_OF_DEATH', 1.0, COUNT(*), SUM(CASE WHEN WD_MONTH_OF_DEATH IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.PATIENT_PERSON -- EPISODE_OF_CARE (OLIDS_COMMON) UNION ALL SELECT 'EPISODE_OF_CARE', 'id', 0.0, COUNT(*), SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.EPISODE_OF_CARE @@ -111,6 +124,9 @@ WITH checks AS ( UNION ALL SELECT 'DIAGNOSTIC_ORDER', 'result_measurement_units_source_concept_id', 5.0, COUNT(*), SUM(CASE WHEN result_measurement_units_source_concept_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.DIAGNOSTIC_ORDER UNION ALL SELECT 'DIAGNOSTIC_ORDER', 'clinical_effective_date_precision_source_concept_id', 5.0, COUNT(*), SUM(CASE WHEN clinical_effective_date_precision_source_concept_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.DIAGNOSTIC_ORDER UNION ALL SELECT 'DIAGNOSTIC_ORDER', 'episodicity_source_concept_id', 5.0, COUNT(*), SUM(CASE WHEN episodicity_source_concept_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.DIAGNOSTIC_ORDER + UNION ALL SELECT 'DIAGNOSTIC_ORDER', 'AGE_AT_EVENT', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.DIAGNOSTIC_ORDER + UNION ALL SELECT 'DIAGNOSTIC_ORDER', 'AGE_AT_EVENT_BABY', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT_BABY IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.DIAGNOSTIC_ORDER + UNION ALL SELECT 'DIAGNOSTIC_ORDER', 'AGE_AT_EVENT_NEONATE', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT_NEONATE IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.DIAGNOSTIC_ORDER -- ENCOUNTER (OLIDS_COMMON) UNION ALL SELECT 'ENCOUNTER', 'id', 0.0, COUNT(*), SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ENCOUNTER @@ -120,6 +136,9 @@ WITH checks AS ( UNION ALL SELECT 'ENCOUNTER', 'clinical_effective_date', 1.0, COUNT(*), SUM(CASE WHEN clinical_effective_date IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ENCOUNTER UNION ALL SELECT 'ENCOUNTER', 'encounter_source_concept_id', 5.0, COUNT(*), SUM(CASE WHEN encounter_source_concept_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ENCOUNTER UNION ALL SELECT 'ENCOUNTER', 'clinical_effective_date_precision_source_concept_id', 5.0, COUNT(*), SUM(CASE WHEN clinical_effective_date_precision_source_concept_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ENCOUNTER + UNION ALL SELECT 'ENCOUNTER', 'AGE_AT_EVENT', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ENCOUNTER + UNION ALL SELECT 'ENCOUNTER', 'AGE_AT_EVENT_BABY', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT_BABY IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ENCOUNTER + UNION ALL SELECT 'ENCOUNTER', 'AGE_AT_EVENT_NEONATE', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT_NEONATE IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ENCOUNTER -- ALLERGY_INTOLERANCE (OLIDS_COMMON) UNION ALL SELECT 'ALLERGY_INTOLERANCE', 'id', 0.0, COUNT(*), SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ALLERGY_INTOLERANCE @@ -129,6 +148,9 @@ WITH checks AS ( UNION ALL SELECT 'ALLERGY_INTOLERANCE', 'clinical_effective_date', 1.0, COUNT(*), SUM(CASE WHEN clinical_effective_date IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ALLERGY_INTOLERANCE UNION ALL SELECT 'ALLERGY_INTOLERANCE', 'allergy_intolerance_source_concept_id', 5.0, COUNT(*), SUM(CASE WHEN allergy_intolerance_source_concept_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ALLERGY_INTOLERANCE UNION ALL SELECT 'ALLERGY_INTOLERANCE', 'clinical_effective_date_precision_source_concept_id', 5.0, COUNT(*), SUM(CASE WHEN clinical_effective_date_precision_source_concept_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ALLERGY_INTOLERANCE + UNION ALL SELECT 'ALLERGY_INTOLERANCE', 'AGE_AT_EVENT', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ALLERGY_INTOLERANCE + UNION ALL SELECT 'ALLERGY_INTOLERANCE', 'AGE_AT_EVENT_BABY', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT_BABY IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ALLERGY_INTOLERANCE + UNION ALL SELECT 'ALLERGY_INTOLERANCE', 'AGE_AT_EVENT_NEONAT', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT_NEONAT IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ALLERGY_INTOLERANCE -- PROCEDURE_REQUEST (OLIDS_COMMON) UNION ALL SELECT 'PROCEDURE_REQUEST', 'id', 0.0, COUNT(*), SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.PROCEDURE_REQUEST @@ -147,6 +169,10 @@ WITH checks AS ( UNION ALL SELECT 'REFERRAL_REQUEST', 'referral_request_priority_source_concept_id', 5.0, COUNT(*), SUM(CASE WHEN referral_request_priority_source_concept_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.REFERRAL_REQUEST UNION ALL SELECT 'REFERRAL_REQUEST', 'referral_request_type_source_concept_id', 5.0, COUNT(*), SUM(CASE WHEN referral_request_type_source_concept_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.REFERRAL_REQUEST UNION ALL SELECT 'REFERRAL_REQUEST', 'referral_request_specialty_source_concept_id', 5.0, COUNT(*), SUM(CASE WHEN referral_request_specialty_source_concept_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.REFERRAL_REQUEST + UNION ALL SELECT 'REFERRAL_REQUEST', 'AGE_AT_EVENT', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.REFERRAL_REQUEST + UNION ALL SELECT 'REFERRAL_REQUEST', 'AGE_AT_EVENT_BABY', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT_BABY IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.REFERRAL_REQUEST + UNION ALL SELECT 'REFERRAL_REQUEST', 'AGE_AT_EVENT_NEONATE', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT_NEONATE IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.REFERRAL_REQUEST + -- LOCATION_CONTACT (OLIDS_COMMON) UNION ALL SELECT 'LOCATION_CONTACT', 'id', 0.0, COUNT(*), SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.LOCATION_CONTACT @@ -159,6 +185,9 @@ WITH checks AS ( UNION ALL SELECT 'APPOINTMENT', 'booking_method_source_concept_id', 5.0, COUNT(*), SUM(CASE WHEN booking_method_source_concept_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.APPOINTMENT UNION ALL SELECT 'APPOINTMENT', 'contact_mode_source_concept_id', 5.0, COUNT(*), SUM(CASE WHEN contact_mode_source_concept_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.APPOINTMENT UNION ALL SELECT 'APPOINTMENT', 'lds_start_datetime', 1.0, COUNT(*), SUM(CASE WHEN lds_start_datetime IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.APPOINTMENT + UNION ALL SELECT 'APPOINTMENT', 'AGE_AT_EVENT', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.APPOINTMENT + UNION ALL SELECT 'APPOINTMENT', 'AGE_AT_EVENT_BABY', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT_BABY IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.APPOINTMENT + UNION ALL SELECT 'APPOINTMENT', 'AGE_AT_EVENT_NEONATE', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT_NEONATE IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.APPOINTMENT -- APPOINTMENT_PRACTITIONER (OLIDS_COMMON) UNION ALL SELECT 'APPOINTMENT_PRACTITIONER', 'id', 0.0, COUNT(*), SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.APPOINTMENT_PRACTITIONER @@ -171,6 +200,7 @@ WITH checks AS ( -- LOCATION (OLIDS_COMMON) UNION ALL SELECT 'LOCATION', 'id', 0.0, COUNT(*), SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.LOCATION UNION ALL SELECT 'LOCATION', 'lds_start_datetime', 1.0, COUNT(*), SUM(CASE WHEN lds_start_datetime IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.LOCATION + UNION ALL SELECT 'LOCATION', 'VALUE', 1.0, COUNT(*), SUM(CASE WHEN VALUE IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.LOCATION -- ORGANISATION (OLIDS_COMMON) UNION ALL SELECT 'ORGANISATION', 'id', 0.0, COUNT(*), SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ORGANISATION diff --git a/OLIDS/Testing/data-quality/test_referential_integrity.sql b/OLIDS/Testing/data-quality/test_referential_integrity.sql index e60a025..3f69df3 100644 --- a/OLIDS/Testing/data-quality/test_referential_integrity.sql +++ b/OLIDS/Testing/data-quality/test_referential_integrity.sql @@ -798,6 +798,28 @@ WITH fk_checks AS ( SUM(CASE WHEN c.practitioner_id IS NOT NULL AND p.id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.SCHEDULE_PRACTITIONER c LEFT JOIN OLIDS_COMMON.PRACTITIONER p ON c.practitioner_id = p.id + + UNION ALL + + -- FLAG -> PATIENT + SELECT 'FLAG', 'patient_id', 'PATIENT', + COUNT(DISTINCT c.patient_id), + SUM(CASE WHEN c.patient_id IS NOT NULL THEN 1 ELSE 0 END), + COUNT(DISTINCT CASE WHEN c.patient_id IS NOT NULL AND p.id IS NULL THEN c.patient_id END), + SUM(CASE WHEN c.patient_id IS NOT NULL AND p.id IS NULL THEN 1 ELSE 0 END) + FROM OLIDS_COMMON.FLAG c + LEFT JOIN OLIDS_MASKED.PATIENT p ON c.patient_id = p.id + + UNION ALL + + -- FLAG -> PERSON + SELECT 'FLAG', 'person_id', 'PERSON', + COUNT(DISTINCT c.person_id), + SUM(CASE WHEN c.person_id IS NOT NULL THEN 1 ELSE 0 END), + COUNT(DISTINCT CASE WHEN c.person_id IS NOT NULL AND p.id IS NULL THEN c.person_id END), + SUM(CASE WHEN c.person_id IS NOT NULL AND p.id IS NULL THEN 1 ELSE 0 END) + FROM OLIDS_COMMON.FLAG c + LEFT JOIN OLIDS_MASKED.PERSON p ON c.person_id = p.id ) -- A relationship passes if no orphans OR >99% of distinct FK values resolve.