Skip to content

Updated Referential integrity and completeness tests for PID #104

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
mpattem wants to merge 4 commits into
base: dbt-snowflake-compatibility
Choose a base branch
from
Draft

Updated Referential integrity and completeness tests for PID #104

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
924894e
Updated Referential integrity and completeness tests for PID
Jul 23, 2026
fdc42ae
reverted uv changes
Jul 23, 2026
a50e6cf
Reverted UV changes
Jul 23, 2026
e9de5f1
Update uv.lock
mpattem Jul 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions OLIDS/Testing/data-quality/test_column_completeness.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,15 +39,22 @@ WITH checks AS (
UNION ALL SELECT 'PATIENT', 'birth_year', 1.0, COUNT(*), SUM(CASE WHEN birth_year IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT
UNION ALL SELECT 'PATIENT', 'birth_month', 1.0, COUNT(*), SUM(CASE WHEN birth_month IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT
UNION ALL SELECT 'PATIENT', 'lds_start_datetime', 1.0, COUNT(*), SUM(CASE WHEN lds_start_datetime IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT
UNION ALL SELECT 'PATIENT', 'NHS_NUMBER_HASH', 1.0, COUNT(*), SUM(CASE WHEN NHS_NUMBER_HASH IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT
UNION ALL SELECT 'PATIENT', 'BIRTH_YEAR', 1.0, COUNT(*), SUM(CASE WHEN BIRTH_YEAR IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT
UNION ALL SELECT 'PATIENT', 'BIRTH_MONTH', 1.0, COUNT(*), SUM(CASE WHEN BIRTH_MONTH IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT
UNION ALL SELECT 'PATIENT', 'DEATH_YEAR', 1.0, COUNT(*), SUM(CASE WHEN DEATH_YEAR IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT
UNION ALL SELECT 'PATIENT', 'DEATH_MONTH', 1.0, COUNT(*), SUM(CASE WHEN DEATH_MONTH IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT

-- PERSON (OLIDS_MASKED)
UNION ALL SELECT 'PERSON', 'id', 0.0, COUNT(*), SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PERSON
UNION ALL SELECT 'PERSON', 'lds_start_datetime', 1.0, COUNT(*), SUM(CASE WHEN lds_start_datetime IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PERSON
UNION ALL SELECT 'PERSON', 'MATCHED_NHS_NUMBER_HASH', 1.0, COUNT(*), SUM(CASE WHEN MATCHED_NHS_NUMBER_HASH IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PERSON

-- PATIENT_ADDRESS (OLIDS_MASKED)
UNION ALL SELECT 'PATIENT_ADDRESS', 'id', 0.0, COUNT(*), SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT_ADDRESS
UNION ALL SELECT 'PATIENT_ADDRESS', 'address_type_source_concept_id', 5.0, COUNT(*), SUM(CASE WHEN address_type_source_concept_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT_ADDRESS
UNION ALL SELECT 'PATIENT_ADDRESS', 'lds_start_datetime', 1.0, COUNT(*), SUM(CASE WHEN lds_start_datetime IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT_ADDRESS
UNION ALL SELECT 'PATIENT_ADDRESS', 'POSTCODE_HASH', 1.0, COUNT(*), SUM(CASE WHEN POSTCODE_HASH IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT_ADDRESS

-- PATIENT_CONTACT (OLIDS_MASKED)
UNION ALL SELECT 'PATIENT_CONTACT', 'id', 0.0, COUNT(*), SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT_CONTACT
Expand All @@ -57,10 +64,16 @@ WITH checks AS (
-- PATIENT_UPRN (OLIDS_MASKED)
UNION ALL SELECT 'PATIENT_UPRN', 'id', 0.0, COUNT(*), SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT_UPRN
UNION ALL SELECT 'PATIENT_UPRN', 'lds_start_datetime', 1.0, COUNT(*), SUM(CASE WHEN lds_start_datetime IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT_UPRN
UNION ALL SELECT 'PATIENT_UPRN', 'MASKED_UPRN', 1.0, COUNT(*), SUM(CASE WHEN MASKED_UPRN IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT_UPRN
UNION ALL SELECT 'PATIENT_UPRN', 'MASKED_USRN', 1.0, COUNT(*), SUM(CASE WHEN MASKED_USRN IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT_UPRN
UNION ALL SELECT 'PATIENT_UPRN', 'MASKED_POSTCODE', 1.0, COUNT(*), SUM(CASE WHEN MASKED_POSTCODE IS NULL THEN 1 ELSE 0 END) FROM OLIDS_MASKED.PATIENT_UPRN

-- PATIENT_PERSON (OLIDS_COMMON) — no ID column in this table
UNION ALL SELECT 'PATIENT_PERSON', 'patient_id', 0.5, COUNT(*), SUM(CASE WHEN patient_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.PATIENT_PERSON
UNION ALL SELECT 'PATIENT_PERSON', 'person_id', 0.5, COUNT(*), SUM(CASE WHEN person_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.PATIENT_PERSON
UNION ALL SELECT 'PATIENT_PERSON', 'WD_NHS_NUMBER_HASH', 1.0, COUNT(*), SUM(CASE WHEN WD_NHS_NUMBER_HASH IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.PATIENT_PERSON
UNION ALL SELECT 'PATIENT_PERSON', 'WD_MONTH_OF_BIRTH', 1.0, COUNT(*), SUM(CASE WHEN WD_MONTH_OF_BIRTH IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.PATIENT_PERSON
UNION ALL SELECT 'PATIENT_PERSON', 'WD_MONTH_OF_DEATH', 1.0, COUNT(*), SUM(CASE WHEN WD_MONTH_OF_DEATH IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.PATIENT_PERSON

-- EPISODE_OF_CARE (OLIDS_COMMON)
UNION ALL SELECT 'EPISODE_OF_CARE', 'id', 0.0, COUNT(*), SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.EPISODE_OF_CARE
Expand Down Expand Up @@ -111,6 +124,9 @@ WITH checks AS (
UNION ALL SELECT 'DIAGNOSTIC_ORDER', 'result_measurement_units_source_concept_id', 5.0, COUNT(*), SUM(CASE WHEN result_measurement_units_source_concept_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.DIAGNOSTIC_ORDER
UNION ALL SELECT 'DIAGNOSTIC_ORDER', 'clinical_effective_date_precision_source_concept_id', 5.0, COUNT(*), SUM(CASE WHEN clinical_effective_date_precision_source_concept_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.DIAGNOSTIC_ORDER
UNION ALL SELECT 'DIAGNOSTIC_ORDER', 'episodicity_source_concept_id', 5.0, COUNT(*), SUM(CASE WHEN episodicity_source_concept_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.DIAGNOSTIC_ORDER
UNION ALL SELECT 'DIAGNOSTIC_ORDER', 'AGE_AT_EVENT', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.DIAGNOSTIC_ORDER
UNION ALL SELECT 'DIAGNOSTIC_ORDER', 'AGE_AT_EVENT_BABY', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT_BABY IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.DIAGNOSTIC_ORDER
UNION ALL SELECT 'DIAGNOSTIC_ORDER', 'AGE_AT_EVENT_NEONATE', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT_NEONATE IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.DIAGNOSTIC_ORDER

-- ENCOUNTER (OLIDS_COMMON)
UNION ALL SELECT 'ENCOUNTER', 'id', 0.0, COUNT(*), SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ENCOUNTER
Expand All @@ -120,6 +136,9 @@ WITH checks AS (
UNION ALL SELECT 'ENCOUNTER', 'clinical_effective_date', 1.0, COUNT(*), SUM(CASE WHEN clinical_effective_date IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ENCOUNTER
UNION ALL SELECT 'ENCOUNTER', 'encounter_source_concept_id', 5.0, COUNT(*), SUM(CASE WHEN encounter_source_concept_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ENCOUNTER
UNION ALL SELECT 'ENCOUNTER', 'clinical_effective_date_precision_source_concept_id', 5.0, COUNT(*), SUM(CASE WHEN clinical_effective_date_precision_source_concept_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ENCOUNTER
UNION ALL SELECT 'ENCOUNTER', 'AGE_AT_EVENT', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ENCOUNTER
UNION ALL SELECT 'ENCOUNTER', 'AGE_AT_EVENT_BABY', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT_BABY IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ENCOUNTER
UNION ALL SELECT 'ENCOUNTER', 'AGE_AT_EVENT_NEONATE', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT_NEONATE IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ENCOUNTER

-- ALLERGY_INTOLERANCE (OLIDS_COMMON)
UNION ALL SELECT 'ALLERGY_INTOLERANCE', 'id', 0.0, COUNT(*), SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ALLERGY_INTOLERANCE
Expand All @@ -129,6 +148,9 @@ WITH checks AS (
UNION ALL SELECT 'ALLERGY_INTOLERANCE', 'clinical_effective_date', 1.0, COUNT(*), SUM(CASE WHEN clinical_effective_date IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ALLERGY_INTOLERANCE
UNION ALL SELECT 'ALLERGY_INTOLERANCE', 'allergy_intolerance_source_concept_id', 5.0, COUNT(*), SUM(CASE WHEN allergy_intolerance_source_concept_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ALLERGY_INTOLERANCE
UNION ALL SELECT 'ALLERGY_INTOLERANCE', 'clinical_effective_date_precision_source_concept_id', 5.0, COUNT(*), SUM(CASE WHEN clinical_effective_date_precision_source_concept_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ALLERGY_INTOLERANCE
UNION ALL SELECT 'ALLERGY_INTOLERANCE', 'AGE_AT_EVENT', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ALLERGY_INTOLERANCE
UNION ALL SELECT 'ALLERGY_INTOLERANCE', 'AGE_AT_EVENT_BABY', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT_BABY IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ALLERGY_INTOLERANCE
UNION ALL SELECT 'ALLERGY_INTOLERANCE', 'AGE_AT_EVENT_NEONAT', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT_NEONAT IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ALLERGY_INTOLERANCE

-- PROCEDURE_REQUEST (OLIDS_COMMON)
UNION ALL SELECT 'PROCEDURE_REQUEST', 'id', 0.0, COUNT(*), SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.PROCEDURE_REQUEST
Expand All @@ -147,6 +169,10 @@ WITH checks AS (
UNION ALL SELECT 'REFERRAL_REQUEST', 'referral_request_priority_source_concept_id', 5.0, COUNT(*), SUM(CASE WHEN referral_request_priority_source_concept_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.REFERRAL_REQUEST
UNION ALL SELECT 'REFERRAL_REQUEST', 'referral_request_type_source_concept_id', 5.0, COUNT(*), SUM(CASE WHEN referral_request_type_source_concept_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.REFERRAL_REQUEST
UNION ALL SELECT 'REFERRAL_REQUEST', 'referral_request_specialty_source_concept_id', 5.0, COUNT(*), SUM(CASE WHEN referral_request_specialty_source_concept_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.REFERRAL_REQUEST
UNION ALL SELECT 'REFERRAL_REQUEST', 'AGE_AT_EVENT', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.REFERRAL_REQUEST
UNION ALL SELECT 'REFERRAL_REQUEST', 'AGE_AT_EVENT_BABY', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT_BABY IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.REFERRAL_REQUEST
UNION ALL SELECT 'REFERRAL_REQUEST', 'AGE_AT_EVENT_NEONATE', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT_NEONATE IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.REFERRAL_REQUEST


-- LOCATION_CONTACT (OLIDS_COMMON)
UNION ALL SELECT 'LOCATION_CONTACT', 'id', 0.0, COUNT(*), SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.LOCATION_CONTACT
Expand All @@ -159,6 +185,9 @@ WITH checks AS (
UNION ALL SELECT 'APPOINTMENT', 'booking_method_source_concept_id', 5.0, COUNT(*), SUM(CASE WHEN booking_method_source_concept_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.APPOINTMENT
UNION ALL SELECT 'APPOINTMENT', 'contact_mode_source_concept_id', 5.0, COUNT(*), SUM(CASE WHEN contact_mode_source_concept_id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.APPOINTMENT
UNION ALL SELECT 'APPOINTMENT', 'lds_start_datetime', 1.0, COUNT(*), SUM(CASE WHEN lds_start_datetime IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.APPOINTMENT
UNION ALL SELECT 'APPOINTMENT', 'AGE_AT_EVENT', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.APPOINTMENT
UNION ALL SELECT 'APPOINTMENT', 'AGE_AT_EVENT_BABY', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT_BABY IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.APPOINTMENT
UNION ALL SELECT 'APPOINTMENT', 'AGE_AT_EVENT_NEONATE', 1.0, COUNT(*), SUM(CASE WHEN AGE_AT_EVENT_NEONATE IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.APPOINTMENT

-- APPOINTMENT_PRACTITIONER (OLIDS_COMMON)
UNION ALL SELECT 'APPOINTMENT_PRACTITIONER', 'id', 0.0, COUNT(*), SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.APPOINTMENT_PRACTITIONER
Expand All @@ -171,6 +200,7 @@ WITH checks AS (
-- LOCATION (OLIDS_COMMON)
UNION ALL SELECT 'LOCATION', 'id', 0.0, COUNT(*), SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.LOCATION
UNION ALL SELECT 'LOCATION', 'lds_start_datetime', 1.0, COUNT(*), SUM(CASE WHEN lds_start_datetime IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.LOCATION
UNION ALL SELECT 'LOCATION', 'VALUE', 1.0, COUNT(*), SUM(CASE WHEN VALUE IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.LOCATION

-- ORGANISATION (OLIDS_COMMON)
UNION ALL SELECT 'ORGANISATION', 'id', 0.0, COUNT(*), SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END) FROM OLIDS_COMMON.ORGANISATION
Expand Down
22 changes: 22 additions & 0 deletions OLIDS/Testing/data-quality/test_referential_integrity.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -798,6 +798,28 @@ WITH fk_checks AS (
SUM(CASE WHEN c.practitioner_id IS NOT NULL AND p.id IS NULL THEN 1 ELSE 0 END)
FROM OLIDS_COMMON.SCHEDULE_PRACTITIONER c
LEFT JOIN OLIDS_COMMON.PRACTITIONER p ON c.practitioner_id = p.id

UNION ALL

-- FLAG -> PATIENT
SELECT 'FLAG', 'patient_id', 'PATIENT',
COUNT(DISTINCT c.patient_id),
SUM(CASE WHEN c.patient_id IS NOT NULL THEN 1 ELSE 0 END),
COUNT(DISTINCT CASE WHEN c.patient_id IS NOT NULL AND p.id IS NULL THEN c.patient_id END),
SUM(CASE WHEN c.patient_id IS NOT NULL AND p.id IS NULL THEN 1 ELSE 0 END)
FROM OLIDS_COMMON.FLAG c
LEFT JOIN OLIDS_MASKED.PATIENT p ON c.patient_id = p.id

UNION ALL

-- FLAG -> PERSON
SELECT 'FLAG', 'person_id', 'PERSON',
COUNT(DISTINCT c.person_id),
SUM(CASE WHEN c.person_id IS NOT NULL THEN 1 ELSE 0 END),
COUNT(DISTINCT CASE WHEN c.person_id IS NOT NULL AND p.id IS NULL THEN c.person_id END),
SUM(CASE WHEN c.person_id IS NOT NULL AND p.id IS NULL THEN 1 ELSE 0 END)
FROM OLIDS_COMMON.FLAG c
LEFT JOIN OLIDS_MASKED.PERSON p ON c.person_id = p.id
)

-- A relationship passes if no orphans OR >99% of distinct FK values resolve.
Expand Down