From 1ca20b168cd71830926c725d45083761f1c96a52 Mon Sep 17 00:00:00 2001 From: William Ridgeway Date: Tue, 14 Jul 2026 11:36:27 -0500 Subject: [PATCH 1/6] Initial commit --- .../provisional-ratio-curves.R | 29 +++++++++++++++---- .../provisional-ratio-curves.sql | 7 +++++ 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/provisional-ratio-curves/provisional-ratio-curves.R b/provisional-ratio-curves/provisional-ratio-curves.R index 9f72aa6..79f6f55 100644 --- a/provisional-ratio-curves/provisional-ratio-curves.R +++ b/provisional-ratio-curves/provisional-ratio-curves.R @@ -8,6 +8,7 @@ library(DBI) library(dplyr) library(ggplot2) library(ggspatial) +library(glue) library(noctua) library(openxlsx) library(prettymapr) @@ -19,7 +20,6 @@ library(sf) library(tidyr) noctua_options(unload = TRUE) - AWS_ATHENA_CONN_NOCTUA <- dbConnect(noctua::athena(), rstudio_conn_tab = FALSE) year <- format(Sys.Date(), "%Y") @@ -29,10 +29,10 @@ year <- format(Sys.Date(), "%Y") # that isn't accessible from the server they need to be copied locally or run # from a local machine. They MUST be named according to the current naming # scheme, and there must be PIN and Desk Review Value columns -data_path <- "O:/CCAODATA/recurring-data-requests/provisional-ratio-curves/" +data_path <- "O:/CCAODATA/recurring-data-requests/provisional-ratio-curves" input_path <- file.path(data_path, "input", year) output_path <- file.path(data_path, "output", year) -files_in <- list.files(input_path, full.names = TRUE) +files_in <- list.files(input_path, full.names = TRUE, pattern = "\\.xlsx$") # Flatfile ---- @@ -47,15 +47,34 @@ dr_vals <- map(files_in, \(x) { }) %>% bind_rows() +# Grab a list of all towns we're processing to use in the SQL query. This is +# based on the first two characters of the file name, which should be the +# township code. Input files *must* be named correctly. +dr_towns <- substr(basename(files_in), 1, 2) +if (!any(dr_towns %in% ccao::town_dict$township_code)) { + stop("One or more township codes in the input files are not valid.") +} + # This SQL query will return townships, neighborhoods, and model values for # every PIN, as well as any associated sales used for training model_vals <- dbGetQuery( - conn = AWS_ATHENA_CONN_NOCTUA, read_file("provisional-ratio-curves.sql") + conn = AWS_ATHENA_CONN_NOCTUA, + glue_sql( + read_file("provisional-ratio-curves.sql"), + .con = AWS_ATHENA_CONN_NOCTUA + ) ) # Attach dr and model values to calculate ratios all_ratios <- dr_vals %>% - left_join(model_vals) + # Valuations provides a bunch of parcels of classes that are not part of the + # modeling pipeline. We strip these parcels out of the analysis. + inner_join( + model_vals %>% + select(pin) + ) %>% + full_join(model_vals, by = "pin") %>% + mutate(sale_excluded = is.na(desk_review_value) & !is.na(sale_price)) walk(unique(all_ratios$township_name), \(x) { # Construct list for outputting multisheet .xlsx diff --git a/provisional-ratio-curves/provisional-ratio-curves.sql b/provisional-ratio-curves/provisional-ratio-curves.sql index ed75580..bc6618a 100644 --- a/provisional-ratio-curves/provisional-ratio-curves.sql +++ b/provisional-ratio-curves/provisional-ratio-curves.sql @@ -45,6 +45,13 @@ most_recent_pin AS ( CAST(YEAR(CURRENT_DATE) - 1 AS VARCHAR), CAST(YEAR(CURRENT_DATE) AS VARCHAR) ) + -- Make sure we only grab parcels valued by the res avm, for the towns + -- we're processing. + AND uni.class IN ( + '202', '203', '204', '205', '206', '207', '208', '209', + '210', '211', '212', '218', '219', '234', '278', '295' + ) + AND uni.township_code IN ({dr_towns*}) -- noqa ) SELECT From 452ddffa3eb86c4fc5d615a505cc2f2a98de212e Mon Sep 17 00:00:00 2001 From: William Ridgeway Date: Tue, 14 Jul 2026 11:39:10 -0500 Subject: [PATCH 2/6] Commenting --- provisional-ratio-curves/provisional-ratio-curves.R | 2 ++ 1 file changed, 2 insertions(+) diff --git a/provisional-ratio-curves/provisional-ratio-curves.R b/provisional-ratio-curves/provisional-ratio-curves.R index 79f6f55..deba8af 100644 --- a/provisional-ratio-curves/provisional-ratio-curves.R +++ b/provisional-ratio-curves/provisional-ratio-curves.R @@ -74,6 +74,8 @@ all_ratios <- dr_vals %>% select(pin) ) %>% full_join(model_vals, by = "pin") %>% + # Define an excluded a sale as one that has a sale price but no provided desk + # review value. mutate(sale_excluded = is.na(desk_review_value) & !is.na(sale_price)) walk(unique(all_ratios$township_name), \(x) { From 10bf5c838ea8b82ac69845438502ad8ea9730686 Mon Sep 17 00:00:00 2001 From: William Ridgeway Date: Tue, 14 Jul 2026 11:45:35 -0500 Subject: [PATCH 3/6] Use ccao.class_dict --- provisional-ratio-curves/provisional-ratio-curves.sql | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/provisional-ratio-curves/provisional-ratio-curves.sql b/provisional-ratio-curves/provisional-ratio-curves.sql index bc6618a..00ab4fe 100644 --- a/provisional-ratio-curves/provisional-ratio-curves.sql +++ b/provisional-ratio-curves/provisional-ratio-curves.sql @@ -41,16 +41,15 @@ most_recent_pin AS ( ORDER BY uni.year DESC ) AS rank FROM default.vw_pin_universe AS uni + -- Make sure we only grab parcels valued by the res avm, for the towns + -- we're processing. + INNER JOIN ccao.class_dict + ON uni.class = class_dict.class_code + AND class_dict.modeling_group IN ('SF', 'MF', 'BB') WHERE uni.year IN ( CAST(YEAR(CURRENT_DATE) - 1 AS VARCHAR), CAST(YEAR(CURRENT_DATE) AS VARCHAR) ) - -- Make sure we only grab parcels valued by the res avm, for the towns - -- we're processing. - AND uni.class IN ( - '202', '203', '204', '205', '206', '207', '208', '209', - '210', '211', '212', '218', '219', '234', '278', '295' - ) AND uni.township_code IN ({dr_towns*}) -- noqa ) From 20299ea7bc32783e589040b3dacbd85f555b7d91 Mon Sep 17 00:00:00 2001 From: William Ridgeway Date: Tue, 14 Jul 2026 12:07:47 -0500 Subject: [PATCH 4/6] Commenting --- provisional-ratio-curves/provisional-ratio-curves.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/provisional-ratio-curves/provisional-ratio-curves.sql b/provisional-ratio-curves/provisional-ratio-curves.sql index 00ab4fe..9476dea 100644 --- a/provisional-ratio-curves/provisional-ratio-curves.sql +++ b/provisional-ratio-curves/provisional-ratio-curves.sql @@ -41,8 +41,8 @@ most_recent_pin AS ( ORDER BY uni.year DESC ) AS rank FROM default.vw_pin_universe AS uni - -- Make sure we only grab parcels valued by the res avm, for the towns - -- we're processing. + -- Make sure we only grab parcels valued by the res avm, for the towns we're + -- processing. INNER JOIN ccao.class_dict ON uni.class = class_dict.class_code AND class_dict.modeling_group IN ('SF', 'MF', 'BB') From 45ce22b9c1e439f9e432a74bd6bb9a94e03ec1f5 Mon Sep 17 00:00:00 2001 From: William Ridgeway Date: Tue, 14 Jul 2026 12:09:41 -0500 Subject: [PATCH 5/6] Commenting --- provisional-ratio-curves/provisional-ratio-curves.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provisional-ratio-curves/provisional-ratio-curves.R b/provisional-ratio-curves/provisional-ratio-curves.R index deba8af..57985fd 100644 --- a/provisional-ratio-curves/provisional-ratio-curves.R +++ b/provisional-ratio-curves/provisional-ratio-curves.R @@ -74,7 +74,7 @@ all_ratios <- dr_vals %>% select(pin) ) %>% full_join(model_vals, by = "pin") %>% - # Define an excluded a sale as one that has a sale price but no provided desk + # Define an excluded sale as one that has a sale price but no provided desk # review value. mutate(sale_excluded = is.na(desk_review_value) & !is.na(sale_price)) From b2581974653a433aa9ac1011334ba2fa6c53c3b0 Mon Sep 17 00:00:00 2001 From: William Ridgeway Date: Wed, 15 Jul 2026 11:01:18 -0500 Subject: [PATCH 6/6] Correct logic check --- provisional-ratio-curves/provisional-ratio-curves.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provisional-ratio-curves/provisional-ratio-curves.R b/provisional-ratio-curves/provisional-ratio-curves.R index 57985fd..61101b3 100644 --- a/provisional-ratio-curves/provisional-ratio-curves.R +++ b/provisional-ratio-curves/provisional-ratio-curves.R @@ -51,7 +51,7 @@ dr_vals <- map(files_in, \(x) { # based on the first two characters of the file name, which should be the # township code. Input files *must* be named correctly. dr_towns <- substr(basename(files_in), 1, 2) -if (!any(dr_towns %in% ccao::town_dict$township_code)) { +if (!all(dr_towns %in% ccao::town_dict$township_code)) { stop("One or more township codes in the input files are not valid.") }