From 3179c3d6aedf203f9f750a9fc194f35ac06a7896 Mon Sep 17 00:00:00 2001 From: Patrick Burns Date: Mon, 13 Jul 2026 10:05:56 -0500 Subject: [PATCH 1/2] fix(lotw): apply confirmations one qsl_lotw_date assignment at a time The dynamic UPDATE seeded qsl_lotw_date = NOW()::date and then appended a second qsl_lotw_date assignment whenever LoTW returned qsl_rcvd_date (which it nearly always does with qso_qsl=yes). Postgres rejects two assignments to the same column in one UPDATE (42601), so the first matched confirmation threw, the whole download was logged failed, and confirmations were never applied. - Assign qsl_lotw_date exactly once: LoTW's qsl_rcvd_date when present, otherwise today. - Isolate per-row apply failures (try/catch + counter) so one bad row can no longer discard an entire batch; surface a failure summary in lotw_download_logs.error_message and the response. - Advance the incremental cursor only from successfully applied confirmations, and use stations.lotw_last_qsl_rcvd_date as qso_qslsince for non-manual downloads (the column was written but never read back). Co-Authored-By: Claude Fable 5 --- src/app/api/lotw/download/route.ts | 84 +++++++++++++++++++++--------- 1 file changed, 59 insertions(+), 25 deletions(-) diff --git a/src/app/api/lotw/download/route.ts b/src/app/api/lotw/download/route.ts index a0d75e7..8500282 100644 --- a/src/app/api/lotw/download/route.ts +++ b/src/app/api/lotw/download/route.ts @@ -37,7 +37,7 @@ export async function POST(request: NextRequest) { // For cron jobs, just verify station exists stationResult = await query( `SELECT s.id, s.callsign, s.lotw_username, s.lotw_password, s.user_id, - u.third_party_services + s.lotw_last_qsl_rcvd_date, u.third_party_services FROM stations s JOIN users u ON s.user_id = u.id WHERE s.id = $1`, @@ -50,7 +50,7 @@ export async function POST(request: NextRequest) { } stationResult = await query( `SELECT s.id, s.callsign, s.lotw_username, s.lotw_password, s.user_id, - u.third_party_services + s.lotw_last_qsl_rcvd_date, u.third_party_services FROM stations s JOIN users u ON s.user_id = u.id WHERE s.id = $1 AND s.user_id = $2`, @@ -125,9 +125,19 @@ export async function POST(request: NextRequest) { ['processing', downloadLogId] ); + // Incremental fetch for scheduled runs: resume from the newest QSL date + // already applied (qso_qslsince filters by QSL date, not QSO date, so + // the local contacts query below must stay unfiltered by this date — + // confirmations can arrive for arbitrarily old QSOs). + let urlDateFrom = date_from; + if (!urlDateFrom && download_method !== 'manual' && station.lotw_last_qsl_rcvd_date) { + const last = station.lotw_last_qsl_rcvd_date; + urlDateFrom = last instanceof Date ? last.toISOString().split('T')[0] : String(last); + } + // Build LoTW download URL const downloadUrl = buildLoTWDownloadUrl(lotwUsername, lotwPassword, { - dateFrom: date_from, + dateFrom: urlDateFrom, dateTo: date_to, ownCallsign: station.callsign, }); @@ -229,6 +239,9 @@ export async function POST(request: NextRequest) { let matchedCount = 0; let unmatchedCount = confirmations.length; + let failedCount = 0; + let firstApplyError: string | null = null; + const appliedQslDates: string[] = []; // Apply each confirmation: set LoTW QSL flags, enrich location fields // (state/county/CQZ/ITUZ/DXCC/country/grid/iota), and cross-flag QRZ @@ -239,7 +252,6 @@ export async function POST(request: NextRequest) { // Build a dynamic UPDATE — only set fields LoTW returned non-empty. const sets: string[] = [ 'qsl_lotw = true', - `qsl_lotw_date = NOW()::date`, `lotw_qsl_rcvd = 'Y'`, 'lotw_match_status = $1', 'updated_at = NOW()', @@ -276,28 +288,43 @@ export async function POST(request: NextRequest) { if (match.contact.qrz_qsl_sent === 'Y') { sets.push(`qrz_qsl_sent = 'M'`); } - // qsl_rcvd_date from LoTW is YYYY-MM-DD already. + // qsl_lotw_date is assigned exactly once — Postgres rejects an UPDATE + // that sets the same column twice. Prefer LoTW's own QSL date + // (YYYY-MM-DD already); fall back to today. if (conf.qsl_rcvd_date) { params.push(conf.qsl_rcvd_date); sets.push(`qsl_lotw_date = $${params.length}::date`); + } else { + sets.push('qsl_lotw_date = NOW()::date'); } params.push(match.contact.id); - await query( - `UPDATE contacts SET ${sets.join(', ')} WHERE id = $${params.length}`, - params - ); - - matchedCount++; - unmatchedCount--; + try { + await query( + `UPDATE contacts SET ${sets.join(', ')} WHERE id = $${params.length}`, + params + ); + matchedCount++; + unmatchedCount--; + if (conf.qsl_rcvd_date) appliedQslDates.push(conf.qsl_rcvd_date); + } catch (applyError) { + // Isolate per-row failures so one bad row can't discard the batch. + failedCount++; + const msg = applyError instanceof Error ? applyError.message : 'Unknown error'; + if (!firstApplyError) firstApplyError = msg; + console.error( + `[LoTW Download] Failed to apply confirmation to contact ${match.contact.id}:`, + applyError + ); + } } // Persist the most recent qsl_rcvd_date so the next download can use it - // as qso_qslsince for an incremental fetch. - if (matches.length > 0) { - const maxDate = matches - .map(m => m.confirmation.qsl_rcvd_date) - .filter((d): d is string => !!d) + // as qso_qslsince for an incremental fetch. Only consider confirmations + // that actually applied — advancing past a failed row would skip it on + // the next incremental run. + if (appliedQslDates.length > 0) { + const maxDate = appliedQslDates .sort() .pop(); if (maxDate) { @@ -310,14 +337,20 @@ export async function POST(request: NextRequest) { } } - // Update download log as completed + // Update download log as completed; surface partial apply failures so + // they are visible in the sync history instead of silently dropped. + const applyErrorMessage = failedCount > 0 + ? `${failedCount} matched confirmation(s) failed to apply: ${firstApplyError}` + : null; + await query( - `UPDATE lotw_download_logs - SET status = 'completed', completed_at = NOW(), - qso_count = $1, confirmations_found = $2, - confirmations_matched = $3, confirmations_unmatched = $4 - WHERE id = $5`, - [contacts.length, confirmations.length, matchedCount, unmatchedCount, downloadLogId] + `UPDATE lotw_download_logs + SET status = 'completed', completed_at = NOW(), + qso_count = $1, confirmations_found = $2, + confirmations_matched = $3, confirmations_unmatched = $4, + error_message = $5 + WHERE id = $6`, + [contacts.length, confirmations.length, matchedCount, unmatchedCount, applyErrorMessage, downloadLogId] ); const response: LotwDownloadResponse = { @@ -325,7 +358,8 @@ export async function POST(request: NextRequest) { download_log_id: downloadLogId, confirmations_found: confirmations.length, confirmations_matched: matchedCount, - confirmations_unmatched: unmatchedCount + confirmations_unmatched: unmatchedCount, + ...(applyErrorMessage ? { error_message: applyErrorMessage } : {}) }; return NextResponse.json(response); From 98dff09230ba8a1b3ce8076c0aec7b16a16fb829 Mon Sep 17 00:00:00 2001 From: Patrick Burns Date: Mon, 13 Jul 2026 12:10:20 -0500 Subject: [PATCH 2/2] fix(lotw): advance qsl watermark for confirmations without a LoTW date When conf.qsl_rcvd_date was missing we wrote NOW()::date to the contact but never recorded it in appliedQslDates, so lotw_last_qsl_rcvd_date never advanced and incremental downloads refetched the same history. Compute the fallback date in JS and record it like any other applied date. Addresses Copilot review feedback on #218. Co-Authored-By: Claude Fable 5 --- src/app/api/lotw/download/route.ts | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/app/api/lotw/download/route.ts b/src/app/api/lotw/download/route.ts index 8500282..be50397 100644 --- a/src/app/api/lotw/download/route.ts +++ b/src/app/api/lotw/download/route.ts @@ -290,13 +290,11 @@ export async function POST(request: NextRequest) { } // qsl_lotw_date is assigned exactly once — Postgres rejects an UPDATE // that sets the same column twice. Prefer LoTW's own QSL date - // (YYYY-MM-DD already); fall back to today. - if (conf.qsl_rcvd_date) { - params.push(conf.qsl_rcvd_date); - sets.push(`qsl_lotw_date = $${params.length}::date`); - } else { - sets.push('qsl_lotw_date = NOW()::date'); - } + // (YYYY-MM-DD already); fall back to today (UTC) so the applied date + // still advances the incremental-download watermark below. + const qslDate = conf.qsl_rcvd_date || new Date().toISOString().slice(0, 10); + params.push(qslDate); + sets.push(`qsl_lotw_date = $${params.length}::date`); params.push(match.contact.id); try { @@ -306,7 +304,7 @@ export async function POST(request: NextRequest) { ); matchedCount++; unmatchedCount--; - if (conf.qsl_rcvd_date) appliedQslDates.push(conf.qsl_rcvd_date); + appliedQslDates.push(qslDate); } catch (applyError) { // Isolate per-row failures so one bad row can't discard the batch. failedCount++;