From 035c6c50ab2f4edcb32c28898adc004d9e278ba4 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 13 Jul 2026 08:15:59 +0545 Subject: [PATCH 1/2] feat(clickhouse): add premium geo columns to audit schema Add 9 optional geo attributes (city, continentCode, subdivisions, isp, autonomousSystemNumber, autonomousSystemOrganization, connectionType, connectionUsageType, connectionOrganization) to the ClickHouse audit adapter so audit rows can carry premium geo data. Low-cardinality dimensions (continentCode, subdivisions, connectionType, connectionUsageType, autonomousSystemNumber) are added to LOW_CARDINALITY_COLUMNS so they map to LowCardinality(Nullable(String)); the high-cardinality ones stay plain Nullable(String). createTable() and getColumnDefinition() pick them up automatically from getAttributes(); no index/sort-key changes. Extend adapter tests to cover the new attributes and their LowCardinality vs String/Nullable column-type classification. --- src/Audit/Adapter/ClickHouse.php | 97 ++++++++++++++++++++++++++ tests/Audit/Adapter/ClickHouseTest.php | 89 ++++++++++++++++++++++- 2 files changed, 185 insertions(+), 1 deletion(-) diff --git a/src/Audit/Adapter/ClickHouse.php b/src/Audit/Adapter/ClickHouse.php index 0ff5880..93a4032 100644 --- a/src/Audit/Adapter/ClickHouse.php +++ b/src/Audit/Adapter/ClickHouse.php @@ -31,6 +31,12 @@ class ClickHouse extends SQL 'actorType', 'resourceType', 'country', + // premium geo + 'continentCode', + 'subdivisions', + 'connectionType', + 'connectionUsageType', + 'autonomousSystemNumber', ]; /** @@ -450,6 +456,97 @@ public function getAttributes(): array 'array' => false, 'filters' => [], ], + // premium geo + [ + '$id' => 'city', + 'type' => Database::VAR_STRING, + 'size' => Database::LENGTH_KEY, + 'required' => false, + 'default' => null, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'continentCode', + 'type' => Database::VAR_STRING, + 'size' => Database::LENGTH_KEY, + 'required' => false, + 'default' => null, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'subdivisions', + 'type' => Database::VAR_STRING, + 'size' => Database::LENGTH_KEY, + 'required' => false, + 'default' => null, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'isp', + 'type' => Database::VAR_STRING, + 'size' => Database::LENGTH_KEY, + 'required' => false, + 'default' => null, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'autonomousSystemNumber', + 'type' => Database::VAR_STRING, + 'size' => Database::LENGTH_KEY, + 'required' => false, + 'default' => null, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'autonomousSystemOrganization', + 'type' => Database::VAR_STRING, + 'size' => Database::LENGTH_KEY, + 'required' => false, + 'default' => null, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'connectionType', + 'type' => Database::VAR_STRING, + 'size' => Database::LENGTH_KEY, + 'required' => false, + 'default' => null, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'connectionUsageType', + 'type' => Database::VAR_STRING, + 'size' => Database::LENGTH_KEY, + 'required' => false, + 'default' => null, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], + [ + '$id' => 'connectionOrganization', + 'type' => Database::VAR_STRING, + 'size' => Database::LENGTH_KEY, + 'required' => false, + 'default' => null, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], [ '$id' => 'projectId', 'type' => Database::VAR_STRING, diff --git a/tests/Audit/Adapter/ClickHouseTest.php b/tests/Audit/Adapter/ClickHouseTest.php index 8074243..8dbfb8f 100644 --- a/tests/Audit/Adapter/ClickHouseTest.php +++ b/tests/Audit/Adapter/ClickHouseTest.php @@ -467,7 +467,17 @@ public function testClickHouseAdapterAttributes(): void 'projectInternalId', 'teamId', 'teamInternalId', - 'hostname' + 'hostname', + // premium geo + 'city', + 'continentCode', + 'subdivisions', + 'isp', + 'autonomousSystemNumber', + 'autonomousSystemOrganization', + 'connectionType', + 'connectionUsageType', + 'connectionOrganization', ]; foreach ($expectedAttributes as $expected) { @@ -475,6 +485,83 @@ public function testClickHouseAdapterAttributes(): void } } + /** + * Test that premium geo attributes are all optional String columns. + */ + public function testPremiumGeoAttributesAreOptionalStrings(): void + { + $adapter = new ClickHouse( + host: 'clickhouse', + username: 'default', + password: 'clickhouse' + ); + + $attributes = $adapter->getAttributes(); + $byId = []; + foreach ($attributes as $attribute) { + $byId[$attribute['$id']] = $attribute; + } + + $geoColumns = [ + 'city', + 'continentCode', + 'subdivisions', + 'isp', + 'autonomousSystemNumber', + 'autonomousSystemOrganization', + 'connectionType', + 'connectionUsageType', + 'connectionOrganization', + ]; + + foreach ($geoColumns as $column) { + $this->assertArrayHasKey($column, $byId, "Premium geo attribute '{$column}' not found"); + $this->assertEquals(\Utopia\Database\Database::VAR_STRING, $byId[$column]['type'], "'{$column}' should be a string"); + $this->assertFalse($byId[$column]['required'], "'{$column}' should be optional"); + $this->assertFalse($byId[$column]['array'], "'{$column}' should not be an array"); + } + } + + /** + * Test that premium geo columns get the correct ClickHouse type: + * low-cardinality dimensions use LowCardinality(Nullable(String)), while + * high-cardinality ones stay plain Nullable(String). + */ + public function testPremiumGeoColumnTypes(): void + { + $adapter = new ClickHouse( + host: 'clickhouse', + username: 'default', + password: 'clickhouse' + ); + + $method = new \ReflectionMethod($adapter, 'getColumnDefinition'); + $method->setAccessible(true); + + $lowCardinality = [ + 'continentCode', + 'subdivisions', + 'connectionType', + 'connectionUsageType', + 'autonomousSystemNumber', + ]; + foreach ($lowCardinality as $column) { + $definition = $method->invoke($adapter, $column); + $this->assertEquals("{$column} LowCardinality(Nullable(String))", $definition); + } + + $highCardinality = [ + 'city', + 'isp', + 'autonomousSystemOrganization', + 'connectionOrganization', + ]; + foreach ($highCardinality as $column) { + $definition = $method->invoke($adapter, $column); + $this->assertEquals("{$column} Nullable(String)", $definition); + } + } + /** * Test that ClickHouse adapter has all required indexes */ From 5cba96766977af5862ae531e2e3a0da131635146 Mon Sep 17 00:00:00 2001 From: Damodar Lohani Date: Mon, 13 Jul 2026 12:14:38 +0545 Subject: [PATCH 2/2] test(clickhouse): geo round-trip test; move ASN out of LowCardinality - autonomousSystemNumber: plain Nullable(String) instead of LowCardinality (~100k ASNs globally can exceed ClickHouse's LowCardinality sweet spot; the remaining low-card geo dims have tight cardinality). Addresses review. - Add testPremiumGeoRoundTrip: writes a log with all 9 geo fields and reads them back unchanged, proving the columns are created/written/selected. --- src/Audit/Adapter/ClickHouse.php | 4 +-- tests/Audit/Adapter/ClickHouseTest.php | 47 +++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/src/Audit/Adapter/ClickHouse.php b/src/Audit/Adapter/ClickHouse.php index 16238c8..539d0d4 100644 --- a/src/Audit/Adapter/ClickHouse.php +++ b/src/Audit/Adapter/ClickHouse.php @@ -32,12 +32,12 @@ class ClickHouse extends SQL 'resourceType', 'country', 'sdk', - // premium geo + // premium geo (autonomousSystemNumber is intentionally NOT low-cardinality: + // ~100k ASNs globally can exceed ClickHouse's LowCardinality sweet spot) 'continentCode', 'subdivisions', 'connectionType', 'connectionUsageType', - 'autonomousSystemNumber', ]; /** diff --git a/tests/Audit/Adapter/ClickHouseTest.php b/tests/Audit/Adapter/ClickHouseTest.php index 71186ff..a2202c8 100644 --- a/tests/Audit/Adapter/ClickHouseTest.php +++ b/tests/Audit/Adapter/ClickHouseTest.php @@ -614,16 +614,18 @@ public function testPremiumGeoColumnTypes(): void 'subdivisions', 'connectionType', 'connectionUsageType', - 'autonomousSystemNumber', ]; foreach ($lowCardinality as $column) { $definition = $method->invoke($adapter, $column); $this->assertEquals("{$column} LowCardinality(Nullable(String))", $definition); } + // autonomousSystemNumber is high-cardinality (~100k ASNs) so it is a + // plain Nullable(String), not LowCardinality. $highCardinality = [ 'city', 'isp', + 'autonomousSystemNumber', 'autonomousSystemOrganization', 'connectionOrganization', ]; @@ -633,6 +635,49 @@ public function testPremiumGeoColumnTypes(): void } } + /** + * Premium geo values must round-trip through a real write/read cycle: + * write a log carrying all 9 geo fields and read it back unchanged. This + * proves the columns are actually created, written and selected (not just + * present in the schema definition). + */ + public function testPremiumGeoRoundTrip(): void + { + $actorId = 'geo-actor-' . uniqid('', true); + $geo = [ + 'city' => 'Mountain View', + 'continentCode' => 'NA', + 'subdivisions' => 'California', + 'isp' => 'Google', + 'autonomousSystemNumber' => '15169', + 'autonomousSystemOrganization' => 'GOOGLE', + 'connectionType' => 'cable', + 'connectionUsageType' => 'residential', + 'connectionOrganization' => 'Google LLC', + ]; + + $batchEvents = [array_merge([ + 'actorId' => $actorId, + 'event' => 'geo.roundtrip', + 'resource' => 'document/geo-1', + 'userAgent' => 'RoundTrip/1.0', + 'ip' => '8.8.8.8', + 'data' => [], + 'time' => \Utopia\Database\DateTime::formatTz(\Utopia\Database\DateTime::now()) ?? '', + ], $geo)]; + + $batchEvents = $this->applyRequiredAttributesToBatch($batchEvents); + $this->assertTrue($this->audit->logBatch($batchEvents)); + + $logs = $this->audit->getLogsByUser($actorId); + $this->assertGreaterThan(0, count($logs), 'geo round-trip log was not persisted'); + + $log = $logs[0]; + foreach ($geo as $key => $expected) { + $this->assertSame($expected, $log->getAttribute($key), "premium geo '{$key}' did not round-trip"); + } + } + /** * Test that ClickHouse adapter has all required indexes */