diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5c7d64..30ce9ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.12", "3.13"] + python-version: ["3.12", "3.13", "3.14"] # TODO: skip spark on Windows #os: [ubuntu-latest, windows-latest] os: [ubuntu-latest] @@ -30,9 +30,9 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install ".[dev,spark]" - wget https://dlcdn.apache.org/spark/spark-4.0.1/spark-4.0.1-bin-hadoop3.tgz - tar -xzf spark-4.0.1-bin-hadoop3.tgz - export SPARK_HOME=$(pwd)/spark-4.0.1-bin-hadoop3 + wget https://dlcdn.apache.org/spark/spark-4.0.4/spark-4.0.4-bin-hadoop3.tgz + tar -xzf spark-4.0.4-bin-hadoop3.tgz + export SPARK_HOME=$(pwd)/spark-4.0.4-bin-hadoop3 export PATH=$SPARK_HOME/sbin:$PATH start-thriftserver.sh - name: Run pytest with coverage diff --git a/pyproject.toml b/pyproject.toml index 5cb61f6..39c6d51 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,10 +4,10 @@ build-backend = "hatchling.build" [project] name = "chronify" -version = "0.7.0" +version = "0.8.0" description = "Time series store and mapping libray" readme = "README.md" -requires-python = ">=3.11, <3.14" +requires-python = ">=3.11, <3.15" license = "BSD-3-Clause" keywords = [] authors = [ @@ -19,14 +19,15 @@ authors = [ classifiers = [ "Development Status :: 4 - Beta", "Programming Language :: Python", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ] dependencies = [ - "duckdb ~= 1.1.0", + "duckdb >= 1.5", "duckdb_engine", "loguru", "pandas >= 2.2, < 3", diff --git a/src/chronify/csv_io.py b/src/chronify/csv_io.py index 5b99b07..75ead8c 100644 --- a/src/chronify/csv_io.py +++ b/src/chronify/csv_io.py @@ -4,6 +4,7 @@ import duckdb from duckdb import DuckDBPyRelation +from chronify.duckdb.types import TIMESTAMP from chronify.models import CsvTableSchema, get_duckdb_type_from_sqlalchemy from chronify.time_configs import DatetimeRange @@ -24,7 +25,7 @@ def read_csv(path: Path | str, schema: CsvTableSchema, **kwargs: Any) -> DuckDBP expr = column if isinstance(time_config, DatetimeRange) and column == time_config.time_column: time_type = rel.types[i] - if time_type == duckdb.typing.TIMESTAMP and not time_config.start_time_is_tz_naive(): # type: ignore + if time_type == TIMESTAMP and not time_config.start_time_is_tz_naive(): expr = f"timezone('{time_config.start.tzinfo.key}', {column}) AS {column}" # type: ignore exprs.append(expr) diff --git a/src/chronify/duckdb/types.py b/src/chronify/duckdb/types.py new file mode 100644 index 0000000..ab296c0 --- /dev/null +++ b/src/chronify/duckdb/types.py @@ -0,0 +1,33 @@ +"""DuckDB type constants used across the codebase.""" + +import duckdb +from _duckdb._sqltypes import DuckDBPyType + +BIGINT = duckdb.sqltype("BIGINT") +BOOLEAN = duckdb.sqltype("BOOLEAN") +DOUBLE = duckdb.sqltype("DOUBLE") +FLOAT = duckdb.sqltype("FLOAT") +INTEGER = duckdb.sqltype("INTEGER") +TINYINT = duckdb.sqltype("TINYINT") +VARCHAR = duckdb.sqltype("VARCHAR") +TIMESTAMP = duckdb.sqltype("TIMESTAMP") +TIMESTAMP_TZ = duckdb.sqltype("TIMESTAMP WITH TIME ZONE") +TIMESTAMP_MS = duckdb.sqltype("TIMESTAMP_MS") +TIMESTAMP_NS = duckdb.sqltype("TIMESTAMP_NS") +TIMESTAMP_S = duckdb.sqltype("TIMESTAMP_S") + +__all__ = [ + "DuckDBPyType", + "BIGINT", + "BOOLEAN", + "DOUBLE", + "FLOAT", + "INTEGER", + "TINYINT", + "VARCHAR", + "TIMESTAMP", + "TIMESTAMP_TZ", + "TIMESTAMP_MS", + "TIMESTAMP_NS", + "TIMESTAMP_S", +] diff --git a/src/chronify/models.py b/src/chronify/models.py index b770200..46ef09b 100644 --- a/src/chronify/models.py +++ b/src/chronify/models.py @@ -1,14 +1,28 @@ import re from typing import Any, Optional -import duckdb.typing +import duckdb import pandas as pd -from duckdb.typing import DuckDBPyType from pydantic import Field, field_validator, model_validator from sqlalchemy import BigInteger, Boolean, DateTime, Double, Float, Integer, SmallInteger, String from typing_extensions import Annotated from chronify.base_models import ChronifyBaseModel +from chronify.duckdb.types import ( + BIGINT, + BOOLEAN, + DOUBLE, + FLOAT, + INTEGER, + TINYINT, + VARCHAR, + TIMESTAMP, + TIMESTAMP_MS, + TIMESTAMP_NS, + TIMESTAMP_S, + TIMESTAMP_TZ, + DuckDBPyType, +) from chronify.exceptions import InvalidParameter, InvalidValue from chronify.time_configs import TimeConfig @@ -156,28 +170,22 @@ def list_columns(self) -> list[str]: _DB_TYPES = {x for x in _COLUMN_TYPES.values()} _DUCKDB_TYPES_TO_SQLALCHEMY_TYPES = { - duckdb.typing.BIGINT.id: BigInteger, # type: ignore - duckdb.typing.BOOLEAN.id: Boolean, # type: ignore - duckdb.typing.DOUBLE.id: Double, # type: ignore - duckdb.typing.FLOAT.id: Float, # type: ignore - duckdb.typing.INTEGER.id: Integer, # type: ignore - duckdb.typing.TINYINT.id: SmallInteger, # type: ignore - duckdb.typing.VARCHAR.id: String, # type: ignore - # Note: timestamp requires special handling because of timezone in sqlalchemy. + BIGINT.id: BigInteger, + BOOLEAN.id: Boolean, + DOUBLE.id: Double, + FLOAT.id: Float, + INTEGER.id: Integer, + TINYINT.id: SmallInteger, + VARCHAR.id: String, } def get_sqlalchemy_type_from_duckdb(duckdb_type: DuckDBPyType) -> Any: """Return the sqlalchemy type for a duckdb type.""" match duckdb_type: - case duckdb.typing.TIMESTAMP_TZ: # type: ignore + case _ if duckdb_type == TIMESTAMP_TZ: sqlalchemy_type = DateTime(timezone=True) - case ( - duckdb.typing.TIMESTAMP # type: ignore - | duckdb.typing.TIMESTAMP_MS # type: ignore - | duckdb.typing.TIMESTAMP_NS # type: ignore - | duckdb.typing.TIMESTAMP_S # type: ignore - ): + case _ if duckdb_type in (TIMESTAMP, TIMESTAMP_MS, TIMESTAMP_NS, TIMESTAMP_S): sqlalchemy_type = DateTime(timezone=False) case _: cls = _DUCKDB_TYPES_TO_SQLALCHEMY_TYPES.get(duckdb_type.id) @@ -192,26 +200,22 @@ def get_sqlalchemy_type_from_duckdb(duckdb_type: DuckDBPyType) -> Any: def get_duckdb_type_from_sqlalchemy(sqlalchemy_type: Any) -> DuckDBPyType: """Return the duckdb type for a sqlalchemy type.""" if isinstance(sqlalchemy_type, DateTime): - duckdb_type = ( - duckdb.typing.TIMESTAMP_TZ # type: ignore - if sqlalchemy_type.timezone - else duckdb.typing.TIMESTAMP # type: ignore - ) + duckdb_type = TIMESTAMP_TZ if sqlalchemy_type.timezone else TIMESTAMP elif isinstance(sqlalchemy_type, BigInteger): - duckdb_type = duckdb.typing.BIGINT # type: ignore + duckdb_type = BIGINT elif isinstance(sqlalchemy_type, Boolean): - duckdb_type = duckdb.typing.BOOLEAN # type: ignore + duckdb_type = BOOLEAN elif isinstance(sqlalchemy_type, Double): - duckdb_type = duckdb.typing.DOUBLE # type: ignore + duckdb_type = DOUBLE elif isinstance(sqlalchemy_type, Integer): - duckdb_type = duckdb.typing.INTEGER # type: ignore + duckdb_type = INTEGER elif isinstance(sqlalchemy_type, String): - duckdb_type = duckdb.typing.VARCHAR # type: ignore + duckdb_type = VARCHAR else: msg = f"There is no duckdb mapping for {sqlalchemy_type=}" raise InvalidParameter(msg) - return duckdb_type # type: ignore + return duckdb_type def get_duckdb_types_from_pandas(df: pd.DataFrame) -> list[DuckDBPyType]: