Add Rowid data type and YQL key generators

ydb/apps/ydbd/ya.make

@@ -55,6 +55,7 @@ PEERDIR(
     ydb/library/yql/udfs/common/hybrid_search
     ydb/library/yql/udfs/common/knn
     ydb/library/yql/udfs/common/roaring
+    ydb/library/yql/udfs/common/rowid
     ydb/library/yql/udfs/statistics_internal
     yql/essentials/parser/pg_wrapper
     yql/essentials/sql/pg

ydb/core/engine/minikql/minikql_engine_host.cpp

@@ -1086,6 +1086,7 @@ NUdf::TUnboxedValue GetCellValue(const TCell& cell, NScheme::TTypeInfo type) {
         case NYql::NProto::TypeIds::Yson:
         case NYql::NProto::TypeIds::Json:
         case NYql::NProto::TypeIds::Uuid:
+        case NYql::NProto::TypeIds::Rowid:
         case NYql::NProto::TypeIds::JsonDocument:
         case NYql::NProto::TypeIds::DyNumber:
             return MakeString(NUdf::TStringRef(cell.Data(), cell.Size()));

ydb/core/engine/mkql_proto.cpp

@@ -7,6 +7,7 @@
 #include <yql/essentials/minikql/computation/mkql_computation_node_holders.h>
 #include <yql/essentials/public/decimal/yql_decimal.h>
 #include <yql/essentials/parser/pg_wrapper/interface/type_desc.h>
+#include <yql/essentials/types/rowid/rowid.h>
 
 #include <ydb/core/scheme_types/scheme_types_defs.h>
 
@@ -223,6 +224,15 @@ bool CellsFromTuple(const NKikimrMiniKQL::TType* tupleType,
             }
             break;
         }
+        case NScheme::NTypeIds::Rowid:
+        {
+            if (!v.HasBytes()) {
+                CHECK_OR_RETURN_ERROR(false, Sprintf("Cannot parse value of type Rowid in tuple at position %" PRIu32, i));
+            }
+            CHECK_OR_RETURN_ERROR(v.GetBytes().size() == NRowid::ROWID_LEN, Sprintf("Invalid Rowid size in tuple at position %" PRIu32, i));
+            c = TCell(v.GetBytes().data(), v.GetBytes().size());
+            break;
+        }
         case NScheme::NTypeIds::Decimal:
         {
             if (v.HasLow128() && v.HasHi128()) {
@@ -366,6 +376,11 @@ bool CellToValue(NScheme::TTypeInfo type, const TCell& c, NKikimrMiniKQL::TValue
         break;
     }
 
+    case NScheme::NTypeIds::Rowid:
+        Y_ABORT_UNLESS(c.Size() == NRowid::ROWID_LEN);
+        val.MutableOptional()->SetBytes(c.Data(), c.Size());
+        break;
+
     default:
         errStr = "Unknown type: " + ToString(typeId);
         return false;

ydb/core/kqp/common/kqp_resolve.cpp

@@ -68,6 +68,10 @@ NUdf::TUnboxedValue MakeDefaultValueByType(NKikimr::NMiniKQL::TType* type) {
             buf.half[1] = 0;
             return NKikimr::NMiniKQL::MakeString(NUdf::TStringRef(buf.bytes, 16));
         }
+        case NUdf::TDataType<NUdf::TRowid>::Id: {
+            char bytes[NUdf::ROWID_SIZE] = {};
+            return NKikimr::NMiniKQL::MakeString(NUdf::TStringRef(bytes, sizeof(bytes)));
+        }
         default:
             return NKikimr::NMiniKQL::MakeString("");
     }

ydb/core/kqp/common/result_set_format/kqp_formats_arrow.cpp

@@ -23,7 +23,9 @@ std::shared_ptr<arrow::DataType> BuildArrowType(NUdf::EDataSlot slot) {
 
 template <>
 std::shared_ptr<arrow::DataType> BuildArrowType<arrow::FixedSizeBinaryType>(NUdf::EDataSlot slot) {
-    Y_UNUSED(slot);
+    if (slot == NUdf::EDataSlot::Rowid) {
+        return arrow::fixed_size_binary(NUdf::ROWID_SIZE);
+    }
     return arrow::fixed_size_binary(NScheme::FSB_SIZE);
 }
 
@@ -376,6 +378,12 @@ void AppendDataValue<arrow::FixedSizeBinaryType>(arrow::ArrayBuilder* builder, N
             break;
         }
 
+        case NUdf::EDataSlot::Rowid: {
+            auto data = value.AsStringRef();
+            status = typedBuilder->Append(data.Data());
+            break;
+        }
+
         case NUdf::EDataSlot::Decimal: {
             auto intVal = value.GetInt128();
             status = typedBuilder->Append(reinterpret_cast<const char*>(&intVal));

ydb/core/kqp/common/result_set_format/kqp_formats_arrow.h

@@ -32,7 +32,7 @@ constexpr size_t MAX_VARIANT_DEPTH = 2;
  * - Temporal types: Date, Datetime, Timestamp, Interval (and their extended variants)
  * - String types: Utf8, Json, JsonDocument (serialized to string), DyNumber (serialized to string) -> arrow::StringType
  * - Binary types: String, Yson -> arrow::BinaryType
- * - Fixed-size binary: Decimal, Uuid -> arrow::FixedSizeBinaryType
+ * - Fixed-size binary: Decimal, Uuid, Rowid -> arrow::FixedSizeBinaryType
  * - Timezone-aware: TzDate, TzDatetime, TzTimestamp -> arrow::StructType<datetimeType, arrow::StringType (serialized name of timezone)>
  *
  * @tparam TFunc Callable type accepting a single template parameter (Arrow type)
@@ -94,6 +94,7 @@ bool SwitchMiniKQLDataTypeToArrowType(NUdf::EDataSlot typeId, TFunc&& callback)
 
         case NUdf::EDataSlot::Decimal:
         case NUdf::EDataSlot::Uuid:
+        case NUdf::EDataSlot::Rowid:
             return callback.template operator()<arrow::FixedSizeBinaryType>();
 
         case NUdf::EDataSlot::TzDate:

ydb/core/kqp/opt/kqp_statistics_transformer.cpp

@@ -781,6 +781,8 @@ double EstimateRowSize(const TStructExprType& rowType, const TString& format, co
                     break;
                 case EDataSlot::Uuid:
                     break;
+                case EDataSlot::Rowid:
+                    break;
                 case EDataSlot::Date:
                     result += decoded ? 2.0 : 1.51;
                     break;

ydb/core/kqp/provider/yql_kikimr_provider.cpp

@@ -17,6 +17,7 @@
 #include <yql/essentials/providers/common/provider/yql_provider.h>
 #include <yql/essentials/providers/common/schema/expr/yql_expr_schema.h>
 #include <yql/essentials/providers/common/transform/yql_visit.h>
+#include <yql/essentials/types/rowid/rowid.h>
 #include <yql/essentials/providers/result/provider/yql_result_provider.h>
 
 namespace NYql {
@@ -833,6 +834,11 @@ void FillLiteralProto(const NNodes::TCoDataCtor& literal, Ydb::TypedValue& proto
             protoValue.set_high_128(uuidData[1]);
             break;
         }
+        case EDataSlot::Rowid: {
+            YQL_ENSURE(value.size() == NKikimr::NRowid::ROWID_LEN, "Invalid Rowid literal size");
+            protoValue.set_bytes_value(value.data(), value.size());
+            break;
+        }
 
         default:
             YQL_ENSURE(false, "Unexpected type slot " << slot);

ydb/core/kqp/runtime/kqp_scan_data.cpp

@@ -72,6 +72,12 @@ TBytesStatistics GetUnboxedValueSize(const NUdf::TUnboxedValue& value, const NSc
             Y_VERIFY_DEBUG_S(size == sizeof(TGUID), "Wrong Uuid size: " << size);
             return { sizeof(NUdf::TUnboxedValue) + size, size };
         }
+        case NTypeIds::Rowid:
+        {
+            const auto size = value.AsStringRef().Size();
+            Y_VERIFY_DEBUG_S(size == NUdf::ROWID_SIZE, "Wrong Rowid size: " << size);
+            return { sizeof(NUdf::TUnboxedValue) + size, size };
+        }
         case NTypeIds::String:
         case NTypeIds::Utf8:
         case NTypeIds::Json:

ydb/core/scheme/scheme_tablecell.h

@@ -342,6 +342,13 @@ inline int CompareTypedCells(const TCell& a, const TCell& b, const NScheme::TTyp
         return CompareCellsAsByteString(a, b, type.IsDescending());
     }
 
+    case NKikimr::NScheme::NTypeIds::Rowid:
+    {
+        Y_ASSERT(a.Size() == 14);
+        Y_ASSERT(b.Size() == 14);
+        return CompareCellsAsByteString(a, b, type.IsDescending());
+    }
+
     case NKikimr::NScheme::NTypeIds::Decimal:
     {
         Y_ASSERT(a.Size() == sizeof(std::pair<ui64, i64>));
@@ -505,6 +512,7 @@ inline ui64 GetValueHash(NScheme::TTypeInfo info, const TCell& cell) {
     case NYql::NProto::TypeIds::JsonDocument:
     case NYql::NProto::TypeIds::DyNumber:
     case NYql::NProto::TypeIds::Uuid:
+    case NYql::NProto::TypeIds::Rowid:
         return ComputeHash(TStringBuf{cell.Data(), cell.Size()});
 
     default:

ydb/core/scheme_types/scheme_type_registry.cpp

@@ -42,6 +42,7 @@ TTypeRegistry::TTypeRegistry()
     RegisterType<TInterval>();
     RegisterType<TDyNumber>();
     RegisterType<TUuid>();
+    RegisterType<TRowid>();
     RegisterType<TDate32>();
     RegisterType<TDatetime64>();
     RegisterType<TTimestamp64>();

ydb/core/scheme_types/scheme_types_defs.cpp

@@ -41,6 +41,7 @@ namespace NNames {
 
     DECLARE_TYPED_TYPE_NAME(DyNumber);
     DECLARE_TYPED_TYPE_NAME(Uuid);
+    DECLARE_TYPED_TYPE_NAME(Rowid);
 }
 
 void WriteEscapedValue(IOutputStream &out, const char *data, size_t size) {

ydb/core/scheme_types/scheme_types_defs.h

@@ -229,6 +229,7 @@ using TLargeBoundedString = TBoundedString<0x200000, NTypeIds::String2m, NNames:
 namespace NNames {
     extern const char Decimal[8];
     extern const char Uuid[5];
+    extern const char Rowid[6];
 }
 
 class TDecimal : public IIntegerPair<ui64, i64, NTypeIds::Decimal, NNames::Decimal> {};
@@ -237,6 +238,10 @@ class TUuid : public TTypedType<char[16], TUuid, NTypeIds::Uuid, NNames::Uuid> {
 public:
 };
 
+class TRowid : public TTypedType<char[14], TRowid, NTypeIds::Rowid, NNames::Rowid> {
+public:
+};
+
 ////////////////////////////////////////////////////////
 /// Datetime types
 namespace NNames {
@@ -289,6 +294,7 @@ class TInterval64 : public IIntegerTypeWithKeyString<i64, NTypeIds::Interval64,
     xx(Interval, TInterval, __VA_ARGS__) \
     xx(DyNumber, TDyNumber, __VA_ARGS__) \
     xx(Uuid, TUuid, __VA_ARGS__) \
+    xx(Rowid, TRowid, __VA_ARGS__) \
     xx(Date32, TDate32, __VA_ARGS__) \
     xx(Datetime64, TDatetime64, __VA_ARGS__) \
     xx(Timestamp64, TTimestamp64, __VA_ARGS__) \

ydb/core/sys_view/show_create/create_table_formatter.cpp

@@ -235,6 +235,12 @@ void TCreateTableFormatter::FormatPrimitive(NYdb::TValueParser& parser) {
             Stream << ")";
             break;
         }
+        case NYdb::EPrimitiveType::Rowid: {
+            Stream << "ROWID(";
+            EscapeString(TString(parser.GetRowid().ToString()), Stream);
+            Stream << ")";
+            break;
+        }
         default:
             ythrow TFormatFail(Ydb::StatusIds::UNSUPPORTED, "Unsupported primitive type for SHOW CREATE TABLE");
     }

ydb/core/viewer/viewer_query.h

@@ -829,6 +829,8 @@ class TJsonQuery : public TViewerPipeClient {
                 return valueParser.GetDyNumber();
             case NYdb::EPrimitiveType::Uuid:
                 return valueParser.GetUuid().ToString();
+            case NYdb::EPrimitiveType::Rowid:
+                return valueParser.GetRowid().ToString();
         }
         return NJson::JSON_UNDEFINED;
     }

ydb/core/viewer/viewer_tabletinfo.h

@@ -186,6 +186,7 @@ class TJsonTabletInfo : public TJsonWhiteboardRequest<TEvWhiteboard::TEvTabletSt
         }
         case NScheme::NTypeIds::DyNumber:        return "DyNumber";
         case NScheme::NTypeIds::Uuid:            return "Uuid";
+        case NScheme::NTypeIds::Rowid:           return "Rowid";
         default:
             return "-";
         }

ydb/core/ydb_convert/ydb_convert.cpp

@@ -12,6 +12,7 @@
 #include <yql/essentials/types/binary_json/read.h>
 #include <yql/essentials/types/binary_json/write.h>
 #include <yql/essentials/types/dynumber/dynumber.h>
+#include <yql/essentials/types/rowid/rowid.h>
 
 #include <yql/essentials/minikql/dom/json.h>
 #include <yql/essentials/minikql/dom/yson.h>
@@ -345,6 +346,11 @@ Y_FORCE_INLINE void ConvertData(NUdf::TDataTypeId typeId, const NKikimrMiniKQL::
             res.set_high_128(value.GetHi128());
             break;
         }
+        case NUdf::TDataType<NUdf::TRowid>::Id: {
+            const auto& stringRef = value.GetBytes();
+            res.set_bytes_value(stringRef.data(), stringRef.size());
+            break;
+        }
         case NUdf::TDataType<NUdf::TJsonDocument>::Id: {
             const auto json = NBinaryJson::SerializeToJson(value.GetBytes());
             res.set_text_value(json);
@@ -511,6 +517,15 @@ Y_FORCE_INLINE void ConvertData(NUdf::TDataTypeId typeId, const Ydb::Value& valu
             res.SetLow128(value.low_128());
             res.SetHi128(value.high_128());
             break;
+        case NUdf::TDataType<NUdf::TRowid>::Id: {
+            CheckTypeId(value.value_case(), Ydb::Value::kBytesValue, "Rowid");
+            const auto& stringRef = value.bytes_value();
+            if (!NRowid::IsValidRowidBytes(TStringBuf(stringRef.data(), stringRef.size()))) {
+                throw yexception() << "Invalid Rowid value";
+            }
+            res.SetBytes(stringRef.data(), stringRef.size());
+            break;
+        }
         case NUdf::TDataType<NUdf::TJsonDocument>::Id: {
             CheckTypeId(value.value_case(), Ydb::Value::kTextValue, "JsonDocument");
             const auto binaryJson = NBinaryJson::SerializeToBinaryJson(value.text_value());
@@ -1162,6 +1177,10 @@ bool CheckValueData(NScheme::TTypeInfo type, const TCell& cell, TString& err) {
         // Uuid value was verified at parsing time
         break;
 
+    case NScheme::NTypeIds::Rowid:
+        ok = NRowid::IsValidRowidBytes(cell.AsBuf());
+        break;
+
     case NScheme::NTypeIds::Pg:
         // no pg validation here
         break;
@@ -1301,6 +1320,20 @@ bool CellFromProtoVal(const NScheme::TTypeInfo& type, i32 typmod, const Ydb::Val
         c = TCell((const char*)&valInPool, sizeof(valInPool));
         break;
     }
+    case NScheme::NTypeIds::Rowid : {
+        if (!val.Hasbytes_value()) {
+            err = "Cannot parse value of type Rowid";
+            return false;
+        }
+        const auto& bytes = val.Getbytes_value();
+        if (!NRowid::IsValidRowidBytes(TStringBuf(bytes.data(), bytes.size()))) {
+            err = "Invalid Rowid value";
+            return false;
+        }
+        const auto bytesInPool = valueDataPool.AppendString(TStringBuf(bytes.data(), bytes.size()));
+        c = TCell(bytesInPool.data(), bytesInPool.size());
+        break;
+    }
     case NScheme::NTypeIds::Pg : {
         TString binary;
         bool isText = false;
@@ -1451,6 +1484,9 @@ void ProtoValueFromCell(NYdb::TValueBuilder& vb, const NScheme::TTypeInfo& typeI
         vb.Uuid(TUuidValue(lo, hi));
         break;
     }
+    case EPrimitiveType::Rowid:
+        vb.Rowid(TRowidValue(cell.AsBuf().data(), cell.AsBuf().size()));
+        break;
     case EPrimitiveType::JsonDocument:
         vb.JsonDocument(NBinaryJson::SerializeToJson(getString()));
         break;

ydb/library/mkql_proto/mkql_proto.cpp

@@ -13,6 +13,7 @@
 #include <yql/essentials/utils/utf8.h>
 #include <yql/essentials/types/binary_json/write.h>
 #include <yql/essentials/types/dynumber/dynumber.h>
+#include <yql/essentials/types/rowid/rowid.h>
 #include <yql/essentials/minikql/dom/yson.h>
 #include <library/cpp/containers/stack_vector/stack_vec.h>
 
@@ -142,6 +143,11 @@ Y_FORCE_INLINE void HandleKindDataExport(const TType* type, const NUdf::TUnboxed
             UuidToYdbProto(stringRef.Data(), stringRef.Size(), res);
             break;
         }
+        case NUdf::TDataType<NUdf::TRowid>::Id: {
+            const auto& stringRef = value.AsStringRef();
+            res.set_bytes_value(stringRef.Data(), stringRef.Size());
+            break;
+        }
         case NUdf::TDataType<NUdf::TJsonDocument>::Id: {
             NUdf::TUnboxedValue json = ValueToString(NUdf::EDataSlot::JsonDocument, value);
             const auto stringRef = json.AsStringRef();
@@ -510,6 +516,11 @@ Y_FORCE_INLINE void HandleKindDataExport(const TType* type, const NUdf::TUnboxed
             UuidToMkqlProto(stringRef.Data(), stringRef.Size(), res);
             break;
         }
+        case NUdf::TDataType<NUdf::TRowid>::Id: {
+            auto stringRef = value.AsStringRef();
+            res.SetBytes(stringRef.Data(), stringRef.Size());
+            break;
+        }
         case NUdf::TDataType<NUdf::TJsonDocument>::Id: {
             auto stringRef = value.AsStringRef();
             res.SetBytes(stringRef.Data(), stringRef.Size());
@@ -843,6 +854,10 @@ Y_FORCE_INLINE NUdf::TUnboxedValue HandleKindDataImport(const TType* type, const
             buf.half[1] = value.GetHi128();
             return MakeString(NUdf::TStringRef(buf.bytes, 16));
         }
+        case NUdf::TDataType<NUdf::TRowid>::Id:
+            MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kBytes);
+            MKQL_ENSURE_S(value.GetBytes().size() == NRowid::ROWID_LEN);
+            return MakeString(value.GetBytes());
         default:
             MKQL_ENSURE_S(oneOfCase == NKikimrMiniKQL::TValue::ValueValueCase::kBytes,
                 "got: " << (int) oneOfCase << ", type: " << (int) dataType->GetSchemeType());
@@ -877,6 +892,7 @@ void ExportPrimitiveTypeToProto(ui32 schemeType, Ydb::Type& output) {
         case NYql::NProto::TypeIds::Yson:
         case NYql::NProto::TypeIds::Json:
         case NYql::NProto::TypeIds::Uuid:
+        case NYql::NProto::TypeIds::Rowid:
         case NYql::NProto::TypeIds::JsonDocument:
         case NYql::NProto::TypeIds::DyNumber:
         case NYql::NProto::TypeIds::Date32:
@@ -1282,6 +1298,10 @@ TNode* TProtoImporter::ImportNodeFromProto(TType* type, const NKikimrMiniKQL::TV
                     dataNode = TDataLiteral::Create(env.NewStringValue(NUdf::TStringRef(buf.bytes, 16)), dataType, env);
                     break;
                 }
+                case NUdf::TDataType<NUdf::TRowid>::Id:
+                    MKQL_ENSURE(value.GetBytes().size() == NRowid::ROWID_LEN, "Invalid Rowid size");
+                    dataNode = TDataLiteral::Create(env.NewStringValue(NUdf::TStringRef(value.GetBytes().data(), value.GetBytes().size())), dataType, env);
+                    break;
                 default:
                     MKQL_ENSURE(false, TStringBuilder() << "Unknown data type: " << schemeType);
             }
@@ -1643,6 +1663,14 @@ Y_FORCE_INLINE NUdf::TUnboxedValue KindDataImport(const TType* type, const Ydb::
             buf.half[1] = value.high_128();
             return MakeString(NUdf::TStringRef(buf.bytes, 16));
         }
+        case NUdf::TDataType<NUdf::TRowid>::Id: {
+            CheckTypeId(value.value_case(), Ydb::Value::kBytesValue, "Rowid");
+            const auto& stringRef = value.bytes_value();
+            if (stringRef.size() != NRowid::ROWID_LEN) {
+                throw yexception() << "Invalid Rowid value";
+            }
+            return MakeString(TStringBuf(stringRef.data(), stringRef.size()));
+        }
         case NUdf::TDataType<NUdf::TJsonDocument>::Id: {
             CheckTypeId(value.value_case(), Ydb::Value::kTextValue, "JsonDocument");
             const auto binaryJson = NBinaryJson::SerializeToBinaryJson(value.text_value());