From 82441c50686a6dd01e3d82f0780b2a8e5ff18a0f Mon Sep 17 00:00:00 2001 From: lixu Date: Thu, 4 Jun 2026 17:29:43 +0800 Subject: [PATCH 001/129] feat(coro_rpc): add URMA RDMA transport support - Add urma_socket_t class implementing ib_socket_t-compatible interface - Add urma_device.hpp with device discovery and management - Add urma_buffer.hpp with buffer pool for memory registration - Add YLT_ENABLE_URMA CMake option (default OFF) - Integrate URMA into socket_wrapper_t visitor dispatch - Add init_urma() method to coro_rpc_server This enables coro_rpc to use URMA as an alternative RDMA transport on Kunpeng hardware for improved performance. --- cmake/config.cmake | 11 + include/ylt/coro_io/socket_wrapper.hpp | 61 ++ include/ylt/coro_io/urma/urma_buffer.hpp | 182 +++++ include/ylt/coro_io/urma/urma_device.hpp | 282 +++++++ include/ylt/coro_io/urma/urma_socket.hpp | 724 ++++++++++++++++++ include/ylt/coro_rpc/impl/coro_rpc_server.hpp | 40 + 6 files changed, 1300 insertions(+) create mode 100644 include/ylt/coro_io/urma/urma_buffer.hpp create mode 100644 include/ylt/coro_io/urma/urma_device.hpp create mode 100644 include/ylt/coro_io/urma/urma_socket.hpp diff --git a/cmake/config.cmake b/cmake/config.cmake index 09f51c6ec..6433cca33 100644 --- a/cmake/config.cmake +++ b/cmake/config.cmake @@ -78,6 +78,17 @@ if (YLT_ENABLE_IBV) target_link_libraries(${ylt_target_name} INTERFACE -libverbs) endif () endif () +option(YLT_ENABLE_URMA "Enable URMA support" OFF) +if (YLT_ENABLE_URMA) + message(STATUS "Enable URMA support") + if(CMAKE_PROJECT_NAME STREQUAL "yaLanTingLibs") + add_compile_definitions("YLT_ENABLE_URMA") + link_libraries(-lurma) + else () + target_compile_definitions(${ylt_target_name} INTERFACE "YLT_ENABLE_URMA") + target_link_libraries(${ylt_target_name} INTERFACE -lurma) + endif () +endif () if (YLT_ENABLE_CUDA) message(STATUS "Enable cuda support") find_package(CUDAToolkit REQUIRED) diff --git a/include/ylt/coro_io/socket_wrapper.hpp b/include/ylt/coro_io/socket_wrapper.hpp index 8016d4d91..55bf17eda 100644 --- a/include/ylt/coro_io/socket_wrapper.hpp +++ b/include/ylt/coro_io/socket_wrapper.hpp @@ -20,6 +20,9 @@ #include "ibverbs/ib_io.hpp" #include "ibverbs/ib_socket.hpp" #endif +#ifdef YLT_ENABLE_URMA +#include "urma/urma_socket.hpp" +#endif #include "io_context_pool.hpp" namespace coro_io { struct socket_wrapper_t { @@ -50,6 +53,15 @@ struct socket_wrapper_t { ib_socket_(std::make_unique(executor_, config)) { ib_socket_->prepare_accpet(std::move(soc)); } +#endif +#ifdef YLT_ENABLE_URMA + socket_wrapper_t(asio::ip::tcp::socket &&soc, + coro_io::ExecutorWrapper<> *executor, + const coro_io::urma_socket_t::config_t &config) + : executor_(executor), + urma_socket_(std::make_unique(executor_, config)) { + urma_socket_->prepare_accept(std::move(soc)); + } #endif void init_tcp_socket() { asio::ip::address addr; @@ -114,6 +126,24 @@ struct socket_wrapper_t { return true; } #endif +#ifdef YLT_ENABLE_URMA + bool init_client(const coro_io::urma_socket_t::config_t &config) { + try { + init_tcp_socket(); + if (urma_socket_) { + *urma_socket_ = urma_socket_t(executor_, config); + } + else { + urma_socket_ = std::make_unique(executor_, config); + } + } catch (const std::exception &e) { + ELOG_WARN << "init urma client failed:" << e.what(); + init_ok_ = false; + return false; + } + return true; + } +#endif void set_local_ip(const std::string &local_ip) { local_ip_ = local_ip; } @@ -127,6 +157,9 @@ struct socket_wrapper_t { #ifdef YLT_ENABLE_IBV std::unique_ptr ib_socket_; +#endif +#ifdef YLT_ENABLE_URMA + std::unique_ptr urma_socket_; #endif bool init_ok_ = true; @@ -145,6 +178,11 @@ struct socket_wrapper_t { return op(*ib_socket_); } #endif +#ifdef YLT_ENABLE_URMA + if (urma_socket_) { + return op(*urma_socket_); + } +#endif #ifdef YLT_ENABLE_SSL if (use_ssl()) { return op(*ssl_stream_); @@ -162,6 +200,12 @@ struct socket_wrapper_t { ib_socket_->close(); return; } +#endif +#ifdef YLT_ENABLE_URMA + if (urma_socket_) { + urma_socket_->close(); + return; + } #endif if (socket_) { socket_->shutdown(asio::ip::tcp::socket::shutdown_both, ignored_ec); @@ -176,6 +220,13 @@ struct socket_wrapper_t { coro_io::endpoint::rdma}; } #endif +#ifdef YLT_ENABLE_URMA + if (urma_socket_) { + return {urma_socket_->get_remote_address(), + urma_socket_->get_remote_qp_num(), + coro_io::endpoint::rdma}; + } +#endif return {socket_->remote_endpoint().address(), socket_->remote_endpoint().port(), coro_io::endpoint::tcp}; @@ -186,6 +237,13 @@ struct socket_wrapper_t { return {ib_socket_->get_local_address(), ib_socket_->get_local_qp_num(), coro_io::endpoint::rdma}; } +#endif +#ifdef YLT_ENABLE_URMA + if (urma_socket_) { + return {urma_socket_->get_local_address(), + urma_socket_->get_local_qp_num(), + coro_io::endpoint::rdma}; + } #endif return {socket_->local_endpoint().address(), socket_->local_endpoint().port(), coro_io::endpoint::tcp}; @@ -208,5 +266,8 @@ struct socket_wrapper_t { #ifdef YLT_ENABLE_IBV using ibv_socket_t = coro_io::ib_socket_t; #endif +#ifdef YLT_ENABLE_URMA + using urma_socket_type = coro_io::urma_socket_t; +#endif }; } // namespace coro_io \ No newline at end of file diff --git a/include/ylt/coro_io/urma/urma_buffer.hpp b/include/ylt/coro_io/urma/urma_buffer.hpp new file mode 100644 index 000000000..731822942 --- /dev/null +++ b/include/ylt/coro_io/urma/urma_buffer.hpp @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2025, Alibaba Group Holding Limited; + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OF CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "ylt/easylog.hpp" + +#ifdef YLT_ENABLE_URMA +#include +#endif + +namespace coro_io { + +struct urma_buffer_t { + void* addr = nullptr; + size_t length = 0; + void* seg = nullptr; // urma_target_seg_t* + + urma_buffer_t() = default; + urma_buffer_t(void* a, size_t l, void* s) : addr(a), length(l), seg(s) {} + explicit operator bool() const { return addr != nullptr && length > 0; } +}; + +class urma_buffer_pool_t { + public: + urma_buffer_pool_t(void* ctx, size_t buffer_size, size_t buffer_count); + ~urma_buffer_pool_t(); + + urma_buffer_pool_t(const urma_buffer_pool_t&) = delete; + urma_buffer_pool_t& operator=(const urma_buffer_pool_t&) = delete; + + urma_buffer_t get_buffer(int gpu_id = -1); + void return_buffer(urma_buffer_t& buffer); + + size_t buffer_size() const { return config_.buffer_size; } + size_t total_buffer_count() const { return config_.buffer_count; } + size_t free_buffer_count() const; + bool memory_out_of_limit() const { return free_buffer_count() == 0; } + void* context() const { return ctx_; } + + struct Config { + size_t buffer_size = 256 * 1024; + size_t buffer_count = 8; + int gpu_id = -1; + }; + const Config& get_config() const { return config_; } + + private: + bool init_buffers(); + + private: + void* ctx_ = nullptr; + Config config_; + std::vector buffers_; + std::queue free_indices_; + std::mutex mutex_; + std::atomic outstanding_buffers_{0}; +}; + +// ============= Implementation (inline in header) ============= + +inline urma_buffer_pool_t::urma_buffer_pool_t(void* ctx, size_t buffer_size, + size_t buffer_count) + : ctx_(ctx), config_({buffer_size, buffer_count, -1}) { + init_buffers(); +} + +inline bool urma_buffer_pool_t::init_buffers() { +#ifdef YLT_ENABLE_URMA + buffers_.reserve(config_.buffer_count); + + for (size_t i = 0; i < config_.buffer_count; ++i) { + void* addr = std::malloc(config_.buffer_size); + if (!addr) { + ELOG_ERROR << "Failed to allocate buffer"; + break; + } + + urma_reg_seg_flag_t flag = {}; + flag.bs.token_policy = URMA_TOKEN_NONE; + flag.bs.cacheable = URMA_NON_CACHEABLE; + flag.bs.access = URMA_ACCESS_READ | URMA_ACCESS_WRITE; + flag.bs.token_id_valid = 0; + + urma_seg_cfg_t seg_cfg = {}; + seg_cfg.va = reinterpret_cast(addr); + seg_cfg.len = config_.buffer_size; + seg_cfg.flag = flag; + + urma_target_seg_t* seg = urma_register_seg( + reinterpret_cast(ctx_), &seg_cfg); + if (!seg) { + ELOG_ERROR << "urma_register_seg failed"; + std::free(addr); + break; + } + + buffers_.push_back(urma_buffer_t(addr, config_.buffer_size, seg)); + free_indices_.push(i); + } + + ELOG_INFO << "URMA buffer pool: " << buffers_.size() + << " buffers of " << config_.buffer_size << " bytes"; + return !buffers_.empty(); +#else + return false; +#endif +} + +inline urma_buffer_pool_t::~urma_buffer_pool_t() { +#ifdef YLT_ENABLE_URMA + std::lock_guard lock(mutex_); + for (auto& buf : buffers_) { + if (buf) { + if (buf.seg) urma_unregister_seg(static_cast(buf.seg)); + std::free(buf.addr); + } + } + buffers_.clear(); +#endif +} + +inline urma_buffer_t urma_buffer_pool_t::get_buffer(int gpu_id) { +#ifdef YLT_ENABLE_URMA + std::lock_guard lock(mutex_); + if (free_indices_.empty()) { + ELOG_WARN << "URMA buffer pool out of buffers"; + return urma_buffer_t{}; + } + size_t idx = free_indices_.front(); + free_indices_.pop(); + outstanding_buffers_++; + return buffers_[idx]; +#else + return urma_buffer_t{}; +#endif +} + +inline void urma_buffer_pool_t::return_buffer(urma_buffer_t& buffer) { +#ifdef YLT_ENABLE_URMA + if (!buffer) return; + std::lock_guard lock(mutex_); + for (size_t i = 0; i < buffers_.size(); ++i) { + if (buffers_[i].addr == buffer.addr) { + free_indices_.push(i); + outstanding_buffers_--; + buffer = urma_buffer_t{}; + return; + } + } +#endif +} + +inline size_t urma_buffer_pool_t::free_buffer_count() const { +#ifdef YLT_ENABLE_URMA + return free_indices_.size(); +#else + return 0; +#endif +} + +} // namespace coro_io diff --git a/include/ylt/coro_io/urma/urma_device.hpp b/include/ylt/coro_io/urma/urma_device.hpp new file mode 100644 index 000000000..82d7c717a --- /dev/null +++ b/include/ylt/coro_io/urma/urma_device.hpp @@ -0,0 +1,282 @@ +/* + * Copyright (c) 2025, Alibaba Group Holding Limited; + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include +#include + +#include "ylt/easylog.hpp" + +// URMA forward declarations +struct urma_context_t; +struct urma_device_t; +struct urma_eid_t; +struct urma_eid_info_t; +struct urma_device_attr_t; +struct urma_init_attr_t; + +#define URMA_EID_SIZE 16 + +#ifdef YLT_ENABLE_URMA +#include +#endif + +namespace coro_io { + +class urma_buffer_pool_t; + +// URMA Device abstraction (similar to ib_device_t) +class urma_device_t { + public: + urma_device_t(); + ~urma_device_t(); + + urma_device_t(const urma_device_t&) = delete; + urma_device_t& operator=(const urma_device_t&) = delete; + + bool init(const std::string& device_name, int eid_index = 0); + void close(); + + urma_context_t* context() const { return context_; } + urma_device_t* device() const { return device_; } + const std::string& name() const { return name_; } + int eid_index() const { return eid_index_; } + const urma_eid_t& eid() const { return eid_; } + uint32_t max_jetty() const { return device_attr_.dev_cap.max_jetty; } + uint32_t max_jfc() const { return device_attr_.dev_cap.max_jfc; } + + std::string eid_string() const; + asio::ip::address gid_address() const; + bool is_valid() const { return context_ != nullptr && device_ != nullptr; } + std::shared_ptr get_buffer_pool() const { return buffer_pool_; } + + private: + std::string name_; + int eid_index_ = -1; + urma_device_t* device_ = nullptr; + urma_context_t* context_ = nullptr; + urma_eid_t eid_{}; + urma_device_attr_t device_attr_{}; + std::shared_ptr buffer_pool_; +}; + +// Global device management +class urma_device_manager { + public: + static urma_device_manager& instance(); + bool init(); + std::shared_ptr get_device(const std::string& device_name = ""); + std::vector> get_all_devices(); + std::shared_ptr get_global_device(); + + private: + urma_device_manager() = default; + ~urma_device_manager(); + bool initialized_ = false; + std::vector> devices_; + std::shared_ptr global_device_; +}; + +inline std::shared_ptr get_global_urma_device() { + return urma_device_manager::instance().get_global_device(); +} + +// ============= Implementation (inline in header) ============= + +inline urma_device_t::urma_device_t() = default; + +inline urma_device_t::~urma_device_t() { close(); } + +inline bool urma_device_t::init(const std::string& device_name, int eid_index) { +#ifdef YLT_ENABLE_URMA + name_ = device_name; + eid_index_ = eid_index; + + int num_devices = 0; + urma_device_t** devices = urma_get_device_list(&num_devices); + if (!devices || num_devices <= 0) { + ELOG_ERROR << "urma_get_device_list failed"; + return false; + } + + urma_device_t* found_device = nullptr; + for (int i = 0; i < num_devices; ++i) { + if (device_name.empty() || std::string(devices[i]->name) == device_name) { + found_device = devices[i]; + break; + } + } + + if (!found_device) { + ELOG_ERROR << "URMA device not found: " << device_name; + urma_free_device_list(devices); + return false; + } + + device_ = found_device; + + uint32_t eid_cnt = 0; + urma_eid_info_t* eid_list = urma_get_eid_list(device_, &eid_cnt); + if (!eid_list || eid_cnt == 0) { + ELOG_ERROR << "urma_get_eid_list failed"; + urma_free_device_list(devices); + return false; + } + + if (eid_index >= 0 && eid_index < (int)eid_cnt) { + eid_index_ = eid_index; + } else { + eid_index_ = 0; + } + eid_ = eid_list[eid_index_].eid; + urma_free_eid_list(eid_list); + + context_ = urma_create_context(device_, eid_index_); + if (!context_) { + ELOG_ERROR << "urma_create_context failed"; + urma_free_device_list(devices); + return false; + } + + if (urma_query_device(device_, &device_attr_) != 0) { + ELOG_ERROR << "urma_query_device failed"; + urma_delete_context(context_); + context_ = nullptr; + urma_free_device_list(devices); + return false; + } + + urma_free_device_list(devices); + ELOG_INFO << "URMA device: " << name_ << ", EID: " << eid_string(); + return true; +#else + ELOG_WARN << "URMA not enabled"; + return false; +#endif +} + +inline void urma_device_t::close() { +#ifdef YLT_ENABLE_URMA + if (context_) { + urma_delete_context(context_); + context_ = nullptr; + } +#endif +} + +inline std::string urma_device_t::eid_string() const { + char buf[64] = {0}; + snprintf(buf, sizeof(buf), + "%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:" + "%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x", + eid_.raw[0], eid_.raw[1], eid_.raw[2], eid_.raw[3], + eid_.raw[4], eid_.raw[5], eid_.raw[6], eid_.raw[7], + eid_.raw[8], eid_.raw[9], eid_.raw[10], eid_.raw[11], + eid_.raw[12], eid_.raw[13], eid_.raw[14], eid_.raw[15]); + return std::string(buf); +} + +inline asio::ip::address urma_device_t::gid_address() const { + char buf[64]; + snprintf(buf, sizeof(buf), "%d.%d.%d.%d", + eid_.raw[0], eid_.raw[1], eid_.raw[2], eid_.raw[3]); + std::error_code ec; + auto addr = asio::ip::make_address(buf, ec); + if (ec) return asio::ip::make_address_v4(0x7F000001); + return addr; +} + +// urma_device_manager +inline urma_device_manager& urma_device_manager::instance() { + static urma_device_manager inst; + return inst; +} + +inline urma_device_manager::~urma_device_manager() { + devices_.clear(); + global_device_.reset(); +} + +inline bool urma_device_manager::init() { +#ifdef YLT_ENABLE_URMA + if (initialized_) return true; + urma_init_attr_t init_attr = {}; + if (urma_init(&init_attr) != URMA_SUCCESS && urma_init(&init_attr) != URMA_EEXIST) { + ELOG_ERROR << "urma_init failed"; + return false; + } + initialized_ = true; + return true; +#else + return false; +#endif +} + +inline std::shared_ptr urma_device_manager::get_device( + const std::string& device_name) { +#ifdef YLT_ENABLE_URMA + if (!initialized_) init(); + + for (auto& dev : devices_) { + if (device_name.empty() || dev->name() == device_name) { + return dev; + } + } + + auto dev = std::make_shared(); + if (!dev->init(device_name)) return nullptr; + + devices_.push_back(dev); + if (!global_device_) global_device_ = dev; + return dev; +#else + return nullptr; +#endif +} + +inline std::vector> +urma_device_manager::get_all_devices() { +#ifdef YLT_ENABLE_URMA + if (!devices_.empty()) return devices_; + if (!initialized_) init(); + + int num_devices = 0; + urma_device_t** urma_devices = urma_get_device_list(&num_devices); + if (!urma_devices || num_devices <= 0) return devices_; + + for (int i = 0; i < num_devices; ++i) { + auto dev = std::make_shared(); + if (dev->init(urma_devices[i]->name)) { + devices_.push_back(dev); + if (!global_device_) global_device_ = dev; + } + } + urma_free_device_list(urma_devices); + return devices_; +#else + return {}; +#endif +} + +inline std::shared_ptr urma_device_manager::get_global_device() { + if (!global_device_) get_all_devices(); + return global_device_; +} + +} // namespace coro_io diff --git a/include/ylt/coro_io/urma/urma_socket.hpp b/include/ylt/coro_io/urma/urma_socket.hpp new file mode 100644 index 000000000..c375f5ec2 --- /dev/null +++ b/include/ylt/coro_io/urma/urma_socket.hpp @@ -0,0 +1,724 @@ +/* + * Copyright (c) 2025, Alibaba Group Holding Limited; + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "asio/dispatch.hpp" +#include "asio/ip/address.hpp" +#include "asio/ip/tcp.hpp" +#include "async_simple/coro/Lazy.h" +#include "async_simple/util/move_only_function.h" +#include "ylt/coro_io/coro_io.hpp" +#include "ylt/easylog.hpp" +#include "ylt/struct_pack.hpp" + +// URMA forward declarations +struct urma_context_t; +struct urma_jfc_t; +struct urma_jfr_t; +struct urma_jetty_t; +struct urma_target_jetty_t; +struct urma_target_seg_t; +struct urma_device_t; +struct urma_eid_t; +struct urma_cr_t; +struct urma_jfs_wr; +struct urma_seg_cfg_t; +struct urma_token_t; +struct urma_init_attr_t; + +enum class urma_transport_type_t : int; + +#define URMA_EID_LEN 16 + +namespace coro_io { +namespace detail { + +struct urma_socket_shared_state_t; + +template +struct circle_buffer { + std::vector queue; + uint32_t front_ = 0, end_ = 0; + bool may_empty = true; + circle_buffer(uint32_t size) { + assert(size > 0); + queue.resize(size); + } + void push(T&& elem) { + assert(!full()); + may_empty = false; + end_ = (end_ + 1) % queue.size(); + queue[end_] = std::move(elem); + } + T pop() { + assert(!empty()); + front_ = (front_ + 1) % queue.size(); + may_empty = true; + return std::move(queue[front_]); + } + T& back() { return queue[end_]; } + T& front() { return queue[(front_ + 1) % queue.size()]; } + bool full() const noexcept { return end_ == front_ && !may_empty; } + bool empty() const noexcept { return end_ == front_ && may_empty; } + std::size_t size() const noexcept { + if (front_ > end_) { + return queue.size() + end_ - front_; + } + else if (front_ == end_) { + return empty() ? 0 : queue.size(); + } + else { + return end_ - front_; + } + } +}; + +// URMA-specific buffer representation (compatible with ibv_sge layout) +struct urma_sge { + uint64_t addr; + uint32_t length; + uint32_t lkey; +}; + +struct urma_buffer_t { + void* addr = nullptr; + size_t length = 0; + uint32_t lkey = 0; // URMA key (used similarly to lkey) + + urma_sge subview(size_t offset = 0, size_t len = 0) const { + urma_sge sge; + sge.addr = reinterpret_cast( + reinterpret_cast(addr) + offset); + sge.length = (len == 0) ? static_cast(length - offset) + : static_cast(len); + sge.lkey = lkey; + return sge; + } + + explicit operator bool() const { return addr != nullptr && length > 0; } +}; + +using callback_t = async_simple::util::move_only_function)>; + +struct urma_socket_shared_state_t + : public std::enable_shared_from_this { + static void resume(std::pair&& arg, + callback_t&& handle) { + if (handle) [[likely]] { + auto handle_tmp = std::move(handle); + handle_tmp(std::move(arg)); + } + } + + coro_io::ExecutorWrapper<>* executor_; + asio::ip::tcp::socket soc_; + + // URMA resources + urma_context_t* urma_context_ = nullptr; + urma_jfc_t* jfc_ = nullptr; + urma_jfr_t* jfr_ = nullptr; + urma_jetty_t* jetty_ = nullptr; + urma_target_jetty_t* remote_jetty_ = nullptr; + + // Buffer management + std::vector recv_buffers_; + std::vector send_buffers_; + circle_buffer recv_queue_; + circle_buffer send_queue_; + circle_buffer> recv_result_; + circle_buffer send_cb_; + + callback_t recv_cb_; + urma_buffer_t recv_buf_; + + std::size_t recv_buffer_cnt_ = 0; + std::size_t send_buffer_cnt_ = 0; + uint32_t send_buffer_data_size_ = 0; + uint32_t buffer_size_ = 256 * 1024; // Default 256KB + + std::atomic has_close_ = {false}; + bool peer_close_ = false; + std::optional> wait_promise_; + + // Remote peer info + std::array remote_eid_; + uint32_t remote_jetty_id_ = 0; + + urma_socket_shared_state_t(coro_io::ExecutorWrapper<>* executor, + std::size_t recv_buffer_cnt, + std::size_t send_buffer_cnt, + std::size_t max_recv_buffer_cnt, + uint32_t buffer_size) + : executor_(executor), + soc_(executor->get_asio_executor()), + recv_buffer_cnt_(recv_buffer_cnt), + send_buffer_cnt_(send_buffer_cnt), + buffer_size_(buffer_size), + recv_queue_(max_recv_buffer_cnt), + send_queue_(send_buffer_cnt + 1), + recv_result_(max_recv_buffer_cnt), + send_cb_(send_buffer_cnt + 2) {} + + urma_socket_shared_state_t(urma_socket_shared_state_t&&) = delete; + urma_socket_shared_state_t& operator=(urma_socket_shared_state_t&&) = delete; + + auto get_executor() const noexcept { return executor_->get_asio_executor(); } + + void return_send_buffer(urma_buffer_t buffer) { + assert(!send_queue_.full()); + send_queue_.push(std::move(buffer)); + } + + void wake_up_if_is_waiting(std::error_code ec) { + if (wait_promise_) { + auto promise = std::move(wait_promise_); + wait_promise_ = std::nullopt; + promise->setValue(ec); + } + } + + async_simple::coro::Lazy waiting_write_over() { + assert(send_cb_.size()); + wait_promise_ = async_simple::Promise(); + auto ec = co_await wait_promise_->getFuture(); + co_return ec; + } + + void cancel() { + assert(executor_->get_asio_executor().running_in_this_thread()); + std::error_code ec; + soc_.cancel(ec); + } + + void close_impl() { + ELOG_TRACE << "jetty closed"; + std::error_code ec; + soc_.cancel(ec); + soc_.close(ec); + } + + void close(bool should_check = true) { + assert(executor_->get_asio_executor().running_in_this_thread()); + + bool has_close = false; + if (should_check) { + has_close = has_close_.exchange(true); + } + if (!has_close) { + shutdown().start([self = shared_from_this()](auto&&) { + self->close_impl(); + }); + } + } + + async_simple::coro::Lazy shutdown() { + ELOG_TRACE << "start to notify peer close"; + co_await coro_io::sleep_for(std::chrono::seconds{1}, executor_); + ELOG_TRACE << "finished to notify peer close"; + co_return; + } + + urma_buffer_t release_send_buffer() noexcept { + assert(send_queue_.size()); + send_buffer_data_size_ = 0; + return send_queue_.pop(); + } + + std::size_t sent_request_count() const noexcept { return send_cb_.size(); } + + void post_send_impl(urma_sge sge, callback_t&& handler, + bool skip_check_close = false) { + ELOG_TRACE << "post send sge length:" << sge.length + << ", address:" << sge.addr; + + if (!skip_check_close && has_close_) [[unlikely]] { + urma_socket_shared_state_t::resume( + std::pair{std::make_error_code(std::errc::operation_canceled), 0}, + std::move(handler)); + return; + } + + // Build URMA send WR + urma_jfs_wr wr{}; + wr.next = nullptr; + wr.sg_list = &sge; + wr.num_sge = sge.length ? 1 : 0; + wr.user_ctx = reinterpret_cast(new callback_t(std::move(handler))); + + urma_jfs_wr* bad_wr = nullptr; + auto status = urma_post_jetty_send_wr(jetty_, &wr, &bad_wr); + if (status != 0) [[unlikely]] { + delete reinterpret_cast(wr.user_ctx); + auto err_code = std::make_error_code(std::errc{std::abs(status)}); + ELOG_ERROR << "urma post send failed: " << err_code.message(); + urma_socket_shared_state_t::resume(std::pair{err_code, std::size_t{0}}, + std::move(handler)); + } + else { + send_cb_.push(callback_t{}); // Placeholder for now + } + } + + void post_recv_impl(callback_t&& handler) { + if (!recv_result_.empty()) { + auto result = recv_result_.pop(); + recv_buf_ = std::move(recv_queue_.pop()); + urma_socket_shared_state_t::resume(std::move(result), std::move(handler)); + return; + } + else if (has_close_) [[unlikely]] { + urma_socket_shared_state_t::resume( + std::pair{std::make_error_code(std::errc::io_error), 0}, + std::move(handler)); + return; + } + recv_cb_ = std::move(handler); + } + + std::error_code poll_completion() { + // Poll JFC for completions + urma_cr_t cr_list[8]; + int num_completed = urma_poll_jfc(jfc_, 8, cr_list); + + if (num_completed < 0) [[unlikely]] { + return std::make_error_code(std::errc::io_error); + } + + std::error_code ec; + for (int i = 0; i < num_completed; ++i) { + auto& cr = cr_list[i]; + ec = (cr.status == 0) ? std::error_code{} + : std::make_error_code(std::errc::io_error); + + if (cr.status != 0) [[unlikely]] { + ELOG_WARN << "urma operation failed with status:" << cr.status; + } + + // Determine if this is a send or recv completion based on context + if (!send_cb_.empty()) { + urma_socket_shared_state_t::resume( + std::pair{ec, static_cast(cr.len)}, + send_cb_.pop()); + } + else if (!recv_cb_) { + recv_result_.push( + std::pair{ec, static_cast(cr.len)}); + } + else { + recv_buf_ = recv_queue_.pop(); + urma_socket_shared_state_t::resume( + std::pair{ec, static_cast(cr.len)}, + std::move(recv_cb_)); + } + } + + return {}; + } +}; + +} // namespace detail + +class urma_socket_t { + public: + struct config_t { + uint32_t cq_size = 128; + uint16_t recv_buffer_cnt = 8; + uint16_t send_buffer_cnt = 4; + uint16_t jetty_cnt = 4; + uint32_t buffer_size = 256 * 1024; // 256KB default + std::string device_name; + int eid_index = 0; + // Shared URMA context (can be nullptr for simple case) + std::shared_ptr urma_context; + }; + + // URMA socket info exchanged during handshake (similar to ib_socket_info) + struct urma_socket_info { + uint8_t eid[URMA_EID_LEN]; // EID + uint32_t jetty_id; // Jetty ID + uint32_t buffer_size; // Buffer size + constexpr static auto struct_pack_config = struct_pack::DISABLE_TYPE_INFO; + }; + + using callback_t = detail::callback_t; + + urma_socket_t(coro_io::ExecutorWrapper<>* executor, const config_t& config) + : executor_(executor) { + init(config); + } + + urma_socket_t(coro_io::ExecutorWrapper<>* executor = coro_io::get_global_executor()) + : executor_(executor) { + init(config_t{}); + } + + urma_socket_t(const config_t& config) + : executor_(coro_io::get_global_executor()) { + init(config); + } + + urma_socket_t(urma_socket_t&&) = default; + urma_socket_t& operator=(urma_socket_t&& o) { + close(); + remote_address_ = std::move(o.remote_address_); + remote_jetty_id_ = o.remote_jetty_id_; + remain_data_ = o.remain_data_; + state_ = std::move(o.state_); + executor_ = o.executor_; + conf_ = std::move(o.conf_); + buffer_size_ = o.buffer_size_; + return *this; + } + + ~urma_socket_t() { close(); } + + bool is_open() const noexcept { + return state_ != nullptr && !state_->has_close_; + } + + // Consume data from receive buffer + std::size_t consume(char* dst, std::size_t sz, int dst_gpu_id) { + auto len = std::min(sz, remain_data_.size()); + if (len) { + memcpy(dst, remain_data_.data(), len); + remain_data_ = remain_data_.substr(len); + if (remain_data_.empty()) { + // Return buffer to pool + } + } + return len; + } + + std::size_t remain_read_buffer_size() { return remain_data_.size(); } + + void set_read_buffer_len(std::size_t has_read_size, std::size_t remain_size) { + remain_data_ = std::string_view{ + reinterpret_cast(state_->recv_buf_.addr) + has_read_size, + remain_size}; + } + + // Get current receive buffer as ibv_sge-compatible structure + urma_sge get_recv_buffer() { + assert(remain_read_buffer_size() == 0); + assert(state_->recv_buf_.addr != nullptr); + return state_->recv_buf_.subview(); + } + + void post_recv(callback_t&& cb) { state_->post_recv_impl(std::move(cb)); } + + void post_send(urma_sge buffer, callback_t&& cb) { + state_->post_send_impl(buffer, std::move(cb)); + } + + // For ib_socket_t compatibility - accept ibv_sge and convert + void post_send(ibv_sge buffer, callback_t&& cb) { + urma_sge sge; + sge.addr = buffer.addr; + sge.length = buffer.length; + sge.lkey = buffer.lkey; + post_send(sge, std::move(cb)); + } + + uint32_t get_buffer_size() const noexcept { return buffer_size_; } + + config_t& get_config() noexcept { return conf_; } + const config_t& get_config() const noexcept { return conf_; } + + async_simple::coro::Lazy waiting_write_over() { + return state_->waiting_write_over(); + } + + // Accept incoming URMA connection via TCP handshake + async_simple::coro::Lazy accept( + std::string_view magic = "") noexcept { + urma_socket_t::urma_socket_info peer_info; + constexpr auto sz = struct_pack::get_needed_size(peer_info); + assert(magic.size() < sz.size()); + + char buffer[sz.size()]; + memcpy(buffer, magic.data(), magic.size()); + + auto [ec, _] = co_await async_read( + state_->soc_, + asio::buffer(buffer + magic.size(), sizeof(buffer) - magic.size())); + if (ec) [[unlikely]] { + co_return ec; + } + + auto ec2 = struct_pack::deserialize_to(peer_info, std::span{buffer}); + if (ec2) [[unlikely]] { + co_return std::make_error_code(std::errc::protocol_error); + } + + ELOG_DEBUG << "Remote Jetty ID = " << peer_info.jetty_id; + remote_jetty_id_ = peer_info.jetty_id; + + // Copy remote EID + std::copy(std::begin(peer_info.eid), std::end(peer_info.eid), + state_->remote_eid_.begin()); + + // Convert EID to address for compatibility + remote_address_ = eid_to_address(peer_info.eid); + ELOG_DEBUG << "Remote EID = " << remote_address_; + + buffer_size_ = std::min(peer_info.buffer_size, conf_.buffer_size); + ELOG_DEBUG << "Final buffer size = " << buffer_size_; + + // Send back our info + urma_socket_info local_info{}; + local_info.jetty_id = get_local_jetty_id(); + local_info.buffer_size = conf_.buffer_size; + get_local_eid(local_info.eid); + + struct_pack::serialize_to((char*)buffer, sz, local_info); + co_await async_write(state_->soc_, asio::buffer(buffer)); + + // Shutdown TCP socket - RDMA is now the data channel + std::error_code ignore_ec; + state_->soc_.shutdown(asio::ip::tcp::socket::shutdown_both, ignore_ec); + state_->soc_.close(ignore_ec); + + co_return std::error_code{}; + } + + void prepare_accept(asio::ip::tcp::socket soc) noexcept { + state_->soc_ = std::move(soc); + } + + async_simple::coro::Lazy accept( + asio::ip::tcp::socket soc) noexcept { + state_->soc_ = std::move(soc); + return accept(); + } + + asio::ip::address get_remote_address() const noexcept { + return remote_address_; + } + + uint32_t get_remote_qp_num() const noexcept { + return remote_jetty_id_; // jetty_id serves as QP number in URMA + } + + asio::ip::address get_local_address() const noexcept { + return local_address_; + } + + uint32_t get_local_qp_num() const noexcept { + return get_local_jetty_id(); + } + + // Magic number for protocol detection (similar to ib_md5_first_header) + constexpr static uint32_t urma_md5_header = + struct_pack::get_type_code(); + constexpr static char urma_md5_first_header = + struct_pack::get_type_code() % 256; + + // Connect to remote URMA endpoint + async_simple::coro::Lazy connect_impl() noexcept { + try { + urma_socket_t::urma_socket_info peer_info{}; + peer_info.jetty_id = get_local_jetty_id(); + peer_info.buffer_size = conf_.buffer_size; + get_local_eid(peer_info.eid); + + constexpr auto sz = struct_pack::get_needed_size(peer_info); + char buffer[sz.size()]; + struct_pack::serialize_to((char*)buffer, sz, peer_info); + + // Send our info + auto [ec, len] = co_await async_write(state_->soc_, + asio::buffer(buffer)); + if (ec) { + co_return std::move(ec); + } + + // Read remote info + std::tie(ec, len) = co_await async_read(state_->soc_, + asio::buffer(buffer)); + std::error_code ignore_ec; + state_->soc_.shutdown(asio::ip::tcp::socket::shutdown_both, ignore_ec); + state_->soc_.close(ignore_ec); + + if (ec) { + co_return std::move(ec); + } + + auto ec2 = struct_pack::deserialize_to(peer_info, std::span{buffer}); + if (ec2) [[unlikely]] { + co_return std::make_error_code(std::errc::protocol_error); + } + + remote_jetty_id_ = peer_info.jetty_id; + std::copy(std::begin(peer_info.eid), std::end(peer_info.eid), + state_->remote_eid_.begin()); + remote_address_ = eid_to_address(peer_info.eid); + buffer_size_ = std::min(peer_info.buffer_size, conf_.buffer_size); + + } catch (const std::system_error& err) { + co_return err.code(); + } + co_return std::error_code{}; + } + + async_simple::coro::Lazy connect( + const std::string& host, const std::string& port) noexcept { + auto ec = co_await async_connect(get_coro_executor(), state_->soc_, + host, port); + if (ec) [[unlikely]] { + co_return std::move(ec); + } + ec = co_await connect_impl(); + if (ec) [[unlikely]] { + close(); + } + co_return ec; + } + + template + async_simple::coro::Lazy connect( + const EndPointSeq& endpoint) noexcept { + auto ec = co_await async_connect(state_->soc_, endpoint); + if (ec) [[unlikely]] { + co_return std::move(ec); + } + ec = co_await connect_impl(); + if (ec) [[unlikely]] { + close(); + } + co_return ec; + } + + void close() { + if (state_) { + if (!state_->has_close_.exchange(true)) { + asio::dispatch(executor_->get_asio_executor(), [state = state_]() { + state->close(false); + }); + } + } + } + + auto get_executor() const { return executor_->get_asio_executor(); } + auto get_coro_executor() const { return executor_; } + + urma_buffer_t release_send_buffer() noexcept { + return state_->release_send_buffer(); + } + + std::size_t sent_request_count() const noexcept { + return state_->sent_request_count(); + } + + std::optional get_send_buffer_view() noexcept { + if (state_->send_queue_.empty()) { + // Get buffer from pool + urma_buffer_t buf; + // TODO: Get from URMA buffer pool + if (!buf) { + ELOG_WARN << "buffer out of limit, get send buffer failed"; + close(); + return std::nullopt; + } + state_->send_queue_.push(std::move(buf)); + } + return state_->send_queue_.front().subview(state_->send_buffer_data_size_); + } + + std::size_t get_free_send_buffer_size() noexcept { + return buffer_size_ - state_->send_buffer_data_size_; + } + + void consume_send_buffer(std::size_t sz) noexcept { + state_->send_buffer_data_size_ += sz; + } + + std::shared_ptr get_state() const noexcept { + return state_; + } + + detail::urma_socket_shared_state_t* get_raw_state() const noexcept { + return state_.get(); + } + + // URMA-specific methods + uint32_t get_local_jetty_id() const { + if (state_ && state_->jetty_) { + return state_->jetty_->jetty_id.id; + } + return 0; + } + + void get_local_eid(uint8_t* eid) const { + if (state_ && state_->jfc_) { + std::copy(std::begin(state_->jfc_->jfc_id.eid.raw), + std::end(state_->jfc_->jfc_id.eid.raw), eid); + } + } + + private: + void init(const config_t& config) { + conf_ = config; + conf_.recv_buffer_cnt = std::max(conf_.recv_buffer_cnt, 1); + conf_.send_buffer_cnt = std::max(conf_.send_buffer_cnt, 1); + + ELOG_INFO << "urma_socket config: recv_buffer_cnt:" << conf_.recv_buffer_cnt + << ", send_buffer_cnt:" << conf_.send_buffer_cnt + << ", buffer_size:" << conf_.buffer_size; + + state_ = std::make_shared( + executor_, conf_.recv_buffer_cnt, conf_.send_buffer_cnt, + conf_.recv_buffer_cnt + 2, conf_.buffer_size); + + buffer_size_ = conf_.buffer_size; + } + + // Convert URMA EID to asio::ip::address for compatibility + static asio::ip::address eid_to_address(const uint8_t* eid) { + // EID is 16 bytes, we can format first 4 bytes as IPv4 for simplicity + // or use a proper conversion + char buf[64]; + snprintf(buf, sizeof(buf), "%d.%d.%d.%d", + eid[0], eid[1], eid[2], eid[3]); + std::error_code ec; + auto addr = asio::ip::make_address(buf, ec); + if (ec) { + // Fallback to localhost if conversion fails + return asio::ip::make_address_v4(0x7F000001); // 127.0.0.1 + } + return addr; + } + + asio::ip::address remote_address_; + uint32_t remote_jetty_id_{0}; + std::string_view remain_data_; + std::shared_ptr state_; + coro_io::ExecutorWrapper<>* executor_; + config_t conf_; + uint32_t buffer_size_{0}; + asio::ip::address local_address_; +}; + +} // namespace coro_io \ No newline at end of file diff --git a/include/ylt/coro_rpc/impl/coro_rpc_server.hpp b/include/ylt/coro_rpc/impl/coro_rpc_server.hpp index 2733fe8f8..4eda9ae5f 100644 --- a/include/ylt/coro_rpc/impl/coro_rpc_server.hpp +++ b/include/ylt/coro_rpc/impl/coro_rpc_server.hpp @@ -140,6 +140,11 @@ class coro_rpc_server_base { acceptors_.push_back(std::make_unique( config.address, config.port)); } +#ifdef YLT_ENABLE_URMA + if (config.urma_config) { + init_urma(config.urma_config.value()); + } +#endif } ~coro_rpc_server_base() { @@ -165,6 +170,11 @@ class coro_rpc_server_base { ibv_dev_lists_ = std::move(ibv_dev_lists); } #endif +#ifdef YLT_ENABLE_URMA + void init_urma(const coro_io::urma_socket_t::config_t &conf = {}) { + urma_config_ = conf; + } +#endif /*! * Start the server in blocking mode @@ -562,6 +572,20 @@ class coro_rpc_server_base { co_return init_ok; } #endif +#ifdef YLT_ENABLE_URMA + async_simple::coro::Lazy update_to_urma(coro_connection *conn) { + bool init_ok = true; + auto &wrapper = conn->socket_wrapper(); + try { + wrapper = {std::move(*wrapper.socket()), wrapper.get_executor(), + urma_config_.value_or(coro_io::urma_socket_t::config_t{})}; + } catch (...) { + ELOG_WARN << "init urma connection failed"; + init_ok = false; + } + co_return init_ok; + } +#endif async_simple::coro::Lazy start_one( std::shared_ptr conn) noexcept { @@ -592,6 +616,19 @@ class coro_rpc_server_base { } break; } +#endif +#ifdef YLT_ENABLE_URMA + if (urma_config_.has_value() && result.magic_number.size() == 1 && + result.magic_number[0] == + coro_io::urma_socket_t::urma_md5_first_header) { + ELOG_TRACE << "protocol is urma, try to update"; + auto result = co_await update_to_urma(conn.get()); + if (!result) { + ELOG_WARN << "urma init failed"; + co_return; + } + break; + } #endif if (connection_transfer_) { ELOG_TRACE @@ -635,6 +672,9 @@ class coro_rpc_server_base { std::vector> ibv_dev_lists_; std::atomic rr_index_ = 0; #endif +#ifdef YLT_ENABLE_URMA + std::optional urma_config_; +#endif std::function client_filter_; }; From 9c33a1f8f22206eee8d9306d3aa1dc2b8c7d1ac7 Mon Sep 17 00:00:00 2001 From: lixu Date: Thu, 4 Jun 2026 17:56:46 +0800 Subject: [PATCH 002/129] feat(urma_example): add URMA example based on rdma_example --- .../examples/urma_example/CMakeLists.txt | 4 + .../examples/urma_example/urma_example.cpp | 129 ++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 src/coro_rpc/examples/urma_example/CMakeLists.txt create mode 100644 src/coro_rpc/examples/urma_example/urma_example.cpp diff --git a/src/coro_rpc/examples/urma_example/CMakeLists.txt b/src/coro_rpc/examples/urma_example/CMakeLists.txt new file mode 100644 index 000000000..6a83f30ce --- /dev/null +++ b/src/coro_rpc/examples/urma_example/CMakeLists.txt @@ -0,0 +1,4 @@ +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/output/examples/coro_rpc) +add_executable(coro_rpc_urma_example + urma_example.cpp + ) diff --git a/src/coro_rpc/examples/urma_example/urma_example.cpp b/src/coro_rpc/examples/urma_example/urma_example.cpp new file mode 100644 index 000000000..428ead974 --- /dev/null +++ b/src/coro_rpc/examples/urma_example/urma_example.cpp @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2025, Alibaba Group Holding Limited; + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "async_simple/coro/SyncAwait.h" +#include "ylt/coro_io/client_pool.hpp" +#include "ylt/coro_io/io_context_pool.hpp" +#include "ylt/coro_io/urma/urma_device.hpp" +#include "ylt/coro_io/urma/urma_socket.hpp" +#include "ylt/coro_rpc/coro_rpc_client.hpp" +#include "ylt/coro_rpc/coro_rpc_server.hpp" +#include "ylt/coro_rpc/impl/coro_rpc_client.hpp" +#include "ylt/easylog.hpp" +#include "ylt/easylog/record.hpp" + +using namespace coro_rpc; +using namespace async_simple::coro; +using namespace std::chrono_literals; + +std::string_view echo(std::string_view data) { return data; } + +// The basic example about how to start a rpc connection over URMA. +void basic_example() { + coro_rpc_client client; + coro_rpc_server server; + + server.init_urma(); + server.register_handler(); + auto future = server.async_start(); + if (future.hasResult()) { + ELOG_ERROR << future.result().value().message(); + return; + } + + // Client connects to server - URMA is auto-initialized from server config + auto ec = syncAwait(client.connect(std::string{server.address()} + ":" + + std::to_string(server.port()))); + if (ec) { + ELOG_ERROR << ec.message(); + return; + } + + std::string data(1024 * 1024 * 10, 'A'); + auto result = syncAwait(client.call(data)); + if (result != data) { + ELOG_ERROR << "echo data err"; + } + else { + ELOG_INFO << "echo ok!"; + } + server.stop(); + return; +} + +// This example is about how to configure the detail URMA option. +void set_option() { + /* init global device, should call before any other call */ + coro_io::get_global_urma_device({ + .dev_name = "" /*URMA device name, default is empty, which means choice + the first URMA device*/ + , + .buffer_pool_config = + { + .buffer_size = 256 * 1024, /*buffer size*/ + .max_memory_usage = 20 * 1024 * 1024, /*max memory usage*/ + .idle_timeout = 5s, + }, + .eid_index = 0 /*EID index to use*/}); + + coro_rpc_client client; + coro_rpc_client::config conf; + auto urma_config = coro_io::urma_socket_t::config_t{ + .recv_buffer_cnt = 4, // buffer cnt of recv queue + .send_buffer_cnt = 4, // buffer cnt of send queue + .buffer_size = 256 * 1024, // buffer size 256KB + .device_name = "", // empty means auto-select + .eid_index = 0 // EID index + }; + conf.socket_config = urma_config; + [[maybe_unused]] bool _ = client.init_config(conf); + + coro_rpc_server server; + server.init_urma(urma_config); + + server.register_handler(); + auto future = server.async_start(); + if (future.hasResult()) { + ELOG_ERROR << future.result().value().message(); + return; + } + + auto ec = syncAwait(client.connect(std::string{server.address()} + ":" + + std::to_string(server.port()))); + if (ec) { + ELOG_ERROR << ec.message(); + return; + } + + std::string data(1024 * 1024 * 10, 'A'); + auto result = syncAwait(client.call(data)); + if (result != data) { + ELOG_ERROR << "echo data err"; + } + else { + ELOG_INFO << "echo ok!"; + } + server.stop(); +} + +int main() { + easylog::logger<>::instance().set_min_severity(easylog::Severity::INFO); + set_option(); + basic_example(); + return 0; +} From 772614a906380f8e91071c2222825e3075313174 Mon Sep 17 00:00:00 2001 From: lixu Date: Thu, 4 Jun 2026 20:05:03 +0800 Subject: [PATCH 003/129] fix(coro_rpc): add urma_example subdirectory and fix urma_socket header include --- include/ylt/coro_io/urma/urma_socket.hpp | 16 ++-------------- src/coro_rpc/examples/CMakeLists.txt | 3 +++ 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/include/ylt/coro_io/urma/urma_socket.hpp b/include/ylt/coro_io/urma/urma_socket.hpp index c375f5ec2..f38924f57 100644 --- a/include/ylt/coro_io/urma/urma_socket.hpp +++ b/include/ylt/coro_io/urma/urma_socket.hpp @@ -32,21 +32,9 @@ #include "ylt/coro_io/coro_io.hpp" #include "ylt/easylog.hpp" #include "ylt/struct_pack.hpp" +#include "ylt/urma/urma_api.h" +#include "ylt/urma/urma_types.h" -// URMA forward declarations -struct urma_context_t; -struct urma_jfc_t; -struct urma_jfr_t; -struct urma_jetty_t; -struct urma_target_jetty_t; -struct urma_target_seg_t; -struct urma_device_t; -struct urma_eid_t; -struct urma_cr_t; -struct urma_jfs_wr; -struct urma_seg_cfg_t; -struct urma_token_t; -struct urma_init_attr_t; enum class urma_transport_type_t : int; diff --git a/src/coro_rpc/examples/CMakeLists.txt b/src/coro_rpc/examples/CMakeLists.txt index 7180da6b6..177240a98 100644 --- a/src/coro_rpc/examples/CMakeLists.txt +++ b/src/coro_rpc/examples/CMakeLists.txt @@ -4,6 +4,9 @@ add_subdirectory(file_transfer) if (YLT_HAVE_IBVERBS) add_subdirectory(rdma_example) endif() +if (YLT_ENABLE_URMA) + add_subdirectory(urma_example) +endif() if (CORO_RPC_USE_OTHER_RPC) add_subdirectory(user_defined_rpc_protocol/rest_rpc) From b60601048ae33d19d38bb6ad0b915943dd2a7907 Mon Sep 17 00:00:00 2001 From: lixu Date: Thu, 4 Jun 2026 20:07:38 +0800 Subject: [PATCH 004/129] add urma head --- include/ylt/urma/urma_api.h | 1147 +++++++++++++++++++++++ include/ylt/urma/urma_cmd.h | 1322 ++++++++++++++++++++++++++ include/ylt/urma/urma_opcode.h | 260 ++++++ include/ylt/urma/urma_perf.h | 72 ++ include/ylt/urma/urma_provider.h | 412 +++++++++ include/ylt/urma/urma_types.h | 1430 +++++++++++++++++++++++++++++ include/ylt/urma/urma_types_str.h | 243 +++++ 7 files changed, 4886 insertions(+) create mode 100644 include/ylt/urma/urma_api.h create mode 100644 include/ylt/urma/urma_cmd.h create mode 100644 include/ylt/urma/urma_opcode.h create mode 100644 include/ylt/urma/urma_perf.h create mode 100644 include/ylt/urma/urma_provider.h create mode 100644 include/ylt/urma/urma_types.h create mode 100644 include/ylt/urma/urma_types_str.h diff --git a/include/ylt/urma/urma_api.h b/include/ylt/urma/urma_api.h new file mode 100644 index 000000000..3f9a84425 --- /dev/null +++ b/include/ylt/urma/urma_api.h @@ -0,0 +1,1147 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) Huawei Technologies Co., Ltd. 2021-2025. All rights reserved. + * Description: URMA API + * Author: Ouyang changchun, Bojie Li, Yan Fangfang, Qian Guoxin + * Create: 2021-07-13 + * Note: + * History: 2021-07-13 Create File + */ +#ifndef URMA_API_H +#define URMA_API_H + +#include +#include + +#include "urma_types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Init urma environment. + * @param[in] [Required] conf: urma init attr, a random uasid will be assigned when conf is null. + * Return: 0 on success, other value on error + */ +urma_status_t urma_init(urma_init_attr_t *conf); + +/** + * Un-init urma environment, it will free uasid. + * Return: 0 on success, other value on error + */ +urma_status_t urma_uninit(void); + +/* Device Manage API */ + /** + * Get device list. + * @param[out] num_devices: number of urma device; + * Return: pointer array of urma_device; NULL means no device returned; + * Note: urma_free_device_list() needs to be called to free memory; + */ +urma_device_t **urma_get_device_list(int *num_devices); + +/** +* free device list. +* @param[in] [Required] device_list: pointer array of urma_device,return value of urma_get_device_list. + Can be called after using urma_device list; +* Return: void; +*/ +void urma_free_device_list(urma_device_t **device_list); + +/** +* Get eid list. +* @param[in] [Required] dev: device pointer +* @param[out] cnt: Return the number of valid eids; +* Return: If it succeeds, it will return the eid_info array pointer, and the number of elements +* is cnt; if it fails, it will return NULL; it will be released by the user calling +*/ +urma_eid_info_t *urma_get_eid_list(urma_device_t *dev, uint32_t *cnt); + +/** +* free eid list. +* @param[in] [Required] eid_list: The eid array pointer to be released +* Return: void; +*/ +void urma_free_eid_list(urma_eid_info_t *eid_list); + +/** + * Get device by device name. + * @param[in] [Required] dev_name: device's name; + * Return: urma_device; NULL means no device returned; + */ +urma_device_t *urma_get_device_by_name(char *dev_name); + + /** + * Get device by device eid. + * @param[in] [Required] eid: device's eid; + * @param[in] [Required] type: device's transport type; + * Return: urma_device; NULL means no device returned; + */ +urma_device_t *urma_get_device_by_eid(urma_eid_t eid, urma_transport_type_t type); + +/** + * Query the attributes and capabilities of urma devices. + * @param[in] [Required] dev: urma_device; + * @param[out] dev_attr: Return device attributes, user needs to allocate and free the memory; + * Return: 0 on success, other value on error + */ +urma_status_t urma_query_device(urma_device_t *dev, urma_device_attr_t *dev_attr); + +/** + * Create an urma context on the urma device. + * @param[in] [Required] dev: urma device, by get_device apis. + * @param[in] [Required] eid_index: device's eid index. + * Return urma context pointer on success, NULL on error. + */ +urma_context_t *urma_create_context(urma_device_t *dev, uint32_t eid_index); + +/** + * Delete the created urma context. + * @param[in] [Required] ctx: handle of the created context. + * Return: 0 on success, other value on error + */ +urma_status_t urma_delete_context(urma_context_t *ctx); + +/** + * Set option of urma context. + * @param[in] [Required] ctx: handle of the created context. + * Return: 0 on success, other value on error + */ +urma_status_t urma_set_context_opt(urma_context_t *ctx, urma_opt_name_t opt_name, const void *opt_value, + size_t opt_len); + +/** + * Create a jetty for completion (jfc). + * @param[in] [Required] ctx: the urma context created before; + * @param[in] [Required] jfc_cfg: configuration including: depth, flag, jfce, user context; + * Return: the handle of created jfc, not NULL on success; NULL on error + */ +urma_jfc_t *urma_create_jfc(urma_context_t *ctx, urma_jfc_cfg_t *jfc_cfg); + +/** + * Modify JFC attributes. + * @param[in] [Required] jfc: specify JFC; + * @param[in] [Required] attr: attributes to be modified; + * Return: 0 on success, other value on error + */ +urma_status_t urma_modify_jfc(urma_jfc_t *jfc, urma_jfc_attr_t *attr); + +/** + * Delete the created jfc. + * @param[in] [Required] jfc: handle of the created jfc; + * Return: 0 on success, other value on error + */ +urma_status_t urma_delete_jfc(urma_jfc_t *jfc); + +/** + * Alloc a jfc. + * @param[in] [Required] urma_ctx: the urma context created before; + * @param[in] [Required] cfg: configuration including: depth, flag, jfce, user context; + * @param[out] [Required] jfc: handle of the allocated jfc; + * Return: 0 on success, other value on error + */ +urma_status_t urma_alloc_jfc(urma_context_t *urma_ctx, urma_jfc_cfg_t *cfg, urma_jfc_t **jfc); + +/** + * Set the opt of jfc. + * @param[in] [Required] jfc: the jfc allocated before; + * @param[in] [Required] opt: the opt to change cfg of jfc; + * @param[in] [Required] len: the len of the opt value(byte); + * @param[in] [Required] buf: the buffer containing the value to set; + * Return: 0 on success, other value on error + */ +urma_status_t urma_set_jfc_opt(urma_jfc_t *jfc, uint64_t opt, void *buf, uint32_t len); + +/** + * Active the allocated jfc. + * @param[in] [Required] jfc: the jfc allocated before; + * Return: 0 on success, other value on error + */ +urma_status_t urma_active_jfc(urma_jfc_t *jfc); + +/** + * Get the opt of jfc. + * @param[in] [Required] jfc: the jfc allocated before; + * @param[in] [Required] opt: the opt to change cfg of jfc; + * @param[in] [Required] len: the len of the opt value(byte); + * @param[out] [Required] buf: the buffer to store the value; + * Return: 0 on success, other value on error + */ +urma_status_t urma_get_jfc_opt(urma_jfc_t *jfc, uint64_t opt, void *buf, uint32_t len); + +/** + * Deactive the created jfc. + * @param[in] [Required] jfc: the jfc actived before; + * Return: 0 on success, other value on error + */ +urma_status_t urma_deactive_jfc(urma_jfc_t *jfc); + +/** + * Free the created jfc. + * @param[in] [Required] jfc: the jfc allocated before; + * After free, the jfc pointer is no longer allowed to be accessed. + * Return: 0 on success, other value on error + */ +urma_status_t urma_free_jfc(urma_jfc_t *jfc); + +/** + * Delete the created jfc in a batch. + * @param[in] [Required] jfc_arr: the array of the jfc pointer; + * @param[in] [Required] jfc_num: array length; + * @param[out] [Required] bad_jfc: the address of the first failed jfc pointer; + * Return: 0 on success, EINVAL on invalid parameter, other value on other batch + * delete errors. + * If delete error happens(except invalid parameter), stop at the first failed + * jfc and return, these jfc before the failed jfc will be deleted normally. + */ +urma_status_t urma_delete_jfc_batch(urma_jfc_t **jfc_arr, int jfc_num, urma_jfc_t **bad_jfc); + +/** + * Create a jetty for send (jfs). + * @param[in] [Required] ctx: the urma context created before; + * @param[in] [Required] jfs_cfg: address to pu the jfs config; + * Return: the handle of created jfs, not NULL on success, NULL on error + */ +urma_jfs_t *urma_create_jfs(urma_context_t *ctx, urma_jfs_cfg_t *jfs_cfg); + +/** + * Modify a jetty for send (jfs). + * @param[in] [Required] jfs: the jfs created before; + * @param[in] [Required] attr: attributes to be modified; + * Return: 0 on success, other value on error + */ +urma_status_t urma_modify_jfs(urma_jfs_t *jfs, urma_jfs_attr_t *attr); + +/** + * Query a jetty for send (jfs). + * @param[in] [Required] jfs: the jfs created before; + * @param[out] [Required] cfg: config of jfs; + * @param[out] [Required] attr: attributes of jfs; + * Return: 0 on success, other value on error + */ +urma_status_t urma_query_jfs(urma_jfs_t *jfs, urma_jfs_cfg_t *cfg, urma_jfs_attr_t *attr); + +/** + * Delete the created jfs. + * @param[in] [Required] jfs: the jfs created before; + * Return: 0 on success, other value on error + */ +urma_status_t urma_delete_jfs(urma_jfs_t *jfs); + +/** + * Delete the created jfs in a batch. + * @param[in] [Required] jfs_arr: the array of the jfs pointer; + * @param[in] [Required] jfs_num: array length; + * @param[out] [Required] bad_jfs: the address of the first failed jfs pointer; + * Return: 0 on success, EINVAL on invalid parameter, other value on other batch + * delete errors. + * If delete error happens(except invalid parameter), stop at the first failed + * jfs and return, these jfs before the failed jfs will be deleted normally. + */ +urma_status_t urma_delete_jfs_batch(urma_jfs_t **jfs_arr, int jfs_num, urma_jfs_t **bad_jfs); + +/** + * Poll the CRs for all the WRs that posted to JFS, but are not completed. + * Call the API after modify JFS to error, or polled a suspened done CR. + * CRs with status of URMA_CR_WR_FLUSH_ERR will be returned on success. + * @param[in] [Required] jfs: the jfs created before; + * @param[in] [Required] cr_cnt: Number of CR expected to be received.; + * @param[out] [Required] cr: Address for storing CR; + * Return: the number of CR returned, 0 means no CR returned, -1 on error + */ +int urma_flush_jfs(urma_jfs_t *jfs, int cr_cnt, urma_cr_t *cr); + +/** + * Alloc a jfs. + * @param[in] [Required] urma_ctx: the urma context created before; + * @param[in] [Required] cfg: configuration including: depth, flag, jfce, user context; + * @param[out] [Required] jfs: handle of the allocated jfs; + * Return: 0 on success, other value on error + */ +urma_status_t urma_alloc_jfs(urma_context_t *urma_ctx, urma_jfs_cfg_t *cfg, urma_jfs_t **jfs); + +/** + * Set the opt of jfs. + * @param[in] [Required] jfs: the jfs allocated before; + * @param[in] [Required] opt: the opt to change cfg of jfs; + * @param[in] [Required] len: the len of the opt value(byte); + * @param[in] [Required] buf: the buffer to store the value; + * Return: 0 on success, other value on error + */ +urma_status_t urma_set_jfs_opt(urma_jfs_t *jfs, uint64_t opt, void *buf, uint32_t len); + +/** + * Active the created jfs. + * @param[in] [Required] jfs: the jfs allocated before; + * Return: 0 on success, other value on error + */ +urma_status_t urma_active_jfs(urma_jfs_t *jfs); + +/** + * Get the opt of jfs. + * @param[in] [Required] jfs: the jfs allocated before; + * @param[in] [Required] opt: the opt to change cfg of jfs; + * @param[in] [Required] len: the len of the opt value(byte); + * @param[out] [Required] buf: the buffer to store the value; + * Return: 0 on success, other value on error + */ +urma_status_t urma_get_jfs_opt(urma_jfs_t *jfs, uint64_t opt, void *buf, uint32_t len); + +/** + * Deactive the created jfs. + * @param[in] [Required] jfs: the jfs actived before; + * Return: 0 on success, other value on error + */ +urma_status_t urma_deactive_jfs(urma_jfs_t *jfs); + +/** + * Free the created jfs. + * @param[in] [Required] jfs: the jfs allocated before; + * After free, the jfs pointer is no longer allowed to be accessed. + * Return: 0 on success, other value on error + */ +urma_status_t urma_free_jfs(urma_jfs_t *jfs); + + /** + * Create a jetty for receive (jfr). + * @param[in] [Required] ctx: the urma context created before; + * @param[in] [Required] jfr_cfg: address to put the jfr config; + * Return: the handle of created jfr, not NULL on success, NULL on error + */ +urma_jfr_t *urma_create_jfr(urma_context_t *ctx, urma_jfr_cfg_t *jfr_cfg); + +/** + * Modify JFR attributes. + * @param[in] [Required] jfr: specify JFR; + * @param[in] [Required] attr: attributes to be modified; + * Return: 0 on success, other value on error + */ +urma_status_t urma_modify_jfr(urma_jfr_t *jfr, urma_jfr_attr_t *attr); + +/** + * Query a jetty for recv(jfr). + * @param[in] [Required] jfr: the jfr created before; + * @param[out] [Required] cfg: config of jfr; + * @param[out] [Required] attr: attributes of jfr; + * Return: 0 on success, other value on error + */ +urma_status_t urma_query_jfr(urma_jfr_t *jfr, urma_jfr_cfg_t *cfg, urma_jfr_attr_t *attr); + +/** + * Delete the created jfr. + * @param[in] [Required] jfr: the jfr created before; + * Return: 0 on success, other value on error + */ +urma_status_t urma_delete_jfr(urma_jfr_t *jfr); + +/** + * Delete the created jfr in a batch. + * @param[in] [Required] jfr_arr: the array of the jfr pointer; + * @param[in] [Required] jfr_num: array length; + * @param[out] [Required] bad_jfr: the address of the first failed jfr pointer; + * Return: 0 on success, EINVAL on invalid parameter, other value on other batch + * delete errors. + * If delete error happens(except invalid parameter), stop at the first failed + * jfr and return, these jfr before the failed jfr will be deleted normally. + */ +urma_status_t urma_delete_jfr_batch(urma_jfr_t **jfr_arr, int jfr_num, urma_jfr_t **bad_jfr); + +/** + * Import a remote jfr to local node. + * @param[in] [Required] ctx: the urma context created before; + * @param[in] [Required] rjfr: the information of remote jfr to import into user node, trans_mode required, + * trans_mode same to create_jfr trans_mode; + * @param[in] [Required] token_value: token to put into output jetty/protection table; + * Return: the address of target jfr, not NULL on success, NULL on error + */ +urma_target_jetty_t *urma_import_jfr(urma_context_t *ctx, urma_rjfr_t *rjfr, urma_token_t *token_value); + +/** + * Import a remote jfr to local node by control plane. + * Note: trans_mode from rjfr should be the same as the trans_mode of get_tp_list, + * users should obey this rule in case of unexpected errors. + * @param[in] [Required] ctx: the urma context created before; + * @param[in] [Required] rjfr: the information of remote jfr to import into user node, trans_mode required, + * trans_mode same to create_jfr trans_mode; + * @param[in] [Required] token_value: token to put into output jetty/protection table; + * @param[in] [Required] cfg: tp active configuration to exchange with target; + * Return: the address of target jfr, not NULL on success, NULL on error + */ +urma_target_jetty_t *urma_import_jfr_ex(urma_context_t *ctx, urma_rjfr_t *rjfr, urma_token_t *token_value, + urma_import_jfr_ex_cfg_t *cfg); + +/** + * Unimport the imported remote jfr. + * @param[in] [Required] target_jfr: the target jfr to unimport; + * Return: 0 on success, other value on error + */ +urma_status_t urma_unimport_jfr(urma_target_jetty_t *target_jfr); + +/** + * Advise jfr: construct the transport channel for jfs and remote jfr. + * @param[in] [Required] jfs: jfs to use to construct the transport channel; + * @param[in] [Required] tjfr: target jfr information including full qualified jfr id; + * Return: 0 on success, URMA_EEXIST if the jfr has been advised, other value on error + */ +urma_status_t urma_advise_jfr(urma_jfs_t *jfs, urma_target_jetty_t *tjfr); + +/** + * Async API for urma_advise_jfr + * Advise jfr: construct the transport channel for jfs and remote jfr. + * @param[in] [Required] jfs: jfs to use to construct the transport channel; + * @param[in] [Required] tjfr: target jfr information including full qulified jfr id; + * @param[in] [Required] cb_func: user defined callback function. + * @param[in] [Required] cb_arg: user defined arguments for the callback function. + * Return: 0 on success, URMA_EEXIST if the jfr has been advised, other value on error. + * Note: User must define callback function to handle result, + * as the async respone will call the cb_func and pass the result to it. + */ +urma_status_t urma_advise_jfr_async(urma_jfs_t *jfs, urma_target_jetty_t *tjfr, urma_advise_async_cb_func cb_fun, + void *cb_arg); + +/** + * Unadvise jfr: disconnect the transport channel for jfs and remote jfr. Optional API for optimization + * @param[in] [Required] jfs: jfs to use to construct the transport channel; + * @param[in] [Required] tjfr: target jfr information including full qualified jfr id; + * Return: 0 on success, other value on error + */ +urma_status_t urma_unadvise_jfr(urma_jfs_t *jfs, urma_target_jetty_t *tjfr); + +/** + * Alloc a jfr. + * @param[in] [Required] urma_ctx: the urma context created before; + * @param[in] [Required] cfg: configuration including: depth, flag, jfce, user context; + * @param[out] [Required] jfr: handle of the allocated jfr; + * Return: 0 on success, other value on error + */ +urma_status_t urma_alloc_jfr(urma_context_t *urma_ctx, urma_jfr_cfg_t *cfg, urma_jfr_t **jfr); + +/** + * Set the opt of jfr. + * @param[in] [Required] jfr: handle of the allocated jfr; + * @param[in] [Required] opt: the opt to change cfg of jfr; + * @param[in] [Required] len: the len of the opt value(byte); + * @param[in] [Required] buf: the buffer to store the value; + * Return: 0 on success, other value on error + */ +urma_status_t urma_set_jfr_opt(urma_jfr_t *jfr, uint64_t opt, void *buf, uint32_t len); + +/** + * Active the allocated jfr. + * @param[in] [Required] jfr: handle of the allocated jfr; + * Return: 0 on success, other value on error + */ +urma_status_t urma_active_jfr(urma_jfr_t *jfr); + +/** + * Get the opt of jfr. + * @param[in] [Required] jfr: handle of the allocated jfr; + * @param[in] [Required] opt: the opt to change cfg of jfr; + * @param[in] [Required] len: the len of the opt value(byte); + * @param[out] [Required] buf: the buffer to store the value; + * Return: 0 on success, other value on error + */ +urma_status_t urma_get_jfr_opt(urma_jfr_t *jfr, uint64_t opt, void *buf, uint32_t len); + +/** + * Deactive the actived jfr. + * @param[in] [Required] jfr: handle of the allocated jfr; + * Return: 0 on success, other value on error + */ +urma_status_t urma_deactive_jfr(urma_jfr_t *jfr); + +/** + * Free the allocated jfr. + * @param[in] [Required] jfr: handle of the allocated jfr; + * After free, the jfr pointer is no longer allowed to be accessed. + * Return: 0 on success, other value on error + */ +urma_status_t urma_free_jfr(urma_jfr_t *jfr); + +/** + ******************** Beginning of URMA JETTY APIs *************************** + */ + +/** + * Create jetty, which is a pair of jfs and jfr + * @param[in] [Required] ctx: the urma context created before; + * @param[in] [Required] jetty_cfg: pointer of the jetty config; + * Return: the handle of created jetty, not NULL on success, NULL on error + */ +urma_jetty_t *urma_create_jetty(urma_context_t *ctx, urma_jetty_cfg_t *jetty_cfg); + +/** + * Modify jetty attributes. + * @param[in] [Required] jetty: specify jetty; + * @param[in] [Required] attr: attributes to be modified; + * Return: 0 on success, other value on error + */ +urma_status_t urma_modify_jetty(urma_jetty_t *jetty, urma_jetty_attr_t *attr); + +/** + * Query jetty attributes. + * @param[in] [Required] jetty: specify jetty; + * @param[out] [Required] cfg: cconfig to query; + * @param[out] [Required] attr: attributes to query; + * Return: 0 on success, other value on error + */ +urma_status_t urma_query_jetty(urma_jetty_t *jetty, urma_jetty_cfg_t *cfg, urma_jetty_attr_t *attr); + +/** + * Delete the created jetty. + * @param[in] [Required] jetty: the jetty created before; + * Return: 0 on success, other value on error + */ +urma_status_t urma_delete_jetty(urma_jetty_t *jetty); + +/** + * Delete the created jetty in a batch. + * @param[in] [Required] jetty_arr: the array of the jetty pointer; + * @param[in] [Required] jetty_num: array length; + * @param[out] [Required] bad_jetty: the address of the first failed jetty pointer; + * Return: 0 on success, EINVAL on invalid parameter, other value on other batch + * delete errors. + * If delete error happens(except invalid parameter), stop at the first failed + * jetty and return, these jetty before the failed jetty will be deleted normally. + */ +urma_status_t urma_delete_jetty_batch(urma_jetty_t **jetty_arr, int jetty_num, urma_jetty_t **bad_jetty); + +/** + * Import a remote jetty. + * @param[in] [Required] ctx: the urma context created before; + * @param[in] [Required] rjetty: information of remote jetty to import, including jetty id and trans_mode, + * trans_mode same to create_jetty trans_mode; + * @param[in] [Required] token_value: token to put into output jetty protection table; + * Return: the address of target jetty, not NULL on success, NULL on error + */ +urma_target_jetty_t *urma_import_jetty(urma_context_t *ctx, urma_rjetty_t *rjetty, urma_token_t *token_value); + +/** + * Import a remote jetty by control plane. + * Note: trans_mode from rjetty should be the same as the trans_mode of get_tp_list, + * users should obey this rule in case of unexpected errors. + * @param[in] [Required] ctx: the urma context created before; + * @param[in] [Required] rjetty: information of remote jetty to import, including jetty id and trans_mode, + * trans_mode same to create_jetty trans_mode; + * @param[in] [Required] token_value: token to put into output jetty protection table; + * @param[in] [Required] cfg: tp active configuration to exchange with target; + * Return: the address of target jetty, not NULL on success, NULL on error + */ +urma_target_jetty_t *urma_import_jetty_ex(urma_context_t *ctx, urma_rjetty_t *rjetty, urma_token_t *token_value, + urma_import_jetty_ex_cfg_t *cfg); + +/** + * Unimport the imported remote jetty. + * @param[in] [Required] tjetty: the target jetty to unimport; + * Return: 0 on success, other value on error + */ +urma_status_t urma_unimport_jetty(urma_target_jetty_t *tjetty); + +/** + * Advise jetty: construct the transport channel between local jetty and remote jetty. + * @param[in] [Required] jetty: local jetty to construct the transport channel; + * @param[in] [Required] tjetty: target jetty imported before; + * Return: 0 on success, URMA_EEXIST if the jetty has been advised, other value on error + * Note: A local jetty can be advised with several remote jetties. A connectionless jetty is free to call the adivse API + */ +/* todo: available after implementing URMA_TM_RM(IB_RC) */ +urma_status_t urma_advise_jetty(urma_jetty_t *jetty, urma_target_jetty_t *tjetty); + +/** + * Unadvise jetty: deconstruct the transport channel between local jetty and remote jetty. + * @param[in] [Required] jetty: local jetty to deconstruct the transport channel; + * @param[in] [Required] tjetty: target jetty imported before; + * Return: 0 on success, other value on error + */ +/* todo: available after implementing URMA_TM_RM(IB_RC) */ +urma_status_t urma_unadvise_jetty(urma_jetty_t *jetty, urma_target_jetty_t *tjetty); + +/** + * Bind jetty: construct the transport channel between local jetty and remote jetty. + * @param[in] [Required] jetty: local jetty to construct the transport channel; + * @param[in] [Required] tjetty: target jetty imported before; + * Return: 0 on success, URMA_EEXIST if the jetty has been binded, other value on error + * Note: A local jetty can be binded with only one remote jetty. Only supported by jetty under URMA_TM_RC. + */ +urma_status_t urma_bind_jetty(urma_jetty_t *jetty, urma_target_jetty_t *tjetty); + +/** + * Bind jetty: construct the transport channel between local jetty and remote jetty by control plane. + * Note: trans_mode from tjetty should be the same as the trans_mode of get_tp_list, + * users should obey this rule in case of unexpected errors. + * @param[in] [Required] jetty: local jetty to construct the transport channel; + * @param[in] [Required] tjetty: target jetty imported before; + * Return: 0 on success, URMA_EEXIST if the jetty has been binded, other value on error; + * @param[in] [Required] cfg: tp active configuration to exchange with target; + * Note: A local jetty can be binded with only one remote jetty. Only supported by jetty under URMA_TM_RC. + */ +urma_status_t urma_bind_jetty_ex(urma_jetty_t *jetty, urma_target_jetty_t *tjetty, urma_bind_jetty_ex_cfg_t *cfg); + +/** + * Unbind jetty: deconstruct the transport channel between local jetty and remote jetty. + * @param[in] [Required] jetty: local jetty to deconstruct the transport channel; + * Return: 0 on success, other value on error + */ +urma_status_t urma_unbind_jetty(urma_jetty_t *jetty); + +/** + * Poll the CRs for all the WRs that posted to Jetty, but are not completed. + * Call the API after modify Jetty to error, or polled a suspened done CR. + * CRs with status of URMA_CR_WR_FLUSH_ERR will be returned on success. + * @param[in] [Required] jetty: the jetty created before; + * @param[in] [Required] cr_cnt: Number of CR expected to be received.; + * @param[out] [Required] cr: Address for storing CR; + * Return: the number of CR returned, 0 means no CR returned, -1 on error + */ +int urma_flush_jetty(urma_jetty_t *jetty, int cr_cnt, urma_cr_t *cr); + +/** + * Import a remote jetty asynchronously. + * @param[in] [Required] notifier: data structure used for sensing asynchronous link establishment results; + * @param[in] [Required] rjetty: information of remote jetty to import, including jetty id and trans_mode, + * trans_mode same to create_jetty trans_mode; + * @param[in] [Required] token_value: token to put into output jetty protection table; + * @param[in] [Required] user_ctx: user_ctx create by user; + * @param[in] [Required] timeout: task timeout set by user (milliseconds); + * Return: the address of target jetty, not NULL on success, NULL on error + */ +urma_target_jetty_t *urma_import_jetty_async(urma_notifier_t *notifier, const urma_rjetty_t *rjetty, + const urma_token_t *token_value, uint64_t user_ctx, int timeout); + +/** + * Unimport the imported remote jetty asynchronously. + * @param[in] [Required] tjetty: the target jetty to unimport; + * Return: 0 on success, other value on error + */ +urma_status_t urma_unimport_jetty_async(urma_target_jetty_t *tjetty); + +/** + * Bind jetty asynchronously: construct the transport channel between local jetty and remote jetty. + * @param[in] [Required] notifier: data structure used for sensing asynchronous link establishment results; + * @param[in] [Required] jetty: local jetty to construct the transport channel; + * @param[in] [Required] tjetty: target jetty imported before; + * @param[in] [Required] user_ctx: user_ctx create by user; + * @param[in] [Required] timeout: task timeout set by user (milliseconds); + * Return: 0 on success, URMA_EEXIST if the jetty has been binded, other value on error + * Note: A local jetty can be binded with only one remote jetty. Only supported by jetty under URMA_TM_RC. + */ +urma_status_t urma_bind_jetty_async(urma_notifier_t *notifier, urma_jetty_t *jetty, urma_target_jetty_t *tjetty, + uint64_t user_ctx, int timeout); + +/** + * Unbind jetty: deconstruct the transport channel between local jetty and remote jetty asynchronously. + * @param[in] [Required] jetty: local jetty to deconstruct the transport channel; + * Return: 0 on success, other value on error + */ +urma_status_t urma_unbind_jetty_async(urma_jetty_t *jetty); + +/** + * Create a data structure for sensing asynchronous link establishment results. + * @param[in] [Required] ctx: the urma context created before; + * Return: the address of urma notifier, not NULL on success, NULL on error + */ +urma_notifier_t *urma_create_notifier(urma_context_t *ctx); + +/** + * Delete the created notifier. + * @param[in] [Required] notifier: data structure used for sensing asynchronous link establishment results; + * Return: 0 on success, other value on error + */ +urma_status_t urma_delete_notifier(urma_notifier_t *notifier); + +/** + * Alloc a jetty. + * @param[in] [Required] urma_ctx: the urma context created before; + * @param[in] [Required] jetty_cfg: configuration including: depth, flag, jfce, user context; + * @param[out] [Required] jetty: handle of the allocated jetty; + * Return: 0 on success, other value on error + */ +urma_status_t urma_alloc_jetty(urma_context_t *urma_ctx, urma_jetty_cfg_t *cfg, urma_jetty_t **jetty); + +/** + * Set the opt of jetty. + * @param[in] [Required] jetty: handle of the allocated jetty; + * @param[in] [Required] opt: the opt to change cfg of jetty; + * @param[in] [Required] len: the len of the opt value(byte); + * @param[in] [Required] buf: the buffer to store the value; + * Return: 0 on success, other value on error + */ +urma_status_t urma_set_jetty_opt(urma_jetty_t *jetty, uint64_t opt, void *buf, uint32_t len); + +/** + * Active the allocated jetty. + * @param[in] [Required] jetty: handle of the allocated jetty; + * Return: 0 on success, other value on error + */ +urma_status_t urma_active_jetty(urma_jetty_t *jetty); + +/** + * Get the opt of jetty. + * @param[in] [Required] jetty: handle of the allocated jetty; + * @param[in] [Required] opt: the opt to change cfg of jetty; + * @param[in] [Required] len: the len of the opt value(byte); + * @param[out] [Required] buf: the buffer to store the value; + * Return: 0 on success, other value on error + */ +urma_status_t urma_get_jetty_opt(urma_jetty_t *jetty, uint64_t opt, void *buf, uint32_t len); + +/** + * Deactive the actived jetty. + * @param[in] [Required] jetty: handle of the allocated jetty; + * Return: 0 on success, other value on error + */ +urma_status_t urma_deactive_jetty(urma_jetty_t *jetty); + +/** + * Free the allocated jetty. + * @param[in] [Required] jetty: handle of the allocated jetty; + * After free, the jfc pointer is no longer allowed to be accessed. + * Return: 0 on success, other value on error + */ +urma_status_t urma_free_jetty(urma_jetty_t *jetty); + +/** + * Wait for asynchronous event notification to obtain the connection establishment result. + * @param[in] [Required] notifier: data structure used for sensing asynchronous link establishment results; + * @param[in] [Required] cnt: expected number of target jetty to return; + * @param[in] [Required] timeout: max time to wait (milliseconds), + timeout = 0: return immediately even if no events are ready, + timeout = -1: an infinite timeout; + * @param[out] [Required] notify: created by user to store target jetty results; + * Return: the number of target jetty returned, 0 means no target jetty returned, -1 on error + */ +int urma_wait_notify(urma_notifier_t *notifier, uint32_t cnt, urma_notify_t *notify, int timeout); + +/** + * This interface is no longer functional and will be removed later. + * Keep parameter checks to ensure the function works as before. + */ +urma_status_t urma_ack_notify(urma_context_t *ctx, uint32_t cnt, urma_notify_t *notify); + +/** + ******************** Beginning of URMA JETTY GROUP APIs *************************** + */ + +/** + * Create jetty group + * @param[in] [Required] ctx: the urma context created before; + * @param[in] [Required] cfg: pointer of the jetty group config; + * Return: the handle of created jetty group, not NULL on success, NULL on error + */ +urma_jetty_grp_t *urma_create_jetty_grp(urma_context_t *ctx, urma_jetty_grp_cfg_t *cfg); + +/** + * Destroy jetty group + * @param[in] [Required] jetty_grp: the Jetty group created before; + * Return: 0 on success, other value on error + */ +urma_status_t urma_delete_jetty_grp(urma_jetty_grp_t *jetty_grp); + +/** + * Create a jfce + * @param[in] [Required] ctx: the urma context created before; + * Return: the address of created jfce, not NULL on success, NULL on error + */ +urma_jfce_t *urma_create_jfce(urma_context_t *ctx); + +/** + * Delete a jfce + * @param[in] [Required] jfce: the jfce to be deleted; + * Return: 0 on success, other value on error + */ +urma_status_t urma_delete_jfce(urma_jfce_t *jfce); + +/** + * Get asyn event. + * @param[in] [Required] ctx: handle of the created urma context; + * @param[out] [Required] event: the address to put event + * Return: 0 on success, other value on error + */ +urma_status_t urma_get_async_event(urma_context_t *ctx, urma_async_event_t *event); + +/** + * Ack asyn event. + * @param[in] [Required] event: the address to ack event; + * Return: void + */ +void urma_ack_async_event(urma_async_event_t *event); + +/** + * Request to assign a token id. token id is used to register the segment with the protection table. + * @param[in] [Required] ctx: specifies the urma context. + * Return: pointer to key id on success, NULL on error. + */ +urma_token_id_t *urma_alloc_token_id(urma_context_t *ctx); + +/** + * Request to assign a token id. token id is used to register multiple segments with the protection table. + * Can use table mode or entry mode based on flag. + * @param[in] [Required] ctx: specifies the urma context. + * @param[in] [Required] flag: decides the mode of token id. use table mode if enable multi_seg in flag. + * Return: pointer to key id on success, NULL on error. + * Note: if use table mode, the VA address page alignment is required when register the segments. + */ +urma_token_id_t *urma_alloc_token_id_ex(urma_context_t *ctx, urma_token_id_flag_t flag); + +/** + * Request to release token id. + * @param[in] [Required] token_id: Specifies the token id to be released. + * Return: 0 on success, other value on error + */ +urma_status_t urma_free_token_id(urma_token_id_t *token_id); + +/** + * Register a memory segment on specified va address for local or remote access. + * @param[in] [Required] ctx: the created urma context pointer; + * @param[in] [Required] seg_cfg: Specify cfg of seg to be registered, including address, len, token, and so on; + * Return: pointer to target segment on success, NULL on error + * And the immedidate data wrote from clients is polled from this common jfc. + */ +urma_target_seg_t *urma_register_seg(urma_context_t *ctx, urma_seg_cfg_t *seg_cfg); + +/** + * Unregister a local memory segment on specified va address. + * @param[in] [Required] target_seg: target segment to be unregistered; + * Return: 0 on success, other value on error + */ +urma_status_t urma_unregister_seg(urma_target_seg_t *target_seg); + +/** + * Import a memory segment on specified ubva address. + * @param[in] [Required] ctx: the created urma context pointer; + * @param[in] [Required] seg: handle of memory segment to import; + * @param[in] [Required] token_value: token of remote side to put into output protection table; + * @param[in] [Optional] addr: the virtual address to which the segment will be mapped; + * @param[in] [Required] flag: flag to indicate the import attribute of memory segment; + * Return: pointer to target segment on success, NULL on error + */ +urma_target_seg_t *urma_import_seg(urma_context_t *ctx, urma_seg_t *seg, urma_token_t *token_value, uint64_t addr, + urma_import_seg_flag_t flag); + +/** + * Unimport a memory segment on specified ubva address. + * @param[in] [Required] tseg: the address of the target segment to unimport; + * Return: 0 on success, other value on error + */ +urma_status_t urma_unimport_seg(urma_target_seg_t *tseg); + +/** + * post a request to read, write, atomic or send data. + * @param[in] jfs: the jfs created before, which is used to put command; + * @param[in] wr: the posting request all information, including src addr, dst addr, len, jfc, flag, ordering etc. + * @param[in] bad_wr: the first of failure request. + * Return: 0 on success, other value on error + */ +urma_status_t urma_post_jfs_wr(urma_jfs_t *jfs, urma_jfs_wr_t *wr, urma_jfs_wr_t **bad_wr); + +/** + * post a request to recv data. + * @param[in] jfr: the jfr created before, which is used to put command; + * @param[in] wr: the posting request all information, including sge, flag. + * @param[in] bad_wr: the first of failure request. + * Return: 0 on success, other value on error + */ +urma_status_t urma_post_jfr_wr(urma_jfr_t *jfr, urma_jfr_wr_t *wr, urma_jfr_wr_t **bad_wr); + +/** + * post a request to read, write, atomic or send data. + * @param[in] jetty: the jetty created before, which is used to put command; + * @param[in] wr: the posting request all information, including src addr, dst addr, len, jfc, flag, ordering etc. + * @param[in] bad_wr: the first of failure request. + * Return: 0 on success, other value on error + */ +urma_status_t urma_post_jetty_send_wr(urma_jetty_t *jetty, urma_jfs_wr_t *wr, urma_jfs_wr_t **bad_wr); + +/** + * post a request to recv data. + * @param[in] jetty: the jetty created before, which is used to put command; + * @param[in] wr: the posting request all information, including sge, flag. + * @param[in] bad_wr: the first of failure request. + * Return: 0 on success, other value on error + */ +urma_status_t urma_post_jetty_recv_wr(urma_jetty_t *jetty, urma_jfr_wr_t *wr, urma_jfr_wr_t **bad_wr); + +/** + * Write data to remote node. + * @param[in] jfs: the jfs created before, which is used to put command; + * @param[in] target_jfr: destination jetty receiver; + * @param[in] dst_tseg: the dst target seg imported before; + * @param[in] src_tseg: the src target seg registered before; + * @param[in] dst: destination address(mapping va on user node or rva in ubva on home node) to be written into + * @param[in] src: source address(local process address space) to fetch data + * @param[in] len: the data len to be written + * @param[in] flag: flag to control jfs work request attritube + * @param[in] user_ctx: the user context, such as request id(rid) etc. + * Return: 0 on success, other value on error + */ +urma_status_t urma_write(urma_jfs_t *jfs, urma_target_jetty_t *target_jfr, urma_target_seg_t *dst_tseg, + urma_target_seg_t *src_tseg, uint64_t dst, uint64_t src, uint32_t len, urma_jfs_wr_flag_t flag, + uint64_t user_ctx); + +/** + * Read data from remote node. + * @param[in] jfs: the jfs created before, which is used to put command; + * @param[in] target_jfr: destination jetty receiver; + * @param[in] dst_tseg: the seg registered before; + * @param[in] src_tseg: the target seg imported before; + * @param[in] dst: destination address(local process address space) to be written into + * @param[in] src: source address(mapping va or rva in ubva) to fetch data + * @param[in] len: the data len to be written + * @param[in] flag: the flag to control jfs work request attritube + * @param[in] user_ctx: the user context, such as request id(rid) etc. + * Return: 0 on success, other value on error + */ +urma_status_t urma_read(urma_jfs_t *jfs, urma_target_jetty_t *target_jfr, urma_target_seg_t *dst_tseg, + urma_target_seg_t *src_tseg, uint64_t dst, uint64_t src, uint32_t len, urma_jfs_wr_flag_t flag, + uint64_t user_ctx); + +/** + * Send data to remote node. + * @param[in] jfs: the jfs created before, which is used to put command; + * @param[in] target_jfr: destination jetty receiver(with full qualifed jfr id); + * @param[in] src_tseg: the seg registered before, can be NULL only when flag.bs.inline_flag == URMA_INLINE_ENABLE + * @param[in] src: source address for sending; + * @param[in] len: data length; + * @param[in] flag: flag to control jfs work request attritube + * @param[in] user_ctx: the user context, such as request id(rid) etc; + * Return: 0 on success, other value on error. + */ +urma_status_t urma_send(urma_jfs_t *jfs, urma_target_jetty_t *target_jfr, urma_target_seg_t *src_tseg, uint64_t src, + uint32_t len, urma_jfs_wr_flag_t flag, uint64_t user_ctx); + +/** + * Assign local buffer to receive data from remote node. + * @param[in] jfr: jetty receiver; + * @param[in] recv_tseg: the locally registered segment before for receiving; + * @param[in] buf: buffer address for receiving; + * @param[in] len: buffer length; + * @param[in] user_ctx: the user context, such as request id(rid) etc; + * Return: 0 on success, other value on error. + */ +urma_status_t urma_recv(urma_jfr_t *jfr, urma_target_seg_t *recv_tseg, uint64_t buf, uint32_t len, uint64_t user_ctx); + +/** + * Poll jfc to get completion record. + * @param[in] jfc: jetty completion queue to poll + * @param[in] cr_cnt: the expected number of completion record to get + * @param[out] cr: the completion record array to fill at least cr_cnt completion records + * Return: the number of completion record returned, 0 means no completion record returned, less than 0 on error + * Note that: at most 16 completion records can be polled for RDMA device + */ +int urma_poll_jfc(urma_jfc_t *jfc, int cr_cnt, urma_cr_t *cr); + +/** + * Arm jfc with interrupt mode. + * @param[in] jfc: jetty completion queue to arm to interrupt mode + * @param[in] solicited_only: indicate it will trigger event only for packets with solicited flag. + * Return: 0 on success, other value on error + */ +urma_status_t urma_rearm_jfc(urma_jfc_t *jfc, bool solicited_only); + +/** + * Wait jfce for event of any completion message is generated. + * @param[in] jfce: jetty event channel to wait on + * @param[in] jfc_cnt: expected jfc count to return + * @param[in] time_out: max time to wait (milliseconds), + * timeout = 0: return immediately even if no events are ready, + * timeout = -1: an infinite timeout + * @param[out] jfc: address to put the jfc handle + * Return: the number of jfc returned, 0 means no jfc returned, -1 on error + * Note: User should check error when wait_jfc returns 0, errno ERESTARTSYS(512) + * means wait operation was interrupted by a signal and this error should + * be ignored, users should continue to wait jfc. + */ +int urma_wait_jfc(urma_jfce_t *jfce, uint32_t jfc_cnt, int time_out, urma_jfc_t *jfc[]); + +/** + * Confirm that a JFC generated event has been processed. + * @param[in] jfc: jfc pointer array to be acknowledged + * @param[in] nevents: event count array to be acknowledged + * @param[in] jfc_cnt: number of elements in the array + * Return: void + */ +void urma_ack_jfc(urma_jfc_t *jfc[], uint32_t nevents[], uint32_t jfc_cnt); + +/** + * Get or allocate a uasid. + * @param[out] uasid: the address to put uasid + * Return: 0 on success, other value on error + */ +urma_status_t urma_get_uasid(uint32_t *uasid); + +/** + * User defined control of the context. + * @param[in] ctx: the created urma context pointer; + * @param[in] in: user ioctl cmd; + * @param[out] out: result of execution; + * Return: 0 on success, other value on error + * Note: This API only supports UB hardware currently. + */ +urma_status_t urma_user_ctl(urma_context_t *ctx, urma_user_ctl_in_t *in, urma_user_ctl_out_t *out); + +/** + * User register own log function, default rsyslog. + * @param[in] func: log callback func; + * Return: 0 on success, other value on error + */ +urma_status_t urma_register_log_func(urma_log_cb_t func); + +/** + * User register location log function with file, function and line info. + * @param[in] func: location log callback function; + * Return: 0 on success, other value on error + * Note: If both urma_register_log_func and urma_register_loc_log_func are called, + * the last registered function will be used. + */ +urma_status_t urma_register_loc_log_func(urma_loc_log_cb func); + +/** + * User unregister own log function, use rsyslog. + * Return: 0 on success, other value on error + */ +urma_status_t urma_unregister_log_func(void); + +/** + * get log level. + * Return: urma_vlog_level_t + */ +urma_vlog_level_t urma_log_get_level(void); + +/** + * set log level. + * @param[in] level: log level to set; + */ +void urma_log_set_level(urma_vlog_level_t level); + +/** + * get log thread tag. + * Return: const char * + */ +const char *urma_log_get_thread_tag(void); + +/** + * set log thread tag. + * @param[in] tag: log tag per thread; + */ +void urma_log_set_thread_tag(const char *tag); + +/** + * User tp only. + * Get tpn of tp created when creating jetty. + * @param[in] jetty: the created jetty pointer; + * Return: >= 0 on success, return as tpn; < 0 on error + */ +int urma_get_tpn(urma_jetty_t *jetty); + +/** + * Get net address info list, user tp only. + * @param[in] ctx: the created urma context pointer; + * @param[out] cnt: numer of net address info; + * Return: pointer of net address list; NULL on error + */ +urma_net_addr_info_t *urma_get_net_addr_list(urma_context_t *ctx, uint32_t *cnt); + +/** + * Free net address info list. + * @param[in] net_addr_list: pointer of net address list + */ +void urma_free_net_addr_list(urma_net_addr_info_t *net_addr_list); + +/** + * Modify tp by user connection. + * @param[in] ctx: the created urma context pointer; + * @param[in] tpn: tpn of tp created before; + * @param[in] cfg: tp configurations filled by user; + * @param[in] attr: tp attributes filled by user; + * @param[in] mask: bitmap configurations for tp attributes; + * Return: 0 on success; other values on error + */ +int urma_modify_tp(urma_context_t *ctx, uint32_t tpn, urma_tp_cfg_t *cfg, urma_tp_attr_t *attr, + urma_tp_attr_mask_t mask); + +/** + * get available tp list from control plane. + * @param[in] [Required] ctx: the created urma context pointer; + * @param[in] [Required] tp_cfg: tp configuration to get; + * @param[in && out] [Required] tp_cnt: tp_cnt is the length of tp_list buffer as in parameter; + * tp_cnt is the number of tp as out parameter; + * @param[out] [Required] tp_list: tp list to get, the buffer is allocated by user; + * Return: 0 on success, other value on error + */ +urma_status_t urma_get_tp_list(urma_context_t *ctx, urma_get_tp_cfg_t *cfg, uint32_t *tp_cnt, urma_tp_info_t *tp_list); + +/** + * set tp attribution values in control plane. + * @param[in] [Required] ctx: the created urma context pointer; + * @param[in] [Required] tp_handle: tp_handle got by urma_get_tp_list; + * @param[in] [Required] tp_attr_cnt: number of tp attributions; + * @param[in] [Required] tp_attr_bitmap: tp attributions bitmap, current bitmap is as follow: + * 0-retry_times_init: 3 bit 1-at: 5 bit 2-SIP: 128 bit + * 3-DIP: 128 bit 4-SMA: 48 bit 5-DMA: 48 bit + * 6-vlan_id: 12 bit 7-vlan_en: 1 bit 8-dscp: 6 bit + * 9-at_times: 5 bit 10-sl: 4 bit 11-ttl: 8 bit + * @param[in] [Required] tp_attr: tp attribution values to set; + * Return: 0 on success, other value on error + */ +urma_status_t urma_set_tp_attr(const urma_context_t *ctx, const uint64_t tp_handle, const uint8_t tp_attr_cnt, + const uint32_t tp_attr_bitmap, const urma_tp_attr_value_t *tp_attr); + +/** + * get tp attribution values in control plane. + * @param[in] [Required] ctx: the created urma context pointer; + * @param[in] [Required] tp_handle: tp_handle got by urma_get_tp_list; + * @param[out] [Required] tp_attr_cnt: number of tp attributions; + * @param[out] [Required] tp_attr_bitmap: tp attributions bitmap, current bitmap is as follow: + * 0-retry_times_init: 3 bit 1-at: 5 bit 2-SIP: 128 bit + * 3-DIP: 128 bit 4-SMA: 48 bit 5-DMA: 48 bit + * 6-vlan_id: 12 bit 7-vlan_en: 1 bit 8-dscp: 6 bit + * 9-at_times: 5 bit 10-sl: 4 bit 11-ttl: 8 bit + * @param[out] [Required] tp_attr: tp attribution values to get; + * Return: 0 on success, other value on error + */ +urma_status_t urma_get_tp_attr(const urma_context_t *ctx, const uint64_t tp_handle, uint8_t *tp_attr_cnt, + uint32_t *tp_attr_bitmap, urma_tp_attr_value_t *tp_attr); + +/** + * get eid by ip info + * @param[in] ctx: the created urma context pointer; + * @param[in] net_addr: the ip info (type and net_addr are valid, vlan, mac, prefix_len will not be used); + * @param[out] eid: device's eid; + * Return: 0 on success, other value on error + */ +urma_status_t urma_get_eid_by_ip(const urma_context_t *ctx, const urma_net_addr_t *net_addr, urma_eid_t *eid); + +/** + * get ip info by eid. + * @param[in] ctx: the created urma context pointer; + * @param[in] eid: device's eid; + * @param[out] net_addr: the ip info (type and net_addr are valid, vlan, mac, prefix_len will not be used); + * Return: 0 on success, other value on error + */ +urma_status_t urma_get_ip_by_eid(const urma_context_t *ctx, const urma_eid_t *eid, urma_net_addr_t *net_addr); + + +/** + * get source mac address. + + * @param[in] ctx: the created urma context pointer; + * @param[out] mac: the mac address of source; + * Return: 0 on success, other value on error + */ +urma_status_t urma_get_smac(const urma_context_t *ctx, uint8_t *mac); + +/** + * get dest mac address. + * @param[in] ctx: the created urma context pointer; + * @param[in] net_addr: the ip info (type and net_addr are valid, vlan, mac, prefix_len will not be used); + * @param[out] mac: the mac address of dest; + * Return: 0 on success, other value on error + */ +urma_status_t urma_get_dmac(const urma_context_t *ctx, const urma_net_addr_t *net_addr, uint8_t *mac); + +#ifdef __cplusplus + } +#endif + +#endif diff --git a/include/ylt/urma/urma_cmd.h b/include/ylt/urma/urma_cmd.h new file mode 100644 index 000000000..0295170c5 --- /dev/null +++ b/include/ylt/urma/urma_cmd.h @@ -0,0 +1,1322 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) Huawei Technologies Co., Ltd. 2021-2025. All rights reserved. + * Description: Public header file of urma cmd + * Author: Qian Guoxin, Yan Fangfang + * Create: 2021-11-12 + * Note: + * History: 2021-11-12: Create file + * History: 2022-07-25: Yan Fangfang Change the prefix ubp_ioctl_ to urma_cmd_ + */ + +#ifndef URMA_CMD_H +#define URMA_CMD_H + +#include + +#include "urma_types.h" + +typedef struct urma_cmd_hdr { + uint32_t command; + uint32_t args_len; + uint64_t args_addr; +} urma_cmd_hdr_t; + +#define URMA_CMD_MAX_ARGS_SIZE 4096 +#define URMA_CMD_EID_SIZE (16) + +/* only for ubcore device ioctl */ +#define URMA_CORE_CMD_MAGIC 'C' +#define URMA_CORE_CMD _IOWR(URMA_CORE_CMD_MAGIC, 1, urma_cmd_hdr_t) +#define URMA_MAX_UASID (1 << 24) +#define URMA_CMD_TP_ATTR_BYTES 128 + +typedef enum urma_core_cmd { + URMA_CORE_CMD_QUERY_STATS = 1, + URMA_CORE_CMD_QUERY_RES, + URMA_CORE_CMD_ADD_EID, + URMA_CORE_CMD_DEL_EID, + URMA_CORE_CMD_SET_EID_MODE, + URMA_CORE_SET_NS_MODE, + URMA_CORE_SET_DEV_NS, + URMA_CORE_EXPOSE_DEV_NS, + URMA_CORE_UNEXPOSE_DEV_NS, + URMA_CORE_SET_DEV_EID_NS, + URMA_CORE_GET_TOPO_INFO, + URMA_CORE_SET_SL, + URMA_CORE_ADMIN_INSERT_MAIN_UE_EID = 35, + URMA_CORE_ADMIN_DELETE_MAIN_UE_EID, + URMA_CORE_ADMIN_LOOKUP_MAIN_UE_EID, + URMA_CORE_ADMIN_FLUSH_MAIN_UE_EID, + URMA_CORE_ADMIN_INSERT_MAIN_UE_EID_BATCH, +} urma_core_cmd_t; + +typedef enum uram_ubagg_cmd { + UBAGG_NL_GET_PHYSICAL_DEVICE = 4, +} uram_ubagg_cmd_t; + +/* only for uburma device ioctl */ +#define URMA_CMD_MAGIC 'U' +#define URMA_CMD _IOWR(URMA_CMD_MAGIC, 1, urma_cmd_hdr_t) + +typedef enum urma_cmd { + URMA_CMD_CREATE_CTX = 1, + URMA_CMD_ALLOC_TOKEN_ID, + URMA_CMD_FREE_TOKEN_ID, + URMA_CMD_REGISTER_SEG, + URMA_CMD_UNREGISTER_SEG, + URMA_CMD_IMPORT_SEG, + URMA_CMD_UNIMPORT_SEG, + URMA_CMD_CREATE_JFS, + URMA_CMD_MODIFY_JFS, + URMA_CMD_QUERY_JFS, + URMA_CMD_DELETE_JFS, + URMA_CMD_CREATE_JFR, + URMA_CMD_MODIFY_JFR, + URMA_CMD_QUERY_JFR, + URMA_CMD_DELETE_JFR, + URMA_CMD_CREATE_JFC, + URMA_CMD_MODIFY_JFC, + URMA_CMD_DELETE_JFC, + URMA_CMD_CREATE_JFCE, + URMA_CMD_IMPORT_JFR, + URMA_CMD_UNIMPORT_JFR, + URMA_CMD_CREATE_JETTY, + URMA_CMD_MODIFY_JETTY, + URMA_CMD_QUERY_JETTY, + URMA_CMD_DELETE_JETTY, + URMA_CMD_IMPORT_JETTY, + URMA_CMD_UNIMPORT_JETTY, + URMA_CMD_ADVISE_JFR, + URMA_CMD_UNADVISE_JFR, + URMA_CMD_ADVISE_JETTY, + URMA_CMD_UNADVISE_JETTY, + URMA_CMD_BIND_JETTY, + URMA_CMD_UNBIND_JETTY, + URMA_CMD_CREATE_JETTY_GRP, + URMA_CMD_DESTROY_JETTY_GRP, + URMA_CMD_USER_CTL, + URMA_CMD_GET_EID_LIST, + URMA_CMD_GET_NETADDR_LIST, + URMA_CMD_MODIFY_TP, + URMA_CMD_QUERY_DEV_ATTR, + URMA_CMD_IMPORT_JETTY_ASYNC, + URMA_CMD_UNIMPORT_JETTY_ASYNC, + URMA_CMD_BIND_JETTY_ASYNC, + URMA_CMD_UNBIND_JETTY_ASYNC, + URMA_CMD_CREATE_NOTIFIER, + URMA_CMD_GET_TP_LIST, + URMA_CMD_IMPORT_JETTY_EX, + URMA_CMD_IMPORT_JFR_EX, + URMA_CMD_BIND_JETTY_EX, + URMA_CMD_DELETE_JFS_BATCH, + URMA_CMD_DELETE_JFR_BATCH, + URMA_CMD_DELETE_JFC_BATCH, + URMA_CMD_DELETE_JETTY_BATCH, + URMA_CMD_SET_TP_ATTR, + URMA_CMD_GET_TP_ATTR, + URMA_CMD_EXCHANGE_TP_INFO, + URMA_CMD_GET_EID_BY_IP, + URMA_CMD_GET_IP_BY_EID, + URMA_CMD_GET_SMAC, + URMA_CMD_GET_DMAC, + URMA_CMD_ALLOC_JFC, + URMA_CMD_FREE_JFC, + URMA_CMD_SET_JFC_OPT, + URMA_CMD_GET_JFC_OPT, + URMA_CMD_ACTIVE_JFC, + URMA_CMD_DEACTIVE_JFC, + URMA_CMD_ALLOC_JFR, + URMA_CMD_FREE_JFR, + URMA_CMD_SET_JFR_OPT, + URMA_CMD_GET_JFR_OPT, + URMA_CMD_ACTIVE_JFR, + URMA_CMD_DEACTIVE_JFR, + URMA_CMD_ALLOC_JFS, + URMA_CMD_FREE_JFS, + URMA_CMD_SET_JFS_OPT, + URMA_CMD_GET_JFS_OPT, + URMA_CMD_ACTIVE_JFS, + URMA_CMD_DEACTIVE_JFS, + URMA_CMD_ALLOC_JETTY, + URMA_CMD_FREE_JETTY, + URMA_CMD_SET_JETTY_OPT, + URMA_CMD_GET_JETTY_OPT, + URMA_CMD_ACTIVE_JETTY, + URMA_CMD_DEACTIVE_JETTY, + URMA_CMD_MAX +} urma_cmd_t; + +#ifndef URMA_CMD_UDRV_PRIV +#define URMA_CMD_UDRV_PRIV +typedef struct urma_cmd_udrv_priv { + uint64_t in_addr; + uint32_t in_len; + uint64_t out_addr; + uint32_t out_len; +} urma_cmd_udrv_priv_t; +#endif + +typedef struct urma_cmd_create_ctx { + struct { + uint8_t eid[URMA_CMD_EID_SIZE]; + uint32_t eid_index; + } in; + struct { + int async_fd; + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_create_ctx_t; + +typedef struct urma_cmd_alloc_token_id { + struct { + urma_token_id_flag_t flag; + } in; + struct { + uint32_t token_id; + uint64_t handle; /* handle of the allocated token_id obj in kernel */ + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_alloc_token_id_t; + +typedef struct urma_cmd_free_token_id { + struct { + uint64_t handle; /* handle of the allocated token_id obj in kernel */ + uint32_t token_id; + } in; + urma_cmd_udrv_priv_t udata; +} urma_cmd_free_token_id_t; + +typedef struct urma_cmd_register_seg { + struct { + uint64_t va; + uint64_t len; + uint32_t token_id; + uint64_t token_id_handle; + uint32_t token; + uint32_t flag; + } in; + struct { + uint32_t token_id; + uint64_t handle; /* handle of the allocated seg obj in kernel */ + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_register_seg_t; + +typedef struct urma_cmd_unregister_seg { + struct { + uint64_t handle; /* handle of seg, used to find seg obj in kernel */ + } in; +} urma_cmd_unregister_seg_t; + +typedef struct urma_cmd_import_seg { + struct { + uint8_t eid[URMA_CMD_EID_SIZE]; + uint64_t va; + uint64_t len; + uint32_t flag; + uint32_t token; + uint32_t token_id; + uint64_t mva; + } in; + struct { + uint64_t handle; /* handle of the allocated tseg obj in kernel */ + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_import_seg_t; + +typedef struct urma_cmd_unimport_seg { + struct { + uint64_t handle; /* handle of the seg to be unimported */ + } in; +} urma_cmd_unimport_seg_t; + +typedef struct urma_cmd_create_jfr { + struct { + uint32_t depth; + uint32_t flag; + uint32_t trans_mode; + uint8_t max_sge; + uint8_t min_rnr_timer; + uint32_t jfc_id; + uint64_t jfc_handle; + uint32_t token; + uint32_t id; + uint64_t urma_jfr; /* urma jfr pointer */ + } in; + struct { + uint32_t id; + uint32_t depth; + uint8_t max_sge; + uint64_t handle; /* handle of the allocated jfr obj in kernel */ + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_create_jfr_t; + +typedef struct urma_cmd_modify_jfr { + struct { + uint64_t handle; /* handle of jfr, used to find jfr obj in kernel */ + uint32_t mask; /* see urma_jfr_attr_mask_t */ + uint32_t rx_threshold; + uint32_t state; + } in; + urma_cmd_udrv_priv_t udata; +} urma_cmd_modify_jfr_t; + +typedef struct urma_cmd_query_jfr { + struct { + uint64_t handle; /* handle of the allocated jfr obj in kernel */ + } in; + struct { + uint32_t depth; + uint32_t flag; + uint32_t trans_mode; + uint8_t max_sge; + uint8_t min_rnr_timer; + uint32_t token; + uint32_t id; + + uint32_t rx_threshold; + uint32_t state; + } out; +} urma_cmd_query_jfr_t; + + +typedef struct urma_cmd_delete_jfr { + struct { + uint64_t handle; /* handle of jfr, used to find jfr obj in kernel */ + } in; + struct { + uint32_t async_events_reported; + } out; +} urma_cmd_delete_jfr_t; + +typedef struct urma_cmd_delete_jfr_batch { + struct { + uint32_t async_events_reported; + uint32_t bad_jfr_index; + } out; + struct { + uint32_t jfr_num; + uint64_t jfr_ptr; + } in; +} urma_cmd_delete_jfr_batch_t; + +typedef struct urma_cmd_alloc_jfr { + struct { + uint32_t depth; + uint32_t flag; + uint32_t trans_mode; + uint8_t max_sge; + uint8_t min_rnr_timer; + uint32_t jfc_id; + uint64_t jfc_handle; + uint32_t token; + uint32_t id; + uint64_t urma_jfr; /* urma jfr pointer */ + } in; + struct { + uint32_t id; + uint32_t depth; + uint64_t handle; /* handle of the allocated jfr obj in kernel */ + uint8_t max_sge; + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_alloc_jfr_t; + +typedef struct urma_cmd_free_jfr { + struct { + uint64_t handle; /* handle of jfc, used to find jfc obj in kernel */ + } in; + struct { + uint32_t async_events_reported; + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_free_jfr_t; + +typedef struct urma_cmd_set_jfr_opt { + struct { + uint64_t handle; + uint64_t opt; + uint64_t buf; + uint32_t len; + } in; + urma_cmd_udrv_priv_t udata; +} urma_cmd_set_jfr_opt_t; + +typedef struct urma_cmd_get_jfr_opt { + struct { + uint64_t handle; + uint64_t opt; + uint64_t buf; + uint32_t len; + } in; + struct { + uint64_t buf; + uint32_t len; + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_get_jfr_opt_t; + +typedef struct urma_cmd_active_jfr { + struct { + uint64_t handle; /* handle of jfr, used to find jfr obj in kernel */ + uint32_t depth; + uint32_t flag; + uint32_t trans_mode; + uint8_t max_sge; + uint8_t min_rnr_timer; + uint32_t jfc_id; + uint64_t jfc_handle; + uint32_t token_value; + uint64_t jfr_opt; + } in; + struct { + uint32_t id; + uint32_t depth; + uint64_t handle; /* handle of the allocated jfr obj in kernel */ + uint8_t max_sge; + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_active_jfr_t; + +typedef struct urma_cmd_deactive_jfr { + struct { + uint64_t handle; /* handle of jfr, used to find jfr obj in kernel */ + } in; + urma_cmd_udrv_priv_t udata; +} urma_cmd_deactive_jfr_t; + +typedef struct urma_cmd_create_jfs { + struct { + uint32_t depth; + uint32_t flag; + uint32_t trans_mode; + uint8_t priority; + uint8_t max_sge; + uint8_t max_rsge; + uint32_t max_inline_data; + uint8_t retry_cnt; + uint8_t rnr_retry; + uint8_t err_timeout; + uint32_t jfc_id; + uint64_t jfc_handle; + uint64_t urma_jfs; /* urma jfs pointer */ + } in; + struct { + uint32_t id; + uint32_t depth; + uint8_t max_sge; + uint8_t max_rsge; + uint32_t max_inline_data; + uint64_t handle; /* handle of the allocated jfs obj in kernel */ + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_create_jfs_t; + +typedef struct urma_cmd_modify_jfs { + struct { + uint64_t handle; /* handle of jfs, used to find jfs obj in kernel */ + uint32_t mask; /* see urma_jfr_attr_mask_t */ + uint32_t state; /* urma_jetty_state_t */ + } in; + urma_cmd_udrv_priv_t udata; +} urma_cmd_modify_jfs_t; + +typedef struct urma_cmd_query_jfs { + struct { + uint64_t handle; /* handle of the allocated jfs obj in kernel */ + } in; + struct { + uint32_t depth; + uint32_t flag; + uint32_t trans_mode; + uint8_t priority; + uint8_t max_sge; + uint8_t max_rsge; + uint32_t max_inline_data; + uint8_t retry_cnt; + uint8_t rnr_retry; + uint8_t err_timeout; + + uint32_t state; + } out; +} urma_cmd_query_jfs_t; + +typedef struct urma_cmd_delete_jfs { + struct { + uint64_t handle; /* handle of jfs, used to find jfs obj in kernel */ + } in; + struct { + uint32_t async_events_reported; + } out; +} urma_cmd_delete_jfs_t; + +typedef struct urma_cmd_delete_jfs_batch { + struct { + uint32_t async_events_reported; + uint32_t bad_jfs_index; + } out; + struct { + uint32_t jfs_num; + uint64_t jfs_ptr; + } in; +} urma_cmd_delete_jfs_batch_t; + +typedef struct urma_cmd_alloc_jfs { + struct { + uint32_t depth; + uint32_t flag; + uint32_t trans_mode; + uint8_t priority; + uint8_t max_sge; + uint8_t max_rsge; + uint32_t max_inline_data; + uint8_t rnr_retry; + uint8_t err_timeout; + uint32_t jfc_id; + uint64_t jfc_handle; + uint64_t urma_jfs; /* urma jfs pointer */ + } in; + struct { + uint32_t id; + uint32_t depth; + uint8_t max_sge; + uint8_t max_rsge; + uint32_t max_inline_data; + uint64_t handle; /* handle of the allocated jfs obj in kernel */ + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_alloc_jfs_t; + +typedef struct urma_cmd_free_jfs { + struct { + uint64_t handle; /* handle of jfs, used to find jfs obj in kernel */ + } in; + struct { + uint32_t async_events_reported; + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_free_jfs_t; + +typedef struct urma_cmd_set_jfs_opt { + struct { + uint64_t handle; + uint64_t opt; + uint64_t buf; + uint32_t len; + } in; + urma_cmd_udrv_priv_t udata; +} urma_cmd_set_jfs_opt_t; + +typedef struct urma_cmd_get_jfs_opt { + struct { + uint64_t handle; + uint64_t opt; + uint64_t buf; + uint32_t len; + } in; + struct { + uint64_t buf; + uint32_t len; + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_get_jfs_opt_t; + + +typedef struct urma_cmd_active_jfs { + struct { + uint64_t handle; /* handle of jfs, used to find jfs obj in kernel */ + uint32_t depth; + uint32_t flag; + uint32_t trans_mode; + uint32_t priority; + uint8_t max_sge; + uint8_t max_rsge; + uint32_t max_inline_data; + uint8_t rnr_retry; + uint8_t err_timeout; + uint32_t jfc_id; + uint64_t jfc_handle; + uint64_t jfs_opt; + } in; + struct { + uint32_t id; + uint32_t depth; + uint8_t max_sge; + uint8_t max_rsge; + uint32_t max_inline_data; + uint64_t handle; /* handle of the allocated jfs obj in kernel */ + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_active_jfs_t; + +typedef struct urma_cmd_deactive_jfs { + struct { + uint64_t handle; /* handle of jfs, used to find jfs obj in kernel */ + } in; + urma_cmd_udrv_priv_t udata; +} urma_cmd_deactive_jfs_t; + +typedef struct urma_cmd_create_jfc { + struct { + uint32_t depth; /* in terms of CQEBB */ + uint32_t flag; + int jfce_fd; + uint64_t urma_jfc; /* urma jfc pointer */ + uint32_t ceqn; /* [Optional] event queue id, less than urma_device_cap_t->ceq_cnt + * set to 0 by default */ + } in; + struct { + uint32_t id; + uint32_t depth; + uint64_t handle; /* handle of the allocated jfc obj in kernel */ + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_create_jfc_t; + +typedef struct urma_cmd_modify_jfc { + struct { + uint64_t handle; /* handle of jfc, used to find jfc obj in kernel */ + uint32_t mask; /* see urma_jfc_attr_mask_t */ + uint16_t moderate_count; + uint16_t moderate_period; /* in micro seconds */ + } in; + urma_cmd_udrv_priv_t udata; +} urma_cmd_modify_jfc_t; + +typedef struct urma_cmd_delete_jfc { + struct { + uint64_t handle; /* handle of jfc, used to find jfc obj in kernel */ + } in; + struct { + uint32_t comp_events_reported; + uint32_t async_events_reported; + } out; +} urma_cmd_delete_jfc_t; + +typedef struct urma_cmd_delete_jfc_batch { + struct { + uint32_t comp_events_reported; + uint32_t async_events_reported; + uint32_t bad_jfc_index; + } out; + struct { + uint32_t jfc_num; + uint64_t jfc_ptr; + } in; +} urma_cmd_delete_jfc_batch_t; + +typedef struct urma_cmd_alloc_jfc { + struct { + uint32_t depth; + uint32_t flag; + int jfce_fd; + uint64_t urma_jfc; /* urma jfc pointer */ + uint32_t ceqn; /* [Optional] event queue id, less than urma_device_cap_t->ceq_cnt + * set to 0 by default */ + } in; + struct { + uint32_t id; + uint32_t depth; + uint64_t handle; /* handle of the allocated jfc obj in kernel */ + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_alloc_jfc_t; + +typedef struct urma_cmd_free_jfc { + struct { + uint64_t handle; /* handle of jfc, used to find jfc obj in kernel */ + } in; + struct { + uint32_t comp_events_reported; + uint32_t async_events_reported; + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_free_jfc_t; + +typedef struct urma_cmd_set_jfc_opt { + struct { + uint64_t handle; + uint64_t opt; + uint64_t buf; + uint32_t len; + } in; + urma_cmd_udrv_priv_t udata; +} urma_cmd_set_jfc_opt_t; + +typedef struct urma_cmd_get_jfc_opt { + struct { + uint64_t handle; + uint64_t opt; + uint64_t buf; + uint32_t len; + } in; + struct { + uint64_t buf; + uint32_t len; + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_get_jfc_opt_t; + +typedef struct urma_cmd_active_jfc { + struct { + uint64_t handle; /* handle of jfc, used to find jfc obj in kernel */ + uint32_t depth; + uint32_t flag; + uint32_t ceqn; + uint64_t urma_jfc_opt; + } in; + struct { + uint32_t id; + uint32_t depth; + uint64_t handle; /* handle of the allocated jfc obj in kernel */ + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_active_jfc_t; + +typedef struct urma_cmd_deactive_jfc { + struct { + uint64_t handle; /* handle of jfc, used to find jfc obj in kernel */ + } in; + urma_cmd_udrv_priv_t udata; +} urma_cmd_deactive_jfc_t; + +typedef struct urma_cmd_create_jfce { + struct { + int fd; + } out; +} urma_cmd_create_jfce_t; + +typedef struct urma_cmd_import_jfr { + struct { + /* correspond to urma_jfr_id */ + uint8_t eid[URMA_CMD_EID_SIZE]; + uint32_t id; + uint32_t flag; + /* correspond to urma_token_t */ + uint32_t token; + uint32_t trans_mode; + uint32_t tp_type; + } in; + struct { + uint32_t tpn; + uint64_t handle; /* handle of the allocated tjfr obj in kernel */ + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_import_jfr_t; + +typedef struct urma_cmd_import_jfr_ex { + struct { + /* correspond to urma_jfr_id */ + uint8_t eid[URMA_CMD_EID_SIZE]; + uint32_t id; + uint32_t flag; /* refer to urma_import_jetty_flag_t */ + /* correspond to urma_token_t */ + uint32_t token; + uint32_t trans_mode; + uint32_t tp_type; + /* correspond to urma_active_tp_cfg_t */ + uint64_t tp_handle; + uint64_t peer_tp_handle; + uint64_t tag; + uint32_t tx_psn; + uint32_t rx_psn; + uint64_t stag; + uint64_t dtag; + } in; + struct { + uint32_t tpn; + uint32_t reserved; + uint64_t handle; /* handle of the allocated tjfr obj in kernel */ + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_import_jfr_ex_t; + +typedef struct urma_cmd_unimport_jfr { + struct { + uint64_t handle; /* handle of tjfr, used to find tjfr obj in kernel */ + } in; +} urma_cmd_unimport_jfr_t; + +typedef struct urma_cmd_create_jetty { + struct { + uint32_t id; /* user may assign id */ + uint32_t jetty_flag; + + uint32_t jfs_depth; + uint32_t jfs_flag; + uint32_t trans_mode; + uint8_t priority; + uint8_t max_send_sge; + uint8_t max_send_rsge; + uint32_t max_inline_data; + uint8_t rnr_retry; + uint8_t err_timeout; + uint32_t send_jfc_id; + uint64_t send_jfc_handle; /* handle of the related send jfc */ + + uint32_t jfr_depth; + uint32_t jfr_flag; + uint8_t max_recv_sge; + uint8_t min_rnr_timer; + + uint32_t recv_jfc_id; + uint64_t recv_jfc_handle; /* handle of the related recv jfc */ + uint32_t token; + + uint32_t jfr_id; /* shared jfr */ + uint64_t jfr_handle; /* handle of the shared jfr */ + + uint64_t jetty_grp_handle; /* handle of the related jetty group */ + uint8_t is_jetty_grp; + + uint64_t urma_jetty; /* urma jetty pointer */ + } in; + struct { + uint32_t id; /* jetty id allocated by ubcore */ + uint64_t handle; /* handle of the allocated jetty obj in kernel */ + uint32_t jfs_depth; + uint32_t jfr_depth; + uint8_t max_send_sge; + uint8_t max_send_rsge; + uint8_t max_recv_sge; + uint32_t max_inline_data; + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_create_jetty_t; + +typedef struct urma_cmd_modify_jetty { + struct { + uint64_t handle; /* handle of jetty, used to find jetty obj in kernel */ + uint32_t mask; /* see urma_jetty_attr_mask_t */ + uint32_t rx_threshold; + uint32_t state; + } in; + urma_cmd_udrv_priv_t udata; +} urma_cmd_modify_jetty_t; + +typedef struct urma_cmd_query_jetty { + struct { + uint64_t handle; /* handle of the allocated jetty obj in kernel */ + } in; + struct { + uint32_t id; /* user may assign id */ + uint32_t jetty_flag; + + uint32_t jfs_depth; + uint32_t jfr_depth; + uint32_t jfs_flag; + uint32_t jfr_flag; + uint32_t trans_mode; + uint8_t max_send_sge; + uint8_t max_send_rsge; + uint8_t max_recv_sge; + uint32_t max_inline_data; + uint8_t priority; + uint8_t retry_cnt; + uint8_t rnr_retry; + uint8_t err_timeout; + uint8_t min_rnr_timer; + uint32_t jfr_id; + uint32_t token; + + uint32_t rx_threshold; + uint32_t state; + } out; +} urma_cmd_query_jetty_t; + +typedef struct urma_cmd_delete_jetty { + struct { + uint64_t handle; /* handle of jetty, used to find jetty obj in kernel */ + } in; + struct { + uint32_t async_events_reported; + } out; +} urma_cmd_delete_jetty_t; + +typedef struct urma_cmd_delete_jetty_batch { + struct { + uint32_t async_events_reported; + uint32_t bad_jetty_index; + } out; + struct { + uint32_t jetty_num; + uint64_t jetty_ptr; + } in; +} urma_cmd_delete_jetty_batch_t; + +typedef struct urma_cmd_import_jetty { + struct { + /* correspond to urma_jetty_id */ + uint8_t eid[URMA_CMD_EID_SIZE]; + uint32_t id; + uint32_t flag; + /* correspond to urma_token_t */ + uint32_t token; + uint32_t trans_mode; + uint32_t policy; + uint32_t type; + uint32_t tp_type; + } in; + struct { + uint32_t tpn; + uint64_t handle; /* handle of the allocated tjetty obj in kernel */ + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_import_jetty_t; + +typedef struct urma_cmd_import_jetty_ex { + struct { + /* correspond to urma_jetty_id */ + uint8_t eid[URMA_CMD_EID_SIZE]; + uint32_t id; + uint32_t flag; + /* correspond to urma_token_t */ + uint32_t token; + uint32_t trans_mode; + uint32_t policy; + uint32_t type; + uint32_t tp_type; + /* correspond to urma_active_tp_cfg_t */ + uint64_t tp_handle; + uint64_t peer_tp_handle; + uint64_t tag; + uint32_t tx_psn; + uint32_t rx_psn; + /* correspond to upper layer business */ + uint64_t stag; + uint64_t dtag; + } in; + struct { + uint32_t tpn; + uint32_t reserved; + uint64_t handle; /* handle of the allocated tjetty obj in kernel */ + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_import_jetty_ex_t; + +typedef struct urma_cmd_unimport_jetty { + struct { + uint64_t handle; /* handle of tjetty, used to find tjetty obj in kernel */ + } in; +} urma_cmd_unimport_jetty_t; + +typedef struct urma_cmd_advise_jetty { + struct { + uint64_t jetty_handle; /* handle of jetty, used to find jetty obj in kernel */ + uint64_t tjetty_handle; /* handle of tjetty, used to find tjetty obj in kernel */ + } in; + urma_cmd_udrv_priv_t udata; +} urma_cmd_advise_jetty_t; + +typedef struct urma_cmd_bind_jetty { + struct { + uint64_t jetty_handle; /* handle of jetty, used to find jetty obj in kernel */ + uint64_t tjetty_handle; /* handle of tjetty, used to find tjetty obj in kernel */ + } in; + struct { + uint32_t tpn; + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_bind_jetty_t; + +typedef struct urma_cmd_bind_jetty_ex { + struct { + uint64_t jetty_handle; /* handle of jetty, used to find jetty obj in kernel */ + uint64_t tjetty_handle; /* handle of tjetty, used to find tjetty obj in kernel */ + /* correspond to urma_active_tp_cfg_t */ + uint64_t tp_handle; + uint64_t peer_tp_handle; + uint64_t tag; + uint32_t tx_psn; + uint32_t rx_psn; + } in; + struct { + uint32_t tpn; + uint32_t reserved; + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_bind_jetty_ex_t; + +typedef struct urma_cmd_unbind_jetty { + struct { + uint64_t jetty_handle; /* handle of jetty, used to find jetty obj in kernel */ + } in; +} urma_cmd_unbind_jetty_t; + +typedef struct urma_cmd_unadvise_jetty { + struct { + uint64_t jetty_handle; /* handle of jetty, used to find jetty obj in kernel */ + uint64_t tjetty_handle; /* handle of tjetty, used to find tjetty obj in kernel */ + } in; +} urma_cmd_unadvise_jetty_t; + +typedef struct urma_cmd_create_jetty_grp { + struct { + char name[URMA_MAX_NAME]; + uint32_t token; + uint32_t id; + uint32_t policy; + uint32_t flag; + uint64_t urma_jetty_grp; /* urma jetty group pointer */ + } in; + struct { + uint32_t id; /* jetty group id allocated by ubcore */ + uint64_t handle; /* handle of the allocated jetty group obj in kernel */ + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_create_jetty_grp_t; + +typedef struct urma_cmd_delete_jetty_grp { + struct { + uint64_t handle; /* handle of jetty group, used to find jetty group obj in kernel */ + } in; + struct { + uint32_t async_events_reported; + } out; +} urma_cmd_delete_jetty_grp_t; + +typedef urma_cmd_create_jetty_t urma_cmd_alloc_jetty_t; + +typedef struct urma_cmd_free_jetty { + struct { + uint64_t handle; /* handle of jetty, used to find jetty obj in kernel */ + } in; + struct { + uint32_t async_events_reported; + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_free_jetty_t; + +typedef struct urma_cmd_set_jetty_opt { + struct { + uint64_t handle; + uint64_t opt; + uint64_t buf; + uint32_t len; + } in; + urma_cmd_udrv_priv_t udata; +} urma_cmd_set_jetty_opt_t; + +typedef struct urma_cmd_get_jetty_opt { + struct { + uint64_t handle; + uint64_t opt; + uint64_t buf; + uint32_t len; + } in; + struct { + uint64_t buf; + uint32_t len; + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_get_jetty_opt_t; + +typedef struct urma_cmd_active_jetty { + struct { + uint32_t flag; + uint64_t handle; + uint64_t send_jfc_handle; /* handle of the related send jfc */ + uint64_t recv_jfc_handle; /* handle of the related recv jfc */ + uint64_t urma_jetty; /* urma jetty pointer */ + uint64_t jetty_opt; + } in; + struct { + uint32_t jetty_id; + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_active_jetty_t; + +typedef struct urma_cmd_deactive_jetty { + struct { + uint64_t handle; /* handle of jetty, used to find jetty obj in kernel */ + } in; + urma_cmd_udrv_priv_t udata; +} urma_cmd_deactive_jetty_t; + +typedef struct urma_cmd_get_eid_list { + struct { + uint32_t max_eid_cnt; + } in; + struct { + uint32_t eid_cnt; + urma_eid_info_t eid_list[URMA_MAX_EID_CNT]; + } out; +} urma_cmd_get_eid_list_t; + +typedef struct urma_cmd_user_ctl { + struct { + uint64_t addr; + uint32_t len; + uint32_t opcode; + } in; /* struct [in] should be consistent with [urma_user_ctl_in_t] */ + struct { + uint64_t addr; + uint32_t len; + uint32_t reserved; + } out; /* struct [out] should be consistent with [urma_user_ctl_out_t] */ + struct { + uint64_t in_addr; + uint32_t in_len; + uint64_t out_addr; + uint32_t out_len; + } udrv; /* struct [udrv] should be consistent with [urma_udrv_t] */ +} urma_cmd_user_ctl_t; + +typedef enum urma_cmd_net_addr_type { + URMA_CMD_NET_ADDR_TYPE_IPV4 = 0, + URMA_CMD_NET_ADDR_TYPE_IPV6 +} urma_cmd_net_addr_type_t; + +union urma_cmd_net_addr_union { + uint8_t raw[URMA_CMD_EID_SIZE]; + struct { + uint64_t reserved1; + uint32_t reserved2; + uint32_t addr; + } in4; + struct { + uint64_t subnet_prefix; + uint64_t interface_id; + } in6; +}; + +typedef struct urma_cmd_net_addr { + urma_cmd_net_addr_type_t type; + union urma_cmd_net_addr_union net_addr; + uint64_t vlan; /* available for UBOE */ + uint8_t mac[URMA_MAC_BYTES]; /* available for UBOE */ + uint32_t prefix_len; +} urma_cmd_net_addr_t; + + +typedef struct urma_cmd_net_addr_info { + urma_cmd_net_addr_t netaddr; + uint32_t index; +} urma_cmd_net_addr_info_t; + +typedef struct urma_cmd_get_net_addr_list { + struct { + uint32_t max_netaddr_cnt; + } in; + struct { + uint32_t netaddr_cnt; + uint64_t addr; /* containing the array of urma_cmd_net_addr_info_t */ + uint64_t len; + } out; +} urma_cmd_get_net_addr_list_t; + + +typedef struct urma_cmd_modify_tp { + struct { + uint32_t tpn; + urma_tp_cfg_t tp_cfg; + urma_tp_attr_t attr; + urma_tp_attr_mask_t mask; + } in; +} urma_cmd_modify_tp_t; /* this struct should be consistent [struct uburma_cmd_modify_tp] */ + +typedef struct urma_cmd_query_device_attr { + struct { + char dev_name[URMA_MAX_DEV_NAME]; + } in; + struct { + urma_device_attr_t attr; + } out; +} urma_cmd_query_device_attr_t; + +typedef struct urma_cmd_import_jetty_async { + struct { + /* correspond to urma_jetty_id */ + uint8_t eid[URMA_CMD_EID_SIZE]; + uint32_t id; + uint32_t flag; + /* correspond to urma_token_t */ + uint32_t token; + uint32_t trans_mode; + uint32_t policy; + uint32_t type; + uint64_t urma_tjetty; /* urma tjetty pointer */ + uint64_t user_ctx; + int fd; + int timeout; + } in; + struct { + uint32_t tpn; + uint64_t handle; /* handle of the allocated tjetty obj in kernel */ + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_import_jetty_async_t; + +typedef struct urma_cmd_unimport_jetty_async { + struct { + uint64_t handle; /* handle of tjetty, used to find tjetty obj in kernel */ + } in; +} urma_cmd_unimport_jetty_async_t; + +typedef struct urma_cmd_bind_jetty_async { + struct { + uint64_t jetty_handle; /* handle of jetty, used to find jetty obj in kernel */ + uint64_t tjetty_handle; /* handle of tjetty, used to find tjetty obj in kernel */ + uint64_t urma_tjetty; /* urma tjetty pointer */ + uint64_t urma_jetty; /* urma jetty pointer */ + int fd; + uint64_t user_ctx; + int timeout; + } in; + struct { + uint32_t tpn; + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_bind_jetty_async_t; + +typedef struct urma_cmd_unbind_jetty_async { + struct { + uint64_t jetty_handle; /* handle of jetty, used to find jetty obj in kernel */ + uint64_t tjetty_handle; /* handle of tjetty, used to find tjetty obj in kernel */ + } in; +} urma_cmd_unbind_jetty_async_t; + + +typedef struct urma_cmd_create_notifier { + struct { + int fd; + } out; +} urma_cmd_create_notifier_t; + +/* only for event ioctl */ +#define MAX_JFCE_EVENT_CNT 16 +#define MAX_NOTIFY_CNT 16 +#define URMA_EVENT_CMD_MAGIC 'E' + +#define JFCE_CMD_WAIT_EVENT 0 +#define URMA_CMD_WAIT_JFC _IOWR(URMA_EVENT_CMD_MAGIC, JFCE_CMD_WAIT_EVENT, urma_cmd_hdr_t) +#define JFAE_CMD_GET_ASYNC_EVENT 0 +#define URMA_CMD_GET_ASYNC_EVENT _IOWR(URMA_EVENT_CMD_MAGIC, JFAE_CMD_GET_ASYNC_EVENT, urma_cmd_hdr_t) +#define NOTIFIER_CMD_WAIT_NOTIFY 0 +#define URMA_CMD_WAIT_NOTIFY _IOWR(URMA_EVENT_CMD_MAGIC, NOTIFIER_CMD_WAIT_NOTIFY, urma_cmd_hdr_t) + +typedef struct urma_cmd_jfce_wait { + struct { + uint32_t max_event_cnt; + int time_out; + } in; + struct { + uint32_t event_cnt; + uint64_t event_data[MAX_JFCE_EVENT_CNT]; + } out; +} urma_cmd_jfce_wait_t; + +typedef struct urma_cmd_async_event { + uint32_t event_type; + uint64_t event_data; + uint32_t pad; +} urma_cmd_async_event_t; + +typedef struct urma_cmd_notify { + urma_notify_type_t type; + urma_status_t status; + uint64_t user_ctx; + uint64_t urma_jetty; + uint32_t vtpn; +} urma_cmd_notify_t; + +typedef struct urma_cmd_wait_notify { + struct { + uint32_t cnt; + int timeout; + } in; + struct { + uint32_t cnt; + urma_cmd_notify_t notify[MAX_NOTIFY_CNT]; + } out; +} urma_cmd_wait_notify_t; + +#define URMA_CMD_MAX_TP_NUM 128 + +typedef struct urma_cmd_get_tp_list { + struct { + uint32_t flag; + uint32_t trans_mode; + uint8_t local_eid[URMA_CMD_EID_SIZE]; + uint8_t peer_eid[URMA_CMD_EID_SIZE]; + uint32_t tp_cnt; + uint32_t reserved; + } in; + struct { + uint32_t tp_cnt; + uint32_t reserved; + uint64_t tp_handle[URMA_CMD_MAX_TP_NUM]; + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_get_tp_list_t; + +typedef struct urma_cmd_exchange_tp_info_t { + struct { + struct urma_get_tp_cfg get_tp_cfg; + uint64_t tp_handle; + uint32_t tx_psn; + } in; + struct { + uint64_t peer_tp_handle; + uint32_t rx_psn; + } out; +} urma_cmd_exchange_tp_info_t; + +typedef struct urma_cmd_set_tp_attr { + struct { + uint64_t tp_handle; + uint8_t tp_attr_cnt; + uint32_t tp_attr_bitmap; + uint8_t tp_attr[URMA_CMD_TP_ATTR_BYTES]; + } in; + urma_cmd_udrv_priv_t udata; +} urma_cmd_set_tp_attr_t; + +typedef struct urma_cmd_get_tp_attr { + struct { + uint64_t tp_handle; + } in; + struct { + uint8_t tp_attr_cnt; + uint32_t tp_attr_bitmap; + uint8_t tp_attr[URMA_CMD_TP_ATTR_BYTES]; + } out; + urma_cmd_udrv_priv_t udata; +} urma_cmd_get_tp_attr_t; + +typedef struct urma_cmd_get_eid_by_ip { + struct { + urma_net_addr_t net_addr; + } in; + struct { + urma_eid_t eid; + } out; +} urma_cmd_get_eid_by_ip_t; + +typedef struct urma_cmd_get_ip_by_eid { + struct { + urma_eid_t eid; + } in; + struct { + urma_net_addr_t net_addr; + } out; +} urma_cmd_get_ip_by_eid_t; + +typedef struct urma_cmd_get_smac { + struct { + uint8_t mac[URMA_MAC_BYTES]; + } out; +} urma_cmd_get_smac_t; + +typedef struct urma_cmd_get_dmac { + struct { + urma_net_addr_t net_addr; + } in; + struct { + uint8_t mac[URMA_MAC_BYTES]; + } out; +} urma_cmd_get_dmac_t; + +#endif diff --git a/include/ylt/urma/urma_opcode.h b/include/ylt/urma/urma_opcode.h new file mode 100644 index 000000000..a707951d2 --- /dev/null +++ b/include/ylt/urma/urma_opcode.h @@ -0,0 +1,260 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) Huawei Technologies Co., Ltd. 2021-2025. All rights reserved. + * Description: URMA opcode header file + * Author: Ouyang Changchun, Yan Fangfang, Qian Guoxin + * Create: 2021-09-26 + * Note: + * History: 2021-09-26 Create File + */ + +#ifndef URMA_OPCODE_H +#define URMA_OPCODE_H + +#include + +/* urma bit field value */ +#define URMA_TOKEN_NONE 0 /* Indicates the verification policy of the key. */ +#define URMA_TOKEN_PLAIN_TEXT 1 +#define URMA_TOKEN_SIGNED 2 +#define URMA_TOKEN_ALL_ENCRYPTED 3 +#define URMA_TOKEN_RESERVED 4 + +#define URMA_TOKEN_ID_INVALID 0 +#define URMA_TOKEN_ID_VALID 1 + +#define URMA_DSVA_DISABLE 0 /* Indicates whether it is a segment of dsva. */ +#define URMA_DSVA_ENABLE 1 + +#define URMA_NON_CACHEABLE 0 /* Indicates whether the segment can be cached by multiple hosts. */ +#define URMA_CACHEABLE 1 + +/* If URMA_ACCESS_LOCAL_ONLY is set, local access will have all the permissions of + * READ, WRITE, and ATOMIC but external access is denied. + * If URMA_ACCESS_LOCAL_ONLY is not set, in addition to having all permissions for local access, + * the configuration of external access permissions is determined by the following three types, and + * it takes effect according to the combination of READ, WRITE, and ATOMIC configured by the user. + */ +#define URMA_ACCESS_LOCAL_ONLY (0x1 << 0) +#define URMA_ACCESS_READ (0x1 << 1) +#define URMA_ACCESS_WRITE (0x1 << 2) +#define URMA_ACCESS_ATOMIC (0x1 << 3) + +#define URMA_LOCAL_MEMORY 0 /* Indicates that the physical memory is remote. */ +#define URMA_REMOTE_MEMORY 1 + +#define URMA_SEG_NOMAP 0 /* Indicates that the current process has mapped this segment */ +#define URMA_SEG_MAPPED 1 + +#define URMA_ADDR_TYPE_MVA 0 +#define URMA_ADDR_TYPE_UBVA 1 + +#define URMA_COMPLETE_ENABLE 1 /* Notify the source after the task is completed. */ +#define URMA_COMPLETE_DISABLE 0 /* Do not notify the source after the task is complete. */ + +#define URMA_COMPLETE_TYPE_JFC 0 /* Complete notification via JFC. */ +#define URMA_COMPLETE_TYPE_CF 1 /* Complete notification via DDR address */ + +#define URMA_DEPENDENCY_NONE 0 /* There is no dependency between commands. */ +#define URMA_DEPENDENCY_FIRST \ + 1 /* Subsequent commands depend on the execution result \ + of the current command. */ +#define URMA_DEPENDENCY_DELAY \ + 2 /* The current command is executed only when the command \ + that the preamble depends on is executed successfully. */ + +#define URMA_NOTIFY_DISABLE 0 /* The destination is not notified when the task is completed. */ +#define URMA_NOTIFY_ENABLE 1 /* Notify the destination when the task is completed. */ + +#define URMA_NOTIFY_TYPE_JFC 0 /* Complete notification via JFC. */ +#define URMA_NOTIFY_TYPE_RVA 1 /* Complete notification via DDR address. */ + +#define URMA_INLINE_DISABLE 0 /* The data is generated by source_address assignment. */ +#define URMA_INLINE_ENABLE 1 /* The data is carried in the command. */ + +#define URMA_SOLICITED_DISABLE 0 /* There is no interruption when notifying through JFC. */ +#define URMA_SOLICITED_ENABLE 1 /* Interrupt occurred while notifying via JFC. */ + +#define URMA_FENCE_DISABLE 0 /* There is no fence. */ +#define URMA_FENCE_ENABLE 1 /* Fence with previous WRs. */ + +#define URMA_REGULAR 1 /* regular, specifies stride format. */ +#define URMA_IRREGULAR 0 /* irregular, specifies S/G format. */ + +#define URMA_NO_TAG_MATCHING 0 +#define URMA_WITH_TAG_MATCHING 1 + +#define URMA_NONPOST_LS 0 +#define URMA_POST_LS 1 + +#define URMA_NO_SHARE_JFR 0 +#define URMA_SHARE_JFR 1 + +#define URMA_TYPICAL_RNR_RETRY 7 /* typical value of rnr retry for jfs cfg */ +#define URMA_TYPICAL_ERR_TIMEOUT 17 /* typical value of err_timeout for jfs cfg */ +#define URMA_TYPICAL_MIN_RNR_TIMER 12 /* typical value of min_rnr_timer for jfr cfg */ +#define URMA_MAX_PRIORITY 15 + +/* operation information */ +typedef enum urma_place_order { + URMA_NO_ORDER = 0, // No order + URMA_RELAX_ORDER, // Relax order + URMA_STRONG_ORDER // Strong order +} urma_place_order_t; + +/* opcode definition */ +typedef enum urma_opcode { + URMA_OPC_WRITE = 0x00, + URMA_OPC_WRITE_IMM = 0x01, + URMA_OPC_WRITE_NOTIFY = 0x02, // not support result will return for URMA_OPC_WRITE_NOTIFY + URMA_OPC_READ = 0x10, + URMA_OPC_CAS = 0x20, + URMA_OPC_SWAP = 0x21, + URMA_OPC_FADD = 0x22, + URMA_OPC_FSUB = 0x23, + URMA_OPC_FAND = 0x24, + URMA_OPC_FOR = 0x25, + URMA_OPC_FXOR = 0x26, + URMA_OPC_SEND = 0x40, // remote JFR/jetty ID + URMA_OPC_SEND_IMM = 0x41, // remote JFR/jetty ID + URMA_OPC_SEND_INVALIDATE = 0x42, // remote JFR/jetty ID and seg token id + URMA_OPC_NOP = 0x51, + URMA_OPC_WRITE_ATOMIC = 0x60, // Non-standard definition of OPCODE + URMA_OPC_LAST +} urma_opcode_t; + +typedef int urma_status_t; +#define URMA_SUCCESS 0 +#define URMA_EAGAIN EAGAIN // Resource temporarily unavailable +#define URMA_ENOMEM ENOMEM // Failed to allocate memory +#define URMA_ENOPERM EPERM // Operation not permitted +#define URMA_ETIMEOUT ETIMEDOUT // Operation time out +#define URMA_EINVAL EINVAL // Invalid argument +#define URMA_EEXIST EEXIST // Exist +#define URMA_EINPROGRESS EINPROGRESS +#define URMA_FAIL 0x1000 /* 0x1000 */ + +/* completion information */ +typedef enum urma_cr_status { // completion record status + URMA_CR_SUCCESS = 0, + URMA_CR_UNSUPPORTED_OPCODE_ERR, /* Opcode in the WR is not supported */ + URMA_CR_LOC_LEN_ERR, /* Local data too long error */ + URMA_CR_LOC_OPERATION_ERR, /* Local operation err */ + URMA_CR_LOC_ACCESS_ERR, /* Access to local memory error */ + URMA_CR_REM_RESP_LEN_ERR, /* Local Operation Error, with sub-status of Remote Response Length Error */ + URMA_CR_REM_UNSUPPORTED_REQ_ERR, + URMA_CR_REM_OPERATION_ERR, /* Error when target jetty can not complete the operation */ + URMA_CR_REM_ACCESS_ABORT_ERR, /* Error when target jetty access memory error or abort the operation */ + URMA_CR_ACK_TIMEOUT_ERR, /* Retransmission exceeds the maximum number of times */ + URMA_CR_RNR_RETRY_CNT_EXC_ERR, /* RNR retries exceeded the maximum number: remote jfr has no buffer */ + URMA_CR_WR_FLUSH_ERR, /* Jetty in the error state, and the hardware has processed the WR. */ + URMA_CR_WR_SUSPEND_DONE, /* Hardware constructs a fake CQE, and user_ctx is invalid. */ + URMA_CR_WR_FLUSH_ERR_DONE, /* Hardware constructs a fake CQE, and user_ctx is invalid. */ + URMA_CR_WR_UNHANDLED, /* Return of flush jetty/jfs, and the hardware has not processed the WR. */ + URMA_CR_LOC_DATA_POISON, /* Local Data Poison */ + URMA_CR_REM_DATA_POISON, /* Remote Data Poison */ +} urma_cr_status_t; + +typedef enum urma_cr_opcode { + URMA_CR_OPC_SEND = 0x00, + URMA_CR_OPC_SEND_WITH_IMM, + URMA_CR_OPC_SEND_WITH_INV, + URMA_CR_OPC_WRITE_WITH_IMM, +} urma_cr_opcode_t; + +/* event information */ +typedef enum urma_async_event_type { + URMA_EVENT_JFC_ERR, + URMA_EVENT_JFS_ERR, + URMA_EVENT_JFR_ERR, + URMA_EVENT_JFR_LIMIT, + URMA_EVENT_JETTY_ERR, + URMA_EVENT_JETTY_LIMIT, + URMA_EVENT_JETTY_GRP_ERR, + URMA_EVENT_PORT_ACTIVE, + URMA_EVENT_PORT_DOWN, + URMA_EVENT_DEV_FATAL, + URMA_EVENT_EID_CHANGE, // eid change, HNM and other management roles will be modified. + URMA_EVENT_ELR_ERR, /* Entity level error */ + URMA_EVENT_ELR_DONE /* Entity flush done */ +} urma_async_event_type_t; + +typedef enum urma_jfc_state { + URMA_JFC_STATE_INVALID = 0, + URMA_JFC_STATE_VALID, + URMA_JFC_STATE_ERROR +} urma_jfc_state_t; + +typedef enum urma_jetty_state { + URMA_JETTY_STATE_RESET = 0, + URMA_JETTY_STATE_READY, + URMA_JETTY_STATE_SUSPENDED, + URMA_JETTY_STATE_ERROR +} urma_jetty_state_t; + +typedef enum urma_jfr_state { + URMA_JFR_STATE_RESET = 0, + URMA_JFR_STATE_READY, + URMA_JFR_STATE_ERROR +} urma_jfr_state_t; + +#define URMA_JFS_DEPTH 0x0001 +#define URMA_JFS_FLAG 0x0002 +#define URMA_JFS_TRANS_MODE 0x0003 +#define URMA_JFS_PRIORITY 0x0004 +#define URMA_JFS_MAX_SGE 0x0005 +#define URMA_JFS_MAX_RSGE 0x0006 +#define URMA_JFS_MAX_INLINE_DATA 0x0007 +#define URMA_JFS_RNR_RETRY 0x0008 +#define URMA_JFS_ERR_TIMEOUT 0x0009 +#define URMA_JFS_BIND_JFC 0x000a +#define URMA_JFS_USER_CTX 0x000b +#define URMA_JFS_SQE_BASE_ADDR 0x000c +#define URMA_JFS_ID 0x000d +#define URMA_JFS_DB_ADDR 0x000e +#define URMA_JFS_DB_STATUS 0x000f +#define URMA_JFS_PI 0x0010 +#define URMA_JFS_PI_TYPE 0x0011 +#define URMA_JFS_CI 0x0012 +#define URMA_JFS_FULL_CTX 0x0013 + +#define URMA_JFR_DEPTH 0x1001 +#define URMA_JFR_FLAG 0x1002 +#define URMA_JFR_TRANS_MODE 0x1003 +#define URMA_JFR_MAX_SGE 0x1004 +#define URMA_JFR_MIN_RNR_TIMER 0x1005 +#define URMA_JFR_BIND_JFC 0x1006 +#define URMA_JFR_TOKEN_VALUE 0x1007 +#define URMA_JFR_USER_CTX 0x1008 +#define URMA_JFR_RQE_BASE_ADDR 0x1009 +#define URMA_JFR_ID 0x100a +#define URMA_JFR_DB_ADDR 0x100b +#define URMA_JFR_DB_STATUS 0x100c +#define URMA_JFR_PI 0x100d +#define URMA_JFR_PI_TYPE 0x100e +#define URMA_JFR_CI 0x100f +#define URMA_JFR_FULL_CTX 0x1010 + +#define URMA_JFC_DEPTH 0x2001 +#define URMA_JFC_CEQN 0x2002 +#define URMA_JFC_FLAG 0x2003 +#define URMA_JFC_BIND_JFCE 0x2004 +#define URMA_JFC_USER_CTX 0x2005 +#define URMA_JFC_CQE_BASE_ADDR 0x2006 +#define URMA_JFC_ID 0x2007 +#define URMA_JFC_DB_ADDR 0x2008 +#define URMA_JFC_DB_STATUS 0x2009 +#define URMA_JFC_PI 0x200a +#define URMA_JFC_PI_TYPE 0x200b +#define URMA_JFC_CI 0x200c +#define URMA_JFC_FULL_CTX 0x200d + +#define URMA_JETTY_ID 0x3001 +#define URMA_JETTY_FLAG 0x3002 +#define URMA_JETTY_BIND_JFR 0x3003 +#define URMA_JETTY_BIND_RX_JFC 0x3004 +#define URMA_JETTY_BIND_JTG 0x3005 +#define URMA_JETTY_USER_CTX 0x3006 +#define URMA_JETTY_FULL_CTX 0x3007 + +#endif // URMA_OPCODE_H diff --git a/include/ylt/urma/urma_perf.h b/include/ylt/urma/urma_perf.h new file mode 100644 index 000000000..a5df5276c --- /dev/null +++ b/include/ylt/urma/urma_perf.h @@ -0,0 +1,72 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) Huawei Technologies Co., Ltd. 2021-2025. All rights reserved. + * Description: Performance monitoring and profiling for URMA + * Author: Tang Zhedong + * Create: 2026-04-08 + * Note: + * History: 2026-04-08 create file + */ + +#ifndef URMA_PERF_H +#define URMA_PERF_H + +#include +#include "urma_opcode.h" + +#define URMA_PERF_THREAD_MAX_NUM (128u) + +typedef enum urma_perf_type { + UB_JETTY_POST_SEND, + BOND_JETTY_POST_SEND, + UB_JFS_POST_SEND, + BOND_JFS_POST_SEND, + UB_JETTY_POST_RECV, + BOND_JETTY_POST_RECV, + UB_POST_JFR_RECV, + BOND_POST_JFR_RECV, + UB_POLL_JFC, + BOND_POLL_JFC, + UB_WAIT_JFC, + BOND_WAIT_JFC, + UB_ACK_JFC, + UB_REARM_JFC, + BOND_REARM_JFC, + URMA_PERF_RECORD_TYPE_MAX, +} urma_perf_record_type_t; + +typedef struct urma_perf_stats { + uint64_t retry_count; + struct { + urma_perf_record_type_t type; + uint64_t sample_num; + uint64_t average; + uint64_t mininum; + uint64_t maxinum; + uint64_t p90; + uint64_t p99; + uint64_t p9999; + } type_record[URMA_PERF_RECORD_TYPE_MAX]; +} urma_perf_stats_t; + +/** + * Start performance monitoring for urma devices. + * Return: 0 on success, other value on error + */ +urma_status_t urma_start_perf(void); + +/** + * Stop performance monitoring for urma devices. + * Return: 0 on success, other value on error + */ +urma_status_t urma_stop_perf(void); + +/** + * Get performance statistics information. + * @param[in] perf_buf: Buffer to store performance information, user needs to allocate the memory; + * @param[in] length: Pointer to buffer length, input as buffer size, output as actual data length; + * Return: 0 on success, other value on error + */ +urma_status_t urma_get_perf_info(char *perf_buf, uint32_t *length); + +#endif diff --git a/include/ylt/urma/urma_provider.h b/include/ylt/urma/urma_provider.h new file mode 100644 index 000000000..0487fbf45 --- /dev/null +++ b/include/ylt/urma/urma_provider.h @@ -0,0 +1,412 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) Huawei Technologies Co., Ltd. 2021-2025. All rights reserved. + * Description: Liburma interface open to provier + * Author: Qian Guoxin + * Create: 2021-07-31 + * Note: + * History: 2021-07-31 create file + */ + +#ifndef URMA_PROVIDER_H +#define URMA_PROVIDER_H + +#include + +#include "urma_api.h" + +#define URMA_SYSFS_DEV_FLAG_DRIVER_CREATED (0x1) +#define URMA_CFG_MASK 0 + +typedef enum { + TARGET_CFG, + TARGET_OPT, + TARGET_JFS_CFG, +} urma_field_target_t; + +typedef struct { + uint64_t opt; /* opt id (eg. URMA_JFC_DEPTH) */ + uint64_t mask; /* bit mask value for this opt (eg. URMA_JFC_DEPTH_MASK) */ + urma_field_target_t tgt; /* which sub-struct the field belongs to */ + size_t offset; /* offsetof(sub-struct, member) */ + size_t size; /* sizeof(member) */ +} opt_map_t; + +extern const opt_map_t JFS_OPT_TABLE[]; +extern const size_t JFS_OPT_MAP_COUNT; +extern const opt_map_t JFR_OPT_TABLE[]; +extern const size_t JFR_OPT_MAP_COUNT; +extern const opt_map_t JFC_OPT_TABLE[]; +extern const size_t JFC_OPT_MAP_COUNT; +extern const opt_map_t JETTY_OPT_TABLE[]; +extern const size_t JETTY_OPT_MAP_COUNT; + +typedef struct urma_match_entry { + uint16_t vendor_id; + uint16_t device_id; +} urma_match_entry_t; + +typedef struct urma_udrv { + uint64_t in_addr; + uint32_t in_len; + uint64_t out_addr; + uint32_t out_len; +} urma_udrv_t; + +typedef struct urma_ops { + /* OPs name */ + const char *name; + + /* Jetty OPs */ + urma_jfc_t *(*create_jfc)(urma_context_t *ctx, urma_jfc_cfg_t *jfc_cfg); + urma_status_t (*modify_jfc)(urma_jfc_t *jfc, urma_jfc_attr_t *attr); + urma_status_t (*delete_jfc)(urma_jfc_t *jfc); + urma_status_t (*delete_jfc_batch)(urma_jfc_t **jfc, int jfc_num, urma_jfc_t **bad_jfc); + urma_status_t (*alloc_jfc)(urma_context_t *ctx, urma_jfc_cfg_t *cfg, urma_jfc_t **jfc); + urma_status_t (*set_jfc_opt)(urma_jfc_t *jfc, uint64_t opt, void *buf, uint32_t len); + urma_status_t (*active_jfc)(urma_jfc_t *jfc); + urma_status_t (*get_jfc_opt)(urma_jfc_t *jfc, uint64_t opt, void *buf, uint32_t len); + urma_status_t (*deactive_jfc)(urma_jfc_t *jfc); + urma_status_t (*free_jfc)(urma_jfc_t *jfc); + urma_jfs_t *(*create_jfs)(urma_context_t *ctx, urma_jfs_cfg_t *jfs); + urma_status_t (*modify_jfs)(urma_jfs_t *jfs, urma_jfs_attr_t *attr); + urma_status_t (*query_jfs)(urma_jfs_t *jfs, urma_jfs_cfg_t *cfg, urma_jfs_attr_t *attr); + int (*flush_jfs)(urma_jfs_t *jfs, int cr_cnt, urma_cr_t *cr); + urma_status_t (*delete_jfs)(urma_jfs_t *jfs); + urma_status_t (*delete_jfs_batch)(urma_jfs_t **jfs_arr, int jfs_num, urma_jfs_t **bad_jfs); + urma_status_t (*alloc_jfs)(urma_context_t *ctx, urma_jfs_cfg_t *cfg, urma_jfs_t **jfs); + urma_status_t (*set_jfs_opt)(urma_jfs_t *jfs, uint64_t opt, void *buf, uint32_t len); + urma_status_t (*active_jfs)(urma_jfs_t *jfs); + urma_status_t (*get_jfs_opt)(urma_jfs_t *jfs, uint64_t opt, void *buf, uint32_t len); + urma_status_t (*deactive_jfs)(urma_jfs_t *jfs); + urma_status_t (*free_jfs)(urma_jfs_t *jfs); + urma_jfr_t *(*create_jfr)(urma_context_t *ctx, urma_jfr_cfg_t *jfr); + urma_status_t (*modify_jfr)(urma_jfr_t *jfr, urma_jfr_attr_t *attr); + urma_status_t (*query_jfr)(urma_jfr_t *jfr, urma_jfr_cfg_t *cfg, urma_jfr_attr_t *attr); + urma_status_t (*delete_jfr)(urma_jfr_t *jfr); + urma_status_t (*delete_jfr_batch)(urma_jfr_t **jfr_arr, int jfr_num, urma_jfr_t **bad_jfr); + urma_target_jetty_t *(*import_jfr)(urma_context_t *ctx, urma_rjfr_t *rjfr, urma_token_t *token); + urma_status_t (*unimport_jfr)(urma_target_jetty_t *target_jfr); + urma_status_t (*advise_jfr)(urma_jfs_t *jfs, urma_target_jetty_t *tjfr); + urma_status_t (*unadvise_jfr)(urma_jfs_t *jfs, urma_target_jetty_t *tjfr); + urma_status_t (*advise_jfr_async)(urma_jfs_t *jfs, urma_target_jetty_t *tjfr, urma_advise_async_cb_func cb_fun, + void *cb_arg); + urma_status_t (*alloc_jfr)(urma_context_t *ctx, urma_jfr_cfg_t *cfg, urma_jfr_t **jfr); + urma_status_t (*set_jfr_opt)(urma_jfr_t *jfr, uint64_t opt, void *buf, uint32_t len); + urma_status_t (*active_jfr)(urma_jfr_t *jfr); + urma_status_t (*get_jfr_opt)(urma_jfr_t *jfr, uint64_t opt, void *buf, uint32_t len); + urma_status_t (*deactive_jfr)(urma_jfr_t *jfr); + urma_status_t (*free_jfr)(urma_jfr_t *jfr); + urma_jetty_t *(*create_jetty)(urma_context_t *ctx, urma_jetty_cfg_t *jetty_cfg); + urma_status_t (*modify_jetty)(urma_jetty_t *jetty, urma_jetty_attr_t *jetty_attr); + urma_status_t (*query_jetty)(urma_jetty_t *jetty, urma_jetty_cfg_t *cfg, urma_jetty_attr_t *attr); + int (*flush_jetty)(urma_jetty_t *jetty, int cr_cnt, urma_cr_t *cr); + urma_status_t (*delete_jetty)(urma_jetty_t *jetty); + urma_status_t (*delete_jetty_batch)(urma_jetty_t **jetty_arr, int jetty_num, urma_jetty_t **bad_jetty); + urma_target_jetty_t *(*import_jetty)(urma_context_t *ctx, urma_rjetty_t *rjetty, urma_token_t *rjetty_token); + urma_status_t (*unimport_jetty)(urma_target_jetty_t *target_jetty); + urma_status_t (*advise_jetty)(urma_jetty_t *jetty, urma_target_jetty_t *tjetty); + urma_status_t (*unadvise_jetty)(urma_jetty_t *jetty, urma_target_jetty_t *tjetty); + urma_status_t (*advise_jetty_async)(urma_jetty_t *jetty, urma_target_jetty_t *tjetty, + urma_advise_async_cb_func cb_fun, void *cb_arg); + urma_status_t (*bind_jetty)(urma_jetty_t *jetty, urma_target_jetty_t *tjetty); + urma_status_t (*unbind_jetty)(urma_jetty_t *jetty); + urma_status_t (*alloc_jetty)(urma_context_t *ctx, urma_jetty_cfg_t *cfg, urma_jetty_t **jetty); + urma_status_t (*set_jetty_opt)(urma_jetty_t *jetty, uint64_t opt, void *buf, uint32_t len); + urma_status_t (*active_jetty)(urma_jetty_t *jetty); + urma_status_t (*get_jetty_opt)(urma_jetty_t *jetty, uint64_t opt, void *buf, uint32_t len); + urma_status_t (*deactive_jetty)(urma_jetty_t *jetty); + urma_status_t (*free_jetty)(urma_jetty_t *jetty); + urma_jetty_grp_t *(*create_jetty_grp)(urma_context_t *ctx, urma_jetty_grp_cfg_t *cfg); + urma_status_t (*delete_jetty_grp)(urma_jetty_grp_t *jetty_grp); + urma_jfce_t *(*create_jfce)(urma_context_t *ctx); + urma_status_t (*delete_jfce)(urma_jfce_t *jfce); + /** + * Get tpn of current jetty + * @param[in] jetty: the jetty pointer created before + * Return: 0 or positive as correct tpn; negative as get tpn failure + */ + int (*get_tpn)(urma_jetty_t *jetty); + int (*modify_tp)(urma_context_t *ctx, uint32_t tpn, urma_tp_cfg_t *cfg, urma_tp_attr_t *attr, + urma_tp_attr_mask_t mask); + /* Control plane OPs */ + urma_status_t (*get_tp_list)(urma_context_t *ctx, urma_get_tp_cfg_t *cfg, uint32_t *tp_cnt, + urma_tp_info_t *tp_list); + urma_status_t (*set_tp_attr)(const urma_context_t *ctx, const uint64_t tp_handle, const uint8_t tp_attr_cnt, + const uint32_t tp_attr_bitmap, const urma_tp_attr_value_t *tp_attr); + urma_status_t (*get_tp_attr)(const urma_context_t *ctx, const uint64_t tp_handle, uint8_t *tp_attr_cnt, + uint32_t *tp_attr_bitmap, urma_tp_attr_value_t *tp_attr); + urma_target_jetty_t *(*import_jetty_ex)(urma_context_t *ctx, urma_rjetty_t *rjetty, urma_token_t *token_value, + urma_active_tp_cfg_t *active_tp_cfg); + urma_target_jetty_t *(*import_jfr_ex)(urma_context_t *ctx, urma_rjfr_t *rjfr, urma_token_t *token_value, + urma_active_tp_cfg_t *active_tp_cfg); + urma_status_t (*bind_jetty_ex)(urma_jetty_t *jetty, urma_target_jetty_t *tjetty, + urma_active_tp_cfg_t *active_tp_cfg); + + /* Segment OPs */ + urma_token_id_t *(*alloc_token_id)(urma_context_t *ctx); + urma_token_id_t *(*alloc_token_id_ex)(urma_context_t *ctx, urma_token_id_flag_t flag); + urma_status_t (*free_token_id)(urma_token_id_t *token_id); + urma_target_seg_t *(*register_seg)(urma_context_t *ctx, urma_seg_cfg_t *seg_cfg); + urma_status_t (*unregister_seg)(urma_target_seg_t *target_seg); + urma_target_seg_t *(*import_seg)(urma_context_t *ctx, urma_seg_t *seg, urma_token_t *token, uint64_t addr, + urma_import_seg_flag_t flag); + urma_status_t (*unimport_seg)(urma_target_seg_t *target_seg); + + /* Events OPs */ + urma_status_t (*get_async_event)(urma_context_t *ctx, urma_async_event_t *event); + void (*ack_async_event)(urma_async_event_t *event); + + /* Other OPs */ + int (*user_ctl)(urma_context_t *ctx, urma_user_ctl_in_t *in, urma_user_ctl_out_t *out); + + /* Dataplane OPs */ + urma_status_t (*post_jfs_wr)(urma_jfs_t *jfs, urma_jfs_wr_t *wr, urma_jfs_wr_t **bad_wr); + urma_status_t (*post_jfr_wr)(urma_jfr_t *jfr, urma_jfr_wr_t *wr, urma_jfr_wr_t **bad_wr); + urma_status_t (*post_jetty_send_wr)(urma_jetty_t *jetty, urma_jfs_wr_t *wr, urma_jfs_wr_t **bad_wr); + urma_status_t (*post_jetty_recv_wr)(urma_jetty_t *jetty, urma_jfr_wr_t *wr, urma_jfr_wr_t **bad_wr); + int (*poll_jfc)(urma_jfc_t *jfc, int cr_cnt, urma_cr_t *cr); + urma_status_t (*rearm_jfc)(urma_jfc_t *jfc, bool solicited_only); + int (*wait_jfc)(urma_jfce_t *jfce, uint32_t jfc_cnt, int time_out, urma_jfc_t *jfc[]); + void (*ack_jfc)(urma_jfc_t *jfc[], uint32_t nevents[], uint32_t jfc_cnt); + + /* Jetty async OPs */ + urma_target_jetty_t *(*import_jetty_async)(urma_notifier_t *notifier, const urma_rjetty_t *rjetty, + const urma_token_t *token_value, uint64_t user_ctx, int timeout); + urma_status_t (*unimport_jetty_async)(urma_target_jetty_t *target_jetty); + + urma_status_t (*bind_jetty_async)(urma_notifier_t *notifier, urma_jetty_t *jetty, urma_target_jetty_t *tjetty, + uint64_t user_ctx, int timeout); + urma_status_t (*unbind_jetty_async)(urma_jetty_t *jetty); + + urma_notifier_t *(*create_notifier)(urma_context_t *ctx); + urma_status_t (*delete_notifier)(urma_notifier_t *notifier); + int (*wait_notify)(urma_notifier_t *notifier, uint32_t cnt, urma_notify_t *notify, int timeout); + void (*ack_notify)(uint32_t cnt, urma_notify_t *notify); + urma_status_t (*get_eid_by_ip)(const urma_context_t *ctx, const urma_net_addr_t *net_addr, urma_eid_t *eid); + urma_status_t (*get_ip_by_eid)(const urma_context_t *ctx, const urma_eid_t *eid, urma_net_addr_t *net_addr); + urma_status_t (*get_smac)(const urma_context_t *ctx, uint8_t *mac); + urma_status_t (*get_dmac)(const urma_context_t *ctx, const urma_net_addr_t *net_addr, uint8_t *mac); +} urma_ops_t; + +typedef struct urma_provider_attr { + uint32_t version; /* compatible with abi verison of kernel driver */ + urma_transport_type_t transport_type; +} urma_provider_attr_t; + +typedef struct urma_provider_ops { + const char *name; + urma_provider_attr_t attr; + urma_match_entry_t *match_table; + urma_status_t (*init)(urma_init_attr_t *conf); + urma_status_t (*uninit)(void); + /* Device OPs */ + urma_status_t (*query_device)(urma_device_t *dev, urma_device_attr_t *dev_attr); + urma_context_t *(*create_context)(urma_device_t *dev, uint32_t eid_index, int dev_fd); + urma_status_t (*delete_context)(urma_context_t *ctx); + urma_status_t (*get_uasid)(uint32_t *uasid); /* obsolete */ +} urma_provider_ops_t; + +typedef struct urma_import_tseg_cfg { + urma_ubva_t ubva; + uint64_t len; + urma_seg_attr_t attr; + uint32_t token_id; + urma_token_t *token; + urma_import_seg_flag_t flag; + uint64_t mva; +} urma_import_tseg_cfg_t; + +typedef struct urma_tjfr_cfg { + urma_jfr_id_t jfr_id; + urma_import_jetty_flag_t flag; + urma_token_t *token; + urma_transport_mode_t trans_mode; + urma_tp_type_t tp_type; +} urma_tjfr_cfg_t; + +typedef struct urma_tjetty_cfg { + urma_jetty_id_t jetty_id; + urma_import_jetty_flag_t flag; + urma_token_t *token; + urma_transport_mode_t trans_mode; + urma_jetty_grp_policy_t policy; + urma_target_type_t type; + urma_tp_type_t tp_type; +} urma_tjetty_cfg_t; + +typedef struct urma_context_cfg { + struct urma_device *dev; + struct urma_ops *ops; + uint32_t eid_index; + int dev_fd; + uint32_t uasid; +} urma_context_cfg_t; + +#ifndef URMA_CMD_UDRV_PRIV +#define URMA_CMD_UDRV_PRIV +typedef struct urma_cmd_udrv_priv { + uint64_t in_addr; + uint32_t in_len; + uint64_t out_addr; + uint32_t out_len; +} urma_cmd_udrv_priv_t; +#endif + +typedef struct urma_post_and_ret_db_in { + bool is_jetty; + union { + urma_jfs_t *jfs; + urma_jetty_t *jetty; + }; + urma_jfs_wr_t *wr; +} urma_post_and_ret_db_in_t; + +typedef struct urma_post_and_ret_db_out { + urma_jfs_wr_t **bad_wr; + uint64_t db_addr; + uint64_t db_data; +} urma_post_and_ret_db_out_t; + +int urma_register_provider_ops(urma_provider_ops_t *provider_ops); +int urma_unregister_provider_ops(urma_provider_ops_t *provider_ops); +ssize_t urma_read_sysfs_file(const char *dir, const char *file, char *buf, size_t size); + +int urma_cmd_create_context(urma_context_t *ctx, urma_context_cfg_t *cfg, urma_cmd_udrv_priv_t *udata); +int urma_cmd_delete_context(urma_context_t *ctx); + +/* Return jfce fd */ +int urma_cmd_create_jfce(urma_context_t *ctx); + +int urma_cmd_create_jfc(urma_context_t *ctx, urma_jfc_t *jfc, urma_jfc_cfg_t *cfg, urma_cmd_udrv_priv_t *udata); +int urma_cmd_modify_jfc(urma_jfc_t *jfc, urma_jfc_attr_t *attr, urma_cmd_udrv_priv_t *udata); +int urma_cmd_delete_jfc(urma_jfc_t *jfc); +int urma_cmd_delete_jfc_batch(urma_jfc_t **jfc_arr, int jfc_num, urma_jfc_t **bad_jfc); + +/* Return number of events on success, -1 on error */ +int urma_cmd_wait_jfc(int jfce_fd, uint32_t jfc_cnt, int time_out, urma_jfc_t *jfc[]); +void urma_cmd_ack_jfc(urma_jfc_t *jfc[], uint32_t nevents[], uint32_t jfc_cnt); +int urma_cmd_alloc_jfc(urma_context_t *ctx, urma_jfc_cfg_t *cfg, urma_jfc_t *jfc, urma_cmd_udrv_priv_t *udata); +int urma_cmd_set_jfc_opt(urma_jfc_t *jfc, uint64_t opt, void *buf, uint32_t len, urma_cmd_udrv_priv_t *udata); +int urma_cmd_active_jfc(urma_jfc_t *jfc, urma_cmd_udrv_priv_t *udata); +int urma_cmd_get_jfc_opt(urma_jfc_t *jfc, uint64_t opt, void *buf, uint32_t len, urma_cmd_udrv_priv_t *udata); +int urma_cmd_deactive_jfc(urma_jfc_t *jfc, urma_cmd_udrv_priv_t *udata); +int urma_cmd_free_jfc(urma_jfc_t *jfc, urma_cmd_udrv_priv_t *udata); + +int urma_cmd_create_jfs(urma_context_t *ctx, urma_jfs_t *jfs, urma_jfs_cfg_t *cfg, urma_cmd_udrv_priv_t *udata); +int urma_cmd_modify_jfs(urma_jfs_t *jfs, urma_jfs_attr_t *attr, urma_cmd_udrv_priv_t *udata); +int urma_cmd_query_jfs(urma_jfs_t *jfs, urma_jfs_cfg_t *cfg, urma_jfs_attr_t *attr); +int urma_cmd_delete_jfs(urma_jfs_t *jfs); +int urma_cmd_delete_jfs_batch(urma_jfs_t **jfs_arr, int jfs_num, urma_jfs_t **bad_jfs); +int urma_cmd_alloc_jfs(urma_context_t *ctx, urma_jfs_cfg_t *cfg, urma_jfs_t *jfs, urma_cmd_udrv_priv_t *udata); +int urma_cmd_set_jfs_opt(urma_jfs_t *jfs, uint64_t opt, void *buf, uint32_t len, urma_cmd_udrv_priv_t *udata); +int urma_cmd_active_jfs(urma_jfs_t *jfs, urma_cmd_udrv_priv_t *udata); +int urma_cmd_get_jfs_opt(urma_jfs_t *jfs, uint64_t opt, void *buf, uint32_t len, urma_cmd_udrv_priv_t *udata); +int urma_cmd_deactive_jfs(urma_jfs_t *jfs, urma_cmd_udrv_priv_t *udata); +int urma_cmd_free_jfs(urma_jfs_t *jfs, urma_cmd_udrv_priv_t *udata); + +int urma_cmd_create_jfr(urma_context_t *ctx, urma_jfr_t *jfr, urma_jfr_cfg_t *cfg, urma_cmd_udrv_priv_t *udata); +int urma_cmd_modify_jfr(urma_jfr_t *jfr, urma_jfr_attr_t *attr, urma_cmd_udrv_priv_t *udata); +int urma_cmd_query_jfr(urma_jfr_t *jfr, urma_jfr_cfg_t *cfg, urma_jfr_attr_t *attr); +int urma_cmd_delete_jfr(urma_jfr_t *jfr); +int urma_cmd_delete_jfr_batch(urma_jfr_t **jfr_arr, int jfr_num, urma_jfr_t **bad_jfr); +int urma_cmd_alloc_jfr(urma_context_t *ctx, urma_jfr_cfg_t *cfg, urma_jfr_t *jfr, urma_cmd_udrv_priv_t *udata); +int urma_cmd_set_jfr_opt(urma_jfr_t *jfr, uint64_t opt, void *buf, uint32_t len, urma_cmd_udrv_priv_t *udata); +int urma_cmd_active_jfr(urma_jfr_t *jfr, urma_cmd_udrv_priv_t *udata); +int urma_cmd_get_jfr_opt(urma_jfr_t *jfr, uint64_t opt, void *buf, uint32_t len, urma_cmd_udrv_priv_t *udata); +int urma_cmd_deactive_jfr(urma_jfr_t *jfr, urma_cmd_udrv_priv_t *udata); +int urma_cmd_free_jfr(urma_jfr_t *jfr, urma_cmd_udrv_priv_t *udata); + +int urma_cmd_import_jfr(urma_context_t *ctx, urma_target_jetty_t *tjfr, urma_tjfr_cfg_t *cfg, + urma_cmd_udrv_priv_t *udata); +int urma_cmd_import_jfr_ex(urma_context_t *ctx, urma_target_jetty_t *tjfr, urma_tjfr_cfg_t *cfg, + urma_import_jfr_ex_cfg_t *ex_cfg, urma_cmd_udrv_priv_t *udata); +int urma_cmd_unimport_jfr(urma_target_jetty_t *tjfr); + +/* Advise cmds */ +int urma_cmd_advise_jfr(urma_jfs_t *jfs, urma_target_jetty_t *tjfr, urma_cmd_udrv_priv_t *udata); +int urma_cmd_unadvise_jfr(urma_jfs_t *jfs, urma_target_jetty_t *tjfr); + +int urma_cmd_create_jetty(urma_context_t *ctx, urma_jetty_t *jetty, urma_jetty_cfg_t *cfg, urma_cmd_udrv_priv_t *udata); +int urma_cmd_modify_jetty(urma_jetty_t *jetty, urma_jetty_attr_t *attr, urma_cmd_udrv_priv_t *udata); +int urma_cmd_query_jetty(urma_jetty_t *jetty, urma_jetty_cfg_t *cfg, urma_jetty_attr_t *attr); +int urma_cmd_delete_jetty(urma_jetty_t *jetty); +int urma_cmd_delete_jetty_batch(urma_jetty_t **jetty_arr, int jetty_num, urma_jetty_t **bad_jetty); +int urma_cmd_alloc_jetty(urma_context_t *ctx, urma_jetty_cfg_t *cfg, urma_jetty_t *jetty, urma_cmd_udrv_priv_t *udata); +int urma_cmd_set_jetty_opt(urma_jetty_t *jetty, uint64_t opt, void *buf, uint32_t len, urma_cmd_udrv_priv_t *udata); +int urma_cmd_active_jetty(urma_jetty_t *jetty, urma_cmd_udrv_priv_t *udata); +int urma_cmd_get_jetty_opt(urma_jetty_t *jetty, uint64_t opt, void *buf, uint32_t len, urma_cmd_udrv_priv_t *udata); +int urma_cmd_deactive_jetty(urma_jetty_t *jetty, urma_cmd_udrv_priv_t *udata); +int urma_cmd_free_jetty(urma_jetty_t *jetty, urma_cmd_udrv_priv_t *udata); + +int urma_cmd_import_jetty(urma_context_t *ctx, urma_target_jetty_t *tjetty, urma_tjetty_cfg_t *cfg, + urma_cmd_udrv_priv_t *udata); +int urma_cmd_import_jetty_ex(urma_context_t *ctx, urma_target_jetty_t *tjetty, urma_tjetty_cfg_t *cfg, + urma_import_jetty_ex_cfg_t *ex_cfg, urma_cmd_udrv_priv_t *udata); +int urma_cmd_unimport_jetty(urma_target_jetty_t *tjetty); + +int urma_cmd_advise_jetty(urma_jetty_t *jetty, urma_target_jetty_t *tjetty, urma_cmd_udrv_priv_t *udata); +int urma_cmd_unadvise_jetty(urma_jetty_t *jetty, urma_target_jetty_t *tjetty); + +int urma_cmd_bind_jetty(urma_jetty_t *jetty, urma_target_jetty_t *tjetty, urma_cmd_udrv_priv_t *udata); +int urma_cmd_bind_jetty_ex(urma_jetty_t *jetty, urma_target_jetty_t *tjetty, urma_bind_jetty_ex_cfg_t *ex_cfg, + urma_cmd_udrv_priv_t *udata); +int urma_cmd_unbind_jetty(urma_jetty_t *jetty); + +int urma_cmd_create_jetty_grp(urma_context_t *ctx, urma_jetty_grp_t *jetty_grp, urma_jetty_grp_cfg_t *cfg, + urma_cmd_udrv_priv_t *udata); +int urma_cmd_delete_jetty_grp(urma_jetty_grp_t *jetty_grp); + +int urma_cmd_alloc_token_id(urma_context_t *ctx, urma_token_id_t *token_id, urma_cmd_udrv_priv_t *udata); +int urma_cmd_alloc_token_id_ex(urma_context_t *ctx, urma_token_id_t *token_id, urma_token_id_flag_t flag, + urma_cmd_udrv_priv_t *udata); +int urma_cmd_free_token_id(urma_token_id_t *token_id); + +int urma_cmd_register_seg(urma_context_t *ctx, urma_target_seg_t *tseg, urma_seg_cfg_t *cfg, + urma_cmd_udrv_priv_t *udata); +int urma_cmd_unregister_seg(urma_target_seg_t *tseg); + +int urma_cmd_import_seg(urma_context_t *ctx, urma_target_seg_t *tseg, urma_import_tseg_cfg_t *cfg, + urma_cmd_udrv_priv_t *udata); +int urma_cmd_unimport_seg(urma_target_seg_t *tseg); + +urma_status_t urma_cmd_get_async_event(urma_context_t *ctx, urma_async_event_t *event); +void urma_cmd_ack_async_event(urma_async_event_t *event); + +/* Return user control res, for 0 on success, others on error */ +int urma_cmd_user_ctl(urma_context_t *ctx, urma_user_ctl_in_t *in, urma_user_ctl_out_t *out, urma_udrv_t *udrv_data); +int urma_cmd_get_eid_list(int dev_fd, uint32_t max_eid_cnt, urma_eid_info_t *eid_list, uint32_t *eid_cnt); +int urma_cmd_get_net_addr_list(urma_context_t *ctx, uint32_t max_netaddr_cnt, urma_net_addr_info_t *net_addr_info, + uint32_t *cnt); +int urma_cmd_modify_tp(urma_context_t *ctx, uint32_t tpn, urma_tp_cfg_t *cfg, urma_tp_attr_t *attr, + urma_tp_attr_mask_t mask); +struct urma_sysfs_dev; +int urma_cmd_query_device_attr(int dev_fd, struct urma_sysfs_dev *sysfs_dev); +int urma_register_sysfs_dev(struct urma_sysfs_dev *dev); + +int urma_cmd_import_jetty_async(urma_notifier_t *notifier, urma_target_jetty_t *tjetty, urma_tjetty_cfg_t *cfg, + uint64_t user_ctx, int timeout, urma_cmd_udrv_priv_t *udata); +int urma_cmd_unimport_jetty_async(urma_target_jetty_t *tjetty); + +int urma_cmd_bind_jetty_async(urma_notifier_t *notifier, urma_jetty_t *jetty, urma_target_jetty_t *tjetty, + uint64_t user_ctx, int timeout, urma_cmd_udrv_priv_t *udata); +int urma_cmd_unbind_jetty_async(urma_jetty_t *jetty); + +int urma_cmd_create_notifier(urma_context_t *ctx); +int urma_cmd_wait_notify(urma_notifier_t *notifier, uint32_t cnt, urma_notify_t *notify, int timeout); + +int urma_cmd_get_tp_list(urma_context_t *ctx, urma_get_tp_cfg_t *cfg, uint32_t *tp_cnt, urma_tp_info_t *tp_list, + urma_cmd_udrv_priv_t *udata); +int urma_cmd_set_tp_attr(const urma_context_t *ctx, const uint64_t tp_handle, const uint8_t tp_attr_cnt, + const uint32_t tp_attr_bitmap, const urma_tp_attr_value_t *tp_attr, + urma_cmd_udrv_priv_t *udata); +int urma_cmd_get_tp_attr(const urma_context_t *ctx, const uint64_t tp_handle, uint8_t *tp_attr_cnt, + uint32_t *tp_attr_bitmap, urma_tp_attr_value_t *tp_attr, urma_cmd_udrv_priv_t *udata); +int urma_cmd_exchange_tp_info(urma_context_t *ctx, urma_get_tp_cfg_t *cfg, uint64_t local_tp_handle, uint32_t tx_psn, + uint64_t *peer_tp_handle, uint32_t *rx_psn); +int urma_cmd_get_eid_by_ip(const urma_context_t *ctx, const urma_net_addr_t *net_addr, urma_eid_t *eid); +int urma_cmd_get_ip_by_eid(const urma_context_t *ctx, const urma_eid_t *eid, urma_net_addr_t *net_addr); +int urma_cmd_get_smac(const urma_context_t *ctx, uint8_t *mac); +int urma_cmd_get_dmac(const urma_context_t *ctx, const urma_net_addr_t *net_addr, uint8_t *mac); + +#endif diff --git a/include/ylt/urma/urma_types.h b/include/ylt/urma/urma_types.h new file mode 100644 index 000000000..e62058b28 --- /dev/null +++ b/include/ylt/urma/urma_types.h @@ -0,0 +1,1430 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) Huawei Technologies Co., Ltd. 2021-2025. All rights reserved. + * Description: URMA type header file + * Author: Ouyang Changchun, Bojie Li, Yan Fangfang, Qian Guoxin + * Create: 2021-07-13 + * Note: + * History: 2021-07-13 Create File + */ + +#ifndef URMA_TYPES_H +#define URMA_TYPES_H + +#include +#include +#include +#include +#include + +#ifndef __cplusplus +#include +#else +#include +#endif + +#include "urma_opcode.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define URMA_GET_VERSION(a, b) (((a) << 16) + ((b) > 65535 ? 65535 : (b))) +#define URMA_API_VERSION ((0 << 16) + 9) // Current Version: 0.9 +#define MAX_PORT_CNT 8 +#define URMA_MAX_JETTY_IN_JETTY_GRP 32U +#define URMA_MAX_NAME 64 +#define URMA_MAX_PATH 4096 +#define URMA_EID_SIZE (16) +#define URMA_MAX_PRIORITY_CNT 16 +#define URMA_IPV4_MAP_IPV6_PREFIX (0x0000ffff) +#define URMA_MAX_EID_CNT 1024 /* refer to UBCORE_MAX_SIP */ +#define URMA_CC_IDX_TABLE_SIZE 81 /* support 9 priorities and 9 algorithms */ + /* same as UBCORE_CC_IDX_TABLE_SIZE */ +#define URMA_OPT_REVERSED_NUM 4 + +#define URMA_EID_STR_LEN (39) +#define EID_FMT "%2.2x%2.2x:%2.2x%2.2x:%2.2x%2.2x:%2.2x%2.2x:%2.2x%2.2x:%2.2x%2.2x:%2.2x%2.2x:%2.2x%2.2x" +#define EID_RAW_ARGS(eid) \ + eid[0], eid[1], eid[2], eid[3], eid[4], eid[5], eid[6], eid[7], eid[8], eid[9], eid[10], eid[11], eid[12], \ + eid[13], eid[14], eid[15] +#define EID_ARGS(eid) EID_RAW_ARGS((eid).raw) +#define URMA_SEG_TOKEN_ID_INVALID 0xffffffff + +/* refer to UBCORE_MAX_DEV_NAME */ +#define URMA_MAX_DEV_NAME 64 +#define URMA_GUID_SIZE (16) + +#define URMA_IP_ADDR_BYTES 16 /* refer to UBCORE_IP_ADDR_BYTES */ +#define URMA_MAC_BYTES 6 /* refer to UBCORE_MAC_BYTES */ + +#define URMA_JFS_SQE_BASE_ADDR_MASK (1ULL << 0) +#define URMA_JFS_ID_MASK (1ULL << 1) +#define URMA_JFS_DB_ADDR_MASK (1ULL << 2) +#define URMA_JFS_DB_STATUS_MASK (1ULL << 3) +#define URMA_JFS_PI_MASK (1ULL << 4) +#define URMA_JFS_PI_TYPE_MASK (1ULL << 5) +#define URMA_JFS_CI_MASK (1ULL << 6) + +#define URMA_JFR_RQE_BASE_ADDR_MASK (1ULL << 0) +#define URMA_JFR_ID_MASK (1ULL << 1) +#define URMA_JFR_DB_ADDR_MASK (1ULL << 2) +#define URMA_JFR_DB_STATUS_MASK (1ULL << 3) +#define URMA_JFR_PI_MASK (1ULL << 4) +#define URMA_JFR_PI_TYPE_MASK (1ULL << 5) +#define URMA_JFR_CI_MASK (1ULL << 6) + +#define URMA_JFC_CQE_BASE_ADDR_MASK (1ULL << 0) +#define URMA_JFC_ID_MASK (1ULL << 1) +#define URMA_JFC_DB_ADDR_MASK (1ULL << 2) +#define URMA_JFC_DB_STATUS_MASK (1ULL << 3) +#define URMA_JFC_PI_MASK (1ULL << 4) +#define URMA_JFC_PI_TYPE_MASK (1ULL << 5) +#define URMA_JFC_CI_MASK (1ULL << 6) + +typedef struct urma_init_attr { + uint64_t token; /* [Optional] security token */ + uint32_t uasid; /* [Optional] uasid to set and reserve. If the parameter is 0, + the system will randomly assign a non-0 value. */ +} urma_init_attr_t; + +/* device information */ +typedef enum urma_mtu { + URMA_MTU_256 = 1, + URMA_MTU_512, + URMA_MTU_1024, + URMA_MTU_2048, + URMA_MTU_4096, + URMA_MTU_8192, +} urma_mtu_t; + +typedef enum urma_port_state { + URMA_PORT_NOP = 0, + URMA_PORT_DOWN, + URMA_PORT_INIT, + URMA_PORT_ARMED, + URMA_PORT_ACTIVE, + URMA_PORT_ACTIVE_DEFER, +} urma_port_state_t; + +typedef enum urma_speed { + URMA_SP_10M = 0, + URMA_SP_100M, + URMA_SP_1G, + URMA_SP_2_5G, + URMA_SP_5G, + URMA_SP_10G, + URMA_SP_14G, + URMA_SP_25G, + URMA_SP_40G, + URMA_SP_50G, + URMA_SP_100G, + URMA_SP_200G, + URMA_SP_400G, + URMA_SP_800G, +} urma_speed_t; + +typedef enum urma_link_width { + URMA_LINK_X1 = 0x1, + URMA_LINK_X2 = 0x1 << 1, + URMA_LINK_X4 = 0x1 << 2, + URMA_LINK_X8 = 0x1 << 3, + URMA_LINK_X16 = 0x1 << 4, + URMA_LINK_X32 = 0x1 << 5, +} urma_link_width_t; + +typedef union urma_eid { + uint8_t raw[URMA_EID_SIZE]; /* Network Order */ + struct { + uint64_t reserved; /* If IPv4 mapped to IPv6, == 0 */ + uint32_t prefix; /* If IPv4 mapped to IPv6, == 0x0000ffff */ + uint32_t addr; /* If IPv4 mapped to IPv6, == IPv4 addr */ + } in4; + struct { + uint64_t subnet_prefix; + uint64_t interface_id; + } in6; +} urma_eid_t; + +void urma_u32_to_eid(uint32_t ipv4, urma_eid_t *eid); +int urma_str_to_eid(const char *buf, urma_eid_t *eid); + +typedef struct urma_ref { +#ifndef __cplusplus + atomic_ulong atomic_cnt; +#else + std::atomic_ulong atomic_cnt; +#endif +} urma_ref_t; + +typedef struct urma_port_attr { + urma_mtu_t max_mtu; /* [Public] MTU_256, MTU_512, MTU_1024 etc. */ + urma_port_state_t state; /* [Public] PORT_DOWN, PORT_INIT, PORT_ACTIVE */ + urma_link_width_t active_width; /* [Public] link width: X1, X2, X4. */ + urma_speed_t active_speed; /* [Public] bandwidth. */ + urma_mtu_t active_mtu; /* [Public] current effective mtu. */ +} urma_port_attr_t; + +union urma_tp_type_en { + struct { + uint32_t rtp : 1; + uint32_t ctp : 1; + uint32_t utp : 1; + uint32_t reserved : 29; + } bs; + uint32_t value; +}; + +struct urma_sl_info { + uint32_t SL; + union urma_tp_type_en tp_type; +}; + +typedef union urma_device_feature { + struct { + uint32_t oor : 1; /* [Public] URMA_OUT_OF_ORDER_RECEIVING. */ + uint32_t jfc_per_wr : 1; /* [Public] URMA_JFC_PER_WR. */ + uint32_t stride_op : 1; /* [Public] URMA_STRIDE_OP. */ + uint32_t load_store_op : 1; /* [Public] URMA_LOAD_STORE_OP. */ + uint32_t non_pin : 1; /* [Public] URMA_NON_PIN. */ + uint32_t pmem : 1; /* [Public] URMA_PERSISTENCE_MEM. */ + uint32_t jfc_inline : 1; /* [Public] URMA_JFC_INLINE. */ + uint32_t spray_en : 1; /* [Public] URMA_SPRAY_ENABLE for UDP port. */ + uint32_t selective_retrans : 1; /* [Public] URMA_SELECTIVE_RETRANS. */ + uint32_t live_migrate : 1; /* [Public] support live migration. */ + uint32_t dca : 1; /* [Public] for user tp */ + uint32_t jetty_grp : 1; /* [Public] support jetty group. */ + uint32_t error_suspend : 1; /* [Public] support suspend jetty or jfs on error. */ + uint32_t outorder_comp : 1; /* [Public] support out-of-order completion. */ + uint32_t mn : 1; /* [Public] for user tp */ + uint32_t clan : 1; /* [Public] for user tp */ + uint32_t muti_seg_per_token_id : 1; + uint32_t ipourma_en : 1; + uint32_t ctp_en : 1; + uint32_t uboe : 1; + uint32_t reserved : 12; + } bs; + uint32_t value; +} urma_device_feature_t; + +typedef union urma_atomic_feature { + struct { + uint32_t cas : 1; + uint32_t swap : 1; + uint32_t fetch_and_add : 1; + uint32_t fetch_and_sub : 1; + uint32_t fetch_and_and : 1; + uint32_t fetch_and_or : 1; + uint32_t fetch_and_xor : 1; + uint32_t reserved : 25; + } bs; + uint32_t value; +} urma_atomic_feature_t; + +typedef union urma_order_type_cap { + struct { + uint32_t ot : 1; + uint32_t oi : 1; + uint32_t ol : 1; + uint32_t no : 1; + uint32_t reserved : 28; + } bs; + uint32_t value; +} urma_order_type_cap_t; + +typedef union urma_tp_type_cap { + struct { + uint32_t rtp : 1; + uint32_t ctp : 1; + uint32_t utp : 1; + uint32_t reserved : 29; + } bs; + uint32_t value; +} urma_tp_type_cap_t; + +typedef union urma_tp_feature { + struct { + uint32_t rm_multi_path : 1; + uint32_t rc_multi_path : 1; + uint32_t reserved : 30; + } bs; + uint32_t value; +} urma_tp_feature_t; + +typedef struct urma_device_cap { + urma_device_feature_t feature; /* [Public] support feature of device, such as OOO, LS etc. */ + uint32_t max_jfc; /* [Public] max number of jfc supported by the device. */ + uint32_t max_jfs; /* [Public] max number of jfs supported by the device. */ + uint32_t max_jfr; /* [Public] max number of jfr supported by the device. */ + uint32_t max_jetty; /* [Public] max number of jetty supported by the device. */ + uint32_t max_jetty_grp; /* [Public] max number of jetty group supported by the device. */ + uint32_t max_jetty_in_jetty_grp; /* [Public] max number of jetty per jetty group supported by the device. */ + uint32_t max_jfc_depth; /* [Public] max depth of jfc supported by the device. */ + uint32_t max_jfs_depth; /* [Public] max depth of jfs supported by the device. */ + uint32_t max_jfr_depth; /* [Public] max depth of jfr supported by the device. */ + uint32_t max_jfs_inline_len; /* [Public] max inline length(byte) supported by the jfs. */ + uint32_t max_jfs_sge; /* [Public] max number of sge supported by the jfs. */ + uint32_t max_jfs_rsge; /* [Public] max number of remote sge supported by the jfs. */ + uint32_t max_jfr_sge; /* [Public] max number of sge supported by the jfr. */ + uint64_t max_msg_size; /* [Public] max message size supported by the device. */ + uint32_t max_read_size; + uint32_t max_write_size; + uint32_t max_cas_size; + uint32_t max_swap_size; + uint32_t max_fetch_and_add_size; + uint32_t max_fetch_and_sub_size; + uint32_t max_fetch_and_and_size; + uint32_t max_fetch_and_or_size; + uint32_t max_fetch_and_xor_size; + urma_atomic_feature_t atomic_feat; /* [Public] support atomic feature of device */ + uint16_t trans_mode; /* [Public] bit OR of supported transport modes */ + uint16_t congestion_ctrl_alg; /* [Public] one or more mode from urma_congestion_ctrl_alg_t */ + uint32_t ceq_cnt; /* [Public] ceq_cnt */ + uint32_t max_tp_in_tpg; /* [Public] max tp in tpg */ + uint32_t max_eid_cnt; /* [Public] max eid count */ + uint64_t page_size_cap; /* [Public] page size capability, must include PAGE_SIZE(4k) */ + uint32_t max_oor_cnt; /* [Public] max OOR window size by packet, only for user tp */ + uint32_t mn; /* [Public] only for user tp */ + uint32_t max_netaddr_cnt; /* [Public] only for user tp */ + urma_order_type_cap_t rm_order_cap; + urma_order_type_cap_t rc_order_cap; + urma_tp_type_cap_t rm_tp_cap; + urma_tp_type_cap_t rc_tp_cap; + urma_tp_type_cap_t um_tp_cap; + urma_tp_feature_t tp_feature; + struct urma_sl_info priority_info[URMA_MAX_PRIORITY_CNT]; +} urma_device_cap_t; + +typedef struct urma_guid { + uint8_t raw[URMA_GUID_SIZE]; +} urma_guid_t; + +typedef struct urma_device_attr { + urma_guid_t guid; /* [Public] */ + urma_device_cap_t dev_cap; /* [Public] capabilities of device. */ + uint8_t port_cnt; /* [Public] port number of device. */ + struct urma_port_attr port_attr[MAX_PORT_CNT]; + uint32_t reserved_jetty_id_min; + uint32_t reserved_jetty_id_max; +} urma_device_attr_t; + +/* security information */ +typedef struct urma_token { + uint32_t token; +} urma_token_t; + +struct urma_sysfs_dev; +struct urma_ref; +struct urma_ops; +struct urma_provider_ops; + +typedef enum urma_transport_type { + URMA_TRANSPORT_INVALID = -1, + URMA_TRANSPORT_UB = 0, + URMA_TRANSPORT_MAX +} urma_transport_type_t; + +typedef enum urma_transport_mode { + URMA_TM_RM = 0x1, /* Reliable message */ + URMA_TM_RC = 0x1 << 1, /* Reliable connection */ + URMA_TM_UM = 0x1 << 2, /* Unreliable message */ +} urma_transport_mode_t; + +typedef enum urma_tp_cc_alg { + URMA_TP_CC_NONE = 0, + URMA_TP_CC_DCQCN, + URMA_TP_CC_DCQCN_AND_NETWORK_CC, + URMA_TP_CC_LDCP, + URMA_TP_CC_LDCP_AND_CAQM, + URMA_TP_CC_LDCP_AND_OPEN_CC, + URMA_TP_CC_HC3, + URMA_TP_CC_DIP, + URMA_TP_CC_ACC, + URMA_TP_CC_NUM, +} urma_tp_cc_alg_t; /* larger means better */ + +typedef enum urma_congestion_ctrl_alg { + URMA_CC_NONE = 0x1 << URMA_TP_CC_NONE, + URMA_CC_DCQCN = 0x1 << URMA_TP_CC_DCQCN, + URMA_CC_DCQCN_AND_NETWORK_CC = 0x1 << URMA_TP_CC_DCQCN_AND_NETWORK_CC, + URMA_CC_LDCP = 0x1 << URMA_TP_CC_LDCP, + URMA_CC_LDCP_AND_CAQM = 0x1 << URMA_TP_CC_LDCP_AND_CAQM, + URMA_CC_LDCP_AND_OPEN_CC = 0x1 << URMA_TP_CC_LDCP_AND_OPEN_CC, + URMA_CC_HC3 = 0x1 << URMA_TP_CC_HC3, + URMA_CC_DIP = 0x1 << URMA_TP_CC_DIP, + URMA_CC_ACC = 0x1 << URMA_TP_CC_ACC +} urma_congestion_ctrl_alg_t; + +typedef struct urma_cc_entry { + urma_tp_cc_alg_t alg; + uint8_t cc_pattern_idx; + uint8_t cc_priority; +} __attribute__((packed)) urma_cc_entry_t; + +typedef struct urma_device { + char name[URMA_MAX_NAME]; /* [Public] urma device's name, the names of devices + in different transport modes are different. */ + char path[URMA_MAX_PATH]; /* [Public] urma device's path in sysfs. */ + urma_transport_type_t type; /* [Public] urma device's transport type. */ + struct urma_provider_ops *ops; /* [Private] urma device driver's ops. */ + struct urma_sysfs_dev *sysfs_dev; /* [Private] internal device corresponding to the urma device */ +} urma_device_t; + +typedef enum urma_context_opt_name { + URMA_OPT_AGGR_MODE, +} urma_opt_name_t; + +typedef enum urma_context_aggr_mode { + URMA_AGGR_MODE_STANDALONE, + URMA_AGGR_MODE_ACTIVE_BACKUP, + URMA_AGGR_MODE_BALANCE, +} urma_context_aggr_mode_t; + +typedef struct urma_context { + struct urma_device *dev; /* [Private] point to the corresponding urma device. */ + struct urma_ops *ops; /* [Private] operation of urma device. */ + int dev_fd; /* [Private] fd of urma device's sysfs file. */ + int async_fd; /* [Private] fd of urma device's async event file. */ + pthread_mutex_t mutex; /* [Private] mutex of urma context. */ + urma_eid_t eid; /* [Public] eid of urma device. */ + uint32_t eid_index; + uint32_t uasid; /* [Public] uasid of current process. */ + struct urma_ref ref; /* [Private] reference count of urma context. */ + urma_context_aggr_mode_t aggr_mode; /* [Public] aggregated mode of urma context. */ +} urma_context_t; + +typedef struct urma_eid_info { + urma_eid_t eid; + uint32_t eid_index; /* 0~UBCORE_MAX_EID_CNT -1 */ +} urma_eid_info_t; + +typedef struct urma_jfce_cfg { + uint32_t depth; + uint64_t user_ctx; +} urma_jfce_cfg_t; + +typedef struct urma_jfce { + urma_context_t *urma_ctx; /* [Private] point to urma context. */ + int fd; /* [Private] fd of completed event. */ + struct urma_ref ref; /* [Private] reference count of urma context. */ +} urma_jfce_t; + +typedef union urma_jfc_flag { + struct { + uint32_t lock_free : 1; + uint32_t jfc_inline : 1; + uint32_t non_blocking : 1; + uint32_t has_drv_ext : 1; + uint32_t reserved : 28; + } bs; + uint32_t value; +} urma_jfc_flag_t; + +typedef struct urma_jfc_cfg { + uint32_t depth; /* [Required] the depth of jfc, no greater than urma_device_cap_t->jfc_depth */ + urma_jfc_flag_t flag; /* [Optional] see urma_jfc_flag_t, set flag.value to be 0 by default */ + uint32_t ceqn; /* [Optional] event queue id, less than urma_device_cap_t->ceq_cnt + set to 0 by default */ + urma_jfce_t *jfce; /* [Required] the event of jfc */ + uint64_t user_ctx; /* [Optional] private data of jfc, set to NULL by default */ +} urma_jfc_cfg_t; + +typedef enum urma_jfc_attr_mask { + JFC_MODERATE_COUNT = 0x1, + JFC_MODERATE_PERIOD = 0x1 << 1 +} urma_jfc_attr_mask_t; + +typedef struct urma_jfc_attr { + uint32_t mask; /* mask value, refer to urma_jfc_attr_mask_t */ + uint16_t moderate_count; + uint16_t moderate_period; /* in micro seconds */ +} urma_jfc_attr_t; + +typedef struct urma_jetty_id { + urma_eid_t eid; + uint32_t uasid; /* maybe zero(stand for kernel) or non-zero(stand for app) */ + uint32_t id; +} urma_jetty_id_t; + +typedef struct urma_jetty_id urma_jfs_id_t; +typedef struct urma_jetty_id urma_jfr_id_t; +typedef struct urma_jetty_id urma_jfc_id_t; + +union urma_jfc_opt_mask { + struct { + uint64_t urma_jfc_cqe_base_addr : 1; + uint64_t urma_jfc_db_addr : 1; + uint64_t urma_jfc_id : 1; + uint64_t urma_jfc_pi : 1; + uint64_t urma_jfc_pi_type : 1; + uint64_t urma_jfc_ci : 1; + uint64_t urma_jfc_db_status : 1; + uint64_t reserved : 57; + } bs; + uint64_t value; +}; + +typedef struct urma_jfc_opt { + union urma_jfc_opt_mask jfc_opt_mask; /* bit0:cqe_base_addr, bit1:db_addr, bit2:id, bit3:pi, bit4:pi_type, + bit5:ci, bit6:db_status */ + bool is_actived; + uint64_t urma_jfc_cqe_base_addr; /* [Optional] CQ Queue Address (VA) */ + uint32_t urma_jfc_id; /* [Optional] the id of jfc */ + uint64_t urma_jfc_db_addr; /* [Optional] CQ Queue Doorbell Address (VA) */ + uint8_t urma_jfc_db_status; /* [Optional] JFC Doorbell status, used for live migration (0 invalid, 1 valid) */ + uint16_t urma_jfc_pi; /* [Optional] PI value of the JFC */ + uint16_t urma_jfc_pi_type; /* [Optional] PI type, 0: absolute value, 1: cumulative value */ + uint16_t urma_jfc_ci; /* [Optional] CI value of the JFC */ + uint64_t reserved[URMA_OPT_REVERSED_NUM]; +} urma_jfc_opt_t; + +typedef struct urma_jfc { + urma_context_t *urma_ctx; /* [Private] point to urma context. */ + urma_jfc_id_t jfc_id; /* [Public] see urma_jetty_id. */ + urma_jfc_cfg_t jfc_cfg; /* [Public] storage jfc config. */ + uint64_t handle; + pthread_mutex_t event_mutex; + pthread_cond_t event_cond; + uint32_t comp_events_acked; + uint32_t async_events_acked; + urma_jfc_opt_t urma_jfc_opt; +} urma_jfc_t; + +typedef enum urma_order_type { + URMA_DEF_ORDER, + URMA_OT, // target ordering + URMA_OI, // initiator ordering + URMA_OL, // low layer ordering + URMA_NO // unreliable non ordering +} urma_order_type_t; + +typedef union urma_jfs_flag { + struct { + uint32_t lock_free : 1; /* default as 0, lock protected */ + uint32_t error_suspend : 1; /* 0: error continue; 1: error suspend */ + uint32_t outorder_comp : 1; /* 0: not support; 1: support out-of-order completion */ + uint32_t order_type : 8; /* (0x0): default, auto config by driver */ + /* (0x1): OT, target ordering */ + /* (0x2): OI, initiator ordering */ + /* (0x3): OL, low layer ordering */ + /* (0x4): UNO, unreliable non ordering */ + uint32_t multi_path : 1; /* 1: multi-path, 0: single path, for ubagg only. */ + uint32_t ctp_rc_mul_path_mode : 1; /* 1: ctp rc mode multi-path */ + uint32_t non_blocking : 1; + uint32_t has_drv_ext : 1; + uint32_t reserved : 17; + } bs; + uint32_t value; +} urma_jfs_flag_t; + +union urma_jfs_opt_mask { + struct { + uint64_t urma_jfs_cqe_base_addr : 1; + uint64_t urma_jfs_db_addr : 1; + uint64_t urma_jfs_id : 1; + uint64_t urma_jfs_pi : 1; + uint64_t urma_jfs_pi_type : 1; + uint64_t urma_jfs_ci : 1; + uint64_t urma_jfs_db_status : 1; + uint64_t reserved : 57; + } bs; + uint64_t value; +}; + +typedef struct urma_jfs_opt { + union urma_jfs_opt_mask jfs_opt_mask; /* bit0:sqe_base_addr, bit1:db_addr, bit2:id, bit3:pi, bit4:pi_type, + bit5:ci, bit6:db_status */ + bool is_actived; + uint64_t urma_jfs_sqe_base_addr; /* [Optional] SQ Queue Address (VA) */ + uint32_t urma_jfs_id; /* [Optional] the id of jfs */ + uint64_t urma_jfs_db_addr; /* [Optional] SQ Queue Doorbell Address (VA) */ + uint8_t urma_jfs_db_status; /* [Optional] JFS Doorbell status, used for live migration (0 invalid, 1 valid) */ + uint16_t urma_jfs_pi; /* [Optional] PI value of the JFS */ + uint16_t urma_jfs_pi_type; /* [Optional] PI type, 0: absolute value, 1: cumulative value */ + uint16_t urma_jfs_ci; /* [Optional] CI value of the JFS */ + uint64_t reserved[URMA_OPT_REVERSED_NUM]; +} urma_jfs_opt_t; + +typedef struct urma_jfs_cfg { + uint32_t depth; /* [Required] the depth of jfs, defaut urma_device_cap_t->jfs_depth */ + urma_jfs_flag_t flag; /* [Optional] see urma_jfs_flag_t definition */ + urma_transport_mode_t trans_mode; /* [Required] transport mode, must be supported by the device */ + uint8_t priority; /* [Optional] set the priority of JFS, ranging from [0, 15] + Services with low delay need to set high priority. */ + uint8_t max_sge; /* [Optional] max sge count in one wr, defaut urma_device_cap_t->max_jfs_sge */ + uint8_t max_rsge; /* [Optional] max remote sge count in one wr, defaut urma_device_cap_t->max_jfs_sge */ + uint32_t max_inline_data; /* [Optional] the max inline data size of JFS. if the parameter is 0, + the system will assign device's max inline data length. */ + uint8_t rnr_retry; /* [Optional] number of times that jfs will resend packets before report error, + when the remote side is not ready to receive (RNR), ranging from [0, 7], + the value 0 means never retry and, + the value 7 means retry infinite number of times for RDMA devices */ + uint8_t err_timeout; /* [Optional] the timeout before report error, ranging from [0, 31], + the actual timeout in usec is caculated by: 4.096*(2^err_timeout) */ + urma_jfc_t *jfc; /* [Required] need to specify jfc */ + uint64_t user_ctx; /* [Optional] private data of jfs */ +} urma_jfs_cfg_t; + +typedef struct urma_jfs { + urma_context_t *urma_ctx; /* [Private] point to urma context. */ + urma_jfs_id_t jfs_id; /* [Public] see urma_jetty_id. */ + urma_jfs_cfg_t jfs_cfg; /* [Public] storage jfs config. */ + uint64_t handle; + pthread_mutex_t event_mutex; + pthread_cond_t event_cond; + uint32_t async_events_acked; + urma_jfs_opt_t urma_jfs_opt; +} urma_jfs_t; + +typedef enum urma_jfs_attr_mask { + JFS_STATE = 0x1 +} urma_jfs_attr_mask_t; + +typedef urma_jetty_state_t urma_jfs_state_t; + +typedef struct urma_jfs_attr { + uint32_t mask; /* mask value refer to urma_jfs_attr_mask_t */ + urma_jfs_state_t state; +} urma_jfs_attr_t; + +typedef union urma_jfr_flag { + struct { + uint32_t token_policy : 3; /* 0: URMA_TOKEN_NONE + 1: URMA_TOKEN_PLAIN_TEXT + 2: URMA_TOKEN_SIGNED + 3: URMA_TOKEN_ALL_ENCRYPTED + 4: URMA_TOKEN_RESERVED */ + uint32_t tag_matching : 1; /* 0: URMA_NO_TAG_MATCHING. + 1: URMA_WITH_TAG_MATCHING. */ + uint32_t lock_free : 1; + uint32_t order_type : 8; /* (0x0): default, auto config by driver */ + /* (0x1): OT, target ordering */ + /* (0x2): OI, initiator ordering */ + /* (0x3): OL, low layer ordering */ + /* (0x4): UNO, unreliable non ordering */ + uint32_t has_drv_ext : 1; + uint32_t reserved : 18; + } bs; + uint32_t value; +} urma_jfr_flag_t; + +union urma_jfr_opt_mask { + struct { + uint64_t urma_jfr_cqe_base_addr : 1; + uint64_t urma_jfr_db_addr : 1; + uint64_t urma_jfr_id : 1; + uint64_t urma_jfr_pi : 1; + uint64_t urma_jfr_pi_type : 1; + uint64_t urma_jfr_ci : 1; + uint64_t urma_jfr_db_status : 1; + uint64_t reserved : 57; + } bs; + uint64_t value; +}; + +typedef struct urma_jfr_opt { + union urma_jfr_opt_mask jfr_opt_mask; /* bit0:rqe_base_addr, bit1:db_addr, bit2:id, bit3:pi, bit4:pi_type, + bit5:ci, bit6:db_status */ + bool is_actived; + uint64_t urma_jfr_rqe_base_addr; /* [Optional] RQ Queue Address (VA) */ + uint32_t urma_jfr_id; /* [Optional] the id of jfr */ + uint64_t urma_jfr_db_addr; /* [Optional] RQ Queue Doorbell Address (VA) */ + uint8_t urma_jfr_db_status; /* [Optional] JFR Doorbell status, used for live migration (0 invalid, 1 valid) */ + uint16_t urma_jfr_pi; /* [Optional] PI value of the JFR */ + uint16_t urma_jfr_pi_type; /* [Optional] PI type, 0: absolute value, 1: cumulative value */ + uint16_t urma_jfr_ci; /* [Optional] CI value of the JFR */ + uint64_t reserved[URMA_OPT_REVERSED_NUM]; +} urma_jfr_opt_t; + +typedef struct urma_jfr_cfg { + uint32_t id; /* [Optional] specify jfr id. If the parameter is 0, + the system will randomly assign a non-0 value. */ + uint32_t depth; /* [Required] total depth, include berth, defaut urma_device_cap_t->jfr_depth. */ + urma_jfr_flag_t flag; /* [Optional] whether is in TAG_matching, whether is in DC/IDC mode. */ + urma_transport_mode_t trans_mode; /* [Required] transport mode, must be supported by the device */ + uint8_t max_sge; /* [Optional] max sge count in one wr, defaut urma_device_cap_t->max_jfr_sge. */ + uint8_t min_rnr_timer; /* [Optional] the minimum RNR NACK timer, ranging from [0, 31], i.e. + the time before jfr sends NACK to the sender for the reason of "ready to receive" */ + urma_jfc_t *jfc; /* [Required] need to specify jfc. */ + urma_token_t token_value; /* [Required] specify token_value for jfr. */ + uint64_t user_ctx; /* [Optional] private data of jfr */ +} urma_jfr_cfg_t; + +typedef enum urma_jfr_attr_mask { + JFR_RX_THRESHOLD = 0x1, + JFR_STATE = 0x1 << 1 +} urma_jfr_attr_mask_t; + +typedef struct urma_jfr_attr { + uint32_t mask; // mask value refer to urma_jfr_attr_mask_t + uint32_t rx_threshold; + urma_jfr_state_t state; +} urma_jfr_attr_t; + +typedef struct urma_jfr { + urma_context_t *urma_ctx; /* [Private] point to urma context. */ + urma_jfr_id_t jfr_id; /* [Public] see urma_jetty_id. */ + urma_jfr_cfg_t jfr_cfg; /* [Public] storage jfr config. */ + uint64_t handle; + pthread_mutex_t event_mutex; + pthread_cond_t event_cond; + uint32_t async_events_acked; + urma_jfr_opt_t urma_jfr_opt; +} urma_jfr_t; + +typedef union urma_import_jetty_flag { + struct { + uint32_t token_policy : 3; + uint32_t order_type : 8; /* (0x0): default, auto config by driver */ + /* (0x1): OT, target ordering */ + /* (0x2): OI, initiator ordering */ + /* (0x3): OL, low layer ordering */ + /* (0x4): UNO, unreliable non ordering */ + uint32_t share_tp : 1; /* 1: shared tp; 0: non-shared tp. When rc mode is not ta dst ordering, + this flag can only be set to 0. */ + uint32_t has_drv_ext : 1; + uint32_t reserved : 19; + } bs; + uint32_t value; + struct { + uint64_t reserved; + uint64_t stag; + uint64_t dtag; + } user_tag; +} urma_import_jetty_flag_t; + +typedef enum urma_tp_type { + URMA_RTP, + URMA_CTP, + URMA_UTP +} urma_tp_type_t; + +typedef struct urma_rjfr { + urma_jfr_id_t jfr_id; /* see urma_jetty_id */ + urma_transport_mode_t trans_mode; + urma_import_jetty_flag_t flag; + urma_tp_type_t tp_type; +} urma_rjfr_t; + +typedef struct urma_tp { + uint32_t tpn; /* vtpn */ +} urma_tp_t; + +typedef union urma_jetty_flag { + struct { + uint32_t share_jfr : 1; /* 0: URMA_NO_SHARE_JFR. + 1: URMA_SHARE_JFR. */ + uint32_t non_blocking : 1; + uint32_t has_drv_ext : 1; + uint32_t reserved : 29; + } bs; + uint32_t value; +} urma_jetty_flag_t; + +typedef struct urma_jetty_grp urma_jetty_grp_t; + +typedef struct urma_jetty_cfg { + uint32_t id; /* [Optional] user specified jetty id. */ + urma_jetty_flag_t flag; /* [Optional] Connection or connection less */ + + /* send configuration */ + urma_jfs_cfg_t jfs_cfg; /* [Required] see urma_jfs_cfg_t */ + + /* recv configuration */ + union { + struct { + urma_jfr_t *jfr; /* [Optional] shared jfr to receive msg */ + urma_jfc_t *jfc; /* [Optional] To replace the jfc related to the above jfr */ + } shared; /* [Required] */ + urma_jfr_cfg_t *jfr_cfg; /* deprecated */ + }; + urma_jetty_grp_t *jetty_grp; /* [Optional] user specified jetty group. */ + uint64_t user_ctx; /* [Optional] private data of jetty */ +} urma_jetty_cfg_t; + +typedef enum urma_jetty_grp_policy { + URMA_JETTY_GRP_POLICY_RR = 0, + URMA_JETTY_GRP_POLICY_HASH_HINT = 1 +} urma_jetty_grp_policy_t; + +typedef enum urma_target_type { + URMA_JFR = 0, + URMA_JETTY, + URMA_JETTY_GROUP +} urma_target_type_t; + +typedef struct urma_rjetty { + urma_jetty_id_t jetty_id; + urma_transport_mode_t trans_mode; + urma_jetty_grp_policy_t policy; + urma_target_type_t type; + urma_import_jetty_flag_t flag; + urma_tp_type_t tp_type; +} urma_rjetty_t; + +typedef struct urma_target_jetty { + urma_context_t *urma_ctx; /* [Private] point to urma context. */ + urma_jetty_id_t id; /* [Private] see urma_jetty_id. */ + uint64_t handle; + urma_transport_mode_t trans_mode; + urma_tp_t tp; + urma_target_type_t type; // todo supplementary target type + urma_import_jetty_flag_t flag; + urma_jetty_grp_policy_t policy; + urma_tp_type_t tp_type; +} urma_target_jetty_t; + +typedef enum urma_jetty_attr_mask { + JETTY_RX_THRESHOLD = 0x1, + JETTY_STATE = 0x1 << 1 +} urma_jetty_attr_mask_t; + +typedef struct urma_jetty_attr { + uint32_t mask; // mask value refer to urma_jetty_attr_mask_t + uint32_t rx_threshold; + urma_jetty_state_t state; +} urma_jetty_attr_t; + +typedef struct urma_jetty_opt { + bool is_actived; + urma_jfs_opt_t jfs_opt; + uint64_t reserved[URMA_OPT_REVERSED_NUM]; +} urma_jetty_opt_t; + +typedef struct urma_jetty { + urma_context_t *urma_ctx; /* [Private] point to urma context. */ + urma_jetty_id_t jetty_id; /* [Public] see urma_jetty_id. */ + urma_target_jetty_t *remote_jetty; /* [Private] Only valid for connection mode Jetty. + After the bind succeeds, the pointer is not null. */ + urma_jetty_cfg_t jetty_cfg; /* [Public] storage jetty config. */ + uint64_t handle; + pthread_mutex_t event_mutex; + pthread_cond_t event_cond; + uint32_t async_events_acked; + urma_jetty_opt_t urma_jetty_opt; +} urma_jetty_t; + +typedef struct urma_notifier { + urma_context_t *urma_ctx; + int fd; + void *incomplete_tjetty_list; +} urma_notifier_t; + +typedef enum urma_notify_type { + URMA_IMPORT_JETTY_NOTIFY = 0, + URMA_BIND_JETTY_NOTIFY +} urma_notify_type_t; + +typedef struct urma_notify { + urma_notify_type_t type; + urma_status_t status; + uint64_t user_ctx; + union { + urma_target_jetty_t *tjetty; /* IMPORT */ + urma_jetty_t *jetty; /* BIND */ + }; +} urma_notify_t; + +typedef union urma_jetty_grp_flag { + struct { + uint32_t token_policy : 3; /* 0: URMA_TOKEN_NONE + 1: URMA_TOKEN_PLAIN_TEXT + 2: URMA_TOKEN_SIGNED + 3: URMA_TOKEN_ALL_ENCRYPTED + 4: URMA_TOKEN_RESERVED */ + uint32_t reserved : 29; + } bs; + uint32_t value; +} urma_jetty_grp_flag_t; + +typedef struct urma_jetty_grp_cfg { + char name[URMA_MAX_NAME]; + urma_jetty_grp_flag_t flag; + urma_token_t token_value; /* [Required] specify token_value for Jetty group. */ + uint32_t id; /* [Optional] specify Jetty group id. + If the parameter is 0, UMDK will assign a non_0 value. */ + urma_jetty_grp_policy_t policy; /* Hash or RR(on default) */ + uint64_t user_ctx; /* [Optional] private data of jetty */ +} urma_jetty_grp_cfg_t; + +struct urma_jetty_grp { + urma_context_t *urma_ctx; + urma_jetty_id_t jetty_grp_id; + urma_jetty_grp_cfg_t cfg; + uint32_t jetty_cnt; + urma_jetty_t **jetty_list; + pthread_mutex_t list_mutex; + uint64_t handle; /* use to quickly get uobj of jetty group in kernel module */ + pthread_mutex_t event_mutex; + pthread_cond_t event_cond; + uint32_t async_events_acked; +}; + +/* memory information */ +typedef struct urma_ubva { + urma_eid_t eid; + uint32_t uasid; // 24 bit for UB + uint64_t va; +} __attribute__((packed)) urma_ubva_t; + + +/* segment definition */ +typedef union urma_reg_seg_flag { + struct { + uint32_t token_policy : 3; /* 0: URMA_TOKEN_NONE. + 1: URMA_TOKEN_PLAIN_TEXT. + 2: URMA_TOKEN_SIGNED. + 3: URMA_TOKEN_ALL_ENCRYPTED. + 4: URMA_TOKEN_RESERVED. */ + uint32_t cacheable : 1; /* 0: URMA_NON_CACHEABLE. + 1: URMA_CACHEABLE. */ + uint32_t dsva : 1; + uint32_t access : 6; /* (0x1): URMA_ACCESS_LOCAL_ONLY. + (0x1 << 1): URMA_ACCESS_READ. + (0x1 << 2): URMA_ACCESS_WRITE. + (0x1 << 3): URMA_ACCESS_ATOMIC. */ + uint32_t non_pin : 1; /* 0: segment pages pinned. + 1: segment pages non-pinned. */ + uint32_t user_iova : 1; /* 0: segment without user iova addr. + 1: segment with user iova addr. */ + uint32_t token_id_valid : 1; /* 0: token id in cfg is invalid. + 1: token id in cfg is valid. */ + uint32_t reserved : 18; + } bs; + uint32_t value; +} urma_reg_seg_flag_t; + +typedef union urma_seg_attr { + struct { + uint32_t token_policy : 3; /* 0: URMA_TOKEN_NONE. + 1: URMA_TOKEN_PLAIN_TEXT. + 2: URMA_TOKEN_SIGNED. + 3: URMA_TOKEN_ALL_ENCRYPTED. + 4: URMA_TOKEN_RESERVED. */ + uint32_t cacheable : 1; /* 0: URMA_NON_CACHEABLE. + 1: URMA_CACHEABLE. */ + uint32_t dsva : 1; + uint32_t access : 6; /* (0x1): URMA_ACCESS_LOCAL_ONLY. + (0x1 << 1): URMA_ACCESS_READ. + (0x1 << 2): URMA_ACCESS_WRITE. + (0x1 << 3): URMA_ACCESS_ATOMIC. */ + uint32_t non_pin : 1; /* 0: segment pages pinned. + 1: segment pages non-pinned. */ + uint32_t user_iova : 1; /* 0: segment without user iova addr. + 1: segment with user iova addr. */ + uint32_t user_token_id : 1; /* 0: token_id is allocated and should be freed by urma. + 1: token_id is allocated by user in urma_seg_cfg. */ + uint32_t reserved : 18; + } bs; + uint32_t value; +} urma_seg_attr_t; + +typedef union urma_import_seg_flag { + struct { + uint32_t cacheable : 1; /* 0: URMA_NON_CACHEABLE. + 1: URMA_CACHEABLE. */ + uint32_t access : 6; /* (0x1): URMA_ACCESS_LOCAL_ONLY. + (0x1 << 1): URMA_ACCESS_READ. + (0x1 << 2): URMA_ACCESS_WRITE. + (0x1 << 3): URMA_ACCESS_ATOMIC. + */ + uint32_t mapping : 1; /* 0: URMA_SEG_NOMAP/ + 1: URMA_SEG_MAPPED. */ + uint32_t reserved : 24; + } bs; + uint32_t value; +} urma_import_seg_flag_t; + +typedef union urma_token_id_flag { + struct { + uint32_t multi_seg : 1; + uint32_t reserved : 31; + } bs; + uint32_t value; +} urma_token_id_flag_t; + +typedef struct urma_token_id { + urma_context_t *urma_ctx; + uint32_t token_id; + uint64_t handle; + urma_ref_t ref; + urma_token_id_flag_t flag; +} urma_token_id_t; + +typedef struct urma_seg_cfg { + uint64_t va; /* specify the address of the segment to be registered */ + uint64_t len; /* specify the length of the segment to be registered */ + urma_token_id_t *token_id; + urma_token_t token_value; /* Security authentication for access */ + urma_reg_seg_flag_t flag; + uint64_t user_ctx; + uint64_t iova; /* user iova, maybe zero-based-address */ +} urma_seg_cfg_t; + +typedef struct urma_seg { + urma_ubva_t ubva; /* [Public] ubva of segment. */ + uint64_t len; /* [Public] length of segment. */ + urma_seg_attr_t attr; /* [Public] include: access flag, token policy, cacheability. */ + uint32_t token_id; /* [Private] match token */ +} urma_seg_t; + +typedef struct urma_target_seg { + urma_seg_t seg; /* [Private] see urma_seg_t. */ + uint64_t user_ctx; /* [Private] private data of segment */ + uint64_t mva; /* [Public] mapping addr when import remote seg. */ + urma_context_t *urma_ctx; /* [Private] point to urma context. */ + urma_token_id_t *token_id; /* When registering seg, it is a valid address; when importing seg, it is NULL */ + uint64_t handle; +} urma_target_seg_t; + +typedef struct urma_user_ctl_in { + uint64_t addr; /* [Required] the address of the input parameter buffer. */ + uint32_t len; /* [Required] the length of the input parameter buffer */ + /* + * Opcode is simultaneously recognized by user and driver. + * User opcode should be distinguished with enum urma_user_ctl_ops_t, which is only used by URMA. + */ + uint32_t opcode; /* [Required] */ +} urma_user_ctl_in_t; + +typedef struct urma_user_ctl_out { + uint64_t addr; /* [Optional] the address of the output parameter buffer. */ + uint32_t len; /* [Optional] the length of the output parameter buffer */ + uint32_t reserved; +} urma_user_ctl_out_t; + +typedef struct urma_user_target_seg { + urma_seg_attr_t attr; + uint32_t token_id; + urma_token_t token_value; +} urma_user_tseg_t; + +typedef struct urma_sge { + uint64_t addr; + uint32_t len; + /* Driver verification + * remote seg: Either tseg or user tseg is not NULL. + * If both of them are not NULL, ignore user_tseg. + * local seg: user_tseg is not supported, tseg must not NULL. + */ + urma_target_seg_t *tseg; + urma_user_tseg_t *user_tseg; /* To support the exemption of import_seg */ +} urma_sge_t; + +typedef struct urma_sg { + urma_sge_t *sge; + uint32_t num_sge; +} urma_sg_t; + +/* wr for batch operations */ +typedef union urma_jfs_wr_flag { + struct { + uint32_t place_order : 2; /* 0: There is no order with other WR + 1: relax order + 2: strong order + 3: reserve */ /* see urma_place_order_t */ + uint32_t comp_order : 1; /* 0: There is no completion order with othwe WR. + 1: Completion order with previous WR. */ + uint32_t fence : 1; /* 0: There is not fence. + 1: Fence with previous read and atomic WR */ + uint32_t solicited_enable : 1; /* 0: There is not solicited. + 1: solicited. It will trigger an event on remote side */ + uint32_t complete_enable : 1; /* 0: Do not notify local process after the task is complete. + 1: Notify local process after the task is completed. */ + uint32_t inline_flag : 1; /* 0: not inline. + 1: inline data. */ + + uint32_t db_bypass : 1; + uint32_t has_drv_ext : 1; + uint32_t reserved : 23; + } bs; + uint32_t value; +} urma_jfs_wr_flag_t; + +typedef union urma_jfr_wr_flag { + struct { + uint32_t complete_type : 1; /* 0: Write completion record to jfc. + 1: Write completion record to complete flag (CF) address */ + uint32_t reserved : 31; + } bs; + uint32_t value; +} urma_jfr_wr_flag_t; + +typedef struct urma_rw_wr { + urma_sg_t src; /* including total data length. src is local va for write, and remote va for read. + only support 1 src sge in read operation. */ + urma_sg_t dst; /* dst is remote va for write, and local va for read. + only support 1 dst sge in write operation. */ + uint8_t target_hint; // required when using jetty group + uint64_t notify_data; // notify data or imm data in host byte order; +} urma_rw_wr_t; + +typedef struct urma_send_wr { + urma_sg_t src; // including total data length + uint8_t target_hint; // required when using jetty group + uint64_t imm_data; // imm_data in host byte order; + urma_target_seg_t *tseg; /* tseg used only when send with invalidate */ +} urma_send_wr_t; + +typedef struct urma_cas_wr { + urma_sge_t *dst; // len is the data length of CAS operation + urma_sge_t *src; // local address for destination original value writeback, len represents the buffer length. + union { // Value compared with destination value + uint64_t cmp_data; // When the len <= 8B, it indicates the CMP value. + uint64_t cmp_addr; // When the len > 8B, it indicates the data address. + }; + union { // If destination value is the same as cmp_data, destination value will be changed to swap_data + uint64_t swap_data; // When the len <= 8B, it indicates the swap value. + uint64_t swap_addr; // When the len > 8B, it indicates the data address. + }; +} urma_cas_wr_t; + +typedef struct urma_faa_wr { + urma_sge_t *dst; // len is the data length of FAA operation + urma_sge_t *src; // local address for destination original value writeback, len represents the buffer length. + union { + uint64_t operand; // When the len <= 8B, it indicates the operand value. + uint64_t operand_addr; // When the len > 8B, it indicates the data address. + }; +} urma_faa_wr_t; + +typedef struct urma_jfs_wr { + urma_opcode_t opcode; + urma_jfs_wr_flag_t flag; + urma_target_jetty_t *tjetty; + uint64_t user_ctx; // completion data + union { + urma_rw_wr_t rw; + urma_send_wr_t send; + urma_cas_wr_t cas; + urma_faa_wr_t faa; + }; + struct urma_jfs_wr *next; +} urma_jfs_wr_t; + +typedef struct urma_jfr_wr { + urma_sg_t src; // includeing buffer length + uint64_t user_ctx; // completion data, eg. wr id + struct urma_jfr_wr *next; +} urma_jfr_wr_t; + +typedef union urma_cr_flag { + struct { + uint8_t s_r : 1; // Indicate CR stands for sending or receiving, 0: send, 1: recv. + uint8_t jetty : 1; // Indicate CR stands for jetty or jfs/jfr, 0: jfs/jfr, 1: jetty. + uint8_t suspend_done : 1; // Real CR associated with the WR, user_ctx is valid + uint8_t flush_err_done : 1; // Real CR associated with the WR, user_ctx is valid + uint8_t reserved : 4; + } bs; + uint8_t value; +} urma_cr_flag_t; + +typedef struct urma_cr_token { + uint32_t token_id; + urma_token_t token_value; +} urma_cr_token_t; + +typedef struct urma_cr { + urma_cr_status_t status; + uint64_t user_ctx; // user_ctx related to a work request + urma_cr_opcode_t opcode; // Only for recv + urma_cr_flag_t flag; // indicate notify data or swap data is valid or not + uint32_t completion_len; // The number of bytes transferred + + uint32_t local_id; // Local jetty ID, or JFS ID, or JFR ID, depends on flag + urma_jetty_id_t remote_id; // Valid only for receiving CR. The remote jetty where the + // received msg comes from Jetty ID or JFS ID, depends on flag. + union { + uint64_t imm_data; // Valid only for receiving CR: send/write/read with imm. + urma_cr_token_t invalid_token; // Valid only for receiving CR: send with invalidate. + }; + uint32_t tpn; // TP number or TPG number + uintptr_t user_data; // e.g. use as pointer to local jetty struct. +} urma_cr_t; + +typedef struct urma_async_event { + /* may be SW queue error, may be HW port error */ + const urma_context_t *urma_ctx; + union { + urma_jfc_t *jfc; + urma_jfs_t *jfs; + urma_jfr_t *jfr; + urma_jetty_t *jetty; + urma_jetty_grp_t *jetty_grp; + uint32_t port_id; + uint32_t eid_idx; + } element; + urma_async_event_type_t event_type; + void *priv; +} urma_async_event_t; + +/* URMA region definition */ +typedef union urma_ur_attr { + struct { + uint32_t reserved : 32; + } bs; + uint32_t value; +} urma_ur_attr_t; + +typedef union urma_import_ur_flag { + struct { + uint32_t mapping : 2; /* 0: URMA_SEG_NOMAP + 1: URMA_SEG_MAPPED_MVA + 2: URMA_SEG_MAPPED_DSVA */ + uint32_t reserved : 30; + } bs; + uint32_t value; +} urma_import_ur_flag_t; + +#define UR_NAME_MAX_LEN 256 +#define JFR_NAME_MAX_LEN 256 +#define URMA_MAX_SEGS_PER_UR_OPT 64 // Max number of SEGS per attach/detach ur + +// In parametre for create UR +typedef struct urma_ur { + char name[UR_NAME_MAX_LEN]; // UR url name + uint64_t size; + urma_ur_attr_t attr; // include: access flag, token policy, cacheability, dsva + uint64_t token; + uint64_t user_ctx; +} urma_ur_t; + +// Out parametre for import UR +typedef struct urma_target_ur { + char name[UR_NAME_MAX_LEN]; // UR url name + uint64_t size; + urma_import_ur_flag_t flag; // include: access flag, token policy, cacheability, dsva + urma_target_seg_t **tseg_list; + uint32_t cnt; +} urma_target_ur_t; + +typedef struct urma_seg_info { + urma_seg_t seg; + uint32_t idx_in_ur; +} urma_seg_info_t; + +// Out parametre for lookup UR +typedef struct urma_ur_info { + char name[UR_NAME_MAX_LEN]; // UR url name + uint64_t size; // limit size, by byte + urma_ur_attr_t attr; // include: access flag, token policy, cacheability, dsva + uint32_t cnt; // + urma_seg_info_t seg_list[0]; // cnt * sizeof(urma_seg_info_t) +} urma_ur_info_t; + +typedef struct urma_jfr_info { + char name[JFR_NAME_MAX_LEN]; // jfr url name + urma_eid_t eid; + uint32_t uasid; + uint32_t id; +} urma_jfr_info_t; + +typedef union urma_tp_cfg_flag { + struct { + uint32_t target : 1; /* 0: initiator, 1: target */ + uint32_t loopback : 1; + uint32_t dca_enable : 1; + /* for the bonding case, the hardware selects the port + * ignoring the port of the tp context and + * selects the port based on the hash value + * along with the information in the bonding group table. + */ + uint32_t bonding : 1; + uint32_t reserved : 28; + } bs; + uint32_t value; +} urma_tp_cfg_flag_t; + +typedef struct urma_tp_cfg { + urma_tp_cfg_flag_t flag; /* flag of initial tp */ + /* transport layer attributes */ + urma_transport_mode_t trans_mode; + uint8_t retry_num; + uint8_t retry_factor; /* for calculate the time slot to retry */ + uint8_t ack_timeout; + uint8_t dscp; + uint32_t oor_cnt; /* OOR window size: by packet */ +} urma_tp_cfg_t; + +typedef union urma_tp_attr_mask { + struct { + uint32_t flag : 1; + uint32_t peer_tpn : 1; + uint32_t state : 1; + uint32_t tx_psn : 1; + uint32_t rx_psn : 1; /* modify both rx psn and tx psn when restore tp */ + uint32_t mtu : 1; + uint32_t cc_pattern_idx : 1; + uint32_t oos_cnt : 1; + uint32_t local_net_addr_idx : 1; + uint32_t peer_net_addr : 1; + uint32_t data_udp_start : 1; + uint32_t ack_udp_start : 1; + uint32_t udp_range : 1; + uint32_t hop_limit : 1; + uint32_t flow_label : 1; + uint32_t port_id : 1; + uint32_t mn : 1; + uint32_t peer_trans_type : 1; + uint32_t reserved : 14; + } bs; + uint32_t value; +} urma_tp_attr_mask_t; + +typedef union urma_tp_mod_flag { + struct { + uint32_t oor_en : 1; /* out of order receive, 0: disable 1: enable */ + uint32_t sr_en : 1; /* selective retransmission, 0: disable 1: enable */ + uint32_t cc_en : 1; /* congestion control algorithm, 0: disable 1: enable */ + uint32_t cc_alg : 4; /* The value is ubcore_tp_cc_alg_t */ + uint32_t spray_en : 1; /* spray with src udp port, 0: disable 1: enable */ + uint32_t clan : 1; /* clan domain, 0: disable 1: enable */ + uint32_t reserved : 23; + } bs; + uint32_t value; +} urma_tp_mod_flag_t; + +typedef enum urma_tp_state { + URMA_TP_STATE_RESET = 0, + URMA_TP_STATE_PASSIVE, + URMA_TP_STATE_ACTIVE, + URMA_TP_STATE_BRAKE, + URMA_TP_STATE_ERROR +} urma_tp_state_t; + +typedef struct urma_net_addr { + sa_family_t sin_family; /* AF_INET/AF_INET6 */ + union { + struct in_addr in4; + struct in6_addr in6; + }; + uint64_t vlan; + uint8_t mac[URMA_MAC_BYTES]; + uint32_t prefix_len; +} urma_net_addr_t; + +typedef struct urma_net_addr_info { + urma_net_addr_t netaddr; + uint32_t index; +} urma_net_addr_info_t; + +typedef struct urma_tp_attr { + urma_tp_mod_flag_t flag; + uint32_t peer_tpn; + urma_tp_state_t state; + uint32_t tx_psn; + uint32_t rx_psn; + urma_mtu_t mtu; + uint8_t cc_pattern_idx; + uint32_t oos_cnt; /* out of standing packet cnt */ + uint32_t local_net_addr_idx; + urma_net_addr_t peer_net_addr; + uint16_t data_udp_start; + uint16_t ack_udp_start; + uint8_t udp_range; + uint8_t hop_limit; + uint32_t flow_label; + uint8_t port_id; + uint8_t mn; /* 0~15, a packet contains only one msg if mn is set as 0 */ + urma_transport_type_t peer_trans_type; +} urma_tp_attr_t; + +typedef union urma_get_tp_cfg_flag { + struct { + uint32_t ctp : 1; + uint32_t rtp : 1; + uint32_t utp : 1; + uint32_t uboe : 1; + uint32_t pre_defined : 1; + uint32_t dynamic_defined : 1; + uint32_t udp : 5; + uint32_t group_id : 15; + uint32_t reserved : 6; + } bs; + uint32_t value; +} urma_get_tp_cfg_flag_t; + +typedef struct urma_get_tp_cfg { + urma_get_tp_cfg_flag_t flag; + urma_transport_mode_t trans_mode; + urma_eid_t local_eid; + urma_eid_t peer_eid; +} urma_get_tp_cfg_t; + +typedef struct urma_tp_info { + uint64_t tp_handle; +} urma_tp_info_t; + +typedef struct urma_active_tp_attr { + uint32_t tx_psn; + uint32_t rx_psn; + uint64_t reserved; +} urma_active_tp_attr_t; + +typedef struct urma_active_tp_cfg { + uint64_t tp_handle; + uint64_t peer_tp_handle; + uint64_t tag; + urma_active_tp_attr_t tp_attr; +} urma_active_tp_cfg_t; + +typedef struct urma_active_tp_cfg urma_import_jetty_ex_cfg_t; +typedef struct urma_active_tp_cfg urma_import_jfr_ex_cfg_t; +typedef struct urma_active_tp_cfg urma_bind_jetty_ex_cfg_t; + +#pragma pack(1) +typedef struct urma_tp_attr_value { + uint8_t retry_times_init : 3; + uint8_t at : 5; + uint8_t sip[URMA_IP_ADDR_BYTES]; + uint8_t dip[URMA_IP_ADDR_BYTES]; + uint8_t sma[URMA_MAC_BYTES]; + uint8_t dma[URMA_MAC_BYTES]; + uint16_t vlan_id : 12; + uint8_t vlan_en : 1; + uint8_t dscp : 6; + uint8_t at_times : 5; + uint8_t sl : 4; + uint8_t ttl; + uint16_t ack_udp_srcport; + uint16_t data_udp_srcport; + uint8_t udp_srcport_range : 4; + uint8_t spray_en : 1; + uint8_t udp_global_en : 1; + uint8_t reserve_0 : 2; + uint16_t sl_bitmap; + uint8_t dscp_config_mode : 1; + uint8_t reserve_1 : 7; + uint8_t reserved[70]; +} urma_tp_attr_value_t; +#pragma pack() + +/* callback information */ +typedef void (*urma_async_event_cb)(urma_async_event_t *event, void *cb_arg); + +/* callback function type for urma_advise_jfr/jetty_async. User must define callback function to handle result. + advise_result is the result of advise jfr or jetty */ +typedef void (*urma_advise_async_cb_func)(urma_status_t advise_result, void *cb_arg); + +typedef enum urma_vlog_level { + URMA_VLOG_LEVEL_EMERG = 0, + URMA_VLOG_LEVEL_ALERT = 1, + URMA_VLOG_LEVEL_CRIT = 2, + URMA_VLOG_LEVEL_ERR = 3, + URMA_VLOG_LEVEL_WARNING = 4, + URMA_VLOG_LEVEL_NOTICE = 5, + URMA_VLOG_LEVEL_INFO = 6, + URMA_VLOG_LEVEL_DEBUG = 7, + URMA_VLOG_LEVEL_MAX = 8, +} urma_vlog_level_t; + +typedef void (*urma_log_cb_t)(int level, char *message); + +/* location log callback function definition */ +typedef void (*urma_loc_log_cb)(int level, const char *file, const char *function, int line, char *message); + +#ifdef __cplusplus +} +#endif + +#endif // URMA_TYPES_H diff --git a/include/ylt/urma/urma_types_str.h b/include/ylt/urma/urma_types_str.h new file mode 100644 index 000000000..e75a27597 --- /dev/null +++ b/include/ylt/urma/urma_types_str.h @@ -0,0 +1,243 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) Huawei Technologies Co., Ltd. 2022-2025. All rights reserved. + * Description: URMA type string header file + * Author: Qian Guoxin + * Create: 2022-10-18 + * Note: + * History: 2022-10-18 Create File + */ + +#ifndef URMA_TYPES_STR_H +#define URMA_TYPES_STR_H +#include "urma_types.h" + +static const char *const g_urma_mtu_str[] = { + [URMA_MTU_256] = "MTU_256", // + [URMA_MTU_512] = "MTU_512", // + [URMA_MTU_1024] = "MTU_1024", // + [URMA_MTU_2048] = "MTU_2048", // + [URMA_MTU_4096] = "MTU_4096", // + [URMA_MTU_8192] = "MTU_8192", // +}; + +static inline const char *urma_mtu_to_string(urma_mtu_t mtu) +{ + if (mtu < URMA_MTU_256 || mtu > URMA_MTU_8192) { + return "Invalid Value"; + } + return g_urma_mtu_str[mtu]; +} + +static const char *const g_urma_port_state_str[] = { + [URMA_PORT_NOP] = "NOP", // + [URMA_PORT_DOWN] = "DOWN", // + [URMA_PORT_INIT] = "INIT", // + [URMA_PORT_ARMED] = "ARMED", // + [URMA_PORT_ACTIVE] = "ACTIVE", // + [URMA_PORT_ACTIVE_DEFER] = "ACTIVE_DEFER", // +}; + +static inline const char *urma_port_state_to_string(urma_port_state_t state) +{ + if (state > URMA_PORT_ACTIVE_DEFER) { + return "Invalid Value"; + } + return g_urma_port_state_str[state]; +} + +static const char *const g_urma_speed_str[] = { + [URMA_SP_10M] = "SP_10M", // + [URMA_SP_100M] = "SP_100M", // + [URMA_SP_1G] = "SP_1G", // + [URMA_SP_2_5G] = "SP_2.5G", // + [URMA_SP_5G] = "SP_5G", // + [URMA_SP_10G] = "SP_10G", // + [URMA_SP_14G] = "SP_14G", // + [URMA_SP_25G] = "SP_25G", // + [URMA_SP_40G] = "SP_40G", // + [URMA_SP_50G] = "SP_50G", // + [URMA_SP_100G] = "SP_100G", // + [URMA_SP_200G] = "SP_200G", // + [URMA_SP_400G] = "SP_400G", // + [URMA_SP_800G] = "SP_800G", // +}; + +static inline const char *urma_speed_to_string(urma_speed_t speed) +{ + if (speed > URMA_SP_800G) { + return "Invalid Value"; + } + return g_urma_speed_str[speed]; +} + +static const char *const g_urma_link_width_str[] = { + [0] = "unknow", + [URMA_LINK_X1] = "LINK_X1", + [URMA_LINK_X2] = "LINK_X2", + [URMA_LINK_X4] = "LINK_X4", + [URMA_LINK_X8] = "LINK_X8", + [URMA_LINK_X16] = "LINK_X16", + [URMA_LINK_X32] = "LINK_X32", +}; + +static inline const char *urma_link_width_to_string(urma_link_width_t width) +{ + if (width > URMA_LINK_X32) { + return "Invalid Value"; + } + return g_urma_link_width_str[width]; +} + +static const char * const g_urma_tp_type_en_str[] = { + [URMA_RTP] = "RTP", + [URMA_CTP] = "CTP", + [URMA_UTP] = "UTP", +}; + +static inline const char *urma_tp_type_en_to_string(union urma_tp_type_en tp_type) +{ + if (tp_type.bs.rtp == 1 && tp_type.bs.ctp == 0 && tp_type.bs.utp == 0) { + return g_urma_tp_type_en_str[URMA_RTP]; + } + if (tp_type.bs.rtp == 0 && tp_type.bs.ctp == 1 && tp_type.bs.utp == 0) { + return g_urma_tp_type_en_str[URMA_CTP]; + } + if (tp_type.bs.rtp == 0 && tp_type.bs.ctp == 0 && tp_type.bs.utp == 1) { + return g_urma_tp_type_en_str[URMA_UTP]; + } + return "Invalid Value"; +} + +#define URMA_DEVICE_FEAT_NUM 9 + +static const char *const g_urma_device_feat_str[URMA_DEVICE_FEAT_NUM] = { + "OUT_OF_ORDER", // + "JFC_PER_WR", // + "STRIDE_OP", // + "LOAD_STORE_OP", // + "NON_PIN", // + "PERSISTENCE_MEM", // + "JFC_INLINE", // + "SPRAY_ENABLE", // + "SELECTIVE_RETRANS", // +}; + +static inline const char *urma_device_feat_to_string(uint8_t bit) +{ + if (bit >= URMA_DEVICE_FEAT_NUM) { + return "Invalid Value"; + } + return g_urma_device_feat_str[bit]; +} + +#define URMA_ATOMIC_FEAT_NUM 7 + +static const char *const g_urma_atomic_feat_str[URMA_ATOMIC_FEAT_NUM] = { + "compare_and_swap", // + "swap", // + "fetch_and_add", // + "fetch_and_sub", // + "fetch_and_and", // + "fetch_and_or", // + "fetch_and_xor", // +}; + +static inline const char *urma_atomic_feat_to_string(uint8_t bit) +{ + if (bit >= URMA_ATOMIC_FEAT_NUM) { + return "Invalid Value"; + } + return g_urma_atomic_feat_str[bit]; +} + +static const char *const g_urma_trans_mode_str[] = { + [URMA_TM_RM] = "RM(Reliable message)", + [URMA_TM_RC] = "RC(Reliable connection)", + [URMA_TM_UM] = "UM(Unreliable message)", +}; + +static inline const char *urma_trans_mode_to_string(urma_transport_mode_t mode) +{ + if (mode > URMA_TM_UM) { + return "Invalid Value"; + } + return g_urma_trans_mode_str[mode]; +} + +static const char *const g_urma_tp_type_str[] = { + [URMA_TRANSPORT_UB] = "UB", +}; + +static inline const char *urma_tp_type_to_string(urma_transport_type_t type) +{ + if (type <= URMA_TRANSPORT_INVALID || type >= URMA_TRANSPORT_MAX) { + return "Invalid Value"; + } + return g_urma_tp_type_str[type]; +} + +static const char *const g_urma_congestion_ctrl_alg_str[] = { + [URMA_TP_CC_NONE] = "NONE", + [URMA_TP_CC_DCQCN] = "DCQCN", + [URMA_TP_CC_DCQCN_AND_NETWORK_CC] = "DCQCN_AND_NETWORK_CC", + [URMA_TP_CC_LDCP] = "LDCP", + [URMA_TP_CC_LDCP_AND_CAQM] = "LDCP_AND_CAQM", + [URMA_TP_CC_LDCP_AND_OPEN_CC] = "LDCP_AND_OPEN_CC", + [URMA_TP_CC_HC3] = "HC3", + [URMA_TP_CC_DIP] = "DIP", + [URMA_TP_CC_ACC] = "ACC", +}; + +static inline const char *urma_congestion_ctrl_alg_to_string(uint8_t bit) +{ + if (bit > URMA_TP_CC_DIP) { + return "Invalid Value"; + } + return g_urma_congestion_ctrl_alg_str[bit]; +} + +static const char *const g_urma_jfc_state[] = { + [URMA_JFC_STATE_INVALID] = "INVALID", + [URMA_JFC_STATE_VALID] = "VALID", + [URMA_JFC_STATE_ERROR] = "ERROR", +}; + +static inline const char *urma_jfc_state_to_string(uint8_t bit) +{ + if (bit > URMA_JFC_STATE_ERROR) { + return "Invalid Value"; + } + return g_urma_jfc_state[bit]; +} + +static const char *const g_urma_jetty_state[] = { + [URMA_JETTY_STATE_RESET] = "RESET", + [URMA_JETTY_STATE_READY] = "READY", + [URMA_JETTY_STATE_SUSPENDED] = "SUSPENDED", + [URMA_JETTY_STATE_ERROR] = "ERROR", +}; + +static inline const char *urma_jetty_state_to_string(uint8_t bit) +{ + if (bit > URMA_JETTY_STATE_ERROR) { + return "Invalid Value"; + } + return g_urma_jetty_state[bit]; +} + +static const char *const g_urma_jfr_state[] = { + [URMA_JFR_STATE_RESET] = "RESET", + [URMA_JFR_STATE_READY] = "READY", + [URMA_JFR_STATE_ERROR] = "ERROR", +}; + +static inline const char *urma_jfr_state_to_string(uint8_t bit) +{ + if (bit > URMA_JFR_STATE_ERROR) { + return "Invalid Value"; + } + return g_urma_jfr_state[bit]; +} + +#endif From 6e71e1be852163bc0818b5a32b36b4f10986e09f Mon Sep 17 00:00:00 2001 From: lixu Date: Thu, 4 Jun 2026 20:16:23 +0800 Subject: [PATCH 005/129] fix bug --- include/ylt/coro_io/ibverbs/ib_socket.hpp | 38 +------------------- include/ylt/coro_io/urma/urma_device.hpp | 2 +- include/ylt/coro_io/urma/urma_socket.hpp | 42 +---------------------- 3 files changed, 3 insertions(+), 79 deletions(-) diff --git a/include/ylt/coro_io/ibverbs/ib_socket.hpp b/include/ylt/coro_io/ibverbs/ib_socket.hpp index 41b97d45a..928d5f943 100644 --- a/include/ylt/coro_io/ibverbs/ib_socket.hpp +++ b/include/ylt/coro_io/ibverbs/ib_socket.hpp @@ -44,6 +44,7 @@ #include "ib_device.hpp" #include "ib_error.hpp" #include "ylt/coro_io/coro_io.hpp" +#include "ylt/coro_io/detail/circle_buffer.hpp" #include "ylt/coro_io/ibverbs/ib_buffer.hpp" #include "ylt/coro_io/io_context_pool.hpp" #include "ylt/easylog.hpp" @@ -53,43 +54,6 @@ namespace coro_io { namespace detail { struct ib_socket_shared_state_t; -template -struct circle_buffer { - std::vector queue; - uint32_t front_ = 0, end_ = 0; - bool may_empty = true; - circle_buffer(uint32_t size) { - assert(size > 0); - queue.resize(size); - } - void push(T&& elem) { - assert(!full()); - may_empty = false; - end_ = (end_ + 1) % queue.size(); - queue[end_] = std::move(elem); - } - T pop() { - assert(!empty()); - front_ = (front_ + 1) % queue.size(); - may_empty = true; - return std::move(queue[front_]); - } - T& back() { return queue[end_]; } - T& front() { return queue[(front_ + 1) % queue.size()]; } - bool full() const noexcept { return end_ == front_ && !may_empty; } - bool empty() const noexcept { return end_ == front_ && may_empty; } - std::size_t size() const noexcept { - if (front_ > end_) { - return queue.size() + end_ - front_; - } - else if (front_ == end_) { - return empty() ? 0 : queue.size(); - } - else { - return end_ - front_; - } - } -}; struct ib_buffer_queue : public circle_buffer { using circle_buffer::circle_buffer; std::error_code post_recv_real(ibv_sge buffer, diff --git a/include/ylt/coro_io/urma/urma_device.hpp b/include/ylt/coro_io/urma/urma_device.hpp index 82d7c717a..88078cf54 100644 --- a/include/ylt/coro_io/urma/urma_device.hpp +++ b/include/ylt/coro_io/urma/urma_device.hpp @@ -34,7 +34,7 @@ struct urma_init_attr_t; #define URMA_EID_SIZE 16 #ifdef YLT_ENABLE_URMA -#include +#include "ylt/urma/urma_api.h" #endif namespace coro_io { diff --git a/include/ylt/coro_io/urma/urma_socket.hpp b/include/ylt/coro_io/urma/urma_socket.hpp index f38924f57..f82c53e9b 100644 --- a/include/ylt/coro_io/urma/urma_socket.hpp +++ b/include/ylt/coro_io/urma/urma_socket.hpp @@ -30,14 +30,12 @@ #include "async_simple/coro/Lazy.h" #include "async_simple/util/move_only_function.h" #include "ylt/coro_io/coro_io.hpp" +#include "ylt/coro_io/detail/circle_buffer.hpp" #include "ylt/easylog.hpp" #include "ylt/struct_pack.hpp" #include "ylt/urma/urma_api.h" #include "ylt/urma/urma_types.h" - -enum class urma_transport_type_t : int; - #define URMA_EID_LEN 16 namespace coro_io { @@ -45,44 +43,6 @@ namespace detail { struct urma_socket_shared_state_t; -template -struct circle_buffer { - std::vector queue; - uint32_t front_ = 0, end_ = 0; - bool may_empty = true; - circle_buffer(uint32_t size) { - assert(size > 0); - queue.resize(size); - } - void push(T&& elem) { - assert(!full()); - may_empty = false; - end_ = (end_ + 1) % queue.size(); - queue[end_] = std::move(elem); - } - T pop() { - assert(!empty()); - front_ = (front_ + 1) % queue.size(); - may_empty = true; - return std::move(queue[front_]); - } - T& back() { return queue[end_]; } - T& front() { return queue[(front_ + 1) % queue.size()]; } - bool full() const noexcept { return end_ == front_ && !may_empty; } - bool empty() const noexcept { return end_ == front_ && may_empty; } - std::size_t size() const noexcept { - if (front_ > end_) { - return queue.size() + end_ - front_; - } - else if (front_ == end_) { - return empty() ? 0 : queue.size(); - } - else { - return end_ - front_; - } - } -}; - // URMA-specific buffer representation (compatible with ibv_sge layout) struct urma_sge { uint64_t addr; From 3a9b59ac990fd301ab337afa0fbd94fed3b238c0 Mon Sep 17 00:00:00 2001 From: lixu Date: Thu, 4 Jun 2026 20:17:01 +0800 Subject: [PATCH 006/129] add circle_buffer.hpp --- include/ylt/coro_io/detail/circle_buffer.hpp | 67 ++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 include/ylt/coro_io/detail/circle_buffer.hpp diff --git a/include/ylt/coro_io/detail/circle_buffer.hpp b/include/ylt/coro_io/detail/circle_buffer.hpp new file mode 100644 index 000000000..0ebdd59e6 --- /dev/null +++ b/include/ylt/coro_io/detail/circle_buffer.hpp @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2025, Alibaba Group Holding Limited; + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef YLT_CORO_IO_DETAIL_CIRCLE_BUFFER_HPP +#define YLT_CORO_IO_DETAIL_CIRCLE_BUFFER_HPP + +#include +#include +#include + +namespace coro_io { +namespace detail { + +template +struct circle_buffer { + std::vector queue; + uint32_t front_ = 0, end_ = 0; + bool may_empty = true; + circle_buffer(uint32_t size) { + assert(size > 0); + queue.resize(size); + } + void push(T&& elem) { + assert(!full()); + may_empty = false; + end_ = (end_ + 1) % queue.size(); + queue[end_] = std::move(elem); + } + T pop() { + assert(!empty()); + front_ = (front_ + 1) % queue.size(); + may_empty = true; + return std::move(queue[front_]); + } + T& back() { return queue[end_]; } + T& front() { return queue[(front_ + 1) % queue.size()]; } + bool full() const noexcept { return end_ == front_ && !may_empty; } + bool empty() const noexcept { return end_ == front_ && may_empty; } + std::size_t size() const noexcept { + if (front_ > end_) { + return queue.size() + end_ - front_; + } + else if (front_ == end_) { + return empty() ? 0 : queue.size(); + } + else { + return end_ - front_; + } + } +}; + +} // namespace detail +} // namespace coro_io + +#endif // YLT_CORO_IO_DETAIL_CIRCLE_BUFFER_HPP \ No newline at end of file From 93634ba0b6c2f5bd162a78545f802a28261b705f Mon Sep 17 00:00:00 2001 From: lixu Date: Thu, 4 Jun 2026 20:39:14 +0800 Subject: [PATCH 007/129] fix(urma): rename urma_device_t to urma_device_wrapper_t to avoid conflict with URMA library type - Rename wrapper class to urma_device_wrapper_t - Use ::urma_device_t to explicitly refer to URMA library type - Add backward compatibility alias using urma_device_t = urma_device_wrapper_t - Fix all references to URMA library functions to use :: scope resolution --- include/ylt/coro_io/urma/urma_device.hpp | 89 +++++++++++------------- include/ylt/coro_io/urma/urma_socket.hpp | 23 +++--- 2 files changed, 51 insertions(+), 61 deletions(-) diff --git a/include/ylt/coro_io/urma/urma_device.hpp b/include/ylt/coro_io/urma/urma_device.hpp index 88078cf54..807dd2a7d 100644 --- a/include/ylt/coro_io/urma/urma_device.hpp +++ b/include/ylt/coro_io/urma/urma_device.hpp @@ -23,16 +23,6 @@ #include "ylt/easylog.hpp" -// URMA forward declarations -struct urma_context_t; -struct urma_device_t; -struct urma_eid_t; -struct urma_eid_info_t; -struct urma_device_attr_t; -struct urma_init_attr_t; - -#define URMA_EID_SIZE 16 - #ifdef YLT_ENABLE_URMA #include "ylt/urma/urma_api.h" #endif @@ -41,20 +31,21 @@ namespace coro_io { class urma_buffer_pool_t; -// URMA Device abstraction (similar to ib_device_t) -class urma_device_t { +// URMA Device abstraction - wrapper around URMA library's ::urma_device_t +// Note: Using ::urma_device_t to explicitly refer to the URMA library type +class urma_device_wrapper_t { public: - urma_device_t(); - ~urma_device_t(); + urma_device_wrapper_t(); + ~urma_device_wrapper_t(); - urma_device_t(const urma_device_t&) = delete; - urma_device_t& operator=(const urma_device_t&) = delete; + urma_device_wrapper_t(const urma_device_wrapper_t&) = delete; + urma_device_wrapper_t& operator=(const urma_device_wrapper_t&) = delete; bool init(const std::string& device_name, int eid_index = 0); void close(); urma_context_t* context() const { return context_; } - urma_device_t* device() const { return device_; } + urma_device_t* device() const { return device_ptr_; } const std::string& name() const { return name_; } int eid_index() const { return eid_index_; } const urma_eid_t& eid() const { return eid_; } @@ -63,59 +54,63 @@ class urma_device_t { std::string eid_string() const; asio::ip::address gid_address() const; - bool is_valid() const { return context_ != nullptr && device_ != nullptr; } + bool is_valid() const { return context_ != nullptr && device_ptr_ != nullptr; } std::shared_ptr get_buffer_pool() const { return buffer_pool_; } private: std::string name_; int eid_index_ = -1; - urma_device_t* device_ = nullptr; + urma_device_t* device_ptr_ = nullptr; // URMA library device handle urma_context_t* context_ = nullptr; urma_eid_t eid_{}; urma_device_attr_t device_attr_{}; std::shared_ptr buffer_pool_; }; +// Backward compatibility alias +using urma_device_t = urma_device_wrapper_t; + // Global device management class urma_device_manager { public: static urma_device_manager& instance(); bool init(); - std::shared_ptr get_device(const std::string& device_name = ""); - std::vector> get_all_devices(); - std::shared_ptr get_global_device(); + std::shared_ptr get_device(const std::string& device_name = ""); + std::vector> get_all_devices(); + std::shared_ptr get_global_device(); private: urma_device_manager() = default; ~urma_device_manager(); bool initialized_ = false; - std::vector> devices_; - std::shared_ptr global_device_; + std::vector> devices_; + std::shared_ptr global_device_; }; -inline std::shared_ptr get_global_urma_device() { +inline std::shared_ptr get_global_urma_device() { return urma_device_manager::instance().get_global_device(); } // ============= Implementation (inline in header) ============= -inline urma_device_t::urma_device_t() = default; +inline urma_device_wrapper_t::urma_device_wrapper_t() = default; -inline urma_device_t::~urma_device_t() { close(); } +inline urma_device_wrapper_t::~urma_device_wrapper_t() { close(); } -inline bool urma_device_t::init(const std::string& device_name, int eid_index) { +inline bool urma_device_wrapper_t::init(const std::string& device_name, int eid_index) { #ifdef YLT_ENABLE_URMA name_ = device_name; eid_index_ = eid_index; int num_devices = 0; - urma_device_t** devices = urma_get_device_list(&num_devices); + // Use :: to explicitly call URMA library function + ::urma_device_t** devices = urma_get_device_list(&num_devices); if (!devices || num_devices <= 0) { ELOG_ERROR << "urma_get_device_list failed"; return false; } - urma_device_t* found_device = nullptr; + ::urma_device_t* found_device = nullptr; for (int i = 0; i < num_devices; ++i) { if (device_name.empty() || std::string(devices[i]->name) == device_name) { found_device = devices[i]; @@ -129,10 +124,10 @@ inline bool urma_device_t::init(const std::string& device_name, int eid_index) { return false; } - device_ = found_device; + device_ptr_ = found_device; uint32_t eid_cnt = 0; - urma_eid_info_t* eid_list = urma_get_eid_list(device_, &eid_cnt); + urma_eid_info_t* eid_list = urma_get_eid_list(device_ptr_, &eid_cnt); if (!eid_list || eid_cnt == 0) { ELOG_ERROR << "urma_get_eid_list failed"; urma_free_device_list(devices); @@ -147,14 +142,14 @@ inline bool urma_device_t::init(const std::string& device_name, int eid_index) { eid_ = eid_list[eid_index_].eid; urma_free_eid_list(eid_list); - context_ = urma_create_context(device_, eid_index_); + context_ = urma_create_context(device_ptr_, eid_index_); if (!context_) { ELOG_ERROR << "urma_create_context failed"; urma_free_device_list(devices); return false; } - if (urma_query_device(device_, &device_attr_) != 0) { + if (urma_query_device(device_ptr_, &device_attr_) != 0) { ELOG_ERROR << "urma_query_device failed"; urma_delete_context(context_); context_ = nullptr; @@ -171,7 +166,7 @@ inline bool urma_device_t::init(const std::string& device_name, int eid_index) { #endif } -inline void urma_device_t::close() { +inline void urma_device_wrapper_t::close() { #ifdef YLT_ENABLE_URMA if (context_) { urma_delete_context(context_); @@ -180,7 +175,7 @@ inline void urma_device_t::close() { #endif } -inline std::string urma_device_t::eid_string() const { +inline std::string urma_device_wrapper_t::eid_string() const { char buf[64] = {0}; snprintf(buf, sizeof(buf), "%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:" @@ -192,7 +187,7 @@ inline std::string urma_device_t::eid_string() const { return std::string(buf); } -inline asio::ip::address urma_device_t::gid_address() const { +inline asio::ip::address urma_device_wrapper_t::gid_address() const { char buf[64]; snprintf(buf, sizeof(buf), "%d.%d.%d.%d", eid_.raw[0], eid_.raw[1], eid_.raw[2], eid_.raw[3]); @@ -228,7 +223,7 @@ inline bool urma_device_manager::init() { #endif } -inline std::shared_ptr urma_device_manager::get_device( +inline std::shared_ptr urma_device_manager::get_device( const std::string& device_name) { #ifdef YLT_ENABLE_URMA if (!initialized_) init(); @@ -239,7 +234,7 @@ inline std::shared_ptr urma_device_manager::get_device( } } - auto dev = std::make_shared(); + auto dev = std::make_shared(); if (!dev->init(device_name)) return nullptr; devices_.push_back(dev); @@ -250,33 +245,33 @@ inline std::shared_ptr urma_device_manager::get_device( #endif } -inline std::vector> +inline std::vector> urma_device_manager::get_all_devices() { #ifdef YLT_ENABLE_URMA if (!devices_.empty()) return devices_; if (!initialized_) init(); int num_devices = 0; - urma_device_t** urma_devices = urma_get_device_list(&num_devices); - if (!urma_devices || num_devices <= 0) return devices_; + ::urma_device_t** urma_dev_list = urma_get_device_list(&num_devices); + if (!urma_dev_list || num_devices <= 0) return devices_; for (int i = 0; i < num_devices; ++i) { - auto dev = std::make_shared(); - if (dev->init(urma_devices[i]->name)) { + auto dev = std::make_shared(); + if (dev->init(urma_dev_list[i]->name)) { devices_.push_back(dev); if (!global_device_) global_device_ = dev; } } - urma_free_device_list(urma_devices); + urma_free_device_list(urma_dev_list); return devices_; #else return {}; #endif } -inline std::shared_ptr urma_device_manager::get_global_device() { +inline std::shared_ptr urma_device_manager::get_global_device() { if (!global_device_) get_all_devices(); return global_device_; } -} // namespace coro_io +} // namespace coro_io \ No newline at end of file diff --git a/include/ylt/coro_io/urma/urma_socket.hpp b/include/ylt/coro_io/urma/urma_socket.hpp index f82c53e9b..e3377bc2c 100644 --- a/include/ylt/coro_io/urma/urma_socket.hpp +++ b/include/ylt/coro_io/urma/urma_socket.hpp @@ -211,9 +211,13 @@ struct urma_socket_shared_state_t // Build URMA send WR urma_jfs_wr wr{}; + wr.opcode = URMA_OPC_SEND; wr.next = nullptr; - wr.sg_list = &sge; - wr.num_sge = sge.length ? 1 : 0; + urma_sge_t sge_list[1]; + sge_list[0].addr = sge.addr; + sge_list[0].len = sge.length; + wr.send.src.sge = sge_list; + wr.send.src.num_sge = sge.length ? 1 : 0; wr.user_ctx = reinterpret_cast(new callback_t(std::move(handler))); urma_jfs_wr* bad_wr = nullptr; @@ -268,17 +272,17 @@ struct urma_socket_shared_state_t // Determine if this is a send or recv completion based on context if (!send_cb_.empty()) { urma_socket_shared_state_t::resume( - std::pair{ec, static_cast(cr.len)}, + std::pair{ec, static_cast(cr.completion_len)}, send_cb_.pop()); } else if (!recv_cb_) { recv_result_.push( - std::pair{ec, static_cast(cr.len)}); + std::pair{ec, static_cast(cr.completion_len)}); } else { recv_buf_ = recv_queue_.pop(); urma_socket_shared_state_t::resume( - std::pair{ec, static_cast(cr.len)}, + std::pair{ec, static_cast(cr.completion_len)}, std::move(recv_cb_)); } } @@ -381,15 +385,6 @@ class urma_socket_t { state_->post_send_impl(buffer, std::move(cb)); } - // For ib_socket_t compatibility - accept ibv_sge and convert - void post_send(ibv_sge buffer, callback_t&& cb) { - urma_sge sge; - sge.addr = buffer.addr; - sge.length = buffer.length; - sge.lkey = buffer.lkey; - post_send(sge, std::move(cb)); - } - uint32_t get_buffer_size() const noexcept { return buffer_size_; } config_t& get_config() noexcept { return conf_; } From 2a963e923588a1f4f9691e22b5760f5d57d1b295 Mon Sep 17 00:00:00 2001 From: lixu Date: Thu, 4 Jun 2026 20:42:22 +0800 Subject: [PATCH 008/129] fix(urma_socket): fix type name conflicts with URMA library - Rename local urma_buffer_t to urma_buf_t to avoid conflict with URMA library type - Fix sge.length to sge.len per URMA API - Remove lkey from subview() as URMA uses UBVA addressing --- include/ylt/coro_io/urma/urma_socket.hpp | 46 +++++++++++------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/include/ylt/coro_io/urma/urma_socket.hpp b/include/ylt/coro_io/urma/urma_socket.hpp index e3377bc2c..ef33ccad7 100644 --- a/include/ylt/coro_io/urma/urma_socket.hpp +++ b/include/ylt/coro_io/urma/urma_socket.hpp @@ -43,25 +43,19 @@ namespace detail { struct urma_socket_shared_state_t; -// URMA-specific buffer representation (compatible with ibv_sge layout) -struct urma_sge { - uint64_t addr; - uint32_t length; - uint32_t lkey; -}; - -struct urma_buffer_t { +// URMA-specific buffer representation for coro_io +// Note: Avoid naming conflicts with URMA library types (urma_sge_t, urma_buf_t) +struct urma_buf_t { void* addr = nullptr; size_t length = 0; - uint32_t lkey = 0; // URMA key (used similarly to lkey) + uint32_t lkey = 0; - urma_sge subview(size_t offset = 0, size_t len = 0) const { - urma_sge sge; + urma_sge_t subview(size_t offset = 0, size_t len = 0) const { + urma_sge_t sge; sge.addr = reinterpret_cast( reinterpret_cast(addr) + offset); - sge.length = (len == 0) ? static_cast(length - offset) - : static_cast(len); - sge.lkey = lkey; + sge.len = (len == 0) ? static_cast(length - offset) + : static_cast(len); return sge; } @@ -92,15 +86,15 @@ struct urma_socket_shared_state_t urma_target_jetty_t* remote_jetty_ = nullptr; // Buffer management - std::vector recv_buffers_; - std::vector send_buffers_; - circle_buffer recv_queue_; - circle_buffer send_queue_; + std::vector recv_buffers_; + std::vector send_buffers_; + circle_buffer recv_queue_; + circle_buffer send_queue_; circle_buffer> recv_result_; circle_buffer send_cb_; callback_t recv_cb_; - urma_buffer_t recv_buf_; + urma_buf_t recv_buf_; std::size_t recv_buffer_cnt_ = 0; std::size_t send_buffer_cnt_ = 0; @@ -135,7 +129,7 @@ struct urma_socket_shared_state_t auto get_executor() const noexcept { return executor_->get_asio_executor(); } - void return_send_buffer(urma_buffer_t buffer) { + void return_send_buffer(urma_buf_t buffer) { assert(!send_queue_.full()); send_queue_.push(std::move(buffer)); } @@ -189,7 +183,7 @@ struct urma_socket_shared_state_t co_return; } - urma_buffer_t release_send_buffer() noexcept { + urma_buf_t release_send_buffer() noexcept { assert(send_queue_.size()); send_buffer_data_size_ = 0; return send_queue_.pop(); @@ -199,7 +193,7 @@ struct urma_socket_shared_state_t void post_send_impl(urma_sge sge, callback_t&& handler, bool skip_check_close = false) { - ELOG_TRACE << "post send sge length:" << sge.length + ELOG_TRACE << "post send sge length:" << sge.len << ", address:" << sge.addr; if (!skip_check_close && has_close_) [[unlikely]] { @@ -215,9 +209,9 @@ struct urma_socket_shared_state_t wr.next = nullptr; urma_sge_t sge_list[1]; sge_list[0].addr = sge.addr; - sge_list[0].len = sge.length; + sge_list[0].len = sge.len; wr.send.src.sge = sge_list; - wr.send.src.num_sge = sge.length ? 1 : 0; + wr.send.src.num_sge = sge.len ? 1 : 0; wr.user_ctx = reinterpret_cast(new callback_t(std::move(handler))); urma_jfs_wr* bad_wr = nullptr; @@ -567,7 +561,7 @@ class urma_socket_t { auto get_executor() const { return executor_->get_asio_executor(); } auto get_coro_executor() const { return executor_; } - urma_buffer_t release_send_buffer() noexcept { + urma_buf_t release_send_buffer() noexcept { return state_->release_send_buffer(); } @@ -578,7 +572,7 @@ class urma_socket_t { std::optional get_send_buffer_view() noexcept { if (state_->send_queue_.empty()) { // Get buffer from pool - urma_buffer_t buf; + urma_buf_t buf; // TODO: Get from URMA buffer pool if (!buf) { ELOG_WARN << "buffer out of limit, get send buffer failed"; From 78a80f97ec724f84766b6fa9161ba07fc781c680 Mon Sep 17 00:00:00 2001 From: lixu Date: Thu, 4 Jun 2026 20:47:38 +0800 Subject: [PATCH 009/129] refactor(urma_socket): restructure for URMA CTP API - Reorganize urma_socket.hpp based on URMA documentation - Use urma_buf_t with urma_target_seg_t instead of lkey - Add proper JFC/JFR/Jetty creation flow - Add register_buffer/unregister_buffer for segment management - Fix urma_sge_t field usage (len instead of length) Note: Still needs fixes for API details (alloc_jfc, jfc_cfg, etc) --- include/ylt/coro_io/urma/urma_socket.hpp | 829 ++++++++++------------- 1 file changed, 371 insertions(+), 458 deletions(-) diff --git a/include/ylt/coro_io/urma/urma_socket.hpp b/include/ylt/coro_io/urma/urma_socket.hpp index ef33ccad7..cf64ec6bb 100644 --- a/include/ylt/coro_io/urma/urma_socket.hpp +++ b/include/ylt/coro_io/urma/urma_socket.hpp @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include @@ -41,21 +40,17 @@ namespace coro_io { namespace detail { -struct urma_socket_shared_state_t; - -// URMA-specific buffer representation for coro_io -// Note: Avoid naming conflicts with URMA library types (urma_sge_t, urma_buf_t) +// URMA buffer wrapper - local representation struct urma_buf_t { void* addr = nullptr; size_t length = 0; - uint32_t lkey = 0; + urma_target_seg_t* seg = nullptr; // URMA segment handle - urma_sge_t subview(size_t offset = 0, size_t len = 0) const { + urma_sge_t to_sge() const { urma_sge_t sge; - sge.addr = reinterpret_cast( - reinterpret_cast(addr) + offset); - sge.len = (len == 0) ? static_cast(length - offset) - : static_cast(len); + sge.addr = reinterpret_cast(addr); + sge.len = static_cast(length); + sge.tseg = seg; return sge; } @@ -65,6 +60,7 @@ struct urma_buf_t { using callback_t = async_simple::util::move_only_function)>; +// URMA CTP Socket state struct urma_socket_shared_state_t : public std::enable_shared_from_this { static void resume(std::pair&& arg, @@ -86,8 +82,6 @@ struct urma_socket_shared_state_t urma_target_jetty_t* remote_jetty_ = nullptr; // Buffer management - std::vector recv_buffers_; - std::vector send_buffers_; circle_buffer recv_queue_; circle_buffer send_queue_; circle_buffer> recv_result_; @@ -98,10 +92,9 @@ struct urma_socket_shared_state_t std::size_t recv_buffer_cnt_ = 0; std::size_t send_buffer_cnt_ = 0; - uint32_t send_buffer_data_size_ = 0; uint32_t buffer_size_ = 256 * 1024; // Default 256KB - std::atomic has_close_ = {false}; + std::atomic has_close_{false}; bool peer_close_ = false; std::optional> wait_promise_; @@ -109,38 +102,36 @@ struct urma_socket_shared_state_t std::array remote_eid_; uint32_t remote_jetty_id_ = 0; - urma_socket_shared_state_t(coro_io::ExecutorWrapper<>* executor, - std::size_t recv_buffer_cnt, - std::size_t send_buffer_cnt, - std::size_t max_recv_buffer_cnt, - uint32_t buffer_size) - : executor_(executor), - soc_(executor->get_asio_executor()), - recv_buffer_cnt_(recv_buffer_cnt), - send_buffer_cnt_(send_buffer_cnt), - buffer_size_(buffer_size), - recv_queue_(max_recv_buffer_cnt), - send_queue_(send_buffer_cnt + 1), - recv_result_(max_recv_buffer_cnt), - send_cb_(send_buffer_cnt + 2) {} + urma_socket_shared_state_t() = default; + ~urma_socket_shared_state_t() { close(); } urma_socket_shared_state_t(urma_socket_shared_state_t&&) = delete; urma_socket_shared_state_t& operator=(urma_socket_shared_state_t&&) = delete; + // Initialize URMA resources + bool init(urma_context_t* ctx, + std::size_t recv_buffer_cnt, + std::size_t send_buffer_cnt, + uint32_t buffer_size); + + void close(); + auto get_executor() const noexcept { return executor_->get_asio_executor(); } - void return_send_buffer(urma_buf_t buffer) { - assert(!send_queue_.full()); - send_queue_.push(std::move(buffer)); - } + // Post receive buffer + void post_recv_impl(callback_t&& handler); - void wake_up_if_is_waiting(std::error_code ec) { - if (wait_promise_) { - auto promise = std::move(wait_promise_); - wait_promise_ = std::nullopt; - promise->setValue(ec); - } - } + // Post send with URMA buffer + void post_send_impl(urma_buf_t buf, callback_t&& handler); + + // Poll for completions + std::error_code poll_completion(); + + // Register memory as URMA segment + urma_buf_t register_buffer(void* addr, size_t len); + + // Unregister segment + void unregister_buffer(urma_buf_t& buf); async_simple::coro::Lazy waiting_write_over() { assert(send_cb_.size()); @@ -149,139 +140,12 @@ struct urma_socket_shared_state_t co_return ec; } - void cancel() { - assert(executor_->get_asio_executor().running_in_this_thread()); - std::error_code ec; - soc_.cancel(ec); - } - - void close_impl() { - ELOG_TRACE << "jetty closed"; - std::error_code ec; - soc_.cancel(ec); - soc_.close(ec); - } - - void close(bool should_check = true) { - assert(executor_->get_asio_executor().running_in_this_thread()); - - bool has_close = false; - if (should_check) { - has_close = has_close_.exchange(true); - } - if (!has_close) { - shutdown().start([self = shared_from_this()](auto&&) { - self->close_impl(); - }); - } - } - - async_simple::coro::Lazy shutdown() { - ELOG_TRACE << "start to notify peer close"; - co_await coro_io::sleep_for(std::chrono::seconds{1}, executor_); - ELOG_TRACE << "finished to notify peer close"; - co_return; - } - - urma_buf_t release_send_buffer() noexcept { - assert(send_queue_.size()); - send_buffer_data_size_ = 0; - return send_queue_.pop(); - } - - std::size_t sent_request_count() const noexcept { return send_cb_.size(); } - - void post_send_impl(urma_sge sge, callback_t&& handler, - bool skip_check_close = false) { - ELOG_TRACE << "post send sge length:" << sge.len - << ", address:" << sge.addr; - - if (!skip_check_close && has_close_) [[unlikely]] { - urma_socket_shared_state_t::resume( - std::pair{std::make_error_code(std::errc::operation_canceled), 0}, - std::move(handler)); - return; - } - - // Build URMA send WR - urma_jfs_wr wr{}; - wr.opcode = URMA_OPC_SEND; - wr.next = nullptr; - urma_sge_t sge_list[1]; - sge_list[0].addr = sge.addr; - sge_list[0].len = sge.len; - wr.send.src.sge = sge_list; - wr.send.src.num_sge = sge.len ? 1 : 0; - wr.user_ctx = reinterpret_cast(new callback_t(std::move(handler))); - - urma_jfs_wr* bad_wr = nullptr; - auto status = urma_post_jetty_send_wr(jetty_, &wr, &bad_wr); - if (status != 0) [[unlikely]] { - delete reinterpret_cast(wr.user_ctx); - auto err_code = std::make_error_code(std::errc{std::abs(status)}); - ELOG_ERROR << "urma post send failed: " << err_code.message(); - urma_socket_shared_state_t::resume(std::pair{err_code, std::size_t{0}}, - std::move(handler)); - } - else { - send_cb_.push(callback_t{}); // Placeholder for now - } - } - - void post_recv_impl(callback_t&& handler) { - if (!recv_result_.empty()) { - auto result = recv_result_.pop(); - recv_buf_ = std::move(recv_queue_.pop()); - urma_socket_shared_state_t::resume(std::move(result), std::move(handler)); - return; - } - else if (has_close_) [[unlikely]] { - urma_socket_shared_state_t::resume( - std::pair{std::make_error_code(std::errc::io_error), 0}, - std::move(handler)); - return; - } - recv_cb_ = std::move(handler); - } - - std::error_code poll_completion() { - // Poll JFC for completions - urma_cr_t cr_list[8]; - int num_completed = urma_poll_jfc(jfc_, 8, cr_list); - - if (num_completed < 0) [[unlikely]] { - return std::make_error_code(std::errc::io_error); - } - - std::error_code ec; - for (int i = 0; i < num_completed; ++i) { - auto& cr = cr_list[i]; - ec = (cr.status == 0) ? std::error_code{} - : std::make_error_code(std::errc::io_error); - - if (cr.status != 0) [[unlikely]] { - ELOG_WARN << "urma operation failed with status:" << cr.status; - } - - // Determine if this is a send or recv completion based on context - if (!send_cb_.empty()) { - urma_socket_shared_state_t::resume( - std::pair{ec, static_cast(cr.completion_len)}, - send_cb_.pop()); - } - else if (!recv_cb_) { - recv_result_.push( - std::pair{ec, static_cast(cr.completion_len)}); - } - else { - recv_buf_ = recv_queue_.pop(); - urma_socket_shared_state_t::resume( - std::pair{ec, static_cast(cr.completion_len)}, - std::move(recv_cb_)); - } + void wake_up_if_is_waiting(std::error_code ec) { + if (wait_promise_) { + auto promise = std::move(wait_promise_); + wait_promise_ = std::nullopt; + promise->setValue(ec); } - - return {}; } }; @@ -293,72 +157,32 @@ class urma_socket_t { uint32_t cq_size = 128; uint16_t recv_buffer_cnt = 8; uint16_t send_buffer_cnt = 4; - uint16_t jetty_cnt = 4; uint32_t buffer_size = 256 * 1024; // 256KB default std::string device_name; int eid_index = 0; - // Shared URMA context (can be nullptr for simple case) - std::shared_ptr urma_context; }; - // URMA socket info exchanged during handshake (similar to ib_socket_info) + // Socket info exchanged during handshake struct urma_socket_info { - uint8_t eid[URMA_EID_LEN]; // EID - uint32_t jetty_id; // Jetty ID - uint32_t buffer_size; // Buffer size + uint8_t eid[URMA_EID_LEN]; + uint32_t jetty_id; + uint32_t buffer_size; constexpr static auto struct_pack_config = struct_pack::DISABLE_TYPE_INFO; }; using callback_t = detail::callback_t; - urma_socket_t(coro_io::ExecutorWrapper<>* executor, const config_t& config) - : executor_(executor) { - init(config); - } - - urma_socket_t(coro_io::ExecutorWrapper<>* executor = coro_io::get_global_executor()) - : executor_(executor) { - init(config_t{}); - } - - urma_socket_t(const config_t& config) - : executor_(coro_io::get_global_executor()) { - init(config); - } - + urma_socket_t() = default; + urma_socket_t(const config_t& config); urma_socket_t(urma_socket_t&&) = default; - urma_socket_t& operator=(urma_socket_t&& o) { - close(); - remote_address_ = std::move(o.remote_address_); - remote_jetty_id_ = o.remote_jetty_id_; - remain_data_ = o.remain_data_; - state_ = std::move(o.state_); - executor_ = o.executor_; - conf_ = std::move(o.conf_); - buffer_size_ = o.buffer_size_; - return *this; - } - + urma_socket_t& operator=(urma_socket_t&&) = default; ~urma_socket_t() { close(); } bool is_open() const noexcept { return state_ != nullptr && !state_->has_close_; } - // Consume data from receive buffer - std::size_t consume(char* dst, std::size_t sz, int dst_gpu_id) { - auto len = std::min(sz, remain_data_.size()); - if (len) { - memcpy(dst, remain_data_.data(), len); - remain_data_ = remain_data_.substr(len); - if (remain_data_.empty()) { - // Return buffer to pool - } - } - return len; - } - - std::size_t remain_read_buffer_size() { return remain_data_.size(); } + std::size_t remain_read_buffer_size() const { return remain_data_.size(); } void set_read_buffer_len(std::size_t has_read_size, std::size_t remain_size) { remain_data_ = std::string_view{ @@ -366,296 +190,385 @@ class urma_socket_t { remain_size}; } - // Get current receive buffer as ibv_sge-compatible structure - urma_sge get_recv_buffer() { - assert(remain_read_buffer_size() == 0); - assert(state_->recv_buf_.addr != nullptr); - return state_->recv_buf_.subview(); - } + uint32_t get_buffer_size() const noexcept { return buffer_size_; } + config_t& get_config() noexcept { return conf_; } + const config_t& get_config() const noexcept { return conf_; } - void post_recv(callback_t&& cb) { state_->post_recv_impl(std::move(cb)); } + // Connection management + async_simple::coro::Lazy connect( + const std::string& addr, const std::string& port); - void post_send(urma_sge buffer, callback_t&& cb) { - state_->post_send_impl(buffer, std::move(cb)); - } + async_simple::coro::Lazy accept() noexcept; - uint32_t get_buffer_size() const noexcept { return buffer_size_; } + void close() noexcept; - config_t& get_config() noexcept { return conf_; } - const config_t& get_config() const noexcept { return conf_; } + // Post receive + void post_recv(callback_t&& cb) { state_->post_recv_impl(std::move(cb)); } - async_simple::coro::Lazy waiting_write_over() { - return state_->waiting_write_over(); + // Post send + void post_send(detail::urma_buf_t buf, callback_t&& cb) { + state_->post_send_impl(buf, std::move(cb)); } - // Accept incoming URMA connection via TCP handshake - async_simple::coro::Lazy accept( - std::string_view magic = "") noexcept { - urma_socket_t::urma_socket_info peer_info; - constexpr auto sz = struct_pack::get_needed_size(peer_info); - assert(magic.size() < sz.size()); - - char buffer[sz.size()]; - memcpy(buffer, magic.data(), magic.size()); - - auto [ec, _] = co_await async_read( - state_->soc_, - asio::buffer(buffer + magic.size(), sizeof(buffer) - magic.size())); - if (ec) [[unlikely]] { - co_return ec; - } + // Get executor + auto get_executor() const { return state_->get_executor(); } - auto ec2 = struct_pack::deserialize_to(peer_info, std::span{buffer}); - if (ec2) [[unlikely]] { - co_return std::make_error_code(std::errc::protocol_error); - } + // Remote address (for logging) + std::string remote_address() const { return remote_address_; } + uint32_t remote_jetty_id() const { return state_->remote_jetty_id_; } - ELOG_DEBUG << "Remote Jetty ID = " << peer_info.jetty_id; - remote_jetty_id_ = peer_info.jetty_id; - - // Copy remote EID - std::copy(std::begin(peer_info.eid), std::end(peer_info.eid), - state_->remote_eid_.begin()); + private: + detail::urma_socket_shared_state_t* state() const { return state_.get(); } - // Convert EID to address for compatibility - remote_address_ = eid_to_address(peer_info.eid); - ELOG_DEBUG << "Remote EID = " << remote_address_; + config_t conf_; + uint32_t buffer_size_ = 256 * 1024; + std::string remote_address_; + std::string_view remain_data_; + std::unique_ptr state_; +}; - buffer_size_ = std::min(peer_info.buffer_size, conf_.buffer_size); - ELOG_DEBUG << "Final buffer size = " << buffer_size_; +// Helper function to convert EID to address string +inline std::string eid_to_address(const uint8_t* eid) { + char buf[64]; + snprintf(buf, sizeof(buf), "%d.%d.%d.%d", eid[0], eid[1], eid[2], eid[3]); + return std::string(buf); +} - // Send back our info - urma_socket_info local_info{}; - local_info.jetty_id = get_local_jetty_id(); - local_info.buffer_size = conf_.buffer_size; - get_local_eid(local_info.eid); +// Implementation - struct_pack::serialize_to((char*)buffer, sz, local_info); - co_await async_write(state_->soc_, asio::buffer(buffer)); +namespace detail { - // Shutdown TCP socket - RDMA is now the data channel - std::error_code ignore_ec; - state_->soc_.shutdown(asio::ip::tcp::socket::shutdown_both, ignore_ec); - state_->soc_.close(ignore_ec); +inline bool urma_socket_shared_state_t::init( + urma_context_t* ctx, + std::size_t recv_buffer_cnt, + std::size_t send_buffer_cnt, + uint32_t buffer_size) { + urma_context_ = ctx; + recv_buffer_cnt_ = recv_buffer_cnt; + send_buffer_cnt_ = send_buffer_cnt; + buffer_size_ = buffer_size; + + // Create JFC (Completion Channel) + urma_jfc_cfg_t jfc_cfg = {}; + jfc_cfg.comp_type = URMA_CQ_TYPE_JFC; + jfc_cfg.queue_size = 64; + jfc_ = urma_create_jfc(ctx, &jfc_cfg); + if (!jfc_) { + ELOG_ERROR << "Failed to create JFC"; + return false; + } + + // Create JFR (Receive Queue) + urma_jfr_cfg_t jfr_cfg = {}; + jfr_cfg.jfc = jfc_; + jfr_cfg.queue_size = static_cast(recv_buffer_cnt); + jfr_ = urma_create_jfr(ctx, &jfr_cfg); + if (!jfr_) { + ELOG_ERROR << "Failed to create JFR"; + return false; + } + + // Create Jetty with shared JFR (CTP mode) + urma_jetty_cfg_t jetty_cfg = {}; + jetty_cfg.flag.bs.share_jfr = 1; + jetty_cfg.jfs_cfg.jfc = jfc_; + jetty_cfg.jfs_cfg.queue_size = static_cast(send_buffer_cnt); + jetty_cfg.shared.jfr = jfr_; + jetty_ = urma_create_jetty(ctx, &jetty_cfg); + if (!jetty_) { + ELOG_ERROR << "Failed to create Jetty"; + return false; + } + + // Initialize buffers + new (&recv_queue_) circle_buffer(recv_buffer_cnt); + new (&send_queue_) circle_buffer(send_buffer_cnt); + new (&recv_result_) circle_buffer>(recv_buffer_cnt); + new (&send_cb_) circle_buffer(send_buffer_cnt + 2); + + ELOG_INFO << "URMA CTP socket initialized, Jetty ID: " << urma_get_jetty_id(jetty_); + return true; +} + +inline void urma_socket_shared_state_t::close() { + if (has_close_.exchange(true)) { + return; + } + + if (jetty_) { + urma_delete_jetty(jetty_); + jetty_ = nullptr; + } + if (jfr_) { + urma_delete_jfr(jfr_); + jfr_ = nullptr; + } + if (jfc_) { + urma_delete_jfc(jfc_); + jfc_ = nullptr; + } +} + +inline urma_buf_t urma_socket_shared_state_t::register_buffer(void* addr, size_t len) { + urma_buf_t buf; + buf.addr = addr; + buf.length = len; + + urma_seg_cfg_t seg_cfg = {}; + seg_cfg.va = reinterpret_cast(addr); + seg_cfg.len = len; + seg_cfg.flag.bs.access = URMA_ACCESS_READ | URMA_ACCESS_WRITE; + seg_cfg.flag.bs.cacheable = URMA_NON_CACHEABLE; + seg_cfg.flag.bs.token_policy = URMA_TOKEN_NONE; + + buf.seg = urma_register_seg(urma_context_, &seg_cfg); + if (!buf.seg) { + ELOG_ERROR << "Failed to register segment"; + buf.addr = nullptr; + } + return buf; +} + +inline void urma_socket_shared_state_t::unregister_buffer(urma_buf_t& buf) { + if (buf.seg) { + urma_unregister_seg(buf.seg); + buf.seg = nullptr; + } + buf.addr = nullptr; + buf.length = 0; +} + +inline void urma_socket_shared_state_t::post_recv_impl(callback_t&& handler) { + if (!recv_result_.empty()) { + auto result = recv_result_.pop(); + recv_buf_ = std::move(recv_queue_.pop()); + urma_socket_shared_state_t::resume(std::move(result), std::move(handler)); + return; + } + else if (has_close_) [[unlikely]] { + urma_socket_shared_state_t::resume( + std::pair{std::make_error_code(std::errc::io_error), 0}, + std::move(handler)); + return; + } + recv_cb_ = std::move(handler); +} + +inline void urma_socket_shared_state_t::post_send_impl(urma_buf_t buf, callback_t&& handler) { + if (has_close_) [[unlikely]] { + urma_socket_shared_state_t::resume( + std::pair{std::make_error_code(std::errc::operation_canceled), 0}, + std::move(handler)); + return; + } + + // Build URMA SEND work request + urma_sge_t sge = buf.to_sge(); + urma_sg_t src_sg; + src_sg.sge = &sge; + src_sg.num_sge = 1; + + urma_send_wr_t send_wr = {}; + send_wr.src = src_sg; + + urma_jfs_wr_t jfs_wr = {}; + jfs_wr.opcode = URMA_OPC_SEND; + jfs_wr.send = send_wr; + jfs_wr.tjetty = remote_jetty_; + jfs_wr.user_ctx = reinterpret_cast(new callback_t(std::move(handler))); + + urma_jfs_wr* bad_wr = nullptr; + auto status = urma_post_jetty_send_wr(jetty_, &jfs_wr, &bad_wr); + if (status != 0) [[unlikely]] { + delete reinterpret_cast(jfs_wr.user_ctx); + auto err_code = std::make_error_code(std::errc{std::abs(status)}); + ELOG_ERROR << "URMA post send failed: " << err_code.message(); + urma_socket_shared_state_t::resume(std::pair{err_code, std::size_t{0}}, + std::move(handler)); + } else { + send_cb_.push(callback_t{}); // Placeholder + } +} + +inline std::error_code urma_socket_shared_state_t::poll_completion() { + urma_cr_t cr_list[8]; + int num_completed = urma_poll_jfc(jfc_, 8, cr_list); + + if (num_completed < 0) [[unlikely]] { + return std::make_error_code(std::errc::io_error); + } + + std::error_code ec; + for (int i = 0; i < num_completed; ++i) { + auto& cr = cr_list[i]; + ec = (cr.status == 0) ? std::error_code{} + : std::make_error_code(std::errc::io_error); + + if (cr.status != 0) [[unlikely]] { + ELOG_WARN << "URMA operation failed with status: " << cr.status; + } - co_return std::error_code{}; + // Determine if send or recv completion based on CR flag + if (cr.flag.bs.s_r == 0) { // Send completion + if (!send_cb_.empty()) { + urma_socket_shared_state_t::resume( + std::pair{ec, static_cast(cr.completion_len)}, + send_cb_.pop()); + } + } else { // Receive completion + if (!recv_cb_) { + recv_result_.push( + std::pair{ec, static_cast(cr.completion_len)}); + } else { + recv_buf_ = std::move(recv_queue_.pop()); + urma_socket_shared_state_t::resume( + std::pair{ec, static_cast(cr.completion_len)}, + std::move(recv_cb_)); + } + } } - void prepare_accept(asio::ip::tcp::socket soc) noexcept { - state_->soc_ = std::move(soc); - } + return {}; +} - async_simple::coro::Lazy accept( - asio::ip::tcp::socket soc) noexcept { - state_->soc_ = std::move(soc); - return accept(); - } +} // namespace detail - asio::ip::address get_remote_address() const noexcept { - return remote_address_; - } +// urma_socket_t implementation - uint32_t get_remote_qp_num() const noexcept { - return remote_jetty_id_; // jetty_id serves as QP number in URMA - } +inline urma_socket_t::urma_socket_t(const config_t& config) + : conf_(config), buffer_size_(config.buffer_size) {} - asio::ip::address get_local_address() const noexcept { - return local_address_; +inline void urma_socket_t::close() noexcept { + if (state_) { + state_->close(); + state_.reset(); } +} - uint32_t get_local_qp_num() const noexcept { - return get_local_jetty_id(); - } +inline async_simple::coro::Lazy urma_socket_t::connect( + const std::string& addr, const std::string& port) { + auto executor = co_await coro_io::get_current_executor; - // Magic number for protocol detection (similar to ib_md5_first_header) - constexpr static uint32_t urma_md5_header = - struct_pack::get_type_code(); - constexpr static char urma_md5_first_header = - struct_pack::get_type_code() % 256; - - // Connect to remote URMA endpoint - async_simple::coro::Lazy connect_impl() noexcept { - try { - urma_socket_t::urma_socket_info peer_info{}; - peer_info.jetty_id = get_local_jetty_id(); - peer_info.buffer_size = conf_.buffer_size; - get_local_eid(peer_info.eid); - - constexpr auto sz = struct_pack::get_needed_size(peer_info); - char buffer[sz.size()]; - struct_pack::serialize_to((char*)buffer, sz, peer_info); - - // Send our info - auto [ec, len] = co_await async_write(state_->soc_, - asio::buffer(buffer)); - if (ec) { - co_return std::move(ec); - } + state_ = std::make_unique(); + state_->executor_ = executor; - // Read remote info - std::tie(ec, len) = co_await async_read(state_->soc_, - asio::buffer(buffer)); - std::error_code ignore_ec; - state_->soc_.shutdown(asio::ip::tcp::socket::shutdown_both, ignore_ec); - state_->soc_.close(ignore_ec); + // Get global URMA device + auto device = get_global_urma_device(); + if (!device || !device->is_valid()) { + co_return std::make_error_code(std::errc::network_unreachable); + } - if (ec) { - co_return std::move(ec); - } + // Initialize URMA resources + if (!state_->init(device->context(), + conf_.recv_buffer_cnt, + conf_.send_buffer_cnt, + conf_.buffer_size)) { + state_.reset(); + co_return std::make_error_code(std::errc::operation_not_supported); + } - auto ec2 = struct_pack::deserialize_to(peer_info, std::span{buffer}); - if (ec2) [[unlikely]] { - co_return std::make_error_code(std::errc::protocol_error); - } + // TCP handshake for connection establishment + asio::ip::tcp::resolver resolver(executor->get_asio_executor()); + auto endpoints = co_await resolver.async_resolve(addr, port); - remote_jetty_id_ = peer_info.jetty_id; - std::copy(std::begin(peer_info.eid), std::end(peer_info.eid), - state_->remote_eid_.begin()); - remote_address_ = eid_to_address(peer_info.eid); - buffer_size_ = std::min(peer_info.buffer_size, conf_.buffer_size); + state_->soc_.connect(asio::ip::tcp::endpoint( + asio::ip::make_address(addr), std::stoi(port))); - } catch (const std::system_error& err) { - co_return err.code(); - } - co_return std::error_code{}; - } + // Exchange socket info + urma_socket_info local_info{}; + local_info.jetty_id = urma_get_jetty_id(state_->jetty_); + local_info.buffer_size = conf_.buffer_size; + // Get local EID from device + auto& local_eid = device->eid(); + std::memcpy(local_info.eid, local_eid.raw, URMA_EID_LEN); - async_simple::coro::Lazy connect( - const std::string& host, const std::string& port) noexcept { - auto ec = co_await async_connect(get_coro_executor(), state_->soc_, - host, port); - if (ec) [[unlikely]] { - co_return std::move(ec); - } - ec = co_await connect_impl(); - if (ec) [[unlikely]] { - close(); - } - co_return ec; - } + // Send our info + char buffer[sizeof(urma_socket_info)]; + std::memcpy(buffer, &local_info, sizeof(local_info)); + co_await async_write(state_->soc_, asio::buffer(buffer)); - template - async_simple::coro::Lazy connect( - const EndPointSeq& endpoint) noexcept { - auto ec = co_await async_connect(state_->soc_, endpoint); - if (ec) [[unlikely]] { - co_return std::move(ec); - } - ec = co_await connect_impl(); - if (ec) [[unlikely]] { - close(); - } + // Receive peer info + urma_socket_info peer_info; + auto [ec, _] = co_await async_read(state_->soc_, asio::buffer(buffer, sizeof(buffer))); + if (ec) [[unlikely]] { co_return ec; } + std::memcpy(&peer_info, buffer, sizeof(peer_info)); - void close() { - if (state_) { - if (!state_->has_close_.exchange(true)) { - asio::dispatch(executor_->get_asio_executor(), [state = state_]() { - state->close(false); - }); - } - } - } - - auto get_executor() const { return executor_->get_asio_executor(); } - auto get_coro_executor() const { return executor_; } + // Import remote jetty + urma_rjetty_t remote_jetty_id = {}; + remote_jetty_id.jetty_id = peer_info.jetty_id; + remote_jetty_id.eid = *reinterpret_cast(peer_info.eid); - urma_buf_t release_send_buffer() noexcept { - return state_->release_send_buffer(); + urma_import_jetty_flag_t flag = {}; + state_->remote_jetty_ = urma_import_jetty(state_->urma_context_, &remote_jetty_id, nullptr); + if (!state_->remote_jetty_) { + co_return std::make_error_code(std::errc::operation_not_supported); } - std::size_t sent_request_count() const noexcept { - return state_->sent_request_count(); - } + remote_address_ = eid_to_address(peer_info.eid); + state_->remote_jetty_id_ = peer_info.jetty_id; - std::optional get_send_buffer_view() noexcept { - if (state_->send_queue_.empty()) { - // Get buffer from pool - urma_buf_t buf; - // TODO: Get from URMA buffer pool - if (!buf) { - ELOG_WARN << "buffer out of limit, get send buffer failed"; - close(); - return std::nullopt; - } - state_->send_queue_.push(std::move(buf)); - } - return state_->send_queue_.front().subview(state_->send_buffer_data_size_); - } + ELOG_INFO << "Connected to URMA peer, Jetty ID: " << peer_info.jetty_id; + co_return std::error_code{}; +} - std::size_t get_free_send_buffer_size() noexcept { - return buffer_size_ - state_->send_buffer_data_size_; - } +inline async_simple::coro::Lazy urma_socket_t::accept() noexcept { + auto executor = co_await coro_io::get_current_executor; - void consume_send_buffer(std::size_t sz) noexcept { - state_->send_buffer_data_size_ += sz; - } + state_ = std::make_unique(); + state_->executor_ = executor; - std::shared_ptr get_state() const noexcept { - return state_; + // Get global URMA device + auto device = get_global_urma_device(); + if (!device || !device->is_valid()) { + co_return std::make_error_code(std::errc::network_unreachable); } - detail::urma_socket_shared_state_t* get_raw_state() const noexcept { - return state_.get(); + // Initialize URMA resources + if (!state_->init(device->context(), + conf_.recv_buffer_cnt, + conf_.send_buffer_cnt, + conf_.buffer_size)) { + state_.reset(); + co_return std::make_error_code(std::errc::operation_not_supported); } - // URMA-specific methods - uint32_t get_local_jetty_id() const { - if (state_ && state_->jetty_) { - return state_->jetty_->jetty_id.id; - } - return 0; + // Receive peer info + char buffer[sizeof(urma_socket_info)]; + auto [ec, _] = co_await async_read(state_->soc_, asio::buffer(buffer)); + if (ec) [[unlikely]] { + co_return ec; } - void get_local_eid(uint8_t* eid) const { - if (state_ && state_->jfc_) { - std::copy(std::begin(state_->jfc_->jfc_id.eid.raw), - std::end(state_->jfc_->jfc_id.eid.raw), eid); - } - } + urma_socket_info peer_info; + std::memcpy(&peer_info, buffer, sizeof(peer_info)); - private: - void init(const config_t& config) { - conf_ = config; - conf_.recv_buffer_cnt = std::max(conf_.recv_buffer_cnt, 1); - conf_.send_buffer_cnt = std::max(conf_.send_buffer_cnt, 1); + // Import remote jetty + urma_rjetty_t remote_jetty_id = {}; + remote_jetty_id.jetty_id = peer_info.jetty_id; + remote_jetty_id.eid = *reinterpret_cast(peer_info.eid); - ELOG_INFO << "urma_socket config: recv_buffer_cnt:" << conf_.recv_buffer_cnt - << ", send_buffer_cnt:" << conf_.send_buffer_cnt - << ", buffer_size:" << conf_.buffer_size; + state_->remote_jetty_ = urma_import_jetty(state_->urma_context_, &remote_jetty_id, nullptr); + if (!state_->remote_jetty_) { + co_return std::make_error_code(std::errc::operation_not_supported); + } - state_ = std::make_shared( - executor_, conf_.recv_buffer_cnt, conf_.send_buffer_cnt, - conf_.recv_buffer_cnt + 2, conf_.buffer_size); + // Send our info + urma_socket_info local_info{}; + local_info.jetty_id = urma_get_jetty_id(state_->jetty_); + local_info.buffer_size = conf_.buffer_size; + auto& local_eid = device->eid(); + std::memcpy(local_info.eid, local_eid.raw, URMA_EID_LEN); - buffer_size_ = conf_.buffer_size; - } + std::memcpy(buffer, &local_info, sizeof(local_info)); + co_await async_write(state_->soc_, asio::buffer(buffer)); - // Convert URMA EID to asio::ip::address for compatibility - static asio::ip::address eid_to_address(const uint8_t* eid) { - // EID is 16 bytes, we can format first 4 bytes as IPv4 for simplicity - // or use a proper conversion - char buf[64]; - snprintf(buf, sizeof(buf), "%d.%d.%d.%d", - eid[0], eid[1], eid[2], eid[3]); - std::error_code ec; - auto addr = asio::ip::make_address(buf, ec); - if (ec) { - // Fallback to localhost if conversion fails - return asio::ip::make_address_v4(0x7F000001); // 127.0.0.1 - } - return addr; - } + remote_address_ = eid_to_address(peer_info.eid); + state_->remote_jetty_id_ = peer_info.jetty_id; - asio::ip::address remote_address_; - uint32_t remote_jetty_id_{0}; - std::string_view remain_data_; - std::shared_ptr state_; - coro_io::ExecutorWrapper<>* executor_; - config_t conf_; - uint32_t buffer_size_{0}; - asio::ip::address local_address_; -}; + ELOG_INFO << "Accepted URMA connection from peer, Jetty ID: " << peer_info.jetty_id; + co_return std::error_code{}; +} } // namespace coro_io \ No newline at end of file From d2e172365cd3a1b46da213a6256eb17ac71d6a4b Mon Sep 17 00:00:00 2001 From: lixu Date: Thu, 4 Jun 2026 20:52:36 +0800 Subject: [PATCH 010/129] refactor(urma_socket): fix JFC/JFR/Jetty creation for CTP mode Known issues remaining: - urma_query_jetty signature incorrect - urma_import_jetty parameters need urma_rjetty_t structure - ASIO coroutine compatibility issues (await_ready) - Socket wrapper interface mismatches --- include/ylt/coro_io/urma/urma_socket.hpp | 29 +++++++++++++++++------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/include/ylt/coro_io/urma/urma_socket.hpp b/include/ylt/coro_io/urma/urma_socket.hpp index cf64ec6bb..9667c4183 100644 --- a/include/ylt/coro_io/urma/urma_socket.hpp +++ b/include/ylt/coro_io/urma/urma_socket.hpp @@ -248,20 +248,27 @@ inline bool urma_socket_shared_state_t::init( send_buffer_cnt_ = send_buffer_cnt; buffer_size_ = buffer_size; - // Create JFC (Completion Channel) + // Create JFC (Completion Channel) - polling mode (jfce = nullptr) urma_jfc_cfg_t jfc_cfg = {}; - jfc_cfg.comp_type = URMA_CQ_TYPE_JFC; - jfc_cfg.queue_size = 64; + jfc_cfg.depth = 64; + jfc_cfg.flag.value = 0; + jfc_cfg.jfce = nullptr; // polling mode + jfc_cfg.user_ctx = 0; jfc_ = urma_create_jfc(ctx, &jfc_cfg); if (!jfc_) { ELOG_ERROR << "Failed to create JFC"; return false; } - // Create JFR (Receive Queue) + // Create JFR (Receive Queue) for CTP mode urma_jfr_cfg_t jfr_cfg = {}; + jfr_cfg.depth = static_cast(recv_buffer_cnt); + jfr_cfg.flag.value = 0; + jfr_cfg.trans_mode = URMA_TM_RM; // CTP uses RM mode + jfr_cfg.max_sge = 1; + jfr_cfg.min_rnr_timer = 12; // typical value jfr_cfg.jfc = jfc_; - jfr_cfg.queue_size = static_cast(recv_buffer_cnt); + jfr_cfg.token_value = {}; // empty token jfr_ = urma_create_jfr(ctx, &jfr_cfg); if (!jfr_) { ELOG_ERROR << "Failed to create JFR"; @@ -270,9 +277,12 @@ inline bool urma_socket_shared_state_t::init( // Create Jetty with shared JFR (CTP mode) urma_jetty_cfg_t jetty_cfg = {}; - jetty_cfg.flag.bs.share_jfr = 1; + jetty_cfg.flag.bs.share_jfr = 1; // CTP requires shared JFR + jetty_cfg.jfs_cfg.depth = static_cast(send_buffer_cnt + 1); + jetty_cfg.jfs_cfg.flag.value = 0; + jetty_cfg.jfs_cfg.trans_mode = URMA_TM_RM; // CTP uses RM mode jetty_cfg.jfs_cfg.jfc = jfc_; - jetty_cfg.jfs_cfg.queue_size = static_cast(send_buffer_cnt); + jetty_cfg.jfs_cfg.user_ctx = 0; jetty_cfg.shared.jfr = jfr_; jetty_ = urma_create_jetty(ctx, &jetty_cfg); if (!jetty_) { @@ -286,7 +296,10 @@ inline bool urma_socket_shared_state_t::init( new (&recv_result_) circle_buffer>(recv_buffer_cnt); new (&send_cb_) circle_buffer(send_buffer_cnt + 2); - ELOG_INFO << "URMA CTP socket initialized, Jetty ID: " << urma_get_jetty_id(jetty_); + // Get Jetty ID for connection establishment + urma_jetty_attr_t jetty_attr = {}; + urma_query_jetty(jetty_, nullptr, &jetty_attr); + ELOG_INFO << "URMA CTP socket initialized, Jetty ID: " << jetty_attr.jetty_id.jetty_id; return true; } From 8c52020758e36cc79fa51b10222f381d75c057e9 Mon Sep 17 00:00:00 2001 From: lixu Date: Thu, 4 Jun 2026 21:14:17 +0800 Subject: [PATCH 011/129] refactor(urma_socket): fix URMA CTP API usage and ASIO coroutine issues - Fix urma_query_jetty call - access Jetty ID directly via jetty_->jetty_id.id - Fix urma_import_jetty parameters - properly initialize urma_rjetty_t with trans_mode, type, tp_type, and flag fields - Fix executor type mismatch - use async_simple::CurrentExecutor{} and proper cast - Add missing urma_socket_t methods: prepare_accept, get_remote_address, get_local_address, get_remote_qp_num, get_local_qp_num - Add urma_md5_header and urma_md5_first_header for protocol identification - Fix waiting_write_over - stub out until Future awaiting is fixed - Initialize socket properly in shared state with placement new Note: urma_example.cpp still needs integration fixes for coro_rpc_client --- .claude/plans/urma-ctp-implementation-plan.md | 142 + .claude/rules/common/agents.md | 50 + .claude/rules/common/code-review.md | 124 + .claude/rules/common/coding-style.md | 90 + .claude/rules/common/development-workflow.md | 44 + .claude/rules/common/git-workflow.md | 24 + .claude/rules/common/hooks.md | 30 + .claude/rules/common/patterns.md | 31 + .claude/rules/common/performance.md | 55 + .claude/rules/common/security.md | 29 + .claude/rules/common/testing.md | 57 + .claude/rules/cpp/coding-style.md | 44 + .claude/rules/cpp/hooks.md | 39 + .claude/rules/cpp/patterns.md | 51 + .claude/rules/cpp/security.md | 51 + .claude/rules/cpp/testing.md | 44 + .claude/settings.json | 44 + .claude/skills/query-urma-docs/SKILL.md | 60 + .../query-urma-docs/URMA API Guide.ch.md | 16525 ++++++++++++++++ .../URMA QuickStart Guide.ch.md | 336 + .../query-urma-docs/URMA User Guide.ch.md | 2522 +++ include/ylt/coro_io/urma/urma_socket.hpp | 117 +- 22 files changed, 20481 insertions(+), 28 deletions(-) create mode 100644 .claude/plans/urma-ctp-implementation-plan.md create mode 100644 .claude/rules/common/agents.md create mode 100644 .claude/rules/common/code-review.md create mode 100644 .claude/rules/common/coding-style.md create mode 100644 .claude/rules/common/development-workflow.md create mode 100644 .claude/rules/common/git-workflow.md create mode 100644 .claude/rules/common/hooks.md create mode 100644 .claude/rules/common/patterns.md create mode 100644 .claude/rules/common/performance.md create mode 100644 .claude/rules/common/security.md create mode 100644 .claude/rules/common/testing.md create mode 100644 .claude/rules/cpp/coding-style.md create mode 100644 .claude/rules/cpp/hooks.md create mode 100644 .claude/rules/cpp/patterns.md create mode 100644 .claude/rules/cpp/security.md create mode 100644 .claude/rules/cpp/testing.md create mode 100644 .claude/settings.json create mode 100644 .claude/skills/query-urma-docs/SKILL.md create mode 100644 .claude/skills/query-urma-docs/URMA API Guide.ch.md create mode 100644 .claude/skills/query-urma-docs/URMA QuickStart Guide.ch.md create mode 100644 .claude/skills/query-urma-docs/URMA User Guide.ch.md diff --git a/.claude/plans/urma-ctp-implementation-plan.md b/.claude/plans/urma-ctp-implementation-plan.md new file mode 100644 index 000000000..c54adc16d --- /dev/null +++ b/.claude/plans/urma-ctp-implementation-plan.md @@ -0,0 +1,142 @@ +# URMA CTP 实现计划 + +## 当前状态 + +### 已完成 +1. `circle_buffer` 提取到 `coro_io/detail/circle_buffer.hpp` +2. `urma_device.hpp` 类型冲突修复（`urma_device_wrapper_t`） +3. `urma_socket.hpp` 基础重构 +4. JFC/JFR/Jetty 创建流程修正 + +### 已知问题 +1. `urma_query_jetty` 签名不正确 +2. `urma_import_jetty` 参数结构错误 +3. ASIO 协程兼容性问题 +4. Socket wrapper 接口不匹配 + +--- + +## URMA CTP API 正确用法 + +### 1. JFC 创建 (Completion Channel) +```c +urma_jfc_cfg_t jfc_cfg = {}; +jfc_cfg.depth = 64; +jfc_cfg.flag.value = 0; +jfc_cfg.jfce = nullptr; // polling mode +jfc_cfg.user_ctx = 0; +urma_jfc_t* jfc = urma_create_jfc(ctx, &jfc_cfg); +``` + +### 2. JFR 创建 (Receive Queue) +```c +urma_jfr_cfg_t jfr_cfg = {}; +jfr_cfg.depth = recv_cnt; +jfr_cfg.flag.value = 0; +jfr_cfg.trans_mode = URMA_TM_RM; +jfr_cfg.max_sge = 1; +jfr_cfg.min_rnr_timer = 12; +jfr_cfg.jfc = jfc; +jfr_cfg.token_value = {}; +urma_jfr_t* jfr = urma_create_jfr(ctx, &jfr_cfg); +``` + +### 3. Jetty 创建 (CTP Mode) +```c +urma_jetty_cfg_t jetty_cfg = {}; +jetty_cfg.flag.bs.share_jfr = 1; +jetty_cfg.jfs_cfg.depth = send_cnt + 1; +jetty_cfg.jfs_cfg.flag.value = 0; +jetty_cfg.jfs_cfg.trans_mode = URMA_TM_RM; +jetty_cfg.jfs_cfg.jfc = jfc; +jetty_cfg.jfs_cfg.user_ctx = 0; +jetty_cfg.shared.jfr = jfr; +urma_jetty_t* jetty = urma_create_jetty(ctx, &jetty_cfg); +``` + +### 4. Jetty ID 获取 +```c +// 通过 jetty->jfs_id.id 获取 +uint32_t jetty_id = jetty->jfs_id.id; +``` + +### 5. Segment 注册 +```c +urma_seg_cfg_t seg_cfg = {}; +seg_cfg.va = (uint64_t)buffer; +seg_cfg.len = buffer_size; +seg_cfg.flag.bs.access = URMA_ACCESS_READ | URMA_ACCESS_WRITE; +seg_cfg.flag.bs.token_policy = URMA_TOKEN_NONE; +urma_target_seg_t* tseg = urma_register_seg(ctx, &seg_cfg); +``` + +### 6. 导入远端 Jetty +```c +urma_rjetty_t remote = {}; +remote.jetty_id.eid = peer_eid; +remote.jetty_id.id = peer_jetty_id; +remote.trans_mode = URMA_TM_RM; +remote.type = URMA_JETTY; +remote.tp_type = URMA_RTP; // or URMA_CTP +urma_target_jetty_t* remote_tjetty = urma_import_jetty(ctx, &remote, nullptr); +``` + +### 7. 发送数据 (SEND) +```c +urma_sge_t sge = { + .addr = (uint64_t)buffer, + .len = data_len, + .tseg = local_tseg +}; +urma_sg_t src = { + .sge = &sge, + .num_sge = 1 +}; +urma_send_wr_t send_wr = { + .src = src +}; +urma_jfs_wr_t jfs_wr = { + .opcode = URMA_OPC_SEND, + .send = send_wr, + .tjetty = remote_tjetty +}; +urma_post_jetty_send_wr(jetty, &jfs_wr, &bad_wr); +``` + +### 8. 轮询完成 +```c +urma_cr_t cr[8]; +int cnt = urma_poll_jfc(jfc, 8, cr); +for (int i = 0; i < cnt; ++i) { + // cr[i].completion_len - 传输字节数 + // cr[i].status - 状态 + // cr[i].flag.bs.s_r - 0=send, 1=recv +} +``` + +--- + +## 待修复清单 + +### 高优先级 +- [ ] `urma_socket.hpp`: 修复 `urma_query_jetty` 调用 +- [ ] `urma_socket.hpp`: 修复 `urma_import_jetty` 参数 +- [ ] `urma_socket.hpp`: 正确获取 Jetty ID +- [ ] `socket_wrapper.hpp`: 添加缺失接口 + +### 中优先级 +- [ ] ASIO 协程兼容性问题 +- [ ] `await_ready` Future 使用错误 +- [ ] `async_read/write` 调用方式 + +### 低优先级 +- [ ] `urma_socket_info` 序列化格式 +- [ ] 连接握手协议 + +--- + +## 参考文档 + +- URMA API Guide: `.claude/skills/query-urma-docs/URMA API Guide.ch.md` +- URMA User Guide: `.claude/skills/query-urma-docs/URMA User Guide.ch.md` +- URMA 头文件: `include/ylt/urma/urma_api.h`, `urma_types.h` \ No newline at end of file diff --git a/.claude/rules/common/agents.md b/.claude/rules/common/agents.md new file mode 100644 index 000000000..09d636489 --- /dev/null +++ b/.claude/rules/common/agents.md @@ -0,0 +1,50 @@ +# Agent Orchestration + +## Available Agents + +Located in `~/.claude/agents/`: + +| Agent | Purpose | When to Use | +|-------|---------|-------------| +| planner | Implementation planning | Complex features, refactoring | +| architect | System design | Architectural decisions | +| tdd-guide | Test-driven development | New features, bug fixes | +| code-reviewer | Code review | After writing code | +| security-reviewer | Security analysis | Before commits | +| build-error-resolver | Fix build errors | When build fails | +| e2e-runner | E2E testing | Critical user flows | +| refactor-cleaner | Dead code cleanup | Code maintenance | +| doc-updater | Documentation | Updating docs | +| rust-reviewer | Rust code review | Rust projects | + +## Immediate Agent Usage + +No user prompt needed: +1. Complex feature requests - Use **planner** agent +2. Code just written/modified - Use **code-reviewer** agent +3. Bug fix or new feature - Use **tdd-guide** agent +4. Architectural decision - Use **architect** agent + +## Parallel Task Execution + +ALWAYS use parallel Task execution for independent operations: + +```markdown +# GOOD: Parallel execution +Launch 3 agents in parallel: +1. Agent 1: Security analysis of auth module +2. Agent 2: Performance review of cache system +3. Agent 3: Type checking of utilities + +# BAD: Sequential when unnecessary +First agent 1, then agent 2, then agent 3 +``` + +## Multi-Perspective Analysis + +For complex problems, use split role sub-agents: +- Factual reviewer +- Senior engineer +- Security expert +- Consistency reviewer +- Redundancy checker diff --git a/.claude/rules/common/code-review.md b/.claude/rules/common/code-review.md new file mode 100644 index 000000000..d79ba9bf0 --- /dev/null +++ b/.claude/rules/common/code-review.md @@ -0,0 +1,124 @@ +# Code Review Standards + +## Purpose + +Code review ensures quality, security, and maintainability before code is merged. This rule defines when and how to conduct code reviews. + +## When to Review + +**MANDATORY review triggers:** + +- After writing or modifying code +- Before any commit to shared branches +- When security-sensitive code is changed (auth, payments, user data) +- When architectural changes are made +- Before merging pull requests + +**Pre-Review Requirements:** + +Before requesting review, ensure: + +- All automated checks (CI/CD) are passing +- Merge conflicts are resolved +- Branch is up to date with target branch + +## Review Checklist + +Before marking code complete: + +- [ ] Code is readable and well-named +- [ ] Functions are focused (<50 lines) +- [ ] Files are cohesive (<800 lines) +- [ ] No deep nesting (>4 levels) +- [ ] Errors are handled explicitly +- [ ] No hardcoded secrets or credentials +- [ ] No console.log or debug statements +- [ ] Tests exist for new functionality +- [ ] Test coverage meets 80% minimum + +## Security Review Triggers + +**STOP and use security-reviewer agent when:** + +- Authentication or authorization code +- User input handling +- Database queries +- File system operations +- External API calls +- Cryptographic operations +- Payment or financial code + +## Review Severity Levels + +| Level | Meaning | Action | +|-------|---------|--------| +| CRITICAL | Security vulnerability or data loss risk | **BLOCK** - Must fix before merge | +| HIGH | Bug or significant quality issue | **WARN** - Should fix before merge | +| MEDIUM | Maintainability concern | **INFO** - Consider fixing | +| LOW | Style or minor suggestion | **NOTE** - Optional | + +## Agent Usage + +Use these agents for code review: + +| Agent | Purpose | +|-------|---------| +| **code-reviewer** | General code quality, patterns, best practices | +| **security-reviewer** | Security vulnerabilities, OWASP Top 10 | +| **typescript-reviewer** | TypeScript/JavaScript specific issues | +| **python-reviewer** | Python specific issues | +| **go-reviewer** | Go specific issues | +| **rust-reviewer** | Rust specific issues | + +## Review Workflow + +``` +1. Run git diff to understand changes +2. Check security checklist first +3. Review code quality checklist +4. Run relevant tests +5. Verify coverage >= 80% +6. Use appropriate agent for detailed review +``` + +## Common Issues to Catch + +### Security + +- Hardcoded credentials (API keys, passwords, tokens) +- SQL injection (string concatenation in queries) +- XSS vulnerabilities (unescaped user input) +- Path traversal (unsanitized file paths) +- CSRF protection missing +- Authentication bypasses + +### Code Quality + +- Large functions (>50 lines) - split into smaller +- Large files (>800 lines) - extract modules +- Deep nesting (>4 levels) - use early returns +- Missing error handling - handle explicitly +- Mutation patterns - prefer immutable operations +- Missing tests - add test coverage + +### Performance + +- N+1 queries - use JOINs or batching +- Missing pagination - add LIMIT to queries +- Unbounded queries - add constraints +- Missing caching - cache expensive operations + +## Approval Criteria + +- **Approve**: No CRITICAL or HIGH issues +- **Warning**: Only HIGH issues (merge with caution) +- **Block**: CRITICAL issues found + +## Integration with Other Rules + +This rule works with: + +- [testing.md](testing.md) - Test coverage requirements +- [security.md](security.md) - Security checklist +- [git-workflow.md](git-workflow.md) - Commit standards +- [agents.md](agents.md) - Agent delegation diff --git a/.claude/rules/common/coding-style.md b/.claude/rules/common/coding-style.md new file mode 100644 index 000000000..e72f3f119 --- /dev/null +++ b/.claude/rules/common/coding-style.md @@ -0,0 +1,90 @@ +# Coding Style + +## Immutability (CRITICAL) + +ALWAYS create new objects, NEVER mutate existing ones: + +``` +// Pseudocode +WRONG: modify(original, field, value) → changes original in-place +CORRECT: update(original, field, value) → returns new copy with change +``` + +Rationale: Immutable data prevents hidden side effects, makes debugging easier, and enables safe concurrency. + +## Core Principles + +### KISS (Keep It Simple) + +- Prefer the simplest solution that actually works +- Avoid premature optimization +- Optimize for clarity over cleverness + +### DRY (Don't Repeat Yourself) + +- Extract repeated logic into shared functions or utilities +- Avoid copy-paste implementation drift +- Introduce abstractions when repetition is real, not speculative + +### YAGNI (You Aren't Gonna Need It) + +- Do not build features or abstractions before they are needed +- Avoid speculative generality +- Start simple, then refactor when the pressure is real + +## File Organization + +MANY SMALL FILES > FEW LARGE FILES: +- High cohesion, low coupling +- 200-400 lines typical, 800 max +- Extract utilities from large modules +- Organize by feature/domain, not by type + +## Error Handling + +ALWAYS handle errors comprehensively: +- Handle errors explicitly at every level +- Provide user-friendly error messages in UI-facing code +- Log detailed error context on the server side +- Never silently swallow errors + +## Input Validation + +ALWAYS validate at system boundaries: +- Validate all user input before processing +- Use schema-based validation where available +- Fail fast with clear error messages +- Never trust external data (API responses, user input, file content) + +## Naming Conventions + +- Variables and functions: `camelCase` with descriptive names +- Booleans: prefer `is`, `has`, `should`, or `can` prefixes +- Interfaces, types, and components: `PascalCase` +- Constants: `UPPER_SNAKE_CASE` +- Custom hooks: `camelCase` with a `use` prefix + +## Code Smells to Avoid + +### Deep Nesting + +Prefer early returns over nested conditionals once the logic starts stacking. + +### Magic Numbers + +Use named constants for meaningful thresholds, delays, and limits. + +### Long Functions + +Split large functions into focused pieces with clear responsibilities. + +## Code Quality Checklist + +Before marking work complete: +- [ ] Code is readable and well-named +- [ ] Functions are small (<50 lines) +- [ ] Files are focused (<800 lines) +- [ ] No deep nesting (>4 levels) +- [ ] Proper error handling +- [ ] No hardcoded values (use constants or config) +- [ ] No mutation (immutable patterns used) diff --git a/.claude/rules/common/development-workflow.md b/.claude/rules/common/development-workflow.md new file mode 100644 index 000000000..ae070be2f --- /dev/null +++ b/.claude/rules/common/development-workflow.md @@ -0,0 +1,44 @@ +# Development Workflow + +> This file extends [common/git-workflow.md](./git-workflow.md) with the full feature development process that happens before git operations. + +The Feature Implementation Workflow describes the development pipeline: research, planning, TDD, code review, and then committing to git. + +## Feature Implementation Workflow + +0. **Research & Reuse** _(mandatory before any new implementation)_ + - **GitHub code search first:** Run `gh search repos` and `gh search code` to find existing implementations, templates, and patterns before writing anything new. + - **Library docs second:** Use Context7 or primary vendor docs to confirm API behavior, package usage, and version-specific details before implementing. + - **Exa only when the first two are insufficient:** Use Exa for broader web research or discovery after GitHub search and primary docs. + - **Check package registries:** Search npm, PyPI, crates.io, and other registries before writing utility code. Prefer battle-tested libraries over hand-rolled solutions. + - **Search for adaptable implementations:** Look for open-source projects that solve 80%+ of the problem and can be forked, ported, or wrapped. + - Prefer adopting or porting a proven approach over writing net-new code when it meets the requirement. + +1. **Plan First** + - Use **planner** agent to create implementation plan + - Generate planning docs before coding: PRD, architecture, system_design, tech_doc, task_list + - Identify dependencies and risks + - Break down into phases + +2. **TDD Approach** + - Use **tdd-guide** agent + - Write tests first (RED) + - Implement to pass tests (GREEN) + - Refactor (IMPROVE) + - Verify 80%+ coverage + +3. **Code Review** + - Use **code-reviewer** agent immediately after writing code + - Address CRITICAL and HIGH issues + - Fix MEDIUM issues when possible + +4. **Commit & Push** + - Detailed commit messages + - Follow conventional commits format + - See [git-workflow.md](./git-workflow.md) for commit message format and PR process + +5. **Pre-Review Checks** + - Verify all automated checks (CI/CD) are passing + - Resolve any merge conflicts + - Ensure branch is up to date with target branch + - Only request review after these checks pass diff --git a/.claude/rules/common/git-workflow.md b/.claude/rules/common/git-workflow.md new file mode 100644 index 000000000..d57d9e281 --- /dev/null +++ b/.claude/rules/common/git-workflow.md @@ -0,0 +1,24 @@ +# Git Workflow + +## Commit Message Format +``` +: + + +``` + +Types: feat, fix, refactor, docs, test, chore, perf, ci + +Note: Attribution disabled globally via ~/.claude/settings.json. + +## Pull Request Workflow + +When creating PRs: +1. Analyze full commit history (not just latest commit) +2. Use `git diff [base-branch]...HEAD` to see all changes +3. Draft comprehensive PR summary +4. Include test plan with TODOs +5. Push with `-u` flag if new branch + +> For the full development process (planning, TDD, code review) before git operations, +> see [development-workflow.md](./development-workflow.md). diff --git a/.claude/rules/common/hooks.md b/.claude/rules/common/hooks.md new file mode 100644 index 000000000..54394083e --- /dev/null +++ b/.claude/rules/common/hooks.md @@ -0,0 +1,30 @@ +# Hooks System + +## Hook Types + +- **PreToolUse**: Before tool execution (validation, parameter modification) +- **PostToolUse**: After tool execution (auto-format, checks) +- **Stop**: When session ends (final verification) + +## Auto-Accept Permissions + +Use with caution: +- Enable for trusted, well-defined plans +- Disable for exploratory work +- Never use dangerously-skip-permissions flag +- Configure `allowedTools` in `~/.claude.json` instead + +## TodoWrite Best Practices + +Use TodoWrite tool to: +- Track progress on multi-step tasks +- Verify understanding of instructions +- Enable real-time steering +- Show granular implementation steps + +Todo list reveals: +- Out of order steps +- Missing items +- Extra unnecessary items +- Wrong granularity +- Misinterpreted requirements diff --git a/.claude/rules/common/patterns.md b/.claude/rules/common/patterns.md new file mode 100644 index 000000000..959939f42 --- /dev/null +++ b/.claude/rules/common/patterns.md @@ -0,0 +1,31 @@ +# Common Patterns + +## Skeleton Projects + +When implementing new functionality: +1. Search for battle-tested skeleton projects +2. Use parallel agents to evaluate options: + - Security assessment + - Extensibility analysis + - Relevance scoring + - Implementation planning +3. Clone best match as foundation +4. Iterate within proven structure + +## Design Patterns + +### Repository Pattern + +Encapsulate data access behind a consistent interface: +- Define standard operations: findAll, findById, create, update, delete +- Concrete implementations handle storage details (database, API, file, etc.) +- Business logic depends on the abstract interface, not the storage mechanism +- Enables easy swapping of data sources and simplifies testing with mocks + +### API Response Format + +Use a consistent envelope for all API responses: +- Include a success/status indicator +- Include the data payload (nullable on error) +- Include an error message field (nullable on success) +- Include metadata for paginated responses (total, page, limit) diff --git a/.claude/rules/common/performance.md b/.claude/rules/common/performance.md new file mode 100644 index 000000000..3ffff1b89 --- /dev/null +++ b/.claude/rules/common/performance.md @@ -0,0 +1,55 @@ +# Performance Optimization + +## Model Selection Strategy + +**Haiku 4.5** (90% of Sonnet capability, 3x cost savings): +- Lightweight agents with frequent invocation +- Pair programming and code generation +- Worker agents in multi-agent systems + +**Sonnet 4.6** (Best coding model): +- Main development work +- Orchestrating multi-agent workflows +- Complex coding tasks + +**Opus 4.5** (Deepest reasoning): +- Complex architectural decisions +- Maximum reasoning requirements +- Research and analysis tasks + +## Context Window Management + +Avoid last 20% of context window for: +- Large-scale refactoring +- Feature implementation spanning multiple files +- Debugging complex interactions + +Lower context sensitivity tasks: +- Single-file edits +- Independent utility creation +- Documentation updates +- Simple bug fixes + +## Extended Thinking + Plan Mode + +Extended thinking is enabled by default, reserving up to 31,999 tokens for internal reasoning. + +Control extended thinking via: +- **Toggle**: Option+T (macOS) / Alt+T (Windows/Linux) +- **Config**: Set `alwaysThinkingEnabled` in `~/.claude/settings.json` +- **Budget cap**: `export MAX_THINKING_TOKENS=10000` +- **Verbose mode**: Ctrl+O to see thinking output + +For complex tasks requiring deep reasoning: +1. Ensure extended thinking is enabled (on by default) +2. Enable **Plan Mode** for structured approach +3. Use multiple critique rounds for thorough analysis +4. Use split role sub-agents for diverse perspectives + +## Build Troubleshooting + +If build fails: +1. Use **build-error-resolver** agent +2. Analyze error messages +3. Fix incrementally +4. Verify after each fix diff --git a/.claude/rules/common/security.md b/.claude/rules/common/security.md new file mode 100644 index 000000000..49624c03a --- /dev/null +++ b/.claude/rules/common/security.md @@ -0,0 +1,29 @@ +# Security Guidelines + +## Mandatory Security Checks + +Before ANY commit: +- [ ] No hardcoded secrets (API keys, passwords, tokens) +- [ ] All user inputs validated +- [ ] SQL injection prevention (parameterized queries) +- [ ] XSS prevention (sanitized HTML) +- [ ] CSRF protection enabled +- [ ] Authentication/authorization verified +- [ ] Rate limiting on all endpoints +- [ ] Error messages don't leak sensitive data + +## Secret Management + +- NEVER hardcode secrets in source code +- ALWAYS use environment variables or a secret manager +- Validate that required secrets are present at startup +- Rotate any secrets that may have been exposed + +## Security Response Protocol + +If security issue found: +1. STOP immediately +2. Use **security-reviewer** agent +3. Fix CRITICAL issues before continuing +4. Rotate any exposed secrets +5. Review entire codebase for similar issues diff --git a/.claude/rules/common/testing.md b/.claude/rules/common/testing.md new file mode 100644 index 000000000..416c1c28c --- /dev/null +++ b/.claude/rules/common/testing.md @@ -0,0 +1,57 @@ +# Testing Requirements + +## Minimum Test Coverage: 80% + +Test Types (ALL required): +1. **Unit Tests** - Individual functions, utilities, components +2. **Integration Tests** - API endpoints, database operations +3. **E2E Tests** - Critical user flows (framework chosen per language) + +## Test-Driven Development + +MANDATORY workflow: +1. Write test first (RED) +2. Run test - it should FAIL +3. Write minimal implementation (GREEN) +4. Run test - it should PASS +5. Refactor (IMPROVE) +6. Verify coverage (80%+) + +## Troubleshooting Test Failures + +1. Use **tdd-guide** agent +2. Check test isolation +3. Verify mocks are correct +4. Fix implementation, not tests (unless tests are wrong) + +## Agent Support + +- **tdd-guide** - Use PROACTIVELY for new features, enforces write-tests-first + +## Test Structure (AAA Pattern) + +Prefer Arrange-Act-Assert structure for tests: + +```typescript +test('calculates similarity correctly', () => { + // Arrange + const vector1 = [1, 0, 0] + const vector2 = [0, 1, 0] + + // Act + const similarity = calculateCosineSimilarity(vector1, vector2) + + // Assert + expect(similarity).toBe(0) +}) +``` + +### Test Naming + +Use descriptive names that explain the behavior under test: + +```typescript +test('returns empty array when no markets match query', () => {}) +test('throws error when API key is missing', () => {}) +test('falls back to substring search when Redis is unavailable', () => {}) +``` diff --git a/.claude/rules/cpp/coding-style.md b/.claude/rules/cpp/coding-style.md new file mode 100644 index 000000000..3550077d5 --- /dev/null +++ b/.claude/rules/cpp/coding-style.md @@ -0,0 +1,44 @@ +--- +paths: + - "**/*.cpp" + - "**/*.hpp" + - "**/*.cc" + - "**/*.hh" + - "**/*.cxx" + - "**/*.h" + - "**/CMakeLists.txt" +--- +# C++ Coding Style + +> This file extends [common/coding-style.md](../common/coding-style.md) with C++ specific content. + +## Modern C++ (C++17/20/23) + +- Prefer **modern C++ features** over C-style constructs +- Use `auto` when the type is obvious from context +- Use `constexpr` for compile-time constants +- Use structured bindings: `auto [key, value] = map_entry;` + +## Resource Management + +- **RAII everywhere** — no manual `new`/`delete` +- Use `std::unique_ptr` for exclusive ownership +- Use `std::shared_ptr` only when shared ownership is truly needed +- Use `std::make_unique` / `std::make_shared` over raw `new` + +## Naming Conventions + +- Types/Classes: `PascalCase` +- Functions/Methods: `snake_case` or `camelCase` (follow project convention) +- Constants: `kPascalCase` or `UPPER_SNAKE_CASE` +- Namespaces: `lowercase` +- Member variables: `snake_case_` (trailing underscore) or `m_` prefix + +## Formatting + +- Use **clang-format** — no style debates +- Run `clang-format -i ` before committing + +## Reference + +See skill: `cpp-coding-standards` for comprehensive C++ coding standards and guidelines. diff --git a/.claude/rules/cpp/hooks.md b/.claude/rules/cpp/hooks.md new file mode 100644 index 000000000..4ab677a03 --- /dev/null +++ b/.claude/rules/cpp/hooks.md @@ -0,0 +1,39 @@ +--- +paths: + - "**/*.cpp" + - "**/*.hpp" + - "**/*.cc" + - "**/*.hh" + - "**/*.cxx" + - "**/*.h" + - "**/CMakeLists.txt" +--- +# C++ Hooks + +> This file extends [common/hooks.md](../common/hooks.md) with C++ specific content. + +## Build Hooks + +Run these checks before committing C++ changes: + +```bash +# Format check +clang-format --dry-run --Werror src/*.cpp src/*.hpp + +# Static analysis +clang-tidy src/*.cpp -- -std=c++17 + +# Build +cmake --build build + +# Tests +ctest --test-dir build --output-on-failure +``` + +## Recommended CI Pipeline + +1. **clang-format** — formatting check +2. **clang-tidy** — static analysis +3. **cppcheck** — additional analysis +4. **cmake build** — compilation +5. **ctest** — test execution with sanitizers diff --git a/.claude/rules/cpp/patterns.md b/.claude/rules/cpp/patterns.md new file mode 100644 index 000000000..0c156e8d9 --- /dev/null +++ b/.claude/rules/cpp/patterns.md @@ -0,0 +1,51 @@ +--- +paths: + - "**/*.cpp" + - "**/*.hpp" + - "**/*.cc" + - "**/*.hh" + - "**/*.cxx" + - "**/*.h" + - "**/CMakeLists.txt" +--- +# C++ Patterns + +> This file extends [common/patterns.md](../common/patterns.md) with C++ specific content. + +## RAII (Resource Acquisition Is Initialization) + +Tie resource lifetime to object lifetime: + +```cpp +class FileHandle { +public: + explicit FileHandle(const std::string& path) : file_(std::fopen(path.c_str(), "r")) {} + ~FileHandle() { if (file_) std::fclose(file_); } + FileHandle(const FileHandle&) = delete; + FileHandle& operator=(const FileHandle&) = delete; +private: + std::FILE* file_; +}; +``` + +## Rule of Five/Zero + +- **Rule of Zero**: Prefer classes that need no custom destructor, copy/move constructors, or assignments +- **Rule of Five**: If you define any of destructor/copy-ctor/copy-assign/move-ctor/move-assign, define all five + +## Value Semantics + +- Pass small/trivial types by value +- Pass large types by `const&` +- Return by value (rely on RVO/NRVO) +- Use move semantics for sink parameters + +## Error Handling + +- Use exceptions for exceptional conditions +- Use `std::optional` for values that may not exist +- Use `std::expected` (C++23) or result types for expected failures + +## Reference + +See skill: `cpp-coding-standards` for comprehensive C++ patterns and anti-patterns. diff --git a/.claude/rules/cpp/security.md b/.claude/rules/cpp/security.md new file mode 100644 index 000000000..0ee9f5f09 --- /dev/null +++ b/.claude/rules/cpp/security.md @@ -0,0 +1,51 @@ +--- +paths: + - "**/*.cpp" + - "**/*.hpp" + - "**/*.cc" + - "**/*.hh" + - "**/*.cxx" + - "**/*.h" + - "**/CMakeLists.txt" +--- +# C++ Security + +> This file extends [common/security.md](../common/security.md) with C++ specific content. + +## Memory Safety + +- Never use raw `new`/`delete` — use smart pointers +- Never use C-style arrays — use `std::array` or `std::vector` +- Never use `malloc`/`free` — use C++ allocation +- Avoid `reinterpret_cast` unless absolutely necessary + +## Buffer Overflows + +- Use `std::string` over `char*` +- Use `.at()` for bounds-checked access when safety matters +- Never use `strcpy`, `strcat`, `sprintf` — use `std::string` or `fmt::format` + +## Undefined Behavior + +- Always initialize variables +- Avoid signed integer overflow +- Never dereference null or dangling pointers +- Use sanitizers in CI: + ```bash + cmake -DCMAKE_CXX_FLAGS="-fsanitize=address,undefined" .. + ``` + +## Static Analysis + +- Use **clang-tidy** for automated checks: + ```bash + clang-tidy --checks='*' src/*.cpp + ``` +- Use **cppcheck** for additional analysis: + ```bash + cppcheck --enable=all src/ + ``` + +## Reference + +See skill: `cpp-coding-standards` for detailed security guidelines. diff --git a/.claude/rules/cpp/testing.md b/.claude/rules/cpp/testing.md new file mode 100644 index 000000000..7c283551a --- /dev/null +++ b/.claude/rules/cpp/testing.md @@ -0,0 +1,44 @@ +--- +paths: + - "**/*.cpp" + - "**/*.hpp" + - "**/*.cc" + - "**/*.hh" + - "**/*.cxx" + - "**/*.h" + - "**/CMakeLists.txt" +--- +# C++ Testing + +> This file extends [common/testing.md](../common/testing.md) with C++ specific content. + +## Framework + +Use **GoogleTest** (gtest/gmock) with **CMake/CTest**. + +## Running Tests + +```bash +cmake --build build && ctest --test-dir build --output-on-failure +``` + +## Coverage + +```bash +cmake -DCMAKE_CXX_FLAGS="--coverage" -DCMAKE_EXE_LINKER_FLAGS="--coverage" .. +cmake --build . +ctest --output-on-failure +lcov --capture --directory . --output-file coverage.info +``` + +## Sanitizers + +Always run tests with sanitizers in CI: + +```bash +cmake -DCMAKE_CXX_FLAGS="-fsanitize=address,undefined" .. +``` + +## Reference + +See skill: `cpp-testing` for detailed C++ testing patterns, TDD workflow, and GoogleTest/GMock usage. diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 000000000..98fda44af --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,44 @@ +{ + "permissions": { + "allow": [ + "Bash(cmake --build build --target coro_rpc_urma_example)", + "Bash(cmake -B build -DBUILD_SHARED_LIBS=ON -DWITH_URMA=ON -DYLT_ENABLE_IBVERBS=OFF)", + "Bash(cmake -B build -DBUILD_SHARED_LIBS=ON -DYLT_ENABLE_URMA=ON -DYLT_HAVE_IBVERBS=OFF)", + "Bash(git add *)", + "Bash(git commit *)", + "Bash(git push *)", + "Bash(mkdir -p /home/lixu/Mooncake/extern/yalantinglibs/.claude/plans)", + "Bash(ECC_GATEGUARD=off find /home/lixu/Mooncake/extern/yalantinglibs -name \"urma_socket.hpp\" -o -name \"urma_example*\")", + "Bash(ECC_GATEGUARD=off cmake --build . --target urma_example)", + "Bash(ECC_GATEGUARD=off ninja -t targets)", + "Bash(ECC_GATEGUARD=off cmake --build . --target coro_rpc_urma_example)", + "Bash(ECC_GATEGUARD=off grep -r \"get_current_executor\" /home/lixu/Mooncake/extern/yalantinglibs/include/ylt/coro_io/*.hpp)", + "Bash(ECC_GATEGUARD=off grep -r \"get_current_executor\" /home/lixu/Mooncake/extern/yalantinglibs/include/ylt/coro_rpc/*.hpp)", + "Bash(ECC_GATEGUARD=off grep -r \"co_await\" /home/lixu/Mooncake/extern/yalantinglibs/include/ylt/coro_io/coro_io.hpp)", + "Bash(ECC_GATEGUARD=off grep -A5 \"async_connect\" /home/lixu/Mooncake/extern/yalantinglibs/include/ylt/coro_io/coro_io.hpp)", + "Bash(ECC_GATEGUARD=off grep -r \"Promise.*getFuture\" /home/lixu/Mooncake/extern/yalantinglibs/include/ylt/thirdparty/async_simple/)", + "Bash(ECC_GATEGUARD=off grep -A10 \"await_transform.*Future\" /home/lixu/Mooncake/extern/yalantinglibs/include/ylt/thirdparty/async_simple/coro/Lazy.h)", + "Bash(ECC_GATEGUARD=off grep -A5 \"class.*Future\" /home/lixu/Mooncake/extern/yalantinglibs/include/ylt/thirdparty/async_simple/Future.h)", + "Bash(ECC_GATEGUARD=off grep -r \"wait_promise\" /home/lixu/Mooncake/extern/yalantinglibs/include/ylt/coro_io/)", + "Bash(ECC_GATEGUARD=off grep -B5 -A10 \"wait_promise.*getFuture\" /home/lixu/Mooncake/extern/yalantinglibs/include/ylt/coro_io/ibverbs/ib_socket.hpp)", + "Bash(ECC_GATEGUARD=off grep -n \"prepare_accpet\\\\|get_remote_address\\\\|remote_address\" /home/lixu/Mooncake/extern/yalantinglibs/include/ylt/coro_io/ibverbs/ib_socket.hpp)", + "Bash(ECC_GATEGUARD=off grep -B5 -A20 \"executor_\" /home/lixu/Mooncake/extern/yalantinglibs/include/ylt/coro_io/ibverbs/ib_socket.hpp)", + "Bash(ECC_GATEGUARD=off grep -B5 -A20 \"async_accept\" /home/lixu/Mooncake/extern/yalantinglibs/include/ylt/coro_io/ibverbs/ib_socket.hpp)", + "Bash(ECC_GATEGUARD=off grep -A30 \"ib_socket_t::connect\" /home/lixu/Mooncake/extern/yalantinglibs/include/ylt/coro_io/ibverbs/ib_socket.hpp)", + "Bash(ECC_GATEGUARD=off grep -r \"async_connect\" /home/lixu/Mooncake/extern/yalantinglibs/include/ylt/coro_io/coro_io.hpp)", + "Bash(ECC_GATEGUARD=off grep -B2 -A20 \"waiting_write_over\" /home/lixu/Mooncake/extern/yalantinglibs/include/ylt/coro_io/ibverbs/ib_socket.hpp)", + "Bash(ECC_GATEGUARD=off grep -n \"urma_md5_first_header\" /home/lixu/Mooncake/extern/yalantinglibs/include/ylt/coro_rpc/impl/coro_rpc_server.hpp)", + "Bash(ECC_GATEGUARD=off grep -n \"md5_first_header\" /home/lixu/Mooncake/extern/yalantinglibs/include/ylt/coro_io/ibverbs/ib_socket.hpp)", + "Bash(ECC_GATEGUARD=off grep -B5 -A5 \"ib_md5_first_header\" /home/lixu/Mooncake/extern/yalantinglibs/include/ylt/coro_io/ibverbs/ib_socket.hpp)", + "Bash(ECC_GATEGUARD=off grep -B5 -A30 \"ib_socket_shared_state_t::init\" /home/lixu/Mooncake/extern/yalantinglibs/include/ylt/coro_io/ibverbs/ib_socket.hpp)", + "Bash(ECC_GATEGUARD=off grep -B3 -A10 \"get_remote_address\\\\|get_local_address\" /home/lixu/Mooncake/extern/yalantinglibs/include/ylt/coro_io/ibverbs/ib_socket.hpp)", + "Bash(ECC_GATEGUARD=off grep -B5 -A20 \"async_connect\" /home/lixu/Mooncake/extern/yalantinglibs/include/ylt/coro_io/coro_io.hpp)", + "Bash(ECC_GATEGUARD=off grep -B5 -A30 \"socket_config\" /home/lixu/Mooncake/extern/yalantinglibs/include/ylt/coro_rpc/coro_rpc_client.hpp)", + "Bash(ECC_GATEGUARD=off grep -r \"socket_config\" /home/lixu/Mooncake/extern/yalantinglibs/include/ylt/coro_rpc/)", + "Bash(ECC_GATEGUARD=off grep -B3 -A10 \"using.*config.*=\" /home/lixu/Mooncake/extern/yalantinglibs/include/ylt/coro_rpc/impl/coro_rpc_client.hpp)" + ], + "additionalDirectories": [ + "/home/lixu/Mooncake/extern/yalantinglibs/.claude" + ] + } +} diff --git a/.claude/skills/query-urma-docs/SKILL.md b/.claude/skills/query-urma-docs/SKILL.md new file mode 100644 index 000000000..3d3224e99 --- /dev/null +++ b/.claude/skills/query-urma-docs/SKILL.md @@ -0,0 +1,60 @@ +--- +name: query-urma-docs +description: Query and search URMA documentation. Use when asked to find URMA API references, look up functions, or search documentation for URMA concepts. +--- + +Query the URMA Chinese documentation at `doc/ch/urma/`. These are markdown docs covering the URMA API guide, user guide, and quickstart guide. + +All paths below are relative to this skill directory (`.claude/skills/query-urma-docs/`). + +## Prerequisite tools + +```bash +grep # built-in +rg --help 2>/dev/null || true # ripgrep, fall back to grep if missing +``` + +## How to query + +**Find a function/API by name:** +```bash +grep -r "urma_create_jfc" . +``` + +**Find a concept (e.g. Jetty, Segment, JFR):** +```bash +grep -r "Jetty" . +``` + +**List all doc files:** +```bash +ls *.md +``` + +**Read the full doc to answer complex questions:** +```bash +cat "URMA API Guide.ch.md" +``` + +## Content summary + +| File | What it covers | +|---|---| +| `URMA API Guide.ch.md` | All URMA API functions: init, device, context, JFC/JFCE/JFAE/JFS/JFR/Jetty, Segment, token, EID | +| `URMA User Guide.ch.md` | URMA architecture, concepts (UB, UBVA, Jetty, Segment), DFX, tools, 生态兼容 | +| `URMA QuickStart Guide.ch.md` | Build from source, install RPM, kernel module, quick verification | + +## Quick answers + +**What's a Jetty?** → `grep -m5 "Jetty" "URMA User Guide.ch.md"` + +**How to create a JFC?** → `grep -A10 "urma_create_jfc" "URMA API Guide.ch.md"` + +**How to compile URMA?** → see `URMA QuickStart Guide.ch.md` §1 + +## Notes + +- All docs are in Chinese with Chinese filenames +- No code or binary to run — this is a reference docs collection +- No build step needed +- Docs are embedded inside the skill directory for portability \ No newline at end of file diff --git a/.claude/skills/query-urma-docs/URMA API Guide.ch.md b/.claude/skills/query-urma-docs/URMA API Guide.ch.md new file mode 100644 index 000000000..6e7930016 --- /dev/null +++ b/.claude/skills/query-urma-docs/URMA API Guide.ch.md @@ -0,0 +1,16525 @@ +# 修订记录 + +1. 修订记录表 + +| 修订时间 | 修订章节 | 修订内容简介 | 修复问题单连接或问题背景 | 修订人员 | +| --- | --- | --- | --- | --- | +| 2026.2.12 | ALL | 文档基线 | | @qianguoxin、@jerry_lilijun、@wuyuyan_98、@pinchen2025、@autoreconf、@heyu_1014、@wdmmsyf | + +--- + +# 目 录 + +- [修订记录](#修订记录) + +- [1 使用约束与限制](#1-使用约束与限制) + - [1.1 版本配套约束](#11-版本配套约束) + +- [2 URMA用户态API](#2-urma用户态api) + - [2.1 编程示例](#21-编程示例) + - [2.1.1 管理面](#211-管理面) + - [2.1.2 控制面](#212-控制面) + - [2.1.3 数据面](#213-数据面) + - [2.1.3.1 单边read/write](#2131-单边readwrite) + - [2.1.3.2 双边send/recv](#2132-双边sendrecv) + - [2.1.4 开源示例](#214-开源示例) + - [2.2 管理面](#22-管理面) + - [2.2.1 初始化](#221-初始化) + - [2.2.1.1 urma_init](#2211-urma_init) + - [2.2.1.1.1 urma_init_attr_t](#22111-urma_init_attr_t) + - [2.2.1.1.2 urma_status_t](#22112-urma_status_t) + - [2.2.1.2 urma_uninit](#2212-urma_uninit) + - [2.2.2 设备及上下文](#222-设备及上下文) + - [2.2.2.1 device](#2221-device) + - [2.2.2.1.1 urma_get_device_list](#22211-urma_get_device_list) + - [2.2.2.1.2 urma_free_device_list](#22212-urma_free_device_list) + - [2.2.2.1.3 urma_get_device_by_name](#22213-urma_get_device_by_name) + - [2.2.2.1.4 urma_get_device_by_eid](#22214-urma_get_device_by_eid) + - [2.2.2.1.5 urma_query_device](#22215-urma_query_device) + - [2.2.2.2 eid](#2222-eid) + - [2.2.2.2.1 urma_get_eid_list](#22221-urma_get_eid_list) + - [2.2.2.2.2 urma_free_eid_list](#22222-urma_free_eid_list) + - [2.2.2.3 uasid](#2223-uasid) + - [2.2.2.3.1 urma_get_uasid](#22231-urma_get_uasid) + - [2.2.2.4 context](#2224-context) + - [2.2.2.4.1 urma_create_context](#22241-urma_create_context) + - [2.2.2.4.2 urma_delete_context](#22242-urma_delete_context) + - [2.2.2.4.3 urma_set_context_opt](#22243-urma_set_context_opt) + - [2.2.2.5 net addr](#2225-net-addr) + - [2.2.2.5.1 urma_get_net_addr_list](#22251-urma_get_net_addr_list) + - [2.2.2.5.2 urma_free_net_addr_list](#22252-urma_free_net_addr_list) + - [2.2.3 安全](#223-安全) + - [2.2.3.1 urma_alloc_token_id](#2231-urma_alloc_token_id) + - [2.2.3.1.1 urma_token_id_t](#22311-urma_token_id_t) + - [2.2.3.1.2 urma_token_id_flag_t](#22312-urma_token_id_flag_t) + - [2.2.3.2 urma_alloc_token_id_ex](#2232-urma_alloc_token_id_ex) + - [2.2.3.3 urma_free_token_id](#2233-urma_free_token_id) + - [2.3 控制面](#23-控制面) + - [2.3.1 Jetty相关](#231-jetty相关) + - [2.3.1.1 JFC](#2311-jfc) + - [2.3.1.1.1 urma_create_jfc](#23111-urma_create_jfc) + - [2.3.1.1.2 urma_modify_jfc](#23112-urma_modify_jfc) + - [2.3.1.1.3 urma_delete_jfc](#23113-urma_delete_jfc) + - [2.3.1.1.4 urma_delete_jfc_batch](#23114-urma_delete_jfc_batch) + - [2.3.1.2 JFCE](#2312-jfce) + - [2.3.1.2.1 urma_create_jfce](#23121-urma_create_jfce) + - [2.3.1.2.2 urma_delete_jfce](#23122-urma_delete_jfce) + - [2.3.1.3 JFAE](#2313-jfae) + - [2.3.1.3.1 urma_get_async_event](#23131-urma_get_async_event) + - [2.3.1.3.2 urma_ack_async_event](#23132-urma_ack_async_event) + - [2.3.1.4 JFS](#2314-jfs) + - [2.3.1.4.1 urma_create_jfs](#23141-urma_create_jfs) + - [2.3.1.4.2 urma_modify_jfs](#23142-urma_modify_jfs) + - [2.3.1.4.3 urma_query_jfs](#23143-urma_query_jfs) + - [2.3.1.4.4 urma_delete_jfs](#23144-urma_delete_jfs) + - [2.3.1.4.5 urma_delete_jfs_batch](#23145-urma_delete_jfs_batch) + - [2.3.1.4.6 urma_flush_jfs](#23146-urma_flush_jfs) + - [2.3.1.5 JFR](#2315-jfr) + - [2.3.1.5.1 urma_create_jfr](#23151-urma_create_jfr) + - [2.3.1.5.2 urma_modify_jfr](#23152-urma_modify_jfr) + - [2.3.1.5.3 urma_query_jfr](#23153-urma_query_jfr) + - [2.3.1.5.4 urma_delete_jfr](#23154-urma_delete_jfr) + - [2.3.1.5.5 urma_delete_jfr_batch](#23155-urma_delete_jfr_batch) + - [2.3.1.5.6 urma_import_jfr](#23156-urma_import_jfr) + - [2.3.1.5.7 urma_import_jfr_ex](#23157-urma_import_jfr_ex) + - [2.3.1.5.8 urma_unimport_jfr](#23158-urma_unimport_jfr) + - [2.3.1.6 Jetty](#2316-jetty) + - [2.3.1.6.1 urma_create_jetty](#23161-urma_create_jetty) + - [2.3.1.6.2 urma_modify_jetty](#23162-urma_modify_jetty) + - [2.3.1.6.3 urma_query_jetty](#23163-urma_query_jetty) + - [2.3.1.6.4 urma_delete_jetty](#23164-urma_delete_jetty) + - [2.3.1.6.5 urma_delete_jetty_batch](#23165-urma_delete_jetty_batch) + - [2.3.1.6.6 urma_import_jetty](#23166-urma_import_jetty) + - [2.3.1.6.7 urma_import_jetty_ex](#23167-urma_import_jetty_ex) + - [2.3.1.6.8 urma_unimport_jetty](#23168-urma_unimport_jetty) + - [2.3.1.6.9 urma_bind_jetty](#23169-urma_bind_jetty) + - [2.3.1.6.10 urma_bind_jetty_ex](#231610-urma_bind_jetty_ex) + - [2.3.1.6.11 urma_unbind_jetty](#231611-urma_unbind_jetty) + - [2.3.1.6.12 urma_flush_jetty](#231612-urma_flush_jetty) + - [2.3.1.6.13 urma_import_jetty_async](#231613-urma_import_jetty_async) + - [2.3.1.6.14 urma_unimport_jetty_async](#231614-urma_unimport_jetty_async) + - [2.3.1.6.15 urma_bind_jetty_async](#231615-urma_bind_jetty_async) + - [2.3.1.6.16 urma_unbind_jetty_async](#231616-urma_unbind_jetty_async) + - [2.3.1.6.17 urma_create_notifier](#231617-urma_create_notifier) + - [2.3.1.6.18 urma_delete_notifier](#231618-urma_delete_notifier) + - [2.3.1.6.19 urma_wait_notify](#231619-urma_wait_notify) + - [2.3.1.6.20 urma_ack_notify](#231620-urma_ack_notify) + - [2.3.1.7 Jetty Group](#2317-jetty-group) + - [2.3.1.7.1 urma_create_jetty_grp](#23171-urma_create_jetty_grp) + - [2.3.1.7.2 urma_delete_jetty_grp](#23172-urma_delete_jetty_grp) + - [2.3.2 Segment](#232-segment) + - [2.3.2.1 urma_register_seg](#2321-urma_register_seg) + - [2.3.2.1.1 urma_seg_cfg_t](#23211-urma_seg_cfg_t) + - [2.3.2.1.2 urma_reg_seg_flag_t](#23212-urma_reg_seg_flag_t) + - [2.3.2.1.3 urma_target_seg_t](#23213-urma_target_seg_t) + - [2.3.2.1.4 urma_seg_t](#23214-urma_seg_t) + - [2.3.2.1.5 urma_ubva_t](#23215-urma_ubva_t) + - [2.3.2.1.6 urma_seg_attr_t](#23216-urma_seg_attr_t) + - [2.3.2.1.7 urma_token_t](#23217-urma_token_t) + - [2.3.2.2 urma_unregister_seg](#2322-urma_unregister_seg) + - [2.3.2.3 urma_import_seg](#2323-urma_import_seg) + - [2.3.2.3.1 urma_import_seg_flag_t](#23231-urma_import_seg_flag_t) + - [2.3.2.4 urma_unimport_seg](#2324-urma_unimport_seg) + - [2.3.3 TP Channel](#233-tp-channel) + - [2.3.3.1 urma_get_tpn](#2331-urma_get_tpn) + - [2.3.3.2 urma_modify_tp](#2332-urma_modify_tp) + - [2.3.3.2.1 urma_tp_cfg_t](#23321-urma_tp_cfg_t) + - [2.3.3.2.2 urma_tp_cfg_flag_t](#23322-urma_tp_cfg_flag_t) + - [2.3.3.2.3 urma_tp_attr_t](#23323-urma_tp_attr_t) + - [2.3.3.2.4 urma_tp_mod_flag_t](#23324-urma_tp_mod_flag_t) + - [2.3.3.2.5 urma_tp_state_t](#23325-urma_tp_state_t) + - [2.3.3.2.6 urma_tp_attr_mask_t](#23326-urma_tp_attr_mask_t) + - [2.3.3.3 urma_get_tp_list](#2333-urma_get_tp_list) + - [2.3.3.3.1 urma_get_tp_cfg_t](#23331-urma_get_tp_cfg_t) + - [2.3.3.3.2 urma_get_tp_cfg_flag_t](#23332-urma_get_tp_cfg_flag_t) + - [2.3.3.3.3 urma_tp_info_t](#23333-urma_tp_info_t) + - [2.3.3.4 urma_get_tp_attr](#2334-urma_get_tp_attr) + - [2.3.3.4.1 urma_tp_attr_value_t](#23341-urma_tp_attr_value_t) + - [2.3.3.5 urma_set_tp_attr](#2335-urma_set_tp_attr) + - [2.4 数据面](#24-数据面) + - [2.4.1 post](#241-post) + - [2.4.1.1 urma_post_jfs_wr](#2411-urma_post_jfs_wr) + - [2.4.1.1.1 urma_jfs_wr_t](#24111-urma_jfs_wr_t) + - [2.4.1.1.2 urma_rw_wr_t](#24112-urma_rw_wr_t) + - [2.4.1.1.3 urma_send_wr_t](#24113-urma_send_wr_t) + - [2.4.1.1.4 urma_cas_wr_t](#24114-urma_cas_wr_t) + - [2.4.1.1.5 urma_faa_wr_t](#24115-urma_faa_wr_t) + - [2.4.1.1.6 urma_opcode_t](#24116-urma_opcode_t) + - [2.4.1.1.7 urma_jfs_wr_flag_t](#24117-urma_jfs_wr_flag_t) + - [2.4.1.1.8 urma_place_order_t](#24118-urma_place_order_t) + - [2.4.1.1.9 urma_sge_t](#24119-urma_sge_t) + - [2.4.1.1.10 urma_sg_t](#241110-urma_sg_t) + - [2.4.1.2 urma_post_jfr_wr](#2412-urma_post_jfr_wr) + - [2.4.1.2.1 urma_jfr_wr_t](#24121-urma_jfr_wr_t) + - [2.4.1.3 urma_post_jetty_send_wr](#2413-urma_post_jetty_send_wr) + - [2.4.1.4 urma_post_jetty_recv_wr](#2414-urma_post_jetty_recv_wr) + - [2.4.2 poll相关](#242-poll相关) + - [2.4.2.1 urma_poll_jfc](#2421-urma_poll_jfc) + - [2.4.2.1.1 urma_cr_t](#24211-urma_cr_t) + - [2.4.2.1.2 urma_cr_status_t](#24212-urma_cr_status_t) + - [2.4.2.1.3 urma_cr_opcode_t](#24213-urma_cr_opcode_t) + - [2.4.2.1.4 urma_cr_flag_t](#24214-urma_cr_flag_t) + - [2.4.2.1.5 urma_cr_token_t](#24215-urma_cr_token_t) + - [2.4.2.2 urma_rearm_jfc](#2422-urma_rearm_jfc) + - [2.4.2.3 urma_wait_jfc](#2423-urma_wait_jfc) + - [2.4.2.4 urma_ack_jfc](#2424-urma_ack_jfc) + - [2.4.3 read/write](#243-readwrite) + - [2.4.3.1 urma_write](#2431-urma_write) + - [2.4.3.2 urma_read](#2432-urma_read) + - [2.4.4 send/recv](#244-sendrecv) + - [2.4.4.1 urma_send](#2441-urma_send) + - [2.4.4.2 urma_recv](#2442-urma_recv) + - [2.5 其他](#25-其他) + - [2.5.1 扩展](#251-扩展) + - [2.5.1.1 urma_user_ctl](#2511-urma_user_ctl) + - [2.5.1.1.1 urma_user_ctl_in_t](#25111-urma_user_ctl_in_t) + - [2.5.1.1.2 urma_user_ctl_out_t](#25112-urma_user_ctl_out_t) + - [2.5.2 日志](#252-日志) + - [2.5.2.1 urma_register_log_func](#2521-urma_register_log_func) + - [2.5.2.1.1 urma_log_cb_t](#25211-urma_log_cb_t) + - [2.5.2.2 urma_unregister_log_func](#2522-urma_unregister_log_func) + - [2.5.2.3 urma_log_get_level](#2523-urma_log_get_level) + - [2.5.2.3.1 urma_vlog_level_t](#25231-urma_vlog_level_t) + - [2.5.2.4 urma_log_set_level](#2524-urma_log_set_level) + - [2.5.2.5 urma_log_get_thread_tag](#2525-urma_log_get_thread_tag) + - [2.5.2.6 urma_log_set_thread_tag](#2526-urma_log_set_thread_tag) + - [2.5.3 宏定义](#253-宏定义) + +- [3 URMA内核态API](#3-urma内核态api) + - [3.1 编程示例](#31-编程示例) + - [3.1.1 管理面](#311-管理面) + - [3.1.2 控制面](#312-控制面) + - [3.1.3 数据面](#313-数据面) + - [3.1.3.1 双边send/recv](#3131-双边sendrecv) + - [3.2 设备及上下文管理](#32-设备及上下文管理) + - [3.2.1 ubcore_register_device](#321-ubcore_register_device) + - [3.2.1.1 ubcore_device](#3211-ubcore_device) + - [3.2.1.2 ubcore_ops](#3212-ubcore_ops) + - [3.2.1.3 ubcore_device_cfg](#3213-ubcore_device_cfg) + - [3.2.1.4 ubcore_device_cfg_mask](#3214-ubcore_device_cfg_mask) + - [3.2.1.5 ubcore_rc_cfg](#3215-ubcore_rc_cfg) + - [3.2.1.6 ubcore_hash_table](#3216-ubcore_hash_table) + - [3.2.1.7 ubcore_ht_param](#3217-ubcore_ht_param) + - [3.2.1.8 ubcore_eid_table](#3218-ubcore_eid_table) + - [3.2.1.9 ubcore_eid_entry](#3219-ubcore_eid_entry) + - [3.2.1.10 ubcore_cg_device](#32110-ubcore_cg_device) + - [3.2.1.11 ubcore_sip_table](#32111-ubcore_sip_table) + - [3.2.1.12 ubcore_sip_entry](#32112-ubcore_sip_entry) + - [3.2.1.13 ubcore_logic_device](#32113-ubcore_logic_device) + - [3.2.1.14 ubcore_port_kobj](#32114-ubcore_port_kobj) + - [3.2.1.15 ubcore_vtp_bitmap](#32115-ubcore_vtp_bitmap) + - [3.2.2 ubcore_unregister_device](#322-ubcore_unregister_device) + - [3.2.3 ubcore_stop_requests](#323-ubcore_stop_requests) + - [3.2.4 ubcore_alloc_ucontext](#324-ubcore_alloc_ucontext) + - [3.2.4.1 ubcore_ucontext](#3241-ubcore_ucontext) + - [3.2.4.2 ubcore_udrv_priv](#3242-ubcore_udrv_priv) + - [3.2.5 ubcore_free_ucontext](#325-ubcore_free_ucontext) + - [3.2.6 ubcore_register_client](#326-ubcore_register_client) + - [3.2.6.1 ubcore_client](#3261-ubcore_client) + - [3.2.7 ubcore_unregister_client](#327-ubcore_unregister_client) + - [3.2.8 ubcore_set_client_ctx_data](#328-ubcore_set_client_ctx_data) + - [3.2.9 ubcore_get_client_ctx_data](#329-ubcore_get_client_ctx_data) + - [3.2.10 ubcore_get_eid_list](#3210-ubcore_get_eid_list) + - [3.2.10.1 ubcore_eid_info](#32101-ubcore_eid_info) + - [3.2.10.2 ubcore_eid](#32102-ubcore_eid) + - [3.2.11 ubcore_free_eid_list](#3211-ubcore_free_eid_list) + - [3.2.12 ubcore_query_device_attr](#3212-ubcore_query_device_attr) + - [3.2.12.1 ubcore_device_attr](#32121-ubcore_device_attr) + - [3.2.12.2 ubcore_pattern](#32122-ubcore_pattern) + - [3.2.12.3 ubcore_guid](#32123-ubcore_guid) + - [3.2.12.4 ubcore_device_cap](#32124-ubcore_device_cap) + - [3.2.12.5 ubcore_device_feat](#32125-ubcore_device_feat) + - [3.2.12.6 ubcore_atomic_feat](#32126-ubcore_atomic_feat) + - [3.2.12.7 ubcore_slice](#32127-ubcore_slice) + - [3.2.12.8 ubcore_congestion_ctrl_alg](#32128-ubcore_congestion_ctrl_alg) + - [3.2.12.9 ubcore_port_attr](#32129-ubcore_port_attr) + - [3.2.13 ubcore_query_device_status](#3213-ubcore_query_device_status) + - [3.2.13.1 ubcore_device_status](#32131-ubcore_device_status) + - [3.2.13.2 ubcore_port_status](#32132-ubcore_port_status) + - [3.2.13.3 ubcore_port_state](#32133-ubcore_port_state) + - [3.2.13.4 ubcore_speed](#32134-ubcore_speed) + - [3.2.13.5 ubcore_link_width](#32135-ubcore_link_width) + - [3.2.14 ubcore_cgroup_reg_dev](#3214-ubcore_cgroup_reg_dev) + - [3.2.15 ubcore_cgroup_unreg_dev](#3215-ubcore_cgroup_unreg_dev) + - [3.2.16 ubcore_cgroup_try_charge](#3216-ubcore_cgroup_try_charge) + - [3.2.16.1 struct ubcore_cg_object](#32161-struct-ubcore_cg_object) + - [3.2.16.2 enum ubcore_resource_type](#32162-enum-ubcore_resource_type) + - [3.2.17 ubcore_cgroup_uncharge](#3217-ubcore_cgroup_uncharge) + - [3.2.18 ubcore_get_mtu](#3218-ubcore_get_mtu) + - [3.2.18.1 ubcore_mtu](#32181-ubcore_mtu) + - [3.2.19 ubcore_recv_req](#3219-ubcore_recv_req) + - [3.2.19.1 ubcore_req_host](#32191-ubcore_req_host) + - [3.2.19.2 ubcore_req](#32192-ubcore_req) + - [3.2.19.3 ubcore_msg_opcode](#32193-ubcore_msg_opcode) + - [3.2.20 ubcore_recv_resp](#3220-ubcore_recv_resp) + - [3.2.20.1 ubcore_resp](#32201-ubcore_resp) + - [3.2.21 ubcore_get_device_by_eid](#3221-ubcore_get_device_by_eid) + - [3.2.21.1 ubcore_transport_type](#32211-ubcore_transport_type) + - [3.3 segment管理](#33-segment管理) + - [3.3.1 ubcore_alloc_token_id](#331-ubcore_alloc_token_id) + - [3.3.1.1 ubcore_token_id_flag](#3311-ubcore_token_id_flag) + - [3.3.1.2 ubcore_udata](#3312-ubcore_udata) + - [3.3.1.3 ubcore_token_id](#3313-ubcore_token_id) + - [3.3.2 ubcore_free_token_id](#332-ubcore_free_token_id) + - [3.3.3 ubcore_register_seg](#333-ubcore_register_seg) + - [3.3.3.1 ubcore_seg_cfg](#3331-ubcore_seg_cfg) + - [3.3.3.2 ubcore_token](#3332-ubcore_token) + - [3.3.3.3 ubcore_reg_seg_flag](#3333-ubcore_reg_seg_flag) + - [3.3.3.4 ubcore_target_seg](#3334-ubcore_target_seg) + - [3.3.3.5 ubcore_seg](#3335-ubcore_seg) + - [3.3.3.6 ubcore_ubva](#3336-ubcore_ubva) + - [3.3.3.7 ubcore_seg_attr](#3337-ubcore_seg_attr) + - [3.3.4 ubcore_unregister_seg](#334-ubcore_unregister_seg) + - [3.3.5 ubcore_import_seg](#335-ubcore_import_seg) + - [3.3.5.1 ubcore_target_seg_cfg](#3351-ubcore_target_seg_cfg) + - [3.3.5.2 ubcore_import_seg_flag](#3352-ubcore_import_seg_flag) + - [3.3.6 ubcore_unimport_seg](#336-ubcore_unimport_seg) + - [3.4 Jetty管理](#34-jetty管理) + - [3.4.1 JFC管理](#341-jfc管理) + - [3.4.1.1 ubcore_create_jfc](#3411-ubcore_create_jfc) + - [3.4.1.1.1 ubcore_jfc_cfg](#34111-ubcore_jfc_cfg) + - [3.4.1.1.2 ubcore_jfc_flag](#34112-ubcore_jfc_flag) + - [3.4.1.1.3 ubcore_comp_callback_t](#34113-ubcore_comp_callback_t) + - [3.4.1.1.4 ubcore_event_callback_t](#34114-ubcore_event_callback_t) + - [3.4.1.1.5 ubcore_jfc](#34115-ubcore_jfc) + - [3.4.1.2 ubcore_modify_jfc](#3412-ubcore_modify_jfc) + - [3.4.1.2.1 ubcore_jfc_attr](#34121-ubcore_jfc_attr) + - [3.4.1.2.2 ubcore_jfc_attr_mask](#34122-ubcore_jfc_attr_mask) + - [3.4.1.3 ubcore_delete_jfc](#3413-ubcore_delete_jfc) + - [3.4.1.4 ubcore_delete_jfc_batch](#3414-ubcore_delete_jfc_batch) + - [3.4.2 JFS管理](#342-jfs管理) + - [3.4.2.1 ubcore_create_jfs](#3421-ubcore_create_jfs) + - [3.4.2.1.1 ubcore_jfs_cfg](#34211-ubcore_jfs_cfg) + - [3.4.2.1.2 ubcore_jfs_flag](#34212-ubcore_jfs_flag) + - [3.4.2.1.3 ubcore_transport_mode](#34213-ubcore_transport_mode) + - [3.4.2.1.4 ubcore_jfs](#34214-ubcore_jfs) + - [3.4.2.1.5 ubcore_jetty_id](#34215-ubcore_jetty_id) + - [3.4.2.2 ubcore_modify_jfs](#3422-ubcore_modify_jfs) + - [3.4.2.2.1 ubcore_jfs_attr](#34221-ubcore_jfs_attr) + - [3.4.2.2.2 ubcore_jfs_attr_mask](#34222-ubcore_jfs_attr_mask) + - [3.4.2.2.3 ubcore_jetty_state](#34223-ubcore_jetty_state) + - [3.4.2.3 ubcore_query_jfs](#3423-ubcore_query_jfs) + - [3.4.2.4 ubcore_delete_jfs](#3424-ubcore_delete_jfs) + - [3.4.2.5 ubcore_delete_jfs_batch](#3425-ubcore_delete_jfs_batch) + - [3.4.2.6 ubcore_flush_jfs](#3426-ubcore_flush_jfs) + - [3.4.2.6.1 ubcore_cr](#34261-ubcore_cr) + - [3.4.2.6.2 ubcore_cr_status](#34262-ubcore_cr_status) + - [3.4.2.6.3 ubcore_cr_opcode](#34263-ubcore_cr_opcode) + - [3.4.2.6.4 ubcore_cr_flag](#34264-ubcore_cr_flag) + - [3.4.2.6.5 ubcore_cr_token](#34265-ubcore_cr_token) + - [3.4.3 JFR管理](#343-jfr管理) + - [3.4.3.1 ubcore_create_jfr](#3431-ubcore_create_jfr) + - [3.4.3.1.1 ubcore_jfr_cfg](#34311-ubcore_jfr_cfg) + - [3.4.3.1.2 ubcore_jfr_flag](#34312-ubcore_jfr_flag) + - [3.4.3.1.3 ubcore_jfr](#34313-ubcore_jfr) + - [3.4.3.2 ubcore_modify_jfr](#3432-ubcore_modify_jfr) + - [3.4.3.2.1 ubcore_jfr_attr](#34321-ubcore_jfr_attr) + - [3.4.3.2.2 ubcore_jfr_attr_mask](#34322-ubcore_jfr_attr_mask) + - [3.4.3.2.3 ubcore_jfr_state](#34323-ubcore_jfr_state) + - [3.4.3.3 ubcore_query_jfr](#3433-ubcore_query_jfr) + - [3.4.3.4 ubcore_delete_jfr](#3434-ubcore_delete_jfr) + - [3.4.3.5 ubcore_delete_jfr_batch](#3435-ubcore_delete_jfr_batch) + - [3.4.3.6 ubcore_import_jfr](#3436-ubcore_import_jfr) + - [3.4.3.6.1 ubcore_tjetty_cfg](#34361-ubcore_tjetty_cfg) + - [3.4.3.6.2 ubcore_import_jetty_flag](#34362-ubcore_import_jetty_flag) + - [3.4.3.6.3 ubcore_target_type](#34363-ubcore_target_type) + - [3.4.3.6.4 ubcore_jetty_grp_policy](#34364-ubcore_jetty_grp_policy) + - [3.4.3.6.5 ubcore_tjetty](#34365-ubcore_tjetty) + - [3.4.3.6.6 ubcore_tp](#34366-ubcore_tp) + - [3.4.3.6.7 ubcore_vtpn](#34367-ubcore_vtpn) + - [3.4.3.6.8 ubcore_vtp_state](#34368-ubcore_vtp_state) + - [3.4.3.7 ubcore_import_jfr_ex](#3437-ubcore_import_jfr_ex) + - [3.4.3.7.1 ubcore_active_tp_cfg](#34371-ubcore_active_tp_cfg) + - [3.4.3.7.2 ubcore_active_tp_attr](#34372-ubcore_active_tp_attr) + - [3.4.3.8 ubcore_unimport_jfr](#3438-ubcore_unimport_jfr) + - [3.4.4 Jetty管理](#344-jetty管理) + - [3.4.4.1 ubcore_create_jetty](#3441-ubcore_create_jetty) + - [3.4.4.1.1 ubcore_jetty_cfg](#34411-ubcore_jetty_cfg) + - [3.4.4.1.2 ubcore_jetty_flag](#34412-ubcore_jetty_flag) + - [3.4.4.1.3 ubcore_jetty](#34413-ubcore_jetty) + - [3.4.4.2 ubcore_modify_jetty](#3442-ubcore_modify_jetty) + - [3.4.4.2.1 ubcore_jetty_attr](#34421-ubcore_jetty_attr) + - [3.4.4.2.2 ubcore_jetty_attr_mask](#34422-ubcore_jetty_attr_mask) + - [3.4.4.3 ubcore_query_jetty](#3443-ubcore_query_jetty) + - [3.4.4.4 ubcore_delete_jetty](#3444-ubcore_delete_jetty) + - [3.4.4.5 ubcore_delete_jetty_batch](#3445-ubcore_delete_jetty_batch) + - [3.4.4.6 ubcore_flush_jetty](#3446-ubcore_flush_jetty) + - [3.4.4.7 ubcore_import_jetty](#3447-ubcore_import_jetty) + - [3.4.4.8 ubcore_import_jetty_ex](#3448-ubcore_import_jetty_ex) + - [3.4.4.9 ubcore_unimport_jetty](#3449-ubcore_unimport_jetty) + - [3.4.4.10 ubcore_bind_jetty](#34410-ubcore_bind_jetty) + - [3.4.4.11 ubcore_bind_jetty_ex](#34411-ubcore_bind_jetty_ex) + - [3.4.4.12 ubcore_unbind_jetty](#34412-ubcore_unbind_jetty) + - [3.4.4.13 ubcore_import_jetty_async](#34413-ubcore_import_jetty_async) + - [3.4.4.13.1 ubcore_import_cb](#344131-ubcore_import_cb) + - [3.4.4.14 ubcore_unimport_jetty_async](#34414-ubcore_unimport_jetty_async) + - [3.4.4.14.1 ubcore_unimport_cb](#344141-ubcore_unimport_cb) + - [3.4.4.15 ubcore_bind_jetty_async](#34415-ubcore_bind_jetty_async) + - [3.4.4.15.1 ubcore_bind_cb](#344151-ubcore_bind_cb) + - [3.4.4.16 ubcore_unbind_jetty_async](#34416-ubcore_unbind_jetty_async) + - [3.4.4.16.1 ubcore_unbind_cb](#344161-ubcore_unbind_cb) + - [3.4.5 Jetty Group管理](#345-jetty-group管理) + - [3.4.5.1 ubcore_create_jetty_grp](#3451-ubcore_create_jetty_grp) + - [3.4.5.1.1 ubcore_jetty_grp_flag](#34511-ubcore_jetty_grp_flag) + - [3.4.5.1.2 ubcore_jetty_grp_cfg](#34512-ubcore_jetty_grp_cfg) + - [3.4.5.1.3 ubcore_jetty_group](#34513-ubcore_jetty_group) + - [3.4.5.2 ubcore_delete_jetty_grp](#3452-ubcore_delete_jetty_grp) + - [3.5 异步事件](#35-异步事件) + - [3.5.1 ubcore_register_event_handler](#351-ubcore_register_event_handler) + - [3.5.1.1 ubcore_event_handler](#3511-ubcore_event_handler) + - [3.5.1.2 ubcore_event](#3512-ubcore_event) + - [3.5.1.3 ubcore_event_type](#3513-ubcore_event_type) + - [3.5.2 ubcore_unregister_event_handler](#352-ubcore_unregister_event_handler) + - [3.6 Post WR操作](#36-post-wr操作) + - [3.6.1 ubcore_post_jfs_wr](#361-ubcore_post_jfs_wr) + - [3.6.1.1 ubcore_jfs_wr](#3611-ubcore_jfs_wr) + - [3.6.1.2 ubcore_opcode](#3612-ubcore_opcode) + - [3.6.1.3 ubcore_jfs_wr_flag](#3613-ubcore_jfs_wr_flag) + - [3.6.1.4 ubcore_rw_wr](#3614-ubcore_rw_wr) + - [3.6.1.5 ubcore_sg](#3615-ubcore_sg) + - [3.6.1.6 ubcore_sge](#3616-ubcore_sge) + - [3.6.1.7 ubcore_send_wr](#3617-ubcore_send_wr) + - [3.6.1.8 ubcore_cas_wr](#3618-ubcore_cas_wr) + - [3.6.1.9 ubcore_faa_wr](#3619-ubcore_faa_wr) + - [3.6.2 ubcore_post_jfr_wr](#362-ubcore_post_jfr_wr) + - [3.6.2.1 ubcore_jfr_wr](#3621-ubcore_jfr_wr) + - [3.6.3 ubcore_post_jetty_send_wr](#363-ubcore_post_jetty_send_wr) + - [3.6.4 ubcore_post_jetty_recv_wr](#364-ubcore_post_jetty_recv_wr) + - [3.7 完成记录](#37-完成记录) + - [3.7.1 ubcore_poll_jfc](#371-ubcore_poll_jfc) + - [3.7.2 ubcore_rearm_jfc](#372-ubcore_rearm_jfc) + - [3.8 ubcore面向UVS接口](#38-ubcore面向uvs接口) + - [3.8.1 ubcore_set_port_netdev](#381-ubcore_set_port_netdev) + - [3.8.2 ubcore_unset_port_netdev](#382-ubcore_unset_port_netdev) + - [3.8.3 ubcore_put_port_netdev](#383-ubcore_put_port_netdev) + - [3.8.4 ubcore_add_ueid](#384-ubcore_add_ueid) + - [3.8.4.1 ubcore_ueid_cfg](#3841-ubcore_ueid_cfg) + - [3.8.5 ubcore_delete_ueid](#385-ubcore_delete_ueid) + - [3.8.6 ubcore_config_device](#386-ubcore_config_device) + - [3.8.7 ubcore_add_sip](#387-ubcore_add_sip) + - [3.8.7.1 ubcore_sip_info](#3871-ubcore_sip_info) + - [3.8.7.2 ubcore_net_addr](#3872-ubcore_net_addr) + - [3.8.7.3 ubcore_net_addr_type](#3873-ubcore_net_addr_type) + - [3.8.7.4 ubcore_net_addr_union](#3874-ubcore_net_addr_union) + - [3.8.8 ubcore_delete_sip](#388-ubcore_delete_sip) + - [3.9 DFX接口](#39-dfx接口) + - [3.9.1 ubcore_query_stats](#391-ubcore_query_stats) + - [3.9.1.1 ubcore_stats_key](#3911-ubcore_stats_key) + - [3.9.1.2 ubcore_stats_val](#3912-ubcore_stats_val) + - [3.9.2 ubcore_query_resource](#392-ubcore_query_resource) + - [3.9.2.1 ubcore_res_key](#3921-ubcore_res_key) + - [3.9.2.2 ubcore_res_val](#3922-ubcore_res_val) + - [3.10 驱动自定义接口](#310-驱动自定义接口) + - [3.10.1 ubcore_user_control](#3101-ubcore_user_control) + - [3.10.1.1 ubcore_user_ctl](#31011-ubcore_user_ctl) + - [3.10.1.2 ubcore_user_ctl_in](#31012-ubcore_user_ctl_in) + - [3.10.1.3 ubcore_user_ctl_out](#31013-ubcore_user_ctl_out) + - [3.11 异步事件分发接口](#311-异步事件分发接口) + - [3.11.1 ubcore_dispatch_async_event](#3111-ubcore_dispatch_async_event) + - [3.12 内存映射接口](#312-内存映射接口) + - [3.12.1 ubcore_umem_get](#3121-ubcore_umem_get) + - [3.12.1.1 ubcore_umem](#31211-ubcore_umem) + - [3.12.1.2 ubcore_umem_flag](#31212-ubcore_umem_flag) + - [3.12.2 ubcore_umem_release](#3122-ubcore_umem_release) + - [3.12.3 ubcore_umem_find_best_page_size](#3123-ubcore_umem_find_best_page_size) + - [3.13 其他API](#313-其他api) + - [3.13.1 ubcore_dispatch_mgmt_event](#3131-ubcore_dispatch_mgmt_event) + - [3.13.1.1 ubcore_mgmt_event](#31311-ubcore_mgmt_event) + - [3.13.1.2 ubcore_mgmt_event_type](#31312-ubcore_mgmt_event_type) + - [3.13.2 ubcore_get_tp_list](#3132-ubcore_get_tp_list) + - [3.13.2.1 ubcore_get_tp_cfg](#31321-ubcore_get_tp_cfg) + - [3.13.2.2 ubcore_get_tp_cfg_flag](#31322-ubcore_get_tp_cfg_flag) + - [3.13.2.3 ubcore_tp_info](#31323-ubcore_tp_info) + - [3.13.2.4 ubcore_tp_handle](#31324-ubcore_tp_handle) + - [3.13.3 ubcore_set_tp_attr](#3133-ubcore_set_tp_attr) + - [3.13.3.1 ubcore_tp_attr_value](#31331-ubcore_tp_attr_value) + - [3.13.4 ubcore_get_tp_attr](#3134-ubcore_get_tp_attr) + - [3.13.5 ubcore_exchange_tp_info](#3135-ubcore_exchange_tp_info) + +- [4 URMA用户态驱动接口](#4-urma用户态驱动接口) + +- [5 URMA内核态驱动接口](#5-urma内核态驱动接口) + - [5.1 内核态UB设备管理接口](#51-内核态ub设备管理接口) + - [5.1.1 UB设备注册接口](#511-ub设备注册接口) + - [5.1.1.1 ubcore_device](#5111-ubcore_device) + - [5.1.1.2 ubcore_eid_entry](#5112-ubcore_eid_entry) + - [5.1.1.3 ubcore_eid_table](#5113-ubcore_eid_table) + - [5.1.1.4 ubcore_sip_info](#5114-ubcore_sip_info) + - [5.1.1.5 ubcore_sip_entry](#5115-ubcore_sip_entry) + - [5.1.1.6 ubcore_sip_table](#5116-ubcore_sip_table) + - [5.1.1.7 ubcore_port_kobj](#5117-ubcore_port_kobj) + - [5.1.1.8 ubcore_eid_attr](#5118-ubcore_eid_attr) + - [5.1.1.9 ubcore_logic_device](#5119-ubcore_logic_device) + - [5.1.1.10 ubcore_vtp_bitmap](#51110-ubcore_vtp_bitmap) + - [5.1.1.11 ubcore_ops](#51111-ubcore_ops) + - [5.1.1.12 ubcore_transport_type](#51112-ubcore_transport_type) + - [5.1.2 UB设备解注册接口](#512-ub设备解注册接口) + - [5.1.3 内存映射接口](#513-内存映射接口) + - [5.1.3.1 ubcore_umem](#5131-ubcore_umem) + - [5.1.3.2 ubcore_umem_flag](#5132-ubcore_umem_flag) + - [5.1.4 内存反映射接口](#514-内存反映射接口) + - [5.1.5 查找内存最优页面大小接口](#515-查找内存最优页面大小接口) + - [5.1.6 获取MTU接口](#516-获取mtu接口) + - [5.1.7 获取ue接口](#517-获取ue接口) + - [5.1.7.1 查询ue_idx接口](#5171-查询ue_idx接口) + - [5.1.7.2 查询ue状态接口](#5172-查询ue状态接口) + - [5.1.8 发送和接口消息接口](#518-发送和接口消息接口) + - [5.1.8.1 发送请求ops接口](#5181-发送请求ops接口) + - [5.1.8.1.1 ubcore_req](#51811-ubcore_req) + - [5.1.8.1.2 ubcore_msg_opcode](#51812-ubcore_msg_opcode) + - [5.1.8.2 发送响应ops接口](#5182-发送响应ops接口) + - [5.1.8.2.1 ubcore_resp_host](#51821-ubcore_resp_host) + - [5.1.8.2.2 ubcore_resp](#51822-ubcore_resp) + - [5.1.8.3 接收请求接口](#5183-接收请求接口) + - [5.1.8.3.1 ubcore_req_host](#51831-ubcore_req_host) + - [5.1.8.4 接收响应接口](#5184-接收响应接口) + - [5.1.8.5 热迁移请求和响应](#5185-热迁移请求和响应) + - [5.1.8.5.1 ubcore_function_mig_req](#51851-ubcore_function_mig_req) + - [5.1.8.5.2 ubcore_mig_resp_status](#51852-ubcore_mig_resp_status) + - [5.1.8.5.3 ubcore_function_mig_resp](#51853-ubcore_function_mig_resp) + - [5.1.9 设备属性查询ops接口](#519-设备属性查询ops接口) + - [5.1.10 设备状态查询ops接口](#5110-设备状态查询ops接口) + - [5.1.11 设备属性配置ops接口](#5111-设备属性配置ops接口) + - [5.1.11.1 ubcore_device_cfg](#51111-ubcore_device_cfg) + - [5.1.11.2 ubcore_device_cfg_mask](#51112-ubcore_device_cfg_mask) + - [5.1.11.3 ubcore_rc_cfg](#51113-ubcore_rc_cfg) + - [5.1.11.4 ubcore_pattern](#51114-ubcore_pattern) + - [5.1.12 设备绑定ops接口](#5112-设备绑定ops接口) + - [5.1.13 设备解除绑定ops接口](#5113-设备解除绑定ops接口) + - [5.1.14 设备添加端口ops接口](#5114-设备添加端口ops接口) + - [5.1.15 配置端口和netdev映射](#5115-配置端口和netdev映射) + - [5.1.16 解除端口和netdev映射](#5116-解除端口和netdev映射) + - [5.2 内核态ID配置、地址配置ops接口](#52-内核态id配置地址配置ops接口) + - [5.2.1 配置网络地址ops接口](#521-配置网络地址ops接口) + - [5.2.1.1 ubcore_net_addr](#5211-ubcore_net_addr) + - [5.2.1.2 ubcore_net_addr_type](#5212-ubcore_net_addr_type) + - [5.2.2 删除网络地址ops接口](#522-删除网络地址ops接口) + - [5.2.3 配置UEID ops接口](#523-配置ueid-ops接口) + - [5.2.3.1 ubcore_ueid_cfg](#5231-ubcore_ueid_cfg) + - [5.2.4 删除UEID ops接口](#524-删除ueid-ops接口) + - [5.2.5 配置Funtion热迁移状态ops接口](#525-配置funtion热迁移状态ops接口) + - [5.2.5.1 ubcore_mig_state](#5251-ubcore_mig_state) + - [5.3 内核态context管理ops接口](#53-内核态context管理ops接口) + - [5.3.1 context创建ops接口](#531-context创建ops接口) + - [5.3.1.1 ubcore_ucontext](#5311-ubcore_ucontext) + - [5.3.1.2 ubcore_udrv_priv](#5312-ubcore_udrv_priv) + - [5.3.1.3 ubcore_udata](#5313-ubcore_udata) + - [5.3.2 context销毁ops接口](#532-context销毁ops接口) + - [5.4 内核态mmap接口](#54-内核态mmap接口) + - [5.4.1 mmap ops接口](#541-mmap-ops接口) + - [5.5 内核态资源管理ops接口](#55-内核态资源管理ops接口) + - [5.5.1 TP协商和配置管理](#551-tp协商和配置管理) + - [5.5.1.1 TPG创建ops接口](#5511-tpg创建ops接口) + - [5.5.1.1.1 ubcore_tpg_cfg](#55111-ubcore_tpg_cfg) + - [5.5.1.1.2 ubcore_tpg_ext](#55112-ubcore_tpg_ext) + - [5.5.1.1.3 ubcore_tpg](#55113-ubcore_tpg) + - [5.5.1.2 TPG销毁ops接口](#5512-tpg销毁ops接口) + - [5.5.1.3 TP创建和使用流程](#5513-tp创建和使用流程) + - [5.5.1.4 TP参数协商](#5514-tp参数协商) + - [5.5.1.5 TP创建ops接口](#5515-tp创建ops接口) + - [5.5.1.5.1 ubcore_tp_cfg](#55151-ubcore_tp_cfg) + - [5.5.1.5.2 ubcore_tp_cfg_flag](#55152-ubcore_tp_cfg_flag) + - [5.5.1.5.3 ubcore_transport_mode](#55153-ubcore_transport_mode) + - [5.5.1.5.4 ubcore_tp_state](#55154-ubcore_tp_state) + - [5.5.1.5.5 ubcore_tp_ext](#55155-ubcore_tp_ext) + - [5.5.1.5.6 ubcore_tp_flag](#55156-ubcore_tp_flag) + - [5.5.1.5.7 ubcore_tp_cc_alg](#55157-ubcore_tp_cc_alg) + - [5.5.1.5.8 ubcore_tp](#55158-ubcore_tp) + - [5.5.1.6 TP修改ops接口](#5516-tp修改ops接口) + - [5.5.1.6.1 ubcore_tp_attr](#55161-ubcore_tp_attr) + - [5.5.1.6.2 ubcore_tp_attr_mask](#55162-ubcore_tp_attr_mask) + - [5.5.1.6.3 ubcore_tp_mod_flag](#55163-ubcore_tp_mod_flag) + - [5.5.1.7 TP销毁ops接口](#5517-tp销毁ops接口) + - [5.5.1.8 多TP创建ops接口](#5518-多tp创建ops接口) + - [5.5.1.9 多TP销毁ops接口](#5519-多tp销毁ops接口) + - [5.5.1.10 多TP修改ops接口](#55110-多tp修改ops接口) + - [5.5.1.11 查询拥塞控制算法模板ops接口](#55111-查询拥塞控制算法模板ops接口) + - [5.5.1.11.1 ubcore_cc_entry](#551111-ubcore_cc_entry) + - [5.5.1.12 VTPN分配ops接口](#55112-vtpn分配ops接口) + - [5.5.1.12.1 ubcore_vtpn](#551121-ubcore_vtpn) + - [5.5.1.13 VTPN释放ops接口](#55113-vtpn释放ops接口) + - [5.5.1.14 VTP创建ops接口](#55114-vtp创建ops接口) + - [5.5.1.14.1 ubcore_vtp_cfg](#551141-ubcore_vtp_cfg) + - [5.5.1.14.2 ubcore_vtp_cfg_flag](#551142-ubcore_vtp_cfg_flag) + - [5.5.1.14.3 ubcore_vtp](#551143-ubcore_vtp) + - [5.5.1.15 VTP销毁ops接口](#55115-vtp销毁ops接口) + - [5.5.1.16 VTP修改ops接口](#55116-vtp修改ops接口) + - [5.5.1.16.1 ubcore_vtp_attr](#551161-ubcore_vtp_attr) + - [5.5.1.16.2 ubcore_vtp_attr_mask](#551162-ubcore_vtp_attr_mask) + - [5.5.1.17 UTP创建ops接口](#55117-utp创建ops接口) + - [5.5.1.17.1 ubcore_utp_cfg](#551171-ubcore_utp_cfg) + - [5.5.1.17.2 ubcore_utp_cfg_flag](#551172-ubcore_utp_cfg_flag) + - [5.5.1.17.3 ubcore_utp](#551173-ubcore_utp) + - [5.5.1.18 UTP销毁ops接口](#55118-utp销毁ops接口) + - [5.5.1.19 CTP创建ops接口](#55119-ctp创建ops接口) + - [5.5.1.19.1 ubcore_ctp_cfg](#551191-ubcore_ctp_cfg) + - [5.5.1.19.2 ubcore_ctp](#551192-ubcore_ctp) + - [5.5.1.20 CTP销毁ops接口](#55120-ctp销毁ops接口) + - [5.5.2 JFC管理接口](#552-jfc管理接口) + - [5.5.2.1 JFC创建ops接口](#5521-jfc创建ops接口) + - [5.5.2.2 JFC修改ops接口](#5522-jfc修改ops接口) + - [5.5.2.3 JFC销毁ops接口](#5523-jfc销毁ops接口) + - [5.5.2.4 JFC rearm ops接口](#5524-jfc-rearm-ops接口) + - [5.5.3 JFS管理接口](#553-jfs管理接口) + - [5.5.3.1 JFS创建ops接口](#5531-jfs创建ops接口) + - [5.5.3.2 JFS修改ops接口](#5532-jfs修改ops接口) + - [5.5.3.3 JFS查询ops接口](#5533-jfs查询ops接口) + - [5.5.3.4 JFS flush ops接口](#5534-jfs-flush-ops接口) + - [5.5.3.5 JFS销毁ops接口](#5535-jfs销毁ops接口) + - [5.5.4 JFR管理接口](#554-jfr管理接口) + - [5.5.4.1 JFR创建ops接口](#5541-jfr创建ops接口) + - [5.5.4.2 JFR修改ops接口](#5542-jfr修改ops接口) + - [5.5.4.3 JFR查询ops接口](#5543-jfr查询ops接口) + - [5.5.4.4 JFR销毁ops接口](#5544-jfr销毁ops接口) + - [5.5.4.5 JFR导入ops接口](#5545-jfr导入ops接口) + - [5.5.4.6 JFR反导入ops接口](#5546-jfr反导入ops接口) + - [5.5.5 Jetty管理接口](#555-jetty管理接口) + - [5.5.5.1 Jetty创建ops接口](#5551-jetty创建ops接口) + - [5.5.5.2 Jetty修改ops接口](#5552-jetty修改ops接口) + - [5.5.5.3 Jetty查询ops接口](#5553-jetty查询ops接口) + - [5.5.5.4 Jetty销毁ops接口](#5554-jetty销毁ops接口) + - [5.5.5.5 Jetty导入ops接口](#5555-jetty导入ops接口) + - [5.5.5.6 Jetty反导入ops接口](#5556-jetty反导入ops接口) + - [5.5.5.7 Jetty bind ops接口](#5557-jetty-bind-ops接口) + - [5.5.5.8 Jetty unbind ops接口](#5558-jetty-unbind-ops接口) + - [5.5.5.9 Jetty flush ops接口](#5559-jetty-flush-ops接口) + - [5.5.6 Jetty group管理接口](#556-jetty-group管理接口) + - [5.5.6.1 Jetty group创建ops接口](#5561-jetty-group创建ops接口) + - [5.5.6.2 Jetty group销毁ops接口](#5562-jetty-group销毁ops接口) + - [5.5.7 segment和token管理接口](#557-segment和token管理接口) + - [5.5.7.1 token_id分配ops接口](#5571-token_id分配ops接口) + - [5.5.7.2 token_id释放ops接口](#5572-token_id释放ops接口) + - [5.5.7.3 segment注册ops接口](#5573-segment注册ops接口) + - [5.5.7.4 segment反注册ops接口](#5574-segment反注册ops接口) + - [5.5.7.5 segment导入ops接口](#5575-segment导入ops接口) + - [5.5.7.6 segment反导入ops接口](#5576-segment反导入ops接口) + - [5.5.8 dscp-vl映射管理接口](#558-dscp-vl映射管理接口) + - [5.5.8.1 dscp-vl映射配置接口](#5581-dscp-vl映射配置接口) + - [5.5.8.2 dscp-vl映射查询接口](#5582-dscp-vl映射查询接口) + - [5.5.9 其他ops接口](#559-其他ops接口) + - [5.5.9.1 驱动自定义控制user_ctl ops接口](#5591-驱动自定义控制user_ctl-ops接口) + - [5.5.10 内核态异常事件上报接口](#5510-内核态异常事件上报接口) + - [5.5.10.1 Jetty异步事件回调接口](#55101-jetty异步事件回调接口) + - [5.5.10.2 异步事件分发接口](#55102-异步事件分发接口) + - [5.5.11 内核态状态查询和DFX 接口](#5511-内核态状态查询和dfx-接口) + - [5.5.11.1 统计查询ops接口](#55111-统计查询ops接口) + - [5.5.11.2 资源查询ops接口](#55112-资源查询ops接口) + - [5.5.12 内核态数据面接口](#5512-内核态数据面接口) + - [5.5.12.1 JFS发送WR ops接口](#55121-jfs发送wr-ops接口) + - [5.5.12.2 JFR接收WR ops接口](#55122-jfr接收wr-ops接口) + - [5.5.12.3 Jetty发送WR ops接口](#55123-jetty发送wr-ops接口) + - [5.5.12.4 Jetty接收WR ops接口](#55124-jetty接收wr-ops接口) + - [5.5.12.5 rearm JFC ops接口](#55125-rearm-jfc-ops接口) + - [5.5.12.6 轮询 JFC ops接口](#55126-轮询-jfc-ops接口) + +- [6 UVS编程接口](#6-uvs编程接口) + - [6.1 uvs_set_topo_info](#61-uvs_set_topo_info) + - [6.2 编程示例](#62-编程示例) + +# 1 使用约束与限制 + +![](figures/urma_warning.png) + +1、数据面参数合法性由API调用者保证，URMA只校验入参的一级指针是否为空；指针对应的类型的成员变量为指针的，不执行空指针校验。 + +2、管理面参数URMA会校验由API调用者构造的参数指针（包含用户构造的二级指针），不会校验由URMA（包含驱动）创建的结构体的内部指针。 + +3、管理面参数指针校验针对URMA和驱动来说遵循谁使用谁校验的基本原则，同时不允许对API调用者授权范围外的环境产生恶意影响。 + +![](figures/urma_info.png) + +urma框架的API不支持任意并发调用，如使用jetty对象与销毁jetty对应并发等会导致不可预期的异常，需要用户保证调用逻辑正确。 + +这些对象包括urma_context、jetty、segment、jfc、jfr、jfs等。 + +## 1.1 版本配套约束 + +--- +# 2 URMA用户态API + +## 2.1 编程示例 + +本节示例主要基于URMA_TM_RM传输模式的jetty，具体配置（如Jetty depth等）需由用户确定。用户也可自行尝试其他用法（如其他传输模式）。 + +图1 基本流程 + +![](figures/urma-api-example-01.png) + +下面分管理面、控制面、数据面分别展示参数填写、API使用。反初始化和初始化相关操作对称，且不涉及复杂参数填写，故不再赘述。 + +### 2.1.1 管理面 + +- Initiator和Target分别调用[3.2.1.1](#2211-urma_init) [urma_init](#2211-urma_init)初始化资源。 + +```c +urma_init_attr_t init_attr = { + .token = 0, + .uasid = 0 +}; +urma_init(&init_attr); +``` + +- Initiator和Target分别获取device和eid（以使用一个device，一个eid为例），查询device属性（规格等）。 + +```c +int dev_num = 1; +urma_device_t **device_list = urma_get_device_list(&dev_num); +// 用户从设备列表中选取设备urma_dev +urma_device_attr_t dev_attr = {0}; +urma_query_device(urma_dev, &dev_attr); +uint32_t eid_cnt = 1; +urma_eid_info_t *eid_list = urma_get_eid_list(urma_dev, &eid_cnt); +// 用户从eid列表中选取eid对应的eid_index，下面以eid_list[0]为例 +uint32_t eid_index = eid_list[0].eid_index; +``` + +- Initiator和Target分别创建 urma_ctx上下文。 + +urma_context_t *urma_ctx = [3.2.2.4.1](#22241-urma_create_context) [urma_create_context](#22241-urma_create_context)(urma_dev, eid_index); + +### 2.1.2 控制面 + +- Initiator和Target分别创建jfc。 + +```c +urma_jfc_cfg_t jfc_cfg = { + .depth = 64, + .flag = {.value = 0}, + .ceqn = 0, + .jfce = nullptr, + .user_ctx = 0, +}; +urma_jfc_t *jfc = urma_create_jfc(urma_ctx, &jfc_cfg); +``` + +- Initiator和Target分别创建jfr。 + +```c +static urma_token_t test_token = { + .token = 0xABCDEF, // 由用户确定 +}; +urma_jfr_cfg_t jfr_cfg = { + .depth = 64, + .flag.bs.tag_matching = URMA_NO_TAG_MATCHING, + .flag.bs.order_type = 0, + .trans_mode = URMA_TM_RM, + .min_rnr_timer = URMA_TYPICAL_MIN_RNR_TIMER, + .jfc = jfc, + .token_value = test_token, + .id = 0, + .max_sge = 1 +}; +urma_jfr_t *jfr = urma_create_jfr(urma_ctx, &jfr_cfg); +``` + +- Initiator和Target分别创建jetty。 + +```c +urma_jfs_cfg_t jfs_cfg = { + .depth = 256, + .flag.bs.order_type = 0, + .flag.bs.multi_path = 0, + .trans_mode = URMA_TM_RM, + .priority = URMA_MAX_PRIORITY, /* Highest priority */ + .max_sge = 1, + .max_inline_data = 0, + .rnr_retry = URMA_TYPICAL_RNR_RETRY, + .err_timeout = URMA_TYPICAL_ERR_TIMEOUT, + .jfc = jfc, + .user_ctx = (uint64_t)NULL +}; +urma_jetty_cfg_t jetty_cfg = { + .flag.bs.share_jfr = 1, + .jfs_cfg = jfs_cfg, + .shared.jfr = jfr +}; +urma_jetty_t *jetty = urma_create_jetty(urma_ctx, &jetty_cfg); +``` + +- Initiator和Target分配各自的数据buffer，并将该数据buffer注册为segment。 + +```c +// 以分配4KB对齐的1GB buffer为例 +#define PAGE_SIZE (0x1 << PAGE_SHIFT) // 4KB +#define MEM_SIZE 0x40000000 // 1GB +void *va = memalign(PAGE_SIZE, MEM_SIZE); +(void)memset(va, 0, MEM_SIZE); +urma_reg_seg_flag_t flag = { + .bs.token_policy = URMA_TOKEN_NONE, + .bs.cacheable = URMA_NON_CACHEABLE, + .bs.access = URMA_ACCESS_READ | URMA_ACCESS_WRITE | URMA_ACCESS_ATOMIC, + .bs.token_id_valid = 0, + .bs.reserved = 0 +}; +urma_seg_cfg_t seg_cfg = { + .va = (uint64_t)va, + .len = MEM_SIZE, + .token_id = NULL, + .token_value = test_token, + .flag = flag, + .user_ctx = (uintptr_t)NULL, + .iova = 0 +}; +urma_target_seg_t *local_tseg = urma_register_seg(urma_ctx, &seg_cfg); +``` + +- Initiator和Target交换jetty和segment信息，可以通过带外socket，或带内公知Jetty通道。 + +```c +// 获取对端info +urma_seg_t remote_seg = { + .ubva.eid = info-\>eid; + .ubva.uasid = info-\>uasid; + .ubva.va = info-\>seg_va; + .len = info-\>seg_len; + .attr.value = info-\>seg_flag; + .token_id = info-\>seg_token_id; +}; +urma_jetty_id_t remote_jetty_id = info-\>jetty_id; +``` + +- Initiator和Target导入对端jetty和segment信息。 + +```c +urma_rjetty_t remote_jetty = { + .jetty_id = remote_jetty_id, + .trans_mode = URMA_TM_RM, + .type = URMA_JETTY, + .tp_type = URMA_RTP, + .flag.bs.order_type = 0, + .flag.bs.share_tp = 0 +}; +urma_target_jetty_t *t_jetty = urma_import_jetty(urma_ctx, &remote_jetty, &test_token); +urma_import_seg_flag_t flag = { + .bs.cacheable = URMA_NON_CACHEABLE, + .bs.access = URMA_ACCESS_READ | URMA_ACCESS_WRITE | URMA_ACCESS_ATOMIC, + .bs.mapping = URMA_SEG_NOMAP, + .bs.reserved = 0 +}; +urma_target_seg_t *import_tseg = urma_import_seg(urma_ctx, &remote_seg, &test_token, 0, flag); +``` + +### 2.1.3 数据面 + +数据面主要分单边read/write、双边send/recv。单边指对端处理器可以不参与，双边则是对端必须参与。 + +#### 2.1.3.1 单边read/write + +read、write所需准备的参数、API使用方式差不多，差别在wr src、dst相反，opcode不同。 + +```c +// read/write共同参数准备 +urma_sge_t src_sge = { + .addr = (uint64_t)va, + .len = MSG_SIZE, + .tseg = local_tseg +}; +urma_sge_t dst_sge = { + .addr = remote_seg.ubva.va, + .len = MSG_SIZE, + .tseg = import_tseg +}; +urma_sg_t src_sg = { + .sge = &src_sge, + .num_sge = 1 +}; +urma_sg_t dst_sg = { + .sge = &dst_sge, + .num_sge = 1 +}; +/* 先示例write */ +// write是把本端src_sg的数据写到对端dst_sg +urma_rw_wr_t rw = { + .src = src_sg, + .dst = dst_sg +}; +urma_jfs_wr_t wr = { + .opcode = URMA_OPC_WRITE, + .flag.bs.complete_enable = 1, + .flag.bs.inline_flag = 0, + .tjetty = t_jetty, + .rw = rw, + .next = NULL +}; +urma_jfs_wr_t *bad_wr = NULL; +urma_post_jetty_send_wr(jetty, &wr, &bad_wr); +urma_cr_t cr = {0}; +urma_poll_jfc(jfc, 1, &cr); +/* read示例 */ +// read是把对端dst_sg数据读到本端src_sg +rw.src = dst_sg; +rw.dst = src_sg; +wr.rw = rw; +wr.opcode = URMA_OPC_READ; +urma_post_jetty_send_wr(ctx-\>jetty, &wr, &bad_wr); +urma_poll_jfc(jfc, 1, &cr); +``` + +poll操作说明：从上面可以看出，post、poll是异步通信，post下发通信任务wr（work request）触发通信，poll尝试获取完成信息cr（completion record）。post后立马poll，通信可能尚未完成，poll的返回值\--cr cnt可能为0，表示暂无cr。因此，需要一些手段在通信完成时再去获取cr，这些手段分为轮询、中断。轮询指循环调poll，直至poll到成功/失败的cr。注意轮询不建议写成死循环，建议在poll间插入sleep，同时设置最大轮询次数。中断指利用硬件中断通知通信完成，相比轮询用起来更复杂，但能更大发挥异步特点。 + +```c +/* 轮询示例 */ +for (int i = 0; i < MAX_POLL_JFC_CNT; i++) { + cnt = urma_poll_jfc(jfc, 1, &cr); + if (cnt < 0) { + fprintf(stderr, "Failed to poll jfc, return_value of urma_poll_jfc is %d\n", cnt); + return -1; + } else if (cnt \> 0) { + if (cr.status == URMA_CR_SUCCESS) { + return 0; + } else { + fprintf(stderr, "Failed to poll jfc, cr_status:%d\n", cr.status); + return -1; + } + } + usleep(SLEEP_TIME); +} +/* 中断示例 */ +cnt = urma_wait_jfc(jfce, 1, TIMEOUT, &ev_jfc); +if (cnt < 0 || (cnt == 1 && jfc != ev_jfc)) { + fprintf(stderr, "Failed to wait jfc\n"); + return -1; +} +cnt = urma_poll_jfc(jfc, 1, &cr); +if (cnt <= 0 || cr.status != URMA_CR_SUCCESS) { + return -1; +} +uint32_t ack_cnt = 1; +urma_ack_jfc((urma_jfc_t **)&ev_jfc, &ack_cnt, 1); +if (urma_rearm_jfc(jfc, false) != URMA_SUCCESS) { + return -1; +} +``` + +#### 2.1.3.2 双边send/recv + +双边操作也是把数据从本端seg发到对端seg，它和单边write的差异主要在于post。双边操作不仅需要本端调[3.4.1.3](#2413-urma_post_jetty_send_wr) [urma_post_jetty_send_wr](#2413-urma_post_jetty_send_wr)下发发送任务，还需要对端调[3.4.1.4](#2414-urma_post_jetty_recv_wr) [urma_post_jetty_recv_wr](#2414-urma_post_jetty_recv_wr)准备接收。图示中post recv先于post send调用，这并非强制要求，本端的报文到对端时若发现recv wr还未下发，会暂时缓存在对端buffer，但为了保证通信性能，并且避免耗尽对端buffer，建议对端预先post一批recv wr，并且每次消耗后要及时补充。至于poll操作，双边操作两端都post了，相应两端都可以poll，poll的具体使用请参考[3.1.3.1](#2131-单边readwrite) [单边read/write](#2131-单边readwrite)，这里不再赘述。 + +Target post recv示例： + +```c +// 预先post一批recv wr +uint64_t offset = MSG_SIZE; +for (int i = 0; i < RECV_BATCH_CNT; i++) { + if (offset + MSG_SIZE \> MEM_SIZE) { + return NULL; + } + src_sge.addr = (uint64_t)va + offset; + src_sge.len = MSG_SIZE; + src_sge.tseg = local_tseg; + src_sg.sge = &src_sge; + src_sg.num_sge = 1; + wr.src = src_sg; + wr.user_ctx = offset; + wr.next = NULL; + if (urma_post_jetty_recv_wr(jetty, &wr, &bad_wr) != URMA_SUCCESS) { + fprintf(stderr, "Failed to recv %i in server jfr thread\n", i); + return NULL; + } + offset += MSG_SIZE; +} +// poll到成功的cr后，补充消耗的recv wr +if (cr.opcode == URMA_CR_OPC_SEND) { + if (urma_post_jetty_recv_wr(jetty, &wr, &bad_wr) != URMA_SUCCESS) { + fprintf(stderr, "Failed to recv in server jetty thread\n"); + return NULL; + } +} +``` + +Initiator post send示例： + +```c +urma_sge_t src_sge = { + .addr = (uint64_t)va + .len = MSG_SIZE, + .tseg = local_tseg +}; +urma_sg_t src_sg = { + .sge = &src_sge, + .num_sge = 1 +}; +urma_send_wr_t send_wr = { + .src = src_sg, + .tseg = local_tseg +}; +urma_jfs_wr_t jfs_wr = { + .opcode = URMA_OPC_SEND, + .flag.bs.complete_enable = 1, + .tjetty = t_jetty, + .send = send_wr, + .next = NULL +}; +urma_jfs_wr_t *bad_jfs_wr = NULL; +urma_post_jetty_send_wr(jetty, &jfs_wr, &bad_jfs_wr); +``` + +### 2.1.4 开源示例 + +开源代码中的编程示例链接： + + + +编译、使用方法可参考： + + + +编译方法也可参考《URMA安装部署用户手册》。 + +手动编译的结果在：src/build/urma/examples/urma_sample + +rpm包安装的结果在：/usr/bin/urma_sample + +## 2.2 管理面 + +### 2.2.1 初始化 + +#### 2.2.1.1 urma_init + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_init([3.2.1.1.1](#22111-urma_init_attr_t) [urma_init_attr_t](#22111-urma_init_attr_t) *conf); + +3. 描述 + +初始化URMA的执行环境。不支持多线程调用此接口。不支持与urma_register_sysfs_dev、urma_unregister_provider_ops并发调用。 + +4. 参数 + +@param[in] [Required] conf: urma init attr, a random uasid will be assigned when conf is null. + +5. 返回值 + +Return: 0 on success, other value on error. + +##### 2.2.1.1.1 urma_init_attr_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_init_attr { + uint64_t token; /* [Optional] security token */ + uint32_t uasid; /* [Optional] uasid to set and reserve. If the parameter is 0, the system will randomly assign a non-0 value. */ +} urma_init_attr_t; +``` + +##### 2.2.1.1.2 urma_status_t + +```c +typedef int urma_status_t; +``` + +定义文件: [urma_opcode.h](../../../src/urma/lib/urma/core/include/urma_opcode.h) + +```c +#define URMA_SUCCESS 0 +#define URMA_EAGAIN EAGAIN // Resource temporarily unavailable +#define URMA_ENOMEM ENOMEM // Failed to allocate memory +#define URMA_ENOPERM EPERM // Operation not permitted +#define URMA_ETIMEOUT ETIMEDOUT // Operation time out +#define URMA_EINVAL EINVAL // Invalid argument +#define URMA_EEXIST EEXIST // Exist +#define URMA_EINPROGRESS EINPROGRESS +#define URMA_FAIL 0x1000 /* 0x1000 */ +``` + +#### 2.2.1.2 urma_uninit + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_uninit(void); + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +3. 描述 + +urma语义环境反初始化。会释放uasid。不支持多线程调用此接口。不支持与urma_register_sysfs_dev、urma_unregister_provider_ops并发调用。 + +4. 参数 + +void + +5. 返回值 + +Return: 0 on success, other value on error. + +### 2.2.2 设备及上下文 + +#### 2.2.2.1 device + +##### 2.2.2.1.1 urma_get_device_list + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[urma_device_t](#_ZH-CN_TOPIC_0000002521872497-chtext) **urma_get_device_list(int *num_devices); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +获取设备列表。支持多线程操作重入操作。 + +4. 参数 + +@param[out] num_devices: number of urma device. + +5. 返回值 + +Return: pointer array of urma_device; NULL means no device returned. + +6. 备注 + +Note: urma_free_device_list() needs to be called to free memory. + +7. [urma_device_t](#_ZH-CN_TOPIC_0000002521872497-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_device { + char name[URMA_MAX_NAME]; /* [Public] urma device's name, the names of devices + in different transport modes are different. */ + char path[URMA_MAX_PATH]; /* [Public] urma device's path in sysfs. */ + urma_transport_type_t type; /* [Public] urma device's transport type. */ + struct urma_provider_ops_t *ops; /* [Private] urma device driver's ops. */ + struct urma_sysfs_dev_t *sysfs_dev; /* [Private] internal device corresponding to the urma device */ +} urma_device_t; +``` + +8. [urma_transport_type_t](#_ZH-CN_TOPIC_0000002489912702-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef enum urma_transport_type { + URMA_TRANSPORT_INVALID = -1, + URMA_TRANSPORT_UB = 0, + URMA_TRANSPORT_MAX +} urma_transport_type_t; +``` + +9. [urma_provider_ops_t](#_ZH-CN_TOPIC_0000002489752726-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_provider_ops { + const char *name; + urma_device_attr_t attr; + urma_match_entry_t *match_table; + urma_status_t (*init)(urma_init_attr_t *conf); + urma_status_t (*uninit)(void); + /* Device OPs */ + urma_status_t (*query_device)(urma_device_t *dev, urma_device_attr_t *dev_attr); + urma_context_t *(*create_context)(urma_device_t *dev, uint32_t eid_index, int dev_fd); + urma_status_t (*delete_context)(urma_context_t *ctx); + urma_status_t (*get_uasid)(uint32_t *uasid); /* obsolete */ +} urma_provider_ops_t; +``` + +10. [urma_match_entry_t](#_ZH-CN_TOPIC_0000002496889932-chtext) + +定义文件: [urma_provider.h](../../../src/urma/lib/urma/core/include/urma_provider.h) + +```c +typedef struct urma_match_entry { + uint16_t vendor_id; + uint16_t device_id; +} urma_match_entry_t; +``` + +11. [urma_sysfs_dev_t](#_ZH-CN_TOPIC_0000002521992509-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_sysfs_dev { + char dev_name[URMA_MAX_NAME]; + char sysfs_path[URMA_MAX_SYSFS_PATH]; + char driver_name[URMA_MAX_NAME]; + urma_transport_type_t transport_type; /* transport type */ + urma_driver_t *driver; + urma_device_t *urma_device; + urma_device_attr_t dev_attr; + uint16_t device_id; + uint16_t vendor_id; + struct ub_list node; /* Add to device list */ + uint32_t flag; + struct timespec time_created; +} urma_sysfs_dev_t; +``` + +12. [urma_driver_t](#_ZH-CN_TOPIC_0000002496570596-chtext) + +```c +typedef struct urma_driver { + struct urma_provider_ops_t *ops; + struct ub_list node; /* Add to driver list */ +} urma_driver_t; +``` + +13. [ub_list](#_ZH-CN_TOPIC_0000002528650589-chtext) + +```c +struct ub_list { + struct ub_list *prev, *next; +}; +``` + +##### 2.2.2.1.2 urma_free_device_list + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +void urma_free_device_list([urma_device_t](#_ZH-CN_TOPIC_0000002521872497-chtext) **device_list); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +释放设备列表。在使用完device_list之后调用。支持多线程操作重入操作。 + +4. 参数 + +@param[in] [Required] device_list: pointer array of urma_device, return value of urma_get_device_list. + +![](figures/urma_notice.png) + +由调用者保证参数device_list来自[3.2.2.1.1](#22211-urma_get_device_list) [urma_get_device_list](#22211-urma_get_device_list)接口返回，参数内部指针等合法性由这些接口保证，本接口不再重复进行校验；否则可能导致调用者进程异常退出。 + +5. 返回值 + +void + +##### 2.2.2.1.3 urma_get_device_by_name + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[urma_device_t](#_ZH-CN_TOPIC_0000002521872497-chtext) *urma_get_device_by_name(char *dev_name); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +根据设备名称dev_name获取其句柄。基于urma_get_device_list封装。支持多线程操作重入操作。 + +4. 参数 + +@param[in] [Required] dev_name: device's name; + +5. 返回值 + +Return: urma_device; NULL means no device returned; + +##### 2.2.2.1.4 urma_get_device_by_eid + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[urma_device_t](#_ZH-CN_TOPIC_0000002521872497-chtext) *urma_get_device_by_eid([urma_eid_t](#_ZH-CN_TOPIC_0000002521872509-chtext) eid, [urma_transport_type_t](#_ZH-CN_TOPIC_0000002489912702-chtext) type); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +根据设备eid获取其句柄。基于[3.2.2.1.1](#22211-urma_get_device_list) [urma_get_device_list](#22211-urma_get_device_list)封装。支持多线程操作重入操作。 + +4. 参数 + +@param[in] [Required] eid: device's eid; + +@param[in] [Required] type: device's transport type; + +5. 返回值 + +Return: pointer of urma_device; NULL means no device returned. + +##### 2.2.2.1.5 urma_query_device + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_query_device([urma_device_t](#_ZH-CN_TOPIC_0000002521872497-chtext) *dev, [urma_device_attr_t](#_ZH-CN_TOPIC_0000002521872503-chtext) *dev_attr); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +查询设备属性dev_attr，包括设备的ID信息（EID、GUID）；最大JFC数量及深度，最大JFS、JFR、Jetty、Jetty group的数量及深度；以及各个物理端口的状态和最大带宽等信息。支持多线程操作重入操作。 + +4. 参数 + +@param[in] [Required] dev: urma_device; + +@param[out] dev_attr: Return device attributes, user needs to allocate and free the memory; + +5. 返回值 + +Return: 0 on success, other value on error. + +6. [urma_device_attr_t](#_ZH-CN_TOPIC_0000002521872503-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_device_attr { + urma_guid_t guid; /* [Public] */ + urma_device_cap_t dev_cap; /* [Public] capabilities of device. */ + uint8_t port_cnt; /* [Public] port number of device. */ + struct urma_port_attr_t port_attr[MAX_PORT_CNT]; + uint32_t reserved_jetty_id_min; + uint32_t reserved_jetty_id_max; +} urma_device_attr_t; +``` + +7. [urma_guid_t](#_ZH-CN_TOPIC_0000002489752730-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_guid { + uint8_t raw[URMA_GUID_SIZE]; +} urma_guid_t; +``` + +8. [urma_device_cap_t](#_ZH-CN_TOPIC_0000002521992515-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_device_cap { + urma_device_feature_t feature; /* [Public] support feature of device, such as OOO, LS etc. */ + uint32_t max_jfc; /* [Public] max number of jfc supported by the device. */ + uint32_t max_jfs; /* [Public] max number of jfs supported by the device. */ + uint32_t max_jfr; /* [Public] max number of jfr supported by the device. */ + uint32_t max_jetty; /* [Public] max number of jetty supported by the device. */ + uint32_t max_jetty_grp; /* [Public] max number of jetty group supported by the device. */ + uint32_t max_jetty_in_jetty_grp; /* [Public] max number of jetty per jetty group supported by the device. */ + uint32_t max_jfc_depth; /* [Public] max depth of jfc supported by the device. */ + uint32_t max_jfs_depth; /* [Public] max depth of jfs supported by the device. */ + uint32_t max_jfr_depth; /* [Public] max depth of jfr supported by the device. */ + uint32_t max_jfs_inline_len; /* [Public] max inline length(byte) supported by the jfs. */ + uint32_t max_jfs_sge; /* [Public] max number of sge supported by the jfs. */ + uint32_t max_jfs_rsge; /* [Public] max number of remote sge supported by the jfs. */ + uint32_t max_jfr_sge; /* [Public] max number of sge supported by the jfr. */ + uint64_t max_msg_size; /* [Public] max message size supported by the device. */ + uint32_t max_read_size; + uint32_t max_write_size; + uint32_t max_cas_size; + uint32_t max_swap_size; + uint32_t max_fetch_and_add_size; + uint32_t max_fetch_and_sub_size; + uint32_t max_fetch_and_and_size; + uint32_t max_fetch_and_or_size; + uint32_t max_fetch_and_xor_size; + urma_atomic_feature_t atomic_feat; /* [Public] support atomic feature of device */ + uint16_t trans_mode; /* [Public] bit OR of supported transport modes */ + uint16_t sub_trans_mode_cap; /* [Public] bit OR of supported transport modes cap, urma_sub_trans_mode_cap_t */ + uint16_t congestion_ctrl_alg; /* [Public] one or more mode from urma_congestion_ctrl_alg_t */ + uint32_t ceq_cnt; /* [Public] ceq_cnt */ + uint32_t max_tp_in_tpg; /* [Public] max tp in tpg */ + uint32_t max_eid_cnt; /* [Public] max eid count */ + uint64_t page_size_cap; /* [Public] page size capability, must include PAGE_SIZE(4k) */ + uint32_t max_oor_cnt; /* [Public] max OOR window size by packet, only for user tp */ + uint32_t mn; /* [Public] only for user tp */ + uint32_t max_netaddr_cnt; /* [Public] only for user tp */ + urma_order_type_cap_t rm_order_cap; + urma_order_type_cap_t rc_order_cap; + urma_tp_type_cap_t rm_tp_cap; + urma_tp_type_cap_t rc_tp_cap; + urma_tp_type_cap_t um_tp_cap; + urma_tp_feature_t tp_feature; +} urma_device_cap_t; +``` + +9. [urma_device_feature_t](#_ZH-CN_TOPIC_0000002489912708-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef union urma_device_feature { + struct { + uint32_t oor : 1; /* [Public] URMA_OUT_OF_ORDER_RECEIVING. */ + uint32_t jfc_per_wr : 1; /* [Public] URMA_JFC_PER_WR. */ + uint32_t stride_op : 1; /* [Public] URMA_STRIDE_OP. */ + uint32_t load_store_op : 1; /* [Public] URMA_LOAD_STORE_OP. */ + uint32_t non_pin : 1; /* [Public] URMA_NON_PIN. */ + uint32_t pmem : 1; /* [Public] URMA_PERSISTENCE_MEM. */ + uint32_t jfc_inline : 1; /* [Public] URMA_JFC_INLINE. */ + uint32_t spray_en : 1; /* [Public] URMA_SPRAY_ENABLE for UDP port. */ + uint32_t selective_retrans : 1; /* [Public] URMA_SELECTIVE_RETRANS. */ + uint32_t live_migrate : 1; /* [Public] support live migration. */ + uint32_t dca : 1; /* [Public] for user tp */ + uint32_t jetty_grp : 1; /* [Public] support jetty group. */ + uint32_t error_suspend : 1; /* [Public] support suspend jetty or jfs on error. */ + uint32_t outorder_comp : 1; /* [Public] support out-of-order completion. */ + uint32_t mn : 1; /* [Public] for user tp */ + uint32_t clan : 1; /* [Public] for user tp */ + uint32_t muti_seg_per_token_id : 1; + uint32_t reserved : 15; + } bs; + uint32_t value; +} urma_device_feature_t; +``` + +10. [urma_atomic_feature_t](#_ZH-CN_TOPIC_0000002489752732-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef union urma_atomic_feature { + struct { + uint32_t cas : 1; + uint32_t swap : 1; + uint32_t fetch_and_add : 1; + uint32_t fetch_and_sub : 1; + uint32_t fetch_and_and : 1; + uint32_t fetch_and_or : 1; + uint32_t fetch_and_xor : 1; + uint32_t reserved : 25; + } bs; + uint32_t value; +} urma_atomic_feature_t; +``` + +11. [urma_sub_trans_mode_cap_t](#_ZH-CN_TOPIC_0000002528411323-chtext) + +```c +typedef enum urma_sub_trans_mode_cap { + URMA_RC_TP_DST_ORDERING = 0x1, /* rc mode with tp dst ordering */ + URMA_RC_TA_DST_ORDERING = 0x1 << 1, /* rc mode with ta dst ordering */ + URMA_RC_USER_TP = 0x1 << 2, /* rc mode with user tp */ +} urma_sub_trans_mode_cap_t; +``` + +12. [urma_congestion_ctrl_alg_t](#_ZH-CN_TOPIC_0000002528409915-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef enum urma_congestion_ctrl_alg { + URMA_CC_NONE = 0x1 << URMA_TP_CC_NONE, + URMA_CC_DCQCN = 0x1 << URMA_TP_CC_DCQCN, + URMA_CC_DCQCN_AND_NETWORK_CC = 0x1 << URMA_TP_CC_DCQCN_AND_NETWORK_CC, + URMA_CC_LDCP = 0x1 << URMA_TP_CC_LDCP, + URMA_CC_LDCP_AND_CAQM = 0x1 << URMA_TP_CC_LDCP_AND_CAQM, + URMA_CC_LDCP_AND_OPEN_CC = 0x1 << URMA_TP_CC_LDCP_AND_OPEN_CC, + URMA_CC_HC3 = 0x1 << URMA_TP_CC_HC3, + URMA_CC_DIP = 0x1 << URMA_TP_CC_DIP, + URMA_CC_ACC = 0x1 << URMA_TP_CC_ACC +} urma_congestion_ctrl_alg_t; +``` + +13. [urma_order_type_cap_t](#_ZH-CN_TOPIC_0000002491667086-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef union urma_order_type_cap { + struct { + uint32_t ot : 1; + uint32_t oi : 1; + uint32_t ol : 1; + uint32_t no : 1; + uint32_t reserved : 28; + } bs; + uint32_t value; +} urma_order_type_cap_t; +``` + +14. [urma_tp_type_cap_t](#_ZH-CN_TOPIC_0000002491827052-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef union urma_tp_type_cap { + struct { + uint32_t rtp : 1; + uint32_t ctp : 1; + uint32_t utp : 1; + uint32_t reserved : 29; + } bs; + uint32_t value; +} urma_tp_type_cap_t; +``` + +15. [urma_tp_feature_t](#_ZH-CN_TOPIC_0000002523906825-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef union urma_tp_feature { + struct { + uint32_t rm_multi_path : 1; + uint32_t rc_multi_path : 1; + uint32_t reserved : 30; + } bs; + uint32_t value; +} urma_tp_feature_t; +``` + +16. [urma_port_attr_t](#_ZH-CN_TOPIC_0000002521872505-chtext) + +一个URMA物理设备可包含一个或者多个port。active mtu不能超过max mtu之值。 + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_port_attr { + urma_mtu_t max_mtu; /* [Public] MTU_256, MTU_512, MTU_1024 etc. */ + urma_port_state_t state; /* [Public] PORT_DOWN, PORT_INIT, PORT_ACTIVE */ + urma_link_width_t active_width; /* [Public] link width: X1, X2, X4. */ + urma_speed_t active_speed; /* [Public] bandwidth. */ + urma_mtu_t active_mtu; /* [Public] current effective mtu. */ +} urma_port_attr_t; +``` + +17. [urma_mtu_t](#_ZH-CN_TOPIC_0000002521872507-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef enum urma_mtu { + URMA_MTU_256 = 1, + URMA_MTU_512, + URMA_MTU_1024, + URMA_MTU_2048, + URMA_MTU_4096, + URMA_MTU_8192, +} urma_mtu_t; +``` + +18. [urma_port_state_t](#_ZH-CN_TOPIC_0000002521992517-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef enum urma_port_state { + URMA_PORT_NOP = 0, + URMA_PORT_DOWN, + URMA_PORT_INIT, + URMA_PORT_ARMED, + URMA_PORT_ACTIVE, + URMA_PORT_ACTIVE_DEFER, +} urma_port_state_t; +``` + +19. [urma_link_width_t](#_ZH-CN_TOPIC_0000002489752734-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef enum urma_link_width { + URMA_LINK_X1 = 0x1, + URMA_LINK_X2 = 0x1 << 1, + URMA_LINK_X4 = 0x1 << 2, + URMA_LINK_X8 = 0x1 << 3, + URMA_LINK_X16 = 0x1 << 4, + URMA_LINK_X32 = 0x1 << 5, +} urma_link_width_t; +``` + +20. [urma_speed_t](#_ZH-CN_TOPIC_0000002489912710-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef enum urma_speed { + URMA_SP_10M = 0, + URMA_SP_100M, + URMA_SP_1G, + URMA_SP_2_5G, + URMA_SP_5G, + URMA_SP_10G, + URMA_SP_14G, + URMA_SP_25G, + URMA_SP_40G, + URMA_SP_50G, + URMA_SP_100G, + URMA_SP_200G, + URMA_SP_400G, + URMA_SP_800G, +} urma_speed_t; +``` + +#### 2.2.2.2 eid + +##### 2.2.2.2.1 urma_get_eid_list + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[urma_eid_info_t](#_ZH-CN_TOPIC_0000002489912704-chtext) * urma_get_eid_list([urma_device_t](#_ZH-CN_TOPIC_0000002521872497-chtext) *dev, uint32_t *cnt); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +查询设备当前有效的eid信息列表。支持多线程操作重入操作。 + +受操作系统sysfs缓存大小限制（一页），每个namespace的每个urma设备最多支持获取80个EID。 + +需用户调用[3.2.2.2.2](#22222-urma_free_eid_list) [urma_free_eid_list](#22222-urma_free_eid_list)()释放相关资源。 + +4. 参数 + +@param[in] [Required] dev: urma_device; + +@param[out] cnt: Return the number of valid eids; + +5. 返回值 + +If it succeeds, it will return the eid_info array pointer. If it fails, it will return NULL. + +6. [urma_eid_info_t](#_ZH-CN_TOPIC_0000002489912704-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_eid_info { + urma_eid_t eid; + uint32_t eid_index; /* 0\~UBCORE_MAX_EID_CNT -1 */ +} urma_eid_info_t; +``` + +7. [urma_eid_t](#_ZH-CN_TOPIC_0000002521872509-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef union urma_eid { + uint8_t raw[URMA_EID_SIZE]; /* Network Order */ + struct { + uint64_t reserved; /* If IPv4 mapped to IPv6, == 0 */ + uint32_t prefix; /* If IPv4 mapped to IPv6, == 0x0000ffff */ + uint32_t addr; /* If IPv4 mapped to IPv6, == IPv4 addr */ + } in4; + struct { + uint64_t subnet_prefix; + uint64_t interface_id; + } in6; +} urma_eid_t; +``` + +##### 2.2.2.2.2 urma_free_eid_list + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +void urma_free_eid_list([urma_eid_info_t](#_ZH-CN_TOPIC_0000002489912704-chtext) *eid_list); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +释放urma_get_eid_list返回的eid信息列表。 + +4. 参数 + +@param[in] [Required] eid_list: The eid array pointer to be released + +![](figures/urma_notice.png) + +由调用者保证参数eid_list来自[3.2.2.2.1](#22221-urma_get_eid_list) [urma_get_eid_list](#22221-urma_get_eid_list)接口返回，参数内部指针等合法性由这些接口保证，本接口不再重复进行校验；否则可能导致调用者进程异常退出。 + +5. 返回值 + +void + +#### 2.2.2.3 uasid + +##### urma_get_smac + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +urma_status_t urma_get_smac(const urma_context_t *ctx, uint8_t *mac); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +获取源MAC地址。 + +4. 参数 + +@param[in] [Required] ctx: the created urma context pointer; + +@param[out] [Required] mac: the mac address of source; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### urma_get_dmac + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +urma_status_t urma_get_dmac(const urma_context_t *ctx, const urma_net_addr_t *net_addr, uint8_t *mac); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +获取目的MAC地址。 + +4. 参数 + +@param[in] [Required] ctx: the created urma context pointer; + +@param[in] [Required] net_addr: the ip info (type and net_addr are valid, vlan, mac, prefix_len will not be used); + +@param[out] [Required] mac: the mac address of dest; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### urma_get_eid_by_ip + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +urma_status_t urma_get_eid_by_ip(const urma_context_t *ctx, const urma_net_addr_t *net_addr, urma_eid_t *eid); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +根据IP地址获取EID。 + +4. 参数 + +@param[in] [Required] ctx: the created urma context pointer; + +@param[in] [Required] net_addr: the ip info (type and net_addr are valid); + +@param[out] [Required] eid: device eid; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### urma_get_ip_by_eid + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +urma_status_t urma_get_ip_by_eid(const urma_context_t *ctx, const urma_eid_t *eid, urma_net_addr_t *net_addr); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +根据EID获取IP地址。 + +4. 参数 + +@param[in] [Required] ctx: the created urma context pointer; + +@param[in] [Required] eid: device eid; + +@param[out] [Required] net_addr: the ip info (type and net_addr are valid); + +5. 返回值 + +Return: 0 on success, other value on error. + +##### 2.2.2.3.1 urma_get_uasid + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_get_uasid(uint32_t *uasid); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +获取uasid。 + +4. 参数 + +@param[out] uasid: the address to put uasid + +5. 返回值 + +Return: 0 on success, other value on error. + +#### 2.2.2.4 context + +##### 2.2.2.4.1 urma_create_context + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[urma_context_t](#_ZH-CN_TOPIC_0000002489912714-chtext) *urma_create_context([urma_device_t](#_ZH-CN_TOPIC_0000002521872497-chtext) *dev, uint32_t eid_index) + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +为设备创建urma上下文。 + +4. 参数 + +@param[in] [Required] dev: urma device, by get_device apis. + +@param[in] [Required] eid_index: device's eid index. + +5. 返回值 + +Return: urma context pointer on success, NULL on error. + +6. [urma_context_t](#_ZH-CN_TOPIC_0000002489912714-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_context { + struct urma_device_t *dev; /* [Private] point to the corresponding urma device. */ + struct urma_ops_t *ops; /* [Private] operation of urma device. */ + int dev_fd; /* [Private] fd of urma device's sysfs file. */ + int async_fd; /* [Private] fd of urma device's async event file. */ + pthread_mutex_t mutex; /* [Private] mutex of urma context. */ + urma_eid_t eid; /* [Public] eid of urma device. */ + uint32_t eid_index; + uint32_t uasid; /* [Public] uasid of current process. */ + struct urma_ref_t ref; /* [Private] reference count of urma context. */ + urma_context_aggr_mode_t aggr_mode; /* [Public] aggregated mode of urma context. */ +} urma_context_t; +``` + +7. [urma_ops_t](#_ZH-CN_TOPIC_0000002524152197-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_ops { + /* OPs name */ + const char *name; + /* Jetty OPs */ + urma_jfc_t *(*create_jfc)(urma_context_t *ctx, urma_jfc_cfg_t *jfc_cfg); + urma_status_t (*modify_jfc)(urma_jfc_t *jfc, urma_jfc_attr_t *attr); + urma_status_t (*delete_jfc)(urma_jfc_t *jfc); + urma_status_t (*delete_jfc_batch)(urma_jfc_t **jfc, int jfc_num, urma_jfc_t **bad_jfc); + urma_jfs_t *(*create_jfs)(urma_context_t *ctx, urma_jfs_cfg_t *jfs); + urma_status_t (*modify_jfs)(urma_jfs_t *jfs, urma_jfs_attr_t *attr); + urma_status_t (*query_jfs)(urma_jfs_t *jfs, urma_jfs_cfg_t *cfg, urma_jfs_attr_t *attr); + int (*flush_jfs)(urma_jfs_t *jfs, int cr_cnt, urma_cr_t *cr); + urma_status_t (*delete_jfs)(urma_jfs_t *jfs); + urma_status_t (*delete_jfs_batch)(urma_jfs_t **jfs_arr, int jfs_num, urma_jfs_t **bad_jfs); + urma_jfr_t *(*create_jfr)(urma_context_t *ctx, urma_jfr_cfg_t *jfr); + urma_status_t (*modify_jfr)(urma_jfr_t *jfr, urma_jfr_attr_t *attr); + urma_status_t (*query_jfr)(urma_jfr_t *jfr, urma_jfr_cfg_t *cfg, urma_jfr_attr_t *attr); + urma_status_t (*delete_jfr)(urma_jfr_t *jfr); + urma_status_t (*delete_jfr_batch)(urma_jfr_t **jfr_arr, int jfr_num, urma_jfr_t **bad_jfr); + urma_target_jetty_t *(*import_jfr)(urma_context_t *ctx, urma_rjfr_t *rjfr, urma_token_t *token); + urma_status_t (*unimport_jfr)(urma_target_jetty_t *target_jfr); + urma_status_t (*advise_jfr)(urma_jfs_t *jfs, urma_target_jetty_t *tjfr); + urma_status_t (*unadvise_jfr)(urma_jfs_t *jfs, urma_target_jetty_t *tjfr); + urma_status_t (*advise_jfr_async)(urma_jfs_t *jfs, urma_target_jetty_t *tjfr, urma_advise_async_cb_func cb_fun, + void *cb_arg); + urma_jetty_t *(*create_jetty)(urma_context_t *ctx, urma_jetty_cfg_t *jetty_cfg); + urma_status_t (*modify_jetty)(urma_jetty_t *jetty, urma_jetty_attr_t *jetty_attr); + urma_status_t (*query_jetty)(urma_jetty_t *jetty, urma_jetty_cfg_t *cfg, urma_jetty_attr_t *attr); + int (*flush_jetty)(urma_jetty_t *jetty, int cr_cnt, urma_cr_t *cr); + urma_status_t (*delete_jetty)(urma_jetty_t *jetty); + urma_status_t (*delete_jetty_batch)(urma_jetty_t **jetty_arr, int jetty_num, urma_jetty_t **bad_jetty); + urma_target_jetty_t *(*import_jetty)(urma_context_t *ctx, urma_rjetty_t *rjetty, urma_token_t *rjetty_token); + urma_status_t (*unimport_jetty)(urma_target_jetty_t *target_jetty); + urma_status_t (*advise_jetty)(urma_jetty_t *jetty, urma_target_jetty_t *tjetty); + urma_status_t (*unadvise_jetty)(urma_jetty_t *jetty, urma_target_jetty_t *tjetty); + urma_status_t (*advise_jetty_async)(urma_jetty_t *jetty, urma_target_jetty_t *tjetty, + urma_advise_async_cb_func cb_fun, void *cb_arg); + urma_status_t (*bind_jetty)(urma_jetty_t *jetty, urma_target_jetty_t *tjetty); + urma_status_t (*unbind_jetty)(urma_jetty_t *jetty); + urma_jetty_grp_t *(*create_jetty_grp)(urma_context_t *ctx, urma_jetty_grp_cfg_t *cfg); + urma_status_t (*delete_jetty_grp)(urma_jetty_grp_t *jetty_grp); + urma_jfce_t *(*create_jfce)(urma_context_t *ctx); + urma_status_t (*delete_jfce)(urma_jfce_t *jfce); + /** + * Get tpn of current jetty + * @param[in] jetty: the jetty pointer created before + * Return: 0 or positive as correct tpn; negative as get tpn failure + */ + int (*get_tpn)(urma_jetty_t *jetty); + int (*modify_tp)(urma_context_t *ctx, uint32_t tpn, urma_tp_cfg_t *cfg, urma_tp_attr_t *attr, + urma_tp_attr_mask_t mask); + /* Control plane OPs */ + urma_status_t (*get_tp_list)(urma_context_t *ctx, urma_get_tp_cfg_t *cfg, uint32_t *tp_cnt, + urma_tp_info_t *tp_list); + urma_status_t (*set_tp_attr)(const urma_context_t *ctx, const uint64_t tp_handle, const uint8_t tp_attr_cnt, + const uint32_t tp_attr_bitmap, const urma_tp_attr_value_t *tp_attr); + urma_status_t (*get_tp_attr)(const urma_context_t *ctx, const uint64_t tp_handle, uint8_t *tp_attr_cnt, + uint32_t *tp_attr_bitmap, urma_tp_attr_value_t *tp_attr); + urma_target_jetty_t *(*import_jetty_ex)(urma_context_t *ctx, urma_rjetty_t *rjetty, urma_token_t *token_value, + urma_active_tp_cfg_t *active_tp_cfg); + urma_target_jetty_t *(*import_jfr_ex)(urma_context_t *ctx, urma_rjfr_t *rjfr, urma_token_t *token_value, + urma_active_tp_cfg_t *active_tp_cfg); + urma_status_t (*bind_jetty_ex)(urma_jetty_t *jetty, urma_target_jetty_t *tjetty, + urma_active_tp_cfg_t *active_tp_cfg); + /* Segment OPs */ + urma_token_id_t *(*alloc_token_id)(urma_context_t *ctx); + urma_token_id_t *(*alloc_token_id_ex)(urma_context_t *ctx, urma_token_id_flag_t flag); + urma_status_t (*free_token_id)(urma_token_id_t *token_id); + urma_target_seg_t *(*register_seg)(urma_context_t *ctx, urma_seg_cfg_t *seg_cfg); + urma_status_t (*unregister_seg)(urma_target_seg_t *target_seg); + urma_target_seg_t *(*import_seg)(urma_context_t *ctx, urma_seg_t *seg, urma_token_t *token, uint64_t addr, + urma_import_seg_flag_t flag); + urma_status_t (*unimport_seg)(urma_target_seg_t *target_seg); + /* Events OPs */ + urma_status_t (*get_async_event)(urma_context_t *ctx, urma_async_event_t *event); + void (*ack_async_event)(urma_async_event_t *event); + /* Other OPs */ + int (*user_ctl)(urma_context_t *ctx, urma_user_ctl_in_t *in, urma_user_ctl_out_t *out); + /* Dataplane OPs */ + urma_status_t (*post_jfs_wr)(urma_jfs_t *jfs, urma_jfs_wr_t *wr, urma_jfs_wr_t **bad_wr); + urma_status_t (*post_jfr_wr)(urma_jfr_t *jfr, urma_jfr_wr_t *wr, urma_jfr_wr_t **bad_wr); + urma_status_t (*post_jetty_send_wr)(urma_jetty_t *jetty, urma_jfs_wr_t *wr, urma_jfs_wr_t **bad_wr); + urma_status_t (*post_jetty_recv_wr)(urma_jetty_t *jetty, urma_jfr_wr_t *wr, urma_jfr_wr_t **bad_wr); + int (*poll_jfc)(urma_jfc_t *jfc, int cr_cnt, urma_cr_t *cr); + urma_status_t (*rearm_jfc)(urma_jfc_t *jfc, bool solicited_only); + int (*wait_jfc)(urma_jfce_t *jfce, uint32_t jfc_cnt, int time_out, urma_jfc_t *jfc[]); + void (*ack_jfc)(urma_jfc_t *jfc[], uint32_t nevents[], uint32_t jfc_cnt); + /* Jetty async OPs */ + urma_target_jetty_t *(*import_jetty_async)(urma_notifier_t *notifier, const urma_rjetty_t *rjetty, + const urma_token_t *token_value, uint64_t user_ctx, int timeout); + urma_status_t (*unimport_jetty_async)(urma_target_jetty_t *target_jetty); + urma_status_t (*bind_jetty_async)(urma_notifier_t *notifier, urma_jetty_t *jetty, urma_target_jetty_t *tjetty, + uint64_t user_ctx, int timeout); + urma_status_t (*unbind_jetty_async)(urma_jetty_t *jetty); + urma_notifier_t *(*create_notifier)(urma_context_t *ctx); + urma_status_t (*delete_notifier)(urma_notifier_t *notifier); + int (*wait_notify)(urma_notifier_t *notifier, uint32_t cnt, urma_notify_t *notify, int timeout); + void (*ack_notify)(uint32_t cnt, urma_notify_t *notify); +} urma_ops_t; +``` + +8. [urma_ref_t](#_ZH-CN_TOPIC_0000002524072163-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_ref { +#ifndef \_\_cplusplus + atomic_ulong atomic_cnt; +#else + std::atomic_ulong atomic_cnt; +#endif +} urma_ref_t; +``` + +9. [urma_context_aggr_mode_t](#_ZH-CN_TOPIC_0000002528412247-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef enum urma_context_aggr_mode { + URMA_AGGR_MODE_STANDALONE, + URMA_AGGR_MODE_ACTIVE_BACKUP, + URMA_AGGR_MODE_BALANCE, +} urma_context_aggr_mode_t; +``` + +##### 2.2.2.4.2 urma_delete_context + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_delete_context([urma_context_t](#_ZH-CN_TOPIC_0000002489912714-chtext) *ctx); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +释放设备上下文。 + +4. 参数 + +@param[in] [Required] ctx: handle of the created context. + +![](figures/urma_notice.png) + +由调用者保证参数ctx来自[3.2.2.4.1](#22241-urma_create_context) [urma_create_context](#22241-urma_create_context)接口返回，参数内部指针等合法性由这些接口保证，本接口不再重复进行校验；否则可能导致调用者进程异常退出。 + +5. 返回值 + +Return: 0 on success, other value on error. + +##### 2.2.2.4.3 urma_set_context_opt + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_set_context_opt([urma_context_t](#_ZH-CN_TOPIC_0000002489912714-chtext) *ctx, [urma_opt_name_t](#_ZH-CN_TOPIC_0000002492112452-chtext) opt_name, const void *opt_value, size_t opt_len); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +设置上下文可选项。 + +4. 参数 + +@param[in] [Required] ctx: handle of the created context. + +@param[in] [Required] opt_name: name of option. + +@param[in] [Required] opt_value: opt value pointer. + +@param[in] [Required] opt_len: len of opt_value. + +5. 返回值 + +Return: 0 on success, other value on error. + +6. [urma_opt_name_t](#_ZH-CN_TOPIC_0000002492112452-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef enum urma_context_opt_name { + URMA_OPT_AGGR_MODE, +} urma_opt_name_t; +``` + +#### 2.2.2.5 net addr + +##### 2.2.2.5.1 urma_get_net_addr_list + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[urma_net_addr_info_t](#_ZH-CN_TOPIC_0000002489752786-chtext) *urma_get_net_addr_list([urma_context_t](#_ZH-CN_TOPIC_0000002489912714-chtext) *ctx, uint32_t *cnt) + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +根据urma上下文，从内核态获取用户建链需要的net address，以数组形式返回。 + +4. 参数 + +@param[in] ctx: the created urma context pointer; + +@param[out] cnt: numer of net address info + +5. 返回值 + +Return: pointer of net address list; NULL on error。 + +6. [urma_net_addr_info_t](#_ZH-CN_TOPIC_0000002489752786-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_net_addr_info { + urma_net_addr_t netaddr; + uint32_t index; +} urma_net_addr_info_t; +``` + +7. [urma_net_addr_t](#_ZH-CN_TOPIC_0000002489912762-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_net_addr { + sa_family_t sin_family; /* AF_INET/AF_INET6 */ + union { + struct in_addr in4; + struct in6_addr in6; + }; + uint64_t vlan; + uint8_t mac[URMA_MAC_BYTES]; + uint32_t prefix_len; +} urma_net_addr_t; +``` + +##### 2.2.2.5.2 urma_free_net_addr_list + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +void urma_free_net_addr_list([urma_net_addr_info_t](#_ZH-CN_TOPIC_0000002489752786-chtext) *net_addr_list); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +释放通过[3.2.2.5.1](#22251-urma_get_net_addr_list) [urma_get_net_addr_list](#22251-urma_get_net_addr_list)()获取的net address信息。 + +4. 参数 + +@param[in] net_addr_list: pointer of net address list + +![](figures/urma_notice.png) + +由调用者保证参数net_addr_list来自[3.2.2.5.1](#22251-urma_get_net_addr_list) [urma_get_net_addr_list](#22251-urma_get_net_addr_list)接口返回，参数内部指针等合法性由这些接口保证，本接口不再重复进行校验；否则可能导致调用者进程异常退出。并且释放之后用户不应再访问该net_addr_list指针。 + +5. 返回值 + +void + +### 2.2.3 安全 + +#### 2.2.3.1 urma_alloc_token_id + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.3.1.1](#22311-urma_token_id_t) [urma_token_id_t](#22311-urma_token_id_t) *urma_alloc_token_id([urma_context_t](#_ZH-CN_TOPIC_0000002489912714-chtext) *ctx); + +3. 描述 + +请求分配token id，用于向protection表注册segment。 + +4. 参数 + +@param[in] [Required] ctx: specifies the urma context; + +5. 返回值 + +Return: pointer to token_id on success, NULL on error. + +##### 2.2.3.1.1 urma_token_id_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_token_id { + urma_context_t *urma_ctx; + uint32_t token_id; + uint64_t handle; + urma_ref_t ref; +} urma_token_id_t; +``` + +##### 2.2.3.1.2 urma_token_id_flag_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef union urma_token_id_flag { + struct { + uint32_t multi_seg : 1; + uint32_t reserved : 31; + } bs; + uint32_t value; +} urma_token_id_flag_t; +``` + +#### 2.2.3.2 urma_alloc_token_id_ex + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.3.1.1](#22311-urma_token_id_t) [urma_token_id_t](#22311-urma_token_id_t) *urma_alloc_token_id_ex([urma_context_t](#_ZH-CN_TOPIC_0000002489912714-chtext) *ctx, [3.2.3.1.2](#22312-urma_token_id_flag_t) [urma_token_id_flag_t](#22312-urma_token_id_flag_t) flag); + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +3. 描述 + +[3.2.3.1](#2231-urma_alloc_token_id) [urma_alloc_token_id](#2231-urma_alloc_token_id)()扩展接口，增加flag参数控制用table mode还是entry mode。若用table mode，注册的seg地址需按页对齐。 + +4. 参数 + +@param[in] [Required] ctx: specifies the urma context; + +@param[in] [Required] flag: decides the mode of token id. use table mode if enable multi_seg in flag. + +5. 返回值 + +Return: pointer to token_id on success, NULL on error. + +#### 2.2.3.3 urma_free_token_id + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_free_token_id([3.2.3.1.1](#22311-urma_token_id_t) [urma_token_id_t](#22311-urma_token_id_t) *token_id) + +3. 描述 + +释放token id。 + +4. 参数 + +@param[in] [Required] token_id: Specifies the token id to be released; + +![](figures/urma_notice.png) + +由调用者保证参数token_id来自[3.2.3.1](#2231-urma_alloc_token_id) [urma_alloc_token_id](#2231-urma_alloc_token_id)接口返回，参数内部指针等合法性由这些接口保证，本接口不再重复进行校验；否则可能导致调用者进程异常退出。 + +5. 返回值 + +Return: 0 on success, other value on error + +## 2.3 控制面 + +### 2.3.1 Jetty相关 + +#### 2.3.1.1 JFC + +##### 2.3.1.1.1 urma_create_jfc + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[urma_jfc_t](#_ZH-CN_TOPIC_0000002521872513-chtext) *urma_create_jfc([urma_context_t](#_ZH-CN_TOPIC_0000002489912714-chtext) *ctx, [urma_jfc_cfg_t](#_ZH-CN_TOPIC_0000002489912716-chtext) *jfc_cfg); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +根据配置创建jfc。 + +4. 参数 + +@param[in] [Required] ctx: the urma context created before; + +@param[in] [Required] jfc_cfg: configuration including: depth, flag, jfce, user context; + +![](figures/urma_caution.png) + +正常时，JFC的队列深度配置不足，可能会影响应用的正常运行。异常时，硬件可能构造错误CR通知应用Jetty或JFS的状态发生了变化，构造的错误CR类型包括URMA_CR_WR_FLUSH_ERR_DONE和URMA_CR_WR_SUSPEND_ERR_DONE，也应为JFC预留足够的空间来存放硬件构造的CR，否则JFC会发生溢出。故推荐按照JFC队列深度 \>= 关联jetty的队列深度总和 / 每多少个WR生成一个CR来配置（默认为1）+ 关联jetty数。 + +5. 返回值 + +Return: the handle of created jfc, not NULL on success; NULL on error. + +6. [urma_jfc_cfg_t](#_ZH-CN_TOPIC_0000002489912716-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_jfc_cfg { + uint32_t depth; /* [Required] the depth of jfc, no greater than urma_device_cap_t-\>jfc_depth */ + urma_jfc_flag_t flag; /* [Optional] see urma_jfc_flag_t, set flag.value to be 0 by default */ + uint32_t ceqn; /* [Optional] event queue id, no greater than urma_device_cap_t-\>ceq_cnt + set to 0 by default */ + urma_jfce_t *jfce; /* [Required] the event of jfc */ + uint64_t user_ctx; /* [Optional] private data of jfc, set to NULL by default */ +} urma_jfc_cfg_t; +``` + +7. [urma_jfc_flag_t](#_ZH-CN_TOPIC_0000002489752740-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef union urma_jfc_flag { + struct { + uint32_t lock_free : 1; + uint32_t jfc_inline : 1; + uint32_t reserved : 30; + } bs; + uint32_t value; +} urma_jfc_flag_t; +``` + +8. [urma_jfce_t](#_ZH-CN_TOPIC_0000002489752796-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_jfce { + urma_context_t *urma_ctx; /* [Private] point to urma context. */ + int fd; /* [Private] fd of completed event. */ + struct urma_ref_t ref; /* [Private] reference count of urma context. */ +} urma_jfce_t; +``` + +9. [urma_jfc_t](#_ZH-CN_TOPIC_0000002521872513-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_jfc { + urma_context_t *urma_ctx; /* [Private] point to urma context. */ + urma_jfc_id_t jfc_id; /* [Public] see urma_jetty_id. */ + urma_jfc_cfg_t jfc_cfg; /* [Public] storage jfc config. */ + uint64_t handle; + pthread_mutex_t event_mutex; + pthread_cond_t event_cond; + uint32_t comp_events_acked; + uint32_t async_events_acked; +} urma_jfc_t; +``` + +10. [urma_jfc_id_t](#_ZH-CN_TOPIC_0000002521992525-chtext) + +typedef struct [urma_jetty_id_t](#_ZH-CN_TOPIC_0000002492112454-chtext) urma_jfc_id_t; + +##### 2.3.1.1.2 urma_modify_jfc + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_modify_jfc([urma_jfc_t](#_ZH-CN_TOPIC_0000002521872513-chtext) *jfc, [3.3.1.1.2](#23112-urma_modify_jfc) [urma_modify_jfc](#23112-urma_modify_jfc) *attr); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +修改JFC属性。 + +4. 参数 + +@param[in] [Required] jfc: specify JFC; + +@param[in] [Required] attr: attributes to be modified. 支持修改JFC的两种中断抑制参数：CQE消息个数、中断间隔时间。 + +5. 返回值 + +Return: 0 on success, other value on error. + +6. ?.1.urma_jfc_attr_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_jfc_attr { + uint32_t mask; /* mask value, refer to urma_jfc_attr_mask_t */ + uint16_t moderate_count; + uint16_t moderate_period; /* in micro seconds */ +} urma_jfc_attr_t; +``` + +7. ?.2.urma_jfc_attr_mask_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef enum urma_jfc_attr_mask { + JFC_MODERATE_COUNT = 0x1, + JFC_MODERATE_PERIOD = 0x1 << 1 +} urma_jfc_attr_mask_t; +``` + +##### 2.3.1.1.3 urma_delete_jfc + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_delete_jfc([urma_jfc_t](#_ZH-CN_TOPIC_0000002521872513-chtext) *jfc); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +删除JFC。删除成功后，JFC不能再被访问。 + +![](figures/urma_caution.png) + +由调用者保证调用urma_delete_jfc接口时，其它依赖jfc的urma对象（例如jetty，jfs和jfr）的生存周期已经结束，否则可能导致use_after_free问题。 + +4. 参数 + +@param[in] [Required] jfc: handle of the created jfc; + +![](figures/urma_notice.png) + +由调用者保证参数jfc来自[3.3.1.1.1](#23111-urma_create_jfc) [urma_create_jfc](#23111-urma_create_jfc)接口返回，参数内部指针等合法性由这些接口保证，本接口不再重复进行校验；否则可能导致调用者进程异常退出。 + +5. 返回值 + +Return: 0 on success, other value on error. + +##### 2.3.1.1.4 urma_delete_jfc_batch + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_delete_jfc_batch([urma_jfc_t](#_ZH-CN_TOPIC_0000002521872513-chtext) **jfc_arr, int jfc_num, urma_jfc_t **bad_jfc); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +批删除JFC。删除成功后，相关JFC都不能再被访问。如果删除失败，返回删除失败的第一个JFC的地址并退出。 + +4. 参数 + +@param[in] [Required] jfc_arr: the array of the jfc pointer; + +@param[in] [Required] jfc_num: array length; + +@param[out] [Required] bad_jfc: the address of the first failed jfc pointer; + +5. 返回值 + +Return: 0 on success, EINVAL on invalid parameter, other value on other batch delete errors. + +![](figures/urma_notice.png) + +1\. 会优先按序对数组中全部输入的jfc进行检查，存在空指针会直接返回bad_jfc，不进入批量删除操作； + +2\. 由于本接口需要将整个数组一次下发删除操作，所以必须保证全部jfc是属于同一个dev； + +3\. 只有接口返回失败才会填写bad_jfc，否则为默认传入值。 + +##### urma_alloc_jfc + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +urma_status_t urma_alloc_jfc(urma_context_t *urma_ctx, urma_jfc_cfg_t *cfg, urma_jfc_t **jfc); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +分配JFC资源。 + +4. 参数 + +@param[in] [Required] ctx: the urma context created before; + +@param[in] [Required] cfg: configuration of jfc; + +@param[out] [Required] jfc: the address to put the handle of allocated jfc; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### urma_free_jfc + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +urma_status_t urma_free_jfc(urma_jfc_t *jfc); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +释放已分配的JFC。释放后，jfc指针不再允许被访问。 + +4. 参数 + +@param[in] [Required] jfc: the jfc allocated before; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### urma_active_jfc + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +urma_status_t urma_active_jfc(urma_jfc_t *jfc); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +激活已分配的JFC。激活后JFC才能接收完成记录。 + +4. 参数 + +@param[in] [Required] jfc: the jfc allocated before; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### urma_deactive_jfc + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +urma_status_t urma_deactive_jfc(urma_jfc_t *jfc); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +去激活已激活的JFC。 + +4. 参数 + +@param[in] [Required] jfc: the jfc actived before; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### urma_set_jfc_opt + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +urma_status_t urma_set_jfc_opt(urma_jfc_t *jfc, uint64_t opt, void *buf, uint32_t len); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +设置JFC选项。 + +4. 参数 + +@param[in] [Required] jfc: the jfc allocated before; + +@param[in] [Required] opt: the opt to change cfg of jfc; + +@param[in] [Required] len: the len of the opt value (byte); + +@param[in] [Required] buf: the buffer containing the value to set; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### urma_get_jfc_opt + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +urma_status_t urma_get_jfc_opt(urma_jfc_t *jfc, uint64_t opt, void *buf, uint32_t len); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +获取JFC选项。 + +4. 参数 + +@param[in] [Required] jfc: the jfc allocated before; + +@param[in] [Required] opt: the opt to change cfg of jfc; + +@param[in] [Required] len: the len of the opt value (byte); + +@param[out] [Required] buf: the buffer to store the value; + +5. 返回值 + +Return: 0 on success, other value on error. + +#### 2.3.1.2 JFCE + +##### 2.3.1.2.1 urma_create_jfce + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[urma_jfce_t](#_ZH-CN_TOPIC_0000002489752796-chtext) *urma_create_jfce([urma_context_t](#_ZH-CN_TOPIC_0000002489912714-chtext) *ctx); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +创建JFCE。同一个进程的多个JFC可以对应一个JFCE。 + +4. 参数 + +@param[in] [Required] ctx: the urma context created before; + +5. 返回值 + +Return: the address of created jfce, not NULL on success, NULL on error. + +##### 2.3.1.2.2 urma_delete_jfce + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_delete_jfce([urma_jfce_t](#_ZH-CN_TOPIC_0000002489752796-chtext) *jfce); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +删除JFCE。删除成功后，JFCE不能再被用于等待JFC事件，无法访问。 + +4. 参数 + +@param[in] [Required] jfce: the jfce to be deleted; + +![](figures/urma_notice.png) + +由调用者保证参数jfc来自[3.3.1.2.1](#23121-urma_create_jfce) [urma_create_jfce](#23121-urma_create_jfce)接口返回，参数内部指针等合法性由这些接口保证，本接口不再重复进行校验；否则可能导致调用者进程异常退出。 + +5. 返回值 + +Return: 0 on success, other value on error + +#### 2.3.1.3 JFAE + +##### 2.3.1.3.1 urma_get_async_event + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_get_async_event([urma_context_t](#_ZH-CN_TOPIC_0000002489912714-chtext) *ctx, [urma_async_event_t](#_ZH-CN_TOPIC_0000002489752798-chtext) *event); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +查询设备或JETTY相关的异步事件。 + +4. 参数 + +@param[in] [Required] ctx: handle of the created urma context; + +@param[out] [Required] event: the address to put event. + +5. 返回值 + +Return: 0 on success, other value on error. + +6. [urma_async_event_t](#_ZH-CN_TOPIC_0000002489752798-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_async_event { + /* may be SW queue error, may be HW port error */ + const urma_context_t *urma_ctx; + union { + urma_jfc_t *jfc; + urma_jfs_t *jfs; + urma_jfr_t *jfr; + urma_jetty_t *jetty; + urma_jetty_grp_t *jetty_grp; + uint32_t port_id; + uint32_t eid_idx; + } element; + urma_async_event_type_t event_type; + void *priv; +} urma_async_event_t; +``` + +7. [urma_async_event_type_t](#_ZH-CN_TOPIC_0000002521872571-chtext) + +定义文件: [urma_opcode.h](../../../src/urma/lib/urma/core/include/urma_opcode.h) + +```c +typedef enum urma_async_event_type { + URMA_EVENT_JFC_ERR, + URMA_EVENT_JFS_ERR, + URMA_EVENT_JFR_ERR, + URMA_EVENT_JFR_LIMIT, + URMA_EVENT_JETTY_ERR, + URMA_EVENT_JETTY_LIMIT, + URMA_EVENT_JETTY_GRP_ERR, + URMA_EVENT_PORT_ACTIVE, + URMA_EVENT_PORT_DOWN, + URMA_EVENT_DEV_FATAL, + URMA_EVENT_EID_CHANGE, // eid change, HNM and other management roles will be modified. + URMA_EVENT_ELR_ERR, /* Entity level error */ + URMA_EVENT_ELR_DONE /* Entity flush done */ +} urma_async_event_type_t; +``` + +##### 2.3.1.3.2 urma_ack_async_event + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +void urma_ack_async_event([urma_async_event_t](#_ZH-CN_TOPIC_0000002489752798-chtext) *event); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +对异步事件做出应答。 + +![](figures/urma_caution.png) + +事件的port_id不应该在调用这个api之后被取消引用。 + +4. 参数 + +@param[in] event: the address to ack event； + +5. 返回值 + +Return: void + +#### 2.3.1.4 JFS + +##### 2.3.1.4.1 urma_create_jfs + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[urma_jfs_t](#_ZH-CN_TOPIC_0000002489752746-chtext) *urma_create_jfs([urma_context_t](#_ZH-CN_TOPIC_0000002489912714-chtext) *ctx, [urma_jfs_cfg_t](#_ZH-CN_TOPIC_0000002521992529-chtext) *jfs_cfg); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +创建JFS。一个JFS只允许一个进程使用，进程中多线程可同时使用。进程根据实际需要创建一个或者多个。JFS的深度以WR为单位。 + +4. 参数 + +@param[in] [Required] ctx: the urma context created before; + +@param[in] [Required] jfs_cfg: address to pu the jfs config; + +5. 返回值 + +Return: the handle of created jfs, not NULL on success, NULL on error. + +6. [urma_jfs_cfg_t](#_ZH-CN_TOPIC_0000002521992529-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_jfs_cfg { + uint32_t depth; /* [Required] the depth of jfs, defaut urma_device_cap_t-\>jfs_depth */ + urma_jfs_flag_t flag; /* [Optional] see urma_jfs_flag_t definition */ + urma_transport_mode_t trans_mode; /* [Required] transport mode, must be supported by the device */ + uint8_t priority; /* [Optional] set the priority of JFS, ranging from [0, 15] + Services with low delay need to set high priority. */ + uint8_t max_sge; /* [Optional] max sge count in one wr, defaut urma_device_cap_t-\>max_jfs_sge */ + uint8_t max_rsge; /* [Optional] max remote sge count in one wr, defaut urma_device_cap_t-\>max_jfs_sge */ + uint32_t max_inline_data; /* [Optional] the max inline data size of JFS. if the parameter is 0, + the system will assign device's max inline data length. */ + uint8_t rnr_retry; /* [Optional] number of times that jfs will resend packets before report error, + when the remote side is not ready to receive (RNR), ranging from [0, 7], + the value 0 means never retry and, + the value 7 means retry infinite number of times for RDMA devices */ + uint8_t err_timeout; /* [Optional] the timeout before report error, ranging from [0, 31], + the actual timeout in usec is caculated by: 4.096*(2^err_timeout) */ + urma_jfc_t *jfc; /* [Required] need to specify jfc */ + uint64_t user_ctx; /* [Optional] private data of jfs */ +} urma_jfs_cfg_t; +``` + +![](figures/urma_info.png) + +err_timeout取值范围0\~31，实际超时值计算方法：Timeout=4.096us*（2^ err_timeout） + +7. [urma_jfs_flag_t](#_ZH-CN_TOPIC_0000002489912722-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef union urma_jfs_flag { + struct { + uint32_t lock_free : 1; /* default as 0, lock protected */ + uint32_t error_suspend : 1; /* 0: error continue; 1: error suspend */ + uint32_t outorder_comp : 1; /* 0: not support; 1: support out-of-order completion */ + uint32_t order_type : 8; /* (0x0): default, auto config by driver */ + /* (0x1): OT, target ordering */ + /* (0x2): OI, initiator ordering */ + /* (0x3): OL, low layer ordering */ + /* (0x4): UNO, unreliable non ordering */ + uint32_t multi_path : 1; /* 1: multi-path, 0: single path, for ubagg only. */ + uint32_t ctp_rc_mul_path_mode : 1; /* 1: ctp rc mode multi-path */ + uint32_t reserved : 19; + } bs; + uint32_t value; +} urma_jfs_flag_t; +``` + +![](figures/urma_info.png) + +1\. error_suspend + +为0表示JFS的Exception Mode为Exception continue，意味着JFS出现错误后，不需要用户任何处理，JFS还可继续工作。 + +为1表示JFS的Exception Mode为Exception suspend，意味着JFS出现错误后，需要被用户处理后才可恢复继续工作。 + +2\. outorder_comp + +是否支持乱序完成。传输模式为RC或RM时支持，UM时不支持。 + +8. ?.3.urma_order_type_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef enum urma_order_type { + URMA_DEF_ORDER, + URMA_OT, // target ordering + URMA_OI, // initiator ordering + URMA_OL, // low layer ordering + URMA_NO // unreliable non ordering +} urma_order_type_t; +``` + +9. [urma_jfs_t](#_ZH-CN_TOPIC_0000002489752746-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_jfs { + urma_context_t *urma_ctx; /* [Private] point to urma context. */ + urma_jfs_id_t jfs_id; /* [Public] see urma_jetty_id. */ + urma_jfs_cfg_t jfs_cfg; /* [Public] storage jfs config. */ + uint64_t handle; + pthread_mutex_t event_mutex; + pthread_cond_t event_cond; + uint32_t async_events_acked; +} urma_jfs_t; +``` + +10. [urma_jfs_id_t](#_ZH-CN_TOPIC_0000002521872519-chtext) + +typedef struct [urma_jetty_id_t](#_ZH-CN_TOPIC_0000002492112454-chtext) urma_jfs_id_t; + +##### 2.3.1.4.2 urma_modify_jfs + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_modify_jfs([urma_jfs_t](#_ZH-CN_TOPIC_0000002489752746-chtext) *jfs, [urma_jfs_attr_t](#_ZH-CN_TOPIC_0000002489912724-chtext) *attr); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +修改JFS的属性。 + +4. 参数 + +@param[in] [Required] jfs: the jfs created before; + +@param[in] [Required]attr: attributes to be modified. + +5. 返回值 + +Return: 0 on success, other value on error. + +6. [urma_jfs_attr_t](#_ZH-CN_TOPIC_0000002489912724-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_jfs_attr { + uint32_t mask; /* mask value refer to urma_jfs_attr_mask_t */ + urma_jfs_state_t state; +} urma_jfs_attr_t; +``` + +7. ?.2.urma_jfs_attr_mask_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef enum urma_jfs_attr_mask { + JFS_STATE = 0x1 +} urma_jfs_attr_mask_t; +``` + +8. [urma_jfs_state_t](#_ZH-CN_TOPIC_0000002521872521-chtext) + +typedef urma_jetty_state_t urma_jfs_state_t; + +##### 2.3.1.4.3 urma_query_jfs + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_query_jfs([urma_jfs_t](#_ZH-CN_TOPIC_0000002489752746-chtext) *jfs, [urma_jfs_cfg_t](#_ZH-CN_TOPIC_0000002521992529-chtext) *cfg, [urma_jfs_attr_t](#_ZH-CN_TOPIC_0000002489912724-chtext) *attr); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +查询JFS的配置和属性。支持多线程操作重入操作。 + +4. 参数 + +@param[in] [Required] jfs: the jfs created before; + +@param[out] [Required]cfg: config of jfs； + +@param[out] [Required]attr: attributes of jfs; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### 2.3.1.4.4 urma_delete_jfs + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_delete_jfs([urma_jfs_t](#_ZH-CN_TOPIC_0000002489752746-chtext) *jfs); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +删除JFS。删除成功后，JFS不能再被访问。 + +4. 参数 + +@param[in] [Required] jfs: the jfs created before; + +![](figures/urma_notice.png) + +由调用者保证参数jfs来自[3.3.1.4.1](#23141-urma_create_jfs) [urma_create_jfs](#23141-urma_create_jfs)接口返回，参数内部指针等合法性由这些接口保证，本接口不再重复进行校验；否则可能导致调用者进程异常退出。 + +5. 返回值 + +Return: 0 on success, other value on error. + +##### 2.3.1.4.5 urma_delete_jfs_batch + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_delete_jfs_batch([urma_jfs_t](#_ZH-CN_TOPIC_0000002489752746-chtext) **jfs_arr, int jfs_num, urma_jfs_t **bad_jfs); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +批删除JFS。删除成功后，相关JFS不能再被访问。如果删除失败，返回删除失败的第一个JFS的地址并退出。 + +4. 参数 + +@param[in] [Required] jfs_arr: the array of the jfs pointer; + +@param[in] [Required] jfs_num: array length; + +@param[out] [Required] bad_jfs: the address of the first failed jfs pointer; + +5. 返回值 + +Return: 0 on success, EINVAL on invalid parameter, other value on other batch delete errors. + +![](figures/urma_notice.png) + +1\. 会优先按序对数组中全部输入的jfs进行检查，存在空指针会直接返回bad_jfs，不进入批量删除操作； + +2\. 由于本接口需要将整个数组一次下发删除操作，所以必须保证全部jfs是属于同一个dev； + +3\. 只有接口返回失败才会填写bad_jfs，否则为默认传入值。 + +##### 2.3.1.4.6 urma_flush_jfs + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +int urma_flush_jfs([3.3.1.4.6](#23146-urma_flush_jfs) [urma_flush_jfs](#23146-urma_flush_jfs) *jfs, int cr_cnt, [3.4.2.1.1](#24211-urma_cr_t) [urma_cr_t](#24211-urma_cr_t) *cr); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +JFS状态切为Error态，或poll到status为URMA_CR_WR_SUSPEND_DONE的cr后调用，将post给JFS但未完成的wr，以cr的形式poll回来。 + +4. 参数 + +@param[in] [Required] jfs: the jfs created before; + +@param[in] [Required] cr_cnt: Number of CR expected to be received; + +@param[out] [Required] cr: Address for storing CR; + +5. 返回值 + +Return: the number of CR returned, 0 means no CR returned, -1 on error. + +![](figures/urma_notice.png) + +若执行成功，则出参cr status为URMA_CR_WR_FLUSH_ERR。 + +##### urma_alloc_jfs + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +urma_status_t urma_alloc_jfs(urma_context_t *urma_ctx, urma_jfs_cfg_t *cfg, urma_jfs_t **jfs); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +分配JFS资源。 + +4. 参数 + +@param[in] [Required] ctx: the urma context created before; + +@param[in] [Required] cfg: configuration of jfs; + +@param[out] [Required] jfs: the address to put the handle of allocated jfs; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### urma_free_jfs + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +urma_status_t urma_free_jfs(urma_jfs_t *jfs); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +释放已分配的JFS。释放后，jfs指针不再允许被访问。 + +4. 参数 + +@param[in] [Required] jfs: the jfs allocated before; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### urma_active_jfs + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +urma_status_t urma_active_jfs(urma_jfs_t *jfs); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +激活已分配的JFS。激活后JFS才能提交工作请求。 + +4. 参数 + +@param[in] [Required] jfs: the jfs allocated before; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### urma_deactive_jfs + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +urma_status_t urma_deactive_jfs(urma_jfs_t *jfs); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +去激活已激活的JFS。 + +4. 参数 + +@param[in] [Required] jfs: the jfs actived before; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### urma_set_jfs_opt + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +urma_status_t urma_set_jfs_opt(urma_jfs_t *jfs, uint64_t opt, void *buf, uint32_t len); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +设置JFS选项。 + +4. 参数 + +@param[in] [Required] jfs: the jfs allocated before; + +@param[in] [Required] opt: the opt to change cfg of jfs; + +@param[in] [Required] len: the len of the opt value (byte); + +@param[in] [Required] buf: the buffer to store the value; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### urma_get_jfs_opt + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +urma_status_t urma_get_jfs_opt(urma_jfs_t *jfs, uint64_t opt, void *buf, uint32_t len); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +获取JFS选项。 + +4. 参数 + +@param[in] [Required] jfs: the jfs allocated before; + +@param[in] [Required] opt: the opt to change cfg of jfs; + +@param[in] [Required] len: the len of the opt value (byte); + +@param[out] [Required] buf: the buffer to store the value; + +5. 返回值 + +Return: 0 on success, other value on error. + +#### 2.3.1.5 JFR + +##### 2.3.1.5.1 urma_create_jfr + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[urma_jfr_t](#_ZH-CN_TOPIC_0000002521992537-chtext) *urma_create_jfr([urma_context_t](#_ZH-CN_TOPIC_0000002489912714-chtext) *ctx, [urma_jfr_cfg_t](#_ZH-CN_TOPIC_0000002489752752-chtext) *jfr_cfg); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +创建JFR。一个JFR只允许一个进程使用，进程中多线程同时使用。进程根据实际需要可创建一个或者多个JFR。JFR的深度以WR为单位。 + +4. 参数 + +@param[in] [Required] ctx: the urma context created before; + +@param[in] [Required] jfr_cfg: address to put the jfr config; + +5. 返回值 + +Return: the handle of created jfr, not NULL on success, NULL on error. + +6. [urma_jfr_cfg_t](#_ZH-CN_TOPIC_0000002489752752-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_jfr_cfg { + uint32_t id; /* [Optional] specify jfr id. If the parameter is 0, + the system will randomly assign a non-0 value. */ + uint32_t depth; /* [Required] total depth, include berth, default urma_device_cap_t-\>jfr_depth. */ + urma_jfr_flag_t flag; /* [Optional] whether is in TAG_matching, whether is in DC/IDC mode. */ + urma_transport_mode_t trans_mode; /* [Required] transport mode, must be supported by the device */ + uint8_t max_sge; /* [Optional] max sge count in one wr, default urma_device_cap_t-\>max_jfr_sge. */ + uint8_t min_rnr_timer; /* [Optional] the minimum RNR NACK timer, ranging from [0, 31], i.e. + the time before jfr sends NACK to the sender for the reason of "ready to receive" */ + urma_jfc_t *jfc; /* [Required] need to specify jfc. */ + urma_token_t token_value; /* [Required] specify token_value for jfr. */ + uint64_t user_ctx; /* [Optional] private data of jfr */ +} urma_jfr_cfg_t; +``` + +![](figures/urma_info.png) + +min_rnr_timer的值对应的时间定义如下： + +5'b00000 :655.36ms 5'b10000 :2.56ms + +5'b00001 :0.01ms 5'b10001 :3.84ms + +5'b00010 :0.02ms 5'b10010 :5.12ms + +5'b00011 :0.03ms 5'b10011 :7.68ms + +5'b00100 :0.04ms 5'b10100 :10.24ms + +5'b00101 :0.06ms 5'b10101 :15.36ms + +5'b00110 :0.08ms 5'b10110 :20.48ms + +5'b00111 :0.12ms 5'b10111 :30.72ms + +5'b01000 :0.16ms 5'b11000 :40.96ms + +5'b01001 :0.24ms 5'b11001 :61.44ms + +5'b01010 :0.32ms 5'b11010 :81.92ms + +5'b01011 :0.48ms 5'b11011 :122.88ms + +5'b01100 :0.64ms 5'b11100 :163.84ms + +5'b01101 :0.96ms 5'b11101 :245.76ms + +5'b01110 :1.28ms 5'b11110 :327.68ms + +5'b01111 :1.92ms 5'b11111 :491.52ms + +7. [urma_jfr_flag_t](#_ZH-CN_TOPIC_0000002521872525-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef union urma_jfr_flag { + struct { + uint32_t token_policy : 3; /* 0: URMA_TOKEN_NONE + 1: URMA_TOKEN_PLAIN_TEXT + 2: URMA_TOKEN_SIGNED + 3: URMA_TOKEN_ALL_ENCRYPTED + 4: URMA_TOKEN_RESERVED */ + uint32_t tag_matching : 1; /* 0: URMA_NO_TAG_MATCHING. + 1: URMA_WITH_TAG_MATCHING. */ + uint32_t lock_free : 1; + uint32_t order_type : 8; /* (0x0): default, auto config by driver */ + /* (0x1): OT, target ordering */ + /* (0x2): OI, initiator ordering */ + /* (0x3): OL, low layer ordering */ + /* (0x4): UNO, unreliable non ordering */ + uint32_t reserved : 19; + } bs; + uint32_t value; +} urma_jfr_flag_t; +``` + +8. [urma_transport_mode_t](#_ZH-CN_TOPIC_0000002521992519-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef enum urma_transport_mode { + URMA_TM_RM = 0x1, /* Reliable message */ + URMA_TM_RC = 0x1 << 1, /* Reliable connection */ + URMA_TM_UM = 0x1 << 2, /* Unreliable message */ +} urma_transport_mode_t; +``` + +9. [urma_jfr_t](#_ZH-CN_TOPIC_0000002521992537-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_jfr { + urma_context_t *urma_ctx; /* [Private] point to urma context. */ + urma_jfr_id_t jfr_id; /* [Public] see urma_jetty_id. */ + urma_jfr_cfg_t jfr_cfg; /* [Public] storage jfr config. */ + uint64_t handle; + pthread_mutex_t event_mutex; + pthread_cond_t event_cond; + uint32_t async_events_acked; +} urma_jfr_t; +``` + +10. [urma_jfr_id_t](#_ZH-CN_TOPIC_0000002489912730-chtext) + +typedef [urma_jetty_id_t](#_ZH-CN_TOPIC_0000002492112454-chtext) urma_jfr_id_t; + +##### 2.3.1.5.2 urma_modify_jfr + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_modify_jfr([urma_jfr_t](#_ZH-CN_TOPIC_0000002521992537-chtext) *jfr, [urma_jfr_attr_t](#_ZH-CN_TOPIC_0000002521872527-chtext) *attr); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +修改JFR的属性。 + +4. 参数 + +@param[in] [Required] jfr: specify JFR; + +@param[in] [Required] attr: attributes to be modified; + +5. 返回值 + +Return: 0 on success, other value on error. + +6. [urma_jfr_attr_t](#_ZH-CN_TOPIC_0000002521872527-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_jfr_attr { + uint32_t mask; // mask value refer to urma_jfr_attr_mask_t + uint32_t rx_threshold; + urma_jfr_state_t state; +} urma_jfr_attr_t; +``` + +7. ?.2.urma_jfr_attr_mask_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef enum urma_jfr_attr_mask { + JFR_RX_THRESHOLD = 0x1, + JFR_STATE = 0x1 << 1 +} urma_jfr_attr_mask_t; +``` + +8. [urma_jfr_state_t](#_ZH-CN_TOPIC_0000002489912732-chtext) + +定义文件: [urma_opcode.h](../../../src/urma/lib/urma/core/include/urma_opcode.h) + +```c +typedef enum urma_jfr_state { + URMA_JFR_STATE_RESET = 0, + URMA_JFR_STATE_READY, + URMA_JFR_STATE_ERROR +} urma_jfr_state_t; +``` + +##### 2.3.1.5.3 urma_query_jfr + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_query_jfr([urma_jfr_t](#_ZH-CN_TOPIC_0000002521992537-chtext) *jfr, [urma_jfr_cfg_t](#_ZH-CN_TOPIC_0000002489752752-chtext) *cfg, [urma_jfr_attr_t](#_ZH-CN_TOPIC_0000002521872527-chtext) *attr); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +查询JFR的配置和属性。 + +4. 参数 + +@param[in] [Required] jfr: specify JFR; + +@param[out] [Required] cfg: config of jfr; + +@param[out] [Required] attr: attributes of jfr; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### 2.3.1.5.4 urma_delete_jfr + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_delete_jfr([urma_jfr_t](#_ZH-CN_TOPIC_0000002521992537-chtext) *jfr); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +删除JFR。 + +4. 参数 + +@param[in] [Required] jfr: the jfr created before; + +5. 返回值 + +Return: 0 on success, other value on error. + +![](figures/urma_notice.png) + +1\. 由调用者保证参数jfr来自[3.3.1.5.1](#23151-urma_create_jfr) [urma_create_jfr](#23151-urma_create_jfr)接口返回，参数内部指针等合法性由这些接口保证，本接口不再重复进行校验；否则可能导致调用者进程异常退出。 + +2\. 由调用者保证调用urma_delete_jfr接口时，其它依赖jfr的urma对象(使用共享jfr的jetty）的生存周期已经结束，否则可能导致use_after_free问题。 + +##### 2.3.1.5.5 urma_delete_jfr_batch + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_delete_jfr_batch([urma_jfr_t](#_ZH-CN_TOPIC_0000002521992537-chtext) **jfr_arr, int jfr_num, urma_jfr_t **bad_jfr); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +批删除JFR。删除成功后，JFR不能再被访问。如果删除失败，返回删除失败的第一个JFR的地址并退出。 + +4. 参数 + +@param[in] [Required] jfr_arr: the array of the jfr pointer; + +@param[in] [Required] jfr_num: array length; + +@param[out] [Required] bad_jfr: the address of the first failed jfr pointer; + +5. 返回值 + +Return: 0 on success, EINVAL on invalid parameter, other value on other batch delete errors. + +![](figures/urma_notice.png) + +1\. 会优先按序对数组中全部输入的jfr进行检查，存在空指针会直接返回bad_jfr，不进入批量删除操作； + +2\. 由于本接口需要将整个数组一次下发删除操作，所以必须保证全部jfr是属于同一个dev； + +3\. 只有接口返回失败才会填写bad_jfr，否则为默认传入值。 + +##### 2.3.1.5.6 urma_import_jfr + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[urma_target_jetty_t](#_ZH-CN_TOPIC_0000002521992545-chtext) *urma_import_jfr([urma_context_t](#_ZH-CN_TOPIC_0000002489912714-chtext) *ctx, [urma_rjfr_t](#_ZH-CN_TOPIC_0000002489752758-chtext) *rjfr, [3.3.2.1.7](#23217-urma_token_t) [urma_token_t](#23217-urma_token_t) *token_value); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +导入远端JFR信息，包括注册其token到本地。 + +4. 参数 + +@param[in] [Required] ctx: the urma context created before; + +@param[in] [Required] rjfr: the information of remote jfr to import into user node, trans_mode required, trans_mode same to create_jfr trans_mode; + +@param[in] [Required] token_value: token_valueto put into output jetty/protection table; + +5. 返回值 + +Return: the address of target jfr, not NULL on success, NULL on error. + +6. [urma_rjfr_t](#_ZH-CN_TOPIC_0000002489752758-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_rjfr { + urma_jfr_id_t jfr_id; /* see urma_jetty_id */ + urma_transport_mode_t trans_mode; + urma_import_jetty_flag_t flag; + urma_tp_type_t tp_type; +} urma_rjfr_t; +``` + +7. [urma_import_jetty_flag_t](#_ZH-CN_TOPIC_0000002491952470-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef union urma_import_jetty_flag { + struct { + uint32_t token_policy : 3; + uint32_t order_type : 8; /* (0x0): default, auto config by driver */ + /* (0x1): OT, target ordering */ + /* (0x2): OI, initiator ordering */ + /* (0x3): OL, low layer ordering */ + /* (0x4): UNO, unreliable non ordering */ + uint32_t share_tp : 1; /* 1: shared tp; 0: non-shared tp. When rc mode is not ta dst ordering, + this flag can only be set to 0. */ + uint32_t reserved : 20; + } bs; + uint32_t value; +} urma_import_jetty_flag_t; +``` + +8. [urma_tp_type_t](#_ZH-CN_TOPIC_0000002524152199-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef enum urma_tp_type { + URMA_RTP, + URMA_CTP, + URMA_UTP +} urma_tp_type_t; +``` + +9. [urma_target_jetty_t](#_ZH-CN_TOPIC_0000002521992545-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_target_jetty { + urma_context_t *urma_ctx; /* [Private] point to urma context. */ + urma_jetty_id_t id; /* [Private] see urma_jetty_id. */ + uint64_t handle; + urma_transport_mode_t trans_mode; + urma_tp_t tp; + urma_target_type_t type; // todo supplementary target type + urma_import_jetty_flag_t flag; + urma_jetty_grp_policy_t policy; + urma_tp_type_t tp_type; +} urma_target_jetty_t; +``` + +10. [urma_jetty_id_t](#_ZH-CN_TOPIC_0000002492112454-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_jetty_id { + urma_eid_t eid; + uint32_t uasid; /* maybe zero(stand for kernel) or non-zero(stand for app) */ + uint32_t id; +} urma_jetty_id_t; +``` + +11. [urma_tp_t](#_ZH-CN_TOPIC_0000002489912738-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_tp { + uint32_t tpn; /* vtpn */ +} urma_tp_t; +``` + +12. [urma_target_type_t](#_ZH-CN_TOPIC_0000002489752762-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef enum urma_target_type { + URMA_JFR = 0, + URMA_JETTY, + URMA_JETTY_GROUP +} urma_target_type_t; +``` + +13. [urma_jetty_grp_policy_t](#_ZH-CN_TOPIC_0000002524072165-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef enum urma_jetty_grp_policy { + URMA_JETTY_GRP_POLICY_RR = 0, + URMA_JETTY_GRP_POLICY_HASH_HINT = 1 +} urma_jetty_grp_policy_t; +``` + +##### 2.3.1.5.7 urma_import_jfr_ex + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[urma_target_jetty_t](#_ZH-CN_TOPIC_0000002521992545-chtext) *urma_import_jfr_ex([urma_context_t](#_ZH-CN_TOPIC_0000002489912714-chtext) *ctx, [urma_rjfr_t](#_ZH-CN_TOPIC_0000002489752758-chtext) *rjfr, [3.3.2.1.7](#23217-urma_token_t) [urma_token_t](#23217-urma_token_t) *token_value, [urma_import_jfr_ex_cfg_t](#_ZH-CN_TOPIC_0000002521872535-chtext) *cfg); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +[3.3.1.5.6](#23156-urma_import_jfr) [urma_import_jfr](#23156-urma_import_jfr)()扩展接口，增加入参urma_import_jfr_ex_cfg_t *cfg。 + +4. 参数 + +@param[in] [Required] ctx: the urma context created before; + +@param[in] [Required] rjfr: the information of remote jfr to import into user node, trans_mode required, trans_mode same to create_jfr trans_mode; + +@param[in] [Required] token_value: token to put into output jetty/protection table; + +@param[in] [Required] cfg: tp active configuration to exchange with target; + +5. 返回值 + +Return: the address of target jfr, not NULL on success, NULL on error + +6. [urma_import_jfr_ex_cfg_t](#_ZH-CN_TOPIC_0000002521872535-chtext) + +typedef struct [urma_active_tp_cfg_t](#_ZH-CN_TOPIC_0000002525470775-chtext) urma_import_jfr_ex_cfg_t; + +7. [urma_active_tp_cfg_t](#_ZH-CN_TOPIC_0000002525470775-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_active_tp_cfg { + uint64_t tp_handle; + uint64_t peer_tp_handle; + uint64_t tag; + urma_active_tp_attr_t tp_attr; +} urma_active_tp_cfg_t; +``` + +8. [urma_active_tp_attr_t](#_ZH-CN_TOPIC_0000002528696683-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_active_tp_attr { + uint32_t tx_psn; + uint32_t rx_psn; + uint64_t reserved; +} urma_active_tp_attr_t; +``` + +##### 2.3.1.5.8 urma_unimport_jfr + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_unimport_jfr([urma_target_jetty_t](#_ZH-CN_TOPIC_0000002521992545-chtext) *target_jfr); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +unimport远端JFR。操作成功后，进程不可访问这个远端JFR。 + +4. 参数 + +@param[in] [Required] target_jfr: the target jfr to unimport; + +![](figures/urma_notice.png) + +由调用者保证参数target_jfr来自[3.3.1.5.6](#23156-urma_import_jfr) [urma_import_jfr](#23156-urma_import_jfr)接口返回，参数内部指针等合法性由这些接口保证，本接口不再重复进行校验；否则可能导致调用者进程异常退出。 + +5. 返回值 + +Return: 0 on success, other value on error. + +##### urma_alloc_jfr + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +urma_status_t urma_alloc_jfr(urma_context_t *urma_ctx, urma_jfr_cfg_t *cfg, urma_jfr_t **jfr); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +分配JFR资源。 + +4. 参数 + +@param[in] [Required] ctx: the urma context created before; + +@param[in] [Required] cfg: configuration of jfr; + +@param[out] [Required] jfr: the address to put the handle of allocated jfr; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### urma_free_jfr + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +urma_status_t urma_free_jfr(urma_jfr_t *jfr); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +释放已分配的JFR。释放后，jfr指针不再允许被访问。 + +4. 参数 + +@param[in] [Required] jfr: handle of the allocated jfr; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### urma_active_jfr + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +urma_status_t urma_active_jfr(urma_jfr_t *jfr); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +激活已分配的JFR。激活后JFR才能接收消息。 + +4. 参数 + +@param[in] [Required] jfr: handle of the allocated jfr; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### urma_deactive_jfr + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +urma_status_t urma_deactive_jfr(urma_jfr_t *jfr); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +去激活已激活的JFR。 + +4. 参数 + +@param[in] [Required] jfr: the jfr actived before; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### urma_set_jfr_opt + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +urma_status_t urma_set_jfr_opt(urma_jfr_t *jfr, uint64_t opt, void *buf, uint32_t len); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +设置JFR选项。 + +4. 参数 + +@param[in] [Required] jfr: handle of the allocated jfr; + +@param[in] [Required] opt: the opt to change cfg of jfr; + +@param[in] [Required] len: the len of the opt value (byte); + +@param[in] [Required] buf: the buffer to store the value; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### urma_get_jfr_opt + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +urma_status_t urma_get_jfr_opt(urma_jfr_t *jfr, uint64_t opt, void *buf, uint32_t len); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +获取JFR选项。 + +4. 参数 + +@param[in] [Required] jfr: handle of the allocated jfr; + +@param[in] [Required] opt: the opt to change cfg of jfr; + +@param[in] [Required] len: the len of the opt value (byte); + +@param[out] [Required] buf: the buffer to store the value; + +5. 返回值 + +Return: 0 on success, other value on error. + +#### 2.3.1.6 Jetty + +##### 2.3.1.6.1 urma_create_jetty + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[urma_jetty_t](#_ZH-CN_TOPIC_0000002489912746-chtext) *urma_create_jetty([urma_context_t](#_ZH-CN_TOPIC_0000002489912714-chtext) *ctx, [urma_jetty_cfg_t](#_ZH-CN_TOPIC_0000002489752766-chtext) *jetty_cfg); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +创建Jetty。一个Jetty只允许一个进程使用，进程中多线程可同时使用。进程根据实际需要创建一个或者多个jetty。 + +4. 参数 + +@param[in] [Required] ctx: the urma context created before; + +@param[in] [Required] jetty_cfg: pointer of the jetty config; + +![](figures/urma_caution.png) + +1\. 用户指定的jetty id可能已被占用，应以实际分配出的id为准。 + +2\. UB设备只支持Jetty绑定共享JFR，用户创建Jetty之前必须先创建接收共享JFR，创建Jetty时，必须传入JFR指针和接收JFC指针。 + +3\. Jetty的深度以WR为单位。jfc的深度要是jfr和jfs的深度和，否则会存在数据丢失的情况。 + +5. 返回值 + +Return: the handle of created jetty, not NULL on success, NULL on error. + +6. [urma_jetty_cfg_t](#_ZH-CN_TOPIC_0000002489752766-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_jetty_cfg { + uint32_t id; /* [Optional] user specified jetty id. */ + urma_jetty_flag_t flag; /* [Optional] Connection or connection less */ + /* send configuration */ + urma_jfs_cfg_t jfs_cfg; /* [Required] see urma_jfs_cfg_t */ + /* recv configuration */ + union { + struct { + urma_jfr_t *jfr; /* [Optional] shared jfr to receive msg */ + urma_jfc_t *jfc; /* [Optional] To replace the jfc related to the above jfr */ + } shared; /* [Required] */ + urma_jfr_cfg_t *jfr_cfg; /* deprecated */ + }; + urma_jetty_grp_t *jetty_grp; /* [Optional] user specified jetty group. */ + uint64_t user_ctx; /* [Optional] private data of jetty */ +} urma_jetty_cfg_t; +``` + +7. [urma_jetty_flag_t](#_ZH-CN_TOPIC_0000002521872541-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef union urma_jetty_flag { + struct { + uint32_t share_jfr : 1; /* 0: URMA_NO_SHARE_JFR. + 1: URMA_SHARE_JFR. */ + uint32_t reserved : 31; + } bs; + uint32_t value; +} urma_jetty_flag_t; +``` + +![](figures/urma_caution.png) + +UB设备只支持share_jfr指定为URMA_SHARE \_JFR. + +8. [urma_jetty_grp_t](#_ZH-CN_TOPIC_0000002524152201-chtext) + +```c +struct urma_jetty_grp { + urma_context_t *urma_ctx; + urma_jetty_id_t jetty_grp_id; + urma_jetty_grp_cfg_t cfg; + uint32_t jetty_cnt; + urma_jetty_t **jetty_list; + pthread_mutex_t list_mutex; + uint64_t handle; /* use to quickly get uobj of jetty group in kernel module */ + pthread_mutex_t event_mutex; + pthread_cond_t event_cond; + uint32_t async_events_acked; +}; +``` + +9. [urma_jetty_grp_cfg_t](#_ZH-CN_TOPIC_0000002527065929-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_jetty_grp_cfg { + char name[URMA_MAX_NAME]; + urma_jetty_grp_flag_t flag; + urma_token_t token_value; /* [Required] specify token_value for Jetty group. */ + uint32_t id; /* [Optional] specify Jetty group id. + If the parameter is 0, UMDK will assign a non_0 value. */ + urma_jetty_grp_policy_t policy; /* Hash or RR(on default) */ + uint64_t user_ctx; /* [Optional] private data of jetty */ +} urma_jetty_grp_cfg_t; +``` + +10. [urma_jetty_grp_flag_t](#_ZH-CN_TOPIC_0000002521872565-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef union urma_jetty_grp_flag { + struct { + uint32_t token_policy : 3; /* 0: URMA_TOKEN_NONE + 1: URMA_TOKEN_PLAIN_TEXT + 2: URMA_TOKEN_SIGNED + 3: URMA_TOKEN_ALL_ENCRYPTED + 4: URMA_TOKEN_RESERVED */ + uint32_t reserved : 29; + } bs; + uint32_t value; +} urma_jetty_grp_flag_t; +``` + +11. [urma_jetty_t](#_ZH-CN_TOPIC_0000002489912746-chtext) + +```c +struct urma_jetty_grp { + urma_context_t *urma_ctx; + urma_jetty_id_t jetty_grp_id; + urma_jetty_grp_cfg_t cfg; + uint32_t jetty_cnt; + urma_jetty_t **jetty_list; + pthread_mutex_t list_mutex; + uint64_t handle; /* use to quickly get uobj of jetty group in kernel module */ + pthread_mutex_t event_mutex; + pthread_cond_t event_cond; + uint32_t async_events_acked; +}; +``` + +##### 2.3.1.6.2 urma_modify_jetty + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_modify_jetty([urma_jetty_t](#_ZH-CN_TOPIC_0000002489912746-chtext) *jetty, [urma_jetty_attr_t](#_ZH-CN_TOPIC_0000002521872543-chtext) *attr); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +修改Jetty属性。 + +![](figures/urma_caution.png) + +UB设备的Jetty只支持使用共享JFR，不支持修改Jetty水线。 + +4. 参数 + +@param[in] [Required] jetty: specify jetty; + +@param[in] [Required] attr: attributes to be modified; + +5. 返回值 + +Return: 0 on success, other value on error. + +6. [urma_jetty_attr_t](#_ZH-CN_TOPIC_0000002521872543-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_jetty_attr { + uint32_t mask; // mask value refer to urma_jetty_attr_mask_t + uint32_t rx_threshold; + urma_jetty_state_t state; +} urma_jetty_attr_t; +``` + +7. ?.2.urma_jetty_attr_mask_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef enum urma_jetty_attr_mask { + JETTY_RX_THRESHOLD = 0x1, + JETTY_STATE = 0x1 << 1 +} urma_jetty_attr_mask_t; +``` + +8. [urma_jetty_state_t](#_ZH-CN_TOPIC_0000002489912748-chtext) + +定义文件: [urma_opcode.h](../../../src/urma/lib/urma/core/include/urma_opcode.h) + +```c +typedef enum urma_jetty_state { + URMA_JETTY_STATE_RESET = 0, + URMA_JETTY_STATE_READY, + URMA_JETTY_STATE_SUSPENDED, + URMA_JETTY_STATE_ERROR +} urma_jetty_state_t; +``` + +##### 2.3.1.6.3 urma_query_jetty + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_query_jetty([urma_jetty_t](#_ZH-CN_TOPIC_0000002489912746-chtext) *jetty, [urma_jetty_cfg_t](#_ZH-CN_TOPIC_0000002489752766-chtext) *cfg, [urma_jetty_attr_t](#_ZH-CN_TOPIC_0000002521872543-chtext) *attr) + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +查询Jetty配置和属性。支持多线程操作重入操作。 + +4. 参数 + +@param[in] [Required] jetty: the jetty created before; + +@param[out] [Required] cfg: config to query; + +@param[out] [Required] attr: attributes to query; + +5. 返回值 + +Return: 0 on success, other value on error. + +![](figures/urma_notice.png) + +对于采用share jfr的jetty，[3.3.1.6.3](#23163-urma_query_jetty) [urma_query_jetty](#23163-urma_query_jetty)不会查询jfr的属性，请使用[3.3.1.5.3](#23153-urma_query_jfr) [urma_query_jfr](#23153-urma_query_jfr)进行查询。 + +##### 2.3.1.6.4 urma_delete_jetty + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_delete_jetty([urma_jetty_t](#_ZH-CN_TOPIC_0000002489912746-chtext) *jetty); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +删除Jetty。删除成功的Jetty不能再被访问。不支持多线程操作重入操作。 + +4. 参数 + +@param[in] [Required] jetty: the jetty created before; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### 2.3.1.6.5 urma_delete_jetty_batch + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_delete_jetty_batch([urma_jetty_t](#_ZH-CN_TOPIC_0000002489912746-chtext) **jetty_arr, int jetty_num, urma_jetty_t **bad_jetty); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +批删除Jetty。删除成功后，相关Jetty不能再被访问。如果删除失败，返回删除失败的第一个Jetty的地址并退出。 + +4. 参数 + +@param[in] [Required] jetty_arr: the array of the jetty pointer; + +@param[in] [Required] jetty_num: array length; + +@param[out] [Required] bad_jetty: the address of the first failed jetty pointer; + +5. 返回值 + +Return: 0 on success, EINVAL on invalid parameter, other value on other batch delete errors. + +![](figures/urma_notice.png) + +1\. 会优先按序对数组中全部输入的jetty进行检查，存在空指针会直接返回bad_jetty，不进入批量删除操作； + +2\. 由于本接口需要将整个数组一次下发删除操作，所以必须保证全部jetty是属于同一个dev； + +3\. 只有接口返回失败才会填写bad_jetty，否则为默认传入值。 + +##### 2.3.1.6.6 urma_import_jetty + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[urma_target_jetty_t](#_ZH-CN_TOPIC_0000002521992545-chtext) *urma_import_jetty([urma_context_t](#_ZH-CN_TOPIC_0000002489912714-chtext) *ctx, [urma_rjetty_t](#_ZH-CN_TOPIC_0000002489912752-chtext) *rjetty, [3.3.2.1.7](#23217-urma_token_t) [urma_token_t](#23217-urma_token_t) *token_value); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +导入远端Jetty信息。 + +支持无连接类型的jetty和连接类型的jetty进行导入操作，导入操作不会和本地的某个jetty建立连接关系。 + +支持多线程操作重入操作。 + +4. 参数 + +@param[in] [Required] ctx: the urma context created before; + +@param[in] [Required] rjetty: information of remote jetty to import, including jetty id and mode, trans_mode same to create_jetty trans_mode; + +@param[in] [Required] token_value: token to put into output jetty protection table; + +5. 返回值 + +Return: the address of target jetty, not NULL on success, NULL on error. + +![](figures/urma_notice.png) + +ubcore等待UVS的最大响应时间为30s，如果import任务发送给UVS后30秒内得不到响应，就会返回import失败给用户。 + +6. [urma_rjetty_t](#_ZH-CN_TOPIC_0000002489912752-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_rjetty { + urma_jetty_id_t jetty_id; + urma_transport_mode_t trans_mode; + urma_jetty_grp_policy_t policy; + urma_target_type_t type; + urma_import_jetty_flag_t flag; + urma_tp_type_t tp_type; +} urma_rjetty_t; +``` + +##### 2.3.1.6.7 urma_import_jetty_ex + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[urma_target_jetty_t](#_ZH-CN_TOPIC_0000002521992545-chtext) *urma_import_jetty_ex([urma_context_t](#_ZH-CN_TOPIC_0000002489912714-chtext) *ctx, [urma_rjetty_t](#_ZH-CN_TOPIC_0000002489912752-chtext) *rjetty, [3.3.2.1.7](#23217-urma_token_t) [urma_token_t](#23217-urma_token_t) *token_value, [urma_import_jetty_ex_cfg_t](#_ZH-CN_TOPIC_0000002521872549-chtext) *cfg); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +[3.3.1.6.6](#23166-urma_import_jetty) [urma_import_jetty](#23166-urma_import_jetty)的扩展接口，增加参数[urma_import_jetty_ex_cfg_t](#_ZH-CN_TOPIC_0000002521872549-chtext) *cfg。 + +4. 参数 + +@param[in] [Required] ctx: the urma context created before; + +@param[in] [Required] rjetty: information of remote jetty to import, including jetty id and trans_mode,trans_mode same to create_jetty trans_mode; + +@param[in] [Required] token_value: token to put into output jetty protection table; + +@param[in] [Required] cfg: tp active configuration to exchange with target; + +5. 返回值 + +Return: the address of target jetty, not NULL on success, NULL on error. + +6. [urma_import_jetty_ex_cfg_t](#_ZH-CN_TOPIC_0000002521872549-chtext) + +typedef struct [urma_active_tp_cfg_t](#_ZH-CN_TOPIC_0000002525470775-chtext) urma_import_jetty_ex_cfg_t; + +##### 2.3.1.6.8 urma_unimport_jetty + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_unimport_jetty([urma_target_jetty_t](#_ZH-CN_TOPIC_0000002521992545-chtext) *tjetty); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +unimport远端Jetty信息。操作成功后，进程不可访问这个远端jetty。支持多线程操作重入操作。 + +4. 参数 + +@param[in] [Required] tjetty: the target jetty to unimport; + +![](figures/urma_info.png) + +由调用者保证参数tjetty来自[3.3.1.6.6](#23166-urma_import_jetty) [urma_import_jetty](#23166-urma_import_jetty)接口返回，参数内部指针等合法性由这些接口保证，本接口不再重复进行校验；否则可能导致调用者进程异常退出。 + +5. 返回值 + +Return: 0 on success, other value on error. + +##### 2.3.1.6.9 urma_bind_jetty + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_bind_jetty([urma_jetty_t](#_ZH-CN_TOPIC_0000002489912746-chtext) *jetty, [urma_target_jetty_t](#_ZH-CN_TOPIC_0000002521992545-chtext) *tjetty) + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +绑定远端target Jetty，建立连接。 + +![](figures/urma_caution.png) + +1\. bind的Jetty和target Jetty必须是RC模式，不支持非RC模式Jetty间进行bind。 + +2\. 只能和一个target Jetty成功建立连接；如需切换target Jetty进行建立新的连接，需要先unbind操作，成功后再与target Jetty建立新的连接关系。即连接是本地的Jetty和远端target Jetty一对一的关系，不允许多对一或者一对多的连接关系。 + +3\. 连接成功后，从Jetty发出的所有消息将发送到tjetty指定的节点。 + +4\. 支持多线程操作重入操作。 + +4. 参数 + +@param[in] [Required] jetty: local jetty to construct the transport channel; + +@param[in] [Required] tjetty: target jetty imported before; + +5. 返回值 + +Return: 0 on success, URMA_EEXIST if the jetty has been binded, other value on error. + +![](figures/urma_info.png) + +1\. jetty或者tjetty的模式不为RC，返回URMA_ENOPERM错误。 + +2\. jetty或者tjetty为NULL，返回URMA_EINVAL。 + +3\. 对jetty和tjetty的重入操作，返回URMA_SUCCESS。 + +##### 2.3.1.6.10 urma_bind_jetty_ex + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_bind_jetty_ex([urma_jetty_t](#_ZH-CN_TOPIC_0000002489912746-chtext) *jetty, [urma_target_jetty_t](#_ZH-CN_TOPIC_0000002521992545-chtext) *tjetty, [urma_bind_jetty_ex_cfg_t](#_ZH-CN_TOPIC_0000002524072167-chtext) *cfg); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +[3.3.1.6.9](#23169-urma_bind_jetty) [urma_bind_jetty](#23169-urma_bind_jetty)扩展接口，增加参数urma_bind_jetty_ex_cfg_t *cfg。 + +4. 参数 + +@param[in] [Required] jetty: local jetty to construct the transport channel; + +@param[in] [Required] tjetty: target jetty imported before; + +@param[in] [Required] cfg: tp active configuration to exchange with target; + +5. 返回值 + +Return: 0 on success, URMA_EEXIST if the jetty has been binded, other value on error. + +6. [urma_bind_jetty_ex_cfg_t](#_ZH-CN_TOPIC_0000002524072167-chtext) + +typedef struct [urma_active_tp_cfg_t](#_ZH-CN_TOPIC_0000002525470775-chtext) urma_bind_jetty_ex_cfg_t; + +##### 2.3.1.6.11 urma_unbind_jetty + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_unbind_jetty([urma_jetty_t](#_ZH-CN_TOPIC_0000002489912746-chtext) *jetty) + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +解绑远端Jetty，断开连接。 + +![](figures/urma_caution.png) + +1\. unbind的Jetty必须是RC模式，不支持非RC模式Jetty进行unbind。 + +2\. 解绑定成功后，Jetty无法发送任何消息，需等待重新和某个target Jetty建立新的绑定关系，才可以通过该Jetty发送消息。 + +3\. 支持多线程操作重入操作 + +4. 参数 + +@param[in] [Required] jetty: local jetty to deconstruct the transport channel; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### 2.3.1.6.12 urma_flush_jetty + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +int urma_flush_jetty([urma_jetty_t](#_ZH-CN_TOPIC_0000002489912746-chtext) *jetty, int cr_cnt, [3.4.2.1.1](#24211-urma_cr_t) [urma_cr_t](#24211-urma_cr_t) *cr); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +Jetty状态切为Error态，或poll到status为URMA_CR_WR_SUSPEND_DONE的cr后调用，将post给Jetty但未完成的wr，以cr的形式poll回来。 + +4. 参数 + +@param[in] [Required] jetty: local jetty to deconstruct the transport channel; + +@param[in] [Required] cr_cnt: Number of CR expected to be received; + +@param[out] [Required] cr: Address for storing CR; + +5. 返回值 + +Return: the number of CR returned, 0 means no CR returned, -1 on error. + +![](figures/urma_notice.png) + +若执行成功，则出参cr status为URMA_CR_WR_FLUSH_ERR。 + +##### 2.3.1.6.13 urma_import_jetty_async + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[urma_target_jetty_t](#_ZH-CN_TOPIC_0000002521992545-chtext) *urma_import_jetty_async([urma_notifier_t](#_ZH-CN_TOPIC_0000002524152205-chtext) *notifier, const [urma_rjetty_t](#_ZH-CN_TOPIC_0000002489912752-chtext) *rjetty, const [3.3.2.1.7](#23217-urma_token_t) [urma_token_t](#23217-urma_token_t) *token_value, uint64_t user_ctx, int timeout); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +[3.3.1.6.6](#23166-urma_import_jetty) [urma_import_jetty](#23166-urma_import_jetty)的异步版本。 + +4. 参数 + +@param[in] [Required] notifier: data structure used for sensing asynchronous link establishment results; + +@param[in] [Required] rjetty: information of remote jetty to import, including jetty id and trans_mode, trans_mode same to create_jetty trans_mode; + +@param[in] [Required] token_value: token to put into output jetty protection table; + +@param[in] [Required] user_ctx: user_ctx create by user; + +@param[in] [Required] timeout: task timeout set by user (milliseconds); + +5. 返回值 + +Return: the address of target jetty, not NULL on success, NULL on error. + +6. [urma_notifier_t](#_ZH-CN_TOPIC_0000002524152205-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_notifier { + urma_context_t *urma_ctx; + int fd; + void *incomplete_tjetty_list; +} urma_notifier_t; +``` + +##### 2.3.1.6.14 urma_unimport_jetty_async + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_unimport_jetty_async([urma_target_jetty_t](#_ZH-CN_TOPIC_0000002521992545-chtext) *tjetty); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +[3.3.1.6.8](#23168-urma_unimport_jetty) [urma_unimport_jetty](#23168-urma_unimport_jetty)的异步版本。 + +4. 参数 + +@param[in] [Required] tjetty: the target jetty to unimport; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### 2.3.1.6.15 urma_bind_jetty_async + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_bind_jetty_async([urma_notifier_t](#_ZH-CN_TOPIC_0000002524152205-chtext) *notifier, [urma_jetty_t](#_ZH-CN_TOPIC_0000002489912746-chtext) *jetty, [urma_target_jetty_t](#_ZH-CN_TOPIC_0000002521992545-chtext) *tjetty, uint64_t user_ctx, int timeout); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +[3.3.1.6.9](#23169-urma_bind_jetty) [urma_bind_jetty](#23169-urma_bind_jetty)的异步版本。 + +4. 参数 + +@param[in] [Required] notifier: data structure used for sensing asynchronous link establishment results; + +@param[in] [Required] jetty: local jetty to construct the transport channel; + +@param[in] [Required] tjetty: target jetty imported before; + +@param[in] [Required] user_ctx: user_ctx create by user; + +@param[in] [Required] timeout: task timeout set by user (milliseconds); + +5. 返回值 + +Return: 0 on success, URMA_EEXIST if the jetty has been binded, other value on error. + +##### 2.3.1.6.16 urma_unbind_jetty_async + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_unbind_jetty_async([urma_jetty_t](#_ZH-CN_TOPIC_0000002489912746-chtext) *jetty)； + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +支持异步unbind。 + +解绑定远端Jetty。 + +1）unbind的Jetty必须是RC模式，不支持非RC模式Jetty进行unbind。 + +2）解绑定成功后，Jetty无法发送任何消息，需等待重新和某个target Jetty建立新的绑定关系，才可以通过该Jetty发送消息。 + +支持多线程操作重入操作。 + +4. 参数 + +@param[in] [Required] jetty: local jetty to deconstruct the transport channel; + +5. 返回值 + +Return: 0 on success, other value on error + +![](figures/urma_notice.png) + +由调用者保证参数jetty来自[3.3.1.6.1](#23161-urma_create_jetty) [urma_create_jetty](#23161-urma_create_jetty)接口返回，tjetty来自[3.3.1.6.16](#231616-urma_unbind_jetty_async) [urma_unbind_jetty_async](#231616-urma_unbind_jetty_async)接口返回，参数内部指针等合法性由这些接口保证，本接口不再重复进行校验；否则可能导致调用者进程异常退出。 + +##### 2.3.1.6.17 urma_create_notifier + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[urma_notifier_t](#_ZH-CN_TOPIC_0000002524152205-chtext) *urma_create_notifier([urma_context_t](#_ZH-CN_TOPIC_0000002489912714-chtext) *ctx); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +创建通知异步建链结果的结构体变量。 + +4. 参数 + +@param[in] [Required] ctx: the urma context created before; + +5. 返回值 + +Return: the address of urma notifier, not NULL on success, NULL on error. + +##### 2.3.1.6.18 urma_delete_notifier + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_delete_notifier([urma_notifier_t](#_ZH-CN_TOPIC_0000002524152205-chtext) *notifier); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +删除通知异步建链结果的结构体变量。 + +4. 参数 + +@param[in] [Required] notifier: data structure used for sensing asynchronous link establishment results; + +![](figures/urma_notice.png) + +由调用者保证参数jetty来自[3.3.1.6.17](#231617-urma_create_notifier) [urma_create_notifier](#231617-urma_create_notifier)接口返回，参数内部指针等合法性由这些接口保证，本接口不再重复进行校验；否则可能导致调用者进程异常退出。 + +5. 返回值 + +Return: 0 on success, other value on error. + +##### 2.3.1.6.19 urma_wait_notify + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +int urma_wait_notify([urma_notifier_t](#_ZH-CN_TOPIC_0000002524152205-chtext) *notifier, uint32_t cnt, [urma_notify_t](#_ZH-CN_TOPIC_0000002492112460-chtext) *notify, int timeout); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +等待异步建链结果。 + +4. 参数 + +@param[in] [Required] notifier: data structure used for sensing asynchronous link establishment results; + +@param[in] [Required] cnt: expected number of target jetty to return; + +@param[out] [Required] notify: created by user to store target jetty results; + +@param[in] [Required] timeout: max time to wait (milliseconds), timeout = 0: return immediately even if no events are ready, timeout = -1: an infinite timeout; + +5. 返回值 + +Return: the number of target jetty returned, 0 means no target jetty returned, -1 on error. + +6. [urma_notify_t](#_ZH-CN_TOPIC_0000002492112460-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_notify { + urma_notify_type_t type; + urma_status_t status; + uint64_t user_ctx; + union { + urma_target_jetty_t *tjetty; /* IMPORT */ + urma_jetty_t *jetty; /* BIND */ + }; +} urma_notify_t; +``` + +7. [urma_notify_type_t](#_ZH-CN_TOPIC_0000002524072171-chtext) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef enum urma_notify_type { + URMA_IMPORT_JETTY_NOTIFY = 0, + URMA_BIND_JETTY_NOTIFY +} urma_notify_type_t; +``` + +##### 2.3.1.6.20 urma_ack_notify + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_ack_notify([urma_context_t](#_ZH-CN_TOPIC_0000002489912714-chtext) *ctx, uint32_t cnt, [urma_notify_t](#_ZH-CN_TOPIC_0000002492112460-chtext) *notify); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +This interface is no longer functional and will be removed later. + +Keep parameter checks to ensure the function works as before. + +4. 参数 + +@param[in] [Required] ctx: the created urma context pointer; + +@param[in] [Required] cnt: notify array count; + +@param[in] [Required] notify: notify array to be acknowledged; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### urma_alloc_jetty + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +urma_status_t urma_alloc_jetty(urma_context_t *urma_ctx, urma_jetty_cfg_t *cfg, urma_jetty_t **jetty); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +分配Jetty资源。Jetty是一对JFS和JFR的集合。 + +4. 参数 + +@param[in] [Required] ctx: the urma context created before; + +@param[in] [Required] cfg: configuration of jetty; + +@param[out] [Required] jetty: the address to put the handle of allocated jetty; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### urma_free_jetty + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +urma_status_t urma_free_jetty(urma_jetty_t *jetty); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +释放已分配的Jetty。释放后，jetty指针不再允许被访问。 + +4. 参数 + +@param[in] [Required] jetty: handle of the allocated jetty; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### urma_active_jetty + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +urma_status_t urma_active_jetty(urma_jetty_t *jetty); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +激活已分配的Jetty。激活后Jetty才能进行数据传输。 + +4. 参数 + +@param[in] [Required] jetty: handle of the allocated jetty; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### urma_deactive_jetty + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +urma_status_t urma_deactive_jetty(urma_jetty_t *jetty); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +去激活已激活的Jetty。 + +4. 参数 + +@param[in] [Required] jetty: handle of the allocated jetty; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### urma_set_jetty_opt + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +urma_status_t urma_set_jetty_opt(urma_jetty_t *jetty, uint64_t opt, void *buf, uint32_t len); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +设置Jetty选项。 + +4. 参数 + +@param[in] [Required] jetty: handle of the allocated jetty; + +@param[in] [Required] opt: the opt to change cfg of jetty; + +@param[in] [Required] len: the len of the opt value (byte); + +@param[in] [Required] buf: the buffer to store the value; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### urma_get_jetty_opt + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +urma_status_t urma_get_jetty_opt(urma_jetty_t *jetty, uint64_t opt, void *buf, uint32_t len); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +获取Jetty选项。 + +4. 参数 + +@param[in] [Required] jetty: handle of the allocated jetty; + +@param[in] [Required] opt: the opt to change cfg of jetty; + +@param[in] [Required] len: the len of the opt value (byte); + +@param[out] [Required] buf: the buffer to store the value; + +5. 返回值 + +Return: 0 on success, other value on error. + +#### 2.3.1.7 Jetty Group + +Jetty group管理API包括创建、删除Jetty group等API。 + +![](figures/urma_warning.png) + +部分芯片不支持Jetty group管理的API。 + +##### 2.3.1.7.1 urma_create_jetty_grp + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[urma_jetty_grp_t](#_ZH-CN_TOPIC_0000002524152201-chtext) *urma_create_jetty_grp([urma_context_t](#_ZH-CN_TOPIC_0000002489912714-chtext) *ctx, [urma_jetty_grp_cfg_t](#_ZH-CN_TOPIC_0000002527065929-chtext) *cfg)； + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +用户创建一个指定名称的Jetty group。Jetty group是多个Jetty的组合，组合中的Jetty必须是同一个节点上的资源。 + +4. 参数 + +@param[in] [Required] ctx: the urma context created before; + +@param[in] [Required] cfg: pointer of the jetty group config; + +5. 返回值 + +Return: 若创建成功，返回非空的urma_jetty_grp_t指针；若同名的Jetty group已经存在，返回已经创建的非空的urma_jetty_grp_t指针。若创建失败，则返回NULL。 + +##### 2.3.1.7.2 urma_delete_jetty_grp + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_delete_jetty_grp([urma_jetty_grp_t](#_ZH-CN_TOPIC_0000002524152201-chtext) *jetty_grp); + +定义文件: [urma_api.h](../../../src/urma/lib/urma/core/include/urma_api.h) + +3. 描述 + +销毁Jetty group。默认只允许Jetty group owner调用该接口。 + +4. 参数 + +@param[in] [Required] jetty_grp: the Jetty group created before; + +![](figures/urma_notice.png) + +由调用者保证参数jetty_grp来自[3.3.1.7.1](#23171-urma_create_jetty_grp) [urma_create_jetty_grp](#23171-urma_create_jetty_grp)接口返回，参数内部指针等合法性由这些接口保证，本接口不再重复进行校验；否则可能导致调用者进程异常退出。 + +5. 返回值 + +Return: 0 on success, other value on error. + +### 2.3.2 Segment + +segment API包括注册和反注册segment，导入和反导入segment等API。 + +#### 2.3.2.1 urma_register_seg + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.3.2.1.3](#23213-urma_target_seg_t) [urma_target_seg_t](#23213-urma_target_seg_t) *urma_register_seg([urma_context_t](#_ZH-CN_TOPIC_0000002489912714-chtext) *ctx, [3.3.2.1.1](#23211-urma_seg_cfg_t) [urma_seg_cfg_t](#23211-urma_seg_cfg_t) *seg_cfg); + +3. 描述 + +向function entity设备注册本端segment信息（包括内存段的UBVA地址和长度）。 + +4. 参数 + +@param[in] [Required] ctx: the created urma context pointer; + +@param[in] [Required] seg_cfg: Specify cfg of seg to be registered, including address, len, key, and so on; + +![](figures/urma_caution.png) + +1\. 鲲鹏950芯片UBMMU模块支持应用不指定key，由ubmmu内部自动产生key。 + +2\. 用户提供的va不可为0。 + +5. 返回值 + +Return: pointer to target segment on success, NULL on error. + +##### 2.3.2.1.1 urma_seg_cfg_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_seg_cfg { + uint64_t va; /* specify the address of the segment to be registered */ + uint64_t len; /* specify the length of the segment to be registered */ + urma_token_id_t *token_id; + urma_token_t token_value; /* Security authentication for access */ + urma_reg_seg_flag_t flag; + uint64_t user_ctx; + uint64_t iova; /* user iova, maybe zero-based-address */ +} urma_seg_cfg_t; +``` + +##### 2.3.2.1.2 urma_reg_seg_flag_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef union urma_reg_seg_flag { + struct { + uint32_t token_policy : 3; /* 0: URMA_TOKEN_NONE. + 1: URMA_TOKEN_PLAIN_TEXT. + 2: URMA_TOKEN_SIGNED. + 3: URMA_TOKEN_ALL_ENCRYPTED. + 4: URMA_TOKEN_RESERVED. */ + uint32_t cacheable : 1; /* 0: URMA_NON_CACHEABLE. + 1: URMA_CACHEABLE. */ + uint32_t dsva : 1; + uint32_t access : 6; /* (0x1): URMA_ACCESS_LOCAL_ONLY. + (0x1 << 1): URMA_ACCESS_READ. + (0x1 << 2): URMA_ACCESS_WRITE. + (0x1 << 3): URMA_ACCESS_ATOMIC. */ + uint32_t non_pin : 1; /* 0: segment pages pinned. + 1: segment pages non-pinned. */ + uint32_t user_iova : 1; /* 0: segment without user iova addr. + 1: segment with user iova addr. */ + uint32_t token_id_valid : 1; /* 0: token id in cfg is invalid. + 1: token id in cfg is valid. */ + uint32_t reserved : 18; + } bs; + uint32_t value; +} urma_reg_seg_flag_t; +``` + +![](figures/urma_caution.png) + +关于urma_reg_seg_flag_t： + +1\. 无论用户指定任何标识，segment权限默认具有本端读、写和原子操作权限； + +2\. 指定URMA_ACCESS_LOCAL_ONLY标识的情况下，不允许再指定其他标识，否则urma将会拦截此错误配置； + +3\. 只有不指定URMA_ACCESS_LOCAL_ONLY，才允许指定其他标识，此时segment权限除默认的本端读、写和原子操作权限，远端权限按照用户配置生效； + +4\. 配置URMA_ACCESS_WRITE标识，必须也配置URMA_ACCESS_READ标识，否则urma将会拦截此错误配置； + +5\. 配置URMA_ACCESS_ATOMIC标识，必须也配置URMA_ACCESS_READ和URMA_ACCESS_WRITE标识，否则urma将会拦截此错误配置。 + +##### 2.3.2.1.3 urma_target_seg_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_target_seg { + urma_seg_t seg; /* [Private] see urma_seg_t. */ + uint64_t user_ctx; /* [Private] private data of segment */ + uint64_t mva; /* [Public] mapping addr when import remote seg. */ + urma_context_t *urma_ctx; /* [Private] point to urma context. */ + urma_token_id_t *token_id; /* When registering seg, it is a valid address; when importing seg, it is NULL */ + uint64_t handle; +} urma_target_seg_t; +``` + +##### 2.3.2.1.4 urma_seg_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_seg { + urma_ubva_t ubva; /* [Public] ubva of segment. */ + uint64_t len; /* [Public] length of segment. */ + urma_seg_attr_t attr; /* [Public] include: access flag, token policy, cacheability. */ + uint32_t token_id; /* [Private] match token */ +} urma_seg_t; +``` + +##### 2.3.2.1.5 urma_ubva_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_ubva { + urma_eid_t eid; + uint32_t uasid; // 24 bit for UB; 16 bit for IB + uint64_t va; +} \_\_attribute\_\_((packed)) urma_ubva_t; +``` + +##### 2.3.2.1.6 urma_seg_attr_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef union urma_seg_attr { + struct { + uint32_t token_policy : 3; /* 0: URMA_TOKEN_NONE. + 1: URMA_TOKEN_PLAIN_TEXT. + 2: URMA_TOKEN_SIGNED. + 3: URMA_TOKEN_ALL_ENCRYPTED. + 4: URMA_TOKEN_RESERVED. */ + uint32_t cacheable : 1; /* 0: URMA_NON_CACHEABLE. + 1: URMA_CACHEABLE. */ + uint32_t dsva : 1; + uint32_t access : 6; /* (0x1): URMA_ACCESS_LOCAL_WRITE. + (0x1 << 1): URMA_ACCESS_REMOTE_READ. + (0x1 << 2): URMA_ACCESS_REMOTE_WRITE. + (0x1 << 3): URMA_ACCESS_REMOTE_ATOMIC. + (0x1 << 4): URMA_ACCESS_REMOTE_INVALIDATE. */ + uint32_t non_pin : 1; /* 0: segment pages pinned. + 1: segment pages non-pinned. */ + uint32_t user_iova : 1; /* 0: segment without user iova addr. + 1: segment with user iova addr. */ + uint32_t user_token_id : 1; /* 0: token_id is allocated and should be freed by urma. + 1: token_id is allocated by user in urma_seg_cfg. */ + uint32_t reserved : 18; + } bs; + uint32_t value; +} urma_seg_attr_t; +``` + +##### 2.3.2.1.7 urma_token_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_token { + uint32_t token; +} urma_token_t; +``` + +#### 2.3.2.2 urma_unregister_seg + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_unregister_seg([3.3.2.1.3](#23213-urma_target_seg_t) [urma_target_seg_t](#23213-urma_target_seg_t) *target_seg) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +3. 描述 + +反注册本端segment。 + +4. 参数 + +@param[in] [Required] target_seg: target segment to be unregistered; + +![](figures/urma_notice.png) + +由调用者保证参数target_seg来自[3.3.2.1](#2321-urma_register_seg) [urma_register_seg](#2321-urma_register_seg)接口返回，参数内部指针等合法性由这些接口保证，本接口不再重复进行校验；否则可能导致调用者进程异常退出。 + +5. 返回值 + +Return: 0 on success, other value on error. + +#### 2.3.2.3 urma_import_seg + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.3.2.1.3](#23213-urma_target_seg_t) [urma_target_seg_t](#23213-urma_target_seg_t) *urma_import_seg([urma_context_t](#_ZH-CN_TOPIC_0000002489912714-chtext) *ctx, [3.3.2.1.4](#23214-urma_seg_t) [urma_seg_t](#23214-urma_seg_t) *seg, [3.3.2.1.7](#23217-urma_token_t) [urma_token_t](#23217-urma_token_t) *token_value, uint64_t addr, [3.3.2.3.1](#23231-urma_import_seg_flag_t) [urma_import_seg_flag_t](#23231-urma_import_seg_flag_t) flag); + +3. 描述 + +向function entity设备注册对端segment信息（包括内存段的UBVA地址和长度）。 + +![](figures/urma_info.png) + +1\. 该接口将填好MMU页表和和UBMMU页表。 + +ctx用于定位TPA的范围，找到对应FUNCTION ENTITY设备上的TPA-UBVA的映射表，并填写。 + +填写MVA-UBVA不依赖于ctx，该表是共享的表。 + +2\. flag若指定映射segment的空间到本地进程空间，将调用mmap分配进程的VA。 + +3\. 支持多线程操作重入操作。 + +4. 参数 + +@param[in] [Required] ctx: the created urma context pointer; + +@param[in] [Required] seg: handle of memory segment to import; + +@param[in] [Required] token_value: token to put into output protection table; + +@param[in] [Optional] addr: the virtual address to which the segment will be mapped; + +@param[in] [Required] flag: flag to indicate the import attribute of memory segment; + +5. 返回值 + +Return: pointer to target segment on success, NULL on error. + +##### 2.3.2.3.1 urma_import_seg_flag_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef union urma_import_seg_flag { + struct { + uint32_t cacheable : 1; /* 0: URMA_NON_CACHEABLE. + 1: URMA_CACHEABLE. */ + uint32_t access : 6; /* (0x1): URMA_ACCESS_LOCAL_WRITE. + (0x1 << 1): URMA_ACCESS_REMOTE_READ. + (0x1 << 2): URMA_ACCESS_REMOTE_WRITE. + (0x1 << 3): URMA_ACCESS_REMOTE_ATOMIC. + (0x1 << 4)：URMA_ACCESS_REMOTE_INVALIDATE. + */ + uint32_t mapping : 1; /* 0: URMA_SEG_NOMAP/ + 1: URMA_SEG_MAPPED. */ + uint32_t reserved : 24; + } bs; + uint32_t value; +} urma_import_seg_flag_t; +``` + +#### 2.3.2.4 urma_unimport_seg + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_unimport_seg([3.3.2.1.3](#23213-urma_target_seg_t) [urma_target_seg_t](#23213-urma_target_seg_t) *tseg); + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +3. 描述 + +unimport对端segment信息。unimport成功后，应用不可继续访问该segment。 + +若segment的空间存在到本地进程的映射。解除过程将调用free释放MVA。解除成功后，MVA不可继续访问。 + +支持多线程操作重入操作。 + +4. 参数 + +@param[in] [Required] tseg: the address of the target segment to unimport; + +5. 返回值 + +Return: 0 on success, other value on error. + +![](figures/urma_notice.png) + +由调用者保证参数tseg来自[3.3.2.3](#2323-urma_import_seg) [urma_import_seg](#2323-urma_import_seg)接口返回，参数内部指针等合法性由这些接口保证，本接口不再重复进行校验；否则可能导致调用者进程异常退出。 + +### 2.3.3 TP Channel + +TP层传输通道资源TP Channel相关接口。 + +#### 2.3.3.1 urma_get_tpn + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +int urma_get_tpn([urma_jetty_t](#_ZH-CN_TOPIC_0000002489912746-chtext) *jetty); + +3. 描述 + +根据用户传入的jetty指针，从驱动中获取对应的tpn。 + +![](figures/urma_info.png) + +该接口只用于UB协议的用户建链方案，应当在调用[3.3.1.6.1](#23161-urma_create_jetty) [urma_create_jetty](#23161-urma_create_jetty)接口之后调用。 + +4. 参数 + +@param[in] [Required] jetty: the created jetty pointer + +5. 返回值 + +Return: \>= 0 on success, return as tpn; < 0 on error。 + +#### 2.3.3.2 urma_modify_tp + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +int urma_modify_tp([urma_context_t](#_ZH-CN_TOPIC_0000002489912714-chtext) *ctx, uint32_t tpn, [3.3.3.2.1](#23321-urma_tp_cfg_t) [urma_tp_cfg_t](#23321-urma_tp_cfg_t) *cfg, [3.3.3.2.3](#23323-urma_tp_attr_t) [urma_tp_attr_t](#23323-urma_tp_attr_t) *attr, [3.3.3.2.6](#23326-urma_tp_attr_mask_t) [urma_tp_attr_mask_t](#23326-urma_tp_attr_mask_t) mask); + +3. 描述 + +修改tp状态。在bind jetty后调用。 + +4. 参数 + +@param[in] ctx: the created urma context pointer; + +@param[in] tpn: tpn of tp created before; + +@param[in] cfg: tp configurations filled by user; + +@param[in] attr: tp attributes filled by user; + +@param[in] mask: bitmap configurations for tp attributes + +5. 返回值 + +Return: 0 on success; other values on error. + +##### 2.3.3.2.1 urma_tp_cfg_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_tp_cfg { + urma_tp_cfg_flag_t flag; /* flag of initial tp */ + /* transport layer attributes */ + urma_transport_mode_t trans_mode; + uint8_t retry_num; + uint8_t retry_factor; /* for calculate the time slot to retry */ + uint8_t ack_timeout; + uint8_t dscp; + uint32_t oor_cnt; /* OOR window size: by packet */ +} urma_tp_cfg_t; +``` + +##### 2.3.3.2.2 urma_tp_cfg_flag_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef union urma_tp_cfg_flag { + struct { + uint32_t target : 1; /* 0: initiator, 1: target */ + uint32_t loopback : 1; + uint32_t dca_enable : 1; + /* for the bonding case, the hardware selects the port + * ignoring the port of the tp context and + * selects the port based on the hash value + * along with the information in the bonding group table. + */ + uint32_t bonding : 1; + uint32_t reserved : 28; + } bs; + uint32_t value; +} urma_tp_cfg_flag_t; +``` + +##### 2.3.3.2.3 urma_tp_attr_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_tp_attr { + urma_tp_mod_flag_t flag; + uint32_t peer_tpn; + urma_tp_state_t state; + uint32_t tx_psn; + uint32_t rx_psn; + urma_mtu_t mtu; + uint8_t cc_pattern_idx; + uint32_t oos_cnt; /* out of standing packet cnt */ + uint32_t local_net_addr_idx; + urma_net_addr_t peer_net_addr; + uint16_t data_udp_start; + uint16_t ack_udp_start; + uint8_t udp_range; + uint8_t hop_limit; + uint32_t flow_label; + uint8_t port_id; + uint8_t mn; /* 0\~15, a packet contains only one msg if mn is set as 0 */ + urma_transport_type_t peer_trans_type; +} urma_tp_attr_t; +``` + +##### 2.3.3.2.4 urma_tp_mod_flag_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef union urma_tp_mod_flag { + struct { + uint32_t oor_en : 1; /* out of order receive, 0: disable 1: enable */ + uint32_t sr_en : 1; /* selective retransmission, 0: disable 1: enable */ + uint32_t cc_en : 1; /* congestion control algorithm, 0: disable 1: enable */ + uint32_t cc_alg : 4; /* The value is ubcore_tp_cc_alg_t */ + uint32_t spray_en : 1; /* spray with src udp port, 0: disable 1: enable */ + uint32_t clan : 1; /* clan domain, 0: disable 1: enable */ + uint32_t reserved : 23; + } bs; + uint32_t value; +} urma_tp_mod_flag_t; +``` + +##### 2.3.3.2.5 urma_tp_state_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef enum urma_tp_state { + URMA_TP_STATE_RESET = 0, + URMA_TP_STATE_PASSIVE, + URMA_TP_STATE_ACTIVE, + URMA_TP_STATE_BRAKE, + URMA_TP_STATE_ERROR +} urma_tp_state_t; +``` + +##### 2.3.3.2.6 urma_tp_attr_mask_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef union urma_tp_attr_mask { + struct { + uint32_t flag : 1; + uint32_t peer_tpn : 1; + uint32_t state : 1; + uint32_t tx_psn : 1; + uint32_t rx_psn : 1; /* modify both rx psn and tx psn when restore tp */ + uint32_t mtu : 1; + uint32_t cc_pattern_idx : 1; + uint32_t oos_cnt : 1; + uint32_t local_net_addr_idx : 1; + uint32_t peer_net_addr : 1; + uint32_t data_udp_start : 1; + uint32_t ack_udp_start : 1; + uint32_t udp_range : 1; + uint32_t hop_limit : 1; + uint32_t flow_label : 1; + uint32_t port_id : 1; + uint32_t mn : 1; + uint32_t peer_trans_type : 1; + uint32_t reserved : 14; + } bs; + uint32_t value; +} urma_tp_attr_mask_t; +``` + +#### 2.3.3.3 urma_get_tp_list + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_get_tp_list([urma_context_t](#_ZH-CN_TOPIC_0000002489912714-chtext) *ctx, [3.3.3.3.1](#23331-urma_get_tp_cfg_t) [urma_get_tp_cfg_t](#23331-urma_get_tp_cfg_t) *cfg, uint32_t *tp_cnt, [3.3.3.3.3](#23333-urma_tp_info_t) [urma_tp_info_t](#23333-urma_tp_info_t) *tp_list); + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +3. 描述 + +获取可用的tp列表。 + +4. 参数 + +@param[in] [Required] ctx: the created urma context pointer; + +@param[in] [Required] tp_cfg: tp configuration to get; + +@param[in && out] [Required] tp_cnt: tp_cnt is the length of tp_list buffer as in parameter; tp_cnt is the number of tp as out parameter; + +@param[out] [Required] tp_list: tp list to get, the buffer is allocated by user; + +5. 返回值 + +Return: 0 on success, other value on error. + +##### 2.3.3.3.1 urma_get_tp_cfg_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_get_tp_cfg { + urma_get_tp_cfg_flag_t flag; + urma_transport_mode_t trans_mode; + urma_eid_t local_eid; + urma_eid_t peer_eid; +} urma_get_tp_cfg_t; +``` + +##### 2.3.3.3.2 urma_get_tp_cfg_flag_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef union urma_get_tp_cfg_flag { + struct { + uint32_t ctp : 1; + uint32_t rtp : 1; + uint32_t utp : 1; + uint32_t uboe : 1; + uint32_t pre_defined : 1; + uint32_t dynamic_defined : 1; + uint32_t reserved : 26; + } bs; + uint32_t value; +} urma_get_tp_cfg_flag_t; +``` + +##### 2.3.3.3.3 urma_tp_info_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_tp_info { + uint64_t tp_handle; +} urma_tp_info_t; +``` + +#### 2.3.3.4 urma_get_tp_attr + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_get_tp_attr(const [urma_context_t](#_ZH-CN_TOPIC_0000002489912714-chtext) *ctx, const uint64_t tp_handle, uint8_t *tp_attr_cnt, uint32_t *tp_attr_bitmap, [3.3.3.4.1](#23341-urma_tp_attr_value_t) [urma_tp_attr_value_t](#23341-urma_tp_attr_value_t) *tp_attr); + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +3. 描述 + +获取tp属性。 + +4. 参数 + +@param[in] [Required] ctx: the created urma context pointer; + +@param[in] [Required] tp_handle: tp_handle got by urma_get_tp_list; + +@param[in] [Required] tp_attr_cnt: number of tp attributions; + +@param[in] [Required] tp_attr_bitmap: tp attributions bitmap + +@param[in] [Required] tp_attr: tp attribution values to set; + +![](figures/urma_info.png) + +tp_attr_bitmap各bit和tp_attr字段对应关系如下（字段含义见[3.3.3.4.1](#23341-urma_tp_attr_value_t) [urma_tp_attr_value_t](#23341-urma_tp_attr_value_t)）： + +0-retry_times_init: 3 bit 1-at: 5 bit 2-SIP: 128 bit + +3-DIP: 128 bit 4-SMA: 48 bit 5-DMA: 48 bit + +6-vlan_id: 12 bit 7-vlan_en: 1 bit 8-dscp: 6 bit + +9-at_times: 5 bit 10-sl: 4 bit 11-ttl: 8 bit + +5. 返回值 + +Return: 0 on success, other value on error. + +##### 2.3.3.4.1 urma_tp_attr_value_t + +```c +#pragma pack(1) +``` + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_tp_attr_value { + uint8_t retry_times_init : 3; + uint8_t at : 5; // ack timeout + uint8_t sip[URMA_IP_ADDR_BYTES]; // src ip + uint8_t dip[URMA_IP_ADDR_BYTES]; // dst ip + uint8_t sma[URMA_MAC_BYTES]; // src mac + uint8_t dma[URMA_MAC_BYTES]; // dst mac + uint16_t vlan_id : 12; + uint8_t vlan_en : 1; + uint8_t dscp : 6; // differentiated services code point + uint8_t at_times : 5; // ack timeout max times + uint8_t sl : 4; // service level + uint8_t ttl; // time to live + uint8_t reserved[78]; +} urma_tp_attr_value_t; +#pragma pack() +``` + +#### 2.3.3.5 urma_set_tp_attr + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_set_tp_attr(const [urma_context_t](#_ZH-CN_TOPIC_0000002489912714-chtext) *ctx, const uint64_t tp_handle, const uint8_t tp_attr_cnt, const uint32_t tp_attr_bitmap, const [3.3.3.4.1](#23341-urma_tp_attr_value_t) [urma_tp_attr_value_t](#23341-urma_tp_attr_value_t) *tp_attr); + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +3. 描述 + +设置tp属性值。 + +4. 参数 + +@param[in] [Required] ctx: the created urma context pointer; + +@param[in] [Required] tp_handle: tp_handle got by urma_get_tp_list; + +@param[in] [Required] tp_attr_cnt: number of tp attributions; + +@param[in] [Required] tp_attr_bitmap: tp attributions bitmap; + +@param[in] [Required] tp_attr: tp attribution values to set; + +5. 返回值 + +Return: 0 on success, other value on error. + +## 2.4 数据面 + +### 2.4.1 post + +#### 2.4.1.1 urma_post_jfs_wr + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_post_jfs_wr([urma_jfs_t](#_ZH-CN_TOPIC_0000002489752746-chtext) *jfs, [3.4.1.1.1](#24111-urma_jfs_wr_t) [urma_jfs_wr_t](#24111-urma_jfs_wr_t) *wr, [3.4.1.1.1](#24111-urma_jfs_wr_t) [urma_jfs_wr_t](#24111-urma_jfs_wr_t) **bad_wr); + +3. 描述 + +发起单边、双边或者原子操作的请求。待操作成功后，应用可poll JFC获得完成消息。 + +可使用JFS关联的JFC查询CR。 + +可以指定ordering和其他的flag。 + +支持多线程操作重入操作。 + +4. 参数 + +@param[in] jfs: the jfs created before, which is used to put command; + +@param[in] wr: the posting request all information, including src addr, dst addr, len, jfc, flag, ordering etc. + +@param[out] bad_wr: the first of failure request. + +5. 返回值 + +Return: 0 on success, other value on error + +![](figures/urma_notice.png) + +由调用者保证参数jfs来自[3.3.1.4.1](#23141-urma_create_jfs) [urma_create_jfs](#23141-urma_create_jfs)接口返回，参数内部指针等合法性已由urma_create_jfs接口保证，该接口不再重复进行校验；否则可能导致调用者进程异常退出。 + +##### 2.4.1.1.1 urma_jfs_wr_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_jfs_wr { + urma_opcode_t opcode; + urma_jfs_wr_flag_t flag; + urma_target_jetty_t *tjetty; + uint64_t user_ctx; // completion data + union { + urma_rw_wr_t rw; + urma_send_wr_t send; + urma_cas_wr_t cas; + urma_faa_wr_t faa; + }; + struct urma_jfs_wr_t *next; +} urma_jfs_wr_t; +``` + +##### 2.4.1.1.2 urma_rw_wr_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_rw_wr { + urma_sg_t src; // including total data length. src is local va for write, and remote va for read. + urma_sg_t dst; // dst is remote va for write, and local va for read. + uint8_t target_hint; // required when using jetty group + uint64_t notify_data; // notify data or imm data in host byte order; +} urma_rw_wr_t; +``` + +![](figures/urma_notice.png) + +- 根据UB协议，write和read操作只支持一个远端sge。因此对于write操作，dst.num_sge必须为1，对于read操作，src.num_sge必须为1。超出部分sge可能被网卡忽略，也可能导致数据面操作失败，取决于底层硬件的实现。 + +- urma还支持write_with_notify特性，在将信息写到远端内存后，在远端不产生CQE，也不消耗RQE。对于write_with_notify，dst.num_sge = 2, notify data表示待写入到对端的notify数据，dst[1]表示notify地址和target segment； dst[0]含义与write一致，表示src sg待写入的远端内存信息。支持dst[0]和dst[1]源于不同的target segment。Notify长度固定为8B。 + +##### 2.4.1.1.3 urma_send_wr_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_send_wr { + urma_sg_t src; // including total data length + uint8_t target_hint; // required when using jetty group + uint64_t imm_data; // imm_data in host byte order; + urma_target_seg_t *tseg; /* tseg used only when send with invalidate */ +} urma_send_wr_t; +``` + +##### 2.4.1.1.4 urma_cas_wr_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_cas_wr { + urma_sge_t *dst; // len is the data length of CAS operation + urma_sge_t *src; // local address for destination original value writeback, len represents the buffer length. + union { // Value compared with destination value + uint64_t cmp_data; // When the len <= 8B, it indicates the CMP value. + uint64_t cmp_addr; // When the len \> 8B, it indicates the data address. + }; + union { // If destination value is the same as cmp_data, destination value will be changed to swap_data + uint64_t swap_data; // When the len <= 8B, it indicates the swap value. + uint64_t swap_addr; // When the len \> 8B, it indicates the data address. + }; +} urma_cas_wr_t; +``` + +##### 2.4.1.1.5 urma_faa_wr_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_faa_wr { + urma_sge_t *dst; // len is the data length of FAA operation + urma_sge_t *src; // local address for destination original value writeback, len represents the buffer length. + union { + uint64_t operand; // When the len <= 8B, it indicates the operand value. + uint64_t operand_addr; // When the len \> 8B, it indicates the data address. + }; +} urma_faa_wr_t; +``` + +##### 2.4.1.1.6 urma_opcode_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef enum urma_opcode { + URMA_OPC_WRITE = 0x00, + URMA_OPC_WRITE_IMM = 0x01, + URMA_OPC_WRITE_NOTIFY = 0x02, // not support result will return for URMA_OPC_WRITE_NOTIFY + URMA_OPC_READ = 0x10, + URMA_OPC_CAS = 0x20, + URMA_OPC_SWAP = 0x21, + URMA_OPC_FADD = 0x22, + URMA_OPC_FSUB = 0x23, + URMA_OPC_FAND = 0x24, + URMA_OPC_FOR = 0x25, + URMA_OPC_FXOR = 0x26, + URMA_OPC_SEND = 0x40, // remote JFR/jetty ID + URMA_OPC_SEND_IMM = 0x41, // remote JFR/jetty ID + URMA_OPC_SEND_INVALIDATE = 0x42, // remote JFR/jetty ID and seg token id + URMA_OPC_NOP = 0x51, + URMA_OPC_WRITE_ATOMIC = 0x60, // Non-standard definition of OPCODE + URMA_OPC_LAST +} urma_opcode_t; +``` + +##### 2.4.1.1.7 urma_jfs_wr_flag_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef union urma_jfs_wr_flag { + struct { + uint32_t place_order : 2; /* 0: There is no order with other WR + 1: relax order + 2: strong order + 3: reserve */ /* see urma_order_type_t */ + uint32_t comp_order : 1; /* 0: There is no completion order with othwe WR. + 1: Completion order with previous WR. */ + uint32_t fence : 1; /* 0: There is not fence. + 1: Fence with previous read and atomic WR */ + uint32_t solicited_enable : 1; /* 0: There is not solicited. + 1: solicited. It will trigger an event on remote side */ + uint32_t complete_enable : 1; /* 0: Do not notify local process after the task is complete. + 1: Notify local process after the task is completed. */ + uint32_t inline_flag : 1; /* 0: not inline. + 1: inline data. */ + uint32_t reserved : 25; + } bs; + uint32_t value; +} urma_jfs_wr_flag_t; +``` + +![](figures/urma_caution.png) + +- 如果jetty/jfs是UM模式，那么WR不支持fence, place_order和comp_order + +- 如果jetty/jfs是完成乱序模式，那么WR不支持fence, place_order和comp_order + +- 当Jetty/JFS是RM或RC模式，并且配置成完成保序模式时，如果WR指定了comp_order，代表接收侧cqe上报是保序的，发送的报文会带comp_order标志。 + +##### 2.4.1.1.8 urma_place_order_t + +定义文件: [urma_opcode.h](../../../src/urma/lib/urma/core/include/urma_opcode.h) + +```c +typedef enum urma_place_order{ + URMA_NO_ORDER = 0, // No order + URMA_RELAX_ORDER, // Relax order + URMA_STRONG_ORDER // Strong order +} urma_place_order_t; +``` + +##### 2.4.1.1.9 urma_sge_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_sge { + uint64_t addr; + uint32_t len; + urma_target_seg_t *tseg; +} urma_sge_t; +``` + +##### 2.4.1.1.10 urma_sg_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_sg { + urma_sge_t *sge; + uint32_t num_sge; +} urma_sg_t; +``` + +#### 2.4.1.2 urma_post_jfr_wr + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_post_jfr_wr([3.4.1.2.1](#24121-urma_jfr_wr_t) [urma_jfr_wr_t](#24121-urma_jfr_wr_t)*jfr, [3.4.1.2.1](#24121-urma_jfr_wr_t) [urma_jfr_wr_t](#24121-urma_jfr_wr_t) *wr, [3.4.1.2.1](#24121-urma_jfr_wr_t) [urma_jfr_wr_t](#24121-urma_jfr_wr_t) **bad_wr) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +3. 描述 + +发起接收操作填recv buffer的请求。待接收操作成功后，应用可poll JFC获得完成消息。 + +可使用JFR关联的JFC存放cqe。 + +支持多线程操作重入操作。 + +4. 参数 + +@param[in] jfr: the jfr created before, which is used to put command; + +@param[in] wr: the posting request all information, including sge, flag. + +@param[out] bad_wr: the first of failure request. + +5. 返回值 + +返回执行结果：0 on success, other value on error。 + +![](figures/urma_notice.png) + +由调用者保证参数jfr来自[3.3.1.5.1](#23151-urma_create_jfr) [urma_create_jfr](#23151-urma_create_jfr)接口返回，参数内部指针等合法性已由urma_create_jfr接口保证，该接口不再重复进行校验；否则可能导致调用者进程异常退出。 + +##### 2.4.1.2.1 urma_jfr_wr_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_jfr_wr { + urma_sg_t src; // includeing buffer length + uint64_t user_ctx; // completion data, eg. wr id + struct urma_jfr_wr_t *next; +} urma_jfr_wr_t; +``` + +![](figures/urma_warning.png) + +IB场景下urma_jfr_wr中src包含len为0的sge时对应的行为由网卡驱动和硬件决定，mlnx cx5环境下可能会产生不可预知的错误。 + +#### 2.4.1.3 urma_post_jetty_send_wr + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_post_jetty_send_wr([urma_jetty_t](#_ZH-CN_TOPIC_0000002489912746-chtext) *jetty, [3.4.1.1.1](#24111-urma_jfs_wr_t) [urma_jfs_wr_t](#24111-urma_jfs_wr_t) *wr, [3.4.1.1.1](#24111-urma_jfs_wr_t) [urma_jfs_wr_t](#24111-urma_jfs_wr_t) **bad_wr) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +3. 描述 + +发起单边、双边或者原子操作的请求。待操作成功后，应用可poll JFC获得完成消息。由于Jetty为共享JFR，此接口实际上向Jetty关联的共享JFR提交接收请求。 + +可使用jetty发送通道关联的JFC查询CR。 + +可以指定ordering和其他的flag。 + +支持多线程操作重入操作。 + +4. 参数 + +@param[in] jetty: the jetty created before, which is used to put command; + +@param[in] wr: the posting request all information, including src addr, dst addr, len, jfc, flag, ordering etc. + +@param[out] bad_wr: the first of failure request. + +5. 返回值 + +Return: 0 on success, other value on error + +![](figures/urma_notice.png) + +由调用者保证参数jetty来自[3.3.1.6.1](#23161-urma_create_jetty) [urma_create_jetty](#23161-urma_create_jetty)接口返回，参数内部指针等合法性已由urma_create_jetty接口保证，该接口不再重复进行校验；否则可能导致调用者进程异常退出。 + +#### 2.4.1.4 urma_post_jetty_recv_wr + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_post_jetty_recv_wr([urma_jetty_t](#_ZH-CN_TOPIC_0000002489912746-chtext) *jetty, [3.4.1.2.1](#24121-urma_jfr_wr_t) [urma_jfr_wr_t](#24121-urma_jfr_wr_t) *wr, [3.4.1.2.1](#24121-urma_jfr_wr_t) [urma_jfr_wr_t](#24121-urma_jfr_wr_t)**bad_wr) + +3. 描述 + +发起接收操作填recv buffer的请求。待接收操作成功后，应用可poll JFC获得完成消息。 + +可使用jetty关联的JFC存放cqe。 + +支持多线程操作重入操作。 + +4. 参数 + +@param[in] jetty: the jetty created before, which is used to put command; + +@param[in] wr: the posting request all information, including sge, flag. + +@param[out] bad_wr: the first of failure request. + +5. 返回值 + +Return: 0 on success, other value on error + +![](figures/urma_notice.png) + +由调用者保证参数jetty来自[3.3.1.6.1](#23161-urma_create_jetty) [urma_create_jetty](#23161-urma_create_jetty)接口返回，参数内部指针等合法性已由urma_create_jetty接口保证，该接口不再重复进行校验；否则可能导致调用者进程异常退出。 + +### 2.4.2 poll相关 + +#### 2.4.2.1 urma_poll_jfc + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +int urma_poll_jfc([urma_jfc_t](#_ZH-CN_TOPIC_0000002521872513-chtext) *jfc, int cr_cnt, [3.4.2.1.1](#24211-urma_cr_t) [urma_cr_t](#24211-urma_cr_t) *cr) + +3. 描述 + +调用urma_poll_jfc轮询JFC，轮询的结果返回到参数cr指定的地址中。cr数据结构包括了请求执行的结果，传输的数据长度，错误类型等信息。 + +支持多线程操作重入操作。 + +4. 参数 + +@param[in] jfc: jetty completion queue to poll + +@param[in] cr_cnt: the expected number of completion record to get + +@param[out] cr: the completion record array to fill at least cr_cnt completion records + +5. 返回值 + +Return: the number of completion record returned, 0 means no completion record returned, -1 on error + +6. 备注 + +Note that: at most 16 completion records can be polled for RDMA device + +![](figures/urma_notice.png) + +由调用者保证参数jfc来自[3.3.1.1.1](#23111-urma_create_jfc) [urma_create_jfc](#23111-urma_create_jfc)接口返回，参数内部指针等合法性已由urma_create_jfc接口保证，该接口不再重复进行校验；否则可能导致调用者进程异常退出。 + +由调用者保证参数cr_cnt与cr指定地址个数一致，接口内仅对cr_cnt做非0校验。否则可能导致调用者进程异常退出。 + +##### 2.4.2.1.1 urma_cr_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_cr { + urma_cr_status_t status; + uint64_t user_ctx; // user_ctx related to a work request + urma_cr_opcode_t opcode; // Only for recv + urma_cr_flag_t flag; // indicate notify data or swap data is valid or not + uint32_t completion_len; // The number of bytes transferred + uint32_t local_id; // Local jetty ID, or JFS ID, or JFR ID, depends on flag + urma_jetty_id_t remote_id; // Valid only for receiving CR. The remote jetty where the + // received msg comes from Jetty ID or JFS ID, depends on flag. + union { + uint64_t imm_data; // Valid only for receiving CR: send/write/read with imm. + urma_cr_token_t invalid_token; // Valid only for receiving CR: send with invalidate. + }; + uint32_t tpn ; // TP number or TPG number + uintptr_t user_data; // e.g. use as pointer to local jetty struct. +} urma_cr_t; +``` + +##### 2.4.2.1.2 urma_cr_status_t + +定义文件: [urma_opcode.h](../../../src/urma/lib/urma/core/include/urma_opcode.h) + +```c +typedef enum urma_cr_status { // completion record status + URMA_CR_SUCCESS = 0, + URMA_CR_UNSUPPORTED_OPCODE_ERR, /* Opcode in the WR is not supported */ + URMA_CR_LOC_LEN_ERR, /* Local data too long error */ + URMA_CR_LOC_OPERATION_ERR, /* Local operation err */ + URMA_CR_LOC_ACCESS_ERR, /* Access to local memory error */ + URMA_CR_REM_RESP_LEN_ERR, /* Local Operation Error, with sub-status of Remote Response Length Error */ + URMA_CR_REM_UNSUPPORTED_REQ_ERR, + URMA_CR_REM_OPERATION_ERR, /* Error when target jetty can not complete the operation */ + URMA_CR_REM_ACCESS_ABORT_ERR, /* Error when target jetty access memory error or abort the operation */ + URMA_CR_ACK_TIMEOUT_ERR, /* Retransmission exceeds the maximum number of times */ + URMA_CR_RNR_RETRY_CNT_EXC_ERR, /* RNR retries exceeded the maximum number: remote jfr has no buffer */ + URMA_CR_WR_FLUSH_ERR, /* Jetty in the error state, and the hardware has processed the WR. */ + URMA_CR_WR_SUSPEND_DONE, /* Hardware constructs a fake CQE, and user_ctx is invalid. */ + URMA_CR_WR_FLUSH_ERR_DONE, /* Hardware constructs a fake CQE, and user_ctx is invalid. */ + URMA_CR_WR_UNHANDLED, /* Return of flush jetty/jfs, and the hardware has not processed the WR. */ + URMA_CR_LOC_DATA_POISON, /* Local Data Poison */ + URMA_CR_REM_DATA_POISON, /* Remote Data Poison */ +} urma_cr_status_t; +``` + +##### 2.4.2.1.3 urma_cr_opcode_t + +定义文件: [urma_opcode.h](../../../src/urma/lib/urma/core/include/urma_opcode.h) + +```c +typedef enum urma_cr_opcode { + URMA_CR_OPC_SEND = 0x00, + URMA_CR_OPC_SEND_WITH_IMM, + URMA_CR_OPC_SEND_WITH_INV, + URMA_CR_OPC_WRITE_WITH_IMM, +} urma_cr_opcode_t; +``` + +##### 2.4.2.1.4 urma_cr_flag_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef union urma_cr_flag { + struct { + uint8_t s_r : 1; // Indicate CR stands for sending or receiving, 0: send, 1: recv. + uint8_t jetty : 1; // Indicate CR stands for jetty or jfs/jfr, 0: jfs/jfr, 1: jetty. + uint8_t suspend_done : 1; // Real CR associated with the WR, user_ctx is valid + uint8_t flush_err_done : 1; // Real CR associated with the WR, user_ctx is valid + uint8_t reserved : 4; + } bs; + uint8_t value; +} urma_cr_flag_t; +``` + +##### 2.4.2.1.5 urma_cr_token_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_cr_token { + uint32_t token_id; + urma_token_t token_value; +} urma_cr_token_t; +``` + +#### 2.4.2.2 urma_rearm_jfc + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_rearm_jfc([urma_jfc_t](#_ZH-CN_TOPIC_0000002521872513-chtext) *jfc, bool solicited_only) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +3. 描述 + +重新设置jfc的事件通知机制。在应用调用urma_wait_jfc返回之后需要调用该函数重新设置通知机制 + +4. 参数 + +@param[in] jfc: jetty completion queue to arm to interrupt mode + +@param[in] solicited_only: indicate it will trigger event only for packets with solicited flag. + +5. 返回值 + +Return: 0 on success, other value on error + +![](figures/urma_notice.png) + +由调用者保证参数jfc来自[3.3.1.1.1](#23111-urma_create_jfc) [urma_create_jfc](#23111-urma_create_jfc)接口返回，参数内部指针等合法性已由urma_create_jfc接口保证，该接口不再重复进行校验；否则可能导致调用者进程异常退出。 + +#### 2.4.2.3 urma_wait_jfc + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +int urma_wait_jfc([urma_jfce_t](#_ZH-CN_TOPIC_0000002489752796-chtext) *jfce, uint32_t jfc_cnt, int time_out, [urma_jfc_t](#_ZH-CN_TOPIC_0000002521872513-chtext) *jfc[]) + +3. 描述 + +等待某个JFC产生新的JFCE，该操作阻塞调用进程。若进程被唤醒，则jfc handle返回到指定的地址中。应用需要进一步调用urma_poll_jfc来获得cr + +![](figures/urma_info.png) + +中断模式的要严格保证urma_wait_jfc，[3.4.2.1](#2421-urma_poll_jfc) [urma_poll_jfc](#2421-urma_poll_jfc)和[3.4.2.4](#2424-urma_ack_jfc) [urma_ack_jfc](#2424-urma_ack_jfc)的相对顺序，否则可能会产生意外结果。 + +4. 参数 + +@param[in] jfce: jetty event channel to wait on + +@param[in] jfc_cnt: expected jfc count to return + +@param[in] time_out: max time to wait (milliseconds), + +timeout = 0: return immediately even if no events are ready， + +timeout=-1: an infinite timeout + +@param[out] jfc: address to put the jfc handle + +5. 返回值 + +Return: the number of jfc returned, 0 means no jfc returned, -1 on error + +![](figures/urma_notice.png) + +由调用者保证参数jfce来自[3.3.1.2.1](#23121-urma_create_jfce) [urma_create_jfce](#23121-urma_create_jfce)接口返回，参数内部指针等合法性已由urma_create_jfce接口保证，该接口不再重复进行校验；否则可能导致调用者进程异常退出。 + +由调用者保证jfc_cnt与jfc中地址个数一致，接口内仅校验jfc_cnt非0。否则可能导致调用者进程异常退出。 + +Repeatedly calling this API without calling [urma_poll_jfc] may lead to number of jfc which is larger than expected in IP provider. This error is controllable. + +#### 2.4.2.4 urma_ack_jfc + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +void urma_ack_jfc([urma_jfc_t](#_ZH-CN_TOPIC_0000002521872513-chtext) *jfc[], uint32_t nevents[], uint32_t jfc_cnt) + +3. 描述 + +确认jfc产生的完成事件被处理完毕。 + +4. 参数 + +@param[in] jfc: jfc pointer array to be acknowledged; + +@param[in] nevents: event count array to be acknowledged; + +@param[in] jfc_cnt: number of elements in the array + +5. 返回值 + +Return: void + +![](figures/urma_notice.png) + +由调用者保证参数jfc来自[3.4.2.3](#2423-urma_wait_jfc) [urma_wait_jfc](#2423-urma_wait_jfc)接口返回，参数内部指针等合法性已由urma_wait_jfc接口保证，该接口不再重复进行校验；否则可能导致调用者进程异常退出。 + +由调用者保证参数jfc的地址个数、nevents的数组元素个数和jfc_cnt数量一致，接口内仅校验jfc_cnt非0。否则可能导致调用者进程异常退出。 + +### 2.4.3 read/write + +#### 2.4.3.1 urma_write + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_write([urma_jfs_t](#_ZH-CN_TOPIC_0000002489752746-chtext) *jfs, [urma_target_jetty_t](#_ZH-CN_TOPIC_0000002521992545-chtext)*target_jfr, [3.3.2.1.3](#23213-urma_target_seg_t) [urma_target_seg_t](#23213-urma_target_seg_t) *dst_tseg, [3.3.2.1.3](#23213-urma_target_seg_t) [urma_target_seg_t](#23213-urma_target_seg_t)*src_tseg, uint64_t dst, uint64_t src, uint32_t len, [3.4.1.1.7](#24117-urma_jfs_wr_flag_t) [urma_jfs_wr_flag_t](#24117-urma_jfs_wr_flag_t) flag, uint64_t user_ctx); + +3. 描述 + +调用urma_write发起单边操作的write操作请求，将指定的本地内存起始位置的数据，发送指定字节数据到指定的目的地址。write请求最终完成之前，应用不能修改本地内存内容。待成功把数据写到远端节点后，应用可poll JFC获得完成消息。 + +使用的默认的JFS进行发送，使用JFS关联的JFC存放cqe，ordering和flag均为默认值 + +支持多线程操作重入操作。 + +4. 参数 + +@param[in] jfs: the jfs created before, which is used to put command; + +@param[in] target_jfr: destination jetty receiver; + +@param[in] dst_tseg: the dst target seg imported before; + +@param[in] src_tseg: the src target seg registered before; + +@param[in] dst: destination address(mapping va on user node or rva in ubva on home node) to be written into + +@param[in] src: source address(local process address space) to fetch data + +@param[in] len: the data len to be written + +@param[in] flag: flag to control jfs work request attribute + +@param[in] user_ctx: the user context, such as request id(rid) etc. + +5. 返回值 + +Return: 0 on success, other value on error + +![](figures/urma_notice.png) + +由调用者保证参数jfs来自[3.3.1.4.1](#23141-urma_create_jfs) [urma_create_jfs](#23141-urma_create_jfs)接口返回，dst_tseg来自[3.3.2.3](#2323-urma_import_seg) [urma_import_seg](#2323-urma_import_seg)接口返回，src_tseg来自[3.3.2.1](#2321-urma_register_seg) [urma_register_seg](#2321-urma_register_seg)接口返回，否则可能导致调用者进程异常退出。参数内部指针等合法性已由这些接口保证，本接口不再重复进行校验。 + +#### 2.4.3.2 urma_read + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_read([urma_jfs_t](#_ZH-CN_TOPIC_0000002489752746-chtext) *jfs, [urma_target_jetty_t](#_ZH-CN_TOPIC_0000002521992545-chtext)*target_jfr, [3.3.2.1.3](#23213-urma_target_seg_t) [urma_target_seg_t](#23213-urma_target_seg_t) *dst_tseg, urma_target_seg_t *src_tseg, uint64_t dst, uint64_t src, uint32_t len, [3.4.1.1.7](#24117-urma_jfs_wr_flag_t) [urma_jfs_wr_flag_t](#24117-urma_jfs_wr_flag_t) flag, uint64_t user_ctx); + +3. 描述 + +调用urma_read发起单边操作的read操作请求，从指定的远端地址读数据到指定的本地缓存起始位置中，参数中指定了读取字节数。read请求最终完成之前，应用不能修改本地缓存内容。待成功把数据从远端节点读到本地内存后，应用可poll JFC获得完成消息。 + +使用的默认的JFS进行发送，使用JFS关联的JFC存放cqe，ordering和flag均为默认值。 + +支持多线程操作重入操作。 + +4. 参数 + +@param[in] jfs: the jfs created before, which is used to put command; + +@param[in] target_jfr: destination jetty receiver; + +@param[in] dst_tseg: the seg registered before; + +@param[in] src_tseg: the target seg imported before; + +@param[in] dst: destination address(local process address space) to be written into + +@param[in] src: source address(mapping va or rva in ubva) to fetch data + +@param[in] len: the data len to be written + +@param[in] flag: the flag to control jfs work request attribute + +@param[in] user_ctx: the user context, such as request id(rid) etc. + +5. 返回值 + +Return: 0 on success, other value on error + +![](figures/urma_notice.png) + +由调用者保证参数jfs来自[3.3.1.4.1](#23141-urma_create_jfs) [urma_create_jfs](#23141-urma_create_jfs)接口返回，dst_tseg来自[3.3.2.3](#2323-urma_import_seg) [urma_import_seg](#2323-urma_import_seg)接口返回，src_tseg来自[3.3.2.1](#2321-urma_register_seg) [urma_register_seg](#2321-urma_register_seg)接口返回，否则可能导致调用者进程异常退出。参数内部指针等合法性已由这些接口保证，本接口不再重复进行校验。 + +### 2.4.4 send/recv + +#### 2.4.4.1 urma_send + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t)urma_send([urma_jfs_t](#_ZH-CN_TOPIC_0000002489752746-chtext) *jfs, [urma_target_jetty_t](#_ZH-CN_TOPIC_0000002521992545-chtext) *target_jfr, [3.3.2.1.3](#23213-urma_target_seg_t) [urma_target_seg_t](#23213-urma_target_seg_t) *src_tseg, uint64_t src, uint32_t len, [3.4.1.1.7](#24117-urma_jfs_wr_flag_t) [urma_jfs_wr_flag_t](#24117-urma_jfs_wr_flag_t) flag, uint64_t user_ctx); + +3. 描述 + +调用urma_send发起双边操作的send操作请求，将指定的本地缓存起始位置的数据，发送指定字节数据到指定的目的地址。send请求最终完成之前，应用不能修改本地缓存内容。待成功把数据写到远端节点后，应用可poll JFC获得完成消息。 + +使用的jetty id和jfc id，ordering和flag均为默认值。 + +支持多线程操作重入操作。 + +4. 参数 + +@param[in] jfs: the jfs created before, which is used to put command; + +@param[in] target_jfr: destination jetty receiver(with full qualified jfr id); + +@param[in] src_tseg: the seg registered before; + +@param[in] src: source address for sending; + +@param[in] len: data length; + +@param[in] flag: flag to control jfs work request attribute + +@param[in] user_ctx: the user context, such as request id(rid) etc; + +5. 返回值 + +Return: 0 on success, other value on error. + +![](figures/urma_notice.png) + +由调用者保证参数jfs来自[3.3.1.4.1](#23141-urma_create_jfs) [urma_create_jfs](#23141-urma_create_jfs)接口返回，target_jfr来自[3.3.1.5.6](#23156-urma_import_jfr) [urma_import_jfr](#23156-urma_import_jfr)接口返回，src_tseg来自[3.3.2.1](#2321-urma_register_seg) [urma_register_seg](#2321-urma_register_seg)接口返回，否则可能导致调用者进程异常退出。参数内部指针等合法性已由这些接口保证，本接口不再重复进行校验。 + +#### 2.4.4.2 urma_recv + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_recv([urma_jfr_t](#_ZH-CN_TOPIC_0000002521992537-chtext) *jfr, [3.3.2.1.3](#23213-urma_target_seg_t) [urma_target_seg_t](#23213-urma_target_seg_t) *recv_tseg, uint64_t buf, uint32_t len, uint64_t user_ctx) + +3. 描述 + +调用urma_recv发起双边操作recv操作请求，将指定的本地缓存，放到jetty对应的接收队列。待成功接收远端发来的数据后，应用可poll JFC或者等待jfc事件获得完成消息。 + +支持多线程操作重入操作。 + +4. 参数 + +@param[in] jfr: jetty receiver; + +@param[in] recv_tseg: the locally registered segment before for receiving; + +@param[in] buf: buffer address for receiving; + +@param[in] len: buffer length; + +@param[in] user_ctx: the user context, such as request id(rid) etc; + +5. 返回值 + +Return: 0 on success, other value on error + +![](figures/urma_notice.png) + +由调用者保证参数jfr来自[3.3.1.5.1](#23151-urma_create_jfr) [urma_create_jfr](#23151-urma_create_jfr)接口返回，recv_tseg来自[3.3.2.1](#2321-urma_register_seg) [urma_register_seg](#2321-urma_register_seg)接口返回，否则可能导致调用者进程异常退出。参数内部指针等合法性已由这些接口保证，本接口不再重复进行校验。 + +## 2.5 其他 + +### 2.5.1 扩展 + +#### 2.5.1.1 urma_user_ctl + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_user_ctl([urma_context_t](#_ZH-CN_TOPIC_0000002489912714-chtext) *ctx, [3.5.1.1.1](#25111-urma_user_ctl_in_t) [urma_user_ctl_in_t](#25111-urma_user_ctl_in_t) *in, [3.5.1.1.2](#25112-urma_user_ctl_out_t) [urma_user_ctl_out_t](#25112-urma_user_ctl_out_t) *out) + +3. 描述 + +开放接口，用户可以直接使用该接口操作驱动，具体输入参数需要符合硬件驱动要求。 + +4. 参数 + +@param[in] ctx: the created urma context pointer; + +@param[in] in: user ioctl cmd; + +@param[out] out: result of execution; + +5. 返回值 + +Return: 0 on success, other value on error + +![](figures/urma_notice.png) + +由调用者保证参数ctx来自[3.2.2.4.1](#22241-urma_create_context) [urma_create_context](#22241-urma_create_context)接口返回，参数内部指针等合法性已由urma_create_context接口保证，该接口不再重复进行校验；否则可能导致调用者进程异常退出。 + +##### 2.5.1.1.1 urma_user_ctl_in_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_user_ctl_in { + uint64_t addr; /* [Required] the address of the input parameter buffer. */ + uint32_t len; /* [Required] the length of the input parameter buffer */ + /* + * Opcode is simultaneously recognized by user and driver. + * User opcode should be distinguished with enum urma_user_ctl_ops_t, which is only used by URMA. + */ + uint32_t opcode; /* [Required] */ +} urma_user_ctl_in_t; +``` + +##### 2.5.1.1.2 urma_user_ctl_out_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef struct urma_user_ctl_out { + uint64_t addr; /* [Optional] the address of the output parameter buffer. */ + uint32_t len; /* [Optional] the length of the output parameter buffer */ + uint32_t reserved; +} urma_user_ctl_out_t; +``` + +### 2.5.2 日志 + +#### 2.5.2.1 urma_register_log_func + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t)urma_register_log_func([3.5.2.1.1](#25211-urma_log_cb_t) [urma_log_cb_t](#25211-urma_log_cb_t) func) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +3. 描述 + +用户注册指定的日志函数 + +4. 参数 + +@param[in] func: log callback func + +5. 返回值 + +Return: 0 on success, other value on error + +##### 2.5.2.1.1 urma_log_cb_t + +typedef void (*urma_log_cb_t)(int level, char *message); + +#### 2.5.2.2 urma_unregister_log_func + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.2.1.1.2](#22112-urma_status_t) [urma_status_t](#22112-urma_status_t) urma_unregister_log_func(void) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +3. 描述 + +解注册用户指定的日志函数，使用urma默认的syslog() + +4. 参数 + +NA + +5. 返回值 + +Return: 0 on success, other value on error + +#### 2.5.2.3 urma_log_get_level + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +[3.5.2.3.1](#25231-urma_vlog_level_t) [urma_vlog_level_t](#25231-urma_vlog_level_t) urma_log_get_level(void) + +3. 描述 + +应用获得日志级别信息。 + +4. 参数 + +NA + +5. 返回值 + +Return: 返回日志级别urma_vlog_level_t + +##### 2.5.2.3.1 urma_vlog_level_t + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +```c +typedef enum urma_vlog_level { + URMA_VLOG_LEVEL_EMERG = 0, + URMA_VLOG_LEVEL_ALERT = 1, + URMA_VLOG_LEVEL_CRIT = 2, + URMA_VLOG_LEVEL_ERR = 3, + URMA_VLOG_LEVEL_WARNING = 4, + URMA_VLOG_LEVEL_NOTICE = 5, + URMA_VLOG_LEVEL_INFO = 6, + URMA_VLOG_LEVEL_DEBUG = 7, + URMA_VLOG_LEVEL_MAX = 8, +} urma_vlog_level_t; +``` + +#### 2.5.2.4 urma_log_set_level + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +void urma_log_set_level([3.5.2.3.1](#25231-urma_vlog_level_t) [urma_vlog_level_t](#25231-urma_vlog_level_t) level) + +定义文件: [urma_types.h](../../../src/urma/lib/urma/core/include/urma_types.h) + +3. 描述 + +应用设置日志级别信息。 + +4. 参数 + +@param[in] level: log level to set; + +5. 返回值 + +Return: void + +#### 2.5.2.5 urma_log_get_thread_tag + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +const char* urma_log_get_thread_tag(void); + +3. 描述 + +获取用户自定义的日志标记（线程粒度）。 + +4. 参数 + +NA + +5. 返回值 + +Return: const char * + +#### 2.5.2.6 urma_log_set_thread_tag + +1. 头文件 + +#include "urma_api.h" + +2. 原型 + +void urma_log_set_thread_tag(const char* tag); + +3. 描述 + +设置用户自定义的日志标记（线程粒度）。 + +4. 参数 + +@param[in] tag: log tag per thread; + +5. 返回值 + +Return: void + +### 2.5.3 宏定义 + +定义文件: [urma_opcode.h](../../../src/urma/lib/urma/core/include/urma_opcode.h) + +```c +#define URMA_TOKEN_NONE 0 /* Indicates the verification policy of the key. */ +#define URMA_TOKEN_PLAIN_TEXT 1 +#define URMA_TOKEN_SIGNED 2 +#define URMA_TOKEN_ALL_ENCRYPTED 3 +#define URMA_TOKEN_RESERVED 4 +#define URMA_TOKEN_ID_INVALID 0 +#define URMA_TOKEN_ID_VALID 1 +#define URMA_DSVA_DISABLE 0 /* Indicates whether it is a segment of dsva. */ +#define URMA_DSVA_ENABLE 1 +#define URMA_NON_CACHEABLE 0 /* Indicates whether the segment can be cached by multiple hosts. */ +#define URMA_CACHEABLE 1 +#define URMA_ACCESS_LOCAL_ONLY (0x1 << 0) +#define URMA_ACCESS_READ (0x1 << 1) +#define URMA_ACCESS_WRITE (0x1 << 2) +#define URMA_ACCESS_ATOMIC (0x1 << 3) +#define URMA_LOCAL_MEMORY 0 /* Indicates that the physical memory is remote. */ +#define URMA_REMOTE_MEMORY 1 +#define URMA_SEG_NOMAP 0 /* Indicates that the current process has mapped this segment */ +#define URMA_SEG_MAPPED 1 +#define URMA_ADDR_TYPE_MVA 0 +#define URMA_ADDR_TYPE_UBVA 1 +#define URMA_COMPLETE_ENABLE 1 /* Notify the source after the task is completed. */ +#define URMA_COMPLETE_DISABLE 0 /* Do not notify the source after the task is complete. */ +#define URMA_COMPLETE_TYPE_JFC 0 /* Complete notification via JFC. */ +#define URMA_COMPLETE_TYPE_CF 1 /* Complete notification via DDR address */ +#define URMA_DEPENDENCY_NONE 0 /* There is no dependency between commands. */ +#define URMA_DEPENDENCY_FIRST 1 /* Subsequent commands depend on the execution result of the current command. */ +#define URMA_DEPENDENCY_DELAY 2 /* The current command is executed only when the command that the preamble depends on is executed \ +successfully. */ +``` + +定义文件: [urma_opcode.h](../../../src/urma/lib/urma/core/include/urma_opcode.h) + +```c +#define URMA_NOTIFY_DISABLE 0 /* The destination is not notified when the task is completed. */ +#define URMA_NOTIFY_ENABLE 1 /* Notify the destination when the task is completed. */ +#define URMA_NOTIFY_TYPE_JFC 0 /* Complete notification via JFC. */ +#define URMA_NOTIFY_TYPE_RVA 1 /* Complete notification via DDR address. */ +#define URMA_INLINE_DISABLE 0 /* The data is generated by source_address assignment. */ +#define URMA_INLINE_ENABLE 1 /* The data is carried in the command. */ +#define URMA_SOLICITED_DISABLE 0 /* There is no interruption when notifying through JFC. */ +#define URMA_SOLICITED_ENABLE 1 /* Interrupt occurred while notifying via JFC. */ +#define URMA_FENCE_DISABLE 0 /* There is no fence. */ +#define URMA_FENCE_ENABLE 1 /* Fence with previous WRs. */ +#define URMA_REGULAR 1 /* regular, specifies stride format. */ +#define URMA_IRREGULAR 0 /* irregular, specifies S/G format. */ +#define URMA_NO_TAG_MATCHING 0 +#define URMA_WITH_TAG_MATCHING 1 +#define URMA_NONPOST_LS 0 +#define URMA_POST_LS 1 +#define URMA_NO_SHARE_JFR 0 +#define URMA_SHARE_JFR 1 +#define URMA_TYPICAL_RNR_RETRY 7 /* typical value of rnr retry for jfs cfg */ +#define URMA_TYPICAL_ERR_TIMEOUT 17 /* typical value of err_timeout for jfs cfg */ +#define URMA_TYPICAL_MIN_RNR_TIMER 12 /* typical value of min_rnr_timer for jfr cfg */ +#define URMA_MAX_PRIORITY 15 +#define URMA_SUCCESS 0 +#define URMA_EAGAIN EAGAIN /* Resource temporarily unavailable */ +#define URMA_ENOMEM ENOMEM /* Failed to allocate memory */ +#define URMA_ENOPERM EPERM /* Operation not permitted */ +#define URMA_ETIMEOUT ETIMEDOUT /* Operation time out */ +#define URMA_EINVAL EINVAL /* Invalid argument */ +#define URMA_EEXIST EEXIST /* Exist */ +#define URMA_EINPROGRESS EINPROGRESS +#define URMA_FAIL 0x1000 /* 0x1000 */ +``` + +--- +# 3 URMA内核态API + +内核态应用，例如smc-r，和支持用户态进程的应用，例如uburma，复用同一套URMA内核态API接口。用 udata区分用户态和内核态，传入udata=NULL表示内核态，udata！=NULL表示用户态。 + +## 3.1 编程示例 + +先注册client，然后回调add函数（根据用户需求，自定义） + +1. 内核态收发包流程图 + +![](figures/urma-api-kernel-example-01.png) + +### 3.1.1 管理面 + +- 内核态URMA通信需要向ubcore注册客户端及回调函数，从而取得ubcore_device用于通信，首先创建客户端结构体 + +```c +static int mymod_ubcore_add_device(struct ubcore_device *ubc_dev); +static void mymod_ubcore_remove_device(struct ubcore_device *ubc_dev, void *client_ctx); +struct ubcore_client g_mymod_ubcore_client = { + .list_node = LIST_HEAD_INIT(g_mymod_ubcore_client.list_node), + .client_name = "mymod", + .add = mymod_ubcore_add_device, + .remove = mymod_ubcore_remove_device, +}; +``` + +- 可以使用全局链表在回调函数中将ubc_device存下来用于后续通信 + +```c +struct mymod_dev_priv { + struct list_head list; + struct ubcore_device *ubc_dev; + /* other priv data + * \... + * \... + * \... */ +}; +DEFINE_SPINLOCK(priv_list_lock); +struct list_head priv_list_head = LIST_HEAD_INIT(priv_list_head); +static int mymod_ubcore_add_device(struct ubcore_device *ubc_dev) +{ + struct mymod_dev_priv *priv = kzalloc(sizeof(struct mymod_dev_priv), GFP_KERNEL); + priv-\>ubc_dev = ubc_dev; + spin_lock(&priv_list_lock); + list_add_tail(&priv-\>list, &priv_list_head); + spin_unlock(&priv_list_lock); + ubcore_set_client_ctx_data(ubc_dev, &g_mymod_ubcore_client, priv); // 第三个参数会记录在client_ctx中，在remove回调时成为入参 +} +``` + +- 反注册回调函数中释放私有结构体，也可以选择遍历链表进行释放 + +```c +static void mymod_ubcore_remove_device(struct ubcore_device *ubc_dev, void *client_ctx) +{ + struct mymod_dev_priv *priv = client_ctx; + if (priv == NULL) + return; + spin_lock(&priv_list_lock); + list_del(&priv-\>list, &priv_list_head); + spin_unlock(&priv_list_lock); + return ; +} +``` + +- 内核模块init/exit时注册/反注册ubcore cilent + +- 注：ubcore_register_client对于已经存在的ubcore_device会直接串行调用回调，若不考虑动态新增设备的复杂情况，注册客户端后就可以使用全局链表中的设备做通信 + +```c +static int \_\_init mymod_init(void) +{ + int ret = 0; + ret = ubcore_register_client(&g_mymod_ubcore_client); + if (ret != 0) { + pr_err("Register ubcore client failed.\n"); + return ret; + } + pr_info("Register ubcore client success.\n"); + /* create communication resources + * \... + * \... + * \...*/ + return 0; +} +static void \_\_exit mymod_exit(void) +{ + /* release communication resources + * \... + * \... + * \...*/ + ubcore_unregister_client(&g_mymod_ubcore_client); +} +``` + +### 3.1.2 控制面 + +- client和server分别创建jfc + +```c +void my_jfce(struct ubcore_jfc *jfc){ + // 用户自定义如何处理cqe +} +struct ubcore_jfc_cfg jfc_cfg = { + .depth = 64, + .flag = {.value = 0}, + .ceqn = 0, + .jfc_context = NULL, +} +struct ubcore_jfc *jfc = ubcore_create_jfc(ubc_dev, &jfc_cfg, my_jfce, NULL, NULL); +``` + +- client和server分别创建jfr + +```c +struct ubcore_jfr_cfg jfr_cfg = { + .depth = 64; + .flag.bs.token_policy = UBCORE_TOKEN_NONE; + .trans_mode = UBCORE_TP_RM; + .eid_index = eid_index; + .max_sge = 1; + .jfc = jfc; +}; +ubcore_create_jfr(urma_dev, &jfr_cfg, NULL, NULL); +``` + +- client和server分别创建jetty + +```c +struct ubcore_jetty_cfg jetty_cfg = { + .id = jetty_id; + .flag.bs.share_jfr = 1; + .trans_mode = UBCORE_TP_RM; + .eid_index = eid_index; + .jfs_depth = 64; + .priority = 0; /* Highest priority */ + .max_send_sge = 1; + .max_send_rsge = 1; + .jfr_depth = 64; + .max_recv_sge = 1; + .send_jfc = tx_jfc; + .recv_jfc = rx_jfc; + .jfr = jfr; +}; +ubcore_create_jetty(ubc_dev, &jetty_cfg, NULL, NULL); +``` + +- client和server分配各自的数据buffer，并将该数据buffer注册为segment + +```c +// 以分配4KB的buffer为例 +#define BUFFER_SIZE (0x1 << PAGE_SHIFT) +union ubcore_reg_seg_flag flag = { + .bs.token_policy = UBCORE_TOKEN_NONE, + .bs.cacheable = UBCORE_NON_CACHEABLE, + .bs.access = (UBCORE_ACCESS_READ | UBCORE_ACCESS_WRITE), + .bs.token_id_valid = 0, + .bs.reserved = 0 +}; +void *va = kzalloc(BUFFER_SIZE, GFP_KERNEL); +struct ubcore_seg_cfg cfg = { + .va = va; + .len = BUFFER_SIZE; + .flag = flag; +}; +struct ubcore_target_seg *local_tseg = ubcore_register_seg(ubc_dev, &cfg, NULL); +``` + +- client和server交换jetty信息，可以通过带外socket，或带内公知Jetty通道。以公知Jetty为例 + +```c +struct ubcore_tjetty_cfg tjetty_cfg = { + .id.eid = dst_eid; + .id.id = jetty_id; + .flag.bs.token_policy = UBCORE_TOKEN_NONE; + .tp_type = UBCORE_CTP; + .trans_mode = UBCORE_TP_RM; + .type = UBCORE_JETTY; + .eid_index = eid_index; +}; +struct ubcore_tjetty *tjetty = ubcore_import_jetty(ubc_dev, &tjetty_cfg, NULL); +``` + +### 3.1.3 数据面 + +#### 3.1.3.1 双边send/recv + +双边操作是把数据从本端seg发到对端seg。双边操作既需要本端调ubcore_post_jetty_send_wr下发发送任务，又需要对端调ubcore_post_jetty_recv_wr准备接收。本端的报文到对端时若发现recv wr还未下发，会暂时缓存在对端buffer，为了保证通信性能，并且避免耗尽对端buffer，建议对端预先post一批recv wr，并且每次消耗后要及时补充。 + +Server端post recv wr + +```c +/* post recv wr */ +struct ubcore_jfr_wr rx_wr; +struct ubcore_sge rx_sge; +struct ubcore_jfr_wr *jfr_bad_wr = NULL; +rx_wr.user_ctx = user_ctx; +rx_wr.src.sge = &rx_sge; +rx_wr.src.num_sge = 1; +rx_sge.tseg = local_tseg; /* 之前register得到的tseg */ +rx_sge.addr = local_tseg-\>seg.ubva.va; +rx_sge.len = BUFFER_SIZE; +ret = ubcore_post_jetty_recv_wr(jetty, &rx_wr, &jfr_bad_wr); +if (ret != 0) { + pr_err("post jetty recv wr failed.\n"); + return NULL; +} +``` + +Client端post send wr + +```c +/* post send wr */ +struct ubcore_sge tx_sge = { 0 }; +struct ubcore_jfs_wr tx_wr = { 0 }; +struct ubcore_jfs_wr *jfs_bad_wr = NULL; +tx_wr.user_ctx = user_ctx; +tx_wr.opcode = UBCORE_OPC_SEND; +tx_wr.send.src.sge = &tx_sge; +tx_wr.flag.bs.complete_enable = 1; +tx_wr.tjetty = tjetty; +tx_sge.tseg = local_tseg; +tx_sge.addr = local_tseg-\>seg.ubva.va; +tx_sge.len = tx_size; /* 实际发送的数据长度 */ +ret = ubcore_post_jetty_send_wr(jetty, &tx_wr, &jfs_bad_wr); +if (ret != 0) { + pr_err("post jetty send wr failed.\n"); + return NULL; +} +``` + +post wr后当发送/接收完成后可以通过poll操作尝试获取完成信息。可以使用轮询或者中断的方式获取（见[3.1.3.1](#2131-单边readwrite) [单边read/write](#2131-单边readwrite)）在内核态建议使用中断。 + +```c +/* 中断+napi 获取并处理接收完成信息，补充recv wr例子 */ +#define MY_NAPI_RX_WEIGHT 64 +struct my_napi_struct { + struct ubcore_jfc *rx_jfc; + struct napi_struct napi; +} +void handle_rx_wc(struct ubcore_cr rx_cr) +{ + /* 用户自定义处理完成信息 */ +} +int my_napi_rx_poll(struct napi_struct *napi, int budget) +{ + struct ubcore_jfc *rx_jfc = container_of(napi, struct my_napi_struct, napi)-\>rx_jfc; + struct ubcore_cr rx_cr[MY_NAPI_RX_WEIGHT]; + int left, max_num, actual_num, i, ret; + int done = 0; + while(done < budget) { + left = budget - done; + max_num = min(MY_NAPI_RX_WEIGHT, left); + actual_num = ubcore_poll_jfc(rx_jfc, max_num, rx_cr); + if (actual_num < 0) { + pr_info("poll jfc failed\n"); + break; + } + for (i = 0; i < actual_num; i++) { + handle_rx_wc(rx_cr[i]); + done++; + } + } + ubcore_rearm_jfc(rx_jfc, false); +} +``` + +## 3.2 设备及上下文管理 + +### 3.2.1 ubcore_register_device + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_register_device([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev); + +3. 描述 + +向 ubcore 注册一个UB设备。 + +4. 参数 + +@param[in] dev: the ubcore device; + +5. 返回值 + +Return: 0 on success, other value on error + +#### 3.2.1.1 ubcore_device + +```c +struct ubcore_device { + struct list_head list_node; /* add to device list */ + /* driver fills start */ + char dev_name[UBCORE_MAX_DEV_NAME]; + struct device *dma_dev; + struct device dev; + struct net_device *netdev; + ubcore_ops *ops; + ubcore_transport_type transport_type; + ubcore_device_attr attr; + struct attribute_group + *group[UBCORE_MAX_ATTR_GROUP]; /* driver may fill group [1] */ + /* driver fills end */ + ubcore_device_cfg cfg; + /* port management */ + struct list_head port_list; + /* For ubcore client */ + struct rw_semaphore client_ctx_rwsem; + struct list_head client_ctx_list; + struct list_head event_handler_list; + struct rw_semaphore event_handler_rwsem; + ubcore_hash_table ht[UBCORE_HT_NUM]; /* to be replaced with uobj */ + /* protect from unregister device */ + atomic_t use_cnt; + struct completion comp; + bool dynamic_eid; /* Assign eid dynamically with netdev notifier */ + ubcore_eid_table eid_table; + ubcore_cg_device cg_device; + ubcore_sip_table sip_table; + /* logic device list and mutex */ + ubcore_logic_device ldev; + struct mutex ldev_mutex; + struct list_head ldev_list; + /* ue_idx to uvs_instance mapping */ + void **ue2uvs_table; + struct rw_semaphore ue2uvs_rwsem; + /* for vtp audit */ + ubcore_vtp_bitmap vtp_bitmap; +}; +``` + +#### 3.2.1.2 ubcore_ops + +```c +struct ubcore_ops { + struct module *owner; /* kernel driver module */ + char driver_name[UBCORE_MAX_DRIVER_NAME]; /* user space driver name */ + uint32_t abi_version; /* abi version of kernel driver */ + /** + * add a function entity id (eid) to ub device (for uvs) + * @param[in] dev: the ubcore_device handle; + * @param[in] ue_idx: ue_idx; + * @param[in] cfg: eid and the upi of ue to which the eid belongs can be specified; + * @return: the index of eid/upi, less than 0 indicating error + */ + int (*add_ueid)(struct ubcore_device *dev, uint16_t ue_idx, + struct ubcore_ueid_cfg *cfg); + /** + * delete a function entity id (eid) to ub device (for uvs) + * @param[in] dev: the ubcore_device handle; + * @param[in] ue_idx: ue_idx; + * @param[in] cfg: eid and the upi of ue to which the eid belongs can be specified; + * @return: 0 on success, other value on error + */ + int (*delete_ueid)(struct ubcore_device *dev, uint16_t ue_idx, + struct ubcore_ueid_cfg *cfg); + /** + * query device attributes + * @param[in] dev: the ub device handle; + * @param[out] attr: attributes for the driver to fill in + * @return: 0 on success, other value on error + */ + int (*query_device_attr)(struct ubcore_device *dev, + struct ubcore_device_attr *attr); + /** + * query device status + * @param[in] dev: the ub device handle; + * @param[out] status: status for the driver to fill in + * @return: 0 on success, other value on error + */ + int (*query_device_status)(struct ubcore_device *dev, + struct ubcore_device_status *status); + /** + * query resource + * @param[in] dev: the ub device handle; + * @param[in] key: resource type and key; + * @param[in/out] val: addr and len of value + * @return: 0 on success, other value on error + */ + int (*query_res)(struct ubcore_device *dev, struct ubcore_res_key *key, + struct ubcore_res_val *val); + /** + * config device + * @param[in] dev: the ub device handle; + * @param[in] cfg: device configuration + * @return: 0 on success, other value on error + */ + int (*config_device)(struct ubcore_device *dev, + struct ubcore_device_cfg *cfg); + /** + * set ub network address + * @param[in] dev: the ub device handle; + * @param[in] net_addr: net_addr to set + * @param[in] index: index by sip table + * @return: 0 on success, other value on error + */ + int (*add_net_addr)(struct ubcore_device *dev, + struct ubcore_net_addr *net_addr, uint32_t index); + /** + * unset ub network address + * @param[in] dev: the ub device handle; + * @param[in] idx: net_addr idx by sip table entry + * @return: 0 on success, other value on error + */ + int (*delete_net_addr)(struct ubcore_device *dev, uint32_t idx); + /** + * allocate a context from ubep for a user process + * @param[in] dev: the ub device handle; + * @param[in] eid: function entity id (eid) index to set; + * @param[in] udrv_data: user space driver data + * @return: pointer to user context on success, null or error, + */ + struct ubcore_ucontext *(*alloc_ucontext)( + struct ubcore_device *dev, uint32_t eid_index, + struct ubcore_udrv_priv *udrv_data); + /** + * free a context to ubep + * @param[in] uctx: the user context created before; + * @return: 0 on success, other value on error + */ + int (*free_ucontext)(struct ubcore_ucontext *uctx); + /** + * mmap doorbell or jetty buffer, etc + * @param[in] uctx: the user context created before; + * @param[in] vma: linux vma including vm_start, vm_pgoff, etc; + * @return: 0 on success, other value on error + */ + int (*mmap)(struct ubcore_ucontext *ctx, struct vm_area_struct *vma); + /* segment part */ + /** alloc token id to ubep + * @param[in] dev: the ub device handle; + * @param[in] flag: token_id_flag; + * @param[in] udata: ucontext and user space driver data + * @return: token id pointer on success, NULL on error + */ + struct ubcore_token_id *(*alloc_token_id)( + struct ubcore_device *dev, union ubcore_token_id_flag flag, + struct ubcore_udata *udata); + /** free key id from ubep + * @param[in] token_id: the token id alloced before; + * @return: 0 on success, other value on error + */ + int (*free_token_id)(struct ubcore_token_id *token_id); + /** register segment to ubep + * @param[in] dev: the ub device handle; + * @param[in] cfg: segment attributes and configurations + * @param[in] udata: ucontext and user space driver data + * @return: target segment pointer on success, NULL on error + */ + struct ubcore_target_seg *(*register_seg)(struct ubcore_device *dev, + struct ubcore_seg_cfg *cfg, + struct ubcore_udata *udata); + /** unregister segment from ubep + * @param[in] tseg: the segment registered before; + * @return: 0 on success, other value on error + */ + int (*unregister_seg)(struct ubcore_target_seg *tseg); + /** import a remote segment to ubep + * @param[in] dev: the ub device handle; + * @param[in] cfg: segment attributes and import configurations + * @param[in] udata: ucontext and user space driver data + * @return: target segment handle on success, NULL on error + */ + struct ubcore_target_seg *(*import_seg)( + struct ubcore_device *dev, struct ubcore_target_seg_cfg *cfg, + struct ubcore_udata *udata); + /** unimport seg from ubep + * @param[in] tseg: the segment imported before; + * @return: 0 on success, other value on error + */ + int (*unimport_seg)(struct ubcore_target_seg *tseg); + /** add port for bound device + * @param[in] dev: the ub device handle; + * @param[in] port_cnt: port count + * @param[in] port_list: port list + * @return: target segment handle on success, NULL on error + */ + int (*add_port)(struct ubcore_device *dev, uint32_t port_cnt, + uint32_t *port_list); + /* jetty part */ + /** + * create jfc with ubep. + * @param[in] dev: the ub device handle; + * @param[in] cfg: jfc attributes and configurations + * @param[in] udata: ucontext and user space driver data + * @return: jfc pointer on success, NULL on error + */ + struct ubcore_jfc *(*create_jfc)(struct ubcore_device *dev, + struct ubcore_jfc_cfg *cfg, + struct ubcore_udata *udata); + /** + * modify jfc from ubep. + * @param[in] jfc: the jfc created before; + * @param[in] attr: ubcore jfc attr; + * @param[in] udata: ucontext and user space driver data + * @return: 0 on success, other value on error + */ + int (*modify_jfc)(struct ubcore_jfc *jfc, struct ubcore_jfc_attr *attr, + struct ubcore_udata *udata); + /** + * destroy jfc from ubep. + * @param[in] jfc: the jfc created before; + * @return: 0 on success, other value on error + */ + int (*destroy_jfc)(struct ubcore_jfc *jfc); + /** + * batch destroy jfc from ubep. + * @param[in] jfc_arr: the jfc array created before; + * @param[in] jfc_num: jfc array length; + * @param[out] bad_jfc_index: when delete err, return jfc index in the array; + * @return: 0 on success, other value on error + */ + int (*destroy_jfc_batch)(struct ubcore_jfc **jfc_arr, int jfc_num, + int *bad_jfc_index); + /** + * rearm jfc. + * @param[in] jfc: the jfc created before; + * @param[in] solicited_only: rearm notify by message marked with solicited flag + * @return: 0 on success, other value on error + */ + int (*rearm_jfc)(struct ubcore_jfc *jfc, bool solicited_only); + /** + * create jfs with ubep. + * @param[in] dev: the ub device handle; + * @param[in] cfg: jfs attributes and configurations + * @param[in] udata: ucontext and user space driver data + * @return: jfs pointer on success, NULL on error + */ + struct ubcore_jfs *(*create_jfs)(struct ubcore_device *dev, + struct ubcore_jfs_cfg *cfg, + struct ubcore_udata *udata); + /** + * modify jfs from ubep. + * @param[in] jfs: the jfs created before; + * @param[in] attr: ubcore jfs attr; + * @param[in] udata: ucontext and user space driver data + * @return: 0 on success, other value on error + */ + int (*modify_jfs)(struct ubcore_jfs *jfs, struct ubcore_jfs_attr *attr, + struct ubcore_udata *udata); + /** + * query jfs from ubep. + * @param[in] jfs: the jfs created before; + * @param[out] cfg: jfs configurations; + * @param[out] attr: ubcore jfs attributes; + * @return: 0 on success, other value on error + */ + int (*query_jfs)(struct ubcore_jfs *jfs, struct ubcore_jfs_cfg *cfg, + struct ubcore_jfs_attr *attr); + /** + * flush jfs from ubep. + * @param[in] jfs: the jfs created before; + * @param[in] cr_cnt: the maximum number of CRs expected to be returned; + * @param[out] cr: the addr of returned CRs; + * @return: the number of CR returned, 0 means no completion record returned, -1 on error + */ + int (*flush_jfs)(struct ubcore_jfs *jfs, int cr_cnt, + struct ubcore_cr *cr); + /** + * destroy jfs from ubep. + * @param[in] jfs: the jfs created before; + * @return: 0 on success, other value on error + */ + int (*destroy_jfs)(struct ubcore_jfs *jfs); + /** + * batch destroy jfs from ubep. + * @param[in] jfs_arr: the jfs array created before; + * @param[in] jfs_num: jfs array length; + * @param[out] bad_jfs_index: when error, return error jfs index in the array; + * @return: 0 on success, other value on error + */ + int (*destroy_jfs_batch)(struct ubcore_jfs **jfs_arr, int jfs_num, + int *bad_jfs_index); + /** + * create jfr with ubep. + * @param[in] dev: the ub device handle; + * @param[in] cfg: jfr attributes and configurations + * @param[in] udata: ucontext and user space driver data + * @return: jfr pointer on success, NULL on error + */ + struct ubcore_jfr *(*create_jfr)(struct ubcore_device *dev, + struct ubcore_jfr_cfg *cfg, + struct ubcore_udata *udata); + /** + * modify jfr from ubep. + * @param[in] jfr: the jfr created before; + * @param[in] attr: ubcore jfr attr; + * @param[in] udata: ucontext and user space driver data + * @return: 0 on success, other value on error + */ + int (*modify_jfr)(struct ubcore_jfr *jfr, struct ubcore_jfr_attr *attr, + struct ubcore_udata *udata); + /** + * query jfr from ubep. + * @param[in] jfr: the jfr created before; + * @param[out] cfg: jfr configurations; + * @param[out] attr: ubcore jfr attributes; + * @return: 0 on success, other value on error + */ + int (*query_jfr)(struct ubcore_jfr *jfr, struct ubcore_jfr_cfg *cfg, + struct ubcore_jfr_attr *attr); + /** + * destroy jfr from ubep. + * @param[in] jfr: the jfr created before; + * @return: 0 on success, other value on error + */ + int (*destroy_jfr)(struct ubcore_jfr *jfr); + /** + * batch destroy jfr from ubep. + * @param[in] jfr_arr: the jfr array created before; + * @param[in] jfr_num: jfr array length; + * @param[out] bad_jfr_index: when error, return error jfr index in the array; + * @return: 0 on success, other value on error + */ + int (*destroy_jfr_batch)(struct ubcore_jfr **jfr_arr, int jfr_num, + int *bad_jfr_index); + /** + * import jfr to ubep. + * @param[in] dev: the ub device handle; + * @param[in] cfg: remote jfr attributes and import configurations + * @param[in] udata: ucontext and user space driver data + * @return: target jfr pointer on success, NULL on error + */ + struct ubcore_tjetty *(*import_jfr)(struct ubcore_device *dev, + struct ubcore_tjetty_cfg *cfg, + struct ubcore_udata *udata); + /** + * import jfr to ubep by control plane. + * @param[in] dev: the ub device handle; + * @param[in] cfg: remote jfr attributes and import configurations; + * @param[in] active_tp_cfg: tp configuration to active; + * @param[in] udata: ucontext and user space driver data + * @return: target jfr pointer on success, NULL on error + */ + struct ubcore_tjetty *(*import_jfr_ex)( + struct ubcore_device *dev, struct ubcore_tjetty_cfg *cfg, + struct ubcore_active_tp_cfg *active_tp_cfg, + struct ubcore_udata *udata); + /** + * unimport jfr from ubep. + * @param[in] tjfr: the target jfr imported before; + * @return: 0 on success, other value on error + */ + int (*unimport_jfr)(struct ubcore_tjetty *tjfr); + /** + * create jetty with ubep. + * @param[in] dev: the ub device handle; + * @param[in] cfg: jetty attributes and configurations + * @param[in] udata: ucontext and user space driver data + * @return: jetty pointer on success, NULL on error + */ + struct ubcore_jetty *(*create_jetty)(struct ubcore_device *dev, + struct ubcore_jetty_cfg *cfg, + struct ubcore_udata *udata); + /** + * modify jetty from ubep. + * @param[in] jetty: the jetty created before; + * @param[in] attr: ubcore jetty attr; + * @param[in] udata: ucontext and user space driver data + * @return: 0 on success, other value on error + */ + int (*modify_jetty)(struct ubcore_jetty *jetty, + struct ubcore_jetty_attr *attr, + struct ubcore_udata *udata); + /** + * query jetty from ubep. + * @param[in] jetty: the jetty created before; + * @param[out] cfg: jetty configurations; + * @param[out] attr: ubcore jetty attributes; + * @return: 0 on success, other value on error + */ + int (*query_jetty)(struct ubcore_jetty *jetty, + struct ubcore_jetty_cfg *cfg, + struct ubcore_jetty_attr *attr); + /** + * flush jetty from ubep. + * @param[in] jetty: the jetty created before; + * @param[in] cr_cnt: the maximum number of CRs expected to be returned; + * @param[out] cr: the addr of returned CRs; + * @return: the number of CR returned, 0 means no completion record returned, -1 on error + */ + int (*flush_jetty)(struct ubcore_jetty *jetty, int cr_cnt, + struct ubcore_cr *cr); + /** + * destroy jetty from ubep. + * @param[in] jetty: the jetty created before; + * @return: 0 on success, other value on error + */ + int (*destroy_jetty)(struct ubcore_jetty *jetty); + /** + * batch destroy jetty from ubep. + * @param[in] jetty_arr: the jetty array created before; + * @param[in] jetty_num: jetty array length; + * @param[out] bad_jetty_index: when error, return error jetty index in the array; + * @return: 0 on success, other value on error + */ + int (*destroy_jetty_batch)(struct ubcore_jetty **jetty_arr, + int jetty_num, int *bad_jetty_index); + /** + * import jetty to ubep. + * @param[in] dev: the ub device handle; + * @param[in] cfg: remote jetty attributes and import configurations + * @param[in] udata: ucontext and user space driver data + * @return: target jetty pointer on success, NULL on error + */ + struct ubcore_tjetty *(*import_jetty)(struct ubcore_device *dev, + struct ubcore_tjetty_cfg *cfg, + struct ubcore_udata *udata); + /** + * import jetty to ubep by control plane. + * @param[in] dev: the ub device handle; + * @param[in] cfg: remote jetty attributes and import configurations + * @param[in] active_tp_cfg: tp configuration to active + * @param[in] udata: ucontext and user space driver data + * @return: target jetty pointer on success, NULL on error + */ + struct ubcore_tjetty *(*import_jetty_ex)( + struct ubcore_device *dev, struct ubcore_tjetty_cfg *cfg, + struct ubcore_active_tp_cfg *active_tp_cfg, + struct ubcore_udata *udata); + /** + * unimport jetty from ubep. + * @param[in] tjetty: the target jetty imported before; + * @return: 0 on success, other value on error + */ + int (*unimport_jetty)(struct ubcore_tjetty *tjetty); + /** + * bind jetty from ubep. + * @param[in] jetty: the jetty created before; + * @param[in] tjetty: the target jetty imported before; + * @param[in] udata: ucontext and user space driver data + * @return: 0 on success, other value on error + */ + int (*bind_jetty)(struct ubcore_jetty *jetty, + struct ubcore_tjetty *tjetty, + struct ubcore_udata *udata); + /** + * bind jetty from ubep by control plane. + * @param[in] jetty: the jetty created before; + * @param[in] tjetty: the target jetty imported before; + * @param[in] active_tp_cfg: tp configuration to active; + * @param[in] udata: ucontext and user space driver data + * @return: 0 on success, other value on error + */ + int (*bind_jetty_ex)(struct ubcore_jetty *jetty, + struct ubcore_tjetty *tjetty, + struct ubcore_active_tp_cfg *active_tp_cfg, + struct ubcore_udata *udata); + /** + * unbind jetty from ubep. + * @param[in] jetty: the jetty binded before; + * @return: 0 on success, other value on error + */ + int (*unbind_jetty)(struct ubcore_jetty *jetty); + /** + * create jetty group to ubep. + * @param[in] dev: the ub device handle; + * @param[in] cfg: pointer of the jetty group config; + * @param[in] udata: ucontext and user space driver data + * @return: jetty group pointer on success, NULL on error + */ + struct ubcore_jetty_group *(*create_jetty_grp)( + struct ubcore_device *dev, struct ubcore_jetty_grp_cfg *cfg, + struct ubcore_udata *udata); + /** + * destroy jetty group to ubep. + * @param[in] jetty_grp: the jetty group created before; + * @return: 0 on success, other value on error + */ + int (*delete_jetty_grp)(struct ubcore_jetty_group *jetty_grp); + /** + * create tpg. + * @param[in] dev: the ub device handle; + * @param[in] cfg: tpg init attributes + * @param[in] udata: ucontext and user space driver data + * @return: tp pointer on success, NULL on error + */ + struct ubcore_tpg *(*create_tpg)(struct ubcore_device *dev, + struct ubcore_tpg_cfg *cfg, + struct ubcore_udata *udata); + /** + * destroy tpg. + * @param[in] tp: tp pointer created before + * @return: 0 on success, other value on error + */ + int (*destroy_tpg)(struct ubcore_tpg *tpg); + /** + * get tpid list by control plane. + * @param[in] dev: ubcore device pointer created before + * @param[in] cfg: tpid configuration to be matched + * @param[in && out] tp_cnt: tp_cnt is the length of tp_list buffer as in parameter; + * tp_cnt is the number of tp as out parameter + * @param[out] tp_list: tp list to get, the buffer is allocated by user; + * @param[in && out] udata: ucontext and user space driver data + * @return: 0 on success, other value on error + */ + int (*get_tp_list)(struct ubcore_device *dev, + struct ubcore_get_tp_cfg *cfg, uint32_t *tp_cnt, + struct ubcore_tp_info *tp_list, + struct ubcore_udata *udata); + /** + * set tp attributions by control plane. + * @param[in] dev: ubcore device pointer created before; + * @param[in] tp_handle: tp_handle got by ubcore_get_tp_list; + * @param[in] tp_attr_cnt: number of tp attributions; + * @param[in] tp_attr_bitmap: tp attributions bitmap, current bitmap is as follow: + * 0-retry_times_init: 3 bit 1-at: 5 bit 2-SIP: 128 bit + * 3-DIP: 128 bit 4-SMA: 48 bit 5-DMA: 48 bit + * 6-vlan_id: 12 bit 7-vlan_en: 1 bit 8-dscp: 6 bit + * 9-at_times: 5 bit 10-sl: 4 bit 11-tti: 8 bit + * @param[in] tp_attr: tp attribution values to set; + * @param[in && out] udata: ucontext and user space driver data; + * @return: 0 on success, other value on error + */ + int (*set_tp_attr)(struct ubcore_device *dev, const uint64_t tp_handle, + const uint8_t tp_attr_cnt, + const uint32_t tp_attr_bitmap, + const struct ubcore_tp_attr_value *tp_attr, + struct ubcore_udata *udata); + /** + * get tp attributions by control plane. + * @param[in] dev: ubcore device pointer created before; + * @param[in] tp_handle: tp_handle got by ubcore_get_tp_list; + * @param[out] tp_attr_cnt: number of tp attributions; + * @param[out] tp_attr_bitmap: tp bitmap, the same as tp_attr_bitmap in set_tp_attr; + * @param[out] tp_attr: tp attribution values to get; + * @param[in && out] udata: ucontext and user space driver data; + * @return: 0 on success, other value on error + */ + int (*get_tp_attr)(struct ubcore_device *dev, const uint64_t tp_handle, + uint8_t *tp_attr_cnt, uint32_t *tp_attr_bitmap, + struct ubcore_tp_attr_value *tp_attr, + struct ubcore_udata *udata); + /** + * active tp by control plane. + * @param[in] dev: ubcore device pointer created before + * @param[in] active_cfg: tp configuration to active + * @return: 0 on success, other value on error + */ + int (*active_tp)(struct ubcore_device *dev, + struct ubcore_active_tp_cfg *active_cfg); + /** + * deactivate tp by control plane. + * @param[in] dev: ubcore device pointer created before + * @param[in] tp_handle: tp_handle value got before + * @param[in] udata: [Optional] udata should be NULL when called + * by kernel application and be valid when called + * by user space application + * @return: 0 on success, other value on error + */ + int (*deactive_tp)(struct ubcore_device *dev, + union ubcore_tp_handle tp_handle, + struct ubcore_udata *udata); + /** + * create tp. + * @param[in] dev: the ub device handle; + * @param[in] cfg: tp init attributes + * @param[in] udata: ucontext and user space driver data + * @return: tp pointer on success, NULL on error + */ + struct ubcore_tp *(*create_tp)(struct ubcore_device *dev, + struct ubcore_tp_cfg *cfg, + struct ubcore_udata *udata); + /** + * modify tp. + * @param[in] tp: tp pointer created before + * @param[in] attr: tp attributes + * @param[in] mask: attr mask indicating the attributes to be modified + * @return: 0 on success, other value on error + */ + int (*modify_tp)(struct ubcore_tp *tp, struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask mask); + /** + * modify user tp. + * @param[in] dev: the ub device handle + * @param[in] tpn: tp number of the tp created before + * @param[in] cfg: user configuration of the tp + * @param[in] attr: tp attributes + * @param[in] mask: attr mask indicating the attributes to be modified + * @return: 0 on success, other value on error + */ + int (*modify_user_tp)(struct ubcore_device *dev, uint32_t tpn, + struct ubcore_tp_cfg *cfg, + struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask mask); + /** + * destroy tp. + * @param[in] tp: tp pointer created before + * @return: 0 on success, other value on error + */ + int (*destroy_tp)(struct ubcore_tp *tp); + /** + * create multi tp. + * @param[in] dev: the ub device handle; + * @param[in] cnt: the number of tp, must be less than or equal to 32; + * @param[in] cfg: array of tp init attributes + * @param[in] udata: array of ucontext and user space driver data + * @param[out] tp: pointer array of tp + * @return: created tp cnt, 0 on error + */ + int (*create_multi_tp)(struct ubcore_device *dev, uint32_t cnt, + struct ubcore_tp_cfg *cfg, + struct ubcore_udata *udata, + struct ubcore_tp **tp); + /** + * modify multi tp. + * @param[in] cnt: the number of tp; + * @param[in] tp: pointer array of tp created before + * @param[in] attr: array of tp attributes + * @param[in] mask: array of attr mask indicating the attributes to be modified + * @param[in] fail_tp: pointer of tp failed to modify + * @return: modified successfully tp cnt, 0 on error + */ + int (*modify_multi_tp)(uint32_t cnt, struct ubcore_tp **tp, + struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask *mask, + struct ubcore_tp **fail_tp); + /** + * destroy multi tp. + * @param[in] cnt: the number of tp; + * @param[in] tp: pointer array of tp created before + * @return: destroyed tp cnt, 0 on error + */ + int (*destroy_multi_tp)(uint32_t cnt, struct ubcore_tp **tp); + /** + * allocate vtp. + * @param[in] dev: the ub device handle; + * @return: vtpn pointer on success, NULL on error + */ + struct ubcore_vtpn *(*alloc_vtpn)(struct ubcore_device *dev); + /** + * free vtpn. + * @param[in] vtpn: vtpn pointer allocated before + * @return: 0 on success, other value on error + */ + int (*free_vtpn)(struct ubcore_vtpn *vtpn); + /** + * create vtp. + * @param[in] dev: the ub device handle; + * @param[in] cfg: vtp init attributes + * @param[in] udata: ucontext and user space driver data + * @return: vtp pointer on success, NULL on error + */ + struct ubcore_vtp *(*create_vtp)(struct ubcore_device *dev, + struct ubcore_vtp_cfg *cfg, + struct ubcore_udata *udata); + /** + * destroy vtp. + * @param[in] vtp: vtp pointer created before + * @return: 0 on success, other value on error + */ + int (*destroy_vtp)(struct ubcore_vtp *vtp); + /** + * create utp. + * @param[in] dev: the ub device handle; + * @param[in] cfg: utp init attributes + * @param[in] udata: ucontext and user space driver data + * @return: utp pointer on success, NULL on error + */ + struct ubcore_utp *(*create_utp)(struct ubcore_device *dev, + struct ubcore_utp_cfg *cfg, + struct ubcore_udata *udata); + /** + * destroy utp. + * @param[in] utp: utp pointer created before + * @return: 0 on success, other value on error + */ + int (*destroy_utp)(struct ubcore_utp *utp); + /** + * create ctp. + * @param[in] dev: the ub device handle; + * @param[in] cfg: ctp init attributes + * @param[in] udata: ucontext and user space driver data + * @return: ctp pointer on success, NULL on error + */ + struct ubcore_ctp *(*create_ctp)(struct ubcore_device *dev, + struct ubcore_ctp_cfg *cfg, + struct ubcore_udata *udata); + /** + * destroy ctp. + * @param[in] ctp: ctp pointer created before + * @return: 0 on success, other value on error + */ + int (*destroy_ctp)(struct ubcore_ctp *ctp); + /** + * UE send msg to MUE device. + * @param[in] dev: UE or MUE device; + * @param[in] msg: msg to send; + * @return: 0 on success, other value on error + */ + int (*send_req)(struct ubcore_device *dev, struct ubcore_req *msg); + /** + * MUE send msg to UE device. + * @param[in] dev: MUE device; + * @param[in] msg: msg to send; + * @return: 0 on success, other value on error + */ + int (*send_resp)(struct ubcore_device *dev, + struct ubcore_resp_host *msg); + /** + * query cc table to get cc pattern idx + * @param[in] dev: the ub device handle; + * @param[in] cc_entry_cnt: cc entry cnt; + * @return: return NULL on fail, otherwise, return cc entry array + */ + struct ubcore_cc_entry *(*query_cc)(struct ubcore_device *dev, + uint32_t *cc_entry_cnt); + /** + * bond slave net device + * @param[in] bond: bond netdev; + * @param[in] slave: slave netdev; + * @param[in] upper_info: change upper event info; + * @return: 0 on success, other value on error + */ + int (*bond_add)(struct net_device *bond, struct net_device *slave, + struct netdev_lag_upper_info *upper_info); + /** + * unbond slave net device + * @param[in] bond: bond netdev; + * @param[in] slave: slave netdev; + * @return: 0 on success, other value on error + */ + int (*bond_remove)(struct net_device *bond, struct net_device *slave); + /** + * update slave net device + * @param[in] bond: bond netdev; + * @param[in] slave: slave netdev; + * @param[in] lower_info: change lower state event info; + * @return: 0 on success, other value on error + */ + int (*slave_update)(struct net_device *bond, struct net_device *slave, + struct netdev_lag_lower_state_info *lower_info); + /** + * operation of user ioctl cmd. + * @param[in] dev: the ub device handle; + * @param[in] user_ctl: kdrv user control command pointer; + * Return: 0 on success, other value on error + */ + int (*user_ctl)(struct ubcore_device *dev, + struct ubcore_user_ctl *user_ctl); + /** data path ops */ + /** + * post jfs wr. + * @param[in] jfs: the jfs created before; + * @param[in] wr: the wr to be posted; + * @param[out] bad_wr: the first failed wr; + * @return: 0 on success, other value on error + */ + int (*post_jfs_wr)(struct ubcore_jfs *jfs, struct ubcore_jfs_wr *wr, + struct ubcore_jfs_wr **bad_wr); + /** + * post jfr wr. + * @param[in] jfr: the jfr created before; + * @param[in] wr: the wr to be posted; + * @param[out] bad_wr: the first failed wr; + * @return: 0 on success, other value on error + */ + int (*post_jfr_wr)(struct ubcore_jfr *jfr, struct ubcore_jfr_wr *wr, + struct ubcore_jfr_wr **bad_wr); + /** + * post jetty send wr. + * @param[in] jetty: the jetty created before; + * @param[in] wr: the wr to be posted; + * @param[out] bad_wr: the first failed wr; + * @return: 0 on success, other value on error + */ + int (*post_jetty_send_wr)(struct ubcore_jetty *jetty, + struct ubcore_jfs_wr *wr, + struct ubcore_jfs_wr **bad_wr); + /** + * post jetty receive wr. + * @param[in] jetty: the jetty created before; + * @param[in] wr: the wr to be posted; + * @param[out] bad_wr: the first failed wr; + * @return: 0 on success, other value on error + */ + int (*post_jetty_recv_wr)(struct ubcore_jetty *jetty, + struct ubcore_jfr_wr *wr, + struct ubcore_jfr_wr **bad_wr); + /** + * poll jfc. + * @param[in] jfc: the jfc created before; + * @param[in] cr_cnt: the maximum number of CRs expected to be polled; + * @return: 0 on success, other value on error + */ + int (*poll_jfc)(struct ubcore_jfc *jfc, int cr_cnt, + struct ubcore_cr *cr); + /** + * query_stats. success to query and buffer length is enough + * @param[in] dev: the ub device handle; + * @param[in] key: type and key value of the ub device to query; + * @param[in/out] val: address and buffer length of query results + * @return: 0 on success, other value on error + */ + int (*query_stats)(struct ubcore_device *dev, + struct ubcore_stats_key *key, + struct ubcore_stats_val *val); + /** + * config function migrate state. + * @param[in] dev: the ub device handle; + * @param[in] ue_idx: ue id; + * @param[in] cnt: config count; + * @param[in] cfg: eid and the upi of ue to which the eid belongs can be specified; + * @param[in] state: config state (start, rollback and finish) + * @return: config success count, -1 on error + */ + int (*config_function_migrate_state)(struct ubcore_device *dev, + uint16_t ue_idx, uint32_t cnt, + struct ubcore_ueid_cfg *cfg, + enum ubcore_mig_state state); + /** + * modify vtp. + * @param[in] vtp: vtp pointer to be modified; + * @param[in] attr: vtp attr, tp that we want to change; + * @param[in] mask: attr mask; + * @return: 0 on success, other value on error + */ + int (*modify_vtp)(struct ubcore_vtp *vtp, struct ubcore_vtp_attr *attr, + union ubcore_vtp_attr_mask *mask); + /** + * query ue index. + * @param[in] dev: the ub device handle; + * @param[in] devid: ue devid to query + * @param[out] ue_idx: ue id; + * @return: 0 on success, other value on error + */ + int (*query_ue_idx)(struct ubcore_device *dev, + struct ubcore_devid *devid, uint16_t *ue_idx); + /** + * config dscp-vl mapping + * @param[in] dev:the ub dev handle; + * @param[in] dscp: the dscp value array + * @param[in] vl: the vl value array + * @param[in] num: array num + * @return: 0 on success, other value on error + */ + int (*config_dscp_vl)(struct ubcore_device *dev, uint8_t *dscp, + uint8_t *vl, uint8_t num); + /** + * query ue stats, for migration currently. + * @param[in] dev: the ub device handle; + * @param[in] cnt: array count; + * @param[in] ue_idx: ue id array; + * @param[out] stats: ue counters + * @return: 0 on success, other value on error + */ + int (*query_ue_stats)(struct ubcore_device *dev, uint32_t cnt, + uint16_t *ue_idx, struct ubcore_ue_stats *stats); + /** + * query dscp-vl mapping + * @param[in] dev:the ub dev handle; + * @param[in] dscp: the dscp value array + * @param[in] num: array num + * @param[out] vl: the vl value array + * @return: 0 on success, other value on error + */ + int (*query_dscp_vl)(struct ubcore_device *dev, uint8_t *dscp, + uint8_t num, uint8_t *vl); + /** + * When UVS or UB dataplane is running: + * 1. disassociate_ucontext != NULL means support rmmod driver. + * 2. disassociate_ucontext == NULL means rmmod driver will fail because module is in use. + * If disassociate_ucontext != NULL: + * 1. When remove MUE/UE device, will call it; + * 2. When remove MUE device, will not call it because there are no uctx. + * @param[in] uctx: the ubcore_ucontext + */ + void (*disassociate_ucontext)(struct ubcore_ucontext *uctx); +}; +``` + +#### 3.2.1.3 ubcore_device_cfg + +```c +struct ubcore_device_cfg { + uint16_t ue_idx; /* ue id or mue id. e.g: bdf id */ + union ubcore_device_cfg_mask mask; + struct ubcore_rc_cfg rc_cfg; + uint32_t slice; /* TA slice size byte */ + uint8_t pattern; /* 0: pattern1; 1: pattern3 */ + bool virtualization; + uint32_t suspend_period; /* us */ + uint32_t suspend_cnt; /* TP resend cnt */ + uint32_t min_jetty_cnt; + uint32_t max_jetty_cnt; + uint32_t min_jfr_cnt; + uint32_t max_jfr_cnt; + uint32_t reserved_jetty_id_min; + uint32_t reserved_jetty_id_max; +}; +``` + +#### 3.2.1.4 ubcore_device_cfg_mask + +```c +union ubcore_device_cfg_mask { + struct { + uint32_t rc_cnt : 1; + uint32_t rc_depth : 1; + uint32_t slice : 1; + uint32_t pattern : 1; + uint32_t virtualization : 1; + uint32_t suspend_period : 1; + uint32_t suspend_cnt : 1; + uint32_t min_jetty_cnt : 1; + uint32_t max_jetty_cnt : 1; + uint32_t min_jfr_cnt : 1; + uint32_t max_jfr_cnt : 1; + uint32_t reserved_jetty_id_min : 1; + uint32_t reserved_jetty_id_max : 1; + uint32_t reserved : 19; + } bs; + uint32_t value; +}; +``` + +#### 3.2.1.5 ubcore_rc_cfg + +```c +struct ubcore_rc_cfg { + uint32_t rc_cnt; /* rc queue count */ + uint32_t depth; +}; +``` + +#### 3.2.1.6 ubcore_hash_table + +```c +struct ubcore_hash_table { + ubcore_ht_param p; + struct hlist_head *head; + /* Prevent the same jetty + * from being bound by different tjetty + */ + ubcore_jetty_id rc_tjetty_id; + spinlock_t lock; + struct kref kref; +}; +``` + +#### 3.2.1.7 ubcore_ht_param + +```c +struct ubcore_ht_param { + uint32_t size; + uint32_t node_offset; /* offset of hlist node in the hash table object */ + uint32_t key_offset; + uint32_t key_size; + int (*cmp_f)(void *obj, const void *key); + void (*free_f)(void *obj); + void (*get_f)(void *obj); +}; +``` + +#### 3.2.1.8 ubcore_eid_table + +```c +struct ubcore_eid_table { + uint32_t eid_cnt; + ubcore_eid_entry *eid_entries; + spinlock_t lock; +}; +``` + +#### 3.2.1.9 ubcore_eid_entry + +```c +struct ubcore_eid_entry { + ubcore_eid eid; + uint32_t eid_index; + struct net *net; + bool valid; +}; +``` + +#### 3.2.1.10 ubcore_cg_device + +```c +struct ubcore_cg_device { +#ifdef CONFIG_CGROUP_RDMA + struct rdmacg_device dev; +#endif +}; +``` + +#### 3.2.1.11 ubcore_sip_table + +```c +struct ubcore_sip_table { + struct mutex lock; + uint32_t max_sip_cnt; + ubcore_sip_entry *entry; + DECLARE_BITMAP(index_bitmap, UBCORE_MAX_SIP); +}; +``` + +#### 3.2.1.12 ubcore_sip_entry + +```c +struct ubcore_sip_entry { + ubcore_sip_info sip_info; + atomic_t uvs_cnt; + uint64_t reserve; +}; +``` + +#### 3.2.1.13 ubcore_logic_device + +```c +struct ubcore_logic_device { + struct device *dev; + ubcore_port_kobj port[UBCORE_MAX_PORT_CNT]; + struct list_head node; /* add to ldev list */ + possible_net_t net; + ubcore_device *ub_dev; + const struct attribute_group *dev_group[UBCORE_ATTR_GROUP_MAX]; +}; +``` + +#### 3.2.1.14 ubcore_port_kobj + +```c +struct ubcore_port_kobj { + struct kobject kobj; + ubcore_device *dev; + uint8_t port_id; +}; +``` + +#### 3.2.1.15 ubcore_vtp_bitmap + +```c +struct ubcore_vtp_bitmap { + struct mutex lock; + uint32_t max_vtp_cnt; + uint64_t *bitmap; +}; +``` + +### 3.2.2 ubcore_unregister_device + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +void ubcore_unregister_device([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev) + +3. 描述 + +UDMA驱动卸载时主动调用，解注册UB设备 + +4. 参数 + +@param[in] dev: the ubcore device; + +5. 返回值 + +void + +### 3.2.3 ubcore_stop_requests + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +void **ubcore_stop_requests**([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev); + +3. 描述 + +UDMA驱动调用该接口停流 + +4. 参数 + +@param[in] dev: the ubcore device; + +5. 返回值 + +void + +### 3.2.4 ubcore_alloc_ucontext + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +[4.2.4.1](#3241-ubcore_ucontext) [ubcore_ucontext](#3241-ubcore_ucontext) *ubcore_alloc_ucontext([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, uint32_t eid_index, [4.2.4.2](#3242-ubcore_udrv_priv) [ubcore_udrv_priv](#3242-ubcore_udrv_priv) *udrv_data); + +3. 描述 + +Application specifies the device to allocate an context. + +4. 参数 + +@param[in] dev: ubcore_device found by add ops in the client. + +@param[in] eid_index: function entity id (eid) index to set; + +@param[in] udrv_data (optional): ucontext and user space driver data + +5. 返回值 + +ubcore_ucontext pointer on success, NULL on fail. + +Note: this API is called only by uburma representing user-space application, not by other kernel modules. + +#### 3.2.4.1 ubcore_ucontext + +```c +struct ubcore_ucontext { + ubcore_device *ub_dev; + ubcore_eid eid; + uint32_t eid_index; + void *jfae; /* jfae uobj */ + [struct ubcore_cg_object](#32161-struct-ubcore_cg_object) cg_obj; + atomic_t use_cnt; +}; +``` + +#### 3.2.4.2 ubcore_udrv_priv + +```c +struct ubcore_udrv_priv { + uint64_t in_addr; + uint32_t in_len; + uint64_t out_addr; + uint32_t out_len; +}; +``` + +### 3.2.5 ubcore_free_ucontext + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +void ubcore_free_ucontext([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, [4.2.4.1](#3241-ubcore_ucontext) [ubcore_ucontext](#3241-ubcore_ucontext) *ucontext); + +3. 描述 + +Free the allocated context. + +4. 参数 + +@param[in] dev: device to free context. + +@param[in] ucontext: handle of the allocated context. + +Note: this API is called only by uburma representing user-space application, not by other kernel modules + +### 3.2.6 ubcore_register_client + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_register_client([4.2.6.1](#3261-ubcore_client) [ubcore_client](#3261-ubcore_client) *new_client); + +3. 描述 + +向 ubcore 注册一个内核态应用客户端，例如uburma。 + +4. 参数 + +@param[in] [Required] dev: the ubcore_device handle; + +@param[in] [Required] new_client: ubcore client to be registered + +5. 返回值 + +Return: 0 on success, other value on error + +#### 3.2.6.1 ubcore_client + +```c +struct ubcore_client { + struct list_head list_node; + char *client_name; + int (*add)(ubcore_device *dev); + void (*remove)(ubcore_device *dev, void *client_ctx); + /* The driver needs to stay and resolve the memory mapping first, */ + /* and then release the jetty resources. */ + void (*stop)(ubcore_device *dev, void *client_ctx); +}; +``` + +### 3.2.7 ubcore_unregister_client + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +void ubcore_unregister_client([4.2.6.1](#3261-ubcore_client) [ubcore_client](#3261-ubcore_client) *rm_client); + +3. 描述 + +注销一个ubcore内核态应用客户端 + +4. 参数 + +@param[in] [Required] rm_client: ubcore client to be unregistered + +5. 返回值 + +NA + +### 3.2.8 ubcore_set_client_ctx_data + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +void ubcore_set_client_ctx_data([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, [4.2.6.1](#3261-ubcore_client) [ubcore_client](#3261-ubcore_client) *client, void *data); + +3. 描述 + +内核态应用客户端设置ubcore设备的私有数据 + +4. 参数 + +@param[in] dev: the ubcore_device handle; + +@param[in] client: ubcore client pointer; + +@param[in] data: client private data to be set; + +5. 返回值 + +NA + +### 3.2.9 ubcore_get_client_ctx_data + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +void *ubcore_get_client_ctx_data([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, [4.2.6.1](#3261-ubcore_client) [ubcore_client](#3261-ubcore_client) *client); + +3. 描述 + +内核态应用客户端获取ubcore设备的私有数据 + +4. 参数 + +@param[in] dev: the ubcore_device handle; + +@param[in] client: ubcore client pointer; + +5. 返回值 + +client private data set before. + +用户调用ubcore_set_client_ctx_data设置过的data指针，可能为NULL + +### 3.2.10 ubcore_get_eid_list + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +[4.2.10.1](#32101-ubcore_eid_info) [ubcore_eid_info](#32101-ubcore_eid_info) *ubcore_get_eid_list([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, uint32_t *cnt); + +3. 描述 + +查询设备当前有效的eid信息（包括eid和eid_index）数组; + +4. 参数 + +@param[in] [Required] dev: the ubcore device; + +@param[out] [Required] cnt: eid cnt; + +5. 返回值 + +成功返回eid_info数组指针，元素个数为cnt；失败返回NULL；由用户调用ubcore_free_eid_list释放 + +#### 3.2.10.1 ubcore_eid_info + +```c +struct ubcore_eid_info { + ubcore_eid eid; + uint32_t eid_index; /* 0\~MAX_EID_CNT -1 */ +}; +``` + +#### 3.2.10.2 ubcore_eid + +```c +union ubcore_eid { + uint8_t raw[UBCORE_EID_SIZE]; + struct { + uint64_t reserved; + uint32_t prefix; + uint32_t addr; + } in4; + struct { + uint64_t subnet_prefix; + uint64_t interface_id; + } in6; +}; +``` + +### 3.2.11 ubcore_free_eid_list + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +void ubcore_free_eid_list([4.2.10.1](#32101-ubcore_eid_info) [ubcore_eid_info](#32101-ubcore_eid_info) *eid_list); + +3. 描述 + +ubcore_free_eid_list释放ubcore_get_eid_list返回的eid_list + +4. 参数 + +@param[in] eid_list: the eid list to be freed; + +5. 返回值 + +void + +### 3.2.12 ubcore_query_device_attr + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_query_device_attr([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, [4.2.12.1](#32121-ubcore_device_attr) [ubcore_device_attr](#32121-ubcore_device_attr) *attr); + +3. 描述 + +查询多个ubep device的属性和功能 + +4. 参数 + +@param[in] [Required] dev: ubcore_device; + +@param[out] attr: Return device attributes, user needs to allocate and free the memory; + +5. 返回值 + +Return: 0 on success, other value on error + +#### 3.2.12.1 ubcore_device_attr + +```c +struct ubcore_device_attr { + ubcore_guid guid; + uint16_t fe_idx; + ubcore_device_cap dev_cap; + uint32_t reserved_jetty_id_min; + uint32_t reserved_jetty_id_max; + ubcore_port_attr port_attr[UBCORE_MAX_PORT_CNT]; + uint8_t port_cnt; + bool virtualization; /* In VM or not, must set by driver when register device */ + bool tp_maintainer; /* device used to maintain TP resource */ + ubcore_pattern pattern; +}; +``` + +#### 3.2.12.2 ubcore_pattern + +```c +enum ubcore_pattern { + UBCORE_PATTERN_1 = 0, + UBCORE_PATTERN_3 +}; +``` + +#### 3.2.12.3 ubcore_guid + +```c +struct ubcore_guid { + uint8_t raw[UBCORE_GUID_SIZE]; +}; +``` + +#### 3.2.12.4 ubcore_device_cap + +```c +struct ubcore_device_cap { + ubcore_device_feat feature; + uint32_t max_jfc; + uint32_t max_jfs; + uint32_t max_jfr; + uint32_t max_jetty; + uint32_t max_tp_cnt; + uint32_t max_tpg_cnt; + /* max_vtp_cnt_per_fe * max_fe_cnt Equal to the number of VTPs on the entire card */ + uint32_t max_vtp_cnt_per_fe; + uint32_t max_jetty_grp; + uint32_t max_jetty_in_jetty_grp; + uint32_t max_rc; /* max rc queues */ + uint32_t max_jfc_depth; + uint32_t max_jfs_depth; + uint32_t max_jfr_depth; + uint32_t max_rc_depth; /* max depth of each rc queue */ + uint32_t max_jfs_inline_size; + uint32_t max_jfs_sge; + uint32_t max_jfs_rsge; + uint32_t max_jfr_sge; + uint64_t max_msg_size; + uint32_t max_read_size; + uint32_t max_write_size; + uint32_t max_cas_size; + uint32_t max_swap_size; + uint32_t max_fetch_and_add_size; + uint32_t max_fetch_and_sub_size; + uint32_t max_fetch_and_and_size; + uint32_t max_fetch_and_or_size; + uint32_t max_fetch_and_xor_size; + /* max read command outstanding count in the function entity */ + uint64_t max_rc_outstd_cnt; + uint32_t max_sip_cnt_per_fe; + uint32_t max_dip_cnt_per_fe; + uint32_t max_seid_cnt_per_fe; + uint16_t trans_mode; /* one or more from ubcore_transport_mode_t */ + uint16_t sub_trans_mode_cap; /* one or more from ubcore_sub_trans_mode_cap */ + uint16_t congestion_ctrl_alg; /* one or more mode from ubcore_congestion_ctrl_alg_t */ + uint16_t ceq_cnt; /* completion vector count */ + uint32_t max_tp_in_tpg; + uint32_t max_utp_cnt; + uint32_t max_oor_cnt; /* max OOR window size by packet */ + uint32_t mn; + uint32_t min_slice; /* 32K, 64K */ + uint32_t max_slice; /* 256K, 64K */ + ubcore_atomic_feat atomic_feat; + uint32_t max_eid_cnt; + uint32_t max_upi_cnt; + uint32_t max_netaddr_cnt; + uint16_t max_fe_cnt; /* PF: greater than or equal to 0; FE: must be 0 */ + uint64_t page_size_cap; +}; +``` + +#### 3.2.12.5 ubcore_device_feat + +```c +union ubcore_device_feat { + struct { + uint32_t oor : 1; + uint32_t jfc_per_wr : 1; + uint32_t stride_op : 1; + uint32_t load_store_op : 1; + uint32_t non_pin : 1; + uint32_t pmem : 1; + uint32_t jfc_inline : 1; + uint32_t spray_en : 1; + uint32_t selective_retrans : 1; + uint32_t live_migrate : 1; + uint32_t dca : 1; + uint32_t jetty_grp : 1; + uint32_t err_suspend : 1; + uint32_t outorder_comp : 1; + uint32_t mn : 1; + uint32_t clan : 1; + uint32_t muti_seg_per_token_id : 1; + uint32_t reserved : 15; + } bs; + uint32_t value; +}; +``` + +#### 3.2.12.6 ubcore_atomic_feat + +```c +union ubcore_atomic_feat { + struct { + uint32_t cas : 1; + uint32_t swap : 1; + uint32_t fetch_and_add : 1; + uint32_t fetch_and_sub : 1; + uint32_t fetch_and_and : 1; + uint32_t fetch_and_or : 1; + uint32_t fetch_and_xor : 1; + uint32_t reserved : 25; + } bs; + uint32_t value; +}; +``` + +#### 3.2.12.7 ubcore_slice + +```c +enum ubcore_slice { + UBCORE_SLICE_32K = 1 << 15, + UBCORE_SLICE_64K = 1 << 16, + UBCORE_SLICE_128K = 1 << 17, + UBCORE_SLICE_256K = 1 << 18 +}; +``` + +#### 3.2.12.8 ubcore_congestion_ctrl_alg + +```c +enum ubcore_congestion_ctrl_alg { + UBCORE_CC_NONE = 0x1 << UBCORE_TP_CC_NONE, + UBCORE_CC_DCQCN = 0x1 << UBCORE_TP_CC_DCQCN, + UBCORE_CC_DCQCN_AND_NETWORK_CC = 0x1 << UBCORE_TP_CC_DCQCN_AND_NETWORK_CC, + UBCORE_CC_LDCP = 0x1 << UBCORE_TP_CC_LDCP, + UBCORE_CC_LDCP_AND_CAQM = 0x1 << UBCORE_TP_CC_LDCP_AND_CAQM, + UBCORE_CC_LDCP_AND_OPEN_CC = 0x1 << UBCORE_TP_CC_LDCP_AND_OPEN_CC, + UBCORE_CC_HC3 = 0x1 << UBCORE_TP_CC_HC3, + UBCORE_CC_DIP = 0x1 << UBCORE_TP_CC_DIP, + UBCORE_CC_ACC = 0x1 << UBCORE_TP_CC_ACC +}; +``` + +#### 3.2.12.9 ubcore_port_attr + +```c +struct ubcore_port_attr { + ubcore_mtu max_mtu; +}; +``` + +### 3.2.13 ubcore_query_device_status + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_query_device_status([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, [4.2.13.1](#32131-ubcore_device_status) [ubcore_device_status](#32131-ubcore_device_status) *status); + +3. 描述 + +应用查询设备状态。 + +4. 参数 + +@param[in] [Required] dev: ubcore device, by get_device apis. + +@param[out] [Required] status: status returned to client. + +5. 返回值 + +0 on success, other value on error + +#### 3.2.13.1 ubcore_device_status + +```c +struct ubcore_device_status { + ubcore_port_status port_status[UBCORE_MAX_PORT_CNT]; +}; +``` + +#### 3.2.13.2 ubcore_port_status + +```c +struct ubcore_port_status { + ubcore_port_state state; /* PORT_DOWN, PORT_INIT, PORT_ACTIVE */ + ubcore_speed active_speed; /* bandwidth */ + ubcore_link_width active_width; /* link width: X1, X2, X4 */ + ubcore_mtu active_mtu; +}; +``` + +#### 3.2.13.3 ubcore_port_state + +```c +enum ubcore_port_state { + UBCORE_PORT_NOP = 0, + UBCORE_PORT_DOWN, + UBCORE_PORT_INIT, + UBCORE_PORT_ARMED, + UBCORE_PORT_ACTIVE, + UBCORE_PORT_ACTIVE_DEFER +}; +``` + +#### 3.2.13.4 ubcore_speed + +```c +enum ubcore_speed { + UBCORE_SP_10M = 0, + UBCORE_SP_100M, + UBCORE_SP_1G, + UBCORE_SP_2_5G, + UBCORE_SP_5G, + UBCORE_SP_10G, + UBCORE_SP_14G, + UBCORE_SP_25G, + UBCORE_SP_40G, + UBCORE_SP_50G, + UBCORE_SP_100G, + UBCORE_SP_200G, + UBCORE_SP_400G, + UBCORE_SP_800G +}; +``` + +#### 3.2.13.5 ubcore_link_width + +```c +enum ubcore_link_width { + UBCORE_LINK_X1 = 0x1, + UBCORE_LINK_X2 = 0x1 << 1, + UBCORE_LINK_X4 = 0x1 << 2, + UBCORE_LINK_X8 = 0x1 << 3, + UBCORE_LINK_X16 = 0x1 << 4, + UBCORE_LINK_X32 = 0x1 << 5 +}; +``` + +### 3.2.14 ubcore_cgroup_reg_dev + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +void ubcore_cgroup_reg_dev([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev); + +3. 描述 + +将设备加入到cgroup，支持对资源进行计数控制 + +4. 参数 + +@param[in] [Required] dev: ubcore device, by get_device apis. + +5. 返回值 + +void + +### 3.2.15 ubcore_cgroup_unreg_dev + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +void ubcore_cgroup_unreg_dev([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev); + +3. 描述 + +将设备从cgroup移除 + +4. 参数 + +@param[in] [Required] dev: ubcore device, by get_device apis. + +5. 返回值 + +无 + +### 3.2.16 ubcore_cgroup_try_charge + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_cgroup_try_charge([4.2.16.1](#32161-struct-ubcore_cg_object) [struct ubcore_cg_object](#32161-struct-ubcore_cg_object) *cg_obj, [4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, [4.2.16.2](#32162-enum-ubcore_resource_type) [enum ubcore_resource_type](#32162-enum-ubcore_resource_type) type); + +3. 描述 + +尝试消耗一次设备上的资源计数 + +4. 参数 + +@param[in] [Required] cg_obj: the cgroup obj; + +@param[in] [Required] dev: the ubcore device handle; + +@param[in] type: the cgroup resource type + +5. 返回值 + +0 on success, other value on error + +#### 3.2.16.1 struct ubcore_cg_object + +```c +struct ubcore_cg_object { +#ifdef CONFIG_CGROUP_RDMA + struct rdma_cgroup *cg; +#endif +}; +``` + +#### 3.2.16.2 enum ubcore_resource_type + +```c +enum ubcore_resource_type { + UBCORE_RESOURCE_HCA_HANDLE = 0, + UBCORE_RESOURCE_HCA_OBJECT, + UBCORE_RESOURCE_HCA_MAX +}; +``` + +当前支持两种资源类型，分别是 + +UBCORE_RESOURCE_HCA_HANDLE ：对context进行计数 + +UBCORE_RESOURCE_HCA_OBJECT： 对如下资源进行计数 + +token, segement, target_seg, jfr, jfs, jfc, target_jfr, jetty, target jetty, jetty_group + +### 3.2.17 ubcore_cgroup_uncharge + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_cgroup_uncharge([4.2.16.1](#32161-struct-ubcore_cg_object) [struct ubcore_cg_object](#32161-struct-ubcore_cg_object) *cg_obj, [4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, [4.2.16.2](#32162-enum-ubcore_resource_type) [enum ubcore_resource_type](#32162-enum-ubcore_resource_type) type); + +3. 描述 + +释放一次设备上的资源计数 + +4. 参数 + +@param[in] [Required] cg_obj: the cgroup obj; + +@param[in] [Required] dev: the ubcore device handle; + +@param[in] type: the cgroup resource type. + +5. 返回值 + +void + +### 3.2.18 ubcore_get_mtu + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +[4.2.18.1](#32181-ubcore_mtu) [ubcore_mtu](#32181-ubcore_mtu) ubcore_get_mtu(int mtu); + +3. 描述 + +获得UB MTU的值。由驱动调用。 + +4. 参数 + +@param[in] [Required] mtu: specifies the MTU value of the NIC interface. + +5. 返回值 + +The MTU of the UB protocol, this value removes the length of the network layer, transport layer, transaction layer header and ICRC. + +#### 3.2.18.1 ubcore_mtu + +```c +enum ubcore_mtu { + UBCORE_MTU_256 = 1, + UBCORE_MTU_512, + UBCORE_MTU_1024, + UBCORE_MTU_2048, + UBCORE_MTU_4096, + UBCORE_MTU_8192 +}; +``` + +### 3.2.19 ubcore_recv_req + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +int ubcore_recv_req([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, [4.2.19.1](#32191-ubcore_req_host) [ubcore_req_host](#32191-ubcore_req_host) *req); + +3. 描述 + +ubcore用于接收请求消息。当驱动接收到消息，调用该接口把数据传递给ubcore。驱动分配和释放msg的空间。 + +4. 参数 + +@param[in] dev: TPF device; + +@param[in] msg: received msg; + +5. 返回值 + +0 on success, other value on error + +#### 3.2.19.1 ubcore_req_host + +```c +struct ubcore_req_host { + uint16_t src_fe_idx; + ubcore_req req; +}; +``` + +#### 3.2.19.2 ubcore_req + +```c +struct ubcore_req { + uint32_t msg_id; + ubcore_msg_opcode opcode; + uint32_t len; + uint8_t data[0]; +}; +``` + +#### 3.2.19.3 ubcore_msg_opcode + +```c +enum ubcore_msg_opcode { + /* 630 Verion msg start */ + UBCORE_MSG_CREATE_VTP = 0x0, + UBCORE_MSG_DESTROY_VTP = 0x1, + UBCORE_MSG_ALLOC_EID = 0x2, + UBCORE_MSG_DEALLOC_EID = 0x3, + UBCORE_MSG_CONFIG_DEVICE = 0x4, + UBCORE_MSG_VTP_STATUS_NOTIFY = 0x5, // TPF notify PF/VF + /* 630 Verion msg end. Do not change! */ + /* 930 Verion msg start. */ + UBCORE_MSG_UPDATE_EID_TABLE_NOTIFY = 0x6, // TPF notify PF/VF + UBCORE_MSG_FE2TPF_TRANSFER = 0x7, // FE-TPF common transfer + /* 930 Verion msg end. */ + /* 630 Verion msg start */ + UBCORE_MSG_STOP_PROC_VTP_MSG = 0x10, // Live migration + UBCORE_MSG_QUERY_VTP_MIG_STATUS = 0x11, // Live migration + UBCORE_MSG_FLOW_STOPPED = 0x12, // Live migration + UBCORE_MSG_MIG_ROLLBACK = 0x13, // Live migration + UBCORE_MSG_MIG_VM_START = 0x14, // Live migration + UBCORE_MSG_NEGO_VER = 0x15, // Verion negotiation, processed by backend ubcore. + /* 630 Verion msg start end. Do not change! */ + /* 930 Verion msg start. */ + UBCORE_MSG_NOTIFY_FASTMSG_DRAIN = 0x16, + /* 930 Verion msg end. */ + UBCORE_MSG_UPDATE_NET_ADDR = 0x17, + UBCORE_MSP_UPDATE_EID = 0x18 +}; +``` + +### 3.2.20 ubcore_recv_resp + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +int ubcore_recv_resp([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, [4.2.20.1](#32201-ubcore_resp) [ubcore_resp](#32201-ubcore_resp)*resp); + +3. 描述 + +ubcore用于接收响应消息。当驱动接收到消息，调用该接口把数据传递给ubcore。驱动分配和释放msg的空间。 + +4. 参数 + +@param[in] dev: VF or PF device; + +@param[in] msg: received msg; + +5. 返回值 + +0 on success, other value on error + +#### 3.2.20.1 ubcore_resp + +```c +struct ubcore_resp { + uint32_t msg_id; + ubcore_msg_opcode opcode; + uint32_t len; + uint8_t data[0]; +}; +``` + +### 3.2.21 ubcore_get_device_by_eid + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +[4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *ubcore_get_device_by_eid([4.2.10.2](#32102-ubcore_eid) [ubcore_eid](#32102-ubcore_eid), [4.2.21.1](#32211-ubcore_transport_type) [ubcore_transport_type](#32211-ubcore_transport_type) type); + +3. 描述 + +根据eid和传输类型找到UB设备 + +4. 参数 + +@param[in] eid: the eid of device; + +@param[in] type: transport type; + +5. 返回值 + +ubcore device pointer on success, NULL on error + +#### 3.2.21.1 ubcore_transport_type + +```c +enum ubcore_transport_type { + UBCORE_TRANSPORT_INVALID = -1, + UBCORE_TRANSPORT_UB = 0, + UBCORE_TRANSPORT_MAX +}; +``` + +## 3.3 segment管理 + +### 3.3.1 ubcore_alloc_token_id + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +[4.3.1.3](#3313-ubcore_token_id) [ubcore_token_id](#3313-ubcore_token_id)*ubcore_alloc_token_id([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, [4.3.1.1](#3311-ubcore_token_id_flag) [ubcore_token_id_flag](#3311-ubcore_token_id_flag) flag, [4.3.1.2](#3312-ubcore_udata) [ubcore_udata](#3312-ubcore_udata) *udata); + +3. 描述 + +内核态应用传入设备指针申请token id，用户态应用通过udata指明token id所属的上下文申请token id。Key id表示一个UBMMU内存地址空间，两个用户态进程不能共享同一个token id，但一个应用的两个segment可以共享同一个token id。内核态的两个segment也可以共享同一个token id。 + +4. 参数 + +@param[in] [Required] dev: the ubcore device handle; + +@param[in] flag: token_id_flag; + +@param[in] [Required] udata：ucontext and user space driver data; + +5. 返回值 + +成功则返回token id指针，失败返回NULL + +#### 3.3.1.1 ubcore_token_id_flag + +```c +union ubcore_token_id_flag { + struct { + uint32_t pa : 1; + uint32_t multi_seg : 1; + uint32_t reserved : 30; + } bs; + uint32_t value; +}; +``` + +#### 3.3.1.2 ubcore_udata + +```c +struct ubcore_udata { + ubcore_ucontext *uctx; + ubcore_udrv_priv *udrv_data; +}; +``` + +#### 3.3.1.3 ubcore_token_id + +```c +struct ubcore_token_id { + ubcore_device *ub_dev; + ubcore_ucontext *uctx; + uint32_t token_id; // driver fill + ubcore_token_id_flag flag; + atomic_t use_cnt; +}; +``` + +### 3.3.2 ubcore_free_token_id + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_free_token_id([4.3.1.3](#3313-ubcore_token_id) [ubcore_token_id](#3313-ubcore_token_id) *token_id); + +3. 描述 + +释放UB设备注册的token id + +4. 参数 + +@param[in] [Required] token_id：the token_id id alloced before; + +5. 返回值 + +0 on success, other value on error + +### 3.3.3 ubcore_register_seg + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +[4.3.3.4](#3334-ubcore_target_seg) [ubcore_target_seg](#3334-ubcore_target_seg) *ubcore_register_seg([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, [4.3.3.1](#3331-ubcore_seg_cfg) [ubcore_seg_cfg](#3331-ubcore_seg_cfg) *cfg, [4.3.1.2](#3312-ubcore_udata) [ubcore_udata](#3312-ubcore_udata) *udata); + +3. 描述 + +指定内核va地址、长度、flag，token value注册内存 + +注册时支持用户指定token id或不指定token id，如果用户没有指定token id，则输出返回新的token id。服务器端应用可以将返回的tseg中的seg字段传递给客户端，供客户端导入和读写。注册segment时，如果指定了remote write或remote atomic权限，那么应用也必须指定local write权限。 + +4. 参数 + +@param[in] [Required] dev: the ubcore device handle; + +@param[in] [Required] cfg：segment configurations; + +@param[in] [Required] udata: ucontext and user space driver data; + +5. 返回值 + +成功返回target segment指针，失败返回NULL + +#### 3.3.3.1 ubcore_seg_cfg + +```c +struct ubcore_seg_cfg { + uint64_t va; + uint64_t len; + uint32_t eid_index; + ubcore_token_id *token_id; + ubcore_token token_value; + ubcore_reg_seg_flag flag; + uint64_t user_ctx; + uint64_t iova; +}; +``` + +#### 3.3.3.2 ubcore_token + +```c +struct ubcore_token { + uint32_t token; +}; +``` + +#### 3.3.3.3 ubcore_reg_seg_flag + +```c +#define UBCORE_ACCESS_LOCAL_ONLY 0x1 +#define UBCORE_ACCESS_READ (0x1 << 1) +#define UBCORE_ACCESS_WRITE (0x1 << 2) +#define UBCORE_ACCESS_ATOMIC (0x1 << 3) +union ubcore_reg_seg_flag { + struct { + uint32_t token_policy : 3; + uint32_t cacheable : 1; + uint32_t dsva : 1; + uint32_t access : 6; + uint32_t non_pin : 1; + uint32_t user_iova : 1; + uint32_t token_id_valid : 1; + uint32_t pa : 1; + uint32_t reserved : 17; + } bs; + uint32_t value; +}; +``` + +![](figures/urma_caution.png) + +1、注册segment时，无论用户指定任何标识，segment权限默认具有本端读、写和原子操作权限； + +2、注册segment时，指定UBCORE_ACCESS_LOCAL_ONLY标识的情况下，不允许再指定其他标识，否则urma将会拦截此错误配置； + +3、注册segment时，只有不指定UBCORE_ACCESS_LOCAL_ONLY，才允许指定其他标识，此时segment权限除默认的本端读、写和原子操作权限，远端权限按照用户配置生效； + +4、注册segment时，配置UBCORE_ACCESS_WRITE标识，必须也配置UBCORE_ACCESS_READ标识，否则urma将会拦截此错误配置； + +5、注册segment时，配置UBCORE_ACCESS_ATOMIC标识，必须也配置UBCORE_ACCESS_READ和UBCORE_ACCESS_WRITE标识，否则urma将会拦截此错误配置。 + +#### 3.3.3.4 ubcore_target_seg + +```c +struct ubcore_target_seg { + ubcore_device *ub_dev; + ubcore_ucontext *uctx; + ubcore_seg seg; + uint64_t mva; + ubcore_token_id *token_id; + atomic_t use_cnt; +}; +``` + +#### 3.3.3.5 ubcore_seg + +```c +struct ubcore_seg { + ubcore_ubva ubva; + uint64_t len; + ubcore_seg_attr attr; + uint32_t token_id; +}; +``` + +#### 3.3.3.6 ubcore_ubva + +```c +struct ubcore_ubva { + ubcore_eid eid; + uint64_t va; +} \_\_packed; +``` + +#### 3.3.3.7 ubcore_seg_attr + +```c +union ubcore_seg_attr { + struct { + uint32_t token_policy : 3; + uint32_t cacheable : 1; + uint32_t dsva : 1; + uint32_t access : 6; + uint32_t non_pin : 1; + uint32_t user_iova : 1; + uint32_t user_token_id : 1; + uint32_t pa : 1; + uint32_t reserved : 17; + } bs; + uint32_t value; +}; +``` + +### 3.3.4 ubcore_unregister_seg + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_unregister_seg([4.3.3.4](#3334-ubcore_target_seg) [ubcore_target_seg](#3334-ubcore_target_seg) *tseg); + +3. 描述 + +注销segment。服务器端应用注销segment之前，应该自行确认客户端应用已经反导入了这个segment；同时应该保证服务器端应用不再使用tseg进行数据面操作。 + +4. 参数 + +@param[in] [Required] tseg: 注册得到的tseg指针; + +5. 返回值 + +Return: 0 on success, other value on erron + +### 3.3.5 ubcore_import_seg + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +[4.3.3.4](#3334-ubcore_target_seg) [ubcore_target_seg](#3334-ubcore_target_seg) *ubcore_import_seg([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, [4.3.5.1](#3351-ubcore_target_seg_cfg) [ubcore_target_seg_cfg](#3351-ubcore_target_seg_cfg) *cfg, [4.3.1.2](#3312-ubcore_udata) [ubcore_udata](#3312-ubcore_udata) *udata); + +3. 描述 + +用户传入UBVA、长度、token id，token value等信息导入远端内存。应用需要保证远端segment配置信息为有效值，例如token value与注册时指定的token value相符。 + +4. 参数 + +@param[in] [Required] dev: ubcore_device指针; + +@param[in] [Required] cfg：待导入的远端segment配置信息; + +@param[in] [Required] udata: 用户态驱动自定义数据。内核态应用直接调用该接口时，填NULL; + +5. 返回值 + +Return: pointer to target segment on success, NULL on error + +#### 3.3.5.1 ubcore_target_seg_cfg + +```c +struct ubcore_target_seg_cfg { + ubcore_seg seg; + ubcore_import_seg_flag flag; + uint64_t mva; /* optional */ + ubcore_token token_value; +}; +``` + +#### 3.3.5.2 ubcore_import_seg_flag + +```c +union ubcore_import_seg_flag { + struct { + uint32_t cacheable : 1; + uint32_t access : 6; + uint32_t mapping : 1; + uint32_t reserved : 24; + } bs; + uint32_t value; +}; +``` + +### 3.3.6 ubcore_unimport_seg + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_unimport_seg([4.3.3.4](#3334-ubcore_target_seg) [ubcore_target_seg](#3334-ubcore_target_seg) *tseg); + +3. 描述 + +反导入远端内存。应用unimport 这个tseg之前，需要保证数据面不再使用tseg读写。 + +4. 参数 + +@param[in] [Required] tseg: the address of the target segment to unimport; + +5. 返回值 + +Return: 0 on success, other value on error + +## 3.4 Jetty管理 + +### 3.4.1 JFC管理 + +#### 3.4.1.1 ubcore_create_jfc + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +[4.4.1.1.5](#34115-ubcore_jfc) [ubcore_jfc](#34115-ubcore_jfc) *ubcore_create_jfc([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, [4.4.1.1.1](#34111-ubcore_jfc_cfg) [ubcore_jfc_cfg](#34111-ubcore_jfc_cfg) *cfg, + +[4.4.1.1.3](#34113-ubcore_comp_callback_t) [ubcore_comp_callback_t](#34113-ubcore_comp_callback_t) jfce_handler, [4.4.1.1.4](#34114-ubcore_event_callback_t) [ubcore_event_callback_t](#34114-ubcore_event_callback_t) jfae_handler, [4.3.1.2](#3312-ubcore_udata) [ubcore_udata](#3312-ubcore_udata) *udata); + +3. 描述 + +用户传入深度（以CR为单位）、EQ ID，flag（lock free，jfc_inline）等参数创建JFC。创建成功返回的实际JFC深度不小于用户的传入的深度。 + +用户创建JFC时，传入完成事件回调函数和异步事件回调函数，驱动调用回调函数将事件通知给用户。用户还可以传入自定义jfc_context。 + +用户可以关联一个JFC创建一个或多个JFS，JFR或jetty，发送和接收完成也可以共享一个JFC。 + +为了防止JFC溢出，建议应用： + +（1）JFC的深度可以设置成所有共享它的JFS、JFR或jetty深度总和。 + +（2）应用应该及时读取完成记录，保证JFC不溢出，否则将会上报JFC异步事件。应用可以使用轮询的方式不间断的读取完成记录；也可以事件触发，在回调函数中读取完成记录。 + +硬件可能构造错误CR通知应用Jetty或JFS的状态发生了变化，构造的错误CR类型包括UBCORE_CR_WR_FLUSH_ERR_DONE和UBCORE_CR_WR_SUSPEND_ERR_DONE，应用应该为JFC预留足够的空间来存放硬件构造的CR，否则JFC会发生溢出。 + +4. 参数 + +@param[in] [Required] dev：ubcore_device指针; + +@param[in] [Required] cfg: jfc配置信息; + +@param[in] [Required] jfce_handler: 完成事件回调函数; + +@param[in] [Required] jfae_handler: 异步事件回调函数; + +@param[in] [Required] udata: 用户态驱动自定义数据。内核态应用直接调用该接口时，填NULL; + +5. 返回值 + +Return: the handle of created jfc, not NULL on success; NULL on error + +##### 3.4.1.1.1 ubcore_jfc_cfg + +```c +struct ubcore_jfc_cfg { + uint32_t depth; + ubcore_jfc_flag flag; + uint32_t ceqn; + void *jfc_context; +}; +``` + +eq_id：表示JFC使用的EQ编号，从0到device中的num_comp_vectors - 1 + +##### 3.4.1.1.2 ubcore_jfc_flag + +```c +union ubcore_jfc_flag { + struct { + uint32_t lock_free : 1; + uint32_t jfc_inline : 1; + uint32_t reserved : 30; + } bs; + uint32_t value; +}; +``` + +##### 3.4.1.1.3 ubcore_comp_callback_t + +typedef void (*ubcore_comp_callback_t)([4.4.1.1.5](#34115-ubcore_jfc) [ubcore_jfc](#34115-ubcore_jfc) *jfc); + +##### 3.4.1.1.4 ubcore_event_callback_t + +```c +typedef void (*ubcore_event_callback_t)(ubcore_event *event, +ubcore_ucontext *ctx); +``` + +##### 3.4.1.1.5 ubcore_jfc + +```c +struct ubcore_jfc { + ubcore_device *ub_dev; + ubcore_ucontext *uctx; + ubcore_jfc_cfg jfc_cfg; + uint32_t id; /* allocated by driver */ + ubcore_comp_callback_t jfce_handler; + ubcore_event_callback_t jfae_handler; + uint64_t urma_jfc; /* user space jfc pointer */ + struct hlist_node hnode; + atomic_t use_cnt; +}; +``` + +#### 3.4.1.2 ubcore_modify_jfc + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_modify_jfc([4.4.1.1.5](#34115-ubcore_jfc) [ubcore_jfc](#34115-ubcore_jfc) *jfc, [4.4.1.2.1](#34121-ubcore_jfc_attr) [ubcore_jfc_attr](#34121-ubcore_jfc_attr)r *attr, [4.3.1.2](#3312-ubcore_udata) [ubcore_udata](#3312-ubcore_udata) *udata); + +3. 描述 + +修改JFC的事件过滤属性，按照CR个数（moderate_count）和间隔（moderate_period）来抑制事件的上报。 + +4. 参数 + +@param[in] [Required] jfc: specify JFC; + +@param[in] [Required] attr: attributes to be modified; + +@param[in] [Required] udata: 用户态驱动自定义数据。内核态应用直接调用该接口时，填NULL; + +5. 返回值 + +Return: 0 on success, other value on error + +##### 3.4.1.2.1 ubcore_jfc_attr + +```c +struct ubcore_jfc_attr { + uint32_t mask; /* mask value refer to enum ubcore_jfc_attr_mask */ + uint16_t moderate_count; + uint16_t moderate_period; /* in micro seconds */ +}; +``` + +##### 3.4.1.2.2 ubcore_jfc_attr_mask + +```c +enum ubcore_jfc_attr_mask { + UBCORE_JFC_MODERATE_COUNT = 0x1, + UBCORE_JFC_MODERATE_PERIOD = 0x1 << 1 +}; +``` + +#### 3.4.1.3 ubcore_delete_jfc + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_delete_jfc([4.4.1.1.5](#34115-ubcore_jfc) [ubcore_jfc](#34115-ubcore_jfc) *jfc); + +3. 描述 + +销毁JFC。如果尚有其他JFS、JFR或jetty还在使用JFC，则返回销毁失败。 + +4. 参数 + +@param[in] [Required] jfc: handle of the created jfc; + +5. 返回值 + +Return: 0 on success, other value on error + +#### 3.4.1.4 ubcore_delete_jfc_batch + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_delete_jfc_batch([4.4.1.1.5](#34115-ubcore_jfc) [ubcore_jfc](#34115-ubcore_jfc) **jfc_arr, int jfc_num, int *bad_jfc_index); + +3. 描述 + +批量销毁jfc + +4. 参数 + +@param[in] jfc_arr: the jfc array created before; + +@param[in] jfc_num: jfc array length; + +@param[out] bad_jfc_index: when error, return error jfc index in the array; + +5. 返回值 + +0 on success, EINVAL on invalid parameter, other value on other batch delete errors. + +![](figures/urma_notice.png) + +1、如果发生删除失败的情况（包括非法的参数），接口会在第一个删除失败的jfc返回，在这个jfc之前的jfc都会被正常删除。 + +### 3.4.2 JFS管理 + +#### 3.4.2.1 ubcore_create_jfs + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +[4.4.2.1.4](#34214-ubcore_jfs) [ubcore_jfs](#34214-ubcore_jfs) *ubcore_create_jfs([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, [4.4.2.1.1](#34211-ubcore_jfs_cfg) [ubcore_jfs_cfg](#34211-ubcore_jfs_cfg) *cfg, + +[4.4.1.1.4](#34114-ubcore_event_callback_t) [ubcore_event_callback_t](#34114-ubcore_event_callback_t) jfae_handler, [4.3.1.2](#3312-ubcore_udata) [ubcore_udata](#3312-ubcore_udata) *udata); + +3. 描述 + +用户传入深度（以WR为单位）、关联的JFC、flag（lock free）、优先级、sge数、最大inline长度、优先级、rnr_rertry, err_timeout等参数创建RM或UM模式的JFS。用户指定的规格，例如sge数、rsge（当前只支持等于1）最大inline长度不能超过设备的规格。创建成功的JFS深度、sge和inline长度都不小于用户指定的规格。 + +用户创建JFS时，可以传入自定义jfs_context和异步事件回调函数。 + +创建JFS时可以配置事务层重传参数，rnr_rertry表示对端接收未就绪导致的重传次数， err_timeout是事务层超时上报错误时间。 + +4. 参数 + +@param[in] [Required] dev：ubcore_device指针; + +@param[in] [Required] cfg: jfs配置信息; + +@param[in] [Required] jfae_handler: 异步事件回调函数; + +@param[in] [Required] udata: 用户态驱动自定义数据。内核态应用直接调用该接口时，填NULL; + +5. 返回值 + +Return: the handle of created jfs, not NULL on success, NULL on error + +##### 3.4.2.1.1 ubcore_jfs_cfg + +```c +struct ubcore_jfs_cfg { + uint32_t depth; + ubcore_jfs_flag flag; + ubcore_transport_mode trans_mode; + uint32_t eid_index; + uint8_t priority; + uint8_t max_sge; + uint8_t max_rsge; + uint32_t max_inline_data; + uint8_t rnr_retry; + uint8_t err_timeout; + void *jfs_context; + ubcore_jfc *jfc; +}; +``` + +##### 3.4.2.1.2 ubcore_jfs_flag + +```c +enum ubcore_order_type { + UBCORE_DEF_ORDER, + UBCORE_OT, // target ordering + UBCORE_OI, // initiator ordering + UBCORE_OL, // low layer ordering + UBCORE_NO // unreliable non ordering +}; +union ubcore_jfs_flag { + struct { + uint32_t lock_free : 1; + uint32_t error_suspend : 1; + uint32_t outorder_comp : 1; + uint32_t order_type : 8; /* (0x0): default, auto config by driver */ + /* (0x1): OT, target ordering */ + /* (0x2): OI, initiator ordering */ + /* (0x3): OL, low layer ordering */ + /* (0x4): UNO, unreliable non ordering */ + uint32_t multi_path : 1; + uint32_t ctp_rc_mul_path_mode : 1; /* 1: ctp rc mode multi-path */ + uint32_t reserved : 19; + } bs; + uint32_t value; +}; +``` + +outorder_comp为1时，表示完成乱序模式，WR不支持fence, place_order和comp_order； + +##### 3.4.2.1.3 ubcore_transport_mode + +```c +enum ubcore_transport_mode { + UBCORE_TP_RM = 0x1, /* Reliable message */ + UBCORE_TP_RC = 0x1 << 1, /* Reliable connection */ + UBCORE_TP_UM = 0x1 << 2 /* Unreliable message */ +}; +``` + +##### 3.4.2.1.4 ubcore_jfs + +```c +struct ubcore_jfs { + ubcore_device *ub_dev; + ubcore_ucontext *uctx; + ubcore_jfs_cfg jfs_cfg; + ubcore_jetty_id jfs_id; /* driver fill jfs_id-\>id */ + ubcore_event_callback_t jfae_handler; + uint64_t urma_jfs; /* user space jfs pointer */ + struct hlist_node hnode; + atomic_t use_cnt; + struct kref ref_cnt; + struct completion comp; + ubcore_hash_table *tptable; /* Only for devices not natively supporting RM mode */ +}; +``` + +##### 3.4.2.1.5 ubcore_jetty_id + +```c +struct ubcore_jetty_id { + ubcore_eid eid; + uint32_t id; +}; +``` + +#### 3.4.2.2 ubcore_modify_jfs + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_modify_jfs( [4.4.2.1.4](#34214-ubcore_jfs) [ubcore_jfs](#34214-ubcore_jfs) *jfs, [4.4.2.2.1](#34221-ubcore_jfs_attr) [ubcore_jfs_attr](#34221-ubcore_jfs_attr) *attr, [4.3.1.2](#3312-ubcore_udata) [ubcore_udata](#3312-ubcore_udata) *udata); + +3. 描述 + +UB Core调用UBN厂商驱动修改JFS，支持修改RX WR的最低水位和状态 + +4. 参数 + +@param[in] [Required] jfs: specify JFs; + +@param[in] [Required] attr: attributes to be modified; + +@param[in] [Required] udata: 用户态驱动自定义数据。内核态应用直接调用该接口时，填NULL; + +5. 返回值 + +Return: 0 on success, other value on error + +##### 3.4.2.2.1 ubcore_jfs_attr + +```c +struct ubcore_jfs_attr { + uint32_t mask; /* mask value refer to ubcore_jfs_attr_mask_t */ + ubcore_jetty_state state; +}; +``` + +##### 3.4.2.2.2 ubcore_jfs_attr_mask + +```c +enum ubcore_jfs_attr_mask { + UBCORE_JFS_STATE = 0x1 +}; +``` + +##### 3.4.2.2.3 ubcore_jetty_state + +```c +enum ubcore_jetty_state { + UBCORE_JETTY_STATE_RESET = 0, + UBCORE_JETTY_STATE_READY, + UBCORE_JETTY_STATE_SUSPENDED, + UBCORE_JETTY_STATE_ERROR +}; +``` + +#### 3.4.2.3 ubcore_query_jfs + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_query_jfs([4.4.2.1.4](#34214-ubcore_jfs) [ubcore_jfs](#34214-ubcore_jfs) *jfs, [4.4.2.1.1](#34211-ubcore_jfs_cfg) [ubcore_jfs_cfg](#34211-ubcore_jfs_cfg) *cfg, [4.4.2.2.1](#34221-ubcore_jfs_attr) [ubcore_jfs_attr](#34221-ubcore_jfs_attr) *attr); + +3. 描述 + +查询JFS配置和属性，需要指定JFS。 + +4. 参数 + +@param[in] [Required] jfs: specify JFs; + +@param[out] [Required] cfg: attributes to be query; + +@param[out] [Required] attr: attributes to be query; + +5. 返回值 + +Return: 0 on success, other value on error + +#### 3.4.2.4 ubcore_delete_jfs + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_delete_jfs([4.4.2.1.4](#34214-ubcore_jfs) [ubcore_jfs](#34214-ubcore_jfs) *jfs); + +3. 描述 + +删除一个已经创建的jfs + +4. 参数 + +@param[in] [Required] jfs: the jfs created before; + +5. 返回值 + +Return: 0 on success, other value on error + +#### 3.4.2.5 ubcore_delete_jfs_batch + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_delete_jfs_batch([4.4.2.1.4](#34214-ubcore_jfs) [ubcore_jfs](#34214-ubcore_jfs) **jfs_arr, int jfs_num, int *bad_jfs_index); + +3. 描述 + +批量删除jfs + +4. 参数 + +@param[in] jfs_arr: the jfs array created before; + +@param[in] jfs_num: jfs array length; + +@param[out] bad_jfs_index: when error, return error jfs index in the array; + +5. 返回值 + +0 on success, EINVAL on invalid parameter, other value on other batch delete errors. + +![](figures/urma_notice.png) + +1、如果发生删除失败的情况（包括非法的参数），接口会在第一个删除失败的jfs返回，在这个jfs之前的jfs都会被正常删除。 + +#### 3.4.2.6 ubcore_flush_jfs + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_flush_jfs([4.4.2.1.4](#34214-ubcore_jfs) [ubcore_jfs](#34214-ubcore_jfs) *jfs, int cr_cnt, [4.4.2.6.1](#34261-ubcore_cr) [ubcore_cr](#34261-ubcore_cr) *cr); + +3. 描述 + +udma驱动把jfs中未被硬件执行的wr，通过cr返回给应用。 + +4. 参数 + +@param[in] [Required] jfs: the jfs created before; + +@param[in] [Required] cr_cnt: 希望收到完成记录的个数; + +@param[out] [Required] cr: 存放完成记录信息的地址; + +5. 返回值 + +the number of completion record returned, 0 means no completion record returned, -1 on error + +##### 3.4.2.6.1 ubcore_cr + +```c +struct ubcore_cr { + ubcore_cr_status status; + uint64_t user_ctx; + ubcore_cr_opcode opcode; + ubcore_cr_flag flag; + uint32_t completion_len; /* The number of bytes transferred */ + uint32_t local_id; /* Local jetty ID, or JFS ID, or JFR ID, depending on flag */ + /* Valid only for receiving CR. The remote jetty where received msg + * comes from, may be jetty ID or JFS ID, depending on flag. + */ + ubcore_jetty_id remote_id; + union { + uint64_t imm_data; /* Valid only for received CR */ + struct ubcore_cr_token invalid_token; + }; + uint32_t tpn; + uintptr_t user_data; /* Use as pointer to local jetty struct */ +}; +``` + +##### 3.4.2.6.2 ubcore_cr_status + +```c +/* Must be consistent with urma_cr_status_t */ +enum ubcore_cr_status { // completion record status + UBCORE_CR_SUCCESS = 0, + UBCORE_CR_UNSUPPORTED_OPCODE_ERR, + UBCORE_CR_LOC_LEN_ERR, // Local data too long error + UBCORE_CR_LOC_OPERATION_ERR, // Local operation err + UBCORE_CR_LOC_ACCESS_ERR, // Access to local memory error when WRITE_WITH_IMM + UBCORE_CR_REM_RESP_LEN_ERR, + UBCORE_CR_REM_UNSUPPORTED_REQ_ERR, + UBCORE_CR_REM_OPERATION_ERR, + /* Memory access protection error occurred in the remote node */ + UBCORE_CR_REM_ACCESS_ABORT_ERR, + UBCORE_CR_ACK_TIMEOUT_ERR, + /* RNR retries exceeded the maximum number: remote jfr has no buffer */ + UBCORE_CR_RNR_RETRY_CNT_EXC_ERR, + UBCORE_CR_FLUSH_ERR, + UBCORE_CR_WR_SUSPEND_DONE, + UBCORE_CR_WR_FLUSH_ERR_DONE, + UBCORE_CR_WR_UNHANDLED, + UBCORE_CR_LOC_DATA_POISON, + UBCORE_CR_REM_DATA_POISON +}; +``` + +##### 3.4.2.6.3 ubcore_cr_opcode + +```c +enum ubcore_cr_opcode { + UBCORE_CR_OPC_SEND = 0x00, + UBCORE_CR_OPC_SEND_WITH_IMM, + UBCORE_CR_OPC_SEND_WITH_INV, + UBCORE_CR_OPC_WRITE_WITH_IMM +}; +``` + +##### 3.4.2.6.4 ubcore_cr_flag + +```c +union ubcore_cr_flag { + struct { + uint8_t s_r : 1; /* Indicate CR stands for sending or receiving */ + uint8_t jetty : 1; /* Indicate id in the CR stands for jetty or JFS/JFR */ + uint8_t suspend_done : 1; + uint8_t flush_err_done : 1; + uint8_t reserved : 4; + } bs; + uint8_t value; +}; +``` + +##### 3.4.2.6.5 ubcore_cr_token + +```c +struct ubcore_cr_token { + uint32_t token_id; + ubcore_token token_value; +}; +``` + +### 3.4.3 JFR管理 + +#### 3.4.3.1 ubcore_create_jfr + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +[4.4.3.1.3](#34313-ubcore_jfr) [ubcore_jfr](#34313-ubcore_jfr) *ubcore_create_jfr([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, [4.4.3.1.1](#34311-ubcore_jfr_cfg) [ubcore_jfr_cfg](#34311-ubcore_jfr_cfg) *cfg, + +[4.4.1.1.4](#34114-ubcore_event_callback_t) [ubcore_event_callback_t](#34114-ubcore_event_callback_t) jfae_handler, [4.3.1.2](#3312-ubcore_udata) [ubcore_udata](#3312-ubcore_udata) *udata); + +3. 描述 + +用户传入深度（以WR为单位）、关联的JFC、flag（lock free、token策略）、sge数、token value、min_rnr_timer等参数创建RM或UM模式的JFR。创建成功的JFR深度、sge都不小于用户指定的规格。 + +JFR ID可以标识一项服务，因此用户可以指定JFR ID创建JFR。用户创建JFR时，可以传入自定义jfr_context和异步事件回调函数。创建JFR时只能配置事务层重传参数，min_rnr_timer表示接收端未就绪超时时间，超过时间发送NACK给发送端。 + +4. 参数 + +@param[in] [Required] dev：ubcore_device指针; + +@param[in] [Required] cfg: jfr配置信息; + +@param[in] [Required] jfae_handler: 异步事件回调函数; + +@param[in] [Required] udata: 用户态驱动自定义数据。内核态应用直接调用该接口时，填NULL; + +5. 返回值 + +Return: the handle of created jfr, not NULL on success, NULL on error + +##### 3.4.3.1.1 ubcore_jfr_cfg + +```c +struct ubcore_jfr_cfg { + uint32_t id; /* user may assign id */ + uint32_t depth; + ubcore_jfr_flag flag; + ubcore_transport_mode trans_mode; + uint32_t eid_index; + uint8_t max_sge; + uint8_t min_rnr_timer; + ubcore_token token_value; + ubcore_jfc *jfc; + void *jfr_context; +}; +``` + +##### 3.4.3.1.2 ubcore_jfr_flag + +```c +union ubcore_jfr_flag { + struct { + /* 0: UBCORE_TOKEN_NONE + * 1: UBCORE_TOKEN_PLAIN_TEXT + * 2: UBCORE_TOKEN_SIGNED + * 3: UBCORE_TOKEN_ALL_ENCRYPTED + * 4: UBCORE_TOKEN_RESERVED + */ + uint32_t token_policy : 3; + uint32_t tag_matching : 1; + uint32_t lock_free : 1; + uint32_t order_type : 8; /* (0x0): default, auto config by driver */ + /* (0x1): OT, target ordering */ + /* (0x2): OI, initiator ordering */ + /* (0x3): OL, low layer ordering */ + /* (0x4): UNO, unreliable non ordering */ + uint32_t reserved : 19; + } bs; + uint32_t value; +}; +``` + +##### 3.4.3.1.3 ubcore_jfr + +```c +struct ubcore_jfr { + ubcore_device *ub_dev; + ubcore_ucontext *uctx; + ubcore_jfr_cfg jfr_cfg; + ubcore_jetty_id jfr_id; /* driver fill jfr_id-\>id */ + ubcore_event_callback_t jfae_handler; + uint64_t urma_jfr; /* user space jfr pointer */ + struct hlist_node hnode; + atomic_t use_cnt; + struct kref ref_cnt; + struct completion comp; + ubcore_hash_table *tptable; /* Only for devices not natively supporting RM mode */ +}; +``` + +#### 3.4.3.2 ubcore_modify_jfr + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_modify_jfr([4.4.3.1.3](#34313-ubcore_jfr) [ubcore_jfr](#34313-ubcore_jfr) *jfr, [4.4.3.2.1](#34321-ubcore_jfr_attr) [ubcore_jfr_attr](#34321-ubcore_jfr_attr) *attr, [4.3.1.2](#3312-ubcore_udata) [ubcore_udata](#3312-ubcore_udata) *udata); + +3. 描述 + +修改JFR水线。当JFR中的WR（recv buffer）低于水线将触发JFR_LIMIT低水线异步事件，调用JFR的异步事件回调函数jfae_handler。应用获取到异步事件后，应该调用post_recv补充WR。应用设置的水线不能超过JFR深度，0表示不希望触发水线机制。修改水线时，应该考虑当前JFR存在的WR数，如果应用设定的水线大于当前JFR中存在的WR数，即刻就会收到低水线事件。 + +4. 参数 + +@param[in] [Required] jfr: specify JFR; + +@param[in] [Required] attr: attributes to be modified; + +@param[in] [Required] udata: 用户态驱动自定义数据。内核态应用直接调用该接口时，填NULL; + +5. 返回值 + +Return: 0 on success, other value on error + +##### 3.4.3.2.1 ubcore_jfr_attr + +```c +struct ubcore_jfr_attr { + uint32_t mask; /* mask value refer to enum ubcore_jfr_attr_mask */ + uint32_t rx_threshold; + ubcore_jfr_state state; +}; +``` + +##### 3.4.3.2.2 ubcore_jfr_attr_mask + +```c +enum ubcore_jfr_attr_mask { + UBCORE_JFR_RX_THRESHOLD = 0x1, + UBCORE_JFR_STATE = 0x1 << 1 +}; +``` + +##### 3.4.3.2.3 ubcore_jfr_state + +```c +enum ubcore_jfr_state { + UBCORE_JFR_STATE_RESET = 0, + UBCORE_JFR_STATE_READY, + UBCORE_JFR_STATE_ERROR +}; +``` + +#### 3.4.3.3 ubcore_query_jfr + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_query_jfr([4.4.3.1.3](#34313-ubcore_jfr) [ubcore_jfr](#34313-ubcore_jfr) *jfr, [4.4.3.1.1](#34311-ubcore_jfr_cfg) [ubcore_jfr_cfg](#34311-ubcore_jfr_cfg) *cfg, [4.4.3.2.1](#34321-ubcore_jfr_attr) [ubcore_jfr_attr](#34321-ubcore_jfr_attr) *attr); + +3. 描述 + +查询JFR配置和属性，需要指定JFR。 + +4. 参数 + +@param[in] [Required] jfr: specify JFR; + +@param[out] [Required] cfg: attributes to be query; + +@param[out] [Required] attr: attributes to be query; + +5. 返回值 + +Return: 0 on success, other value on error + +#### 3.4.3.4 ubcore_delete_jfr + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_delete_jfr([4.4.3.1.3](#34313-ubcore_jfr) [ubcore_jfr](#34313-ubcore_jfr) *jfr); + +3. 描述 + +destroy jfr from ubcore device. + +4. 参数 + +@param[in] [Required] jfr: the jfr created before; + +5. 返回值 + +Return: 0 on success, other value on error + +#### 3.4.3.5 ubcore_delete_jfr_batch + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_delete_jfr_batch([4.4.3.1.3](#34313-ubcore_jfr) [ubcore_jfr](#34313-ubcore_jfr) **jfr_arr, int jfr_num, int *bad_jfr_index); + +3. 描述 + +批量删除jfr + +4. 参数 + +@param[in] jfr_arr: the jfr array created before; + +@param[in] jfr_num: jfr array length; + +@param[out] bad_jfr_index: when error, return error jfr index in the array; + +5. 返回值 + +0 on success, EINVAL on invalid parameter, other value on other batch delete errors. + +![](figures/urma_notice.png) + +1、如果发生删除失败的情况（包括非法的参数），接口会在第一个删除失败的jfr返回，在这个jfr之前的jfr都会被正常删除。 + +#### 3.4.3.6 ubcore_import_jfr + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +[4.4.3.6.5](#34365-ubcore_tjetty) [ubcore_tjetty](#34365-ubcore_tjetty) *ubcore_import_jfr([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, [4.4.3.6.1](#34361-ubcore_tjetty_cfg) [ubcore_tjetty_cfg](#34361-ubcore_tjetty_cfg) *cfg, [4.3.1.2](#3312-ubcore_udata) [ubcore_udata](#3312-ubcore_udata) *udata); + +3. 描述 + +导入用户输入远端JFR信息，包括jfr id（包含eid），token value，传输模式等导入JFR，接口返回target jetty指针。应用应该保证传入的远端JFR信息真实有效，否则使用target jetty向远端JFR发送消息将会失败。 + +导入RM类型的JFR隐含与远端节点建链功能。导入UM类型的JFR隐含创建unreliable tp（其实是远端地址句柄）功能，记录在target jetty的tp中。 + +4. 参数 + +@param[in] [Required] dev：ubcore_device指针; + +@param[in] [Required] cfg: jfr配置信息; + +@param[in] [Required] udata: 用户态驱动自定义数据。内核态应用直接调用该接口时，填NULL; + +5. 返回值 + +Return: the address of target jfr, not NULL on success, NULL on error + +##### 3.4.3.6.1 ubcore_tjetty_cfg + +```c +struct ubcore_tjetty_cfg { + ubcore_jetty_id + id; /* jfr, jetty or jetty group id to be imported */ + ubcore_import_jetty_flag flag; + ubcore_transport_mode trans_mode; + uint32_t eid_index; + ubcore_target_type type; + ubcore_jetty_grp_policy policy; + ubcore_token token_value; /* jfr, jetty or jetty group token_value to be imported */ +}; +``` + +##### 3.4.3.6.2 ubcore_import_jetty_flag + +```c +union ubcore_import_jetty_flag { + struct { + uint32_t token_policy : 3; + uint32_t order_type : 8; /* (0x0): default, auto config by driver */ + /* (0x1): OT, target ordering */ + /* (0x2): OI, initiator ordering */ + /* (0x3): OL, low layer ordering */ + /* (0x4): UNO, unreliable non ordering */ + uint32_t share_tp : 1; + uint32_t reserved : 20; + } bs; + uint32_t value; +}; +``` + +##### 3.4.3.6.3 ubcore_target_type + +enum ubcore_target_type { UBCORE_JFR = 0, UBCORE_JETTY, UBCORE_JETTY_GROUP }; + +##### 3.4.3.6.4 ubcore_jetty_grp_policy + +```c +enum ubcore_jetty_grp_policy { + UBCORE_JETTY_GRP_POLICY_RR = 0, + UBCORE_JETTY_GRP_POLICY_HASH_HINT = 1 +}; +``` + +##### 3.4.3.6.5 ubcore_tjetty + +```c +struct ubcore_tjetty { + ubcore_device *ub_dev; + ubcore_ucontext *uctx; + ubcore_tjetty_cfg cfg; + ubcore_tp *tp; + ubcore_vtpn *vtpn; + atomic_t use_cnt; + struct mutex lock; +}; +``` + +##### 3.4.3.6.6 ubcore_tp + +```c +struct ubcore_tp { + uint32_t tpn; /* driver assigned in creating tp */ + uint32_t peer_tpn; + ubcore_device *ub_dev; + ubcore_tp_flag flag; /* indicate initiator or target, etc */ + uint32_t local_net_addr_idx; + struct ubcore_net_addr peer_net_addr; + /* only for RC START */ + union { + ubcore_eid local_eid; + ubcore_jetty_id local_jetty; + }; + union { + union ubcore_eid peer_eid; + struct ubcore_jetty_id peer_jetty; + }; + /* only for RC END */ + ubcore_transport_mode trans_mode; + ubcore_tp_state state; + uint32_t rx_psn; + uint32_t tx_psn; + ubcore_mtu mtu; + uint16_t data_udp_start; /* src udp port start, for multipath data */ + uint16_t ack_udp_start; /* src udp port start, for multipath ack */ + uint8_t udp_range; /* src udp port range, for both multipath data and ack */ + uint8_t port_id; /* optional, physical port, only for non-bonding */ + uint8_t retry_num; + uint8_t retry_factor; + uint8_t ack_timeout; + uint8_t dscp; + uint8_t cc_pattern_idx; + uint8_t hop_limit; + struct ubcore_tpg *tpg; /* NULL if no tpg, eg. UM mode */ + uint32_t oor_cnt; /* out of order window size for recv: packet cnt */ + uint32_t oos_cnt; /* out of order window size for send: packet cnt */ + struct ubcore_tp_ext tp_ext; /* driver fill in creating tp */ + struct ubcore_tp_ext peer_ext; /* ubcore fill before modifying tp */ + atomic_t use_cnt; + struct hlist_node hnode; /* driver inaccessible */ + struct kref ref_cnt; + struct completion comp; + uint32_t flow_label; + uint8_t mn; /* 0\~15, a packet contains only one msg if mn is set as 0 */ + ubcore_transport_type + peer_trans_type; /* Only for user tp connection */ + struct mutex lock; /* protect TP state */ + void *priv; /* ubcore private data for tp management */ + uint32_t ue_idx; +}; +``` + +##### 3.4.3.6.7 ubcore_vtpn + +```c +struct ubcore_vtpn { + uint32_t vtpn; /* driver fills */ + ubcore_device *ub_dev; + /* ubcore private, inaccessible to driver */ + ubcore_transport_mode trans_mode; + /* vtpn key start */ + ubcore_eid local_eid; + union ubcore_eid peer_eid; + uint32_t local_jetty; + uint32_t peer_jetty; + /* vtpn key end */ + uint32_t eid_index; + struct mutex state_lock; + ubcore_vtp_state state; /* protect by state_lock */ + struct hlist_node hnode; /* key: eid + jetty */ + struct hlist_node vtpn_hnode; /* key: vtpn */ + atomic_t use_cnt; + struct kref ref_cnt; + struct completion comp; + struct list_head node; /* vtpn node in vtpn_wait_list */ + struct list_head list; /* vtpn head to restore tjetty/jetty/cb node */ + struct list_head + disconnect_list; /* vtpn head to restore disconnect vtpn node */ + uint64_t tp_handle; + uint64_t peer_tp_handle; + uint64_t tag; + bool uspace; /* true: user space; false: kernel space */ +}; +``` + +##### 3.4.3.6.8 ubcore_vtp_state + +```c +enum ubcore_vtp_state { + UBCORE_VTPS_RESET = 0, + UBCORE_VTPS_READY = 1, + UBCORE_VTPS_WAIT_DESTROY = 2, +}; +``` + +#### 3.4.3.7 ubcore_import_jfr_ex + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +```c +struct ubcore_tjetty * +ubcore_import_jfr_ex(ubcore_device *dev, +ubcore_tjetty_cfg *cfg, +ubcore_active_tp_cfg *active_tp_cfg, +ubcore_udata *udata); +``` + +3. 描述 + +通过管控面导入用户输入远端JFR信息，包括jfr id（包含eid），token value，传输模式等导入JFR，接口返回target jetty指针。应用应该保证传入的远端JFR信息真实有效，否则使用target jetty向远端JFR发送消息将会失败。 + +导入RM类型的JFR隐含与远端节点建链功能。导入UM类型的JFR隐含创建unreliable tp（其实是远端地址句柄）功能，记录在target jetty的tp中。 + +4. 参数 + +@param[in] dev: the ubcore device handle; + +@param[in] cfg: remote jfr attributes and import configurations; + +@param[in] active_tp_cfg: tp configuration to active; + +@param[in] udata (optional): ucontext and user space driver data； + +5. 返回值 + +target jfr pointer on success, NULL on error + +##### 3.4.3.7.1 ubcore_active_tp_cfg + +```c +struct ubcore_active_tp_cfg { + ubcore_tp_handle tp_handle; + ubcore_tp_handle peer_tp_handle; + uint64_t tag; + ubcore_active_tp_attr tp_attr; +}; +``` + +##### 3.4.3.7.2 ubcore_active_tp_attr + +```c +struct ubcore_active_tp_attr { + uint32_t tx_psn; + uint32_t rx_psn; + uint64_t reserved; +}; +``` + +#### 3.4.3.8 ubcore_unimport_jfr + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_unimport_jfr([4.4.3.6.5](#34365-ubcore_tjetty) [ubcore_tjetty](#34365-ubcore_tjetty) *tjfr); + +3. 描述 + +反导入远端JFR。unimport JFR时，隐含：减少tp的引用计数，减到零时触发拆链流程。 + +反导入target jetty将会释放tjfr结构体，应用需要保证已经使用target JFR发起的请求都已经poll JFC得到完成记录（包括错误）。 + +4. 参数 + +@param[in] [Required] tjfr: the target jfr imported before; + +5. 返回值 + +Return: 0 on success, other value on error + +### 3.4.4 Jetty管理 + +#### 3.4.4.1 ubcore_create_jetty + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +[4.4.4.1.3](#34413-ubcore_jetty) [ubcore_jetty](#34413-ubcore_jetty) *ubcore_create_jetty([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, [4.4.4.1.1](#34411-ubcore_jetty_cfg) [ubcore_jetty_cfg](#34411-ubcore_jetty_cfg) *cfg, + +[4.4.1.1.4](#34114-ubcore_event_callback_t) [ubcore_event_callback_t](#34114-ubcore_event_callback_t) jfae_handler, [4.3.1.2](#3312-ubcore_udata) [ubcore_udata](#3312-ubcore_udata) *udata); + +3. 描述 + +创建RM\RC\UM模式的Jetty。创建Jetty时，用户需要传入发送配置参数，例如发送深度、关联的发送JFC、发送sge数、最大inline长度、优先级、rnr_rertry、 err_timeout等参数。用户创建Jetty时，可以传入自定义jetty_context和异步事件回调函数。 + +UB设备只支持Jetty绑定共享JFR，用户创建Jetty之前必须先创建接收共享JFR，创建Jetty时，必须传入JFR指针和接收JFC指针。 + +创建成功的jetty的发送深度、发送sge和inline长度，独占式的接收深度、接收sge不小于用户指定的规格。 + +与创建JFR类似，用户可以指定Jetty ID创建Jetty。对于每个PF或VF来说，Jetty ID和JFR ID复用同一个空间，不能重复。 + +支持cfg中指定jetty group指针，新创建的jetty自动加入jetty group。销毁jetty时，自动将jetty从jetty group中删除。 + +4. 参数 + +@param[in] [Required] dev：ubcore_device指针; + +@param[in] [Required] cfg: jetty配置信息; + +@param[in] [Required] jfae_handler: 异步事件回调函数; + +@param[in] [Required] udata: 用户态驱动自定义数据。内核态应用直接调用该接口时，填NULL; + +5. 返回值 + +Return: the handle of created jetty, not NULL on success, NULL on error + +##### 3.4.4.1.1 ubcore_jetty_cfg + +```c +struct ubcore_jetty_cfg { + uint32_t id; /* user may assign id */ + ubcore_jetty_flag flag; + ubcore_transport_mode trans_mode; + uint32_t eid_index; + uint32_t jfs_depth; + uint8_t priority; + uint8_t max_send_sge; + uint8_t max_send_rsge; + uint32_t max_inline_data; + uint8_t rnr_retry; + uint8_t err_timeout; + uint32_t jfr_depth; /* deprecated */ + uint8_t min_rnr_timer; /* deprecated */ + uint8_t max_recv_sge; /* deprecated */ + ubcore_token token_value; /* deprecated */ + ubcore_jfc *send_jfc; + ubcore_jfc *recv_jfc; /* must set */ + ubcore_jfr *jfr; /* must set, shared jfr */ + ubcore_jetty_group + *jetty_grp; /* [Optional] user specified jetty group */ + void *jetty_context; +}; +``` + +##### 3.4.4.1.2 ubcore_jetty_flag + +```c +union ubcore_jetty_flag { + struct { + uint32_t share_jfr : 1; /* 0: URMA_NO_SHARE_JFR. 1: URMA_SHARE_JFR. */ + uint32_t lock_free : 1; + uint32_t error_suspend : 1; + uint32_t outorder_comp : 1; + uint32_t order_type : 8; /* (0x0): default, auto config by driver */ + /* (0x1): OT, target ordering */ + /* (0x2): OI, initiator ordering */ + /* (0x3): OL, low layer ordering */ + /* (0x4): UNO, unreliable non ordering */ + uint32_t multi_path : 1; + uint32_t reserved : 19; + } bs; + uint32_t value; +}; +``` + +##### 3.4.4.1.3 ubcore_jetty + +```c +struct ubcore_jetty { + ubcore_device *ub_dev; + ubcore_ucontext *uctx; + ubcore_jetty_cfg jetty_cfg; + ubcore_jetty_id jetty_id; /* driver fill jetty_id-\>id */ + ubcore_tjetty *remote_jetty; // bind to remote jetty + ubcore_event_callback_t jfae_handler; + uint64_t urma_jetty; /* user space jetty pointer */ + struct hlist_node hnode; + atomic_t use_cnt; + struct kref ref_cnt; + struct completion comp; + ubcore_hash_table *tptable; /* Only for devices not natively supporting RM mode */ +}; +``` + +#### 3.4.4.2 ubcore_modify_jetty + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_modify_jetty([4.4.4.1.3](#34413-ubcore_jetty) [ubcore_jetty](#34413-ubcore_jetty) *jetty, [4.4.4.2.1](#34421-ubcore_jetty_attr) [ubcore_jetty_attr](#34421-ubcore_jetty_attr) *attr, [4.3.1.2](#3312-ubcore_udata) [ubcore_udata](#3312-ubcore_udata) *udata); + +3. 描述 + +修改jetty水线，只支持修改非共享JFR类型的jetty水线。当jetty中的WR（recv buffer）低于水线将触发JETTY_LIMIT异步事件，调用jetty的异步事件回调函数jfae_handler。应用获取到异步事件后，应该调用post_recv补充WR。应用设置的水线不能超过jetty的接收深度，0表示不希望触发水线机制。如果应用设定的水线大于当前jetty中存在的接收WR数，即刻就会收到低水线事件。 + +4. 参数 + +@param[in] [Required] jetty: specify jetty; + +@param[in] [Required] attr: attributes to be modified; + +@param[in] [Required] udata: 用户态驱动自定义数据。内核态应用直接调用该接口时，填NULL; + +5. 返回值 + +Return: 0 on success, other value on error + +##### 3.4.4.2.1 ubcore_jetty_attr + +```c +struct ubcore_jetty_attr { + uint32_t mask; /* mask value refer to enum ubcore_jetty_attr_mask */ + uint32_t rx_threshold; + ubcore_jetty_state state; +}; +``` + +##### 3.4.4.2.2 ubcore_jetty_attr_mask + +```c +enum ubcore_jetty_attr_mask { + UBCORE_JETTY_RX_THRESHOLD = 0x1, + UBCORE_JETTY_STATE = 0x1 << 1 +}; +``` + +#### 3.4.4.3 ubcore_query_jetty + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_query_jetty([4.4.4.1.3](#34413-ubcore_jetty) [ubcore_jetty](#34413-ubcore_jetty) *jetty, [4.4.4.1.1](#34411-ubcore_jetty_cfg) [ubcore_jetty_cfg](#34411-ubcore_jetty_cfg) *cfg, [4.4.4.2.1](#34421-ubcore_jetty_attr) [ubcore_jetty_attr](#34421-ubcore_jetty_attr) *attr); + +3. 描述 + +查询Jetty配置和属性，需要指定Jetty。 + +4. 参数 + +@param[in] [Required] jetty: specify jetty; + +@param[out] [Required] cfg: cfg to be query; + +@param[out] [Required] attr: attributes to be query; + +5. 返回值 + +Return: 0 on success, other value on error + +#### 3.4.4.4 ubcore_delete_jetty + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_delete_jetty([4.4.4.1.3](#34413-ubcore_jetty) [ubcore_jetty](#34413-ubcore_jetty) *jetty); + +3. 描述 + +销毁jetty。删除jetty之前，应用应该保证没有其他节点的应用还在使用这个jetty。 + +4. 参数 + +@param[in] [Required] jetty: the jetty created before; + +5. 返回值 + +Return: 0 on success, other value on error + +![](figures/urma_notice.png) + +由调用者保证参数jetty来自ubcore_create_jetty接口返回，参数内部指针等合法性已由ubcore_create_jetty接口保证，该接口不再重复进行校验；否则可能导致调用者进程异常退出。 + +#### 3.4.4.5 ubcore_delete_jetty_batch + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_delete_jetty_batch([4.4.4.1.3](#34413-ubcore_jetty) [ubcore_jetty](#34413-ubcore_jetty) **jetty_arr, int jetty_num, int *bad_jetty_index); + +3. 描述 + +批量销毁jetty。删除jetty之前，应用应该保证没有其他节点的应用还在使用这个jetty。 + +4. 参数 + +@param[in] jetty_arr: the jetty array created before; + +@param[in] jetty_num: jetty array length; + +@param[out] bad_jetty_index: when error, return error jetty index in the array; + +5. 返回值 + +0 on success, EINVAL on invalid parameter, other value on other batch delete errors. + +![](figures/urma_notice.png) + +1、由调用者保证参数jetty来自ubcore_create_jetty接口返回，参数内部指针等合法性已由ubcore_create_jetty接口保证，该接口不再重复进行校验；否则可能导致调用者进程异常退出。 + +2、如果发生删除失败的情况（包括非法的参数），接口会在第一个删除失败的jetty返回，在这个jetty之前的jetty都会被正常删除。 + +#### 3.4.4.6 ubcore_flush_jetty + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_flush_jetty([4.4.4.1.3](#34413-ubcore_jetty) [ubcore_jetty](#34413-ubcore_jetty) *jetty, int cr_cnt, [4.4.2.6.1](#34261-ubcore_cr) [ubcore_cr](#34261-ubcore_cr) *cr); + +3. 描述 + +udma驱动把jetty中未被硬件执行的wr，通过cr返回给应用。 + +4. 参数 + +@param[in] [Required] jetty: the jetty created before; + +@param[in] [Required] cr_cnt: the maximum number of CRs expected to be returned; + +@param[out] [Required] cr: the addr of returned CRs + +5. 返回值 + +Return: the number of completion record returned, 0 means no completion record returned, -1 on error + +#### 3.4.4.7 ubcore_import_jetty + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +[4.4.3.6.5](#34365-ubcore_tjetty) [ubcore_tjetty](#34365-ubcore_tjetty) *ubcore_import_jetty([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, [4.4.4.1.1](#34411-ubcore_jetty_cfg) [ubcore_jetty_cfg](#34411-ubcore_jetty_cfg) *cfg, [4.3.1.2](#3312-ubcore_udata) [ubcore_udata](#3312-ubcore_udata) *udata); + +3. 描述 + +用户输入远端jetty或者jetty group信息，包括jetty id（包含eid），token value，传输模式等导入jetty，接口返回target jetty指针。导入RM类型的jetty隐含与远端节点建链功能。导入UM类型的jetty隐含创建unreliable tp（其实是远端地址句柄）功能，记录在target jetty的tp中。 + +当导入jetty_group，只支持导入RM和UM类型的jetty group。如果导入RM模式的jetty，如果尚未与对端建链，隐含与对端建链，创建的tp指针保存在target jetty指针中。如果导入UM类的数据结构，将会创建目的地址句柄。支持多次导入相同配置的jetty group。应用需要保证jetty group配置真实有效，否则数据面无法将数据通过tjetty发送到对端。 + +4. 参数 + +@param[in] [Required] dev：the ubcore device handle; + +@param[in] [Required] cfg: remote jetty attributes and import configurations; + +@param[in] [Required] udata: 用户态驱动自定义数据。内核态应用直接调用该接口时，填NULL; + +5. 返回值 + +Return: the address of target jetty, not NULL on success, NULL on error + +#### 3.4.4.8 ubcore_import_jetty_ex + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +```c +struct ubcore_tjetty *ubcore_import_jetty_ex(ubcore_device *dev, +ubcore_tjetty_cfg *cfg, ubcore_active_tp_cfg *active_tp_cfg, +ubcore_udata *udata); +``` + +3. 描述 + +通过管控面，用户输入远端jetty或者jetty group信息，包括jetty id（包含eid），token value，传输模式等导入jetty，接口返回target jetty指针。导入RM类型的jetty隐含与远端节点建链功能。导入UM类型的jetty隐含创建unreliable tp（其实是远端地址句柄）功能，记录在target jetty的tp中。 + +当导入jetty_group，只支持导入RM和UM类型的jetty group。如果导入RM模式的jetty，如果尚未与对端建链，隐含与对端建链，创建的tp指针保存在target jetty指针中。如果导入UM类的数据结构，将会创建目的地址句柄。支持多次导入相同配置的jetty group。应用需要保证jetty group配置真实有效，否则数据面无法将数据通过tjetty发送到对端。 + +4. 参数 + +@param[in] dev: the ubcore device handle; + +@param[in] cfg: remote jetty attributes and import configurations + +@param[in] active_tp_cfg: tp configuration to active + +@param[in] udata (optional): ucontext and user space driver data + +5. 返回值 + +target jetty pointer on success, NULL on error + +#### 3.4.4.9 ubcore_unimport_jetty + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_unimport_jetty([4.4.3.6.5](#34365-ubcore_tjetty) [ubcore_tjetty](#34365-ubcore_tjetty) *tjetty); + +3. 描述 + +反导入远端jetty或者jetty group。unimport Jetty隐含：减少tjetty保存的tp的引用计数，减到零时触发拆链流程。 + +反导入target jetty将会释放tjetty结构体，应用需要保证已经使用target jetty发起的请求都已经poll JFC得到完成记录（包括错误）。 + +4. 参数 + +@param[in] [Required] tjetty: the target jetty to unimport; + +5. 返回值 + +Return: 0 on success, other value on error + +#### 3.4.4.10 ubcore_bind_jetty + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_bind_jetty(struct [4.4.4.1.3](#34413-ubcore_jetty) [ubcore_jetty](#34413-ubcore_jetty) *jetty, [4.4.3.6.5](#34365-ubcore_tjetty) [ubcore_tjetty](#34365-ubcore_tjetty) *tjetty, [4.3.1.2](#3312-ubcore_udata) [ubcore_udata](#3312-ubcore_udata) *udata); + +3. 描述 + +bind接口针对RC类型的jetty，将本端RC类型jetty指针与已经导入的RC类型target jetty指针，进行一对一绑定。 + +bind可以一方单独完成，不依赖对端已经导入jetty，也不要求对端同时调用bind jetty接口。bind功能还隐含创建RC类型TP，保存在tjetty中。主动调用bind完成者才具有发送和接收功能。被动响应bind请求的一方，底层会创建RC类型的TP与jetty关联，能接收消息、但不能发送消息。 + +4. 参数 + +@param[in] [Required] jetty: local jetty to construct the transport channel; + +@param[in] [Required] tjetty: target jetty imported before; + +@param[in] [Required] udata: 用户态驱动自定义数据。内核态应用直接调用该接口时，填NULL; + +5. 返回值 + +Return: 0 on success, other value on error + +6. 备注 + +Note: A local jetty can be binded with only one remote jetty. Only supported by jetty under URMA_TM_RC. + +#### 3.4.4.11 ubcore_bind_jetty_ex + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +```c +int ubcore_bind_jetty_ex(ubcore_jetty *jetty, ubcore_tjetty *tjetty, +ubcore_active_tp_cfg *active_tp_cfg, ubcore_udata *udata); +``` + +3. 描述 + +通过管控面，bind接口针对RC类型的jetty，将本端RC类型jetty指针与已经导入的RC类型target jetty指针，进行一对一绑定。 + +bind可以一方单独完成，不依赖对端已经导入jetty，也不要求对端同时调用bind jetty接口。bind功能还隐含创建RC类型TP，保存在tjetty中。主动调用bind完成者才具有发送和接收功能。被动响应bind请求的一方，底层会创建RC类型的TP与jetty关联，能接收消息、但不能发送消息。 + +4. 参数 + +@param[in] jetty: local jetty to bind; + +@param[in] tjetty: target jetty imported before; + +@param[in] active_tp_cfg: tp configuration to active; + +@param[in] udata (optional): ucontext and user space driver data + +5. 返回值 + +0 on success, other value on error + +6. 备注 + +Note: A local jetty can be binded with only one remote jetty. Only supported by jetty under URMA_TM_RC. + +#### 3.4.4.12 ubcore_unbind_jetty + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_unbind_jetty([4.4.4.1.3](#34413-ubcore_jetty) [ubcore_jetty](#34413-ubcore_jetty) *jetty); + +3. 描述 + +解除本地jetty与远端jetty的绑定关系。通信的双方必须各自调用该接口，解除已经建立的绑定关系。隐含销毁jetty关联的RC TP（保存在tjetty中），但不销毁对端的RC TP。 + +4. 参数 + +@param[in] [Required] jetty: local jetty to unbind; + +5. 返回值 + +Return: 0 on success, other value on error + +#### 3.4.4.13 ubcore_import_jetty_async + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +[4.4.4.1.3](#34413-ubcore_jetty) [ubcore_jetty](#34413-ubcore_jetty) *ubcore_import_jetty_async([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, [4.4.3.6.1](#34361-ubcore_tjetty_cfg) [ubcore_tjetty_cfg](#34361-ubcore_tjetty_cfg) *cfg, int timeout, [4.4.4.13.1](#344131-ubcore_import_cb) [ubcore_import_cb](#344131-ubcore_import_cb) *cb,[4.3.1.2](#3312-ubcore_udata) [ubcore_udata](#3312-ubcore_udata) *udata); + +3. 描述 + +支持异步import。 + +用户输入远端jetty或者jetty group信息，包括jetty id（包含eid），token value，传输模式等导入jetty，接口返回target jetty指针。导入RM类型的jetty隐含与远端节点建链功能。导入UM类型的jetty隐含创建unreliable tp（其实是远端地址句柄）功能，记录在target jetty的tp中。 + +当导入jetty_group，只支持导入RM和UM类型的jetty group。如果导入RM模式的jetty，如果尚未与对端建链，隐含与对端建链，创建的tp指针保存在target jetty指针中。如果导入UM类的数据结构，将会创建目的地址句柄。支持多次导入相同配置的jetty group。应用需要保证jetty group配置真实有效，否则数据面无法将数据通过tjetty发送到对端。 + +4. 参数 + +@param[in] [Required] dev：the ubcore device handle; + +@param[in] [Required] cfg: remote jetty attributes and import configurations; + +@param[in] [Required] timeout: max time to wait (milliseconds) + +@param[in] [Required] cb: callback function pointer with custom user argument + +@param[in] [Required] udata: 用户态驱动自定义数据。内核态应用直接调用该接口时，填NULL; + +5. 返回值 + +@return: target jetty pointer on success, NULL on error + +##### 3.4.4.13.1 ubcore_import_cb + +```c +struct ubcore_import_cb { + void *user_arg; /* uburma_tjetty */ + void (*callback)(ubcore_tjetty *tjetty, int status, + void *user_arg); +}; +``` + +#### 3.4.4.14 ubcore_unimport_jetty_async + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_unimport_jetty_async([4.4.3.6.5](#34365-ubcore_tjetty) [ubcore_tjetty](#34365-ubcore_tjetty) *tjetty, int timeout, [4.4.4.14.1](#344141-ubcore_unimport_cb) [ubcore_unimport_cb](#344141-ubcore_unimport_cb) *cb); + +3. 描述 + +反导入远端jetty或者jetty group。unimport Jetty隐含：减少tjetty保存的tp的引用计数，减到零时触发拆链流程。 + +反导入target jetty将会释放tjetty结构体，应用需要保证已经使用target jetty发起的请求都已经poll JFC得到完成记录（包括错误）。 + +4. 参数 + +@param[in] [Required] tjetty: the target jetty to unimport; + +@param[in] [Required] timeout: max time to wait (milliseconds); + +@param[in] [Required] cb: callback function pointer with custom user argument; + +5. 返回值 + +@return: 0 on success, other value on error + +##### 3.4.4.14.1 ubcore_unimport_cb + +```c +struct ubcore_unimport_cb { + void *user_arg; + void (*callback)(int status, void *user_arg); +}; +``` + +#### 3.4.4.15 ubcore_bind_jetty_async + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_bind_jetty_async([4.4.4.1.3](#34413-ubcore_jetty) [ubcore_jetty](#34413-ubcore_jetty) *jetty, [4.4.3.6.5](#34365-ubcore_tjetty) [ubcore_tjetty](#34365-ubcore_tjetty) *tjetty, int timeout, [4.4.4.15.1](#344151-ubcore_bind_cb) [ubcore_bind_cb](#344151-ubcore_bind_cb) *cb, [4.3.1.2](#3312-ubcore_udata) [ubcore_udata](#3312-ubcore_udata) *udata); + +3. 描述 + +bind接口针对RC类型的jetty，将本端RC类型jetty指针与已经导入的RC类型target jetty指针，进行一对一绑定。 + +bind可以一方单独完成，不依赖对端已经导入jetty，也不要求对端同时调用bind jetty接口。bind功能还隐含创建RC类型TP，保存在tjetty中。主动调用bind完成者才具有发送和接收功能。被动响应bind请求的一方，底层会创建RC类型的TP与jetty关联，能接收消息、但不能发送消息。 + +4. 参数 + +@param[in] [Required] jetty: local jetty to construct the transport channel; + +@param[in] [Required] tjetty: target jetty imported before; + +@param[in] [Required] timeout: max time to wait (milliseconds); + +@param[in] [Required] cb: callback function pointer with custom user argument; + +@param[in] [Required] udata: 用户态驱动自定义数据。内核态应用直接调用该接口时，填NULL; + +5. 返回值 + +Return: 0 on success, other value on error + +6. 备注 + +Note: A local jetty can be binded with only one remote jetty. Only supported by jetty under URMA_TM_RC. + +##### 3.4.4.15.1 ubcore_bind_cb + +```c +struct ubcore_bind_cb { + void *user_arg; /* uburma_tjetty */ + void (*callback)(ubcore_jetty *jetty, + ubcore_tjetty *tjetty, int status, + void *user_arg); +}; +``` + +#### 3.4.4.16 ubcore_unbind_jetty_async + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_unbind_jetty_async([4.4.4.1.3](#34413-ubcore_jetty) [ubcore_jetty](#34413-ubcore_jetty) *jetty, int timeout, [4.4.4.16.1](#344161-ubcore_unbind_cb) [ubcore_unbind_cb](#344161-ubcore_unbind_cb) *cb); + +3. 描述 + +解除本地jetty与远端jetty的绑定关系。通信的双方必须各自调用该接口，解除已经建立的绑定关系。隐含销毁jetty关联的RC TP（保存在tjetty中），但不销毁对端的RC TP。 + +4. 参数 + +@param[in] [Required] jetty: local jetty to unbind; + +@param[in] [Required] timeout: max time to wait (milliseconds); + +@param[in] [Required] cb: callback function pointer with custom user argument; + +5. 返回值 + +Return: 0 on success, other value on error + +##### 3.4.4.16.1 ubcore_unbind_cb + +```c +struct ubcore_unbind_cb { + void *user_arg; + void (*callback)(int status, void *user_arg); +}; +``` + +### 3.4.5 Jetty Group管理 + +#### 3.4.5.1 ubcore_create_jetty_grp + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +[4.4.5.1.3](#34513-ubcore_jetty_group) [ubcore_jetty_group](#34513-ubcore_jetty_group) *ubcore_create_jetty_grp([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, [4.4.5.1.2](#34512-ubcore_jetty_grp_cfg) [ubcore_jetty_grp_cfg](#34512-ubcore_jetty_grp_cfg) *cfg, + +[4.4.1.1.4](#34114-ubcore_event_callback_t) [ubcore_event_callback_t](#34114-ubcore_event_callback_t) jfae_handler, [4.3.1.2](#3312-ubcore_udata) [ubcore_udata](#3312-ubcore_udata) *udata); + +3. 描述 + +用户指定token、最大jetty数、传输模式等参数创建jetty group，返回jetty group数据结构中包含分配的jetty group id，后续应用可以传入返回的jetty group指针创建jetty。应用可以通过查询device capability得到单个jetty group支持的最大jetty数。 + +目前只支持RM类型的jetty group。Jetty group中创建的jetty必须与group的传输模式相符。服务器端应用可以将jetty group的id，token和传输模式传输到客户端应用，供后者导入jetty group。 + +Jetty group收到消息，但jetty group没有有效的jetty时，将调用jfae_handler回调函数上报异步事件。 + +4. 参数 + +@param[in] [Required] dev：ubcore_device指针; + +@param[in] [Required] cfg: jetty group配置信息; + +@param[in] [Required] jfae_handler: 异步事件回调函数; + +@param[in] [Required] udata: 用户态驱动自定义数据。内核态应用直接调用该接口时，填NULL; + +5. 返回值 + +Return: etty group pointer on success, NULL on error + +##### 3.4.5.1.1 ubcore_jetty_grp_flag + +```c +union ubcore_jetty_grp_flag { + struct { + uint32_t token_policy : 3; + uint32_t reserved : 29; + } bs; + uint32_t value; +}; +``` + +##### 3.4.5.1.2 ubcore_jetty_grp_cfg + +```c +struct ubcore_jetty_grp_cfg { + char name[UBCORE_JETTY_GRP_MAX_NAME]; + uint32_t eid_index; + ubcore_jetty_grp_flag flag; + ubcore_token token_value; + uint32_t id; + ubcore_jetty_grp_policy policy; + uint64_t user_ctx; +}; +``` + +##### 3.4.5.1.3 ubcore_jetty_group + +```c +struct ubcore_jetty_group { + ubcore_device *ub_dev; + ubcore_ucontext *uctx; + ubcore_jetty_grp_cfg jetty_grp_cfg; + ubcore_jetty_id jetty_grp_id; /* driver fill jetty_grp_id-\>id */ + uint32_t jetty_cnt; /* current jetty cnt in the jetty group */ + ubcore_jetty **jetty; + ubcore_event_callback_t jfae_handler; + uint64_t urma_jetty_grp; /* user space jetty_grp pointer */ + struct mutex lock; /* Protect jetty array */ +}; +``` + +#### 3.4.5.2 ubcore_delete_jetty_grp + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_delete_jetty_grp([4.4.5.1.3](#34513-ubcore_jetty_group) [ubcore_jetty_group](#34513-ubcore_jetty_group) *jetty_grp); + +3. 描述 + +删除jetty group。应用必须保证删除jetty group之前，已经删除其中的所有jetty。 + +4. 参数 + +@param[in] [Required] jetty_grp: 待删除的jetty group指针; + +5. 返回值 + +Return: 0 on success, other value on error + +## 3.5 异步事件 + +### 3.5.1 ubcore_register_event_handler + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +void ubcore_register_event_handler([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, [4.5.1.1](#3511-ubcore_event_handler) [ubcore_event_handler](#3511-ubcore_event_handler) *handler); + +3. 描述 + +注册异常事件处理句柄。ubcore_register_event_handler由内核应用或者ubcore的client端，例如uburma模块调用，向ubcore注册，UB Core将handler添加到设备的handler链表中。只有端口、TP、ID_change和设备类型的异步事件，才会调用处理句柄中的event_callback函数。 + +4. 参数 + +@param[in] [Required] dev：包含厂商自定义的设备私有数据; + +@param[out] [Required] event: async_event handler to be unregistered + +5. 返回值 + +Return: void + +#### 3.5.1.1 ubcore_event_handler + +```c +struct ubcore_event_handler { + void (*event_callback)(ubcore_event *event, + ubcore_event_handler *handler); + struct list_head node; +}; +``` + +#### 3.5.1.2 ubcore_event + +```c +struct ubcore_event { + struct ubcore_device *ub_dev; + union { + ubcore_jfc *jfc; + ubcore_jfs *jfs; + ubcore_jfr *jfr; + ubcore_jetty *jetty; + ubcore_jetty_group *jetty_grp; + ubcore_tp *tp; + ubcore_vtp *vtp; + uint32_t port_id; + uint32_t eid_idx; + } element; + ubcore_event_type event_type; +}; +``` + +#### 3.5.1.3 ubcore_event_type + +```c +enum ubcore_event_type { + UBCORE_EVENT_JFC_ERR = 0, + UBCORE_EVENT_JFS_ERR, + UBCORE_EVENT_JFR_ERR, + UBCORE_EVENT_JFR_LIMIT_REACHED, + UBCORE_EVENT_JETTY_ERR, + UBCORE_EVENT_JETTY_LIMIT_REACHED, + UBCORE_EVENT_JETTY_GRP_ERR, + UBCORE_EVENT_PORT_ACTIVE, + UBCORE_EVENT_PORT_DOWN, + UBCORE_EVENT_DEV_FATAL, + UBCORE_EVENT_EID_CHANGE, + UBCORE_EVENT_TP_ERR, + UBCORE_EVENT_TP_SUSPEND, + UBCORE_EVENT_TP_FLUSH_DONE, + UBCORE_EVENT_ELR_ERR, + UBCORE_EVENT_ELR_DONE, + UBCORE_EVENT_MIGRATE_VTP_SWITCH, + UBCORE_EVENT_MIGRATE_VTP_ROLLBACK +}; +``` + +### 3.5.2 ubcore_unregister_event_handler + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +void ubcore_unregister_event_handler([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, [4.5.1.1](#3511-ubcore_event_handler) [ubcore_event_handler](#3511-ubcore_event_handler) *handler); + +3. 描述 + +注销异步事件事件处理接口。ubcore_unregister_event_handler由内核应用或者ubcore的client端调用，向UBN协议栈反注册，UBcore将handler从设备的handler链表中移除。 + +4. 参数 + +@param[in] [Required] dev：包含厂商自定义的设备私有数据; + +@param[in] [Required] handler: async_event handler to be unregistered + +5. 返回值 + +Return: void + +## 3.6 Post WR操作 + +### 3.6.1 ubcore_post_jfs_wr + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_post_jfs_wr([4.4.2.1.4](#34214-ubcore_jfs) [ubcore_jfs](#34214-ubcore_jfs) *jfs, [4.6.1.1](#3611-ubcore_jfs_wr) [ubcore_jfs_wr](#3611-ubcore_jfs_wr) *wr, [4.6.1.1](#3611-ubcore_jfs_wr) [ubcore_jfs_wr](#3611-ubcore_jfs_wr) **bad_wr); + +3. 描述 + +向JFS发起单边、双边或者原子操作等请求。支持使用next指针批量下发请求。应用也可以通过该接口发起invalid segment请求。 + +应用可poll JFS关联的JFC获得完成记录（包括执行错误）。如果创建JFC时指定了jfce_handler，操作完成或错误时，jfce_handler被调用通知到应用。获得完成记录前，应用不能unimport请求中包含的target jetty、target segment，不能使用WR涉及到的segment内存区间。如果应用指定inline发送，inline长度不能超过JFS的max_inline_data，则接口返回即可修改WR涉及到的内存。 + +应用保证on-flight（积发起未获取到完成记录）的请求不超过JFS的深度。如果多个JFS关联到同一个JFC，应用还应该保证JFC不会发生溢出。 + +JFS只能向相同传输（RM或UM）类型的远端JFR（已经导入为target jetty）发送请求。 + +4. 参数 + +@param[in] jfs: the jfs created before, which is used to put command; + +@param[in] wr: the posting request all information, including src addr, dst addr, len, jfc, flag, ordering etc; + +@param[out] bad_wr: the first of failure request. + +5. 返回值 + +Return: 0 on success, other value on error + +#### 3.6.1.1 ubcore_jfs_wr + +```c +struct ubcore_jfs_wr { + ubcore_opcode opcode; + ubcore_jfs_wr_flag flag; + uint64_t user_ctx; + ubcore_tjetty *tjetty; + union { + ubcore_rw_wr rw; + ubcore_send_wr send; + ubcore_cas_wr cas; + ubcore_faa_wr faa; + }; + struct ubcore_jfs_wr *next; +}; +``` + +#### 3.6.1.2 ubcore_opcode + +```c +enum ubcore_opcode { + UBCORE_OPC_WRITE = 0x00, + UBCORE_OPC_WRITE_IMM = 0x01, + UBCORE_OPC_WRITE_NOTIFY = 0x02, + UBCORE_OPC_READ = 0x10, + UBCORE_OPC_CAS = 0x20, + UBCORE_OPC_SWAP = 0x21, + UBCORE_OPC_FADD = 0x22, + UBCORE_OPC_FSUB = 0x23, + UBCORE_OPC_FAND = 0x24, + UBCORE_OPC_FOR = 0x25, + UBCORE_OPC_FXOR = 0x26, + UBCORE_OPC_SEND = 0x40, // remote JFR/jetty ID + UBCORE_OPC_SEND_IMM = 0x41, // remote JFR/jetty ID + UBCORE_OPC_SEND_INVALIDATE = 0x42, // remote JFR/jetty ID and seg token id + UBCORE_OPC_NOP = 0x51, + UBCORE_OPC_WRITE_ATOMIC = 0x60, // Non-standard definition of OPCODE + UBCORE_OPC_LAST +}; +``` + +#### 3.6.1.3 ubcore_jfs_wr_flag + +```c +union ubcore_jfs_wr_flag { + struct { + /* 0: There is no order with other WR. + * 1: relax order. + * 2: strong order. + * 3: reserve. + */ + uint32_t place_order : 2; + /* 0: There is no completion order with other WR + * 1: Completion order with previous WR. + */ + uint32_t comp_order : 1; + /* 0: There is no fence. + * 1: Fence with previous read and atomic WR + */ + uint32_t fence : 1; + /* 0: not solicited. + * 1: solicited. It will trigger an event + * on remote side + */ + uint32_t solicited_enable : 1; + /* 0: Do not notify local process + * after the task is complete. + * 1: Notify local process + * after the task is completed. + */ + uint32_t complete_enable : 1; + /* 0: No inline. + * 1: Inline data. + */ + uint32_t inline_flag : 1; + uint32_t reserved : 25; + } bs; + uint32_t value; +}; +``` + +![](figures/urma_info.png) + +- solicited_enable：表示希望对端接收到该请求时，产生完成记录的同时也产生完成事件。 + +- complete_enable：本端产生完成记录 + +- inline_flag：inline发送，对应的内存免注册，可以不填本地target segmet；接口返回后，应用即可以访问相应的内存 + +- fence：等待前面的read和atomic完成后，才发送当前WR + +- comp_order：如果WR指定了comp_order，代表接收侧r-cqe上报是保序的，发送的报文会带comp_order标志； + +- place_order: 事务序，控制是否按序执行请求，strong表示等待前面的标注relax和strong的WR完成后，才能执行改WR；relax请求之间没有序的保证；no order没有任何执行序的限制 + +![](figures/urma_caution.png) + +- 如果jetty/jfs是UM模式，那么WR不支持fence, place_order和comp_order + +- 如果jetty/jfs是完成乱序模式，那么WR不支持fence, place_order和comp_order + +- 当Jetty/JFS是RM或RC模式，并且配置成完成保序模式时，如果WR指定了comp_order，代表接收侧cqe上报是保序的，发送的报文会带comp_order标志。 + +#### 3.6.1.4 ubcore_rw_wr + +```c +struct ubcore_rw_wr { + ubcore_sg src; + ubcore_sg dst; + uint8_t target_hint; /* hint of jetty in a target jetty group */ + uint64_t notify_data; /* notify data or immeditate data in host byte order */ +}; +``` + +![](figures/urma_info.png) + +对于write和write_imm操作，src表示本地内存信息，dst[0]表示远端目的内存信息，dst.num_sge = 1，只能写入到连续远端地址； + +对于read操作，src表示远端源内存信息，src.num_sge = 1, 只能从连续远端地址读数据， dst表示本地目的内存信息； + +对于write notify操作，dst.num_sge = 2, notify data表示待写入到对端的notify数据，dst[1]表示notify地址和target segen； dst[0]含义与write一致，表示src sg待写入的远端内存信息。支持dst[0]和dst[1]源于不同的target segment。 + +#### 3.6.1.5 ubcore_sg + +```c +struct ubcore_sg { + ubcore_sge *sge; + uint32_t num_sge; +}; +``` + +#### 3.6.1.6 ubcore_sge + +```c +struct ubcore_sge { + uint64_t addr; + uint32_t len; + ubcore_target_seg *tseg; +}; +``` + +#### 3.6.1.7 ubcore_send_wr + +```c +struct ubcore_send_wr { + ubcore_sg src; + uint8_t target_hint; /* hint of jetty in a target jetty group */ + uint64_t imm_data; /* immeditate data in host byte order */ + ubcore_target_seg *tseg; /* Used only when send with invalidate */ +}; +``` + +#### 3.6.1.8 ubcore_cas_wr + +```c +struct ubcore_cas_wr { + ubcore_sge *dst; /* len is the data length of CAS operation */ + ubcore_sge *src; /* Local address for destination original value written back */ + union { + uint64_t cmp_data; /* When the len <= 8B, it indicates the CMP value. */ + uint64_t cmp_addr; /* When the len \> 8B, it indicates the data address. */ + }; + union { + /* If destination value is the same as cmp_data, + * destination value will be change to swap_data. + */ + uint64_t swap_data; + uint64_t swap_addr; + }; +}; +``` + +#### 3.6.1.9 ubcore_faa_wr + +```c +struct ubcore_faa_wr { + ubcore_sge *dst; /* len in the sge is the length of faa at remote side */ + ubcore_sge *src; /* Local address for destination original value written back */ + union { + uint64_t operand; /* Addend */ + uint64_t operand_addr; + }; +}; +``` + +### 3.6.2 ubcore_post_jfr_wr + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_post_jfr_wr([4.4.3.1.3](#34313-ubcore_jfr) [ubcore_jfr](#34313-ubcore_jfr) *jfr, [4.6.2.1](#3621-ubcore_jfr_wr) [ubcore_jfr_wr](#3621-ubcore_jfr_wr) *wr, [4.6.2.1](#3621-ubcore_jfr_wr) [ubcore_jfr_wr](#3621-ubcore_jfr_wr) **bad_wr); + +3. 描述 + +向JFR发起接收操作填recv buffer的请求。应用可poll JFC获得完成消息（包括错误），也可以通过JFC的jfce_handler获得完成事件。支持以next方式批量下发接收请求。未获得完成记录之前应用不能访问已经提交的recv buffer。每收到一个对端发来的Send，Send with invalidate, Send/Write immediate，Write notify都会消耗一个WR内的recv buffer，应用应该保证recv buffer长度不小于接收到的消息长度。应用应该保证JFR不会溢出，需要及时补充recv buffer来保证收到消息时，JFR中有可用的recv buffer。 + +4. 参数 + +@param[in] jfr: the jfr created before, which is used to put command; + +@param[in] wr: the posting request all information, including sge, flag; + +@param[out] bad_wr: the first of failure request. + +5. 返回值 + +成功返回零，失败返回非零 + +#### 3.6.2.1 ubcore_jfr_wr + +```c +struct ubcore_jfr_wr { + ubcore_sg src; + uint64_t user_ctx; + struct ubcore_jfr_wr *next; +}; +``` + +### 3.6.3 ubcore_post_jetty_send_wr + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_post_jetty_send_wr([4.4.4.1.3](#34413-ubcore_jetty) [ubcore_jetty](#34413-ubcore_jetty) *jetty, [4.6.1.1](#3611-ubcore_jfs_wr) [ubcore_jfs_wr](#3611-ubcore_jfs_wr) *wr, [4.6.1.1](#3611-ubcore_jfs_wr) [ubcore_jfs_wr](#3611-ubcore_jfs_wr) **bad_wr); + +3. 描述 + +向jetty发起单边、双边或者原子操作等请求，功能与post_jfs_wr类似。Jetty只能向相同传输（RM、RC或UM）类型的远端jetty发送请求。 + +4. 参数 + +@param[in] jetty: the jetty created before, which is used to put command; + +@param[in] wr: the posting request all information, including src addr, dst addr, len, jfc, flag, ordering etc; + +@param[out] bad_wr: the first of failure request. + +5. 返回值 + +Return: 0 on success, other value on error + +### 3.6.4 ubcore_post_jetty_recv_wr + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_post_jetty_recv_wr([4.4.4.1.3](#34413-ubcore_jetty) [ubcore_jetty](#34413-ubcore_jetty) *jetty, [4.6.2.1](#3621-ubcore_jfr_wr) [ubcore_jfr_wr](#3621-ubcore_jfr_wr) *wr, [4.6.2.1](#3621-ubcore_jfr_wr) [ubcore_jfr_wr](#3621-ubcore_jfr_wr) **bad_wr); + +3. 描述 + +向jetty发起接收操作填recv buffer的请求。功能与post_jfr_recv_wr类似。由于Jetty为共享JFR，此接口实际上上Jetty关联的共享JFR提交接收请求。 + +4. 参数 + +@param[in] jetty: the jetty created before, which is used to put command; + +@param[in] wr: the posting request all information, including sge, flag; + +@param[out] bad_wr: the first of failure request. + +5. 返回值 + +Return: 0 on success, other value on error + +## 3.7 完成记录 + +### 3.7.1 ubcore_poll_jfc + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_poll_jfc([4.4.1.1.5](#34115-ubcore_jfc) [ubcore_jfc](#34115-ubcore_jfc) *jfc, int cr_cnt, [4.4.2.6.1](#34261-ubcore_cr) [ubcore_cr](#34261-ubcore_cr) *cr); + +3. 描述 + +应用从JFC中读取完成记录（CR）。应用提交请求后，应该及时从关联的JFC获取完成记录，避免CR溢出。 + +4. 参数 + +@param[in] jfc: jetty completion queue to poll; + +@param[in] cr_cnt: the maximum number of CRs expected to be polled; + +@param[out] cr: the completion record array to fill at least cr_cnt completion records. + +5. 返回值 + +Return: the number of completion record returned, 0 means no completion record returned, -1 on error + +6. 备注 + +Note that: at most 16 completion records can be polled for RDMA device + +### 3.7.2 ubcore_rearm_jfc + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_rearm_jfc([4.4.1.1.5](#34115-ubcore_jfc) [ubcore_jfc](#34115-ubcore_jfc) *jfc, bool solicited_only); + +3. 描述 + +使能JFC的事件通知机制，当JFC新产生一个完成记录时，完成事件聚合条件满足时，JFC对应的CEQ中也会放置一个新的完成事件，后续驱动处理中断时，将调用JFC的jfce_handler通知新的完成事件。调用该接口时JFC中已经存在的完成记录，是否产生完成事件由驱动自定义实现，不保证产生完成事件。一次调用只能产生一个事件通知，得到事件通知后，应用需要调用该函数重新设置通知机制。 + +solicited_only置位后，接收到"发送端设置了solicited的WR，包括Send，Send/Write immediate，Send with invalidate"并产生完成记录时，才在接收端上报事件。Rearm对失败的发送或接收WR有效，不受solicited_only约束。 + +4. 参数 + +@param[in] jfc: jetty completion queue to arm to interrupt mode; + +@param[in] solicited_only: indicate it will trigger event only for packets with solicited flag. + +5. 返回值 + +Return: 0 on success, other value on error + +## 3.8 ubcore面向UVS接口 + +### 3.8.1 ubcore_set_port_netdev + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +int ubcore_set_port_netdev([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, struct net_device *ndev, unsigned int port_id); + +3. 描述 + +Invoke ndev bind port_id, called only by driver + +4. 参数 + +@param[in] dev: the ubcore device; + +@param[in] ndev: The netdev corresponding to the initial port; + +@param[in] port_id: The physical port_id is the same as the port_id presented in the sysfs file, and port_id is configured in TP during link establishment. + +5. 返回值 + +Return: 0 on success, other value on error + +### 3.8.2 ubcore_unset_port_netdev + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +int ubcore_unset_port_netdev([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, struct net_device *ndev, unsigned int port_id); + +3. 描述 + +Invoke ndev unbind port_id, called only by driver + +4. 参数 + +@param[in] dev: the ubcore device; + +@param[in] ndev: The netdev corresponding to the initial port; + +@param[in] port_id: The physical port_id is the same as the port_id presented in the sysfs file, and port_id is configured in TP during link establishment. + +5. 返回值 + +Return: 0 on success, other value on error + +### 3.8.3 ubcore_put_port_netdev + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +void ubcore_put_port_netdev([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev); + +3. 描述 + +Invoke ndev unbind port_id, called only by driver + +4. 参数 + +@param[in] dev：ubcore_device指针. + +5. 返回值 + +Return: void + +### 3.8.4 ubcore_add_ueid + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_add_ueid([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, uint16_t ue_idx, [4.8.4.1](#3841-ubcore_ueid_cfg) [ubcore_ueid_cfg](#3841-ubcore_ueid_cfg) *cfg); + +3. 描述 + +由uvs调用，给设备新增EID；虚拟化场景下，可以指定新增EID所属UPI + +4. 参数 + +@param[in] dev：ubcore_device指针; + +@param[in] ue_idx：ue_eid; + +@param[in] cfg：待添加的UEID信息，包含EID、UPI值和EID index. + +5. 返回值 + +Return: the index of eid/upi, less than 0 indicating error + +#### 3.8.4.1 ubcore_ueid_cfg + +```c +struct ubcore_ueid_cfg { + ubcore_eid eid; + uint32_t upi; + uint32_t eid_index; + guid_t guid; +}; +``` + +### 3.8.5 ubcore_delete_ueid + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_delete_ueid([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, uint16_t ue_idx, [4.8.4.1](#3841-ubcore_ueid_cfg) [ubcore_ueid_cfg](#3841-ubcore_ueid_cfg) *cfg); + +3. 描述 + +由uvs调用，从设备删除一个EID. + +4. 参数 + +@param[in] dev：ubcore_device指针; + +@param[in] ue_idx：ue_idx; + +@param[in] cfg：待删除的UEID信息，包含EID、UPI值和EID index. + +5. 返回值 + +Return: 0 on success, other value on error + +### 3.8.6 ubcore_config_device + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_config_device([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, [4.2.1.3](#3213-ubcore_device_cfg) [ubcore_device_cfg](#3213-ubcore_device_cfg) *cfg); + +3. 描述 + +uvs配置function属性 + +4. 参数 + +@param[in] dev：ubcore_device指针; + +@param[in] cfg：设备配置信息. + +5. 返回值 + +Return: 0 on success, other value on error + +### 3.8.7 ubcore_add_sip + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +int ubcore_add_sip([4.8.7.1](#3871-ubcore_sip_info) [ubcore_sip_info](#3871-ubcore_sip_info) *sip, uint32_t *sip_idx); + +3. 描述 + +云管理系统通过Uvs调用ubcore的接口设置建链使用的设备名和sip信息。 + +4. 参数 + +@param[in] sip：Specify the sip information used to establish the link, including device name, Specify the sip information used to establish the link, including device name; + +@param[in] sip_idx: sip_idx. + +5. 返回值 + +Return: 0 on success, other value on error + +#### 3.8.7.1 ubcore_sip_info + +```c +struct ubcore_sip_info { + char dev_name[UBCORE_MAX_DEV_NAME]; + ubcore_net_addr addr; + uint32_t prefix_len; + uint8_t port_cnt; + uint8_t port_id[UBCORE_MAX_PORT_CNT]; + uint32_t mtu; + char netdev_name[UBCORE_MAX_DEV_NAME]; /* for change mtu */ + bool is_active; +}; +``` + +#### 3.8.7.2 ubcore_net_addr + +```c +struct ubcore_net_addr { + ubcore_net_addr_type type; + ubcore_net_addr_union net_addr; + uint64_t vlan; /* available for UBOE */ + uint8_t mac[UBCORE_MAC_BYTES]; /* available for UBOE */ + uint32_t prefix_len; +}; +``` + +#### 3.8.7.3 ubcore_net_addr_type + +```c +enum ubcore_net_addr_type { + UBCORE_NET_ADDR_TYPE_IPV4 = 0, + UBCORE_NET_ADDR_TYPE_IPV6 +}; +``` + +#### 3.8.7.4 ubcore_net_addr_union + +```c +union ubcore_net_addr_union { + uint8_t raw[UBCORE_NET_ADDR_BYTES]; + struct { + uint64_t reserved1; + uint32_t reserved2; + uint32_t addr; + } in4; + struct { + uint64_t subnet_prefix; + uint64_t interface_id; + } in6; +}; +``` + +### 3.8.8 ubcore_delete_sip + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +int ubcore_delete_sip([4.8.7.1](#3871-ubcore_sip_info) [ubcore_sip_info](#3871-ubcore_sip_info) *sip); + +3. 描述 + +云管理系统通过Uvs调用ubcore的接口删除sip信息。 + +4. 参数 + +@param[in] sip：Specify the sip information used to establish the link, including device name, sip, mac, vlan, physical port list. + +5. 返回值 + +Return: 0 on success, other value on error + +## 3.9 DFX接口 + +本节只定义DFX函数，DFX接口中的数据结构定义见《UMDK URMA驱动编程接口说明书》 + +### 3.9.1 ubcore_query_stats + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_query_stats([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, [4.9.1.1](#3911-ubcore_stats_key) [ubcore_stats_key](#3911-ubcore_stats_key) *key, [4.9.1.2](#3912-ubcore_stats_val) [ubcore_stats_val](#3912-ubcore_stats_val) *val); + +3. 描述 + +查询设备的统计信息 + +4. 参数 + +@param[in] [Required] dev：ubcore_device指针; + +@param[in] [Required] key：指定待查询信息的类型和key值; + +@param[in/out] [Required] val：返回查询结果的存放地址和实际数据的长度. + +5. 返回值 + +0表示查询成功，非0表示查询失败。Val中指定的buffer足够时，查询操作成功，返回实际填写的数据长度；Buffer不足时，返回失败，驱动填期望的buffer长度。 + +#### 3.9.1.1 ubcore_stats_key + +```c +struct ubcore_stats_key { + uint8_t type; /* stats type, refer to enum ubcore_stats_key_type */ + uint32_t key; /* key can be tpn/tpgn/jetty_id/token_id/ctx_id/etc */ +}; +``` + +#### 3.9.1.2 ubcore_stats_val + +```c +struct ubcore_stats_val { + /* this addr is alloc and free by ubcore, + * refer to struct ubcore_stats_com_val + */ + uint64_t addr; + /* [in/out] real length filled when success + * to query and buffer length enough; + * expected length filled and return failure when buffer length not enough + */ + uint32_t len; +}; +``` + +### 3.9.2 ubcore_query_resource + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_query_resource([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, [4.9.2.1](#3921-ubcore_res_key) [ubcore_res_key](#3921-ubcore_res_key) *key, [4.9.2.2](#3922-ubcore_res_val) [ubcore_res_val](#3922-ubcore_res_val) *val); + +3. 描述 + +query_res为一个函数指针，是ubcore_ops_t结构体成员，在ubcore_register_device接口调用时由厂商驱动向UBN协议栈注册。query_res提供设备级资源使用情况查询，例如Jetty创建数目和VF的UPI等。 + +4. 参数 + +@param[in] [Required] dev：ubcore_device指针; + +@param[in] [Required] key：指定待查询信息的类型和key值; + +@param[in/out] [Required] val：返回查询结果的存放地址和实际数据的长度. + +5. 返回值 + +0表示查询成功，非0表示查询失败。Val中指定的buffer足够时，查询操作成功，返回实际填写的数据长度；Buffer不足时，返回失败，驱动填期望的buffer长度。 + +#### 3.9.2.1 ubcore_res_key + +```c +struct ubcore_res_key { + uint8_t type; /* refer to ubcore_res_key_type_t */ + uint32_t key; /* as UPI, key is ue_idx */ + uint32_t key_ext; /* only for vtp */ + uint32_t key_cnt; /* only for rc */ +}; +``` + +#### 3.9.2.2 ubcore_res_val + +```c +struct ubcore_res_val { + uint64_t addr; /* allocated and free by ubcore */ + /* in&out. As a input parameter, + * it indicates the length allocated by the ubcore + * As a output parameter, it indicates the actual data length. + */ + uint32_t len; +}; +``` + +## 3.10 驱动自定义接口 + +本节只定义DFX函数，DFX接口中的数据结构定义见《UMDK URMA驱动编程接口说明书》 + +### 3.10.1 ubcore_user_control + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int ubcore_user_control([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, [4.10.1.1](#31011-ubcore_user_ctl) [ubcore_user_ctl](#31011-ubcore_user_ctl) *k_user_ctl); + +3. 描述 + +内核态应用调用该接口来让UDMA驱动驱动执行user_ctl命令 + +4. 参数 + +@param[in] dev: the ubcore device handle; + +@param[in] k_user_ctl: async_event handler to be registered. Note: the handler will be called when driver reports an async_event with ubcore_dispatch_async_event. + +5. 返回值 + +0 on success, other value on error + +#### 3.10.1.1 ubcore_user_ctl + +```c +struct ubcore_user_ctl { + ubcore_ucontext *uctx; + ubcore_user_ctl_in in; + ubcore_user_ctl_out out; + ubcore_udrv_priv udrv_data; +}; +``` + +#### 3.10.1.2 ubcore_user_ctl_in + +```c +struct ubcore_user_ctl_in { + uint64_t addr; + uint32_t len; + uint32_t opcode; +}; +``` + +#### 3.10.1.3 ubcore_user_ctl_out + +```c +struct ubcore_user_ctl_out { + uint64_t addr; + uint32_t len; + uint32_t reserved; +}; +``` + +## 3.11 异步事件分发接口 + +### 3.11.1 ubcore_dispatch_async_event + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +void ubcore_dispatch_async_event([4.5.1.2](#3512-ubcore_event) [ubcore_event](#3512-ubcore_event) *event); + +3. 描述 + +用于异步事件的分发。一旦发生设备异常，芯片会通过硬件中断上报给UDMA驱动，UDMA驱动解析中断信息，查找到异常设备，并调用ubcore_dispatch_async_event将异常上报给UB Core协议栈；UB Core协议栈进一步通知更上层的client。 + +设备异常包括Link状态切换、RAS异常等。 + +4. 参数 + +@param[in] event: asynchronous event. + +5. 返回值 + +void + +## 3.12 内存映射接口 + +### 3.12.1 ubcore_umem_get + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +[4.12.1.1](#31211-ubcore_umem) [ubcore_umem](#31211-ubcore_umem) *ubcore_umem_get([4.2.1.1](#3211-ubcore_device) [ubcore_device](#3211-ubcore_device) *dev, uint64_t va, + +uint64_t len, [4.12.1.2](#31212-ubcore_umem_flag) [ubcore_umem_flag](#31212-ubcore_umem_flag) flag) + +3. 描述 + +UDMA驱动需要分配Jetty队列和非sva情况下segment的物理内存并进行DMA映射时主动调用。 + +4. 参数 + +@param[in] dev: the ubcore device; + +@param[in] va: the VA address to be mapped; + +@param[in] len: Length of the address space to be allocated and mapped by DMA; + +@param[in] flag: Attribute flags. + +5. 返回值 + +umem ptr on success, ERR_PTR on error + +#### 3.12.1.1 ubcore_umem + +```c +struct ubcore_umem { + struct ubcore_device *ub_dev; + struct mm_struct *owning_mm; + uint64_t length; + uint64_t va; + union ubcore_umem_flag flag; + struct sg_table sg_head; + uint32_t nmap; +}; +``` + +#### 3.12.1.2 ubcore_umem_flag + +```c +union ubcore_umem_flag { + struct { + uint32_t non_pin : 1; /* 0: pinned to physical memory. 1: non pin. */ + uint32_t writable : 1; /* 0: read-only. 1: writable. */ + uint32_t reserved : 30; + } bs; + uint32_t value; +}; +``` + +### 3.12.2 ubcore_umem_release + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +void ubcore_umem_release([4.12.1.1](#31211-ubcore_umem) [ubcore_umem](#31211-ubcore_umem) *umem); + +3. 描述 + +释放umem物理内存，解除DMA映射 + +4. 参数 + +@param[in] [Required]umem: the ubcore umem created before. + +5. 返回值 + +void + +### 3.12.3 ubcore_umem_find_best_page_size + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +uint64_t ubcore_umem_find_best_page_size([4.12.1.1](#31211-ubcore_umem) [ubcore_umem](#31211-ubcore_umem) *umem, uint64_t page_size_bitmap, uint64_t va) + +3. 描述 + +查找当前segment最优页面大小。 + +4. 参数 + +@param[in] umem: umem struct, return of ubcore_umem_get; + +@param[in] page_size_bitmap: bitmap of HW supported page sizes, must include PAGE_SIZE; + +@param[in] va: Initial address of this segment. + +5. 返回值 + +Return: 早于5.3的内核版本，返回固定大小4k；5.3及之后的内核版本，当内存不支持获取最优页面大小，返回0；否则返回最优页面大小。 + +## 3.13 其他API + +### 3.13.1 ubcore_dispatch_mgmt_event + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +void ubcore_dispatch_mgmt_event([4.13.1.1](#31311-ubcore_mgmt_event) [ubcore_mgmt_event](#31311-ubcore_mgmt_event) *event); + +3. 描述 + +UB设备分发驱动管理事件。由UDMA调用。 + +4. 参数 + +@param[in] event: event message, including event type, eid_info, etc. + +5. 返回值 + +void + +#### 3.13.1.1 ubcore_mgmt_event + +```c +struct ubcore_mgmt_event { + ubcore_device *ub_dev; + union { + ubcore_eid_info *eid_info; + } element; + ubcore_mgmt_event_type event_type; +}; +``` + +#### 3.13.1.2 ubcore_mgmt_event_type + +```c +enum ubcore_mgmt_event_type { + UBCORE_MGMT_EVENT_EID_ADD, + UBCORE_MGMT_EVENT_EID_RMV, +}; +``` + +### 3.13.2 ubcore_get_tp_list + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +```c +int ubcore_get_tp_list(ubcore_device *dev, ubcore_get_tp_cfg *cfg, +uint32_t *tp_cnt, ubcore_tp_info *tp_list, ubcore_udata *udata); +``` + +3. 描述 + +从UB设备中获取tp列表 + +4. 参数 + +@param[in] dev: ubcore device pointer created before; + +@param[in] cfg: configuration to be matched; + +@param[in && out] tp_cnt: tp_cnt is the length of tp_list buffer as in parameter; tp_cnt is the number of tp as out parameter; + +@param[out] tp_list: tp info list to get; + +@param[in && out] udata: ucontext and user space driver data. + +5. 返回值 + +0 on success, other value on error + +#### 3.13.2.1 ubcore_get_tp_cfg + +```c +struct ubcore_get_tp_cfg { + ubcore_get_tp_cfg_flag flag; + ubcore_transport_mode trans_mode; + ubcore_eid local_eid; + ubcore_eid peer_eid; +}; +``` + +#### 3.13.2.2 ubcore_get_tp_cfg_flag + +```c +union ubcore_get_tp_cfg_flag { + struct { + uint32_t ctp : 1; + uint32_t rtp : 1; + uint32_t utp : 1; + uint32_t uboe : 1; + uint32_t pre_defined : 1; + uint32_t dynamic_defined : 1; + uint32_t reserved : 26; + } bs; + uint32_t value; +}; +``` + +#### 3.13.2.3 ubcore_tp_info + +```c +struct ubcore_tp_info { + ubcore_tp_handle tp_handle; +}; +``` + +#### 3.13.2.4 ubcore_tp_handle + +```c +union ubcore_tp_handle { + struct { + uint64_t tpid : 24; + uint64_t tpn_start : 24; + uint64_t tp_cnt : 5; + uint64_t ctp : 1; + uint64_t rtp : 1; + uint64_t utp : 1; + uint64_t uboe : 1; + uint64_t pre_defined : 1; + uint64_t dynamic_defined : 1; + uint64_t reserved : 5; + } bs; + uint64_t value; +}; +``` + +### 3.13.3 ubcore_set_tp_attr + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +```c +int ubcore_set_tp_attr(ubcore_device *dev, const uint64_t tp_handle, +const uint8_t tp_attr_cnt, const uint32_t tp_attr_bitmap, +const ubcore_tp_attr_value *tp_attr, ubcore_udata *udata); +``` + +3. 描述 + +通过管控面设置tp属性 + +4. 参数 + +@param[in] dev: ubcore device pointer created before; + +@param[in] tp_handle: tp_handle got by ubcore_get_tp_list; + +@param[in] tp_attr_cnt: number of tp attributions; + +@param[in] tp_attr_bitmap: tp attributions bitmap, current bitmap is as follow: + +```c +* 0-retry_times_init: 3 bit 1-at: 5 bit 2-SIP: 128 bit +* 3-DIP: 128 bit 4-SMA: 48 bit 5-DMA: 48 bit +* 6-vlan_id: 12 bit 7-vlan_en: 1 bit 8-dscp: 6 bit +``` + +* 9-at_times: 5 bit 10-sl: 4 bit 11-tti: 8 bit + +@param[in] tp_attr: tp attribution values to set; + +@param[in && out] udata: ucontext and user space driver data. + +5. 返回值 + +0 on success, other value on error + +#### 3.13.3.1 ubcore_tp_attr_value + +```c +struct ubcore_tp_attr_value { + uint8_t retry_times_init : 3; + uint8_t at : 5; + uint8_t sip[UBCORE_IP_ADDR_BYTES]; + uint8_t dip[UBCORE_IP_ADDR_BYTES]; + uint8_t sma[UBCORE_MAC_BYTES]; + uint8_t dma[UBCORE_MAC_BYTES]; + uint16_t vlan_id : 12; + uint8_t vlan_en : 1; + uint8_t dscp : 6; + uint8_t at_times : 5; + uint8_t sl : 4; + uint8_t ttl; + uint8_t reserved[78]; +}; +``` + +### 3.13.4 ubcore_get_tp_attr + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +```c +int ubcore_get_tp_attr(ubcore_device *dev, const uint64_t tp_handle, uint8_t *tp_attr_cnt, +uint32_t *tp_attr_bitmap, ubcore_tp_attr_value *tp_attr, ubcore_udata *udata); +``` + +3. 描述 + +通过管控面获取tp属性 + +4. 参数 + +@param[in] dev: ubcore device pointer created before; + +@param[in] tp_handle: tp_handle got by ubcore_get_tp_list; + +@param[in] tp_attr_cnt: number of tp attributions; + +@param[in] tp_attr_bitmap: tp attributions bitmap, current bitmap is as follow: + +```c +* 0-retry_times_init: 3 bit 1-at: 5 bit 2-SIP: 128 bit +* 3-DIP: 128 bit 4-SMA: 48 bit 5-DMA: 48 bit +* 6-vlan_id: 12 bit 7-vlan_en: 1 bit 8-dscp: 6 bit +``` + +* 9-at_times: 5 bit 10-sl: 4 bit 11-tti: 8 bit + +@param[in] tp_attr: tp attribution values to set; + +@param[in && out] udata: ucontext and user space driver data. + +5. 返回值 + +0 on success, other value on error + +### 3.13.5 ubcore_exchange_tp_info + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +```c +int ubcore_exchange_tp_info(ubcore_device *dev, +ubcore_get_tp_cfg *cfg, uint64_t tp_handle, +uint32_t tx_psn, uint64_t *peer_tp_handle, +uint32_t *rx_psn, ubcore_udata *udata); +``` + +3. 描述 + +交换UB设备的tp信息 + +4. 参数 + +@param[in] dev: ubcore device pointer created before; + +@param[in] cfg: configuration to be matched; + +@param[in] tp_handle: local tp handle; + +@param[in] tx_psn: local packet sequence number; + +@param[out] peer_tp_handle: tp_handle got by ubcore_exchange_tp_info; + +@param[out] rx_psn: remote packet sequence number; + +@param[in] [Optional] udata: udata should be NULL when called by kernel application and be valid when called by user space application. + +5. 返回值 + +0 on success, other value on error + +--- +# 4 URMA用户态驱动接口 + +\-\-\--暂不刷新 + +--- +# 5 URMA内核态驱动接口 + +## 5.1 内核态UB设备管理接口 + +UDMA驱动加载时注册设备到UB Core中，同时提供ops函数。 + +### 5.1.1 UB设备注册接口 + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +int ubcore_register_device(struct ubcore_device *dev); + +3. 描述 + +向 ubcore 注册一个UB设备。 + +4. 参数 + +@param[in] dev: the ubcore device; + +5. 返回值 + +Return: 0 on success, other value on error + +#### 5.1.1.1 ubcore_device + +```c +struct ubcore_device { + struct list_head list_node; /* add to device list */ + /* driver fills start */ + char dev_name[UBCORE_MAX_DEV_NAME]; + struct device *dma_dev; + struct device dev; + struct net_device *netdev; + struct ubcore_ops *ops; + enum ubcore_transport_type transport_type; + struct ubcore_device_attr attr; + struct attribute_group *group[UBCORE_MAX_ATTR_GROUP]; /* driver may fill group [1] */ + /* driver fills end */ + struct ubcore_device_cfg cfg; + /* port management */ + struct list_head port_list; + /* For ubcore client */ + struct rw_semaphore client_ctx_rwsem; + struct list_head client_ctx_list; + struct list_head event_handler_list; + struct rw_semaphore event_handler_rwsem; + struct ubcore_hash_table ht[UBCORE_HT_NUM]; /* to be replaced with uobj */ + /* protect from unregister device */ + atomic_t use_cnt; + struct completion comp; + bool dynamic_eid; /* Assign eid dynamically with netdev notifier */ + struct ubcore_eid_table eid_table; + struct ubcore_cg_device cg_device; + struct ubcore_sip_table sip_table; + /* logic device list and mutex */ + struct ubcore_logic_device ldev; + struct mutex ldev_mutex; + struct list_head ldev_list; + /* ue_idx to uvs_instance mapping */ + void **ue2uvs_table; + struct rw_semaphore ue2uvs_rwsem; + /* for vtp audit */ + struct ubcore_vtp_bitmap vtp_bitmap; +}; +``` + +#### 5.1.1.2 ubcore_eid_entry + +```c +struct ubcore_eid_entry { + union ubcore_eid eid; + uint32_t eid_index; + struct net *net; + bool valid; +}; +``` + +#### 5.1.1.3 ubcore_eid_table + +```c +struct ubcore_eid_table { + uint32_t eid_cnt; + struct ubcore_eid_entry *eid_entries; + spinlock_t lock; +}; +``` + +#### 5.1.1.4 ubcore_sip_info + +```c +struct ubcore_sip_info { + char dev_name[UBCORE_MAX_DEV_NAME]; + struct ubcore_net_addr addr; + uint32_t prefix_len; + uint8_t port_cnt; + uint8_t port_id[UBCORE_MAX_PORT_CNT]; + uint32_t mtu; + char netdev_name[UBCORE_MAX_DEV_NAME]; */* for change mtu */* + bool is_active; +}; +``` + +#### 5.1.1.5 ubcore_sip_entry + +```c +struct ubcore_sip_entry { + struct ubcore_sip_info sip_info; + atomic_t uvs_cnt; + uint64_t reserve; +}; +``` + +#### 5.1.1.6 ubcore_sip_table + +```c +struct ubcore_sip_table { + struct mutex lock; + uint32_t max_sip_cnt; + struct ubcore_sip_entry *entry; + DECLARE_BITMAP(index_bitmap, UBCORE_MAX_SIP); +}; +``` + +#### 5.1.1.7 ubcore_port_kobj + +```c +struct ubcore_port_kobj { + struct kobject kobj; + struct ubcore_device *dev; + uint8_t port_id; +}; +``` + +#### 5.1.1.8 ubcore_eid_attr + +```c +struct ubcore_eid_attr { + char name[UBCORE_EID_GROUP_NAME_LEN]; + uint32_t eid_idx; + struct device_attribute attr; +}; +``` + +#### 5.1.1.9 ubcore_logic_device + +```c +struct ubcore_logic_device { + struct device *dev; + struct ubcore_port_kobj port[UBCORE_MAX_PORT_CNT]; + struct list_head node; */* add to ldev list */* + possible_net_t net; + struct ubcore_device *ub_dev; + const struct attribute_group *dev_group[UBCORE_ATTR_GROUP_MAX]; +}; +``` + +#### 5.1.1.10 ubcore_vtp_bitmap + +```c +struct ubcore_vtp_bitmap { + struct mutex lock; + uint32_t max_vtp_cnt; + uint64_t *bitmap; +}; +``` + +#### 5.1.1.11 ubcore_ops + +```c +struct ubcore_ops { + struct module *owner; */* kernel driver module */* + char driver_name[UBCORE_MAX_DRIVER_NAME]; */* user space driver name */* + uint32_t abi_version; */* abi version of kernel driver */* + */*** + ** add a function entity id (eid) to ub device (for uvs)* + *** @param*[in] dev: the ubcore_device handle;* + *** @param*[in] ue_idx: ue_idx;* + *** @param*[in] cfg: eid and the upi of ue to which the eid belongs can be specified;* + *** @return*: the index of eid/upi, less than 0 indicating error* + **/* + int (*add_ueid)(struct ubcore_device *dev, uint16_t ue_idx, + struct ubcore_ueid_cfg *cfg); + */*** + ** delete a function entity id (eid) to ub device (for uvs)* + *** @param*[in] dev: the ubcore_device handle;* + *** @param*[in] ue_idx: ue_idx;* + *** @param*[in] cfg: eid and the upi of ue to which the eid belongs can be specified;* + *** @return*: 0 on success, other value on error* + **/* + int (*delete_ueid)(struct ubcore_device *dev, uint16_t ue_idx, + struct ubcore_ueid_cfg *cfg); + */*** + ** query device attributes* + *** @param*[in] dev: the ub device handle;* + *** @param*[out] attr: attributes for the driver to fill in* + *** @return*: 0 on success, other value on error* + **/* + int (*query_device_attr)(struct ubcore_device *dev, + struct ubcore_device_attr *attr); + */*** + ** query device status* + *** @param*[in] dev: the ub device handle;* + *** @param*[out] status: status for the driver to fill in* + *** @return*: 0 on success, other value on error* + **/* + int (*query_device_status)(struct ubcore_device *dev, + struct ubcore_device_status *status); + */*** + ** query resource* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] key: resource type and key;* + *** @param*[in/out] val: addr and len of value* + *** @return*: 0 on success, other value on error* + **/* + int (*query_res)(struct ubcore_device *dev, struct ubcore_res_key *key, + struct ubcore_res_val *val); + */*** + ** config device* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] cfg: device configuration* + *** @return*: 0 on success, other value on error* + **/* + int (*config_device)(struct ubcore_device *dev, + struct ubcore_device_cfg *cfg); + */*** + ** set ub network address* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] net_addr: net_addr to set* + *** @param*[in] index: index by sip table* + *** @return*: 0 on success, other value on error* + **/* + int (*add_net_addr)(struct ubcore_device *dev, + struct ubcore_net_addr *net_addr, uint32_t index); + */*** + ** unset ub network address* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] idx: net_addr idx by sip table entry* + *** @return*: 0 on success, other value on error* + **/* + int (*delete_net_addr)(struct ubcore_device *dev, uint32_t idx); + */*** + ** allocate a context from ubep for a user process* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] eid: function entity id (eid) index to set;* + *** @param*[in] udrv_data: user space driver data* + *** @return*: pointer to user context on success, null or error,* + **/* + struct ubcore_ucontext *(*alloc_ucontext)( + struct ubcore_device *dev, uint32_t eid_index, + struct ubcore_udrv_priv *udrv_data); + */*** + ** free a context to ubep* + *** @param*[in] uctx: the user context created before;* + *** @return*: 0 on success, other value on error* + **/* + int (*free_ucontext)(struct ubcore_ucontext *uctx); + */*** + ** mmap doorbell or jetty buffer, etc* + *** @param*[in] uctx: the user context created before;* + *** @param*[in] vma: linux vma including vm_start, vm_pgoff, etc;* + *** @return*: 0 on success, other value on error* + **/* + int (*mmap)(struct ubcore_ucontext *ctx, struct vm_area_struct *vma); + */* segment part */* + */** alloc token id to ubep* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] flag: token_id_flag;* + *** @param*[in] udata: ucontext and user space driver data* + *** @return*: token id pointer on success, NULL on error* + **/* + struct ubcore_token_id *(*alloc_token_id)( + struct ubcore_device *dev, union ubcore_token_id_flag flag, + struct ubcore_udata *udata); + */** free key id from ubep* + *** @param*[in] token_id: the token id alloced before;* + *** @return*: 0 on success, other value on error* + **/* + int (*free_token_id)(struct ubcore_token_id *token_id); + */** register segment to ubep* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] cfg: segment attributes and configurations* + *** @param*[in] udata: ucontext and user space driver data* + *** @return*: target segment pointer on success, NULL on error* + **/* + struct ubcore_target_seg *(*register_seg)(struct ubcore_device *dev, + struct ubcore_seg_cfg *cfg, + struct ubcore_udata *udata); + */** unregister segment from ubep* + *** @param*[in] tseg: the segment registered before;* + *** @return*: 0 on success, other value on error* + **/* + int (*unregister_seg)(struct ubcore_target_seg *tseg); + */** import a remote segment to ubep* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] cfg: segment attributes and import configurations* + *** @param*[in] udata: ucontext and user space driver data* + *** @return*: target segment handle on success, NULL on error* + **/* + struct ubcore_target_seg *(*import_seg)( + struct ubcore_device *dev, struct ubcore_target_seg_cfg *cfg, + struct ubcore_udata *udata); + */** unimport seg from ubep* + *** @param*[in] tseg: the segment imported before;* + *** @return*: 0 on success, other value on error* + **/* + int (*unimport_seg)(struct ubcore_target_seg *tseg); + */** add port for bound device* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] port_cnt: port count* + *** @param*[in] port_list: port list* + *** @return*: target segment handle on success, NULL on error* + **/* + int (*add_port)(struct ubcore_device *dev, uint32_t port_cnt, + uint32_t *port_list); + */* jetty part */* + */*** + ** create jfc with ubep.* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] cfg: jfc attributes and configurations* + *** @param*[in] udata: ucontext and user space driver data* + *** @return*: jfc pointer on success, NULL on error* + **/* + struct ubcore_jfc *(*create_jfc)(struct ubcore_device *dev, + struct ubcore_jfc_cfg *cfg, + struct ubcore_udata *udata); + */*** + ** modify jfc from ubep.* + *** @param*[in] jfc: the jfc created before;* + *** @param*[in] attr: ubcore jfc attr;* + *** @param*[in] udata: ucontext and user space driver data* + *** @return*: 0 on success, other value on error* + **/* + int (*modify_jfc)(struct ubcore_jfc *jfc, struct ubcore_jfc_attr *attr, + struct ubcore_udata *udata); + */*** + ** destroy jfc from ubep.* + *** @param*[in] jfc: the jfc created before;* + *** @return*: 0 on success, other value on error* + **/* + int (*destroy_jfc)(struct ubcore_jfc *jfc); + */*** + ** batch destroy jfc from ubep.* + *** @param*[in] jfc_arr: the jfc array created before;* + *** @param*[in] jfc_num: jfc array length;* + *** @param*[out] bad_jfc_index: when delete err, return jfc index in the array;* + *** @return*: 0 on success, other value on error* + **/* + int (*destroy_jfc_batch)(struct ubcore_jfc **jfc_arr, int jfc_num, + int *bad_jfc_index); + */*** + ** rearm jfc.* + *** @param*[in] jfc: the jfc created before;* + *** @param*[in] solicited_only: rearm notify by message marked with solicited flag* + *** @return*: 0 on success, other value on error* + **/* + int (*rearm_jfc)(struct ubcore_jfc *jfc, bool solicited_only); + */*** + ** create jfs with ubep.* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] cfg: jfs attributes and configurations* + *** @param*[in] udata: ucontext and user space driver data* + *** @return*: jfs pointer on success, NULL on error* + **/* + struct ubcore_jfs *(*create_jfs)(struct ubcore_device *dev, + struct ubcore_jfs_cfg *cfg, + struct ubcore_udata *udata); + */*** + ** modify jfs from ubep.* + *** @param*[in] jfs: the jfs created before;* + *** @param*[in] attr: ubcore jfs attr;* + *** @param*[in] udata: ucontext and user space driver data* + *** @return*: 0 on success, other value on error* + **/* + int (*modify_jfs)(struct ubcore_jfs *jfs, struct ubcore_jfs_attr *attr, + struct ubcore_udata *udata); + */*** + ** query jfs from ubep.* + *** @param*[in] jfs: the jfs created before;* + *** @param*[out] cfg: jfs configurations;* + *** @param*[out] attr: ubcore jfs attributes;* + *** @return*: 0 on success, other value on error* + **/* + int (*query_jfs)(struct ubcore_jfs *jfs, struct ubcore_jfs_cfg *cfg, + struct ubcore_jfs_attr *attr); + */*** + ** flush jfs from ubep.* + *** @param*[in] jfs: the jfs created before;* + *** @param*[in] cr_cnt: the maximum number of CRs expected to be returned;* + *** @param*[out] cr: the addr of returned CRs;* + *** @return*: the number of CR returned, 0 means no completion record returned, -1 on error* + **/* + int (*flush_jfs)(struct ubcore_jfs *jfs, int cr_cnt, + struct ubcore_cr *cr); + */*** + ** destroy jfs from ubep.* + *** @param*[in] jfs: the jfs created before;* + *** @return*: 0 on success, other value on error* + **/* + int (*destroy_jfs)(struct ubcore_jfs *jfs); + */*** + ** batch destroy jfs from ubep.* + *** @param*[in] jfs_arr: the jfs array created before;* + *** @param*[in] jfs_num: jfs array length;* + *** @param*[out] bad_jfs_index: when error, return error jfs index in the array;* + *** @return*: 0 on success, other value on error* + **/* + int (*destroy_jfs_batch)(struct ubcore_jfs **jfs_arr, int jfs_num, + int *bad_jfs_index); + */*** + ** create jfr with ubep.* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] cfg: jfr attributes and configurations* + *** @param*[in] udata: ucontext and user space driver data* + *** @return*: jfr pointer on success, NULL on error* + **/* + struct ubcore_jfr *(*create_jfr)(struct ubcore_device *dev, + struct ubcore_jfr_cfg *cfg, + struct ubcore_udata *udata); + */*** + ** modify jfr from ubep.* + *** @param*[in] jfr: the jfr created before;* + *** @param*[in] attr: ubcore jfr attr;* + *** @param*[in] udata: ucontext and user space driver data* + *** @return*: 0 on success, other value on error* + **/* + int (*modify_jfr)(struct ubcore_jfr *jfr, struct ubcore_jfr_attr *attr, + struct ubcore_udata *udata); + */*** + ** query jfr from ubep.* + *** @param*[in] jfr: the jfr created before;* + *** @param*[out] cfg: jfr configurations;* + *** @param*[out] attr: ubcore jfr attributes;* + *** @return*: 0 on success, other value on error* + **/* + int (*query_jfr)(struct ubcore_jfr *jfr, struct ubcore_jfr_cfg *cfg, + struct ubcore_jfr_attr *attr); + */*** + ** destroy jfr from ubep.* + *** @param*[in] jfr: the jfr created before;* + *** @return*: 0 on success, other value on error* + **/* + int (*destroy_jfr)(struct ubcore_jfr *jfr); + */*** + ** batch destroy jfr from ubep.* + *** @param*[in] jfr_arr: the jfr array created before;* + *** @param*[in] jfr_num: jfr array length;* + *** @param*[out] bad_jfr_index: when error, return error jfr index in the array;* + *** @return*: 0 on success, other value on error* + **/* + int (*destroy_jfr_batch)(struct ubcore_jfr **jfr_arr, int jfr_num, + int *bad_jfr_index); + */*** + ** import jfr to ubep.* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] cfg: remote jfr attributes and import configurations* + *** @param*[in] udata: ucontext and user space driver data* + *** @return*: target jfr pointer on success, NULL on error* + **/* + struct ubcore_tjetty *(*import_jfr)(struct ubcore_device *dev, + struct ubcore_tjetty_cfg *cfg, + struct ubcore_udata *udata); + */*** + ** import jfr to ubep by control plane.* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] cfg: remote jfr attributes and import configurations;* + *** @param*[in] active_tp_cfg: tp configuration to active;* + *** @param*[in] udata: ucontext and user space driver data* + *** @return*: target jfr pointer on success, NULL on error* + **/* + struct ubcore_tjetty *(*import_jfr_ex)( + struct ubcore_device *dev, struct ubcore_tjetty_cfg *cfg, + struct ubcore_active_tp_cfg *active_tp_cfg, + struct ubcore_udata *udata); + */*** + ** unimport jfr from ubep.* + *** @param*[in] tjfr: the target jfr imported before;* + *** @return*: 0 on success, other value on error* + **/* + int (*unimport_jfr)(struct ubcore_tjetty *tjfr); + */*** + ** create jetty with ubep.* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] cfg: jetty attributes and configurations* + *** @param*[in] udata: ucontext and user space driver data* + *** @return*: jetty pointer on success, NULL on error* + **/* + struct ubcore_jetty *(*create_jetty)(struct ubcore_device *dev, + struct ubcore_jetty_cfg *cfg, + struct ubcore_udata *udata); + */*** + ** modify jetty from ubep.* + *** @param*[in] jetty: the jetty created before;* + *** @param*[in] attr: ubcore jetty attr;* + *** @param*[in] udata: ucontext and user space driver data* + *** @return*: 0 on success, other value on error* + **/* + int (*modify_jetty)(struct ubcore_jetty *jetty, + struct ubcore_jetty_attr *attr, + struct ubcore_udata *udata); + */*** + ** query jetty from ubep.* + *** @param*[in] jetty: the jetty created before;* + *** @param*[out] cfg: jetty configurations;* + *** @param*[out] attr: ubcore jetty attributes;* + *** @return*: 0 on success, other value on error* + **/* + int (*query_jetty)(struct ubcore_jetty *jetty, + struct ubcore_jetty_cfg *cfg, + struct ubcore_jetty_attr *attr); + */*** + ** flush jetty from ubep.* + *** @param*[in] jetty: the jetty created before;* + *** @param*[in] cr_cnt: the maximum number of CRs expected to be returned;* + *** @param*[out] cr: the addr of returned CRs;* + *** @return*: the number of CR returned, 0 means no completion record returned, -1 on error* + **/* + int (*flush_jetty)(struct ubcore_jetty *jetty, int cr_cnt, + struct ubcore_cr *cr); + */*** + ** destroy jetty from ubep.* + *** @param*[in] jetty: the jetty created before;* + *** @return*: 0 on success, other value on error* + **/* + int (*destroy_jetty)(struct ubcore_jetty *jetty); + */*** + ** batch destroy jetty from ubep.* + *** @param*[in] jetty_arr: the jetty array created before;* + *** @param*[in] jetty_num: jetty array length;* + *** @param*[out] bad_jetty_index: when error, return error jetty index in the array;* + *** @return*: 0 on success, other value on error* + **/* + int (*destroy_jetty_batch)(struct ubcore_jetty **jetty_arr, + int jetty_num, int *bad_jetty_index); + */*** + ** import jetty to ubep.* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] cfg: remote jetty attributes and import configurations* + *** @param*[in] udata: ucontext and user space driver data* + *** @return*: target jetty pointer on success, NULL on error* + **/* + struct ubcore_tjetty *(*import_jetty)(struct ubcore_device *dev, + struct ubcore_tjetty_cfg *cfg, + struct ubcore_udata *udata); + */*** + ** import jetty to ubep by control plane.* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] cfg: remote jetty attributes and import configurations* + *** @param*[in] active_tp_cfg: tp configuration to active* + *** @param*[in] udata: ucontext and user space driver data* + *** @return*: target jetty pointer on success, NULL on error* + **/* + struct ubcore_tjetty *(*import_jetty_ex)( + struct ubcore_device *dev, struct ubcore_tjetty_cfg *cfg, + struct ubcore_active_tp_cfg *active_tp_cfg, + struct ubcore_udata *udata); + */*** + ** unimport jetty from ubep.* + *** @param*[in] tjetty: the target jetty imported before;* + *** @return*: 0 on success, other value on error* + **/* + int (*unimport_jetty)(struct ubcore_tjetty *tjetty); + */*** + ** bind jetty from ubep.* + *** @param*[in] jetty: the jetty created before;* + *** @param*[in] tjetty: the target jetty imported before;* + *** @param*[in] udata: ucontext and user space driver data* + *** @return*: 0 on success, other value on error* + **/* + int (*bind_jetty)(struct ubcore_jetty *jetty, + struct ubcore_tjetty *tjetty, + struct ubcore_udata *udata); + */*** + ** bind jetty from ubep by control plane.* + *** @param*[in] jetty: the jetty created before;* + *** @param*[in] tjetty: the target jetty imported before;* + *** @param*[in] active_tp_cfg: tp configuration to active;* + *** @param*[in] udata: ucontext and user space driver data* + *** @return*: 0 on success, other value on error* + **/* + int (*bind_jetty_ex)(struct ubcore_jetty *jetty, + struct ubcore_tjetty *tjetty, + struct ubcore_active_tp_cfg *active_tp_cfg, + struct ubcore_udata *udata); + */*** + ** unbind jetty from ubep.* + *** @param*[in] jetty: the jetty binded before;* + *** @return*: 0 on success, other value on error* + **/* + int (*unbind_jetty)(struct ubcore_jetty *jetty); + */*** + ** create jetty group to ubep.* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] cfg: pointer of the jetty group config;* + *** @param*[in] udata: ucontext and user space driver data* + *** @return*: jetty group pointer on success, NULL on error* + **/* + struct ubcore_jetty_group *(*create_jetty_grp)( + struct ubcore_device *dev, struct ubcore_jetty_grp_cfg *cfg, + struct ubcore_udata *udata); + */*** + ** destroy jetty group to ubep.* + *** @param*[in] jetty_grp: the jetty group created before;* + *** @return*: 0 on success, other value on error* + **/* + int (*delete_jetty_grp)(struct ubcore_jetty_group *jetty_grp); + */*** + ** create tpg.* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] cfg: tpg init attributes* + *** @param*[in] udata: ucontext and user space driver data* + *** @return*: tp pointer on success, NULL on error* + **/* + struct ubcore_tpg *(*create_tpg)(struct ubcore_device *dev, + struct ubcore_tpg_cfg *cfg, + struct ubcore_udata *udata); + */*** + ** destroy tpg.* + *** @param*[in] tp: tp pointer created before* + *** @return*: 0 on success, other value on error* + **/* + int (*destroy_tpg)(struct ubcore_tpg *tpg); + */*** + ** get tpid list by control plane.* + *** @param*[in] dev: ubcore device pointer created before* + *** @param*[in] cfg: tpid configuration to be matched* + *** @param*[in && out] tp_cnt: tp_cnt is the length of tp_list buffer as in parameter;* + ** tp_cnt is the number of tp as out parameter* + *** @param*[out] tp_list: tp list to get, the buffer is allocated by user;* + *** @param*[in && out] udata: ucontext and user space driver data* + *** @return*: 0 on success, other value on error* + **/* + int (*get_tp_list)(struct ubcore_device *dev, + struct ubcore_get_tp_cfg *cfg, uint32_t *tp_cnt, + struct ubcore_tp_info *tp_list, + struct ubcore_udata *udata); + */*** + ** set tp attributions by control plane.* + *** @param*[in] dev: ubcore device pointer created before;* + *** @param*[in] tp_handle: tp_handle got by ubcore_get_tp_list;* + *** @param*[in] tp_attr_cnt: number of tp attributions;* + *** @param*[in] tp_attr_bitmap: tp attributions bitmap, current bitmap is as follow:* + ** 0-retry_times_init: 3 bit 1-at: 5 bit 2-*SIP: *128 bit* + ** 3-*DIP: *128 bit 4-*SMA: *48 bit 5-*DMA: *48 bit* + ** 6-vlan_id: 12 bit 7-vlan_en: 1 bit 8-dscp: 6 bit* + ** 9-at_times: 5 bit 10-sl: 4 bit 11-tti: 8 bit* + *** @param*[in] tp_attr: tp attribution values to set;* + *** @param*[in && out] udata: ucontext and user space driver data;* + *** @return*: 0 on success, other value on error* + **/* + int (*set_tp_attr)(struct ubcore_device *dev, const uint64_t tp_handle, + const uint8_t tp_attr_cnt, + const uint32_t tp_attr_bitmap, + const struct ubcore_tp_attr_value *tp_attr, + struct ubcore_udata *udata); + */*** + ** get tp attributions by control plane.* + *** @param*[in] dev: ubcore device pointer created before;* + *** @param*[in] tp_handle: tp_handle got by ubcore_get_tp_list;* + *** @param*[out] tp_attr_cnt: number of tp attributions;* + *** @param*[out] tp_attr_bitmap: tp bitmap, the same as tp_attr_bitmap in set_tp_attr;* + *** @param*[out] tp_attr: tp attribution values to get;* + *** @param*[in && out] udata: ucontext and user space driver data;* + *** @return*: 0 on success, other value on error* + **/* + int (*get_tp_attr)(struct ubcore_device *dev, const uint64_t tp_handle, + uint8_t *tp_attr_cnt, uint32_t *tp_attr_bitmap, + struct ubcore_tp_attr_value *tp_attr, + struct ubcore_udata *udata); + */*** + ** active tp by control plane.* + *** @param*[in] dev: ubcore device pointer created before* + *** @param*[in] active_cfg: tp configuration to active* + *** @return*: 0 on success, other value on error* + **/* + int (*active_tp)(struct ubcore_device *dev, + struct ubcore_active_tp_cfg *active_cfg); + */*** + ** deactivate tp by control plane.* + *** @param*[in] dev: ubcore device pointer created before* + *** @param*[in] tp_handle: tp_handle value got before* + *** @param*[in] udata: [Optional] udata should be NULL when called* + ** by kernel application and be valid when called* + ** by user space application* + *** @return*: 0 on success, other value on error* + **/* + int (*deactive_tp)(struct ubcore_device *dev, + union ubcore_tp_handle tp_handle, + struct ubcore_udata *udata); + */*** + ** create tp.* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] cfg: tp init attributes* + *** @param*[in] udata: ucontext and user space driver data* + *** @return*: tp pointer on success, NULL on error* + **/* + struct ubcore_tp *(*create_tp)(struct ubcore_device *dev, + struct ubcore_tp_cfg *cfg, + struct ubcore_udata *udata); + */*** + ** modify tp.* + *** @param*[in] tp: tp pointer created before* + *** @param*[in] attr: tp attributes* + *** @param*[in] mask: attr mask indicating the attributes to be modified* + *** @return*: 0 on success, other value on error* + **/* + int (*modify_tp)(struct ubcore_tp *tp, struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask mask); + */*** + ** modify user tp.* + *** @param*[in] dev: the ub device handle* + *** @param*[in] tpn: tp number of the tp created before* + *** @param*[in] cfg: user configuration of the tp* + *** @param*[in] attr: tp attributes* + *** @param*[in] mask: attr mask indicating the attributes to be modified* + *** @return*: 0 on success, other value on error* + **/* + int (*modify_user_tp)(struct ubcore_device *dev, uint32_t tpn, + struct ubcore_tp_cfg *cfg, + struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask mask); + */*** + ** destroy tp.* + *** @param*[in] tp: tp pointer created before* + *** @return*: 0 on success, other value on error* + **/* + int (*destroy_tp)(struct ubcore_tp *tp); + */*** + ** create multi tp.* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] cnt: the number of tp, must be less than or equal to 32;* + *** @param*[in] cfg: array of tp init attributes* + *** @param*[in] udata: array of ucontext and user space driver data* + *** @param*[out] tp: pointer array of tp* + *** @return*: created tp cnt, 0 on error* + **/* + int (*create_multi_tp)(struct ubcore_device *dev, uint32_t cnt, + struct ubcore_tp_cfg *cfg, + struct ubcore_udata *udata, + struct ubcore_tp **tp); + */*** + ** modify multi tp.* + *** @param*[in] cnt: the number of tp;* + *** @param*[in] tp: pointer array of tp created before* + *** @param*[in] attr: array of tp attributes* + *** @param*[in] mask: array of attr mask indicating the attributes to be modified* + *** @param*[in] fail_tp: pointer of tp failed to modify* + *** @return*: modified successfully tp cnt, 0 on error* + **/* + int (*modify_multi_tp)(uint32_t cnt, struct ubcore_tp **tp, + struct ubcore_tp_attr *attr, + union ubcore_tp_attr_mask *mask, + struct ubcore_tp **fail_tp); + */*** + ** destroy multi tp.* + *** @param*[in] cnt: the number of tp;* + *** @param*[in] tp: pointer array of tp created before* + *** @return*: destroyed tp cnt, 0 on error* + **/* + int (*destroy_multi_tp)(uint32_t cnt, struct ubcore_tp **tp); + */*** + ** allocate vtp.* + *** @param*[in] dev: the ub device handle;* + *** @return*: vtpn pointer on success, NULL on error* + **/* + struct ubcore_vtpn *(*alloc_vtpn)(struct ubcore_device *dev); + */*** + ** free vtpn.* + *** @param*[in] vtpn: vtpn pointer allocated before* + *** @return*: 0 on success, other value on error* + **/* + int (*free_vtpn)(struct ubcore_vtpn *vtpn); + */*** + ** create vtp.* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] cfg: vtp init attributes* + *** @param*[in] udata: ucontext and user space driver data* + *** @return*: vtp pointer on success, NULL on error* + **/* + struct ubcore_vtp *(*create_vtp)(struct ubcore_device *dev, + struct ubcore_vtp_cfg *cfg, + struct ubcore_udata *udata); + */*** + ** destroy vtp.* + *** @param*[in] vtp: vtp pointer created before* + *** @return*: 0 on success, other value on error* + **/* + int (*destroy_vtp)(struct ubcore_vtp *vtp); + */*** + ** create utp.* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] cfg: utp init attributes* + *** @param*[in] udata: ucontext and user space driver data* + *** @return*: utp pointer on success, NULL on error* + **/* + struct ubcore_utp *(*create_utp)(struct ubcore_device *dev, + struct ubcore_utp_cfg *cfg, + struct ubcore_udata *udata); + */*** + ** destroy utp.* + *** @param*[in] utp: utp pointer created before* + *** @return*: 0 on success, other value on error* + **/* + int (*destroy_utp)(struct ubcore_utp *utp); + */*** + ** create ctp.* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] cfg: ctp init attributes* + *** @param*[in] udata: ucontext and user space driver data* + *** @return*: ctp pointer on success, NULL on error* + **/* + struct ubcore_ctp *(*create_ctp)(struct ubcore_device *dev, + struct ubcore_ctp_cfg *cfg, + struct ubcore_udata *udata); + */*** + ** destroy ctp.* + *** @param*[in] ctp: ctp pointer created before* + *** @return*: 0 on success, other value on error* + **/* + int (*destroy_ctp)(struct ubcore_ctp *ctp); + */*** + ** UE send msg to MUE device.* + *** @param*[in] dev: UE or MUE device;* + *** @param*[in] msg: msg to send;* + *** @return*: 0 on success, other value on error* + **/* + int (*send_req)(struct ubcore_device *dev, struct ubcore_req *msg); + */*** + ** MUE send msg to UE device.* + *** @param*[in] dev: MUE device;* + *** @param*[in] msg: msg to send;* + *** @return*: 0 on success, other value on error* + **/* + int (*send_resp)(struct ubcore_device *dev, + struct ubcore_resp_host *msg); + */*** + ** query cc table to get cc pattern idx* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] cc_entry_cnt: cc entry cnt;* + *** @return*: return NULL on fail, otherwise, return cc entry array* + **/* + struct ubcore_cc_entry *(*query_cc)(struct ubcore_device *dev, + uint32_t *cc_entry_cnt); + */*** + ** bond slave net device* + *** @param*[in] bond: bond netdev;* + *** @param*[in] slave: slave netdev;* + *** @param*[in] upper_info: change upper event info;* + *** @return*: 0 on success, other value on error* + **/* + int (*bond_add)(struct net_device *bond, struct net_device *slave, + struct netdev_lag_upper_info *upper_info); + */*** + ** unbond slave net device* + *** @param*[in] bond: bond netdev;* + *** @param*[in] slave: slave netdev;* + *** @return*: 0 on success, other value on error* + **/* + int (*bond_remove)(struct net_device *bond, struct net_device *slave); + */*** + ** update slave net device* + *** @param*[in] bond: bond netdev;* + *** @param*[in] slave: slave netdev;* + *** @param*[in] lower_info: change lower state event info;* + *** @return*: 0 on success, other value on error* + **/* + int (*slave_update)(struct net_device *bond, struct net_device *slave, + struct netdev_lag_lower_state_info *lower_info); + */*** + ** operation of user ioctl cmd.* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] user_ctl: kdrv user control command pointer;* + ** Return: 0 on success, other value on error* + **/* + int (*user_ctl)(struct ubcore_device *dev, + struct ubcore_user_ctl *user_ctl); + */** data path ops */* + */*** + ** post jfs wr.* + *** @param*[in] jfs: the jfs created before;* + *** @param*[in] wr: the wr to be posted;* + *** @param*[out] bad_wr: the first failed wr;* + *** @return*: 0 on success, other value on error* + **/* + int (*post_jfs_wr)(struct ubcore_jfs *jfs, struct ubcore_jfs_wr *wr, + struct ubcore_jfs_wr **bad_wr); + */*** + ** post jfr wr.* + *** @param*[in] jfr: the jfr created before;* + *** @param*[in] wr: the wr to be posted;* + *** @param*[out] bad_wr: the first failed wr;* + *** @return*: 0 on success, other value on error* + **/* + int (*post_jfr_wr)(struct ubcore_jfr *jfr, struct ubcore_jfr_wr *wr, + struct ubcore_jfr_wr **bad_wr); + */*** + ** post jetty send wr.* + *** @param*[in] jetty: the jetty created before;* + *** @param*[in] wr: the wr to be posted;* + *** @param*[out] bad_wr: the first failed wr;* + *** @return*: 0 on success, other value on error* + **/* + int (*post_jetty_send_wr)(struct ubcore_jetty *jetty, + struct ubcore_jfs_wr *wr, + struct ubcore_jfs_wr **bad_wr); + */*** + ** post jetty receive wr.* + *** @param*[in] jetty: the jetty created before;* + *** @param*[in] wr: the wr to be posted;* + *** @param*[out] bad_wr: the first failed wr;* + *** @return*: 0 on success, other value on error* + **/* + int (*post_jetty_recv_wr)(struct ubcore_jetty *jetty, + struct ubcore_jfr_wr *wr, + struct ubcore_jfr_wr **bad_wr); + */*** + ** poll jfc.* + *** @param*[in] jfc: the jfc created before;* + *** @param*[in] cr_cnt: the maximum number of CRs expected to be polled;* + *** @return*: 0 on success, other value on error* + **/* + int (*poll_jfc)(struct ubcore_jfc *jfc, int cr_cnt, + struct ubcore_cr *cr); + */*** + ** query_stats. success to query and buffer length is enough* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] key: type and key value of the ub device to query;* + *** @param*[in/out] val: address and buffer length of query results* + *** @return*: 0 on success, other value on error* + **/* + int (*query_stats)(struct ubcore_device *dev, + struct ubcore_stats_key *key, + struct ubcore_stats_val *val); + */*** + ** config function migrate state.* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] ue_idx: ue id;* + *** @param*[in] cnt: config count;* + *** @param*[in] cfg: eid and the upi of ue to which the eid belongs can be specified;* + *** @param*[in] state: config state (start, rollback and finish)* + *** @return*: config success count, -1 on error* + **/* + int (*config_function_migrate_state)(struct ubcore_device *dev, + uint16_t ue_idx, uint32_t cnt, + struct ubcore_ueid_cfg *cfg, + enum ubcore_mig_state state); + */*** + ** modify vtp.* + *** @param*[in] vtp: vtp pointer to be modified;* + *** @param*[in] attr: vtp attr, tp that we want to change;* + *** @param*[in] mask: attr mask;* + *** @return*: 0 on success, other value on error* + **/* + int (*modify_vtp)(struct ubcore_vtp *vtp, struct ubcore_vtp_attr *attr, + union ubcore_vtp_attr_mask *mask); + */*** + ** query ue index.* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] devid: ue devid to query* + *** @param*[out] ue_idx: ue id;* + *** @return*: 0 on success, other value on error* + **/* + int (*query_ue_idx)(struct ubcore_device *dev, + struct ubcore_devid *devid, uint16_t *ue_idx); + */*** + ** config dscp-vl mapping* + *** @param*[in] dev:the ub dev handle;* + *** @param*[in] dscp: the dscp value array* + *** @param*[in] vl: the vl value array* + *** @param*[in] num: array num* + *** @return*: 0 on success, other value on error* + **/* + int (*config_dscp_vl)(struct ubcore_device *dev, uint8_t *dscp, + uint8_t *vl, uint8_t num); + */*** + ** query ue stats, for migration currently.* + *** @param*[in] dev: the ub device handle;* + *** @param*[in] cnt: array count;* + *** @param*[in] ue_idx: ue id array;* + *** @param*[out] stats: ue counters* + *** @return*: 0 on success, other value on error* + **/* + int (*query_ue_stats)(struct ubcore_device *dev, uint32_t cnt, + uint16_t *ue_idx, struct ubcore_ue_stats *stats); + */*** + ** query dscp-vl mapping* + *** @param*[in] dev:the ub dev handle;* + *** @param*[in] dscp: the dscp value array* + *** @param*[in] num: array num* + *** @param*[out] vl: the vl value array* + *** @return*: 0 on success, other value on error* + **/* + int (*query_dscp_vl)(struct ubcore_device *dev, uint8_t *dscp, + uint8_t num, uint8_t *vl); + */*** + ** When UVS or UB dataplane is running:* + ** 1. disassociate_ucontext != NULL means support rmmod driver.* + ** 2. disassociate_ucontext == NULL means rmmod driver will fail because module is in use.* + ** If disassociate_ucontext !=* NULL: + ** 1. When remove MUE/UE device, will call it;* + ** 2. When remove MUE device, will not call it because there are no uctx.* + *** @param*[in] uctx: the ubcore_ucontext* + **/* + void (*disassociate_ucontext)(struct ubcore_ucontext *uctx); +}; +``` + +#### 5.1.1.12 ubcore_transport_type + +```c +enum ubcore_transport_type { + UBCORE_TRANSPORT_INVALID = -1, + UBCORE_TRANSPORT_UB = 0, + UBCORE_TRANSPORT_MAX +}; +``` + +### 5.1.2 UB设备解注册接口 + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +void ubcore_unregister_device(struct ubcore_device *dev) + +3. 描述 + +UDMA驱动卸载时主动调用，解注册UB设备 + +4. 参数 + +@param[in] dev: the ubcore device; + +5. 返回值 + +NA + +### 5.1.3 内存映射接口 + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +struct ubcore_umem *ubcore_umem_get(struct ubcore_device *dev, uint64_t va, + +uint64_t len, union ubcore_umem_flag flag) + +3. 描述 + +UDMA驱动需要分配Jetty队列和非sva情况下segment的物理内存并进行DMA映射时主动调用。 + +4. 参数 + +@param[in] dev: the ubcore device; + +@param[in] va: the VA address to be mapped. + +@param[in] len: Length of the address space to be allocated and mapped by DMA. + +@param[in] flag: Attribute flags + +5. 返回值 + +umem ptr on success, ERR_PTR on error + +#### 5.1.3.1 ubcore_umem + +```c +struct ubcore_umem { + struct ubcore_device *ub_dev; + struct mm_struct *owning_mm; + uint64_t length; + uint64_t va; + union ubcore_umem_flag flag; + struct sg_table sg_head; + uint32_t nmap; +}; +``` + +#### 5.1.3.2 ubcore_umem_flag + +```c +union ubcore_umem_flag { + struct { + uint32_t non_pin : 1; /* 0: pinned to physical memory. 1: non pin. */ + uint32_t writable : 1; /* 0: read-only. 1: writable. */ + uint32_t reserved : 30; + } bs; + uint32_t value; +}; +``` + +### 5.1.4 内存反映射接口 + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +void ubcore_umem_release(struct ubcore_umem *umem); + +3. 描述 + +释放umem物理内存，解除DMA映射 + +4. 参数 + +@param[in] [Required]umem: the ubcore umem created before + +5. 返回值 + +NA + +### 5.1.5 查找内存最优页面大小接口 + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +uint64_t ubcore_umem_find_best_page_size(struct ubcore_umem *umem, uint64_t page_size_bitmap, uint64_t va) + +3. 描述 + +查找当前segment最优页面大小。 + +4. 参数 + +@param[in] umem: umem struct, return of ubcore_umem_get; + +@param[in] page_size_bitmap: bitmap of HW supported page sizes, must include PAGE_SIZE; + +@param[in] va: Initial address of this segment. + +5. 返回值 + +Return: 早于5.3的内核版本，返回固定大小4k；5.3及之后的内核版本，当内存不支持获取最优页面大小，返回0；否则返回最优页面大小。 + +### 5.1.6 获取MTU接口 + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +enum ubcore_mtu ubcore_get_mtu(int mtu); + +3. 描述 + +获得UB MTU的值。由驱动调用。 + +4. 参数 + +@param[in] [Required] mtu: specifies the MTU value of the NIC interface. + +5. 返回值 + +The MTU of the UB protocol, this value removes the length of the network layer, transport layer, transaction layer header and ICRC. + +### 5.1.7 获取ue接口 + +#### 5.1.7.1 查询ue_idx接口 + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +int (*query_ue_idx)(struct ubcore_device *dev, struct ubcore_devid *devid, uint16_t *ue_idx); + +3. 描述 + +查询ue_idx的值。 + +4. 参数 + +@param[in] dev: the ub device handle; + +@param[in] devid: ue devid to query + +@param[out] ue_idx: ue id; + +5. 返回值 + +*0 on success, other value on error*。 + +#### 5.1.7.2 查询ue状态接口 + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +int (*query_ue_stats)(struct ubcore_device *dev, uint32_t cnt, + +uint16_t *ue_idx, struct ubcore_ue_stats *stats); + +3. 描述 + +查询ue状态。 + +4. 参数 + +@param[in] dev: the ub device handle; + +@param[in] cnt: array count; + +@param[in] ue_idx: ue id array; + +@param[out] stats: ue counters + +5. 返回值 + +*0 on success, other value on error*。 + +### 5.1.8 发送和接口消息接口 + +#### 5.1.8.1 发送请求ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*send_req)(struct ubcore_device *dev, struct ubcore_req *msg); + +3. 描述 + +用于在VM向HOST或者SPU发送请求消息 + +4. 参数 + +@param[in] dev: VF or PF device; + +@param[in] msg: msg to send; + +5. 返回值 + +0 on success, other value on error + +##### 5.1.8.1.1 ubcore_req + +```c +struct ubcore_req { + uint32_t msg_id; + enum ubcore_msg_opcode opcode; + uint32_t len; + uint8_t data[]; +}; +``` + +##### 5.1.8.1.2 ubcore_msg_opcode + +```c +enum ubcore_msg_opcode { + */* 630 Verion msg start */* + UBCORE_MSG_CREATE_VTP = 0x0, + UBCORE_MSG_DESTROY_VTP = 0x1, + UBCORE_MSG_ALLOC_EID = 0x2, + UBCORE_MSG_DEALLOC_EID = 0x3, + UBCORE_MSG_CONFIG_DEVICE = 0x4, + UBCORE_MSG_VTP_STATUS_NOTIFY = 0x5, *// MUE notify MUE/UE* + UBCORE_MSG_UPDATE_EID_TABLE_NOTIFY = 0x6, *// MUE notify MUE/UE* + UBCORE_MSG_UE2MUE_TRANSFER = 0x7, *// UE-MUE common transfer* + UBCORE_MSG_STOP_PROC_VTP_MSG = 0x10, *// Live migration* + UBCORE_MSG_QUERY_VTP_MIG_STATUS = 0x11, *// Live migration* + UBCORE_MSG_FLOW_STOPPED = 0x12, *// Live migration* + UBCORE_MSG_MIG_ROLLBACK = 0x13, *// Live migration* + UBCORE_MSG_MIG_VM_START = 0x14, *// Live migration* + UBCORE_MSG_NEGO_VER = 0x15, *// Verion negotiation, processed by backend ubcore.* + UBCORE_MSG_NOTIFY_FASTMSG_DRAIN = 0x16, + UBCORE_MSG_UPDATE_NET_ADDR = 0x17, + UBCORE_MSP_UPDATE_EID = 0x18 +}; +``` + +#### 5.1.8.2 发送响应ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*send_resp)(struct ubcore_device *dev, struct ubcore_resp_host *msg); + +3. 描述 + +用于HOST或者SPU向VM发送响应消息 + +4. 参数 + +@param[in] dev: TPF device; + +@param[in] msg: msg to send; + +5. 返回值 + +0 on success, other value on error + +##### 5.1.8.2.1 ubcore_resp_host + +```c +struct ubcore_req_host { + uint16_t src_fe_idx; + struct ubcore_req req; +}; +``` + +##### 5.1.8.2.2 ubcore_resp + +```c +struct ubcore_resp { + uint32_t msg_id; + enum ubcore_msg_opcode opcode; + uint32_t len; + uint8_t data[]; +}; +``` + +#### 5.1.8.3 接收请求接口 + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +int ubcore_recv_req(struct ubcore_device *dev, struct ubcore_req_host *req); + +3. 描述 + +ubcore用于接收请求消息。当驱动接收到消息，调用该接口把数据传递给ubcore。驱动分配和释放msg的空间。 + +4. 参数 + +@param[in] dev: TPF device; + +@param[in] req: received msg; + +5. 返回值 + +0 on success, other value on error + +##### 5.1.8.3.1 ubcore_req_host + +```c +struct ubcore_req_host { + uint16_t src_fe_idx; + struct ubcore_req req; +}; +``` + +#### 5.1.8.4 接收响应接口 + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +int ubcore_recv_resp(struct ubcore_device *dev, struct ubcore_resp *resp); + +3. 描述 + +ubcore用于接收响应消息。当驱动接收到消息，调用该接口把数据传递给ubcore。驱动分配和释放msg的空间。 + +4. 参数 + +@param[in] dev: VF or PF device; + +@param[in] msg: received msg; + +5. 返回值 + +0 on success, other value on error + +#### 5.1.8.5 热迁移请求和响应 + +迁移驱动和qemu对接，感知迁移的各个状态：在停流时要通知UVS停止处理删除和新建连接请求，需要查询UVS是否完成了迁移目的端新建连接，停流完成时通知UVS停流完成，回滚时通知UVS发生回滚，目的端虚机起来后通知UVS. + +##### 5.1.8.5.1 ubcore_function_mig_req + +```c +struct ubcore_function_mig_req { + uint16_t mig_fe_idx; +}; +``` + +##### 5.1.8.5.2 ubcore_mig_resp_status + +```c +enum ubcore_mig_resp_status { + UBCORE_MIG_MSG_PROC_SUCCESS, + UBCORE_MIG_MSG_PROC_FAILURE, + UBCORE_VTP_MIG_COMPLETE, + UBCORE_VTP_MIG_UNCOMPLETE +}; +``` + +##### 5.1.8.5.3 ubcore_function_mig_resp + +```c +struct ubcore_function_mig_resp { + uint16_t mig_fe_idx; + enum ubcore_mig_resp_status status; +}; +``` + +### 5.1.9 设备属性查询ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*query_device_attr)(struct ubcore_device *dev, struct ubcore_device_attr *attr); + +3. 描述 + +ubcore在UB设备注册后，调用该接口查询并保存设备的属性。 + +4. 参数 + +@param[in] dev: the ub device handle; + +@param[out] attr: attributes for the driver to fill in + +5. 返回值 + +0 on success, other value on error + +### 5.1.10 设备状态查询ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*query_device_status)(struct ubcore_device *dev, struct ubcore_device_status *status); + +3. 描述 + +query_device_status为一个函数指针，是ubcore_ops_t结构体成员，在ubcore_register_device接口调用时由厂商驱动向UB协议栈注册；用户接口层面，UB协议栈向用户提供查询相应的查询API接口，UB协议栈调用该接口，从驱动得到设备状态后直接返回给应用。 + +4. 参数 + +@param[in] dev: the ub device handle; + +@param[out] status: status for the driver to fill in + +5. 返回值 + +0 on success, other value on error + +### 5.1.11 设备属性配置ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*config_device)(struct ubcore_device *dev, struct ubcore_device_cfg *cfg); + +3. 描述 + +管理员或者云管理系统通过uvs，调用该接口配置设备属性。 + +4. 参数 + +@param[in] dev: the ub device handle; + +@param[in] cfg: device configuration + +5. 返回值 + +0 on success, other value on error + +#### 5.1.11.1 ubcore_device_cfg + +```c +struct ubcore_device_cfg { + uint16_t ue_idx; */* ue id or mue id. e.g: bdf id */* + union ubcore_device_cfg_maskmask; + struct ubcore_rc_cfg rc_cfg; + uint32_t slice; */* TA slice size byte */* + uint8_t pattern; */* 0: pattern1; 1: pattern3 */* + bool virtualization; + uint32_t suspend_period; */* us */* + uint32_t suspend_cnt; */* TP resend cnt */* + uint32_t min_jetty_cnt; + uint32_t max_jetty_cnt; + uint32_t min_jfr_cnt; + uint32_t max_jfr_cnt; + uint32_t reserved_jetty_id_min; + uint32_t reserved_jetty_id_max; +}; +``` + +#### 5.1.11.2 ubcore_device_cfg_mask + +```c +union ubcore_device_cfg_mask { + struct { + uint32_t rc_cnt : 1; + uint32_t rc_depth : 1; + uint32_t slice : 1; + uint32_t pattern : 1; + uint32_t virtualization : 1; + uint32_t suspend_period : 1; + uint32_t suspend_cnt : 1; + uint32_t min_jetty_cnt : 1; + uint32_t max_jetty_cnt : 1; + uint32_t min_jfr_cnt : 1; + uint32_t max_jfr_cnt : 1; + uint32_t reserved_jetty_id_min : 1; + uint32_t reserved_jetty_id_max : 1; + uint32_t reserved : 19; + } bs; + uint32_t value; +}; +``` + +#### 5.1.11.3 ubcore_rc_cfg + +```c +struct ubcore_rc_cfg { + uint32_t rc_cnt; /* rc queue count */ + uint32_t depth; +}; +``` + +#### 5.1.11.4 ubcore_pattern + +```c +enum ubcore_pattern { + UBCORE_PATTERN_1 = 0, + UBCORE_PATTERN_3 +}; +``` + +### 5.1.12 设备绑定ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*bond_add)(struct net_device *bond, struct net_device *slave, struct netdev_lag_upper_info *upper_info); + +3. 描述 + +bond slave net devic + +4. 参数 + +@param[in] bond: bond netdev; + +@param[in] slave: slave netdev; + +@param[in] upper_info: change upper event info; + +5. 返回值 + +0 on success, other value on error + +### 5.1.13 设备解除绑定ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*bond_remove)(struct net_device *bond, struct net_device *slave); + +3. 描述 + +unbond slave net device + +4. 参数 + +@param[in] bond: bond netdev; + +@param[in] slave: slave netdev; + +5. 返回值 + +0 on success, other value on error + +### 5.1.14 设备添加端口ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*add_port)(struct ubcore_device *dev, uint32_t port_cnt, uint32_t *port_list); + +3. 描述 + +ubcore接收到添加端口的命令后，调用udma驱动对bond设备添加端口。Udma驱动将端口信息添加到bonding group table中。 + +4. 参数 + +@param[in] dev: the ub device handle; + +@param[in] port_cnt: port count + +@param[in] port_list: port list + +5. 返回值 + +0 on success, other value on error + +### 5.1.15 配置端口和netdev映射 + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +int ubcore_set_port_netdev(struct ubcore_device *dev, struct net_device *ndev, + +unsigned int port_id); + +3. 描述 + +配置ubcore_device的物理端口与netdev设备的绑定关系； + +4. 参数 + +@param[in] dev: the ubcore device; + +@param[in] ndev: The netdev corresponding to the initial port + +@param[in] port_id: The physical port_id is the same as the port_id presented in the sysfs file, and port_id is configured in TP during link establishment. + +5. 返回值 + +0 on success, other value on error + +### 5.1.16 解除端口和netdev映射 + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +int ubcore_unset_port_netdev(struct ubcore_device *dev, struct net_device *ndev, + +unsigned int port_id); + +3. 描述 + +解除ubcore_device的物理端口与netdev设备的绑定关系； + +4. 参数 + +@param[in] dev: the ubcore device; + +@param[in] ndev: The netdev corresponding to the initial port + +@param[in] port_id: The physical port_id is the same as the port_id presented in the sysfs file, and port_id is configured in TP during link establishment. + +5. 返回值 + +0 on success, other value on error + +## 5.2 内核态ID配置、地址配置ops接口 + +本节描述ubcore和UDMA驱动之间EID及网络地址配置接口。 + +### 5.2.1 配置网络地址ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*add_net_addr)(struct ubcore_device *dev, struct ubcore_net_addr *net_addr, + +uint32_t index); + +3. 描述 + +UB Core调用UDMA驱动设置网络地址等信息。 + +4. 参数 + +@param[in] dev: the ub device handle; + +@param[in] net_addr: net_addr to set + +@param[in] index: index by sip table + +5. 返回值 + +0 on success, other value on error + +#### 5.2.1.1 ubcore_net_addr + +```c +struct ubcore_net_addr { + enum ubcore_net_addr_type type; + union ubcore_net_addr_union net_addr; + uint64_t vlan; */* available for UBOE */* + uint8_t mac[UBCORE_MAC_BYTES]; */* available for UBOE */* + uint32_t prefix_len; +}; +``` + +#### 5.2.1.2 ubcore_net_addr_type + +```c +enum ubcore_net_addr_type { + UBCORE_NET_ADDR_TYPE_IPV4 = 0, + UBCORE_NET_ADDR_TYPE_IPV6 +}; +``` + +### 5.2.2 删除网络地址ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*delete_net_addr)(struct ubcore_device *dev, uint32_t idx); + +3. 描述 + +UB Core调用UDMA驱动删除网络地址等信息。 + +4. 参数 + +@param[in] dev: the ub device handle; + +@param[in] idx: net_addr idx by sip table entry + +5. 返回值 + +0 on success, other value on error + +### 5.2.3 配置UEID ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*add_ueid)(struct ubcore_device *dev, uint16_t ue_idx, struct ubcore_ueid_cfg *cfg); + +3. 描述 + +UB Core统调用UDMA驱动配置function的eid和upi。驱动在此接口实现以下功能： + +（1）添加seid表，由于eid idx是前端带到后端的，网卡需要支持根据指定的idx添加eid，需要在add ueid的接口中增加入参idx；eid idx是VF内编址的。 + +（2）配置UPI + EID -\> FE_IDX的映射表，主要用于收包方向查找VF + +4. 参数 + +@param[in] dev: the ubcore_device handle; + +@param[in] ue_idx: ue_idx; + +@param[in] cfg: eid and the upi of fe to which the eid belongs can be specified; + +5. 返回值 + +the index of eid/upi, less than 0 indicating error + +![](figures/urma_caution.png) + +UB Core统一使用add_ueid接口配置eid和upi，包括虚拟化场景和非虚拟化场景，pattern1和pattern3。 + +（1）虚拟化场景只能通过TPF配置VF的EID + +（2）非虚拟化配置pattern3 EID，使用function本身对应的dev指针配置，fe_idx写入0xFFFF，upi填有效值 + +（3）非虚拟化配置pattern1 EID，使用function本身对应的dev指针配置，fe_idx写入0xFFFF，upi填无效值（0） + +#### 5.2.3.1 ubcore_ueid_cfg + +```c +struct ubcore_ueid_cfg { + union ubcore_eid eid; + uint32_t upi; + uint32_t eid_index; + guid_t guid; +}; +``` + +### 5.2.4 删除UEID ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*delete_ueid)(struct ubcore_device *dev, uint16_t ue_idx, struct ubcore_ueid_cfg *cfg); + +3. 描述 + +UB Core调用UDMA驱动删除VF或者PF的EID和UPI值。 + +4. 参数 + +@param[in] dev: the ubcore_device handle; + +@param[in] ue_idx: ue_idx; + +@param[in] cfg: eid and the upi of fe to which the eid belongs can be specified; + +5. 返回值 + +0 on success, other value on error + +### 5.2.5 配置Funtion热迁移状态ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*config_function_migrate_state)( + +struct ubcore_device *dev, uint16_t ue_idx, uint32_t cnt, + +struct ubcore_ueid_cfg *cfg, enum ubcore_mig_state state); + +3. 描述 + +迁移时，迁移源端虚机被销毁时，由于第三方节点没有完成vtp到tp的切换，因此，需要额外配置表项标示该虚机正在迁移，当迁移源端收到该虚机的消息时，不会因为虚机被销毁而丢消息。同理，发生回滚时，迁移目的端需要设置为回滚状态，迁移源端需要将迁移状态恢复为普通状态。 + +4. 参数 + +@param[in] dev: the ub device handle; + +@param[in] ue_idx: ue id; + +@param[in] cnt: config count; + +@param[in] cfg: eid and the upi of fe to which the eid belongs can be specified; + +@param[in] state: config state (start, rollback and finish) + +5. 返回值 + +config success count, -1 on error + +#### 5.2.5.1 ubcore_mig_state + +```c +enum ubcore_mig_state { + UBCORE_MIG_STATE_START, + UBCORE_MIG_STATE_ROLLBACK, + UBCORE_MIG_STATE_FINISH +}; +``` + +## 5.3 内核态context管理ops接口 + +本节描述UBCore和UB内核态驱动之间URMA context生命周期接口。 + +### 5.3.1 context创建ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +struct ubcore_ucontext *(*alloc_ucontext)(struct ubcore_device *dev, + +uint32_t eid_index, struct ubcore_udrv_priv *udrv_data); + +3. 描述 + +UB Core调用UDMA驱动创建URMA context。 + +4. 参数 + +@param[in] dev: the ub device handle; + +@param[in] eid: function entity id (eid) index to set; + +@param[in] udrv_data: user space driver data + +5. 返回值 + +pointer to user context on success, null or error, + +#### 5.3.1.1 ubcore_ucontext + +```c +struct ubcore_ucontext { + struct ubcore_device *ub_dev; + union ubcore_eid eid; + uint32_t eid_index; + void *jfae; */* jfae uobj */* + struct ubcore_cg_object cg_obj; + atomic_t use_cnt; +}; +``` + +#### 5.3.1.2 ubcore_udrv_priv + +```c +struct ubcore_udrv_priv { + uint64_t in_addr; + uint32_t in_len; + uint64_t out_addr; + uint32_t out_len; +}; +``` + +#### 5.3.1.3 ubcore_udata + +```c +struct ubcore_udata { + struct ubcore_ucontext *uctx; + struct ubcore_udrv_priv *udrv_data; +}; +``` + +### 5.3.2 context销毁ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*free_ucontext)(struct ubcore_ucontext *uctx); + +3. 描述 + +UB Core调用UDMA驱动销毁URMA Context。 + +4. 参数 + +@param[in] uctx: the user context created before; + +5. 返回值 + +0 on success, other value on error + +## 5.4 内核态mmap接口 + +### 5.4.1 mmap ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*mmap)(struct ubcore_ucontext *ctx, struct vm_area_struct *vma); + +3. 描述 + +UB Core调用UDMA驱动映射jetty的doorbell等物理空间。 + +4. 参数 + +@param[in] uctx: the user context created before; + +@param[in] vma: linux vma including vm_start, vm_pgoff, etc; + +5. 返回值 + +0 on success, other value on error + +## 5.5 内核态资源管理ops接口 + +本节描述UBCore和UB内核态驱动之间的控制面接口。主要包括Jetty资源的生命周期和Segment生命周期（包括安全token）的接口。UBCore负责连接关系协商和路由通信关系建立，驱动负责硬件强相关的真正的连接资源管理。 + +### 5.5.1 TP协商和配置管理 + +TP是两个node/multi-path dev之间的物理连接群或者Jetty之间逻辑连接。由transport type来指定。 + +![](figures/urma-api-kernel-tp-01.png) + +#### 5.5.1.1 TPG创建ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +struct ubcore_tpg *(*create_tpg)(struct ubcore_device *dev, struct ubcore_tpg_cfg *cfg, struct ubcore_udata *udata); + +3. 描述 + +创建TP Group，支持RC或者RM类型 + +4. 参数 + +@param[in] dev: the ub device handle; + +DPU智能网卡上，dev是TPF的设备指针，或者用于建链的设备指针；鲲鹏950上dev可能为PF。 + +@param[in] cfg: tpg init attributes + +@param[in] udata: ucontext and user space driver data + +5. 返回值 + +tp pointer on success, NULL on error + +##### 5.5.1.1.1 ubcore_tpg_cfg + +```c +struct ubcore_tpg_cfg { + */* transaction layer attributes */* + union ubcore_eidlocal_eid; + union ubcore_eidpeer_eid; + */* tranport layer attributes */* + enum ubcore_transport_modetrans_mode; + uint8_t dscp; + enum ubcore_tp_cc_alg cc_alg; + uint8_t cc_pattern_idx; + uint32_t tp_cnt; + struct ubcore_net_addrlocal_net_addr; +}; +``` + +##### 5.5.1.1.2 ubcore_tpg_ext + +```c +struct ubcore_tpg_ext { + uint64_t addr; + uint32_t len; +}; +``` + +##### 5.5.1.1.3 ubcore_tpg + +```c +struct ubcore_tpg { + uint32_t tpgn; + struct ubcore_device *ub_dev; + struct ubcore_tpg_cfgtpg_cfg; */* filled by ubcore when creating tp */* + struct ubcore_tpg_exttpg_ext; */* filled by ubn driver when creating tp */* + struct ubcore_tpg_ext peer_ext; */* filled by ubcore before modifying tp */* + struct ubcore_tp * + tp_list[UBCORE_MAX_TP_CNT_IN_GRP]; *// UBCORE_MAX_TP_CNT_IN_GRP=32* + struct hlist_node hnode; */* driver inaccessible */* + struct kref ref_cnt; + struct mutex mutex; + uint32_t ue_idx; + uint32_t peer_tpgn; *// Only for tpg audit with peer, driver inaccessible* +}; +``` + +#### 5.5.1.2 TPG销毁ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*destroy_tpg)(struct ubcore_tpg *tpg); + +3. 描述 + +destroy_tpg为一个函数指针，是ubcore_ops_t结构体成员，在ubcore_register_device接口调用时由厂商驱动向UB协议栈注册 + +4. 参数 + +@param[in] tp: tp pointer created before + +5. 返回值 + +0 on success, other value on error + +#### 5.5.1.3 TP创建和使用流程 + +![](figures/urma-api-kernel-tp-02.png) + +1、客户端和服务端创建进程上下文、Jetty队列和Segment资源； + +2、两端应用先进行EID/VA/TOKEN等必要的双边通信信息参数协商。 + +3、客户端随后调用urma_import_jfr/urma_import_jetty函数；这两个函数的入参均包含远端地址信息，它们都会隐式调用create_tpg和create_tp接口创建TPG和TP（相同的远端地址信息，只会在第一次创建时生成TPG和TP的上下文，后续返回已创建的TPG/TP Index并增加TPG/TP引用计数即可），将TPG/TP index返回用户态驱动和lib； + +4、应用通过Post Send接口下发WQE，用户态驱动需要在WQE中填入tpgn，芯片根据tpgn选路。 + +#### 5.5.1.4 TP参数协商 + +TP参数协商可以采用公知Jetty或Socket方式进行TP参数协商，协商参数参考TP属性域段表。 + +![](figures/urma-api-kernel-tp-03.png) + +| TP属性域段 | 域段描述 | +| --- | --- | +| local flag | 包含：本地节点拥塞控制算法 | +| remote flag | 包含：远端拥塞控制算法 | +| local_net_addr | 本地节点network address | +| peer_net_addr | 远端节点network address | +| local_eid | 本地节点EID, TA连接时使用 | +| peer_eid | 远端节点EID, TA连接时使用 | +| trans_mode | 传输服务类型，分三种：URMA_TM_RM，URMA_TM_RC，URMA_TM_UM | +| State | TP状态，分四种：Reset, RTR, RTS, Error | +| rx_psn | 期待接收的PSN | +| tx_psn | 发送方向初始PSN | +| local MTU | 本地节点maximum transfer unit，最大传输单元 | +| remote MTU | 远端maximum transfer unit，最大传输单元 | +| local tpn | 本地节点TP编号 | +| remote tpn | 远端TP编号 | +| tp cnt | tp数量（需要协商，但不放到tp context中） | +| slice | 报文分片大小 | + +#### 5.5.1.5 TP创建ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +struct ubcore_tp *(*create_tp)(struct ubcore_device *dev, struct ubcore_tp_cfg *cfg, struct ubcore_udata *udata); + +3. 描述 + +若指定tpg，则从tpg中创建一个tp；否则，直接创建tp。 + +对于鲲鹏950、DPU智能网卡等UB设备，创建RC模式和RM模式的tp是，tpg必须为非NULL；但鲲鹏920F ，tpg可以为NULL。 + +4. 参数 + +@param[in] dev: the ub device handle; + +@param[in] cfg: tp init attributes + +@param[in] udata: ucontext and user space driver data + +5. 返回值 + +返回TP对象。成功，返回ubcore_tp_t的结构体地址；失败，返回NULL。 + +##### 5.5.1.5.1 ubcore_tp_cfg + +```c +struct ubcore_tp_cfg { + union ubcore_tp_cfg_flag flag; /* flag of initial tp */ + /* transaction layer attributes */ + union { + union ubcore_eid local_eid; + struct ubcore_jetty_idlocal_jetty; + }; + uint16_t ue_idx; /* rc mode only */ + union { + union ubcore_tp_cfg peer_eid; + struct ubcore_jetty_id peer_jetty; + }; + /* tranport layer attributes */ + enum ubcore_transport_modetrans_mode; + uint8_t retry_num; + uint8_t retry_factor; /* for calculate the time slot to retry */ + uint8_t ack_timeout; + uint8_t dscp; /* priority */ + uint32_t oor_cnt; /* OOR window size: by packet */ + struct ubcore_tpg *tpg; /* NULL if no tpg, eg.UM mode */ +}; +``` + +##### 5.5.1.5.2 ubcore_tp_cfg_flag + +```c +union ubcore_tp_cfg_flag { + struct { + uint32_t target : 1; /* 0: initiator, 1: target */ + uint32_t loopback : 1; + uint32_t ack_resp : 1; + uint32_t dca_enable : 1; + /* for the bonding case, the hardware selects the port + * ignoring the port of the tp context and + * selects the port based on the hash value + * along with the information in the bonding group table. + */ + uint32_t bonding : 1; + uint32_t reserved : 27; + } bs; + uint32_t value; +}; +``` + +##### 5.5.1.5.3 ubcore_transport_mode + +```c +enum ubcore_transport_mode { + UBCORE_TP_RM = 0x1, /* Reliable message */ + UBCORE_TP_RC = 0x1 << 1, /* Reliable connection */ + UBCORE_TP_UM = 0x1 << 2 /* Unreliable message */ +}; +``` + +##### 5.5.1.5.4 ubcore_tp_state + +```c +enum ubcore_tp_state { + UBCORE_TP_STATE_RESET = 0, + UBCORE_TP_STATE_RTR, + UBCORE_TP_STATE_RTS, + UBCORE_TP_STATE_SUSPENDED, + UBCORE_TP_STATE_ERR +}; +``` + +##### 5.5.1.5.5 ubcore_tp_ext + +```c +struct ubcore_tp_ext { + uint64_t addr; + uint32_t len; +}; +``` + +##### 5.5.1.5.6 ubcore_tp_flag + +```c +union ubcore_tp_flag { + struct { + uint32_t target : 1; */* 0: initiator, 1: target */* + uint32_t oor_en : 1; */* out of order receive, 0: disable 1: enable */* + uint32_t sr_en : 1; */* selective retransmission, 0: disable 1: enable */* + uint32_t cc_en : 1; */* congestion control algorithm, 0: disable 1: enable */* + uint32_t cc_alg : 4; */* The value is ubcore_tp_cc_alg_t */* + uint32_t spray_en : 1; */* spray with src udp port, 0: disable 1: enable */* + uint32_t loopback : 1; + uint32_t ack_resp : 1; + uint32_t dca_enable : 1; */* dynamic connection, 0: disable 1: enable */* + uint32_t bonding : 1; + uint32_t clan : 1; + uint32_t reserved : 18; + } bs; + uint32_t value; +}; +``` + +##### 5.5.1.5.7 ubcore_tp_cc_alg + +```c +enum ubcore_tp_cc_alg { + UBCORE_TP_CC_NONE = 0, + UBCORE_TP_CC_DCQCN, + UBCORE_TP_CC_DCQCN_AND_NETWORK_CC, + UBCORE_TP_CC_LDCP, + UBCORE_TP_CC_LDCP_AND_CAQM, + UBCORE_TP_CC_LDCP_AND_OPEN_CC, + UBCORE_TP_CC_HC3, + UBCORE_TP_CC_DIP, + UBCORE_TP_CC_ACC, + UBCORE_TP_CC_NUM +}; +``` + +##### 5.5.1.5.8 ubcore_tp + +```c +struct ubcore_tp { + uint32_t tpn; */* driver assigned in creating tp */* + uint32_t peer_tpn; + struct ubcore_device *ub_dev; + union ubcore_tp_flag flag; */* indicate initiator or target, etc */* + uint32_t local_net_addr_idx; + struct ubcore_net_addr peer_net_addr; + */* only for RC START */* + union { + union ubcore_eid local_eid; + struct ubcore_jetty_id local_jetty; + }; + union { + union ubcore_eid peer_eid; + struct ubcore_jetty_id peer_jetty; + }; + */* only for RC END */* + enum ubcore_transport_mode trans_mode; + enum ubcore_tp_state state; + uint32_t rx_psn; + uint32_t tx_psn; + enum ubcore_mtu mtu; + uint16_t data_udp_start; */* src udp port start, for multipath data */* + uint16_t ack_udp_start; */* src udp port start, for multipath ack */* + uint8_t udp_range; */* src udp port range, for both multipath data and ack */* + uint8_t port_id; */* optional, physical port, only for non-bonding */* + uint8_t retry_num; + uint8_t retry_factor; + uint8_t ack_timeout; + uint8_t dscp; + uint8_t cc_pattern_idx; + uint8_t hop_limit; + struct ubcore_tpg *tpg; */* NULL if no tpg, eg. UM mode */* + uint32_t oor_cnt; */* out of order window size for recv: packet cnt */* + uint32_t oos_cnt; */* out of order window size for send: packet cnt */* + struct ubcore_tp_ext tp_ext; */* driver fill in creating tp */* + struct ubcore_tp_ext peer_ext; */* ubcore fill before modifying tp */* + atomic_t use_cnt; + struct hlist_node hnode; */* driver inaccessible */* + struct kref ref_cnt; + struct completion comp; + uint32_t flow_label; + uint8_t mn; */* 0\~15, a packet contains only one msg if mn is set as 0 */* + enum ubcore_transport_type + peer_trans_type; */* Only for user tp connection */* + struct mutex lock; */* protect TP state */* + void *priv; */* ubcore private data for tp management */* + uint32_t ue_idx; +}; +``` + +#### 5.5.1.6 TP修改ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*modify_tp)(struct ubcore_tp *tp, struct ubcore_tp_attr *attr, union ubcore_tp_attr_mask mask); + +3. 描述 + +修改TP的属性、状态。UDMA驱动根据参数修改TP context，并通过消息通道通知芯片修改TP属性和状态。驱动返回modify成功，ubcore根据attr和mask值，修改入参tp中的信息。 + +4. 参数 + +@param[in] tp: tp pointer created before + +@param[in] attr: tp attributes + +@param[in] mask: attr mask indicating the attributes to be modified + +5. 返回值 + +0 on success, other value on error + +##### 5.5.1.6.1 ubcore_tp_attr + +```c +struct ubcore_tp_attr { + union ubcore_tp_mod_flag flag; + uint32_t peer_tpn; + enum ubcore_tp_state state; + uint32_t tx_psn; + uint32_t rx_psn; + enum ubcore_mtu mtu; + uint8_t cc_pattern_idx; + struct ubcore_tp_ext peer_ext; + uint32_t oos_cnt; */* out of standing packet cnt */* + uint32_t local_net_addr_idx; + struct ubcore_net_addr peer_net_addr; + uint16_t data_udp_start; + uint16_t ack_udp_start; + uint8_t udp_range; + uint8_t hop_limit; + uint32_t flow_label; + uint8_t port_id; + uint8_t mn; */* 0\~15, a packet contains only one msg if mn is set as 0 */* + enum ubcore_transport_type + peer_trans_type; */* Only for user tp connection */* +}; +``` + +##### 5.5.1.6.2 ubcore_tp_attr_mask + +```c +union ubcore_tp_attr_mask { + struct { + uint32_t flag : 1; + uint32_t peer_tpn : 1; + uint32_t state : 1; + uint32_t tx_psn : 1; + uint32_t rx_psn : 1; */* modify both rx psn and tx psn when restore tp */* + uint32_t mtu : 1; + uint32_t cc_pattern_idx : 1; + uint32_t peer_ext : 1; + uint32_t oos_cnt : 1; + uint32_t local_net_addr_idx : 1; + uint32_t peer_net_addr : 1; + uint32_t data_udp_start : 1; + uint32_t ack_udp_start : 1; + uint32_t udp_range : 1; + uint32_t hop_limit : 1; + uint32_t flow_label : 1; + uint32_t port_id : 1; + uint32_t mn : 1; + uint32_t peer_trans_type : 1; */* user tp only */* + uint32_t reserved : 13; + } bs; + uint32_t value; +}; +``` + +##### 5.5.1.6.3 ubcore_tp_mod_flag + +```c +union ubcore_tp_mod_flag { + struct { + uint32_t oor_en : 1; */* out of order receive, 0: disable 1: enable */* + uint32_t sr_en : 1; */* selective retransmission, 0: disable 1: enable */* + uint32_t cc_en : 1; */* congestion control algorithm, 0: disable 1: enable */* + uint32_t cc_alg : 4; */* The value is ubcore_tp_cc_alg_t */* + uint32_t spray_en : 1; */* spray with src udp port, 0: disable 1: enable */* + uint32_t clan : 1; */* clan domain, 0: disable 1: enable */* + uint32_t reserved : 23; + } bs; + uint32_t value; +}; +``` + +#### 5.5.1.7 TP销毁ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*destroy_tp)(struct ubcore_tp *tp); + +3. 描述 + +destroy_tp为一个函数指针，是ubcore_ops_t结构体成员，在ubcore_register_device接口调用时由厂商驱动向UB协议栈注册。 + +4. 参数 + +@param[in] tp: tp pointer created before + +5. 返回值 + +0 on success, other value on error + +#### 5.5.1.8 多TP创建ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*create_multi_tp)(struct ubcore_device *dev, uint32_t cnt, struct ubcore_tp_cfg *cfg, + +struct ubcore_udata *udata, struct ubcore_tp **tp); + +3. 描述 + +从指定tpg中或者直接创建多个tp，新建的tp可以属于相同或者不同的tpg。UDMA第一次创建tp失败时即返回 + +4. 参数 + +@param[in] dev: the ub device handle; + +@param[in] cnt: the number of tp, must be less than or equal to 32; + +@param[in] cfg: array of tp init attributes + +@param[in] udata: array of ucontext and user space driver data + +@param[out] tp: pointer array of tp + +5. 返回值 + +created tp cnt, 0 on error + +#### 5.5.1.9 多TP销毁ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*destroy_multi_tp)(uint32_t cnt, struct ubcore_tp **tp); + +3. 描述 + +销毁多个tp，UDMA第一次销毁tp失败时即返回。 + +4. 参数 + +@param[in] cnt: the number of tp; + +@param[in] tp: pointer array of tp created before + +5. 返回值 + +destroyed tp cnt, 0 on error + +#### 5.5.1.10 多TP修改ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*modify_multi_tp)(uint32_t cnt, struct ubcore_tp **tp, struct ubcore_tp_attr *attr, + +union ubcore_tp_attr_mask *mask, struct ubcore_tp **fail_tp); + +3. 描述 + +Modify多个tp的属性。UDMA第一次修改失败即返回 + +4. 参数 + +@param[in] cnt: the number of tp; + +@param[in] tp: pointer array of tp created before + +@param[in] attr: array of tp attributes + +@param[in] mask: array of attr mask indicating the attributes to be modified + +@param[in] fail_tp: pointer of tp failed to modify + +5. 返回值 + +modified successfully tp cnt, 0 on error + +#### 5.5.1.11 查询拥塞控制算法模板ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +struct ubcore_cc_entry *(*query_cc)(struct ubcore_device *dev, uint32_t *cc_entry_cnt); + +3. 描述 + +查询拥塞控制模板，例如： + +![](figures/urma-api-kernel-tp-04.png) + +4. 参数 + +@param[in] dev: the ub device handle; + +@param[in] cc_entry_cnt: cc entry cnt; + +5. 返回值 + +return NULL on fail, otherwise, return cc entry array + +##### 5.5.1.11.1 ubcore_cc_entry + +```c +struct ubcore_cc_entry { + enum ubcore_tp_cc_alg alg; + uint8_t cc_pattern_idx; + uint8_t cc_priority; +} \_\_packed; +``` + +#### 5.5.1.12 VTPN分配ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +struct ubcore_vtpn *(*alloc_vtpn)(struct ubcore_device *dev); + +3. 描述 + +分配一个空闲的vtpn号。 + +4. 参数 + +@param[in] dev: the ub device handle; + +5. 返回值 + +vtpn pointer on success, NULL on error + +##### 5.5.1.12.1 ubcore_vtpn + +```c +struct ubcore_vtpn { + uint32_t vtpn; */* driver fills */* + struct ubcore_device *ub_dev; + */* ubcore private, inaccessible to driver */* + enum ubcore_transport_mode trans_mode; + */* vtpn key start */* + union ubcore_eid local_eid; + union ubcore_eid peer_eid; + uint32_t local_jetty; + uint32_t peer_jetty; + */* vtpn key end */* + uint32_t eid_index; + struct mutex state_lock; + enum ubcore_vtp_state state; */* protect by state_lock */* + struct hlist_node hnode; */* key: eid + jetty */* + struct hlist_node vtpn_hnode; */* key: vtpn */* + atomic_t use_cnt; + struct kref ref_cnt; + struct completion comp; + struct list_head node; */* vtpn node in vtpn_wait_list */* + struct list_head list; */* vtpn head to restore tjetty/jetty/cb node */* + struct list_head + disconnect_list; */* vtpn head to restore disconnect vtpn node */* + uint64_t tp_handle; + uint64_t peer_tp_handle; + uint64_t tag; + bool uspace; */* true: user space; false: kernel space */* +}; +``` + +#### 5.5.1.13 VTPN释放ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*free_vtpn)(struct ubcore_vtpn *vtpn); + +3. 描述 + +@param[in] vtpn: vtpn pointer allocated before + +4. 参数 + +@param[in] [Required] rm_client: 待注销的内核态应用客户端. + +5. 返回值 + +0 on success, other value on error + +#### 5.5.1.14 VTP创建ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +struct ubcore_vtp *(*create_vtp)(struct ubcore_device *dev, + +struct ubcore_vtp_cfg *cfg, struct ubcore_udata *udata); + +3. 描述 + +将VTP映射到TPG、UTP或Clan domain TP。 + +4. 参数 + +@param[in] dev: the ub device handle; + +@param[in] cfg: vtp init attributes + +@param[in] udata: ucontext and user space driver data + +5. 返回值 + +vtp pointer on success, NULL on error + +##### 5.5.1.14.1 ubcore_vtp_cfg + +```c +struct ubcore_vtp_cfg { + uint16_t fe_idx; // vfid or pfid + uint32_t vtpn; + uint32_t local_jetty; + /* key start */ + union ubcore_eid local_eid; + union ubcore_eid peer_eid; + uint32_t peer_jetty; + /* key end */ + union ubcore_vtp_cfg_flag flag; + enum ubcore_transport_mode trans_mode; + union { + struct ubcore_tpg *tpg; + struct ubcore_tp *tp; + struct ubcore_utp *utp; // idx of dip + struct ubcore_ctp *ctp; /* valid when clan is true */ + }; +}; +``` + +##### 5.5.1.14.2 ubcore_vtp_cfg_flag + +```c +union ubcore_vtp_cfg_flag { + struct { + uint32_t clan_tp : 1; + uint32_t migrate : 1; + uint32_t reserve : 30; + } bs; + uint32_t value; +}; +``` + +##### 5.5.1.14.3 ubcore_vtp + +```c +struct ubcore_vtp { + struct ubcore_device *ub_dev; + struct ubcore_vtp_cfg cfg; */* driver fills */* + struct hlist_node hnode; */* driver inaccessible */* + uint32_t role; */* current side is initiator, target or duplex */* + struct kref ref_cnt; + uint32_t eid_idx; + uint32_t upi; + bool share_mode; +}; +``` + +#### 5.5.1.15 VTP销毁ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*destroy_vtp)(struct ubcore_vtp *vtp); + +3. 描述 + +销毁指定的VTP，解除VTP到TPG、UTP或Clan domain TP的映射关系。 + +4. 参数 + +@param[in] vtp: vtp pointer created before + +5. 返回值 + +0 on success, other value on error + +#### 5.5.1.16 VTP修改ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*modify_vtp)(struct ubcore_vtp *vtp, struct ubcore_vtp_attr *attr, + +union ubcore_vtp_attr_mask *mask); + +3. 描述 + +UB数据面发送报文的时候，根据wqe中携带的vtpn找到vtp表项中记录的TP进行发包；迁移后，需要将vtp表中记录的TP切换为到迁移目的端的TP，如果依赖云管理面刷新位置表触发TP的切换，当第三方节点规模大的时候，表项刷新时间长，会导致停流时间变长。因此，当迁移源端停流以后，收到来自第三方节点的数据面消息，此时在回复的响应消息中携带迁移状态，第三方节点的网卡上报异步事件通知UVS修改vtp表中的tp； + +4. 参数 + +@param[in] vtp: vtp pointer to be modified; + +@param[in] attr: vtp attr, tp that we want to change; + +@param[in] mask: attr mask; + +5. 返回值 + +0 on success, other value on error + +##### 5.5.1.16.1 ubcore_vtp_attr + +```c +struct ubcore_vtp_attr { + union { + struct ubcore_tpg *tpg; + struct ubcore_tp *tp; + struct ubcore_utp *utp; // idx of dip + struct ubcore_ctp *ctp; /* clan domain */ + } tp; +}; +``` + +##### 5.5.1.16.2 ubcore_vtp_attr_mask + +```c +union ubcore_vtp_attr_mask { + struct { + uint32_t tp : 1; + uint32_t reserved : 31; + } bs; + uint32_t value; +}; +``` + +#### 5.5.1.17 UTP创建ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +struct ubcore_utp *(*create_utp)(struct ubcore_device *dev, + +struct ubcore_utp_cfg *cfg, struct ubcore_udata *udata); + +3. 描述 + +创建UM类型TP，即UTP。UDMA驱动添加UTP表项，配置SIP index，DIP等信息 + +4. 参数 + +@param[in] dev: the ub device handle; + +@param[in] cfg: utp init attributes + +@param[in] udata: ucontext and user space driver data + +5. 返回值 + +utp pointer on success, NULL on error + +##### 5.5.1.17.1 ubcore_utp_cfg + +```c +struct ubcore_utp_cfg { + /* transaction layer attributes */ + union ubcore_utp_cfg_flag flag; + uint16_t udp_start; // src udp port start + uint8_t udp_range; // src udp port range + uint32_t local_net_addr_idx; + struct ubcore_net_addr peer_net_addr; + uint32_t flow_label; + uint8_t dscp; + uint8_t hop_limit; + uint32_t port_id; + enum ubcore_mtu mtu; +}; +``` + +##### 5.5.1.17.2 ubcore_utp_cfg_flag + +```c +union ubcore_utp_cfg_flag { + struct { + uint32_t loopback : 1; + uint32_t spray_en : 1; + uint32_t clan : 1; + uint32_t reserved : 29; + } bs; + uint32_t value; +}; +``` + +##### 5.5.1.17.3 ubcore_utp + +```c +struct ubcore_utp { + uint32_t utpn; */* driver fills */* + struct ubcore_device *ub_dev; + struct ubcore_utp_cfg utp_cfg; */* filled by ubcore when createing utp. */* + struct hlist_node hnode; + struct kref ref_cnt; + uint32_t ue_idx; +}; +``` + +#### 5.5.1.18 UTP销毁ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*destroy_utp)(struct ubcore_utp *utp); + +3. 描述 + +销毁UTP，UDMA驱动删除UTP表项。 + +4. 参数 + +@param[in] utp: utp pointer created before + +5. 返回值 + +0 on success, other value on error + +#### 5.5.1.19 CTP创建ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +struct ubcore_ctp *(*create_ctp)(struct ubcore_device *dev, + +struct ubcore_ctp_cfg *cfg, struct ubcore_udata *udata); + +3. 描述 + +创建clan domain简易TP（即CTP） + +4. 参数 + +@param[in] dev: the ub device handle; + +@param[in] cfg: ctp init attributes + +@param[in] udata: ucontext and user space driver data + +5. 返回值 + +ctp pointer on success, NULL on error + +##### 5.5.1.19.1 ubcore_ctp_cfg + +```c +struct ubcore_ctp_cfg { + struct ubcore_net_addr peer_net_addr; + uint32_t cna_len; +}; +``` + +##### 5.5.1.19.2 ubcore_ctp + +```c +struct ubcore_ctp { + uint32_t ctpn; */* driver fills */* + struct ubcore_device *ub_dev; + struct ubcore_ctp_cfg ctp_cfg; */* filled by ubcore when createing cp. */* + atomic_t use_cnt; + struct hlist_node hnode; + struct kref ref_cnt; + struct completion comp; + uint32_t ue_idx; +}; +``` + +#### 5.5.1.20 CTP销毁ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*destroy_ctp)(struct ubcore_ctp *ctp); + +3. 描述 + +销毁clan domain简易TP。 + +4. 参数 + +@param[in] ctp: ctp pointer created before + +5. 返回值 + +0 on success, other value on error + +### 5.5.2 JFC管理接口 + +#### 5.5.2.1 JFC创建ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +struct ubcore_jfc *(*create_jfc)(struct ubcore_device *dev, struct ubcore_jfc_cfg *cfg, struct ubcore_udata *udata); + +3. 描述 + +UB Core调用UDMA驱动创建JFC。 + +4. 参数 + +@param[in] dev: the ub device handle; + +@param[in] cfg: jfc attributes and configurations + +@param[in] udata: ucontext and user space driver data + +5. 返回值 + +jfc pointer on success, NULL on error + +#### 5.5.2.2 JFC修改ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*modify_jfc)(struct ubcore_jfc *jfc, struct ubcore_jfc_attr *attr, struct ubcore_udata *udata); + +3. 描述 + +UB Core调用UDMA驱动修改JFC + +4. 参数 + +@param[in] jfc: the jfc created before; + +@param[in] attr: ubcore jfc attr; + +@param[in] udata: ucontext and user space driver data + +5. 返回值 + +0 on success, other value on error + +#### 5.5.2.3 JFC销毁ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*destroy_jfc)(struct ubcore_jfc *jfc); + +3. 描述 + +注销一个ubcore内核态应用客户端 + +4. 参数 + +@param[in] jfc: the jfc created before; + +5. 返回值 + +@return: 0 on success, other value on error + +#### 5.5.2.4 JFC rearm ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*rearm_jfc)(struct ubcore_jfc *jfc, bool solicited_only); + +3. 描述 + +UB Core调用UDMA驱动使能JFC的中断。 + +4. 参数 + +@param[in] jfc: the jfc created before; + +@param[in] solicited_only: rearm notify by message marked with solicited flag + +指示是否solicited，即接收到的请求带有solicited标志产生的cqe才产生事件。 + +5. 返回值 + +0 on success, other value on error + +### 5.5.3 JFS管理接口 + +#### 5.5.3.1 JFS创建ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +struct ubcore_jfs *(*create_jfs)(struct ubcore_device *dev, struct ubcore_jfs_cfg *cfg, struct ubcore_udata *udata); + +3. 描述 + +UB Core调用UDMA驱动创建JFS。 + +4. 参数 + +@param[in] dev: the ub device handle; + +@param[in] cfg: jfs attributes and configurations + +@param[in] udata: ucontext and user space driver data + +5. 返回值 + +jfs pointer on success, NULL on error + +#### 5.5.3.2 JFS修改ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*modify_jfs)(struct ubcore_jfs *jfs, struct ubcore_jfs_attr *attr, + +struct ubcore_udata *udata); + +3. 描述 + +UB Core调用UDMA驱动修改JFS，支持修改RX WR/WQE的最低水位和状态。 + +4. 参数 + +@param[in] jfs: the jfs created before; + +@param[in] attr: ubcore jfs attr; + +@param[in] udata: ucontext and user space driver data + +5. 返回值 + +0 on success, other value on error + +#### 5.5.3.3 JFS查询ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*query_jfs)(struct ubcore_jfs *jfs, struct ubcore_jfs_cfg *cfg, struct ubcore_jfs_attr *attr); + +3. 描述 + +指定JFS查询JFS配置和属性。支持多线程操作重入操作。 + +4. 参数 + +@param[in] jfs: the jfs created before; + +@param[out] cfg: jfs configurations; + +@param[out] attr: ubcore jfs attributes; + +5. 返回值 + +0 on success, other value on error + +#### 5.5.3.4 JFS flush ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*flush_jfs)(struct ubcore_jfs *jfs, int cr_cnt, struct ubcore_cr *cr); + +3. 描述 + +UDMA驱动把jfs中未被硬件执行的wr，通过cr返回给应用。 + +4. 参数 + +@param[in] jfs: the jfs created before; + +@param[in] cr_cnt: the maximum number of CRs expected to be returned; + +@param[out] cr: the addr of returned CRs; + +5. 返回值 + +the number of CR returned, 0 means no completion record returned, -1 on error + +#### 5.5.3.5 JFS销毁ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*destroy_jfs)(struct ubcore_jfs *jfs); + +3. 描述 + +UB Core调用UDMA驱动销毁JFS。 + +4. 参数 + +@param[in] jfs: the jfs created before; + +5. 返回值 + +0 on success, other value on error + +### 5.5.4 JFR管理接口 + +#### 5.5.4.1 JFR创建ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +struct ubcore_jfr *(*create_jfr)(struct ubcore_device *dev, + +struct ubcore_jfr_cfg *cfg, + +struct ubcore_udata *udata); + +3. 描述 + +UB Core调用UDMA驱动创建JFR。 + +4. 参数 + +@param[in] dev: the ub device handle; + +@param[in] cfg: jfr attributes and configurations + +@param[in] udata: ucontext and user space driver data + +5. 返回值 + +jfr pointer on success, NULL on error + +#### 5.5.4.2 JFR修改ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*modify_jfr)(struct ubcore_jfr *jfr, struct ubcore_jfr_attr *attr, + +struct ubcore_udata *udata); + +3. 描述 + +UB Core调用UDMA驱动修改JFR + +4. 参数 + +@param[in] jfr: the jfr created before; + +@param[in] attr: ubcore jfr attr; + +@param[in] udata: ucontext and user space driver data + +5. 返回值 + +0 on success, other value on error + +#### 5.5.4.3 JFR查询ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*query_jfr)(struct ubcore_jfr *jfr, struct ubcore_jfr_cfg *cfg, + +struct ubcore_jfr_attr *attr); + +3. 描述 + +指定JFR查询JFR配置和属性。支持多线程操作重入操作。 + +4. 参数 + +@param[in] jfr: the jfr created before; + +@param[out] cfg: jfr configurations; + +@param[out] attr: ubcore jfr attributes; + +5. 返回值 + +0 on success, other value on error + +#### 5.5.4.4 JFR销毁ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*destroy_jfr)(struct ubcore_jfr *jfr); + +3. 描述 + +UB Core调用UDMA驱动销毁JFR。 + +4. 参数 + +@param[in] jfr: the jfr created before; + +5. 返回值 + +0 on success, other value on error + +#### 5.5.4.5 JFR导入ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +struct ubcore_tjetty *(*import_jfr)(struct ubcore_device *dev, + +struct ubcore_tjetty_cfg *cfg, + +struct ubcore_udata *udata); + +3. 描述 + +UB Core调用UDMA驱动导入JFR。 + +4. 参数 + +@param[in] dev: the ub device handle; + +@param[in] cfg: remote jfr attributes and import configurations + +@param[in] udata: ucontext and user space driver data + +5. 返回值 + +target jfr pointer on success, NULL on error + +#### 5.5.4.6 JFR反导入ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*unimport_jfr)(struct ubcore_tjetty *tjfr); + +3. 描述 + +UB Core调用UDMA驱动反导入JFR + +4. 参数 + +@param[in] tjfr: the target jfr imported before; + +5. 返回值 + +0 on success, other value on error + +### 5.5.5 Jetty管理接口 + +#### 5.5.5.1 Jetty创建ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +struct ubcore_jetty *(*create_jetty)(struct ubcore_device *dev, + +struct ubcore_jetty_cfg *cfg, + +struct ubcore_udata *udata); + +3. 描述 + +UB Core调用UDMA驱动创建Jetty。 + +4. 参数ubco + +@param[in] dev: the ub device handle; + +@param[in] cfg: jetty attributes and configurations + +@param[in] udata: ucontext and user space driver data + +5. 返回值 + +jetty pointer on success, NULL on error + +#### 5.5.5.2 Jetty修改ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*modify_jetty)(struct ubcore_jetty *jetty, struct ubcore_jetty_attr *attr, + +struct ubcore_udata *udata); + +3. 描述 + +UB Core调用UDMA驱动修改Jetty，支持修改RX WR的最低水位和状态。 + +4. 参数 + +@param[in] jetty: the jetty created before; + +@param[in] attr: ubcore jetty attr; + +@param[in] udata: ucontext and user space driver data + +5. 返回值 + +0 on success, other value on error + +#### 5.5.5.3 Jetty查询ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*query_jetty)(struct ubcore_jetty *jetty, struct ubcore_jetty_cfg *cfg, + +struct ubcore_jetty_attr *attr); + +3. 描述 + +指定Jetty查询Jetty配置和属性。 + +4. 参数 + +@param[in] jetty: the jetty created before; + +@param[out] cfg: jetty configurations; + +@param[out] attr: ubcore jetty attributes; + +5. 返回值 + +0 on success, other value on error + +#### 5.5.5.4 Jetty销毁ops接口 + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int (*destroy_jetty)(struct ubcore_jetty *jetty); + +3. 描述 + +UB Core调用UDMA驱动销毁Jetty。 + +4. 参数 + +@param[in] jetty: the jetty created before; + +5. 返回值 + +0 on success, other value on error + +#### 5.5.5.5 Jetty导入ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +struct ubcore_tjetty *(*import_jetty)(struct ubcore_device *dev, + +struct ubcore_tjetty_cfg *cfg, + +struct ubcore_udata *udata); + +3. 描述 + +UB Core调用UDMA驱动导入Jetty或者Jetty group。 + +4. 参数 + +@param[in] dev: the ub device handle; + +@param[in] cfg: remote jetty attributes and import configurations + +@param[in] udata: ucontext and user space driver data + +5. 返回值 + +target jetty pointer on success, NULL on error + +#### 5.5.5.6 Jetty反导入ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*unimport_jetty)(struct ubcore_tjetty *tjetty); + +3. 描述 + +UB Core调用UDMA驱动反导入Jetty或者Jetty group。 + +4. 参数 + +@param[in] tjetty: the target jetty imported before; + +5. 返回值 + +0 on success, other value on error + +#### 5.5.5.7 Jetty bind ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*bind_jetty)(struct ubcore_jetty *jetty, struct ubcore_tjetty *tjetty, + +struct ubcore_udata *udata); + +3. 描述 + +UB Core调用UDMA驱动导入bind Jetty。 + +4. 参数 + +@param[in] jetty: the jetty created before; + +@param[in] tjetty: the target jetty imported before; + +@param[in] udata: ucontext and user space driver data + +5. 返回值 + +0 on success, other value on error + +#### 5.5.5.8 Jetty unbind ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*unbind_jetty)(struct ubcore_jetty *jetty); + +3. 描述 + +UB Core调用UDMA驱动unbind Jetty。 + +4. 参数 + +@param[in] jetty: the jetty binded before; + +5. 返回值 + +0 on success, other value on error + +#### 5.5.5.9 Jetty flush ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*flush_jetty)(struct ubcore_jetty *jetty, int cr_cnt, struct ubcore_cr *cr); + +3. 描述 + +注销一个ubcore内核态应用客户端 + +4. 参数 + +@param[in] jetty: the jetty created before; + +@param[in] cr_cnt: the maximum number of CRs expected to be returned; + +@param[out] cr: the addr of returned CRs; + +5. 返回值 + +the number of CR returned, 0 means no completion record returned, -1 on error + +### 5.5.6 Jetty group管理接口 + +#### 5.5.6.1 Jetty group创建ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +struct ubcore_jetty_group *(*create_jetty_grp)(struct ubcore_device *dev, + +struct ubcore_jetty_grp_cfg *cfg, struct ubcore_udata *udata); + +3. 描述 + +创建新的jetty group，支持指定id和token。只支持RM模式。 + +4. 参数 + +@param[in] dev: the ub device handle; + +@param[in] cfg: pointer of the jetty group config; + +@param[in] udata: ucontext and user space driver data + +5. 返回值 + +jetty group pointer on success, NULL on error + +#### 5.5.6.2 Jetty group销毁ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*delete_jetty_grp)(struct ubcore_jetty_group *jetty_grp); + +3. 描述 + +调用该API销毁Jetty group。销毁成功， Jetty group不能再被用于执行接收消息和存放接收buffer。 + +4. 参数 + +@param[in] jetty_grp: the jetty group created before; + +5. 返回值 + +0 on success, other value on error + +### 5.5.7 segment和token管理接口 + +#### 5.5.7.1 token_id分配ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +struct ubcore_token_id *(*alloc_token_id)(struct ubcore_device *dev, union ubcore_token_id_flag flag, + +struct ubcore_udata *udata); + +3. 描述 + +UB Core调用UDMA驱动分配token id。 + +4. 参数 + +@param[in] dev: the ub device handle; + +**@**param*[in] flag: token_id_flag;* + +@param[in] udata: ucontext and user space driver data + +5. 返回值 + +token id pointer on success, NULL on error + +#### 5.5.7.2 token_id释放ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*free_token_id)(struct ubcore_token_id *token_id); + +3. 描述 + +UB Core调用UDMA驱动释放token id。 + +4. 参数 + +@param[in] token_id: the token id alloced before; + +5. 返回值 + +0 on success, other value on error + +#### 5.5.7.3 segment注册ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +struct ubcore_target_seg *(*register_seg)(struct ubcore_device *dev, + +struct ubcore_seg_cfg *cfg, + +struct ubcore_udata *udata); + +3. 描述 + +UB Core调用UDMA驱动注册segment + +4. 参数 + +@param[in] dev: the ub device handle; + +@param[in] cfg: segment attributes and configurations + +@param[in] udata: ucontext and user space driver data + +5. 返回值 + +target segment pointer on success, NULL on error + +#### 5.5.7.4 segment反注册ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*unregister_seg)(struct ubcore_target_seg *tseg); + +3. 描述 + +UB Core调用UDMA驱动反注册segment和token。 + +4. 参数 + +@param[in] tseg: the segment registered before; + +5. 返回值 + +0 on success, other value on error + +#### 5.5.7.5 segment导入ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +struct ubcore_target_seg *(*import_seg)(struct ubcore_device *dev, + +struct ubcore_target_seg_cfg *cfg, + +struct ubcore_udata *udata); + +3. 描述 + +UB Core调用UDMA驱动导入segment。 + +4. 参数 + +@param[in] dev: the ub device handle; + +@param[in] cfg: segment attributes and import configurations + +@param[in] udata: ucontext and user space driver data + +5. 返回值 + +target segment handle on success, NULL on error + +#### 5.5.7.6 segment反导入ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*unimport_seg)(struct ubcore_target_seg *tseg); + +3. 描述 + +UB Core调用UDMA驱动反导入远端segment和token。 + +4. 参数 + +@param[in] tseg: the segment imported before; + +5. 返回值 + +0 on success, other value on error + +### 5.5.8 dscp-vl映射管理接口 + +#### 5.5.8.1 dscp-vl映射配置接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*config_dscp_vl)(struct ubcore_device *dev, uint8_t *dscp, + +uint8_t *vl, uint8_t num); + +3. 描述 + +配置dscp-vl映射关系。 + +4. 参数 + +@param[in] dev:the ub dev handle; + +@param[in] dscp: the dscp value array + +@param[in] vl: the vl value array + +@param[in] num: array num + +5. 返回值 + +0 on success, other value on error + +#### 5.5.8.2 dscp-vl映射查询接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*query_dscp_vl)(struct ubcore_device *dev, uint8_t *dscp, + +uint8_t num, uint8_t *vl); + +3. 描述 + +查询dscp-vl映射关系。 + +4. 参数 + +@param[in] dev:the ub dev handle; + +@param[in] dscp: the dscp value array + +@param[in] num: array num + +@param[out] vl: the vl value array + +5. 返回值 + +0 on success, other value on error + +### 5.5.9 其他ops接口 + +#### 5.5.9.1 驱动自定义控制user_ctl ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*user_ctl)(struct ubcore_device *dev, struct ubcore_user_ctl *user_ctl); + +3. 描述 + +UB Core调用UDMA驱动执行user ctl命令 + +4. 参数 + +@param[in] dev: the ub device handle; + +@param[in] user_ctl: kdrv user control command pointer; + +5. 返回值 + +0 on success, other value on error + +### 5.5.10 内核态异常事件上报接口 + +#### 5.5.10.1 Jetty异步事件回调接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +typedef void (*ubcore_event_callback_t)(struct ubcore_event *event, struct ubcore_ucontext *ctx); + +3. 描述 + +ubcore_event_callback_t为一个函数指针，是JFS、JFR、Jetty、JFC和Jetty group结构体成员，在创建Jetty或者JFC时由UB Core协议栈注册到内核态驱动。一旦发生JFS、JFR、Jetty、JFC和Jetty group异常，芯片会通过硬件中断上报给UDMA驱动，UDMA驱动解析中断信息，查找到异常JFS、JFR、Jetty、JFC和Jetty group，并调用ubcore_event_callback将异常上报给UB Core协议栈。 + +4. 参数 + +@param[in] [Required] event: 异常事件描述结构体，包括设备编号、Jetty类型编号和异常事件类型等，UB Core根据该域段选择不同的后处理策略。 + +@param[in] [Required] ctx: UB Core协议栈定义的上下文信息，通过该域段可以找到对应的JFAE（Jetty For Asyn Event） + +5. 返回值 + +NA + +#### 5.5.10.2 异步事件分发接口 + +1. 头文件 + +#include "ubcore_api.h" + +2. 原型 + +void ubcore_dispatch_async_event(struct ubcore_event *event); + +3. 描述 + +用于异步事件的分发。一旦发生设备异常，芯片会通过硬件中断上报给UDMA驱动，UDMA驱动解析中断信息，查找到异常设备，并调用ubcore_dispatch_async_event将异常上报给UB Core协议栈；UB Core协议栈进一步通知更上层的client。 + +设备异常包括Link状态切换、RAS异常等。 + +4. 参数 + +@param[in] event: asynchronous event; + +5. 返回值 + +NA + +### 5.5.11 内核态状态查询和DFX 接口 + +芯片状态查询需要下沉至内核态，实现内核态驱动与用户态DFX工具通信。 + +#### 5.5.11.1 统计查询ops接口 + +1. 头文件 + +#include "ubcore_uapi.h" + +2. 原型 + +int (*query_stats)(struct ubcore_device *dev, struct ubcore_stats_key *key, + +struct ubcore_stats_val *val); + +3. 描述 + +query_stats为一个函数指针。query_stats提供设备级统计信息查询，统计信息包括报文收发包、设备实时运行状态等。 + +4. 参数 + +@param[in] dev: the ub device handle; + +@param[in] key: type and key value of the ub device to query; + +@param[in/out] val: address and buffer length of query results + +5. 返回值 + +0 on success, other value on error + +#### 5.5.11.2 资源查询ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*query_res)(struct ubcore_device *dev, struct ubcore_res_key *key, + +struct ubcore_res_val *val); + +3. 描述 + +query_res为一个函数指针。query_res提供设备级资源使用情况查询，例如Jetty创建数目和VF的UPI等。 + +4. 参数 + +@param[in] dev: the ub device handle; + +@param[in] key: resource type and key; + +@param[in/out] val: addr and len of value + +5. 返回值 + +0 on success, other value on error + +### 5.5.12 内核态数据面接口 + +#### 5.5.12.1 JFS发送WR ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*post_jfs_wr)(struct ubcore_jfs *jfs, struct ubcore_jfs_wr *wr, + +struct ubcore_jfs_wr **bad_wr); + +3. 描述 + +发起单边、双边或者原子操作的请求。待操作成功后，应用可poll JFC获得完成消息。可以指定ordering和其他的flag。 + +4. 参数 + +@param[in] jfs: the jfs created before; + +@param[in] wr: the wr to be posted; + +@param[out] bad_wr: the first failed wr; + +5. 返回值 + +0 on success, other value on error + +#### 5.5.12.2 JFR接收WR ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*post_jfr_wr)(struct ubcore_jfr *jfr, struct ubcore_jfr_wr *wr, + +struct ubcore_jfr_wr **bad_wr); + +3. 描述 + +发起接收操作填recv buffer的请求。待接收操作成功后，应用可poll JFC获得完成消息。 + +4. 参数 + +@param[in] jfr: the jfr created before; + +@param[in] wr: the wr to be posted; + +@param[out] bad_wr: the first failed wr; + +5. 返回值 + +0 on success, other value on error + +#### 5.5.12.3 Jetty发送WR ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*post_jetty_send_wr)(struct ubcore_jetty *jetty, struct ubcore_jfs_wr *wr, + +struct ubcore_jfs_wr **bad_wr); + +3. 描述 + +发起单边、双边或者原子操作的请求。待操作成功后，应用可poll JFC获得完成消息。可以指定ordering和其他的flag。 + +4. 参数 + +@param[in] jetty: the jetty created before; + +@param[in] wr: the wr to be posted; + +@param[out] bad_wr: the first failed wr; + +5. 返回值 + +0 on success, other value on error + +#### 5.5.12.4 Jetty接收WR ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*post_jetty_recv_wr)(struct ubcore_jetty *jetty, struct ubcore_jfr_wr *wr, + +struct ubcore_jfr_wr **bad_wr); + +3. 描述 + +发起接收操作填recv buffer的请求。待接收操作成功后，应用可poll JFC获得完成消息。 + +4. 参数 + +@param[in] jetty: the jetty created before; + +@param[in] wr: the wr to be posted; + +@param[out] bad_wr: the first failed wr; + +5. 返回值 + +0 on success, other value on error + +#### 5.5.12.5 rearm JFC ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*rearm_jfc)(struct ubcore_jfc *jfc, bool solicited_only); + +3. 描述 + +重新设置jfc的事件通知机制。在应用调用wait_jfc返回之后需要调用该函数重新设置通知机制. + +4. 参数 + +@param[in] jfc: the jfc created before; + +@param[in] solicited_only: rearm notify by message marked with solicited flag + +5. 返回值 + +0 on success, other value on error + +#### 5.5.12.6 轮询 JFC ops接口 + +1. 头文件 + +#include "ubcore_types.h" + +2. 原型 + +int (*poll_jfc)(struct ubcore_jfc *jfc, int cr_cnt, struct ubcore_cr *cr); + +3. 描述 + +轮询JFC，轮询的结果返回到参数cr指定的地址中。cr数据结构包括了请求执行的结果，传输的数据长度，错误类型等信息。 + +4. 参数 + +@param[in] jfc: the jfc created before; + +@param[in] cr_cnt: the maximum number of CRs expected to be polled; + +5. 返回值 + +0 on success, other value on error + +--- +# 6 UVS编程接口 + +## 6.1 uvs_set_topo_info + +1. 头文件 + +#include "uvs_api.h" + +2. 原型 + +int **uvs_set_topo_info**(void **topo*, uint32_t *topo_num*); + +3. 场景 + +计算场景 + +4. 描述 + +设置拓扑信息到ubagg模块和ubcore模块。拓扑信息由MXE模块传入。 + +5. 参数 + +*@param[in] topo: topo info of one bonding device* + +*@param[in] topo_num: number of bonding devices* + +6. 返回值 + +*Return: 0 on success, other value on error* + +## 6.2 编程示例 diff --git a/.claude/skills/query-urma-docs/URMA QuickStart Guide.ch.md b/.claude/skills/query-urma-docs/URMA QuickStart Guide.ch.md new file mode 100644 index 000000000..5ef733c07 --- /dev/null +++ b/.claude/skills/query-urma-docs/URMA QuickStart Guide.ch.md @@ -0,0 +1,336 @@ +# 修订记录 + +| 修订时间 | 修订章节 | 修订内容简介 | 修复问题单连接或问题背景 | 修订人员 | +|----|----|----|----|----| +| 2026.2.12 | ALL | 文档基线 | | @qianguoxin、@jerry_lilijun、@guguguo0127 | +--- + +# 目 录 + +- [修订记录](#修订记录) + +- [1 编译指南](#1-编译指南) + - [1.1 组件简介](#11-组件简介) + - [1.2 用户态组件单独编译](#12-用户态组件单独编译) + - [1.3 内核态组件单独编译](#13-内核态组件单独编译) + +- [2 安装指南](#2-安装指南) + - [2.1 安装包概述](#21-安装包概述) + - [2.2 安装依赖](#22-安装依赖) + - [2.3 用户态安装](#23-用户态安装) + - [2.4 URMA RPM包安装](#24-urma-rpm包安装) + - [2.5 内核态ko安装](#25-内核态ko安装) + +- [3 功能依赖](#3-功能依赖) + +- [4 验证与运行示例](#4-验证与运行示例) + - [4.1 设备验证](#41-设备验证) + - [4.2 性能测试示例](#42-性能测试示例) + +# 1 编译指南 + +## 1.1 组件简介 + +URMA组件是一个高性能通信组件，分为用户态和内核态两部分： + +- 用户态：提供应用程序接口，有独立的源码仓库：https://gitcode.com/openeuler/umdk + +- 内核态：位于OpenEuler内核源码的 drivers/ub/urma 目录中：https://gitcode.com/openeuler/kernel + +## 1.2 用户态组件单独编译 + +**编译步骤** + +1. 安装编译工具和软件包 + +```bash +yum install -y git rpm-build make cmake gcc glibc-devel kernel-devel libnl3-devel openssl-devel +``` + +2. 下载源码，进入源码src/路径下，创建并进入build构建目录 + +```bash +mkdir build +cd build +``` + +3. 执行配置与编译 + +```bash +cmake -DCMAKE_VERBOSE_MAKEFILE=on \ +-DCMAKE_INSTALL_PREFIX=/usr \ +-DBUILD_ALL=disable \ +-DBUILD_URMA=enable \ +-DBUILD_UDMA=disable \ +-DBUILD_UMS=disable \ +.. +make -j$(nproc) +``` + +**参数说明** + +- `-DCMAKE_VERBOSE_MAKEFILE=on`：显示详细的编译信息，便于排查问题 + +- `-DCMAKE_INSTALL_PREFIX=/usr`：指定安装路径为系统目录 + +- `-DBUILD_URMA=enable`：明确启用URMA模块编译 + +## 1.3 内核态组件单独编译 + +**前提条件** + +在单独编译内核态组件前，必须完整运行一次内核的全量编译，确保依赖文件已正确生成。 + +**编译步骤** + +1. 安装编译工具 + +```bash +yum install -y dpkg dpkg-devel openssl openssl-devel +yum install -y ncurses ncurses-devel bison flex bc libdrm build elfutils-libelf-devel +``` + +2. 进入内核源码目录 + +```bash +cd kernel +``` + +3. 配置内核（如果尚未配置） + +```bash +make openeuler_defconfig +``` + +4. 单独编译URMA内核模块 + +```bash +make M=drivers/ub/urma -j$(nproc) +``` + +**编译结果** + +编译完成后，会在 drivers/ub/urma 目录下生成 .ko 内核模块文件，包括 ubcore.ko、ubagg.ko、uburma.ko。可通过如下命令验证： + +```bash +cd drivers/ub/urma +find . -type f -name "*.ko" +# ./drivers/ub/urma/ubcore/ubcore.ko +# ./drivers/ub/urma/ubagg/ubagg.ko +# ./drivers/ub/urma/uburma/uburma.ko +``` + +--- +# 2 安装指南 + +## 2.1 安装包概述 + +URMA安装包分为aarch64和x86_64两种，分别支持ARM平台和X86平台。详细RPM包内容见下表： + +**UMDK安装包描述** + + +++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
组件安装包备注
urmaumdk-urma-lib-xxx.rpmurma用户态安装包
umdk-urma-bin-xxx.rpmurma内核模块安装包，需要与内核配套使用
umdk-urma-devel-xxx.rpmurma开发包，包含开发头文件等
umdk-urma-tools-xxx.rpmurma工具包，包含urma_admin、urma_perftest等辅助命令
umdk-urma-examples-xxx.rpm包含urma用户态编程API的使用示例
+ +![](figures/urma_caution.png) + +> urma组件间暂不支持跨发布版本单独升级 + +## 2.2 安装依赖 + +```bash +yum install -y rpm-build +yum install -y make +yum install -y cmake +yum install -y gcc +yum install -y gcc-c++ +yum install -y glibc-devel +yum install -y openssl-devel +yum install -y glib2-devel +yum install -y libnl3-devel +yum install -y kernel-devel # ubcore依赖，来自openEuler内核 +``` + +## 2.3 用户态安装 + +### 方法1：使用 make install 编译安装 + +```bash +cd src +mkdir build +cd build +cmake .. -D BUILD_ALL=disable -D BUILD_URMA=enable +make install -j +``` + +### 方法2：单独编译RPM包 + +RPM包生成在路径 `/root/rpmbuild/RPMS/aarch64` 下： + +```bash +mkdir -p /root/rpmbuild/SOURCES/ +cd /UMDK +tar -czf /root/rpmbuild/SOURCES/umdk-25.12.0.tar.gz --exclude=.git `ls -A` +rpmbuild -ba umdk.spec --with urma +cd /root/rpmbuild/RPMS/aarch64 +rpm -Uvh umdk-urma-lib-25.12.0-0.aarch64.rpm --force --nodeps +rpm -Uvh umdk-urma-bin-25.12.0-0.aarch64.rpm --force --nodeps +rpm -Uvh umdk-urma-tools-25.12.0-0.aarch64.rpm --force --nodeps +rpm -Uvh umdk-urma-example-25.12.0-0.aarch64.rpm --force --nodeps +rpm -Uvh umdk-urma-devel-25.12.0-0.aarch64.rpm --force --nodeps +``` + +### 方法3：yum安装 + +```bash +yum install -y umdk-urma-lib-25.12.0-0.aarch64 +yum install -y umdk-urma-bin-25.12.0-0.aarch64 +yum install -y umdk-urma-example-25.12.0-0.aarch64 +yum install -y umdk-urma-tools-25.12.0-0.aarch64 +yum install -y umdk-urma-devel-25.12.0-0.aarch64 +``` + +## 2.4 URMA RPM包安装 + +URMA子系统在UBUS系统中提供高带宽低时延的数据服务，支持在UBUS原生硬件平台上运行，UBUS原生硬件的驱动需要由海思提供。 + +**URMA支持平台组件图** + +![](figures/urma-platform-arch.png) + +![](figures/urma_notice.png) + +URMA的安装通过RPM的方式，安装需要root权限。 + +![](figures/urma_warning.png) + +**安装要点：** + +1. URMA组件包含ubcore等内核模块，liburma-udma.so等各驱动版本需要强配套使用。 + +2. URMA推荐使用rpm包的方式安装，用户态组件（包含liburma.so、liburma_common.so等）默认安装在 `/usr/lib64/` 下，用户态驱动（liburma-udma.so）默认安装在 `/usr/lib64/urma/` 目录下。 + +3. 由于URMA子系统的组件liburma.so会打开安装同级目录的urma子目录上的驱动，如果应用需要指定urma安装目录，驱动需要按照以下格式安装： + + `/XXX/YYY/urma/liburma-udma.so` + +URMA子系统对外提供的统一运行组件为 umdk-urma-lib 和 umdk-urma-bin 两个RPM包，umdk-urma-tools 包提供了URMA运行时管理工具。如果需要基于URMA进行开发，需要进一步安装 umdk-urma-devel 包。 + +**RPM安装命令：** + +```bash +rpm -ivh umdk-urma-lib-25.12.0-B004.oe2403sp3.aarch64.rpm +rpm -ivh umdk-urma-bin-25.12.0-B004.oe2403sp3.aarch64.rpm +rpm -ivh umdk-urma-devel-25.12.0-B004.oe2403sp3.aarch64.rpm +rpm -ivh umdk-urma-tools-25.12.0-B004.oe2403sp3.aarch64.rpm +rpm -ivh umdk-urma-examples-25.12.0-B004.oe2403sp3.aarch64.rpm +``` + +## 2.5 内核态ko安装 + +安装RPM包后需要加载内核模块，ubcore、ubagg和uburma模块为必选加载，另外需要加载海思内核模块udma.ko（通过modprobe或insmod加载udma.ko，具体加载命令以海思提供为准）。 + +```bash +modprobe ubcore +modprobe uburma +modprobe ubagg +``` + +> **补充说明**：在某些平台上，可能需要更详细的内核模块加载顺序和参数。以下是包含海思内核模块的完整加载示例： +> +> ```bash +> cd /lib/modules/$(uname -r)/kernel/drivers +> insmod ub/ubfi/ubfi.ko.xz cluster=1 # 使用VF网卡时需移除 cluster=1 参数 +> insmod iommu/ummu-core/ummu-core.ko.xz +> insmod ub/hisi-ub/kernelspace/ummu/drivers/ummu.ko.xz +> insmod ub/hisi-ub/kernelspace/ubus/ubus.ko.xz cc_en=0 um_entry_size=1 +> insmod ub/hisi-ub/kernelspace/ubus/vendor/hisi/hisi_ubus.ko.xz msg_wait=2000 fe_msg=1 um_entry_size1=0 cfg_entry_offset=512 +> insmod ub/hisi-ub/kernelspace/ubase/ubase.ko.xz +> insmod ub/hisi-ub/kernelspace/unic/unic.ko.xz tx_timeout_reset_bypass=1 +> insmod ub/hisi-ub/kernelspace/cdma/cdma.ko.xz +> modprobe ubcore uburma +> modprobe udma dfx_switch=1 jfc_arm_mode=2 is_active=0 fast_destroy_tp=0 +> modprobe ubagg +> ``` + +--- +# 3 功能依赖 + +- **系统要求**：OpenEuler 24.03 SP3 或更高版本 + +- **内核版本**：与编译所用内核一致。举例：编译 OpenEuler 6.6.0 需使用对应 Linux-6.6.0 主线版本的 Linux 内核。 + +- **运行时依赖**： + - liburma.so、liburma_common.so、liburma_ubagg.so（用户态库） + - ubcore.ko、ubagg.ko、uburma.ko（内核模块） + +--- +# 4 验证与运行示例 + +## 4.1 设备验证 + +使用 urma_admin 工具检查设备是否正常扫描： + +```bash +urma_admin show +``` + +输出示例： + +``` +num ubep_dev tp_type eid link +--- ---------------- -------- -------------------------------------------- -------- +0 udma3 UB eid0 0000:0000:0000:00xx:00xx:00xx:00xx:1001 ACTIVE +1 udma3 UB eid1 0000:0000:0000:00xx:00xx:00xx:00xx:1002 ACTIVE +2 udma5 UB eid0 0000:0000:0000:00xx:00xx:00xx:00xx:1003 ACTIVE +3 udma5 UB eid1 0000:0000:0000:00xx:00xx:00xx:00xx:1004 ACTIVE +4 udma2 UB eid0 0000:0000:0000:00xx:00xx:00xx:00xx:1005 ACTIVE +5 udma4 UB eid0 0000:0000:0000:00xx:00xx:00xx:00xx:1006 ACTIVE +``` + +## 4.2 性能测试示例 + +```bash +systemctl start scbus-daemon.service + +# 启动服务端 +urma_perftest send_bw -d bonding_dev_0 -s 2 -n 10 -I 128 -p 1 + +# 启动客户端（替换 为实际服务端IP） +urma_perftest send_bw -d bonding_dev_0 -s 2 -n 10 -I 128 -p 1 -S +``` diff --git a/.claude/skills/query-urma-docs/URMA User Guide.ch.md b/.claude/skills/query-urma-docs/URMA User Guide.ch.md new file mode 100644 index 000000000..c6be6a427 --- /dev/null +++ b/.claude/skills/query-urma-docs/URMA User Guide.ch.md @@ -0,0 +1,2522 @@ +# 修订记录 + +| 修订时间 | 修订章节 | 修订内容简介 | 修复问题单连接或问题背景 | 修订人员 | +| --- | --- | --- | --- | --- | +| 2026.2.12 | ALL | 文档基线 | | @qianguoxin、@lairuilang、@wanghang73、@eingesch、@chenjingwei0113、@duelu、@autoreconf、@wdmmsyf | + +--- + +# 目 录 + +- [修订记录](#修订记录) + +- [1 UMDK综述](#1-umdk综述) + +- [2 URMA简介](#2-urma简介) + - [2.1 基础概念](#21-基础概念) + - [2.1.1 UB](#211-ub) + - [2.1.2 UBVA地址模型](#212-ubva地址模型) + - [2.1.3 Segment](#213-segment) + - [2.1.4 Jetty](#214-jetty) + - [2.1.5 UBoE](#215-uboe) + +- [3 编译安装](#3-编译安装) + - [3.1 RPM包编译](#31-rpm包编译) + - [3.2 URMA RPM安装](#32-urma-rpm安装) + +- [4 Quick Start](#4-quick-start) + +- [5 URMA架构](#5-urma架构) + - [5.1 管理面](#51-管理面) + - [5.1.1 分布式管控](#511-分布式管控) + - [5.1.1.1 可靠建链协议](#5111-可靠建链协议) + - [5.1.1.2 共享传输层](#5112-共享传输层) + - [5.1.1.3 前后端分离建链](#5113-前后端分离建链) + - [5.1.1.4 建链状态机管理](#5114-建链状态机管理) + - [5.1.2 集中式管控](#512-集中式管控) + - [5.1.2.1 端侧-管理面建链](#5121-端侧-管理面建链) + - [5.1.2.2 感知传输层的建链](#5122-感知传输层的建链) + - [5.1.2.3 不感知传输层的建链](#5123-不感知传输层的建链) + - [5.2 控制面](#52-控制面) + - [5.2.1 上下文管理](#521-上下文管理) + - [5.2.2 Jetty管理](#522-jetty管理) + - [5.2.3 Segment管理](#523-segment管理) + - [5.2.4 异常事件](#524-异常事件) + - [5.2.4.1 flush jetty](#5241-flush-jetty) + - [5.2.5 设备属性](#525-设备属性) + - [5.2.6 token安全传输](#526-token安全传输) + - [5.3 数据面](#53-数据面) + - [5.3.1 单边操作](#531-单边操作) + - [5.3.2 双边操作](#532-双边操作) + - [5.3.3 完成记录](#533-完成记录) + +- [6 关键特性介绍](#6-关键特性介绍) + - [6.1 特性树](#61-特性树) + - [6.2 设备聚合](#62-设备聚合) + - [6.2.1 聚合设备基本概念](#621-聚合设备基本概念) + - [6.2.2 聚合设备基本使用流程](#622-聚合设备基本使用流程) + - [6.2.3 聚合设备特性列表和约束](#623-聚合设备特性列表和约束) + - [6.3 虚拟化](#63-虚拟化) + - [6.3.1 容器](#631-容器) + - [6.3.2 虚拟机](#632-虚拟机) + - [6.4 工具手册](#64-工具手册) + - [6.4.1 urma_perftest](#641-urma_perftest) + - [6.4.2 urma_admin](#642-urma_admin) + - [6.5 DFX维测](#65-dfx维测) + - [6.5.1 URMA日志](#651-urma日志) + +- [7 生态兼容](#7-生态兼容) + - [7.1 RoUB](#71-roub) + - [7.2 IPoURMA](#72-ipourma) + - [7.3 UMS](#73-ums) + +- [8 性能规格](#8-性能规格) + +- [9 网络安全](#9-网络安全) + - [9.1 UB访问控制](#91-ub访问控制) + - [9.1.1 应用场景](#911-应用场景) + - [9.1.2 功能原理](#912-功能原理) + - [9.1.3 权限分配流程](#913-权限分配流程) + - [9.1.4 权限无效化流程](#914-权限无效化流程) + - [9.2 内存访问控制](#92-内存访问控制) + +# 1 UMDK综述 + +1. + +![](figures/urma-overview-01.png) + +灵衢内存语义开发包（Unified Memory Development Kit, UMDK）是一套以内存语义为核心的分布式通信软件库，为数据中心网络内的主机之间、设备之间以及主机和设备之间提供高性能的通信接口，使能和释放灵衢总线的硬件能力。UMDK包含以下组件： + +1. URMA（Unified Remote Memory Access）：UB（Unified Bus）通信基础库，通过屏蔽底层不同硬件驱动的差异，为上层用户提供了统一的单边、双边、原子等操作远端内存的方式，是应用之间通信的基础。为此，URMA提供两大类接口，一是北向应用编程接口，为应用提供通信API，二是南向驱动编程接口，为驱动开发者提供接入UMDK的API。 + +2. URPC（Unified Remote Process Call）：高性能RPC库，支持灵衢原生高性能主机间和设备间RPC通信，以及RPC加速。 + +3. ULOCK（Unified Lock）：高性能分布式锁，支持灵衢原生高性能状态同步，加速数据库等分布式应用全局资源分配。 + +4. USOCK（Unified Socket）：灵衢通信生态构建，兼容标准Socket编程接口，使能TCP应用零修改提升网络通信性能。 + +--- +# 2 URMA简介 + +URMA（Unified Remote Memory Access，统一远程内存访问）是UMDK的核心基础通信库，其设计理念是通过统一的内存访问语义，实现高效、灵活的分布式内存操作。URMA为上层应用提供了统一的编程接口，支持单边操作、双边消息、原子操作等多种访问远端内存的方式，屏蔽了底层不同硬件的驱动差异。 + +在系统架构上，URMA提供南北向两类接口：北向面向应用，提供简洁的通信API；南向面向驱动开发者，定义标准的接入规范，便于不同硬件融入UB生态。 + +在实际业务中，URMA不仅为数据中心的各类业务提供了高带宽、低时延的消息通信与数据转发基础能力，也为上层更高级的语义编排功能奠定了基础。它能够显著降低大数据业务的端到端通信时延，并为HPC、AI等高性能计算场景提供关键的高性能数据服务支撑。 + +URMA及其周边组件架构如下图所示： + +![](figures/urma-intro-01.png) + +- ubcore.ko：URMA 核心模块，提供基础能力，向上为内核态应用提供接口，向下支持内核态驱动接入。 + +- uburma.ko：将 ubcore.ko 的功能封装为系统调用，供用户态使用。 + +- liburma.so（CMD API）：用户态驱动接口层，封装 uburma.ko 的系统调用，为用户态驱动提供调用入口。 + +- liburma.so（USER API）：用户态应用接口层，向上为用户程序提供接口，向下支持用户态驱动注册。 + +## 2.1 基础概念 + +### 2.1.1 UB + +Unified Bus，灵衢统一总线，包含终端、交换机和软件。 + +### 2.1.2 UBVA地址模型 + +UBVA，Unified Bus Virtual Address，是UBUS总线上的分级的虚拟地址，支持对总线的多个节点共享内存进行统一编址，打破了各个节点地址边界，允许应用通过VA进行跨节点寻址和数据访问。由EID和VA地址两个部分组成。 + +### 2.1.3 Segment + +Segment是一段连续的VA地址空间，同时分配物理内存来对应到一个segment。由segment home节点创建。User侧的APP把segment映射到进程虚拟地址空间，通过被映射地址直接访问远端内存。segment的VA地址和user进程映射的VA可以相同，也可以不同。VA地址相同的场景，即DSVA场景。 + +### 2.1.4 Jetty + +Jetty 是事务层的统一操作接口，可视为事务执行的"港口"，用于管理提交的IO任务或接收的消息的队列。Jetty 主要分为以下几类： + +1. JFS（Jetty for send）：用于提交发送任务（WQE，Work Queue Element）。 + +2. JFR（Jetty for receive）：用于提交接收任务。 + +3. JFC（Jetty for completion）：用于存放发送、接收任务的完成队列记录（CQE，Completion Queue Element）。 + +4. Jetty：具有JFS、JFR两者的功能，同时支持提交发送和接收任务。 + +### 2.1.5 UBoE + +UB over Ethernet, UBoE是指UB事务层和传输层语义承载在Ethernet/IP上的报文格式。如下图所示，ETH头之后可选的，可以使用OPtag携带增强的负载均衡、拥塞控制和网络隔离特性字段。Optag格式定义在ETH网链路层协议族维护，此处仅为参考。 + +![](figures/urma-intro-02.png) + +![](figures/urma-intro-03.png) + +--- +# 3 编译安装 + +## 3.1 RPM包编译 + +**方法一：独立编译URMA RPM包** + +1. 进入UMDK工程根目录下 + +2. 打包源码： + +```bash +tar -czf /root/rpmbuild/SOURCES/umdk-25.12.0.tar.gz --exclude=.git `ls -A` +``` + +3. 编译RPM包： + +```bash +rpmbuild -ba umdk.spec --with urma +``` + +**方法二：make install 编译安装** + +1. 进入UMDK/src工程根目录下 + +2. 创建并进入构建目录： + +```bash +mkdir build +cd build +``` + +3. 配置并编译安装： + +```bash +cmake .. -D BUILD_ALL=disable -D BUILD_URMA=enable +make install -j +``` + +## 3.2 URMA RPM安装 + +说明：URMA需要调用URMA组件的能力，需要提前安装好URMA软件。 + +```bash +rpm -ivh /root/rpmbuild/RPMS/aarch64/umdk-urma-*.rpm +``` + +--- +# 4 Quick start + +本章介绍了URMA通信的基本流程。为使读者在深入细节前建立整体认识，本章以客户端-服务器模型为例，展示URMA通信的四个核心阶段：资源准备、连接建立、数据传输和资源释放。 + +URMA通信流程可分为四个主要阶段： + +**阶段一：资源准备** + +在此阶段，应用程序需要完成底层通信框架的初始化和关键资源的创建。首先调用初始化函数创建上下文，后续的URMA操作都是在上下文粒度进行的。随后创建通信端点（Jetty、JFR、JFS、JFC），为后续的数据收发提供通道。同时，应用程序还需在本地注册用于数据交换的内存区域（Segment），这些内存将被暴露给UB硬件访问。 + +**阶段二：连接建立** + +连接建立阶段负责构建通信双方的数据通路。应用程序需要自行获取对等端资源的ID、地址和访问权限信息，这些信息通常通过带外机制传输。获得必要信息后，应用程序通过导入操作将远程资源（包括Jetty和Segment ）映射到本地。这个过程建立了端到端的逻辑连接，使得本地应用能够像操作本地内存一样引用远程资源。 + +**阶段三：数据传输** + +数据传输阶段是实现核心功能的关键环节。应用程序通过提交工作请求（WR，Work Request）到Jetty来启动数据传输操作，这些请求描述了操作类型、源目标地址和大小等参数。 系统异步处理这些请求，应用程序则通过轮询JFC来确认操作执行结果。主要操作类型包括： + +- 双边SEND/RECV操作：基于接收方缓冲区的传统消息传递模式，注意发送端的SEND操作需要在接收端已下发RECV操作之后才能成功。 + +- 单边READ/WRITE操作：直接读写远端内存，无需远程CPU参与。 + +**阶段四：流程终止** + +流程终止阶段确保系统资源的正确释放和环境的清理。按照与创建相反的顺序，应用程序首先销毁Jetty和Segment，最后销毁上下文并卸载URMA通信框架。 + +以一个简单的客户端-服务器模型为例，应用程序流程如下： + +![](figures/urma-quickstart-01.png) + +示例程序参见： + +URMA用户态编程示例 + +--- +# 5 URMA架构 + +URMA架构主要包括：管理面、控制面和数据面三部分。 + +URMA的控制面和数据面类比于RDMA，是基于UB事务层概念的功能平面，其中控制面（Control Plane）用于管理UB的jetty、segment等事务层对象，数据面（Data Plane）负责基于UB事务层的数据面传输，是URMA高性能的核心。管理面（Management Plane）是管理事务层与传输层对应管理和传输层管理的平面，具备灵活的部署形态，是管理URMA建链的核心平面。 + +## 5.1 管理面 + +URMA管理面是基于UB事务层Jetty事务对象，提供传输层连接管理服务的软件模块。管理面在UB协议栈的层级关系如下所示： + +![](figures/urma-arch-mgmt-01.png) + +上图中，UBFM是 UB Domain的管理者，负责 Domain内的互连、通信和计算资源管理，动态处理系统运行过程中产生的事件。 + +UMMU是UB的内存管理单元（UB Memory Management Unit）。 + +管理面是指是管理事务层与传输层对应管理和传输层管理的平面。 + +URMA协议栈根据底层硬件差异分别支持分布式管控和集中式管控。区别如下图所示。 + +![](figures/urma-arch-mgmt-02.png) + +分布式管控 + +![](figures/urma-arch-mgmt-03.png) + +集中式管控 + +### 5.1.1 分布式管控 + +URMA协议的事务层主要通过URMA API呈现，传输层通常不对外暴露，用户在建链过程中会创建或复用传输层，因此需要有一个控制平面负责维护传输层的创建、状态机等生命周期的维护，因此需要创建了管控面的概念。 + +分布式管控是指管理面与事务层和传输层的协议软件采用分布式的方式部署在端侧，用于完成端侧事务层建链。当前SDI6.0的硬件形态支持分布式管控。 + +分布式管控的核心在于基于事务层与传输层分离的软件架构进行建链管理。 + +#### 5.1.1.1 可靠建链协议 + +URMA管控面负责针对可靠连接模式提供建链管理手段，此协议行为称为可靠建链协议。 + +发起端与目的端的可靠建链协议 + +（1）发起端节点（Initiator）建链流程中，创建本地传输层，连接管理模块向目的端节点（Target）连接管理模块发送连接请求，如下图 CONN_REQ所示 + +（2）目的端节点连接管理模块接收到连接请求后，通知协议框架和协议驱动，在本节点创建传输层，并切换传输层至RTR状态；然后目的端节点连接管理模块向发起端节点连接管理模块发送连接响应，如下图 CONN_REP所示。 + +（3）发起端节点连接管理模块收到连接响应后，通知协议框架和协议驱动，将本端节点传输层切换至RTS状态；然后发送端连接管理模块向目的端连接管理节点发送连接确认，如下图CONN_ACK所示。 + +![](figures/urma-arch-mgmt-dist-01.png) + +#### 5.1.1.2 共享传输层 + +两端同时发起建链的场景，为了节省传输层资源，创建共享传输层协议。 + +共享传输层是指发起端节点与目的端节点完成建链的基础上，目的端节点向发起端节点发起建链的操作。下图所示为此场景的共享对等传输层，即节点A向节点B发起建链创建的传输层，在节点B向节点A发起建链时，将会共享此传输层。 + +![](figures/urma-arch-mgmt-dist-02.png) + +下图所示为此场景共享传输层的协议，交互流程如下： + +（1）目的端节点发起建链，经查询已存在传输层，则共享传输层，不再创建传输层；目的端节点连接管理模块向发起端节点连接管理模块发送共享对等传输层连接请求，如下图CONN_REUSE_REQ所示； + +（2）发起端连接管理模块收到共享对等传输层连接请求后，查询本节点已存在传输层，则共享传输层，不再创建传输层；发起端连接管理模块向目的端连接管理模块发送共享对等传输层连接响应，如下图CONN_REUSE_REP所示。 + +![](figures/urma-arch-mgmt-dist-03.png) + +此外，管理面还提供了拒绝连接协议和拆链协议，流程如下： + +- 拒绝连接协议 + +下图所示为本场景的连接拒绝行为。协议交互流程如下： + +（1）发起端节点发起建链，此节点上连接管理模块向目的端节点的连接管理模块发送连接请求，如下图CONN_REQ所示； + +（2）目的端节点连接管理模块接收到连接请求，在本节点检查连接请求，包括必要的事务层和传输层的配置权限检查，本流程所述场景，目的端节点检查不通过，此节点连接管理模块回复连接拒绝消息，如下图CONN_REJ所示； + +（3）发起端节点连接管理模块收到连接拒绝消息，执行错误回滚流程，包括本端传输层资源选择性销毁（在传输层复用场景不销毁）、事务层资源销毁，并返回建链失败执行结果，流程结束。 + +![](figures/urma-arch-mgmt-dist-04.png) + +- 拆链通知协议 + +本流程描述了拆链场景的各节点行为，本流程特点是对等节点仅发送单条拆链通知。协议交互流程如下： + +（1）发起端节点用户进程发起拆链，或进程退出等场景协议框架自动发起拆链，若两端处于共享对等传输层场景，则用户触发拆链后，本节点连接管理模块向目的端连接管理模块发送拆链通知，如下图DCONN_NOTIFY所示；拆链消息发送完成，本节点自动切换到作为目的节点的建链状态，可参考下章节的建链状态机管理，等待目的端节点的拆链通知； + +（2）目的端节点连接管理模块收到拆链通知，通知协议框架、协议驱动进入拆链流程，本节点自动切换到发起端建链状态；在本节点发起拆链操作时，本节点从发起端建链状态切换到REST状态，销毁本地传输层资源；同时目的端节点的连接管理模块向发起端节点的连接管理模块发送拆链通知，如下图 DCONN_NOTIFY所示； + +（3）发起端节点连接管理模块收到拆链通知，通知协议框架、协议驱动进入拆链流程，本节点在流程（1）切换到目的端建链状态，此后切换到拆链完成状态，销毁本地传输层资源，拆链流程结束。 + +![](figures/urma-arch-mgmt-dist-05.png) + +#### 5.1.1.3 前后端分离建链 + +为了保证安全隔离，URMA协议栈设置了前后端分离的部署方式，前端负责对接用户管理事务层资源，后端管控面负责传输层生命周期的维护。 + +下图所示为连接管理模块与应用进程前后端分离部署形态，在A、B两节点上分别启动虚拟机，应用进程运行在虚拟机VM上。在A、B两节点上部署有附属设备，所述附属设备可采用的形式包括但不限于：（1）节点硬件附属设备，如虚拟化领域的DPU设备；（2）部署在软件隔离内存的虚拟设备等。附属设备通常称后端设备，该部署模式下，连接管理模块的配置与应用进程进行系统隔离。在附属设备上部署守护进程，连接管理模块在守护进程中运行，为前端VM中的多应用进程提供建链和拆链服务。系统管理员在后端设备上访问连接管理模块，通过命令行等方式完成连接管理模块的查询和组网配置等管理。 + +![](figures/urma-arch-mgmt-dist-06.png) + +下图所示为连接管理模块采用独立部署方式场景，隔离恶意攻击的实现方法，以VM前后端实施例进行说明。应用进程部署在VM前端节点，与外部网络、数据交互，存在被恶意攻击的风险，连接管理模块部署在后端的附属设备上，与应用进程部署在不同的操作系统上。当应用进程受到外部恶意攻击时，通过系统隔离攻击无法渗透到连接管理模块，可保证部署在节点设备上的事务层、传输层和用户信息安全。 + +独立灵活部署的连接管理模块，优势如下： + +（1）隔离部署限制攻击范围，提升安全性 + +云安全场景采用前后端+虚机分离部署，前端用户进程与后续管理模块隔离，攻击者针对用户进程进行恶意攻击时，恶意代码难以渗透到后端管理模块，控制面核心功能不受影响，同时能够限制攻击范围。 + +（2）前后端隔离，防止权限提升 + +只有经过授权的人员和进程才有权限访问和操作后端管理模块，云安全场景，针对前端进程仅分配所需最低权限，针对用户进程的攻击限制在虚机和容器范围，能够防止攻击权限提升从而访问后端系统资源。 + +（3）系统稳定，部署灵活 + +后端管理模块采用灵活的部署方式，如跨节点、分布式或节点集中式管控等方式，用于满足不同组网场景的需求。管理模块采用独立部署形态，能够降低用户进程异常崩溃等影响。管理模块独立进程部署形态，便于维护人员开展故障排查和维护管理。 + +![](figures/urma-arch-mgmt-dist-07.png) + +#### 5.1.1.4 建链状态机管理 + +下图为建链状态机管理的示例，建链状态机管理流程如下： + +结合图14说明建链、拆链和共享对等传输层、连接拒绝等流程的状态机切换： + +（1）发起端用户触发建链，本地创建传输层资源，发起端处于reset状态，发送连接请求至目的端（1401），发起端切换为REQ sent状态； + +（2）目的端用户触发建链，处于reset状态，目的端接收连接请求完成后（1404），切换为REQ received状态；目的端创建传输层资源，向发起端发送连接回复（1405），发送完成后切换为REP sent状态； + +（3）发起端未收到连接回复而触发超时（1402），切换到timeout的暂态，然后自动切换（1403）到reset，建链流程失败； + +（4）发起端处于REQ sent状态时，若收到连接请求消息（1408），则切换到peer compare状态，比较两端信息判断本节点处于发起端还是目的端，若此时未收到连接请求消息，则判断属于发起端（1409）,仍然自动切换回REQ sent状态；若此时已经收到连接请求消息，则判断处于目的端（1410），自动切换到REQ received状态； + +（5）发起端处于REQ sent状态时，若收到连接响应消息（1411），则切换到REP received状态，完成本端传输层状态切换后，自动发送连接确认（1412），自动切换到Initiator established状态； + +（6）本步骤接步骤（2）目的端流程，目的端处于REP sent状态时，若收到连接确认（1413），则切换到Target established状态；若未收到连接确认而触发超时（1406），则切换到timeout瞬态，并自动切换（1407）到reset状态，目的端建链流程失败；至此建链流程结束； + +（7）在共享对等传输层场景，目的端节点用户触发建链操作时，此节点发送共享对等传输层连接请求（1414），然后切换到REUSE sent状态，等待对应连接响应； + +（8）发起端节点收到共享对等传输层连接请求（1415），切换到REUSE received瞬态；如果在发起端节点共享连接请求检查通过则发送共享对等传输层连接响应到目的端（1416），并切换到reused状态；如果在发起端节点共享连接请求检查不通过则发送连接共享对等传输层连接拒绝消息（1419），并切换回Initiator established状态； + +（9）本步骤接步骤（7）的目的端流程，目的端节点处于REUSE sent状态，若未收到共享连接回复消息而触发超时（1417），则切换回Target established状态；若收到共享对等传输层连接回复（1418），则切换到reused暂态；至此共享对等传输层流程结束； + +（10）处于reused状态的任一节点，若用户先触发拆链，此节点发送拆链通知（1420），则此节点切换到NOTIFY sent on reuse暂态，此节点自动切换（1423）到Target established状态；此节点若收到拆链通知（1424），则切换到NOTIFY received瞬态，销毁传输层资源后，自动切换（1407）到reset状态； + +（10）处于reused状态的任一节点，若先收到拆链通知（1421），则此节点切换到NOTIFY received on reuse暂态，此节点自动切换（1422）到Initiator established状态；若此节点上用户触发拆链并发送拆链通知（1424），则此节点切换到NOTIFY sent瞬态，并启动切换（1425）到reset状态，至此拆链流程结束。 + +图示说明如下： + +**(1)reset：**建链初始状态； + +**(2)REQ sent：**发送连接请求完成状态； + +**(3)timeout：**超时状态； + +**(4)peer compare：**发起端/目的端切换比较状态； + +**(5)REP received：**接收连接响应完成状态； + +**(6)Initiator established：**发起端建链完成状态； + +**(7)NOTIFY sent：**拆链通知发送完成状态； + +**(8)Reuse received：**接收共享对等传输层连接请求完成状态； + +**(9)NOTIFY received on reuse：**共享对等传输层接收拆链通知完成状态； + +**(10)NOTIFY sent on reuse：**共享对等传输层发送拆链通知完成状态； + +**(11)reused：**共享对等传输层状态； + +**(12)REQ received：**接收连接请求完成状态； + +**(13)REP sent：**发送连接响应完成状态； + +**(14)Target established：**目的端建链完成状态； + +**(15)NOTIFY received：**接收拆链通知完成状态； + +**(16)Reuse sent：**发送共享对等传输层连接请求完成状态。 + +![](figures/urma-arch-mgmt-dist-08.png) + +### 5.1.2 集中式管控 + +集中式管控是指管理面与事务层和传输层的协议软件采用分离的方式部署，其中管理面部署在管控节点，协议软件部署在端侧节点，这种形态称为带外部署。在集中式管控形态中，也具有与带外形态不同的部署方式，即类似于分布式管控，管理面与协议软件均部署在端侧节点，无额外的管控节点，这种形态称为带内部署。当前KunPeng CPU、NPU等硬件形态支持集中式管控。 + +#### 5.1.2.1 端侧-管理面建链 + +集中式管控的主要流程是端侧用户通过URMA协议栈向管理面申请分配传输层资源，完成传输层信息交换之后，端侧触发传输层的状态切换，切换为激活态，则UB协议栈具备通信能力；拆链流程中，端侧触发传输层的状态切换，切换为去激活态，则连接断开，不具备通信能力。 + +#### 5.1.2.2 感知传输层的建链 + +基于建链元语，用户可选择使用感知传输层的建链。建链流程大致如下： + +1. 获取传输层； + +2. 交换事务层、传输层信息； + +3. 导入对端事务层对象，完成建链。 + +建链和拆链流程参考下图： + +![](figures/urma-arch-mgmt-cent-01.png) + +相关的URMA API（用户态为例） + +```c +/** +* get available tp list from control plane. +* @param[in] [Required] ctx: the created urma context pointer; +* @param[in] [Required] tp_cfg: tp configuration to get; +* @param[in && out] [Required] tp_cnt: tp_cnt is the length of tp_list buffer as in parameter; +* tp_cnt is the number of tp as out parameter; +* @param[out] [Required] tp_list: tp list to get, the buffer is allocated by user; +* Return: 0 on success, other value on error +*/ +urma_status_t urma_get_tp_list(urma_context_t *ctx, urma_get_tp_cfg_t *cfg, uint32_t *tp_cnt, +urma_tp_info_t *tp_list); +/** +* Import a remote jetty by control plane. +* Note: trans_mode from rjetty should be the same as the trans_mode of get_tp_list, +* users should obey this rule in case of unexpected errors. +* @param[in] [Required] ctx: the urma context created before; +* @param[in] [Required] rjetty: information of remote jetty to import, including jetty id and trans_mode, +* trans_mode same to create_jetty trans_mode; +* @param[in] [Required] token_value: token to put into output jetty protection table; +* @param[in] [Required] cfg: tp active configuration to exchange with target; +* Return: the address of target jetty, not NULL on success, NULL on error +*/ +urma_target_jetty_t *urma_import_jetty_ex(urma_context_t *ctx, urma_rjetty_t *rjetty, +urma_token_t *token_value, urma_import_jetty_ex_cfg_t *cfg); +/** +* Bind jetty: construct the transport channel between local jetty and remote jetty by control plane. +* Note: trans_mode from tjetty should be the same as the trans_mode of get_tp_list, +* users should obey this rule in case of unexpected errors. +* @param[in] [Required] jetty: local jetty to construct the transport channel; +* @param[in] [Required] tjetty: target jetty imported before; +* Return: 0 on success, URMA_EEXIST if the jetty has been binded, other value on error; +* @param[in] [Required] cfg: tp active configuration to exchange with target; +* Note: A local jetty can be binded with only one remote jetty. Only supported by jetty under URMA_TM_RC. +*/ +urma_status_t urma_bind_jetty_ex(urma_jetty_t *jetty, urma_target_jetty_t *tjetty, +urma_bind_jetty_ex_cfg_t *cfg); +``` + +用户关注基于感知传输层的urma API使用流程。 + +#### 5.1.2.3 不感知传输层的建链 + +在集中式管控建链方案中，URMA还提供了不感知传输层的建链方案，即用户无需进行传输层相关的API操作，创建事务层jetty等资源后，通过不感知传输层API触发建链流程，URMA内部适配层基于感知传输层的流程封装，具体流程如下： + +![](figures/urma-arch-mgmt-cent-02.png) + +相关的URMA API + +```c +/** +* Import a remote jetty. +* @param[in] [Required] ctx: the urma context created before; +* @param[in] [Required] rjetty: information of remote jetty to import, including jetty id and trans_mode, +* trans_mode same to create_jetty trans_mode; +* @param[in] [Required] token_value: token to put into output jetty protection table; +* Return: the address of target jetty, not NULL on success, NULL on error +*/ +urma_target_jetty_t *urma_import_jetty(urma_context_t *ctx, urma_rjetty_t *rjetty, +urma_token_t *token_value); +/** +* Bind jetty: construct the transport channel between local jetty and remote jetty. +* @param[in] [Required] jetty: local jetty to construct the transport channel; +* @param[in] [Required] tjetty: target jetty imported before; +* Return: 0 on success, URMA_EEXIST if the jetty has been binded, other value on error +* Note: A local jetty can be binded with only one remote jetty. Only supported by jetty under URMA_TM_RC. +*/ +urma_status_t urma_bind_jetty(urma_jetty_t *jetty, urma_target_jetty_t *tjetty); +``` + +用户关注基于不感知传输层的URMA API使用流程。 + +## 5.2 控制面 + +### 5.2.1 上下文管理 + +1. 概述 + +URMA支持不同的硬件平台，在初始化时需要配置对应的provider，同时指定使用的设备，创建出上下文。 + +2. 应用场景 + +URMA的上下文管理需要在应用运行初期执行，后续的Jetty、Segment管理和数据面操作都依赖该操作。 + +3. 使用说明 + + 1. 调用*urma_init*函数配置使用的平台和配置uasid。uasid不指定时由系统随机分配，指定uasid时可能导致函数执行失败。 + + + + 1. 调用*urma_query_device*函数查询设备的属性，获取eid等信息。如果应用已经获取设备的eid，则该步骤可不执行。 + + 2. 调用*urma_create_context*函数创建设备上下文。一个进程创建的资源总和（包括硬件doorbell寄存器，Jetty，Segment等），与其他进程相互隔离。 + +\-\-\--结束 + +![](figures/urma-arch-ctrl-ctx-01.png) + +```c +typedef struct urma_context { + struct urma_device *dev; /* [Private] point to the corresponding urma device. */ + struct urma_ops *ops; /* [Private] operation of urma device. */ + int dev_fd; /* [Private] fd of urma device's sysfs file. */ + int async_fd; /* [Private] fd of urma device's async event file. */ + pthread_mutex_t mutex; /* [Private] mutex of urma context. */ + urma_eid_t eid; /* [Public] eid of urma device. */ + uint32_t eid_index; + uint32_t uasid; /* [Public] uasid of current process. */ + struct urma_ref ref; /* [Private] reference count of urma context. */ +} urma_context_t; +``` + +1. struct urma_device *dev: 这是一个指向 urma_device 结构体的指针，它包含了与特定 URMA 设备相关的信息，如设备的属性、操作函数等。 + +2. struct urma_ops *ops: 这也是一个指针，指向 urma_ops 结构体，它定义了与 URMA 设备交互的操作集合，如打开、关闭、读写等。 + +3. int dev_fd: 这是一个整型变量，表示到 URMA 设备 sysfs 文件的文件描述符。sysfs 是 Linux 内核提供的一种接口，允许用户空间程序通过文件系统接口访问内核数据结构，如设备的状态信息。 + +4. int async_fd: 这也是一个整型变量，表示到 URMA 设备异步事件文件的文件描述符。这个文件描述符用于接收设备产生的异步事件通知，比如数据传输完成、错误发生等。 + +5. pthread_mutex_t mutex: 这是一个互斥锁，用于同步对 urma_context_t 结构体中数据的访问。在多线程环境中，互斥锁保证了在任何时候只有一个线程可以修改结构体中的数据，防止数据竞争。 + +6. urma_eid_t eid: 这是一个 urma_eid_t 类型的变量，表示 URMA 设备的全局唯一标识符（eid，Endpoint ID）。在 RoCE（RDMA over Converged Ethernet）网络中，eid 是用来标识网络上的一个端点。 + +7. uint32_t eid_index: 这是一个无符号32位整数，可能用于索引或标识与 eid 相关的额外信息，比如在设备上下文中这个eid的特定位置。 + +8. uint32_t uasid: 这也是一个无符号32位整数，表示当前进程的 User Assisted Segment Identifier (UASID)。 + +9. struct urma_ref ref: 这是一个 urma_ref 结构体的实例，用于跟踪 urma_context_t 的引用计数。 + +### 5.2.2 Jetty管理 + +1. 概述 + +URMA执行资源管理通过Jetty进行管理。Jetty为URMA软件操作对象，借助Jetty UBEP和软件实现消息交互。Jetty主要用于消息语义接收、发送以及内存语义的命令下发。Jetty为进程独享，根据用途的不同Jetty可细分为Jetty、Jetty For Send（JFS）、Jetty For Receive（JFR）、Jetty For Complete（JFC）。 + +1. **Jetty（码头，港口）**：Jetty是URMA中的全功能通信对象，它既支持发送（发送数据到其他Jetty）也支持接收（接收来自其他Jetty的数据）。Jetty对象包含一个发送队列（SQ，Send Queue Buffer）用于提交工作单元（WQE，Work Queue Entry），同时它还关联了一个JFC（Jetty for Complete，完成的Jetty）来管理数据传输的完成状态。Jetty可以独立使用，或者在单向模型中，作为JFS和JFR的组合。 + +2. **JFS（Jetty for Sending）**：JFS是Jetty的一个变体，它专用于发送操作。在单向模型的Initiator（发起者）侧，JFS用于提交DMA任务或者发送消息。JFS仅包含一个SQ，不支持接收操作，只做Send操作或单边UDMA操作。JFS通常与JFR配合使用，节省接收缓冲资源。 + +3. **JFR（Jetty for Receiving）**：JFR是另一个Jetty的变体，专用于接收操作。在单向模型的Target（目标）侧，JFR用于准备接收消息的资源，它包含一个接收队列（RQ，Receive Queue Buffer）。JFR仅做Recv操作，不支持发送。JFR通常与JFS一起工作，提供单向通信的接收端点。 + +4. **JFC（Jetty for Complete）**：JFC是Jetty的辅助实体，它不直接参与数据传输，但对Jetty、JFS和JFR的完成状态进行管理。每个Jetty、JFS或JFR都需要一个JFC来记录数据传输的完成情况。JFC包含一个完成队列（CQ，Complete Queue Buffer），用于poll完成事件（CQE，Complete Queue Entry）。 + + 1. 应用场景 + +在进行具体的read，write，send，receive等操作前需要建立相关的jetty资源，后续的read,write,send,receive等操作都依赖创建的jetty资源。 + +2. 注意事项 + +创建JFC时指定相关的JFCE，才能以中断模式等待完成事件和获取完成记录，JFCE:接收完成事件的通道，内核态JFCE实现为文件，用户态实现为打开的JFCE文件句柄 + +3. 使用说明 + +1\. 使用Jetty实现消息语义的编程框架如下图所示，其中JFC为轮询模式，没有绑定JFCE： + +1. 消息语义使用示例 + +![](figures/urma-arch-ctrl-jetty-01.png) + +![](figures/urma-arch-ctrl-jetty-02.png) + +2.中断模式使用JFCE的编程框架如下图所示： + +2. JFC中断模式编程示例 + +![](figures/urma-arch-ctrl-jetty-03.png) + +### 5.2.3 Segment管理 + +1. 概述 + +Segment是URMA对被内存事务指令访问的内存进行管理和访问的抽象数据结构。Segment是一段连续的UBA地址空间，同时分配物理内存来对应到一个segment。在URMA单边内存访问的编程模型中，Segment是基础的内存管理对象，Target侧创建Segment并注册，Initiator侧申请使用远端的Segment，构造出Target Segment（包含TokenId，ubva等），然后才能继续访问远端内存。 + +**UBVA地址模型** + +2. 应用场景 + +urma语义的内存管理 + +**注册内存registe_seg：允许设备访问进程的一段内存** + +- 权限：本地写、远程读、远程写、远程原子 + +- 与RDMA不同点：无需提前创建PD，需要传入key（可能修改） + +**访问远端内存之前，先导入内存import_seg** + +- 添加Segment访问表项，验证本进程具有访问segment的权限 + +- RDMA应用无需导入mr，直接通过带外通道将server进程的va和rkey交换到到client，client即可使用va和rkey进行rdma操作 + +- URMA每个client注册segment传入不同的派生key + +- 使用bond设备进行import_segment时，seg中的eid不能为空，否则发不到对端 ；使用裸udma设备并不会有此限制且udma并不会对seg中的内容进行检查判空 + + 1. 注意事项 + +\(1\) 本地用户对远端内存读写时，本地buf和远端内存必须提前调用urma_register_seg注册到设备, 不使用必须调用urma_unregister_seg注销。 + +\(2\) 应用使用远端内存读写之前，必须调用urma_import_seg获取target_segment + +\(3\) 注册segment时，如果声明了remote write或者remote atomic权限，那么应用也必须同时声明local write权限，否则注册失败 + +2. 使用说明 + +\(1\) 使用本地内存：申请va，调用urma_register_seg注册segment。 + +\(2\) 释放本地内存：调用urma_unregister_seg注销segment。 + +\(3\) 使用远端内存：urma_import_segment获取targ_segment和mva。 + +\(4\) 释放远端内存：unimport_segment注销segment。 + +```c +typedef struct urma_seg { + urma_ubva_t ubva; /* [Public] ubva of segment. */ + uint64_t len; /* [Public] length of segment. */ + urma_seg_attr_t attr; /* [Public] include: access flag, token policy, cacheability. */ + uint32_t token_id; /* [Private] match token */ +} urma_seg_t; +typedef struct urma_target_seg { + urma_seg_t seg; /* [Private] see urma_seg_t. */ + uint64_t user_ctx; /* [Private] private data of segment */ + uint64_t mva; /* [Public] mapping addr when import remote seg. */ + urma_context_t *urma_ctx; /* [Private] point to urma context. */ + urma_token_id_t *token_id; /* When registering seg, it is a valid address; when importing seg, it is NULL */ + uint64_t handle; +} urma_target_seg_t; +``` + +register与import出的对象为urma_target_seg_t + +基于segment实现内存语义的编程框架如下图所示： + +1. 内存语义编程示例 + +![](figures/urma-arch-ctrl-seg-01.png) + +![](figures/urma-arch-ctrl-seg-02.png) + +![](figures/urma-arch-ctrl-seg-03.png) + +**单端内存访问**： + +- **Read**: + + + +- 用户（Initiator）在Jetty的发送队列（SQ）中发起一次URMA读事务。 + +- udma驱动从SQ队列中读取一个服务队列元素（SQE），解析其中的指令信息。 + +- 用户端的事务处理单元（TP）将读请求封装成报文，发送给目标端（Target）。 + +- 目标端接收到请求后，事务引擎从指定的内存区域读取数据。 + +- 目标端的TP将数据封装成报文，发送回用户端。 + +- 用户端接收数据后，将其放置到mr指定的缓冲区。 + + + +- **Write**: + + + +- 用户在Jetty的发送队列中发起一次URMA写事务。 + +- 用户从SQ读取SQE并解析信息。 + +- 用户根据SQE的信息，从本地内存中获取数据，组装成报文，发送给目标端。 + +- 目标端接收到报文后，事务引擎将数据存储到指定的内存区域。 + +- 目标端执行完操作后，发送事务完成确认（TAACK）给用户端。 + +- 用户端的TP接收到TAACK后，将其封装成报文并发送给用户 + +### 5.2.4 异常事件 + +1. 概述 + +应用发送硬件无法处理的WR，访问超出本端或远端内存的权限，驱动方面cq、sq、rq队列溢出，驱动卸载，驱动elr复位，端口状态异常等情况下，硬件将上报异常事件。 + +应用获取发生的异常类型、具体的异常的对象：异常的上下文、端口、JFS、JFC、JFR等。异常事件处理是通过urma_get_async_event和urma_ack_async_event两个接口来实现的。这两个接口主要用于在用户态处理来自内核态的异步事件通知。 + +![](figures/urma_info.png) + +```c +/* softub currently only report URMA_EVENT_JFC_ERR and URMA_EVENT_JETTY_ERR; other events are not handled in softub;*/ +URMA_EVENT_JFC_ERR, +URMA_EVENT_JFS_ERR, +URMA_EVENT_JFR_ERR, +URMA_EVENT_JFR_LIMIT, /* Jfr Flow Record,over flow */ +URMA_EVENT_JETTY_ERR, +URMA_EVENT_JETTY_LIMIT, /* Jetty Flow Record,over flow */ +URMA_EVENT_JETTY_GRP_ERR, /* Jetty Asynchronous Error Event Reporting */ +URMA_EVENT_PORT_ACTIVE, /* The port status is currently active */ +URMA_EVENT_PORT_DOWN, /* The port status is currently down */ +URMA_EVENT_DEV_FATAL, +URMA_EVENT_EID_CHANGE, /* eid change, HNM and other management roles will be modified */ +URMA_EVENT_ELR_ERR, /* ELR Reset,Entity level error */ +``` + +URMA_EVENT_ELR_DONE /* Entity flush done */ + +2. 应用场景 + +urma异常场景 + +3. 注意事项 + +应用删除某个对象（例如JFS，JFR，JFC，Jetty）之前，如果获得过该对象产生的异常事件时，必须调用确认异常接口（urma_ack_async_event），然后才能删除该对象。 + +4. 使用说明 + +（1）用户调用urma_get_async_event接口获取异常事件； + +（2）用户根据异常事件类型，进行分类处理，例如打印log信息； + +（3）用户调用urma_ack_async_event接口，通知UMDK已经处理完异常； + +1. **urma_get_async_event:** + +* 功能：这个接口用于从URMA的异步事件队列中获取事件。当内核检测到与URMA资源（如Jetty、JFC、JFS等）相关的异常或状态变化时，它会将这些事件记录到异步事件队列中。用户态的驱动程序通过调用urma_get_async_event来轮询这个队列，获取最新的异常事件。 + +* 参数：通常需要一个urma_context_t类型的上下文指针，这个指针是在urma_create_context时创建的，用于与内核进行通信。 + +* 返回值：urma_get_async_event会返回urma_status_t告知用户函数的完成情况，调用成功返回0，其他值表示接口调用失败，参数中的event的指针指向获取的事件。 + +2. **urma_ack_async_event:** + +* 功能：urma_ack_async_event用于向内核确认已经处理了某个异常事件。当用户态驱动程序通过urma_get_async_event获取到一个事件后，处理完相关逻辑，通常需要调用urma_ack_async_event来告诉内核这个事件已经被处理，可以被清理或进一步处理 + +* 参数：通常需要提供之前从urma_get_async_event获取的urma_async_event_t结构体，这样内核可以知道哪个事件已经被确认 + +#### 5.2.4.1 flush jetty + +URMA支持对jetty队列里面的wr进行flush，具体使用分成以下4种场景： + +1. tp error导致jetty切换到suspended状态，有outstanding wr。 + +![](figures/urma-arch-ctrl-event-02.png)（1）有保序需求时，应用才需要flush，目的是根据顺序重试WR + +（2）JFS无序模式，应用从supspend done可以直接到ready，继续发送 + +（3）SO类型WQE由于等待，虽然将wqe拿到TP，但是没有处理，这时上报ERR + +2. tp error导致jetty切换到suspended状态，硬件无outstanding wr，硬件构造suspend done。 + +![](figures/urma-arch-ctrl-event-03.png) + +硬件构造suspend done的场景： + +（1）应用modify jetty到suspend状态完成 + +3. 硬件将jetty切换到error状态，有outstanding wr。 + +![](figures/urma-arch-ctrl-event-04.png) + +必须等待jetty所有tx和rx outstanding rqe都完成，并上报cqe之后，才能上报flush err done，原因如下： + +（1）如果报文指定到jetty，硬件已经从JFR取到rqe，即认为是outstanding rqe； + +（2）如果不等outstanding rqe完成，软件无法安全的释放rqe内存，因为可能硬件还在访问这段内存； + +（3）如果不等outstanding rqe完成，硬件就上报flush done，还有两种后果： （a）软件可能删除jetty，那么rqe完成时，硬件无法上报rx cqe；（b）软件还可能创建新的jetty，恰好jetty id相同，那么硬件将上报错误rx cqe，导致严重错误。 + +4. 硬件或应用将jetty切换到error状态，硬件构造flush_err_done, 硬件无outstanding wr。 + +![](figures/urma-arch-ctrl-event-05.png) + +硬件构造flush_err_done的场景： + +（1）应用modify jetty 到err完成； + +（2）硬件主动将jetty置错，但是没有outstanding wr。 + +### 5.2.5 设备属性 + +UB设备属性包含大致分为三类：只读且不变的设备资源规格、可读可写的设备配置信息、只读且可变的设备端口状态。目前URMA框架通过sysfs文件系统统一呈现，路径如下，这些文件可以直接通过cat、echo等命令进行操作： + +![](figures/urma-arch-ctrl-dev-01.png) + +设备属性的详细情况如下： + +![](figures/urma_info.png) + +UB传输层的设备属性依赖于UB驱动的实现。 + +1. URMA设备属性 + +| 属性 | 可读 | 可写 | 可变 | 备注 | UB | IB | IP | +| --- | --- | --- | --- | --- | --- | --- | --- | +| eid | √ | √ | √ | 设备eid | √ | √ | √ | +| guid | √ | x | x | 设备guid | √ | x | x | +| feature | √ | x | x | 设备支持的feature，包含OOO(out_of_order)、jfc_per_wr、stride_op、load_store_op、non_pin、pmem（persistence_mem）等 | √ | √ | √ | +| max_jfc | √ | x | x | 设备支持创建jfc的最大个数 | √ | √ | √ | +| max_jfs | √ | x | x | 设备支持创建jfs的最大个数 | √ | √ | √ | +| max_jfr | √ | x | x | 设备支持创建jfr的最大个数 | √ | √ | √ | +| max_jfc_depth | √ | x | x | jfc支持配置队列深度的最大值 | √ | √ | √ | +| max_jfs_depth | √ | x | x | jfs支持配置队列深度的最大值 | √ | √ | √ | +| max_jfr_depth | √ | x | x | jfr支持配置队列深度的最大值 | √ | √ | √ | +| max_jfs_inline_len | √ | x | x | jfs支持消息inline的最大值，单位byte | √ | √ | x | +| max_jfs_sge | √ | x | x | jfs支持单个wr里面包含sge的最大个数 | √ | √ | √ | +| max_jfr_sge | √ | x | x | jfr支持单个wr里面包含sge的最大个数 | √ | √ | √ | +| max_msg_size | √ | x | x | 设备支持传输消息的最大值，单位byte | √ | √ | x | +| tp_mode | √ | x | x | 设备传输层模式，枚举值：SRM（shared reliable message）、RC（reliable connection）、UM（unreliable message） | √ | √ | √ | +| port_count | √ | x | x | 设备拥有的port数 | √ | √ | √ | +| max_mtu | √ | x | x | 端口可配置的最大MTU值，枚举值：MTU_256, MTU_512, MTU_1024等 | √ | √ | √ | +| state | √ | x | √ | 端口状态，枚举值：PORT_DOWN, PORT_INIT, PORT_ARMED，PORT_ACTIVE，PORT_ACTIVE_DEFER | √ | √ | √ | +| active_width | √ | x | √ | 端口活跃的链路带宽，枚举值：WIDTH_X1，WIDTH_X2，WIDTH_X4 | √ | √ | x | +| active_speed | √ | x | √ | 端口活跃的速率，枚举值：SP_10M， SP_100M，SP_1G，SP_10G，SP_25G，SP_40G，SP_100G等 | √ | √ | x | +| active_mtu | √ | x | √ | 网卡设备端口活跃的MTU值, 枚举值：MTU_256, MTU_512, MTU_1024等 | √ | √ | √ | + +URMA设备属性可以通过urma_admin工具查询和配置，具体使用方式见工具演示章节 + +### 5.2.6 token安全传输 + +## 5.3 数据面 + +**传输模式-RM、RC、UM** + +**URMA_TM_RC：Reliable Connection** + +- 建立一对一的绑定关系，只能向绑定的jetty发送消息，可以访问目标进程内的segment。 + +- **保证可靠，支持保序** + +- 一个jetty只能与一个目标进程建立连接，发送消息。不支持一对多通信。 + +**URMA_TM_RM：Reliable Message** + +- Jetty之间/jfs与jfr之间建立多个连接关系，可以向多个节点不同目标进程的jetty/jfr发送消息，也可以访问多个节点不同进程的segment。 + +- **保证可靠，支持保序/不保序** + +- JFS到目标进程间只能通过源端保序，时延增大。 + +**URMA_TM_UM：Unreliable Message** + +- Jetty之间/jfs与jfr之间不存在连接关系，可以向多个节点不同目标进程的jetty/jfr发送消息，不支持单边语义。 + +- **不保证可靠和有序** + +- 不具备连接无法实现保序，底层不保证可靠。 + +**可靠与保序介绍** + +**URMA_TM_RC：Reliable Connection** + +- Jetty之间由唯一TP连接，顺序发出的WR在目的端被顺序执行。 + +- 原生支持FENCE保序，执行序，完成序。 + +- 底层对数据进行ACK和失败重传，保证可靠性。 + +**URMA_TM_UM：Unreliable Message** + +- 同一个jetty/jfs到目的进程不创建连接。 + +- 不保序。 + +- 底层使用不保证可靠性。 + +**URMA_TM_RM：Reliable Message** + +- 应用角度一个jetty/jfs可以和多个远端jetty/jfr通信，反映为无连接。 + +- 基于XRC的实现 + + + +- 同一个jetty/jfs到目的进程只创建一个jetty连接，顺序发出的WR在目的端被顺序执行。 + +- 原生支持FENCE保序，执行序，完成序。 + + + +- 基于RC的实现：不支持执行序，完成序 + + + +- 同一个jetty/jfs到目的进程创建多个QP连接，无法保证WR被顺序执行。 + +不同传输模式如下图所示： + +![](figures/urma-arch-data-01.png) + +单边、双边操作对连续/非连续操作地址的支持如下表格所示： + +1. + +![](figures/urma-arch-data-02.png) + +![](figures/urma_info.png) + +URMA over IP 发送报文最大支持1G，即1个WR里面所有sge长度相加之和最大不超过1G。 + +### 5.3.1 单边操作 + +1. 概述 + +UMDK单边操作提供了read write语义，类似于IB的read/write接口，需要知道本地的地址和对端的地址，进行单边操作时只有本端进程在操作，不需要对端的应用感知。 + +UMDK单边操作缓存支持本端连续内存、非连续内存和远端连续内存。urma_read，urma_write只支持连续地址的读写。urma_post_jfs_wr支持本端以sgl的形式访问非连续地址。 + +UMDK支持立即数的写操作，见urma_post_jfs_wr接口，所写的立即数将出现在接收端的完成记录（completion record）中。 + +根据UB协议，write和read操作只支持一个远端sge。因此对于write操作，dst.num_sge必须为1，对于read操作，src.num_sge必须为1。超出部分sge网卡将忽略。 + +单边操作的write操作中，需要设置urma_jfs_wr_flag中的fence标志开启是否保序，单边操作最大值可以在环境上通过cat /sys/class/ubcore/udma1/max_write_size查看 + +![](figures/urma-arch-data-one-sided-01.png) + +2. 应用场景 + +UMDK单边操作不需要对端的CPU参与，不同于双边操作send/recv一般用于传输一些控制信息，单边操作read/write适用于大规模的数据传输。 + +3. 注意事项 + +（1）用户发送和接收的本地缓存必须事先调用urma_register_seg注册到设备 + +（2）对于IB传输层，JFS向某个JFR发送消息之前，必须调用urma_advise_jfr通知UMDK建立从JFS到JFR的传输通道。UB JFS天然具有一对多通信能力，发送消息之前无需调用urma_advise_jfr这个步骤。 + +（3）不同的传输层最大发送消息大小有所不同，可以通过查询设备属性获取发送消息的规格。 + +（4）上述单边、双边和原子操作都是非阻塞的，操作返回成功仅表示命令已经添加到发送或者接受队列，并不意味着已经全部完成。UMDK支持以轮询或中断方式获知单边、双边或原子操作是否已经完成。完成记录（completion record）用来描述操作完成信息。操作完成后，硬件会将完成记录写到JFC完成队列中。当用户轮询JFC时，UMDK读取完成队列的完成记录返回给用户。单边、双边、原子等操作的完成记录将默认写入JFS或JFR所关联的JFC中。UB设备支持在JFS command（即WQE）中指定完成记录待写入的JFC id。 + +![](figures/urma_caution.png) + +URMA的单边语义报文参考业界RDMA通用实现，存在与RDMA类似的安全风险，受限于数据中心信任网络内使用。 + +4. 使用说明 + +UMDK单边读/写的过程为： + +1 调用urma_read，urma_write或urma_post_jfs_wr提交一个读或写的请求至先前注册好的jfs。 + +2 调用urma_poll_jfc进行轮询，查看jfc中是否有cqe到来，当urma_poll_jfc返回值大于0时，即表示轮询到有cqe,表示此次读操作完成。请求完成后，用户才能重新使用（修改或释放）发送消息缓存 + +- **urma_post_jetty_send_wr**：这个函数用于发起单边操作的请求，比如写入远程内存。函数参数包括jetty（命令执行的端口）、wr（包含源地址、目的地址、长度等信息的发送请求）和bad_wr（用于存储发送失败的wr）。如果操作成功，函数返回0，否则返回错误代码。 + +- **urma_read**：这个函数允许应用程序从远程内存中读取数据，同样不需要远程进程的参与。 + +- **urma_write**：这个函数允许应用程序向远程内存写入数据，同样不需要远程进程的响应。注意，URMA的单边写操作不支持notify远端，但支持携带IMM（Immediate）数据，这是一种可以附加到消息中的小块数据，用于传递额外的信息。 + +- ![](figures/urma-arch-data-one-sided-03.png) + +### 5.3.2 双边操作 + +1. 概述 + +消息语义提供了双边Messaging服务，类似于UDP/TCP socket接口或IB的send/receive接口。UMDK的消息语义是异步非阻塞的，消息接收端需要显示地接收消息，接收完成后读取消息继续其他处理。 + +UMDK支持一对多消息语义：从同一个JFS向不同的JFR发送消息，这些JFR可能位于不同的远端节点或进程。 + +UMDK支持以inline方式发送消息，当消息小于UMDK inline阈值时，将UMDK将自动以inline方式发送消息，减少DMA开销以提高发送性能。 + +UMDK双边操作对本端和远端均支持连续内存和非连续内存。urma_send与urma_recv只支持连续地址。urma_post_jfs_wr和urma_post_jfr_wr支持本端和远端使用连续地址或sgl类型的非连续地址。 + +UMDK支持向接收端发送立即数，见urma_post_jfs_wr接口，所发送的立即数将出现在接收端的完成记录（completion record）中。 + +2. 应用场景 + +消息语义应用广泛，例如实现MPI send、recv消息发送，RPC语义，实现UCX的am消息语义等 + +3. 注意事项 + +（1）用户发送和接收的本地缓存必须事先调用urma_register_seg注册到设备 + +（2）对于IB传输层，JFS向某个JFR发送消息之前，必须调用urma_advise_jfr通知UMDK建立从JFS到JFR的传输通道。UB JFS天然具有一对多通信能力，发送消息之前无需调用urma_advise_jfr这个步骤。 + +（3）不同的传输层最大发送消息大小有所不同，可以通过查询设备属性获取发送消息的规格。 + +4. 使用说明 + +接收消息过程为 + +（1）调用urma recv或urma_post_jfr_wr提交一个接收请求，将本地接收缓存添加到jfr中 + +（2）调用urma_poll_jfc轮询接收请求，请求完成后，用户才能从接收缓存中读取消息内容 + +为了提高吞吐量服务器端可以批量提交多个接收请求。每成功接收到一个消息后，向JFR补充新的接收请求。或者当JFR的接收请求数低于某个阈值时，向JFR补充新的接收请求。 + +接收端通过完成记录中的接收长度获得具体收到的有效消息长度，也通过完成记录获知发送端是否发送了立即数。 + +发送消息过程为： + +（1）用户调用urma send或urma_post_jfs_wr通过JFS提交一个发送请求， + +（2）调用urma_poll_jfc轮询接收请求，请求完成后，用户才能重新使用（修改或释放）发送消息缓存 + +- **urma_post_jetty_send_wr**：在双边操作中，这个函数同样用于发起请求，但发送的数据可能会被远程进程接收和处理。 + +- **urma_recv**：接收方使用这个函数从远程内存接收数据。 + +- **urma_send**：发送方使用这个函数向远程内存发送数据，支持携带IMM数据，并且可以设置为with invalid，这意味着即使目标地址无效，操作也会继续执行。![](figures/urma-arch-data-two-sided-01.png) + +### 5.3.3 完成记录 + +1. 概述 + +上述单边、双边和原子操作都是非阻塞的，操作返回成功仅表示命令已经添加到发送或者接受队列，并不意味着已经全部完成。UMDK支持以轮询或中断方式获知单边、双边或原子操作是否已经完成。完成记录（completion record）用来描述操作完成信息。操作完成后，硬件会将完成记录写到JFC完成队列中。当用户轮询JFC时，UMDK读取完成队列的完成记录返回给用户。 + +单边、双边、原子等操作的完成记录将默认写入JFS或JFR所关联的JFC中。UB设备支持在JFS command（即WQE）中指定完成记录待写入的JFC id。 + +2. 应用场景 + +轮询方式应用于低时延场景，用户通过不断查询完成记录，获取操作的执行状态以进行下一步操作，不断轮询操作将提高CPU占用率。中断方式应用于通信不太频繁的场景，用户线程以睡眠状态等待完成事件，CPU开销小，当完成事件发生时，UMDK将唤醒等待的线程。 + +3. 注意事项 + +（1）用户调用urma_recv时，接收完成时总会产生一个完成记录。用户调用urma_read/write/cas/fao/send时，默认操作将会产生一个完成记录；如果JFC处于事件使能状态（armed）也默认将产生一个完成事件。 + +（2）如果用户使用urma_post_jfs_wr批量发送请求时，用户可以指定是否产生完成记录或者完成事件； + +（3）用户提交操作（包括单边、双边、原子等）时，需要自行保证完成记录待写入的JFC不会溢出 + +（4）如果JFC中尚有未读取的完成记录，那么urma_rearm_jfc将返回失败 + +（5）urma_modify_jetty/jfs切换至ERROR/SUSPEND状态会产生完成记录，需要自行保证完成记录待写入的JFC不会溢出 + +4. 使用说明 + +中断模式等待完成事件的流程如下： + +（1）调用urma_rearm_jfc使能完成事件； + +（2）提交JFS操作（包括单边、双边、原子等），指定需要完成记录和完成时间 + +（3）调用urma_wait_jfc阻塞等待一个完成事件，返回产生完成事件的JFC；UMDK将默认去使能JFC完成事件 + +（4）判断返回的JFC和提交JFS操作所有的JFC相符合 + +（5）循环调用urma_poll_jfc读取完成记录，直到没有新的完成记录为止 + +（6）回到步骤（1）重新开启事件 + +![](figures/urma-arch-ctrl-jetty-03.png) + +轮询方式poll到完成事件的流程如下： + +用户调用urma_poll_jfc以轮询方式查询完成记录，轮询是一种非阻塞的查询完成记录的方式，如果完成队列为空，则用户获取不到完成记录。完成记录的使用说明如下： + +（1）用户通过完成记录的状态字段得知操作是否成功完成，如果出错，完成记录的状态字段反应出操作出错的原因； + +（2）完成长度表示已经成功执行的数据长度，例如发送长度或接收消息长度 + +（3）如果完成记录为JFS类型，则用户可以修改或释放操作对应的本地缓存 + +（4）如果完成记录为JFR类型，表示用户可以从接收缓存中读取消息； + +（5）如果notify_data标志位使能，则完成记录中还携带了立即数 + +（6）用户通过完成记录的completion_record_data中的操作上下文（例如urma_read api中的user_ctx参数），关联到具体某个操作 + +**中断接口推荐顺序：** + +![](figures/urma-arch-data-comp-01.png) + +5. 触发场景 + +以下场景将触发JFC生成一条完成记录： + +1、在执行常规数据面操作过程中，若遇到不支持的操作类型、消息长度超出系统允许上限、请求格式不符合规范要求，或本端及对端访问的内存资源已被注销等异常情况，在提交工作请求的操作成功前提下，会生成一条完成记录。 + +2、Flush 操作正常完成，未发生任何异常或错误 + +3、Jetty/JFS 成功切换至 SUSPEND 状态 + +4、Jetty/JFS 成功切换至 ERROR 状态 + +--- +# 6 关键特性介绍 + +## 6.1 特性树 + +#### 管理面接口 + +| Feature L1 | Feature L2 | Feature L3 | Feature L4 | Feature L5 | Feature Description | KP950 | Ascend910D | +|---|---|---|---|---|---|---|---| +| 管理面接口 | URMA初始化 | init/uninit | 基本能力 | — | 初始化/反初始化URMA运行环境 | √ | √ | +| 管理面接口 | URMA初始化 | init/uninit | token | — | 指定安全token | × | × | +| 管理面接口 | URMA初始化 | init/uninit | uasid | — | 指定进程的uasid | × | × | +| 管理面接口 | URMA设备及上下文管理 | get/free_device_list | — | — | 获取/释放设备列表 | √ | √ | +| 管理面接口 | URMA设备及上下文管理 | get_device_by_name | — | — | 通过设备名称获取设备 | √ | √ | +| 管理面接口 | URMA设备及上下文管理 | get_device_by_eid | — | — | 通过EID获取设备 | √ | √ | +| 管理面接口 | URMA设备及上下文管理 | query_device | — | — | 查询设备属性 | √ | √ | +| 管理面接口 | URMA设备及上下文管理 | create/delete_context | — | — | 创建/删除设备上下文 | √ | √ | +| 管理面接口 | JFC基础能力管理 | create/delete_jfc | 基本能力 | — | 创建/删除JFC | √ | √ | +| 管理面接口 | JFC基础能力管理 | create/delete_jfc | cfg.flag.jfc_inline | — | 支持inline配置 | √ | √ | +| 管理面接口 | JFC基础能力管理 | delete_jfc_batch | — | — | 批量删除JFC | × | × | +| 管理面接口 | JFC基础能力管理 | modify_jfc | moderate_count/moderate_period | — | 修改JFC中断抑制参数 | × | × | +| 管理面接口 | JFS基础能力管理 | create/delete_jfs | 基本能力 | — | 创建/删除JFS | √ | √ | +| 管理面接口 | JFS基础能力管理 | create/delete_jfs | lock_free | — | 免锁模式 | √ | √ | +| 管理面接口 | JFS基础能力管理 | create/delete_jfs | error_suspend | — | 数据面异常置错 | √ | √ | +| 管理面接口 | JFS基础能力管理 | create/delete_jfs | outorder_comp | — | 乱序上报完成事件 | √ | √ | +| 管理面接口 | JFS基础能力管理 | create/delete_jfs | order_type | OT | target ordering 目的端保序 | √ | √ | +| 管理面接口 | JFS基础能力管理 | create/delete_jfs | order_type | OI | initiator ordering 源端保序 | √ | √ | +| 管理面接口 | JFS基础能力管理 | create/delete_jfs | order_type | OL | low layer ordering 通道保序 | √ | √ | +| 管理面接口 | JFS基础能力管理 | create/delete_jfs | order_type | UNO | unreliable non ordering 无序 | √ | √ | +| 管理面接口 | JFS基础能力管理 | create/delete_jfs | multi_path | — | 设备多路径能力 | √ | √ | +| 管理面接口 | JFS基础能力管理 | create/delete_jfs | ctp_rc_mul_path_mode | — | RC模式CTP多路径能力 | √ | √ | +| 管理面接口 | JFS基础能力管理 | create/delete_jfs | trans_mode | RM | 无连接可靠传输模式 | √ | √ | +| 管理面接口 | JFS基础能力管理 | create/delete_jfs | trans_mode | RC | 连接可靠传输模式 | √ | √ | +| 管理面接口 | JFS基础能力管理 | create/delete_jfs | trans_mode | UM | 无连接不可靠传输模式 | √ | √ | +| 管理面接口 | JFS基础能力管理 | create/delete_jfs | priority | — | 配置优先级 | √ | √ | +| 管理面接口 | JFS基础能力管理 | create/delete_jfs | max_inline_data | — | 配置inline大小 | √ | √ | +| 管理面接口 | JFS基础能力管理 | create/delete_jfs | rnr_retry | — | 配置rnr重试次数 | √ | √ | +| 管理面接口 | JFS基础能力管理 | create/delete_jfs | err_timeout | — | 配置错误超时时间 | √ | √ | +| 管理面接口 | JFS基础能力管理 | delete_jfs_batch | — | — | 批量删除JFS | √ | √ | +| 管理面接口 | JFS基础能力管理 | modify_jfs | state | — | 修改JFS的状态机 | √ | √ | +| 管理面接口 | JFR基础能力管理 | create/delete_jfr | 基本能力 | — | 创建/删除JFR | √ | √ | +| 管理面接口 | JFR基础能力管理 | create/delete_jfr | 指定jfr_id | — | 指定JFR ID | √ | √ | +| 管理面接口 | JFR基础能力管理 | create/delete_jfr | token_policy | NONE | 不携带TokenValue | √ | √ | +| 管理面接口 | JFR基础能力管理 | create/delete_jfr | token_policy | PLAIN_TEXT | 传输TokenValue明文 | √ | √ | +| 管理面接口 | JFR基础能力管理 | create/delete_jfr | token_policy | SIGNED | 加密TokenValue，明文传输PLD | √ | √ | +| 管理面接口 | JFR基础能力管理 | create/delete_jfr | token_policy | ALL_ENCRYPTED | 加密TokenValue和PLD | × | × | +| 管理面接口 | JFR基础能力管理 | create/delete_jfr | lock_free | — | 免锁模式 | √ | √ | +| 管理面接口 | JFR基础能力管理 | create/delete_jfr | order_type | OT | target ordering 目的端保序 | √ | √ | +| 管理面接口 | JFR基础能力管理 | create/delete_jfr | order_type | OI | initiator ordering 源端保序 | √ | √ | +| 管理面接口 | JFR基础能力管理 | create/delete_jfr | order_type | OL | low layer ordering 通道保序 | √ | √ | +| 管理面接口 | JFR基础能力管理 | create/delete_jfr | order_type | UNO | unreliable non ordering 无序 | √ | √ | +| 管理面接口 | JFR基础能力管理 | create/delete_jfr | trans_mode | RM | 无连接可靠传输模式 | √ | √ | +| 管理面接口 | JFR基础能力管理 | create/delete_jfr | trans_mode | RC | 连接可靠传输模式 | √ | √ | +| 管理面接口 | JFR基础能力管理 | create/delete_jfr | trans_mode | UM | 无连接不可靠传输模式 | √ | √ | +| 管理面接口 | JFR基础能力管理 | create/delete_jfr | min_rnr_timer | — | 配置rnr最小重试时间 | √ | √ | +| 管理面接口 | JFR基础能力管理 | delete_jfr_batch | — | — | 批量删除JFR | √ | √ | +| 管理面接口 | JFR基础能力管理 | modify_jfr | rx_threshold | — | 修改RX WQ的最低水位 | √ | √ | +| 管理面接口 | JFR基础能力管理 | modify_jfr | state | — | 修改JFR的状态机 | √ | √ | +| 管理面接口 | JFR基础能力管理 | import/unimport_jfr | — | — | 导入/反导入远端JFR | √ | √ | +| 管理面接口 | JFR基础能力管理 | import_jfr_async | — | — | 异步导入源端JFR | √ | √ | +| 管理面接口 | jetty基础能力管理 | create/delete_jetty | 基本能力 | — | 创建/删除Jetty | √ | √ | +| 管理面接口 | jetty基础能力管理 | create/delete_jetty | jetty_cfg | — | 配置特性与JFS&JFR一致 | √ | √ | +| 管理面接口 | jetty基础能力管理 | create/delete_jetty | jetty_id | — | 指定jetty id | √ | √ | +| 管理面接口 | jetty基础能力管理 | create/delete_jetty | share_jfr | — | 配置共享JFR | √ | √ | +| 管理面接口 | jetty基础能力管理 | create/delete_jetty | jetty_grp | — | 配置Jetty Group | × | × | +| 管理面接口 | jetty基础能力管理 | delete_jetty_batch | — | — | 批量删除Jetty | √ | √ | +| 管理面接口 | jetty基础能力管理 | modify_jetty | state | — | 修改Jetty状态 | √ | √ | +| 管理面接口 | jetty基础能力管理 | modify_jetty | rx_threshold | — | 修改RX WQ的最低水位 | × | × | +| 管理面接口 | jetty基础能力管理 | import/unimport_jetty | 基本能力 | — | 导入/反导入远端jetty | √ | √ | +| 管理面接口 | jetty基础能力管理 | import/unimport_jetty | tp_type | RTP | 可靠传输模式 | √ | √ | +| 管理面接口 | jetty基础能力管理 | import/unimport_jetty | tp_type | CTP | 轻量级传输模式 | √ | √ | +| 管理面接口 | jetty基础能力管理 | import/unimport_jetty | tp_type | UTP | 不可靠传输模式 | √ | √ | +| 管理面接口 | jetty基础能力管理 | bind/unbind_jetty | — | — | bind/unbind远端jetty | √ | √ | +| 管理面接口 | jetty基础能力管理 | import_jetty_async | — | — | 异步导入远端Jetty | × | × | +| 管理面接口 | jetty基础能力管理 | bind_jetty_async | — | — | 异步bind源端Jetty | × | × | +| 管理面接口 | JFCE基础能力管理 | create/delete_jfce | — | — | 创建/删除JFCE | √ | √ | +| 管理面接口 | 异步事件上报 | get_async_event | 查询jetty异常 | — | 查询jetty异步异常 | √ | √ | +| 管理面接口 | 异步事件上报 | get_async_event | port异常 | — | 端口异步异常 | √ | √ | +| 管理面接口 | 异步事件上报 | get_async_event | 设备异常 | — | 设备异步异常 | √ | √ | +| 管理面接口 | 异步事件上报 | get_async_event | Entity异常 | — | Entity异步异常 | √ | √ | +| 管理面接口 | 异步事件上报 | ack_async_event | — | — | 用户响应异步事件 | √ | √ | +| 管理面接口 | segment管理 | register/unregister_seg | 基本功能 | — | 注册/反注册内存 | √ | √ | +| 管理面接口 | segment管理 | register/unregister_seg | token_policy | — | 指定key验证策略 | √ | √ | +| 管理面接口 | segment管理 | register/unregister_seg | cacheable | — | 是否使能缓存 | √ | √ | +| 管理面接口 | segment管理 | register/unregister_seg | access | LOCAL_ONLY | 仅本地访问 | √ | √ | +| 管理面接口 | segment管理 | register/unregister_seg | access | ACCESS_READ | 远程读访问 | √ | √ | +| 管理面接口 | segment管理 | register/unregister_seg | access | ACCESS_WRITE | 远程写访问 | √ | √ | +| 管理面接口 | segment管理 | register/unregister_seg | access | ACCESS_ATOMIC | 远程原子访问 | √ | √ | +| 管理面接口 | segment管理 | register/unregister_seg | non_pin | — | 是否支持no_pin住内存 | × | × | +| 管理面接口 | segment管理 | register/unregister_seg | user_iova | — | 是否使用iova | × | × | +| 管理面接口 | segment管理 | register/unregister_seg | token_id_valid | — | 是否指定token_id | √ | √ | +| 管理面接口 | segment管理 | import/unimport_seg | 基本功能 | — | 导入/导出远端内存 | √ | √ | +| 管理面接口 | segment管理 | import/unimport_seg | cacheable | — | 是否使能缓存 | √ | √ | +| 管理面接口 | segment管理 | import/unimport_seg | mapping | — | 是否映射到本地地址 | √ | √ | +| 管理面接口 | JFC拓展能力管理 | alloc/free_jfc | — | — | 申请/释放JFC内存 | √ | √ | +| 管理面接口 | JFC拓展能力管理 | active/deactive_jfc | — | — | 硬件使能/失效JFC | √ | √ | +| 管理面接口 | JFC拓展能力管理 | set/get_jfc_opt | 基本能力 | — | 设置/获取JFC属性 | √ | √ | +| 管理面接口 | JFC拓展能力管理 | set/get_jfc_opt | 基本属性 | — | 创建JFC的基本属性 | √ | √ | +| 管理面接口 | JFC拓展能力管理 | set/get_jfc_opt | CQE_BASE_ADDR | — | CQE的地址 | √ | √ | +| 管理面接口 | JFC拓展能力管理 | set/get_jfc_opt | JFC_ID | — | JFC的ID | √ | √ | +| 管理面接口 | JFC拓展能力管理 | set/get_jfc_opt | JFC_DB_ADDR | — | CQ队列Doorbell地址 | √ | √ | +| 管理面接口 | JFC拓展能力管理 | set/get_jfc_opt | JFC_DB_STATUS | — | JFC Doorbell的状态 | × | × | +| 管理面接口 | JFC拓展能力管理 | set/get_jfc_opt | JFC_PI | — | JFC任务队列的PI值 | √ | √ | +| 管理面接口 | JFC拓展能力管理 | set/get_jfc_opt | JFC_PI_TYPE | — | JFC队列PI类型 | √ | √ | +| 管理面接口 | JFC拓展能力管理 | set/get_jfc_opt | JFC_CI | — | JFC任务队列的CI值 | √ | √ | +| 管理面接口 | JFS拓展能力管理 | alloc/free_jfs | — | — | 申请/释放JFS内存 | √ | √ | +| 管理面接口 | JFS拓展能力管理 | active/deactive_jfs | — | — | 硬件使能/失效JFS | √ | √ | +| 管理面接口 | JFS拓展能力管理 | set/get_jfs_opt | 基本能力 | — | 设置/获取JFS属性 | √ | √ | +| 管理面接口 | JFR拓展能力管理 | alloc/free_jfr | — | — | 申请/释放JFR内存 | √ | √ | +| 管理面接口 | JFR拓展能力管理 | active/deactive_jfr | — | — | 硬件使能/失效JFR | √ | √ | +| 管理面接口 | JFR拓展能力管理 | set/get_jfr_opt | 基本能力 | — | 设置/获取JFR属性 | √ | √ | +| 管理面接口 | Jetty拓展能力管理 | alloc/free_jetty | — | — | 申请/释放JETTY内存 | √ | √ | +| 管理面接口 | Jetty拓展能力管理 | active/deactive_jetty | — | — | 硬件使能/失效JETTY | √ | √ | +| 管理面接口 | Jetty拓展能力管理 | set/get_jetty_opt | 基本能力 | — | 设置/获取JETTY属性 | √ | √ | +| 管理面接口 | 感知TP建链 | get_tp_list | — | — | 静态查询可用TP | √ | √ | +| 管理面接口 | 感知TP建链 | get/set_tp_attr | — | — | 获取/设置TP属性 | √ | √ | +| 管理面接口 | 感知TP建链 | import_jetty_ex | — | — | 指定TP建链 | √ | √ | +| 管理面接口 | 设备聚合管理 | 基础能力 | — | — | 使用聚合设备创建上下文 | √ | √ | +| 管理面接口 | 设备聚合管理 | 配置拓扑信息 | — | — | 配置设备聚合拓扑信息 | √ | √ | +| 管理面接口 | IP over URMA | 基础能力 | — | — | 呈现TCP/IP协议栈设备，对接URMA | √ | √ | +| 管理面接口 | Verbs over URMA | 基础能力 | — | — | 呈现RDMA协议栈设备，对接URMA | √ | √ | + +#### 数据面接口 + +| Feature L1 | Feature L2 | Feature L3 | Feature L4 | Feature L5 | Feature Description | KP950 | Ascend910D | +|---|---|---|---|---|---|---|---| +| 数据面接口 | post操作 | post_jfs_wr | — | — | 提交JFS工作请求 | √ | √ | +| 数据面接口 | post操作 | post_jfr_wr | — | — | 提交JFR工作请求 | √ | √ | +| 数据面接口 | post操作 | post_jetty_send_wr | — | — | 提交Jetty发送工作请求 | √ | √ | +| 数据面接口 | post操作 | post_jetty_recv_wr | — | — | 提交Jetty接收工作请求 | √ | √ | +| 数据面接口 | 发送配置 | 执行序 | none | — | 无序 | √ | √ | +| 数据面接口 | 发送配置 | 执行序 | RO | — | 弱保序 | √ | √ | +| 数据面接口 | 发送配置 | 执行序 | SO | — | 强保序 | √ | √ | +| 数据面接口 | 发送配置 | 完成序 | — | — | 是否保序 | √ | √ | +| 数据面接口 | 发送配置 | fence操作 | — | — | Fence操作 | √ | √ | +| 数据面接口 | 发送配置 | solicited使能 | — | — | Solicited使能 | √ | √ | +| 数据面接口 | 发送配置 | 配置完成事件产生 | — | — | 配置完成事件产生 | √ | √ | +| 数据面接口 | 发送配置 | 配置是否inline | — | — | 配置是否inline | √ | √ | +| 数据面接口 | 发送配置 | 支持免import seg | — | — | 支持免import seg | todo | todo | +| 数据面接口 | 内存连续 | sgl配置 | 远端send/recv支持不连续，本端read/write/send/recv支持不连续 | — | SGL支持 | √ | √ | +| 数据面接口 | 内存连续 | wr_list配置 | — | — | WR列表配置 | √ | √ | +| 数据面接口 | 单边操作 | read | 仅支持1个src sge | — | Read操作 | √ | √ | +| 数据面接口 | 单边操作 | write | 仅支持1个dst sge | — | Write操作 | √ | √ | +| 数据面接口 | 单边操作 | write | 支持notify远端 | — | 支持notify远端 | √ | √ | +| 数据面接口 | 单边操作 | write | 支持携带IMM数据 | — | 支持携带IMM数据 | √ | √ | +| 数据面接口 | 单边操作 | write | 支持write_with_atomic_add | — | Write with atomic add | √ | √ | +| 数据面接口 | 双边操作 | send | — | — | Send操作 | √ | √ | +| 数据面接口 | 双边操作 | send | 支持携带IMM数据 | — | 支持携带IMM数据 | √ | √ | +| 数据面接口 | 双边操作 | send | with invalid | — | Send with invalid | √ | √ | +| 数据面接口 | 双边操作 | recv | — | — | Receive操作 | √ | √ | +| 数据面接口 | 原子操作 | cas | — | — | Compare and swap | × | × | +| 数据面接口 | 原子操作 | cas | 支持mask | — | CAS with mask | × | × | +| 数据面接口 | 原子操作 | faa | — | — | Fetch and add | × | × | +| 数据面接口 | 原子操作 | faa | 支持mask | — | FAA with mask | × | × | +| 数据面接口 | 完成操作 | poll_jfc | 支持产生多个CR | — | Poll JFC获取完成记录 | √ | √ | +| 数据面接口 | 完成操作 | poll_jfc | CR解析 | status | 携带完成状态 | √ | √ | +| 数据面接口 | 完成操作 | poll_jfc | CR解析 | flag.s_r | 携带是接收还是发送完成 | √ | √ | +| 数据面接口 | 完成操作 | poll_jfc | CR解析 | opcode | 接收端CR携带发送端操作类型 | √ | √ | +| 数据面接口 | 完成操作 | poll_jfc | CR解析 | user_ctx | 携带user_ctx私有数据 | √ | √ | +| 数据面接口 | 完成操作 | poll_jfc | CR解析 | completion_len | 携带完成长度 | √ | √ | +| 数据面接口 | 完成操作 | poll_jfc | CR解析 | local_id | 携带本端jetty_id | √ | √ | +| 数据面接口 | 完成操作 | poll_jfc | CR解析 | remote_id | 接收端携带远端jetty信息 | √ | √ | +| 数据面接口 | 完成操作 | poll_jfc | CR解析 | imm_data | 携带IMM数据 | √ | √ | +| 数据面接口 | 完成操作 | rearm_jfc | — | — | Rearm JFC | √ | √ | +| 数据面接口 | 完成操作 | rearm_jfc | solicited_only | — | 仅对solicited报文产生事件 | √ | √ | +| 数据面接口 | 完成操作 | wait_jfc | — | — | 等待JFCE的多个JFC产生事件 | √ | √ | +| 数据面接口 | 完成操作 | ack_jfc | — | — | 确认事件处理完毕 | √ | √ | + +#### DFX接口 + +| Feature L1 | Feature L2 | Feature L3 | Feature L4 | Feature L5 | Feature Description | KP950 | Ascend910D | +|---|---|---|---|---|---|---|---| +| DFX接口 | urma_admin配置 | eid | — | — | 配置EID | × | × | +| DFX接口 | urma_admin配置 | upi | PF配置 | — | PF UPI配置 | × | × | +| DFX接口 | urma_admin配置 | upi | VF配置 | — | VF UPI配置 | × | × | +| DFX接口 | urma_admin查询 | 设备列表 | — | — | 查询所有设备基本信息 | — | — | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | feature | 硬件特性 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | max_jfc | 硬件支持创建jfc个数上限 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | max_jfs | 硬件支持创建jfs个数上限 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | max_jfr | 硬件支持创建jfr数上限 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | max_jetty | 硬件支持创建jetty个数上限 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | max_jetty_in_jetty_grp | 硬件支持创建jetty group个数上限 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | jfc_depth | 硬件支持创建jfc最大深度 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | jfs_depth | 硬件支持创建jfs最大深度 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | jfr_depth | 硬件支持创建jfr最大深度 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | max_jfs_inline_size | 硬件支持最大发送inline大小 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | max_jfs_sge | 硬件支持jfs中最大seg个数 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | max_jfs_rsge | 硬件支持jfs中最大rseg个数 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | max_jfr_sge | 硬件支持jfr中最大seg个数 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | max_msg_size | 硬件支持最大messages大小 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | max_read_size | 硬件支持read语义内容上限 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | max_write_size | 硬件支持write语义内容上限 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | max_cas_size | 硬件支持cas语义内容上限 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | max_swap_size | 硬件支持swap语义内容上限 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | max_fetch_and_add_size | 硬件支持fetch_and_add语义内容上限 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | max_fetch_and_sub_size | 硬件支持fetch_and_sub语义内容上限 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | max_fetch_and_and_size | 硬件支持fetch_and_and语义内容上限 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | max_fetch_and_or_size | 硬件支持fetch_and_or语义内容上限 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | max_fetch_and_xor_size | 硬件支持fetch_and_xor语义内容上限 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | atomic_feat | 原子操作能力 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | trans_mode | 传输模式 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | congestion_ctrl_alg | 拥塞控制算法 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | ceq_cnt | CEQ数量 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | max_tp_in_tpg | TP组最大TP数 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | max_eid_cnt | 最大EID数量 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | page_size_cap | 页大小能力 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | max_oor_cnt | 最大乱序窗口大小 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | mn | 制造商名称 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | max_netaddr_cnt | 最大网络地址数量 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | rm_order_cap | RM保序能力 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | rc_order_cap | RC保序能力 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | rm_tp_cap | RM TP能力 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | rc_tp_cap | RC TP能力 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | um_tp_cap | UM TP能力 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | tp_feature | TP特性 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | device_cap | priority_info | 优先级信息 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | port_cnt | — | 端口数量 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | port_attr | max_mtu | 最大MTU | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | port_attr | state | 端口状态 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | port_attr | active_width | 活跃链路宽度 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | port_attr | active_speed | 活跃速率 | √ | √ | +| DFX接口 | urma_admin查询 | 设备属性 | port_attr | active_mtu | 活跃MTU | √ | √ | +| DFX接口 | urma_admin查询 | upi | — | — | UPI查询 | — | — | +| DFX接口 | urma_perftest | perftest测试类型配置 | send_lat | — | 发送时延测试 | √ | √ | +| DFX接口 | urma_perftest | perftest测试类型配置 | read_lat | — | 读时延测试 | √ | √ | +| DFX接口 | urma_perftest | perftest测试类型配置 | write_lat | — | 写时延测试 | √ | √ | +| DFX接口 | urma_perftest | perftest测试类型配置 | atomic_lat | — | 原子操作时延测试 | √ | √ | +| DFX接口 | urma_perftest | perftest测试类型配置 | send_bw | — | 发送带宽测试 | √ | √ | +| DFX接口 | urma_perftest | perftest测试类型配置 | read_bw | — | 读带宽测试 | √ | √ | +| DFX接口 | urma_perftest | perftest测试类型配置 | write_bw | — | 写带宽测试 | √ | √ | +| DFX接口 | urma_perftest | perftest测试类型配置 | atomic_bw | — | 原子操作带宽测试 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | all | — | 运行2~2^15范围内2整数次幂递增长度 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | atomic_type | — | 选择原子操作类型：cas/faa | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | simplex_mode | — | 选择单工模式jfs/jfr运行 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | bidirectional | — | 选择双向模式运行 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | jfc_depth | — | 配置jfc深度 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | dev | — | 指定设备名称 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | duration | — | 指定运行时长 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | use_jfce | — | 选择中断通知方式 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | eid_idx | — | 指定EID index | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | err_timeout | — | 指定异常上报时间 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | user_flat_api | — | 使用flat类型API运行 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | cpu_freq_f | — | CPU频率差异告警 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | help | — | 帮助模式 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | inline_size | — | 配置inline值 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | share_jfr | — | 创建jetty时指定共享jfr | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | jettys | — | 配置jetty/jfs/jfr数量 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | token_policy | — | 配置token传输策略 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | iters | — | 配置运行循环次数 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | no_peak | — | 不输出峰值 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | jfs_post_list | — | 发送端wr链表串联数量 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | lock_free | — | jetty免锁模式 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | priority | — | jetty调度优先级 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | trans_mode | — | 传输模式：RC/RM/UM | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | port | — | 指定端口号 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | cq_num | — | 发送端指定数量wr产生一个cr | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | jfs_post_list | — | 接收端wr链表串联数量 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | jfr_depth | — | 配置jfr深度 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | size | — | 配置传输字节长度 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | server | — | server端IP地址 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | jfs_depth | — | 配置jfs深度 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | warm_up | — | 预热模式 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | infinite | — | 无限测试模式 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | single_path | — | 单路径模式 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | inf_period_ms | — | ms级统计打印 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | rate_limit | — | 速率上限 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | rate_units | — | 速率单位 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | burst_size | — | 单循环发包大小上限 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | enable_ipv6 | — | 使能ipv6监听 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | enable_credit | — | 使能发包限制 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | credit_threshold | — | 发包限速水线 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | credit_notify_cnt | — | 接收指定数量后通知发送者 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | jettys_pre_jfr | — | JFR共享次数 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | seg_pre_jetty | — | jetty的sge个数 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | enable_imm | — | 使能immediate语义 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | enable_err_continue | — | 异常继续特性 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | notify_data | — | write_with_notify | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | sge_num | — | wr内sge个数 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | enable_write_dirty | — | 写脏功能 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | pair_num | — | 多路径连接个数 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | async_import | — | 异步建链 | × | × | +| DFX接口 | urma_perftest | perftest运行参数配置 | tp_aware | — | 感知tp建链 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | tp_reuse | — | tp复用模式 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | ctp | — | ctp传输层 | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | jetty_id | — | 指定jetty id | √ | √ | +| DFX接口 | urma_perftest | perftest运行参数配置 | wait_jfc_timeout | — | 中断模式等待时间 | √ | √ | + +#### 运行环境 + +| Feature L1 | Feature L2 | Feature L3 | Feature L4 | Feature L5 | Feature Description | KP950 | Ascend910D | +|---|---|---|---|---|---|---|---| +| 运行环境 | OS | EulerOS | V2R8 | — | EulerOS V2R8 支持 | — | — | +| 运行环境 | OS | EulerOS | V2R9 | — | EulerOS V2R9 支持 | — | — | +| 运行环境 | OS | EulerOS | V2R10 | — | EulerOS V2R10 支持 | — | — | +| 运行环境 | OS | openEuler | 22.03 | — | openEuler 22.03 支持 | — | — | +| 运行环境 | OS | HCE | 2.0 2403 | — | HCE 2.0 2403 支持 | — | — | +| 运行环境 | OS | HCE | 3.0.2506 | — | HCE 3.0.2506 支持 | — | — | +| 运行环境 | 平台 | 物理机 | — | — | 物理机支持 | — | — | +| 运行环境 | 平台 | 虚拟机 | — | — | 虚拟机支持 | — | — | +| 运行环境 | 平台 | 容器 | — | — | 容器支持 | — | — | +## 6.2 设备聚合 + +urma提供了基于多设备聚合能力，达成带宽倍增、故障切换和负载均衡效果。另外，聚合设备能够屏蔽复杂的组网拓扑，简化用法，提供用户友好的UB基础通信能力。 + +### 6.2.1 聚合设备基本概念 + +通算业务中，用户通常使用2平面*8节点的1D Full-Mesh组网、2平面*16节点的2D Full-mesh组网两种部署形态。 + +![](figures/urma-feat-aggr-concept-01.png) ![](figures/urma-feat-aggr-concept-07.png) + +以1D Full-Mesh组网为例，该拓扑包含8个节点，每个节点有2个IODie，分别位于两个完全对称的平面。每个IODie配备9个物理端口，其中7个端口用于与同平面内其他7个节点的IODie直连，两平面连接方式完全一致。在此拓扑中，同一平面内任意两个节点IODie之间均有且只有一个直连的物理端口。 + +每个IODie上有一个或多个UB设备，每个UB设备上配置了两类EID：primary EID和port EID。port EID仅具备访问其对应直连端口的权限，支持CTP和RTP通信；primary EID则可访问该IODie所有物理端口，支持与同平面所有节点进行CTP通信。 + +为简化用户对复杂拓扑及多类EID的感知，URMA将同一节点两个IODie上的UB设备聚合为一个虚拟URMA设备，称为聚合设备，并将两个UB设备上的所有EID聚合成一个统一的bonding EID。用户可直接通过聚合设备的bonding EID进行通信，无需了解底层拓扑结构或区分primary EID跟port EID。 + +聚合设备根据用户配置的通信模式，查询拓扑选择合适的EID进行通信，在此基础上，还实现了带宽聚合、故障切换功能。URMA聚合设备支持三种聚合模式： + +**单设备模式/Standalone**：最简单的聚合形式，实际仅使用一个物理设备。主要用于屏蔽拓扑和EID类型。 + +**主备模式/ActiveBackup**：高可用性方案，实际仅使用一个物理设备。当主设备故障时，将流量切换到备用设备。 + +**负载均衡模式/Balance**：带宽聚合方案，同时使用多个设备提升吞吐量，支持RR轮询等方式实现负载均衡。此外当其中一个设备故障时，将其上流量切换到其他设备。 + +主备模式/ActiveBackup： + +![](figures/urma-feat-aggr-concept-02.png) + +负载均衡模式/Balance： + +![](figures/urma-feat-aggr-concept-03.png) + +相关接口参见：URMA编程API用户手册的详细说明，包括：urma_set_context_opt等API。 + +如果想要直接使用UB设备通信，需要查询组网拓扑。URMA管理工具提供了urma_admin show topo命令来查询拓扑信息，该工具展示聚合设备EID（bondingEID）、primaryEID和portEID之间的逻辑组网关系。**注意：物理端口能否通信与硬件状态有关。此工具仅支持查询EID之间的逻辑组网关系，不代表按照此逻辑组网对应的物理端口一定能够通信。** + +**show topo命令使用示例** + +本示例为8节点、2个IODie的硬件环境，查询node2、node4之间直连的一对普通设备EID。 + +在node4执行urma_admin show topo，输出格式如下： + +![](figures/urma-feat-aggr-concept-04.png) + +![](figures/urma-feat-aggr-concept-05.png) + +node2执行urma_admin show topo： + +![](figures/urma-feat-aggr-concept-06.png) + +以上输出说明node2-node4之间 + +IODIE0： + +- （node4, port4）(node2, port4)连通 + +- :::40:10:00:dfdf:8c5和:::40:10:00:dfdf:845连通 + +IODIE1： + +- （node4, port4）和(node2, port4)连通 + +- :::40:10:00:dfdf:8c5和:::40:10:00:dfdf:845连通 + +### 6.2.2 聚合设备基本使用流程 + +- 自举建链场景 + +自举建链指的是使用 URMA 公知 jetty 作为建链信息交换通道的建链方法。整体流程如图所示： + +![](figures/urma-feat-aggr-flow-01.png) + +自举建链场景需要使用公知 jetty，公知 jetty 必须配置 trans_mode = URMA_TM_RM 并且启用多路径选项。在有数据收发保序要求的场景下，建议设置 jfs_wr 中的 place_order = URMA_STRONG_ORDER + +- 一般数据传输场景 + +一般数据传输的场景推荐使用非公知 jetty 并且使用单路径 RC 模式。 + +需要注意的是在这种传输场景下，必须双方都完成 bind_jetty 操作之后两端才能进行双边通信。 + +整体代码与上面一致，下图为 urma_bind_jetty 的流程： + +![](figures/urma-feat-aggr-flow-02.png) + +UB采用了事务层与传输层分离的架构，协议定义了传输层的类型：RTP（可靠传输层）、CTP（简易传输层）、UTP（不可靠传输层）。 + +单路径是RC模式基于RTP，多路径是RM模式基于CTP。 + +单路径RM模式接口只知道对端jetty信息但不知道本端绑定哪个jetty，用户使用RQE进行双端操作，RQE无法确定从哪个jfr中收取数据 + +RC模式下的bind操作可以绑定双端jetty + +bonding设备不感知TP。 + +### 6.2.3 聚合设备特性列表和约束 + +- 使用约束 + +1\. 在 鲲鹏950 的场景下，聚合设备只有一个，并且名称一定为 bonding_dev_0； + +2\. 单路径模式的 jetty 和多路径模式的 jetty 之间无法通信，只有两端的 jetty 单路径、多路径的选项参数一致才能通信； + +3\. 不同 jetty 的单路径、多路径模式以及传输模式可以支持的能力有所差异； + +4\. 在使用聚合设备的场景下，传输层使用 TP/CTP 的选择仅和创建 jetty 和 jfs/jfr 的时候设定的参数有关，urma_import_jetty 中传入的 rjetty flag 中的 CTP 参数会被忽略。 + +- 聚合设备的特性列表 + + 1. 聚合设备的特性列表 + +| Device Name | EID Count | Mode | RTP/CTP | Transport Mode | Loopback | Max Send Pkt Size | Reliability | Reachable EID | +|---|---|---|---|---|---|---|---|---| +| bonding_dev_0 | 1 | jetty多路径 | CTP | RM（ROI保序） | 否 | 4kB | 有TA ACK | 任意节点的agg EID | +| bonding_dev_0 | 1 | jetty多路径 | CTP | RM（ROI保序） | 否 | 4kB | 无重传 | 任意节点的agg EID | +| bonding_dev_0 | 1 | jetty多路径 | CTP | RC（ROL保序） | 否 | 4kB | 无可靠保障机制 | 任意节点的agg EID | +| bonding_dev_0 | 1 | jetty单路径 | RTP | RC（ROL保序） | 是 | 64kB | 与单设备RC模式一致 | 任意节点的agg EID | +| bonding_dev_0 | 1 | jfs/jfr多路径 | CTP | RM（ROI保序） | 否 | 4kB | 有TA ACK | 任意节点的agg EID | +| bonding_dev_0 | 1 | jfs/jfr多路径 | CTP | RM（ROI保序） | 否 | 4kB | 无重传 | 任意节点的agg EID | +- 聚合设备支持urma特性API梳理 + +聚合设备已经支持大部分URMA API，现列举不支持的URMA API及其影响： + +1. 聚合设备不支持urma特性API + +| Unsupported URMA API | Function and Impact | +|---|---| +| urma_query_jfs | 用于查询jfs的状态，主要为DFX功能，非必要功能接口 | +| urma_flush_jfs | 清空jfs软件队列，功能接口，应用不要求所有下发的sqe必须上报，则不依赖此功能 | +| urma_advise_jfr/urma_unadvise_jfr | 非UB功能API | +| urma_query_jetty | 用于查询jetty的状态和接收端配置水线，主要为DFX功能，非必要功能接口。说明：jetty作为接收端的配置水线功能，鲲鹏950硬件当前不支持 | +| urma_flush_jetty | 清空jetty发送软件队列，功能接口，应用不要求所有下发的sqe必须上报，则不依赖此功能 | +| urma_advise_jetty_async | 非UB功能API | +| urma_create_jetty_grp/urma_delete_jetty_grp | 创建和删除jetty grp，功能接口 | +| urma_get_tpn | 当前管控面不支持此能力 | +| urma_modify_tp | 当前管控面不支持此能力 | +## 6.3 虚拟化 + +一个UB物理设备可以包含一份或多份设备资源（UE，UB Entity），同时也可以拥有一个或多个物理端口（Port）。在Host访问设备资源时，访问请求可以从任意一个端口进入并访问到相应的设备资源，这意味着这些端口是由多份设备资源共享的。当系统将多份设备资源分配给不同用户使用时，这些资源之间需要具备一定的隔离性。 + +UB设备支持实现多个Port，不同UB设备之间可以通过这些端口进行通信。UE是UB设备中一份具备隔离性的设备资源集合，它不仅是一个可寻址的实体，还提供了特定功能，并描述了该实体所占用的设备资源。通过设备间的Port互联，软件可以访问到UE所对应的设备资源。UE作为UB设备对自身资源进行划分的基本单元，为用户提供了管理设备资源的方式。UB设备允许用户对资源进行更细粒度的划分，因此在实际使用中可能出现多个UE同时依赖某一类资源配置来提供服务的情况。 + +### 6.3.1 容器 + +在容器环境中，为实现URMA物理设备的灵活管理，URMA引入了逻辑设备机制。每个物理设备仅属于一个命名空间，但可以在不同的命名空间中创建逻辑设备，每个逻辑设备同样只属于一个命名空间。逻辑设备作为独立的访问接口，仅在所属命名空间内可见，且名称与物理设备相同。此外，URMA设备上的EID可配置命名空间属性，并绑定到对应命名空间的逻辑设备上，EID同样仅在其所属的命名空间中可见。删除逻辑设备时，其上的EID会自动迁移回原物理设备。 + +基于逻辑设备的管理机制，容器中的URMA支持以下两种工作模式： + +- **自动模式**：系统自动管理逻辑设备的生命周期。当创建或销毁一个命名空间时，系统会自动在该命名空间中创建或销毁所有URMA物理设备对应的逻辑设备。所有容器均可直接识别并使用这些URMA设备，无需用户进行额外配置。 + +- **手动模式**：逻辑设备的创建与删除完全由用户主动控制。用户可根据需要，为指定的URMA物理设备在特定命名空间中创建逻辑设备，并可灵活调整物理设备所属的命名空间。 + +![](figures/urma-feat-virt-container-01.png) + +URMA提供了一组命令来配置设备和EID。 + +创建、删除逻辑设备： + +urma_admin dev create_logic_dev + +urma_admin dev delete_logic_dev + +设置EID的命名空间： + +urma_admin dev set_logic_dev_eid + +设置逻辑设备模式：手动模式、自动模式 + +urma_admin dev set_logic_dev_mode + +设置物理设备的命名空间 + +urma_admin dev set_ns + +### 6.3.2 虚拟机 + +UB网卡的UE直通到虚机中，虚机和裸机中的URMA使用没有区别 + +![](figures/urma-feat-virt-vm-01.png) + +## 6.4 工具手册 + +![](figures/urma_notice.png) + +命令行参数分为必选和可选两类，必选项采用<\>描述，可选项采用[]描述。 + +### 6.4.1 urma_perftest + +用于urma时延和带宽测试的性能工具。涵盖收发、读、写、原子操作等四类语义，每种语义支持时延和带宽测试。区分server端和client端分别起urma_perftest进程开展测试并输出测试结果。 + +1. 命令格式 + +``` +Usage: urma_perftest command [command options] +urma_perftest URMA perftest tool +Command syntax: +read_lat Test for read latency. +write_lat Test for write latency. +send_lat Test for send latency. +atomic_lat Test for atomic latency. +read_bw Test for read bandwidth. +write_bw Test for write bandwidth. +send_bw Test for send bandwidth. +atomic_bw Test for atomic bandwidth. +``` + +2. 功能描述 + +``` +收发时延测试server端：urma_perftest send_lat -d -s [SIZE] -n [ITERATIONS] +收发时延测试client端：urma_perftest send_lat -d -s [SIZE] -n [ITERATIONS] -S +读时延测试server端：urma_perftest read_lat -d -s [SIZE] -n [ITERATIONS] +读时延测试client端：urma_perftest read_lat -d -s [SIZE] -n [ITERATIONS] -S +写时延测试server端：urma_perftest write_lat -d -s [SIZE] -n [ITERATIONS] +写时延测试client端：urma_perftest write_lat -d -s [SIZE] -n [ITERATIONS] -S +原子操作时延测试server端：urma_perftest atomic_lat -d -s [SIZE] -n [ITERATIONS] -A cas +原子操作时延测试client端：urma_perftest atomic_lat -d -s [SIZE] -n [ITERATIONS] -S -A cas +收发带宽测试server端：urma_perftest send_bw -d -s [SIZE] -n [ITERATIONS] +收发带宽测试client端：urma_perftest send_bw -d -s [SIZE] -n [ITERATIONS] -S +读带宽测试server端：urma_perftest read_bw -d -s [SIZE] -n [ITERATIONS] +读带宽测试client端：urma_perftest read_bw -d -s [SIZE] -n [ITERATIONS] -S +写带宽测试server端：urma_perftest write_bw -d -s [SIZE] -n [ITERATIONS] +写带宽测试client端：urma_perftest write_bw -d -s [SIZE] -n [ITERATIONS] -S +原子操作带宽测试server端：urma_perftest atomic_bw -d -s [SIZE] -n [ITERATIONS] -A cas +原子操作带宽测试client端：urma_perftest atomic_bw -d -s [SIZE] -n [ITERATIONS] -S -A cas +``` + +3. 参数说明 + +urma_perftest参数 + +``` +Options: + -a, --all[order] Run sizes from 2 till 2^23 (default 2^16), order: exponent of 2. + -A, --atomic_type Specify atomic type, {cas|faa}. + -b, --simplex_mode Run with simplex mode(jfs/jfr), duplex jetty mode for reserved. + -B, --bidirection Measure bidirectional bandwidth (default unidirectional). + -c, --jfc_inline Enable jfc_inline to upgrade latency performance. + -C, --jfc_depth Size of jfc depth (default 4096 for bw, 1024 for ip bw, 1 for lat). + -d, --dev The name of ubep device. + -D, --duration Run test for a customized period of seconds, this cfg covers iters. + -e, --use_jfce use jfc event. + --eid_idx Specified eid index of device. + -E, --err_timeout