Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 31 additions & 8 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ jobs:
- {os: ubuntu-latest, hdf5: '2.1'}
- {os: macos-latest, hdf5: '1.14'}
- {os: macos-latest, hdf5: '2.1'}
- {os: windows-latest, hdf5: '1.14'}
- {os: windows-latest, hdf5: '2.1'}

steps:
- name: Checkout
Expand All @@ -50,22 +52,39 @@ jobs:
# it only searches system paths otherwise. Warnings are errors in CI
# only, so new compiler versions don't break downstream builds.
run: |
if [ "$RUNNER_OS" = "Windows" ]; then
export PREFIX_PATH="$CONDA_PREFIX/Library"
else
export PREFIX_PATH="$CONDA_PREFIX"
fi
cmake -B build -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
-DCMAKE_PREFIX_PATH="$CONDA_PREFIX" -DHDF5_ROOT="$CONDA_PREFIX"
-DCMAKE_PREFIX_PATH="$PREFIX_PATH" \
-DHDF5_ROOT="$PREFIX_PATH" \
-DHDF5_USE_STATIC_LIBRARIES=OFF

- name: Build
run: cmake --build build --config Release -j 2

- name: Run the test suite (hermetic — no network)
run: ctest --test-dir build --output-on-failure
run: ctest --test-dir build -C Release --output-on-failure

- name: Run the DANDI smoke test (reads a real NWB file over HTTPS)
run: ./build/test_remfile
run: |
if [ "${{ matrix.os }}" = "windows-latest" ]; then
./build/Release/test_remfile.exe
else
./build/test_remfile
fi

- name: Check installation and find_package consumption
run: |
cmake --install build --prefix "$PWD/install"
if [ "$RUNNER_OS" = "Windows" ]; then
export PREFIX_PATH="$CONDA_PREFIX/Library"
else
export PREFIX_PATH="$CONDA_PREFIX"
fi
cmake --install build --config Release --prefix "$PWD/install"
# A downstream project must be able to find and link remfile::remfile
mkdir -p consumer && cd consumer
cat > CMakeLists.txt <<'EOF'
Expand All @@ -83,10 +102,14 @@ jobs:
return H5Pset_fapl_remfile(fapl, NULL) < 0 ? 1 : 0;
}
EOF
cmake -B build -DCMAKE_PREFIX_PATH="$PWD/../install;$CONDA_PREFIX" \
-DHDF5_ROOT="$CONDA_PREFIX"
cmake --build build
./build/consumer
cmake -B build -DCMAKE_PREFIX_PATH="../install;$PREFIX_PATH" \
-DHDF5_ROOT="$PREFIX_PATH" -DHDF5_USE_STATIC_LIBRARIES=OFF
cmake --build build --config Release
if [ "${{ matrix.os }}" = "windows-latest" ]; then
./build/Release/consumer.exe
else
./build/consumer
fi

# The dynamically loadable VFD plugin requires HDF5 >= 1.14. Verify it can be
# loaded by name via HDF5_PLUGIN_PATH, with nothing linked against remfile.
Expand Down
6 changes: 3 additions & 3 deletions src/remfile_vfd.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -304,9 +304,9 @@ bool load_chunk(RemFile *f, uint64_t chunk_index, uint64_t chunks_needed)
if (i * min_chunk >= data.size())
break;
size_t piece_start = i * min_chunk;
size_t piece_end = std::min(piece_start + min_chunk, data.size());
size_t piece_end = (std::min)(piece_start + min_chunk, data.size());
f->chunks[chunk_index + i] =
std::vector<uint8_t>(data.begin() + static_cast<std::ptrdiff_t>(piece_start), data.begin() + static_cast<std::ptrdiff_t>(piece_end));
std::vector<uint8_t>(data.data() + piece_start, data.data() + piece_end);
f->chunk_order.push_back(chunk_index + i);
}
}
Expand Down Expand Up @@ -348,7 +348,7 @@ bool remfile_read_bytes(RemFile *f, uint64_t position, size_t size, void *buf)
for (uint64_t ci = chunk_start_index; ci <= chunk_end_index; ci++) {
const std::vector<uint8_t> &chunk = f->chunks[ci];
size_t chunk_offset = (ci == chunk_start_index) ? static_cast<size_t>(position % min_chunk) : 0;
size_t chunk_length = std::min(chunk.size() - chunk_offset, size - written);
size_t chunk_length = (std::min)(chunk.size() - chunk_offset, size - written);
memcpy(out + written, chunk.data() + chunk_offset, chunk_length);
written += chunk_length;
}
Expand Down
54 changes: 37 additions & 17 deletions tests/http_test_server.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,22 @@
#ifndef REMFILE_HTTP_TEST_SERVER_HPP
#define REMFILE_HTTP_TEST_SERVER_HPP

#ifdef _WIN32
#include <winsock2.h>
#include <ws2tcpip.h>
#pragma comment(lib, "ws2_32.lib")
typedef int socklen_t;
typedef SOCKET socket_t;
#define SHUT_RDWR SD_BOTH
#define close_socket closesocket
#else
#include <arpa/inet.h>
#include <netinet/in.h>
#include <sys/socket.h>
#include <unistd.h>
typedef int socket_t;
#define close_socket ::close
#endif

#include <atomic>
#include <cstdio>
Expand All @@ -43,12 +55,16 @@ class HttpTestServer
explicit HttpTestServer(std::vector<uint8_t> data)
: m_data(std::move(data))
{
#ifdef _WIN32
WSADATA wsaData;
WSAStartup(MAKEWORD(2, 2), &wsaData);
#endif
m_listen_fd = ::socket(AF_INET, SOCK_STREAM, 0);
if (m_listen_fd < 0)
if (m_listen_fd == (socket_t)-1)
throw std::runtime_error("HttpTestServer: socket() failed");

int yes = 1;
::setsockopt(m_listen_fd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes));
::setsockopt(m_listen_fd, SOL_SOCKET, SO_REUSEADDR, (const char*)&yes, sizeof(yes));

sockaddr_in addr {};
addr.sin_family = AF_INET;
Expand All @@ -70,7 +86,7 @@ class HttpTestServer
{
m_stop = true;
::shutdown(m_listen_fd, SHUT_RDWR);
::close(m_listen_fd);
close_socket(m_listen_fd);
if (m_thread.joinable())
m_thread.join();

Expand All @@ -85,12 +101,16 @@ class HttpTestServer
* the failure. */
{
std::lock_guard<std::mutex> lock(m_conn_mutex);
for (int fd : m_conn_fds)
for (socket_t fd : m_conn_fds)
::shutdown(fd, SHUT_RDWR);
}
for (auto& t : m_conn_threads)
if (t.joinable())
t.join();

#ifdef _WIN32
WSACleanup();
#endif
}

HttpTestServer(const HttpTestServer&) = delete;
Expand Down Expand Up @@ -135,8 +155,8 @@ class HttpTestServer
void run()
{
while (!m_stop) {
int fd = ::accept(m_listen_fd, nullptr, nullptr);
if (fd < 0) {
socket_t fd = ::accept(m_listen_fd, nullptr, nullptr);
if (fd == (socket_t)-1) {
if (m_stop)
break;
continue;
Expand All @@ -151,19 +171,19 @@ class HttpTestServer
[this, fd]
{
handle_connection(fd);
::close(fd);
close_socket(fd);
});
}
}

/* Read a full request header block (keep-alive: one connection may carry
* several requests, which is exactly what the driver's curl handle does). */
void handle_connection(int fd)
void handle_connection(socket_t fd)
{
std::string buf;
char chunk[4096];
while (!m_stop) {
ssize_t n = ::recv(fd, chunk, sizeof(chunk), 0);
int n = ::recv(fd, chunk, sizeof(chunk), 0);
if (n <= 0)
return;
buf.append(chunk, (size_t)n);
Expand All @@ -180,7 +200,7 @@ class HttpTestServer
}

/* Returns false if the connection should be closed. */
bool handle_request(int fd, const std::string& request)
bool handle_request(socket_t fd, const std::string& request)
{
m_request_count++;

Expand All @@ -190,7 +210,7 @@ class HttpTestServer
"HTTP/1.1 500 Internal Server Error\r\n"
"Content-Length: 0\r\n"
"Connection: keep-alive\r\n\r\n";
::send(fd, resp.data(), resp.size(), 0);
::send(fd, resp.data(), (int)resp.size(), 0);
return true;
}

Expand All @@ -199,7 +219,7 @@ class HttpTestServer
"HTTP/1.1 404 Not Found\r\n"
"Content-Length: 0\r\n"
"Connection: keep-alive\r\n\r\n";
::send(fd, resp.data(), resp.size(), 0);
::send(fd, resp.data(), (int)resp.size(), 0);
return true;
}

Expand All @@ -226,7 +246,7 @@ class HttpTestServer
"HTTP/1.1 416 Range Not Satisfiable\r\n"
"Content-Length: 0\r\n"
"Connection: keep-alive\r\n\r\n";
::send(fd, resp.data(), resp.size(), 0);
::send(fd, resp.data(), (int)resp.size(), 0);
return true;
}
if (end >= m_data.size())
Expand Down Expand Up @@ -259,12 +279,12 @@ class HttpTestServer
"Connection: keep-alive\r\n\r\n",
(unsigned long long)len);
}
if (::send(fd, header, (size_t)hlen, 0) < 0)
if (::send(fd, header, hlen, 0) < 0)
return false;

size_t sent = 0;
while (sent < len) {
ssize_t n = ::send(fd, m_data.data() + start + sent, (size_t)(len - sent), 0);
int n = ::send(fd, (const char*)(m_data.data() + start + sent), (int)(len - sent), 0);
if (n <= 0)
return false;
sent += (size_t)n;
Expand All @@ -284,7 +304,7 @@ class HttpTestServer
}

std::vector<uint8_t> m_data;
int m_listen_fd = -1;
socket_t m_listen_fd = (socket_t)-1;
int m_port = 0;
std::thread m_thread;
std::atomic<bool> m_stop {false};
Expand All @@ -297,7 +317,7 @@ class HttpTestServer
std::vector<std::pair<uint64_t, uint64_t>> m_ranges;
std::mutex m_conn_mutex;
std::vector<std::thread> m_conn_threads;
std::vector<int> m_conn_fds;
std::vector<socket_t> m_conn_fds;
};

} // namespace remfile_test
Expand Down
11 changes: 11 additions & 0 deletions tests/test_remfile.c
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@
#include <string.h>
#include <time.h>

#ifdef _WIN32
#include <windows.h>
#endif

#include <hdf5.h>
#include <remfile/remfile_vfd.h>

Expand All @@ -22,9 +26,16 @@ static const char *default_dataset =

static double now_seconds(void)
{
#ifdef _WIN32
LARGE_INTEGER count, freq;
QueryPerformanceCounter(&count);
QueryPerformanceFrequency(&freq);
return (double)count.QuadPart / (double)freq.QuadPart;
#else
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return (double)ts.tv_sec + (double)ts.tv_nsec / 1e9;
#endif
}

/* H5Literate2 / H5L_info2_t are only available from HDF5 1.12 onwards. */
Expand Down
12 changes: 6 additions & 6 deletions tests/test_remfile_vfd.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -92,10 +92,10 @@ std::vector<uint8_t> build_test_file(const std::string& path)

hsize_t sdim = 5;
hid_t sspace = H5Screate_simple(1, &sdim, nullptr);
int small[5] = {10, 20, 30, 40, 50};
int small_data[5] = {10, 20, 30, 40, 50};
dset = H5Dcreate2(file, "small", H5T_NATIVE_INT, sspace, H5P_DEFAULT,
H5P_DEFAULT, H5P_DEFAULT);
REQUIRE(H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, small) >= 0);
REQUIRE(H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, small_data) >= 0);
H5Dclose(dset);

hid_t ascalar = H5Screate(H5S_SCALAR);
Expand Down Expand Up @@ -237,10 +237,10 @@ TEST_CASE("attributes and small datasets read correctly", "[vfd]")

hid_t dset = H5Dopen2(file, "small", H5P_DEFAULT);
REQUIRE(dset >= 0);
int small[5] = {0};
REQUIRE(H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, small) >= 0);
REQUIRE(small[0] == 10);
REQUIRE(small[4] == 50);
int small_data[5] = {0};
REQUIRE(H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, small_data) >= 0);
REQUIRE(small_data[0] == 10);
REQUIRE(small_data[4] == 50);
H5Dclose(dset);
H5Fclose(file);
}
Expand Down
Loading