From 6ab00a50a507ce7ccdd260cba25bd7095dd404b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Satabin?= Date: Mon, 29 Jun 2026 11:49:19 +0200 Subject: [PATCH 1/2] =?UTF-8?q?Optimisation=20des=20acc=C3=A8s=20au=20stoc?= =?UTF-8?q?kage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 7 ++++ src/storage/S3Context.cpp | 60 +++++++++++++++++++++------- src/storage/SwiftContext.cpp | 60 ++++++++++++++++++++++------ src/storage/ceph/CephPoolContext.cpp | 27 ++++++++++--- src/utils/Level.cpp | 6 +++ 5 files changed, 126 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index deb74fe9..f89e998f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ Le format est basé sur [Keep a Changelog](https://keepachangelog.com/) et ce pr ## [Unreleased] +### Fixed +- `Storage` : on attend uniquement entre deux tentatives de lecture ou d'écriture, et pas après la dernière tentative. + +### Changed + +- `Level` : on n'essaye même pas de faire la lecture d'une tuile si on est hors limite (éviter des 404 prévisibles) + ## [4.0.0] - 2026-06-26 ### Changed diff --git a/src/storage/S3Context.cpp b/src/storage/S3Context.cpp index 4261c985..70491af1 100644 --- a/src/storage/S3Context.cpp +++ b/src/storage/S3Context.cpp @@ -278,7 +278,7 @@ int S3Context::read(uint8_t *data, int offset, int size, std::string name) { BOOST_LOG_TRIVIAL(debug) << "S3 read : " << size << " bytes (from the " << offset << " one) in the object " << bucket_name << "@" << ((cluster_name != "") ? cluster_name : host) << " / " << name; int attempt = 1; - while (attempt <= read_attempts) { + while (attempt) { // On constitue le moyen de récupération des informations (avec les structures de LibcurlStruct) CURLcode res; @@ -360,8 +360,13 @@ int S3Context::read(uint8_t *data, int offset, int size, std::string name) { BOOST_LOG_TRIVIAL(error) << "Try " << attempt << " failed" ; BOOST_LOG_TRIVIAL(error) << curl_easy_strerror(res); attempt++; - sleep(waiting_time); - continue; + + if (attempt > read_attempts) { + break; + } else { + sleep(waiting_time); + continue; + } } long http_code = 0; @@ -370,8 +375,13 @@ int S3Context::read(uint8_t *data, int offset, int size, std::string name) { BOOST_LOG_TRIVIAL(error) << "Try " << attempt << " failed" ; BOOST_LOG_TRIVIAL(error) << "Response HTTP code : " << http_code; attempt++; - sleep(waiting_time); - continue; + + if (attempt > read_attempts) { + break; + } else { + sleep(waiting_time); + continue; + } } memcpy(data, chunk.data, chunk.size); @@ -391,7 +401,7 @@ uint8_t *S3Context::read_full(int &size, std::string name) { // On constitue le moyen de récupération des informations (avec les structures de LibcurlStruct) int attempt = 1; - while (attempt <= read_attempts) { + while (attempt) { CURLcode res; struct curl_slist *list = NULL; DataStruct chunk; @@ -463,8 +473,13 @@ uint8_t *S3Context::read_full(int &size, std::string name) { BOOST_LOG_TRIVIAL(error) << "Try " << attempt << " failed" ; BOOST_LOG_TRIVIAL(error) << curl_easy_strerror(res); attempt++; - sleep(waiting_time); - continue; + + if (attempt > read_attempts) { + break; + } else { + sleep(waiting_time); + continue; + } } long http_code = 0; @@ -473,8 +488,13 @@ uint8_t *S3Context::read_full(int &size, std::string name) { BOOST_LOG_TRIVIAL(error) << "Try " << attempt << " failed" ; BOOST_LOG_TRIVIAL(error) << "Response HTTP code : " << http_code; attempt++; - sleep(waiting_time); - continue; + + if (attempt > read_attempts) { + break; + } else { + sleep(waiting_time); + continue; + } } size = chunk.size; @@ -574,7 +594,7 @@ bool S3Context::close_to_write(std::string name) { BOOST_LOG_TRIVIAL(debug) << "Write buffered " << it1->second->size() << " bytes in the S3 object " << name; int attempt = 1; - while (attempt <= write_attempts) { + while (attempt) { CURLcode res; struct curl_slist *list = NULL; @@ -645,8 +665,13 @@ bool S3Context::close_to_write(std::string name) { BOOST_LOG_TRIVIAL(error) << "Try " << attempt << " failed" ; BOOST_LOG_TRIVIAL(error) << curl_easy_strerror(res); attempt++; - sleep(waiting_time); - continue; + + if (attempt > write_attempts) { + break; + } else { + sleep(waiting_time); + continue; + } } long http_code = 0; @@ -655,8 +680,13 @@ bool S3Context::close_to_write(std::string name) { BOOST_LOG_TRIVIAL(error) << "Try " << attempt << " failed" ; BOOST_LOG_TRIVIAL(error) << "Response HTTP code : " << http_code; attempt++; - sleep(waiting_time); - continue; + + if (attempt > write_attempts) { + break; + } else { + sleep(waiting_time); + continue; + } } BOOST_LOG_TRIVIAL(debug) << "Erase the flushed buffer"; diff --git a/src/storage/SwiftContext.cpp b/src/storage/SwiftContext.cpp index 8c557d41..6eb7610c 100644 --- a/src/storage/SwiftContext.cpp +++ b/src/storage/SwiftContext.cpp @@ -301,7 +301,7 @@ int SwiftContext::read(uint8_t* data, int offset, int size, std::string name) { int attempt = 1; bool reconnection = false; - while (attempt <= read_attempts) { + while (attempt) { CURLcode res; struct curl_slist *list = NULL; @@ -347,7 +347,14 @@ int SwiftContext::read(uint8_t* data, int offset, int size, std::string name) { if( CURLE_OK != res) { BOOST_LOG_TRIVIAL(error) << "Cannot read data from Swift : " << size << " bytes (from the " << offset << " one) in the object " << name; BOOST_LOG_TRIVIAL(error) << curl_easy_strerror(res); - return -1; + attempt++; + + if (attempt > write_attempts) { + break; + } else { + sleep(waiting_time); + continue; + } } long http_code = 0; @@ -373,8 +380,13 @@ int SwiftContext::read(uint8_t* data, int offset, int size, std::string name) { BOOST_LOG_TRIVIAL(error) << "Try " << attempt << " failed" ; BOOST_LOG_TRIVIAL(error) << "Response HTTP code : " << http_code; attempt++; - sleep(waiting_time); - continue; + + if (attempt > read_attempts) { + break; + } else { + sleep(waiting_time); + continue; + } } memcpy(data, chunk.data, chunk.size); @@ -399,7 +411,7 @@ uint8_t* SwiftContext::read_full(int& size, std::string name) { int attempt = 1; bool reconnection = false; - while (attempt <= read_attempts) { + while (attempt) { CURLcode res; struct curl_slist *list = NULL; @@ -437,7 +449,14 @@ uint8_t* SwiftContext::read_full(int& size, std::string name) { if( CURLE_OK != res) { BOOST_LOG_TRIVIAL(error) << "Cannot read full object from Swift : " << name; BOOST_LOG_TRIVIAL(error) << curl_easy_strerror(res); - return NULL; + attempt++; + + if (attempt > write_attempts) { + break; + } else { + sleep(waiting_time); + continue; + } } long http_code = 0; @@ -463,8 +482,13 @@ uint8_t* SwiftContext::read_full(int& size, std::string name) { BOOST_LOG_TRIVIAL(error) << "Try " << attempt << " failed" ; BOOST_LOG_TRIVIAL(error) << "Response HTTP code : " << http_code; attempt++; - sleep(waiting_time); - continue; + + if (attempt > read_attempts) { + break; + } else { + sleep(waiting_time); + continue; + } } uint8_t* data = new uint8_t[chunk.size]; @@ -559,7 +583,7 @@ bool SwiftContext::close_to_write(std::string name) { int attempt = 1; bool reconnection = false; - while (attempt <= write_attempts) { + while (attempt) { CURLcode res; struct curl_slist *list = NULL; CURL* curl = CurlPool::get_curl_env(); @@ -592,8 +616,13 @@ bool SwiftContext::close_to_write(std::string name) { BOOST_LOG_TRIVIAL(error) << "Try " << attempt << " failed" ; BOOST_LOG_TRIVIAL(error) << curl_easy_strerror(res); attempt++; - sleep(waiting_time); - continue; + + if (attempt > write_attempts) { + break; + } else { + sleep(waiting_time); + continue; + } } long http_code = 0; @@ -620,8 +649,13 @@ bool SwiftContext::close_to_write(std::string name) { BOOST_LOG_TRIVIAL(error) << "Try " << attempt << " failed" ; BOOST_LOG_TRIVIAL(error) << "Response HTTP code : " << http_code; attempt++; - sleep(waiting_time); - continue; + + if (attempt > write_attempts) { + break; + } else { + sleep(waiting_time); + continue; + } } BOOST_LOG_TRIVIAL(debug) << "Erase the flushed buffer"; diff --git a/src/storage/ceph/CephPoolContext.cpp b/src/storage/ceph/CephPoolContext.cpp index bc834f1e..db51304b 100644 --- a/src/storage/ceph/CephPoolContext.cpp +++ b/src/storage/ceph/CephPoolContext.cpp @@ -134,7 +134,7 @@ int CephPoolContext::read(uint8_t* data, int offset, int size, std::string name) int readSize; int attempt = 1; bool error = false; - while(attempt <= read_attempts) { + while(attempt) { readSize = rados_read(io_ctx, name.c_str(), (char*) data, size, offset); if (readSize < 0) { @@ -154,7 +154,12 @@ int CephPoolContext::read(uint8_t* data, int offset, int size, std::string name) } attempt++; - sleep(waiting_time); + + if (attempt > read_attempts) { + break; + } else { + sleep(waiting_time); + } } if (error) { @@ -190,7 +195,7 @@ uint8_t* CephPoolContext::read_full(int& size, std::string name) { int attempt = 1; bool error = false; - while(attempt <= read_attempts) { + while(attempt) { size = rados_read(io_ctx, name.c_str(), (char*) data, fullSize, 0); if (size < 0) { @@ -210,7 +215,12 @@ uint8_t* CephPoolContext::read_full(int& size, std::string name) { } attempt++; - sleep(waiting_time); + + if (attempt > read_attempts) { + break; + } else { + sleep(waiting_time); + } } if (error) { @@ -299,7 +309,7 @@ bool CephPoolContext::close_to_write(std::string name) { bool ok = true; int attempt = 1; - while(attempt <= write_attempts) { + while(attempt) { int err = rados_write_full(io_ctx,name.c_str(), &((*(it1->second))[0]), it1->second->size()); if (err < 0) { ok = false; @@ -312,7 +322,12 @@ bool CephPoolContext::close_to_write(std::string name) { } attempt++; - sleep(waiting_time); + + if (attempt > write_attempts) { + break; + } else { + sleep(waiting_time); + } } if (ok) { diff --git a/src/utils/Level.cpp b/src/utils/Level.cpp index 3495ea00..b1e7cdd8 100644 --- a/src/utils/Level.cpp +++ b/src/utils/Level.cpp @@ -550,6 +550,12 @@ std::string Level::get_path ( int tilex, int tiley) { */ DataSource* Level::get_encoded_tile ( int x, int y ) { // TODO: return 0 sur des cas d'erreur.. + if (!tm_limits.contain_tile(x,y)) { + // On est hors tuiles + BOOST_LOG_TRIVIAL(warning) << "get_encoded_tile out limits"; + return NULL; + } + //on stocke une dalle // Index de la tuile (cf. ordre de rangement des tuiles) int n = ( y % tiles_per_height ) * tiles_per_width + ( x % tiles_per_width ); From fc105eea6e4d25c378a53beabfc8fdb72e4a4d6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Satabin?= Date: Mon, 29 Jun 2026 11:51:04 +0200 Subject: [PATCH 2/2] Changelog pour release 4.1.0 --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f89e998f..dfb1bb01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ Le format est basé sur [Keep a Changelog](https://keepachangelog.com/) et ce pr ## [Unreleased] +## [4.1.0] - 2026-06-29 + ### Fixed - `Storage` : on attend uniquement entre deux tentatives de lecture ou d'écriture, et pas après la dernière tentative. @@ -239,7 +241,8 @@ Les librairies sont gérées de manière indépendantes, conditionnées pour êt [1.0.3]: https://github.com/rok4/core-cpp/releases/tag/1.0.3 -[Unreleased]: https://github.com/rok4/core-cpp/compare/4.0.0...HEAD +[Unreleased]: https://github.com/rok4/core-cpp/compare/4.1.0...HEAD +[4.1.0]: https://github.com/rok4/core-cpp/compare/4.0.0...4.1.0 [4.0.0]: https://github.com/rok4/core-cpp/compare/3.1.0...4.0.0 [3.1.0]: https://github.com/rok4/core-cpp/compare/3.0.0...3.1.0