From 00aa3527f3dd2a26fecdd6851ff3ce141b9fd856 Mon Sep 17 00:00:00 2001 From: Taylor Richards Date: Mon, 3 Oct 2022 15:15:45 -0400 Subject: [PATCH 01/21] add subfolder support to cfile --- code/cfile/cfile.cpp | 54 +++- code/cfile/cfilesystem.cpp | 242 +++++++++++++----- code/def_files/data/scripts/cfile_require.lua | 2 +- 3 files changed, 221 insertions(+), 77 deletions(-) diff --git a/code/cfile/cfile.cpp b/code/cfile/cfile.cpp index cf99770ee38..1db075c585f 100644 --- a/code/cfile/cfile.cpp +++ b/code/cfile/cfile.cpp @@ -1848,28 +1848,56 @@ int cfile_get_path_type(const SCP_string& dir) { SCP_string buf = dir; - // Remove trailing slashes; avoid buffer overflow on 1-char strings - while (buf.size() > 0 && (buf[buf.size() - 1] == '\\' || buf[buf.size() - 1] == '/')) { - buf.resize(buf.size() - 1); + // remove leading and trailing slashes + if ( !buf.empty() ) { + auto start = buf.find_first_not_of("\\/"); + auto end = buf.find_last_not_of("\\/"); + + if ( (start > 0) || (end < buf.length()-1) ) { + buf = buf.substr(start, end-start+1); + } } - // Remove leading slashes - while (buf.size() > 0 && (buf[0] == '\\' || buf[0] == '/')) { - buf = buf.substr(1); + if (buf.empty()) { + return CF_TYPE_ROOT; } // Use official DIR_SEPARATOR_CHAR - for (char& c : buf) { - if (c == '\\' || c == '/') { - c = DIR_SEPARATOR_CHAR; - } + char bad_sep = '/'; + + if (bad_sep == DIR_SEPARATOR_CHAR) { + bad_sep = '\\'; } + std::replace(buf.begin(), buf.end(), bad_sep, DIR_SEPARATOR_CHAR); + + // identify path type + auto best_match = CF_TYPE_INVALID; + for (auto& Pathtype : Pathtypes) { - if (Pathtype.path != nullptr && buf == Pathtype.path) { - return Pathtype.index; + if ( !Pathtype.path ) { + continue; + } + + // skip root, it should have been detected before getting here + if (Pathtype.index == CF_TYPE_ROOT) { + continue; + } + + // don't allow unknown subdirs directly under data + if (Pathtype.index == CF_TYPE_DATA) { + if (buf == Pathtype.path) { + best_match = CF_TYPE_DATA; + } + + continue; + } + + // for everything else just find closest match, allowing for unknown subdirectories + if ( !buf.compare(0, strlen(Pathtype.path), Pathtype.path) ) { + best_match = Pathtype.index; } } - return CF_TYPE_INVALID; + return best_match; } diff --git a/code/cfile/cfilesystem.cpp b/code/cfile/cfilesystem.cpp index f2e10d46c38..7260016394f 100644 --- a/code/cfile/cfilesystem.cpp +++ b/code/cfile/cfilesystem.cpp @@ -96,6 +96,7 @@ typedef struct cf_file { int size; // How big it is in bytes int pack_offset; // For pack files, where it is at. 0 if not in a pack file. This can be used to tell if in a pack file. SCP_string real_name; // For real files, the full path + SCP_string sub_path; // subfolder path off of pathtype root (should include trailing directory seperator) const void* data; // For in-memory files, the data pointer cf_file() : root_index(0), pathtype_index(0), write_time(0), size(0), pack_offset(0), data(nullptr) {} @@ -188,34 +189,52 @@ cf_root *cf_create_root() struct _file_list_t { SCP_string name; + SCP_string sub_path; time_t m_time; size_t size; }; static bool sort_file_list(const _file_list_t &a, const _file_list_t &b) { + if ( a.sub_path.empty() && !b.sub_path.empty() ) { + return stricmp(a.name.c_str(), b.sub_path.c_str()) < 0; + } + + if ( !a.sub_path.empty() && b.sub_path.empty() ) { + return stricmp(a.sub_path.c_str(), b.name.c_str()) < 0; + } + + if ( !a.sub_path.empty() && !b.sub_path.empty() ) { + int rc = stricmp(a.sub_path.c_str(), b.sub_path.c_str()); + + if (rc) { + return rc < 0; + } + } + return stricmp(a.name.c_str(), b.name.c_str()) < 0; } -static size_t cf_get_list_of_files(SCP_string &path, SCP_vector<_file_list_t> &files, const char *filter = nullptr) +static size_t cf_get_list_of_files(const SCP_string &in_path, SCP_vector<_file_list_t> &files, const char *filter = nullptr, bool recursive = false, const char *subpath = nullptr) { _file_list_t nfile; + SCP_string path = in_path; + + if (path.back() != DIR_SEPARATOR_CHAR) { + path += DIR_SEPARATOR_CHAR; + } + + if (subpath) { + path += subpath; + path += DIR_SEPARATOR_CHAR; + } #if defined _WIN32 - SCP_string tmppath = path; intptr_t find_handle; _finddata_t find; - if (filter) { - if (tmppath.back() != DIR_SEPARATOR_CHAR) { - tmppath += DIR_SEPARATOR_CHAR; - } - - tmppath += filter; - } - - find_handle = _findfirst(tmppath.c_str(), &find); + find_handle = _findfirst(path.c_str(), &find); if (find_handle == -1) { return 0; @@ -223,6 +242,11 @@ static size_t cf_get_list_of_files(SCP_string &path, SCP_vector<_file_list_t> &f do { if (find.attrib & _A_SUBDIR) { + cf_get_list_of_files(path, files, filter, recursive, find.name); + continue; + } + + if (filter && !PathMatchSpec(find.name, filter)) { continue; } @@ -230,6 +254,11 @@ static size_t cf_get_list_of_files(SCP_string &path, SCP_vector<_file_list_t> &f nfile.m_time = find.time_write; nfile.size = find.size; + if (subpath) { + nfile.sub_path = subpath; + nfile.sub_path += DIR_SEPARATOR_CHAR; + } + files.push_back(nfile); } while ( !_findnext(find_handle, &find) ); @@ -248,16 +277,7 @@ static size_t cf_get_list_of_files(SCP_string &path, SCP_vector<_file_list_t> &f } while ((dir = readdir(dirp)) != nullptr) { - if (filter && fnmatch(filter, dir->d_name, 0)) { - continue; - } - filepath = path; - - if (filepath.back() != DIR_SEPARATOR_CHAR) { - filepath += DIR_SEPARATOR_CHAR; - } - filepath += dir->d_name; struct stat buf; @@ -266,14 +286,32 @@ static size_t cf_get_list_of_files(SCP_string &path, SCP_vector<_file_list_t> &f continue; } + if ( recursive && S_ISDIR(buf.st_mode) && strcmp(dir->d_name, ".") && strcmp(dir->d_name, "..") ) { + cf_get_list_of_files(path, files, filter, recursive, dir->d_name); + continue; + } + if ( !S_ISREG(buf.st_mode) ) { continue; } + if ( !buf.st_size ) { + continue; + } + + if (filter && fnmatch(filter, dir->d_name, 0)) { + continue; + } + nfile.name = dir->d_name; nfile.m_time = buf.st_mtime; nfile.size = buf.st_size; + if (subpath) { + nfile.sub_path = subpath; + nfile.sub_path += DIR_SEPARATOR_CHAR; + } + files.push_back(nfile); } @@ -281,11 +319,26 @@ static size_t cf_get_list_of_files(SCP_string &path, SCP_vector<_file_list_t> &f #endif - std::sort(files.begin(), files.end(), sort_file_list); + if ( !subpath ) { + std::sort(files.begin(), files.end(), sort_file_list); + } return files.size(); } +static bool cf_should_scan_subdirs(int pathtype) +{ + if ((pathtype <= CF_TYPE_DATA) || (pathtype >= CF_MAX_PATH_TYPES)) { + return false; + } + + if (pathtype == CF_TYPE_PLAYERS) { + return false; + } + + return true; +} + static void cf_init_root_pathtypes(cf_root *root) { @@ -726,7 +779,7 @@ void cf_search_root_path(int root_index) SCP_vector<_file_list_t> files; - cf_get_list_of_files(search_path, files, "*.*"); + cf_get_list_of_files(search_path, files, "*.*", cf_should_scan_subdirs(i)); for (auto &file : files) { auto ext_idx = file.name.rfind('.'); @@ -743,7 +796,8 @@ void cf_search_root_path(int root_index) cfile->write_time = file.m_time; cfile->size = static_cast(file.size); cfile->pack_offset = 0; - cfile->real_name = search_path + DIR_SEPARATOR_STR + file.name; + cfile->real_name = search_path + DIR_SEPARATOR_STR + file.sub_path + file.name; + cfile->sub_path = file.sub_path; ++num_files; } @@ -805,17 +859,18 @@ void cf_search_root_pack(int root_index) // Read index info fseek(fp, VP_header.index_offset, SEEK_SET); - char search_path[CF_MAX_PATHNAME_LENGTH]; - strcpy_s( search_path, "" ); - + SCP_string search_path; + SCP_string sub_path; + int path_type = CF_TYPE_INVALID; + // Go through all the files int i; for (i=0; i search_path) && (*p != DIR_SEPARATOR_CHAR) ) { - p--; + if ( !stricmp(find.filename, "..")) { + auto end = search_path.find_last_of(DIR_SEPARATOR_CHAR); + + if (end != SCP_string::npos) { + search_path.erase(end); + } else { + search_path.clear(); } - *p = 0; } else { - if ( search_path_len && (search_path[search_path_len-1] != DIR_SEPARATOR_CHAR) ) { - strcat_s( search_path, DIR_SEPARATOR_STR ); + if ( !search_path.empty() && (search_path.back() != DIR_SEPARATOR_CHAR) ) { + search_path += DIR_SEPARATOR_CHAR; } - strcat_s( search_path, find.filename ); + + search_path += find.filename; } - //mprintf(( "Current dir = '%s'\n", search_path )); + path_type = cfile_get_path_type(search_path); + + if ( (path_type > CF_TYPE_DATA) && Pathtypes[path_type].path && + (search_path.length() > strlen(Pathtypes[path_type].path)) ) + { + sub_path = search_path.substr(strlen(Pathtypes[path_type].path)+1) + DIR_SEPARATOR_STR; + } else { + sub_path.clear(); + } + + //mprintf(( "Current dir = '%s'\n", search_path.c_str() )); } else { - - int j; - - for (j=CF_TYPE_ROOT; jname_ext = find.filename; - file->root_index = root_index; - file->pathtype_index = j; - file->write_time = (time_t)find.write_time; - file->size = find.size; - file->pack_offset = find.offset; // Mark as a packed file - - num_files++; - //mprintf(( "Found pack file '%s'\n", file->name_ext )); - } + if (path_type != CF_TYPE_INVALID) { + char *ext = strrchr( find.filename, '.' ); + if ( ext ) { + if ( is_ext_in_list( Pathtypes[path_type].extensions, ext ) ) { + // Found a file!!!! + cf_file *file = cf_create_file(); + + file->name_ext = find.filename; + file->root_index = root_index; + file->pathtype_index = path_type; + file->write_time = (time_t)find.write_time; + file->size = find.size; + file->pack_offset = find.offset; // Mark as a packed file + file->sub_path = sub_path; + + num_files++; + //mprintf(( "Found pack file '%s'\n", file->name_ext )); } } } @@ -965,6 +1027,34 @@ void cf_free_secondary_filelist() Num_files = 0; } +static bool is_absolute_path(const char *path) +{ + if ( !path || !strlen(path) ) { + return false; + } + +#ifdef WIN32 + return (PathIsRelative(path) == FALSE); +#else + return (*path == '/'); +#endif +} + +static bool sub_path_match(const SCP_string &search, const SCP_string &index) +{ + // if search path is empty then we only care about the file name + if (search.empty()) { + return true; + } + + // if we have search but no index then fail + if (index.empty()) { + return false; + } + + return !stricmp(search.c_str(), index.c_str()); +} + /** * Searches for a file. * @@ -993,10 +1083,9 @@ CFileLocation cf_find_file_location(const char* filespec, int pathtype, bool loc // of the file // NOTE: full path should also include localization, if so desired - auto last_separator = strrchr(filespec, DIR_SEPARATOR_CHAR); // do we have a full path already? - if ( last_separator ) { + if (is_absolute_path(filespec)) { FILE *fp = fopen(filespec, "rb" ); if (fp) { CFileLocation res(true); @@ -1004,6 +1093,10 @@ CFileLocation cf_find_file_location(const char* filespec, int pathtype, bool loc fclose(fp); + auto last_separator = strrchr(filespec, DIR_SEPARATOR_CHAR); + + Assertion(last_separator != nullptr, "We have full path, but no separator!?"); + res.offset = 0; res.full_name = filespec; res.name_ext = last_separator + 1; @@ -1074,6 +1167,28 @@ CFileLocation cf_find_file_location(const char* filespec, int pathtype, bool loc } } + // fixup filename and sub directory path, if needed + SCP_string filename = filespec; + SCP_string sub_path; + + auto seperator = filename.find_last_of("\\/"); + + if (seperator != SCP_string::npos) { + sub_path = filename.substr(0, seperator); + sub_path += DIR_SEPARATOR_STR; + + filename.erase(0, seperator+1); + + // fix separators in sub path + char bad_sep = '/'; + + if (bad_sep == DIR_SEPARATOR_CHAR) { + bad_sep = '\\'; + } + + std::replace(sub_path.begin(), sub_path.end(), bad_sep, DIR_SEPARATOR_CHAR); + } + // Search the pak files and CD-ROM. for (ui = 0; ui < Num_files; ui++ ) { cf_file *f = cf_get_file(ui); @@ -1125,7 +1240,7 @@ CFileLocation cf_find_file_location(const char* filespec, int pathtype, bool loc } // file either not localized or localized version not found - if ( !stricmp(filespec, f->name_ext.c_str()) ) { + if ( !stricmp(filename.c_str(), f->name_ext.c_str()) && sub_path_match(sub_path, f->sub_path) ) { CFileLocation res(true); res.size = static_cast(f->size); res.offset = (size_t)f->pack_offset; @@ -1136,6 +1251,7 @@ CFileLocation cf_find_file_location(const char* filespec, int pathtype, bool loc // This is an in-memory file so we just copy the pathtype name + file name res.full_name = Pathtypes[f->pathtype_index].path; res.full_name += DIR_SEPARATOR_STR; + res.full_name += f->sub_path; res.full_name += f->name_ext; } else if (f->pack_offset < 1) { // This is a real file, return the actual file path @@ -2032,7 +2148,7 @@ int cf_create_default_path_string(SCP_string& path, int pathtype, const char* fi { uint32_t location_flags = _location_flags; - if ( filename && strchr(filename, DIR_SEPARATOR_CHAR) ) { + if ( filename && is_absolute_path(filename)) { // Already has full path path.assign(filename); diff --git a/code/def_files/data/scripts/cfile_require.lua b/code/def_files/data/scripts/cfile_require.lua index 06629421e21..8a1a7f3f912 100644 --- a/code/def_files/data/scripts/cfile_require.lua +++ b/code/def_files/data/scripts/cfile_require.lua @@ -8,7 +8,7 @@ local function cfileLoader(name) local file = nil for i,v in ipairs(exts) do - local real_name = name .. v + local real_name = string.gsub(name, "%.", "/") .. v file = cf.openFile(real_name, "r", "data/scripts") if (file:isValid()) then From 8c02e9137d68522170328ad0587760cc52ce63fe Mon Sep 17 00:00:00 2001 From: Taylor Richards Date: Sun, 9 Oct 2022 01:26:09 -0400 Subject: [PATCH 02/21] sort files in VP archives before indexing them --- code/cfile/cfilesystem.cpp | 69 +++++++++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 13 deletions(-) diff --git a/code/cfile/cfilesystem.cpp b/code/cfile/cfilesystem.cpp index 7260016394f..c7373d9fc8c 100644 --- a/code/cfile/cfilesystem.cpp +++ b/code/cfile/cfilesystem.cpp @@ -192,6 +192,10 @@ struct _file_list_t { SCP_string sub_path; time_t m_time; size_t size; + int pathtype; + int offset; + + _file_list_t() : m_time(0), size(0), pathtype(CF_TYPE_INVALID), offset(0) {} }; static bool sort_file_list(const _file_list_t &a, const _file_list_t &b) @@ -821,6 +825,29 @@ typedef struct VP_FILE { _fs_time_t write_time; } VP_FILE; +static int cf_add_pack_files(const int root_index, SCP_vector<_file_list_t> &files) +{ + if (files.empty()) { + return 0; + } + + std::sort(files.begin(), files.end(), sort_file_list); + + for (auto &file : files) { + cf_file *pf = cf_create_file(); + + pf->name_ext = file.name; + pf->root_index = root_index; + pf->pathtype_index = file.pathtype; + pf->write_time = file.m_time; + pf->size = file.size; + pf->pack_offset = file.offset; // Mark as a packed file + pf->sub_path = file.sub_path; + } + + return static_cast(files.size()); +} + void cf_search_root_pack(int root_index) { int num_files = 0; @@ -864,6 +891,10 @@ void cf_search_root_pack(int root_index) SCP_string sub_path; int path_type = CF_TYPE_INVALID; + SCP_vector<_file_list_t> files; + + files.reserve(256); // should be set to a good baseline of files per path + // Go through all the files int i; for (i=0; i CF_TYPE_DATA) && Pathtypes[path_type].path && (search_path.length() > strlen(Pathtypes[path_type].path)) ) @@ -913,24 +952,28 @@ void cf_search_root_pack(int root_index) if ( ext ) { if ( is_ext_in_list( Pathtypes[path_type].extensions, ext ) ) { // Found a file!!!! - cf_file *file = cf_create_file(); - - file->name_ext = find.filename; - file->root_index = root_index; - file->pathtype_index = path_type; - file->write_time = (time_t)find.write_time; - file->size = find.size; - file->pack_offset = find.offset; // Mark as a packed file - file->sub_path = sub_path; - - num_files++; - //mprintf(( "Found pack file '%s'\n", file->name_ext )); + _file_list_t file; + + file.name = find.filename; + file.m_time = find.write_time; + file.size = find.size; + file.sub_path = sub_path; + file.pathtype = path_type; + file.offset = find.offset; + + files.push_back(file); + + //mprintf(( "Found pack file '%s'\n", find.filename )); } } } } } + // add final set of files + num_files += cf_add_pack_files(root_index, files); + files.clear(); + fclose(fp); mprintf(( "%i files\n", num_files )); From ea126d614b3be7c1ad25701aefc9a78c4a81e48a Mon Sep 17 00:00:00 2001 From: Taylor Richards Date: Sun, 9 Oct 2022 11:55:33 -0400 Subject: [PATCH 03/21] fix indexing of sub-subfolders --- code/cfile/cfilesystem.cpp | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/code/cfile/cfilesystem.cpp b/code/cfile/cfilesystem.cpp index c7373d9fc8c..8432cf5e6a5 100644 --- a/code/cfile/cfilesystem.cpp +++ b/code/cfile/cfilesystem.cpp @@ -246,7 +246,17 @@ static size_t cf_get_list_of_files(const SCP_string &in_path, SCP_vector<_file_l do { if (find.attrib & _A_SUBDIR) { - cf_get_list_of_files(path, files, filter, recursive, find.name); + SCP_string sub; + + if (subpath) { + sub = subpath; + sub += DIR_SEPARATOR_CHAR; + } + + sub += find.name; + + cf_get_list_of_files(in_path, files, filter, recursive, sub.c_str()); + continue; } @@ -291,7 +301,17 @@ static size_t cf_get_list_of_files(const SCP_string &in_path, SCP_vector<_file_l } if ( recursive && S_ISDIR(buf.st_mode) && strcmp(dir->d_name, ".") && strcmp(dir->d_name, "..") ) { - cf_get_list_of_files(path, files, filter, recursive, dir->d_name); + SCP_string sub; + + if (subpath) { + sub = subpath; + sub += DIR_SEPARATOR_CHAR; + } + + sub += dir->d_name; + + cf_get_list_of_files(in_path, files, filter, recursive, sub.c_str()); + continue; } From 2e0a8d6bb681bf74e4ee375c82544a6902043e6b Mon Sep 17 00:00:00 2001 From: Taylor Richards Date: Sun, 9 Oct 2022 11:57:41 -0400 Subject: [PATCH 04/21] remove unused 8B22K & 16B11K sound path types --- code/cfile/cfile.cpp | 2 -- code/cfile/cfile.h | 66 +++++++++++++++++++++----------------------- 2 files changed, 32 insertions(+), 36 deletions(-) diff --git a/code/cfile/cfile.cpp b/code/cfile/cfile.cpp index 1db075c585f..d33394afebc 100644 --- a/code/cfile/cfile.cpp +++ b/code/cfile/cfile.cpp @@ -59,8 +59,6 @@ cf_pathtype Pathtypes[CF_MAX_PATH_TYPES] = { { CF_TYPE_MODELS, "data" DIR_SEPARATOR_STR "models", ".pof", CF_TYPE_DATA }, { CF_TYPE_TABLES, "data" DIR_SEPARATOR_STR "tables", ".tbl .tbm .lua", CF_TYPE_DATA }, { CF_TYPE_SOUNDS, "data" DIR_SEPARATOR_STR "sounds", ".wav .ogg", CF_TYPE_DATA }, - { CF_TYPE_SOUNDS_8B22K, "data" DIR_SEPARATOR_STR "sounds" DIR_SEPARATOR_STR "8b22k", ".wav .ogg", CF_TYPE_SOUNDS }, - { CF_TYPE_SOUNDS_16B11K, "data" DIR_SEPARATOR_STR "sounds" DIR_SEPARATOR_STR "16b11k", ".wav .ogg", CF_TYPE_SOUNDS }, { CF_TYPE_VOICE, "data" DIR_SEPARATOR_STR "voice", "", CF_TYPE_DATA }, { CF_TYPE_VOICE_BRIEFINGS, "data" DIR_SEPARATOR_STR "voice" DIR_SEPARATOR_STR "briefing", ".wav .ogg", CF_TYPE_VOICE }, { CF_TYPE_VOICE_CMD_BRIEF, "data" DIR_SEPARATOR_STR "voice" DIR_SEPARATOR_STR "command_briefings", ".wav .ogg", CF_TYPE_VOICE }, diff --git a/code/cfile/cfile.h b/code/cfile/cfile.h index 17d32ff4a22..e41c7b33d02 100644 --- a/code/cfile/cfile.h +++ b/code/cfile/cfile.h @@ -49,42 +49,40 @@ typedef struct { #define CF_TYPE_MODELS 5 #define CF_TYPE_TABLES 6 #define CF_TYPE_SOUNDS 7 -#define CF_TYPE_SOUNDS_8B22K 8 -#define CF_TYPE_SOUNDS_16B11K 9 -#define CF_TYPE_VOICE 10 -#define CF_TYPE_VOICE_BRIEFINGS 11 -#define CF_TYPE_VOICE_CMD_BRIEF 12 -#define CF_TYPE_VOICE_DEBRIEFINGS 13 -#define CF_TYPE_VOICE_PERSONAS 14 -#define CF_TYPE_VOICE_SPECIAL 15 -#define CF_TYPE_VOICE_TRAINING 16 -#define CF_TYPE_MUSIC 17 -#define CF_TYPE_MOVIES 18 -#define CF_TYPE_INTERFACE 19 -#define CF_TYPE_FONT 20 -#define CF_TYPE_EFFECTS 21 -#define CF_TYPE_HUD 22 -#define CF_TYPE_PLAYERS 23 -#define CF_TYPE_PLAYER_IMAGES 24 -#define CF_TYPE_SQUAD_IMAGES 25 -#define CF_TYPE_SINGLE_PLAYERS 26 -#define CF_TYPE_MULTI_PLAYERS 27 -#define CF_TYPE_CACHE 28 -#define CF_TYPE_MULTI_CACHE 29 -#define CF_TYPE_MISSIONS 30 -#define CF_TYPE_CONFIG 31 -#define CF_TYPE_DEMOS 32 -#define CF_TYPE_CBANIMS 33 -#define CF_TYPE_INTEL_ANIMS 34 -#define CF_TYPE_SCRIPTS 35 -#define CF_TYPE_FICTION 36 -#define CF_TYPE_FREDDOCS 37 -#define CF_TYPE_INTERFACE_MARKUP 38 -#define CF_TYPE_INTERFACE_CSS 39 -#define CF_TYPE_PLAYER_BINDS 40 +#define CF_TYPE_VOICE 8 +#define CF_TYPE_VOICE_BRIEFINGS 9 +#define CF_TYPE_VOICE_CMD_BRIEF 10 +#define CF_TYPE_VOICE_DEBRIEFINGS 11 +#define CF_TYPE_VOICE_PERSONAS 12 +#define CF_TYPE_VOICE_SPECIAL 13 +#define CF_TYPE_VOICE_TRAINING 14 +#define CF_TYPE_MUSIC 15 +#define CF_TYPE_MOVIES 16 +#define CF_TYPE_INTERFACE 17 +#define CF_TYPE_FONT 18 +#define CF_TYPE_EFFECTS 19 +#define CF_TYPE_HUD 20 +#define CF_TYPE_PLAYERS 21 +#define CF_TYPE_PLAYER_IMAGES 22 +#define CF_TYPE_SQUAD_IMAGES 23 +#define CF_TYPE_SINGLE_PLAYERS 24 +#define CF_TYPE_MULTI_PLAYERS 25 +#define CF_TYPE_CACHE 26 +#define CF_TYPE_MULTI_CACHE 27 +#define CF_TYPE_MISSIONS 28 +#define CF_TYPE_CONFIG 29 +#define CF_TYPE_DEMOS 30 +#define CF_TYPE_CBANIMS 31 +#define CF_TYPE_INTEL_ANIMS 32 +#define CF_TYPE_SCRIPTS 33 +#define CF_TYPE_FICTION 34 +#define CF_TYPE_FREDDOCS 35 +#define CF_TYPE_INTERFACE_MARKUP 36 +#define CF_TYPE_INTERFACE_CSS 37 +#define CF_TYPE_PLAYER_BINDS 38 #define CF_MAX_PATH_TYPES \ - 41 // Can be as high as you'd like //DTP; yeah but beware alot of things uses CF_MAX_PATH_TYPES + 39 // Can be as high as you'd like //DTP; yeah but beware alot of things uses CF_MAX_PATH_TYPES // TRUE if type is specified and valid #define CF_TYPE_SPECIFIED(path_type) (((path_type)>CF_TYPE_INVALID) && ((path_type) Date: Sun, 9 Oct 2022 12:02:01 -0400 Subject: [PATCH 05/21] add more items to skip subfolder scanning --- code/cfile/cfilesystem.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/code/cfile/cfilesystem.cpp b/code/cfile/cfilesystem.cpp index 8432cf5e6a5..fe896ebc0ea 100644 --- a/code/cfile/cfilesystem.cpp +++ b/code/cfile/cfilesystem.cpp @@ -356,8 +356,18 @@ static bool cf_should_scan_subdirs(int pathtype) return false; } - if (pathtype == CF_TYPE_PLAYERS) { - return false; + switch (pathtype) { + // all pilot file related directories, except for images, should be ignored + case CF_TYPE_PLAYERS: + case CF_TYPE_SINGLE_PLAYERS: + case CF_TYPE_MULTI_PLAYERS: + case CF_TYPE_PLAYER_BINDS: + // voice, missing extensions, has no automatic subfolder support + case CF_TYPE_VOICE: + return false; + + default: + break; } return true; From 46c30f02d90591c745203b260c7d8affdb6ff1e4 Mon Sep 17 00:00:00 2001 From: Taylor Richards Date: Sun, 9 Oct 2022 21:10:22 -0400 Subject: [PATCH 06/21] fix windows build issues --- code/cfile/cfilesystem.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/code/cfile/cfilesystem.cpp b/code/cfile/cfilesystem.cpp index fe896ebc0ea..0c860dd233f 100644 --- a/code/cfile/cfilesystem.cpp +++ b/code/cfile/cfilesystem.cpp @@ -21,6 +21,7 @@ #include #include #include /* needed for memory mapping of file functions */ +#include #endif #ifdef SCP_UNIX @@ -870,7 +871,7 @@ static int cf_add_pack_files(const int root_index, SCP_vector<_file_list_t> &fil pf->root_index = root_index; pf->pathtype_index = file.pathtype; pf->write_time = file.m_time; - pf->size = file.size; + pf->size = static_cast(file.size); pf->pack_offset = file.offset; // Mark as a packed file pf->sub_path = file.sub_path; } From 048c14cfa81a4f0fba573eaf69f51b488d441065 Mon Sep 17 00:00:00 2001 From: Taylor Richards Date: Mon, 10 Oct 2022 13:13:35 -0400 Subject: [PATCH 07/21] fix unit tests by allowing 0-byte files to be indexed --- code/cfile/cfilesystem.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/code/cfile/cfilesystem.cpp b/code/cfile/cfilesystem.cpp index 0c860dd233f..7d78923645e 100644 --- a/code/cfile/cfilesystem.cpp +++ b/code/cfile/cfilesystem.cpp @@ -320,9 +320,10 @@ static size_t cf_get_list_of_files(const SCP_string &in_path, SCP_vector<_file_l continue; } - if ( !buf.st_size ) { - continue; - } + // zero byte files shouldn't be indexed, but that breaks unit tests + // if ( !buf.st_size ) { + // continue; + // } if (filter && fnmatch(filter, dir->d_name, 0)) { continue; From c126d13a8c39864401e130b97545a2ef8e3af98e Mon Sep 17 00:00:00 2001 From: Taylor Richards Date: Mon, 10 Oct 2022 23:52:37 -0400 Subject: [PATCH 08/21] fix file indexing on windows --- code/cfile/cfilesystem.cpp | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/code/cfile/cfilesystem.cpp b/code/cfile/cfilesystem.cpp index 7d78923645e..3609be2bd2e 100644 --- a/code/cfile/cfilesystem.cpp +++ b/code/cfile/cfilesystem.cpp @@ -239,6 +239,9 @@ static size_t cf_get_list_of_files(const SCP_string &in_path, SCP_vector<_file_l intptr_t find_handle; _finddata_t find; + // make sure we return all entries by default + path += "*"; + find_handle = _findfirst(path.c_str(), &find); if (find_handle == -1) { @@ -246,17 +249,20 @@ static size_t cf_get_list_of_files(const SCP_string &in_path, SCP_vector<_file_l } do { + if (find.attrib & _A_SUBDIR) { - SCP_string sub; + if ( recursive && strcmp(find.name, ".") && strcmp(find.name, "..") ) { + SCP_string sub; - if (subpath) { - sub = subpath; - sub += DIR_SEPARATOR_CHAR; - } + if (subpath) { + sub = subpath; + sub += DIR_SEPARATOR_CHAR; + } - sub += find.name; + sub += find.name; - cf_get_list_of_files(in_path, files, filter, recursive, sub.c_str()); + cf_get_list_of_files(in_path, files, filter, recursive, sub.c_str()); + } continue; } @@ -449,7 +455,7 @@ static void cf_init_root_pathtypes(cf_root *root) #endif } -static SCP_string cf_get_root_pathtype(const cf_root *root, const int type) +static SCP_string cf_get_root_pathtype(__UNUSED const cf_root *root, const int type) { #ifdef SCP_UNIX auto parentPathIter = root->pathTypeToRealPath.find(type); From 562f443d02f07d0aa2b7483ddbe7a6a7e08d3171 Mon Sep 17 00:00:00 2001 From: Taylor Richards Date: Tue, 11 Oct 2022 00:36:14 -0400 Subject: [PATCH 09/21] add test for cfile subfolders --- test/src/cfile/cfile.cpp | 17 +++++++++++++++++ .../cfile/subfolders/data/tables/file.tbl | 0 .../cfile/subfolders/data/tables/sub/file2.tbl | 0 .../subfolders/data/tables/sub/folder/file3.tbl | 0 4 files changed, 17 insertions(+) create mode 100644 test/test_data/cfile/subfolders/data/tables/file.tbl create mode 100644 test/test_data/cfile/subfolders/data/tables/sub/file2.tbl create mode 100644 test/test_data/cfile/subfolders/data/tables/sub/folder/file3.tbl diff --git a/test/src/cfile/cfile.cpp b/test/src/cfile/cfile.cpp index 1e11ae76650..b4f93be9278 100644 --- a/test/src/cfile/cfile.cpp +++ b/test/src/cfile/cfile.cpp @@ -152,3 +152,20 @@ TEST_F(CFileTest, test_get_path_type) ASSERT_EQ(CF_TYPE_INTERFACE_MARKUP, cfile_get_path_type("data/interface/markup\\\\\\")); ASSERT_EQ(CF_TYPE_INTERFACE_MARKUP, cfile_get_path_type("////data/interface/markup\\\\\\")); } + +TEST_F(CFileTest, subfolders) +{ + // base check for all files + ASSERT_TRUE(cf_exists("file.tbl", CF_TYPE_TABLES)); + ASSERT_TRUE(cf_exists("file2.tbl", CF_TYPE_TABLES)); + ASSERT_TRUE(cf_exists("file3.tbl", CF_TYPE_TABLES)); + + // good direct subfolder check + ASSERT_TRUE(cf_exists("sub/file2.tbl", CF_TYPE_TABLES)); + + // bad direct subfolder check + ASSERT_FALSE(cf_exists("sub/file3.tbl", CF_TYPE_TABLES)); + + // sub-subfolder check + ASSERT_TRUE(cf_exists("sub/folder/file3.tbl", CF_TYPE_TABLES)); +} \ No newline at end of file diff --git a/test/test_data/cfile/subfolders/data/tables/file.tbl b/test/test_data/cfile/subfolders/data/tables/file.tbl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/test_data/cfile/subfolders/data/tables/sub/file2.tbl b/test/test_data/cfile/subfolders/data/tables/sub/file2.tbl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/test_data/cfile/subfolders/data/tables/sub/folder/file3.tbl b/test/test_data/cfile/subfolders/data/tables/sub/folder/file3.tbl new file mode 100644 index 00000000000..e69de29bb2d From 10148c53e5c21e81e18bd991eb263c8a2b7dd84d Mon Sep 17 00:00:00 2001 From: Taylor Richards Date: Wed, 12 Oct 2022 02:49:45 -0400 Subject: [PATCH 10/21] remove specific 'markup' and 'css' interface subfolders They aren't directly referenced anywhere by pathtype and the new subfolder code will continue to index and load files from them --- code/cfile/cfile.cpp | 8 +++----- code/cfile/cfile.h | 26 ++++++++++++-------------- test/src/cfile/cfile.cpp | 16 ++++++++-------- 3 files changed, 23 insertions(+), 27 deletions(-) diff --git a/code/cfile/cfile.cpp b/code/cfile/cfile.cpp index d33394afebc..ecef35ad581 100644 --- a/code/cfile/cfile.cpp +++ b/code/cfile/cfile.cpp @@ -68,7 +68,7 @@ cf_pathtype Pathtypes[CF_MAX_PATH_TYPES] = { { CF_TYPE_VOICE_TRAINING, "data" DIR_SEPARATOR_STR "voice" DIR_SEPARATOR_STR "training", ".wav .ogg", CF_TYPE_VOICE }, { CF_TYPE_MUSIC, "data" DIR_SEPARATOR_STR "music", ".wav .ogg", CF_TYPE_DATA }, { CF_TYPE_MOVIES, "data" DIR_SEPARATOR_STR "movies", ".mve .msb .ogg .mp4 .srt .webm .png",CF_TYPE_DATA }, - { CF_TYPE_INTERFACE, "data" DIR_SEPARATOR_STR "interface", ".pcx .ani .dds .tga .eff .png .jpg", CF_TYPE_DATA }, + { CF_TYPE_INTERFACE, "data" DIR_SEPARATOR_STR "interface", ".pcx .ani .dds .tga .eff .png .jpg .rml .rcss", CF_TYPE_DATA }, { CF_TYPE_FONT, "data" DIR_SEPARATOR_STR "fonts", ".vf .ttf .otf", CF_TYPE_DATA }, { CF_TYPE_EFFECTS, "data" DIR_SEPARATOR_STR "effects", ".ani .eff .pcx .neb .tga .jpg .png .dds .sdr", CF_TYPE_DATA }, { CF_TYPE_HUD, "data" DIR_SEPARATOR_STR "hud", ".pcx .ani .eff .tga .jpg .png .dds", CF_TYPE_DATA }, @@ -77,6 +77,7 @@ cf_pathtype Pathtypes[CF_MAX_PATH_TYPES] = { { CF_TYPE_SQUAD_IMAGES, "data" DIR_SEPARATOR_STR "players" DIR_SEPARATOR_STR "squads", ".pcx .png .dds", CF_TYPE_PLAYERS }, { CF_TYPE_SINGLE_PLAYERS, "data" DIR_SEPARATOR_STR "players" DIR_SEPARATOR_STR "single", ".pl2 .cs2 .plr .csg .css .json", CF_TYPE_PLAYERS }, { CF_TYPE_MULTI_PLAYERS, "data" DIR_SEPARATOR_STR "players" DIR_SEPARATOR_STR "multi", ".plr .json", CF_TYPE_PLAYERS }, + { CF_TYPE_PLAYER_BINDS, "data" DIR_SEPARATOR_STR "players" DIR_SEPARATOR_STR "presets", ".json", CF_TYPE_PLAYERS }, { CF_TYPE_CACHE, "data" DIR_SEPARATOR_STR "cache", ".clr .tmp .bx", CF_TYPE_DATA }, //clr=cached color { CF_TYPE_MULTI_CACHE, "data" DIR_SEPARATOR_STR "multidata", ".pcx .png .jpg .dds .fs2 .txt", CF_TYPE_DATA }, { CF_TYPE_MISSIONS, "data" DIR_SEPARATOR_STR "missions", ".fs2 .fc2 .ntl .ssv", CF_TYPE_DATA }, @@ -86,10 +87,7 @@ cf_pathtype Pathtypes[CF_MAX_PATH_TYPES] = { { CF_TYPE_INTEL_ANIMS, "data" DIR_SEPARATOR_STR "intelanims", ".pcx .ani .eff .tga .jpg .png .dds", CF_TYPE_DATA }, { CF_TYPE_SCRIPTS, "data" DIR_SEPARATOR_STR "scripts", ".lua .lc .fnl", CF_TYPE_DATA }, { CF_TYPE_FICTION, "data" DIR_SEPARATOR_STR "fiction", ".txt", CF_TYPE_DATA }, - { CF_TYPE_FREDDOCS, "data" DIR_SEPARATOR_STR "freddocs", ".html", CF_TYPE_DATA }, - { CF_TYPE_INTERFACE_MARKUP, "data" DIR_SEPARATOR_STR "interface" DIR_SEPARATOR_STR "markup", ".rml", CF_TYPE_INTERFACE }, - { CF_TYPE_INTERFACE_CSS, "data" DIR_SEPARATOR_STR "interface" DIR_SEPARATOR_STR "css", ".rcss", CF_TYPE_INTERFACE }, - { CF_TYPE_PLAYER_BINDS, "data" DIR_SEPARATOR_STR "players" DIR_SEPARATOR_STR "presets", ".json", CF_TYPE_PLAYERS }, + { CF_TYPE_FREDDOCS, "data" DIR_SEPARATOR_STR "freddocs", ".html", CF_TYPE_DATA } }; // clang-format on diff --git a/code/cfile/cfile.h b/code/cfile/cfile.h index e41c7b33d02..841895541a5 100644 --- a/code/cfile/cfile.h +++ b/code/cfile/cfile.h @@ -67,22 +67,20 @@ typedef struct { #define CF_TYPE_SQUAD_IMAGES 23 #define CF_TYPE_SINGLE_PLAYERS 24 #define CF_TYPE_MULTI_PLAYERS 25 -#define CF_TYPE_CACHE 26 -#define CF_TYPE_MULTI_CACHE 27 -#define CF_TYPE_MISSIONS 28 -#define CF_TYPE_CONFIG 29 -#define CF_TYPE_DEMOS 30 -#define CF_TYPE_CBANIMS 31 -#define CF_TYPE_INTEL_ANIMS 32 -#define CF_TYPE_SCRIPTS 33 -#define CF_TYPE_FICTION 34 -#define CF_TYPE_FREDDOCS 35 -#define CF_TYPE_INTERFACE_MARKUP 36 -#define CF_TYPE_INTERFACE_CSS 37 -#define CF_TYPE_PLAYER_BINDS 38 +#define CF_TYPE_PLAYER_BINDS 26 +#define CF_TYPE_CACHE 27 +#define CF_TYPE_MULTI_CACHE 28 +#define CF_TYPE_MISSIONS 29 +#define CF_TYPE_CONFIG 30 +#define CF_TYPE_DEMOS 31 +#define CF_TYPE_CBANIMS 32 +#define CF_TYPE_INTEL_ANIMS 33 +#define CF_TYPE_SCRIPTS 34 +#define CF_TYPE_FICTION 35 +#define CF_TYPE_FREDDOCS 36 #define CF_MAX_PATH_TYPES \ - 39 // Can be as high as you'd like //DTP; yeah but beware alot of things uses CF_MAX_PATH_TYPES + 37 // Can be as high as you'd like //DTP; yeah but beware alot of things uses CF_MAX_PATH_TYPES // TRUE if type is specified and valid #define CF_TYPE_SPECIFIED(path_type) (((path_type)>CF_TYPE_INVALID) && ((path_type) Date: Wed, 12 Oct 2022 02:53:10 -0400 Subject: [PATCH 11/21] add player binds to list of always-root locations --- code/cfile/cfilesystem.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/code/cfile/cfilesystem.cpp b/code/cfile/cfilesystem.cpp index 3609be2bd2e..862fd56ede9 100644 --- a/code/cfile/cfilesystem.cpp +++ b/code/cfile/cfilesystem.cpp @@ -2242,6 +2242,7 @@ int cf_create_default_path_string(SCP_string& path, int pathtype, const char* fi case CF_TYPE_PLAYERS: case CF_TYPE_MULTI_PLAYERS: case CF_TYPE_SINGLE_PLAYERS: + case CF_TYPE_PLAYER_BINDS: location_flags = CF_LOCATION_ROOT_USER | CF_LOCATION_ROOT_GAME | CF_LOCATION_TYPE_ROOT; break; } From 1d766e09f60a95175630025e41f69ce3f6cce555 Mon Sep 17 00:00:00 2001 From: Taylor Richards Date: Sat, 15 Oct 2022 01:31:55 -0400 Subject: [PATCH 12/21] remove file-based localization support An alternative is left in place for old fonts, as that was the only place it appeared to be used. This method of file localization conflicts with subfolder support however so if it's needed in the future then changes to the subfolder code will be necessary. --- code/bmpman/bmpman.cpp | 2 +- code/cfile/cfile.cpp | 12 ++- code/cfile/cfile.h | 5 +- code/cfile/cfilesystem.cpp | 108 +++++-------------------- code/cfile/cfilesystem.h | 4 +- code/graphics/generic.cpp | 2 +- code/graphics/software/FontManager.cpp | 16 +++- code/localization/localize.cpp | 31 +++++++ code/localization/localize.h | 1 + code/pilotfile/csg_convert.cpp | 2 +- code/pilotfile/plr_convert.cpp | 2 +- code/sound/ffmpeg/FFmpegWaveFile.cpp | 4 +- fred2/campaigneditordlg.cpp | 2 +- 13 files changed, 80 insertions(+), 111 deletions(-) diff --git a/code/bmpman/bmpman.cpp b/code/bmpman/bmpman.cpp index 5c9a13b4b2c..2d45cf25467 100644 --- a/code/bmpman/bmpman.cpp +++ b/code/bmpman/bmpman.cpp @@ -1835,7 +1835,7 @@ int bm_load_sub_fast(const char *real_filename, int *handle, int dir_type, bool } int bm_load_sub_slow(const char *real_filename, const int num_ext, const char **ext_list, CFILE **img_cfp, int dir_type) { - auto res = cf_find_file_location_ext(real_filename, num_ext, ext_list, dir_type, false); + auto res = cf_find_file_location_ext(real_filename, num_ext, ext_list, dir_type); // could not be found, or is invalid for some reason if (!res.found) diff --git a/code/cfile/cfile.cpp b/code/cfile/cfile.cpp index ecef35ad581..9aee12172ac 100644 --- a/code/cfile/cfile.cpp +++ b/code/cfile/cfile.cpp @@ -469,7 +469,7 @@ int cf_delete(const char *filename, int path_type, uint32_t location_flags) Assert(CF_TYPE_SPECIFIED(path_type)); - cf_create_default_path_string(longname, path_type, filename, false, location_flags); + cf_create_default_path_string(longname, path_type, filename, location_flags); return (_unlink(longname.c_str()) != -1); } @@ -619,7 +619,7 @@ void cf_create_directory(int dir_type, uint32_t location_flags) int i; for (i=num_dirs-1; i>=0; i-- ) { - cf_create_default_path_string(longname, dir_tree[i], nullptr, false, location_flags); + cf_create_default_path_string(longname, dir_tree[i], nullptr, location_flags); if (stat(longname.c_str(), &statbuf) != 0) { mprintf(( "CFILE: Creating new directory '%s'\n", longname.c_str() )); mkdir_recursive(longname.c_str()); @@ -644,7 +644,7 @@ void cf_create_directory(int dir_type, uint32_t location_flags) // CFILE* _cfopen(const char* source, int line, const char* file_path, const char* mode, int type, int dir_type, - bool localize, uint32_t location_flags) + bool /* localize */, uint32_t location_flags) { /* Bobboau, what is this doing here? 31 is way too short... - Goober5000 if( strlen(file_path) > 31 ) @@ -685,7 +685,7 @@ CFILE* _cfopen(const char* source, int line, const char* file_path, const char* // Create the directory if necessary cf_create_directory(dir_type, location_flags); - cf_create_default_path_string(longname, dir_type, file_path, false, location_flags); + cf_create_default_path_string(longname, dir_type, file_path, location_flags); } Assert( !(type & CFILE_MEMORY_MAPPED) ); @@ -735,10 +735,8 @@ CFILE* _cfopen(const char* source, int line, const char* file_path, const char* //================================================ // Search for file on disk, on cdrom, or in a packfile - char copy_file_path[MAX_PATH_LEN]; // FIX change in memory from cf_find_file_location - strcpy_s(copy_file_path, file_path); + auto find_res = cf_find_file_location(file_path, dir_type, location_flags); - auto find_res = cf_find_file_location( copy_file_path, dir_type, localize, location_flags ); if ( find_res.found ) { // Fount it, now create a cfile out of it diff --git a/code/cfile/cfile.h b/code/cfile/cfile.h index 841895541a5..4754f65bea5 100644 --- a/code/cfile/cfile.h +++ b/code/cfile/cfile.h @@ -391,7 +391,7 @@ struct CFileLocation { // size - File size // offset - Offset into pack file. 0 if not a packfile. // Returns: If not found returns 0. -CFileLocation cf_find_file_location(const char* filespec, int pathtype, bool localize = false, +CFileLocation cf_find_file_location(const char* filespec, int pathtype, uint32_t location_flags = CF_LOCATION_ALL); struct CFileLocationExt : public CFileLocation { @@ -415,8 +415,7 @@ struct CFileLocationExt : public CFileLocation { // offset - Offset into pack file. 0 if not a packfile. // Returns: If not found returns -1, else returns offset into ext_list. // (NOTE: This function is exponentially slow, so don't use it unless truely needed!!) -CFileLocationExt cf_find_file_location_ext(const char* filename, const int ext_num, const char** ext_list, int pathtype, - bool localize = false); +CFileLocationExt cf_find_file_location_ext(const char* filename, const int ext_num, const char** ext_list, int pathtype); // Functions to change directories int cfile_chdir(const char *dir); diff --git a/code/cfile/cfilesystem.cpp b/code/cfile/cfilesystem.cpp index 862fd56ede9..ee43ba0be53 100644 --- a/code/cfile/cfilesystem.cpp +++ b/code/cfile/cfilesystem.cpp @@ -38,7 +38,6 @@ #include "cmdline/cmdline.h" #include "globalincs/pstypes.h" #include "def_files/def_files.h" -#include "localization/localize.h" #include "osapi/osapi.h" #include "parse/parselo.h" @@ -1143,17 +1142,16 @@ static bool sub_path_match(const SCP_string &search, const SCP_string &index) * * @param filespec Filename & extension * @param pathtype See CF_TYPE_ defines in CFILE.H - * @param localize Undertake localization * @param location_flags Specifies where to search for the specified flag * * @return A structure which describes the found file */ -CFileLocation cf_find_file_location(const char* filespec, int pathtype, bool localize, uint32_t location_flags) +CFileLocation cf_find_file_location(const char* filespec, int pathtype, uint32_t location_flags) { int i; uint ui; int cfs_slow_search = 0; - char longname[MAX_PATH_LEN]; + SCP_string longname; Assert( (filespec != NULL) && (strlen(filespec) > 0) ); //-V805 @@ -1201,9 +1199,6 @@ CFileLocation cf_find_file_location(const char* filespec, int pathtype, bool loc } } - memset( longname, 0, sizeof(longname) ); - - for (ui=0; uiname_ext.c_str()) ) { - CFileLocation res(true); - res.size = static_cast(f->size); - res.offset = (size_t)f->pack_offset; - res.data_ptr = f->data; - res.name_ext = f->name_ext; - - if (f->data != nullptr) { - // This is an in-memory file so we just copy the pathtype name + file name - res.full_name = Pathtypes[f->pathtype_index].path; - res.full_name += DIR_SEPARATOR_STR; - res.full_name += f->name_ext; - } else if (f->pack_offset < 1) { - // This is a real file, return the actual file path - res.full_name = f->real_name; - } else { - // File is in a pack file - cf_root *r = cf_get_root(f->root_index); - - res.full_name = r->path; - } - - return res; - } - } + if ( !sub_path_match(sub_path, f->sub_path) ) { + continue; } // file either not localized or localized version not found - if ( !stricmp(filename.c_str(), f->name_ext.c_str()) && sub_path_match(sub_path, f->sub_path) ) { + if ( !stricmp(filename.c_str(), f->name_ext.c_str()) ) { CFileLocation res(true); res.size = static_cast(f->size); res.offset = (size_t)f->pack_offset; @@ -1365,16 +1332,15 @@ extern char *stristr(char *str, const char *substr); * @param ext_list Extension filter list * @param pathtype See CF_TYPE_ defines in CFILE.H * @param max_out Maximum string length that should be stuffed into pack_filename - * @param localize Undertake localization * * @return A structure containing information about the found file */ -CFileLocationExt cf_find_file_location_ext( const char *filename, const int ext_num, const char **ext_list, int pathtype, bool localize) +CFileLocationExt cf_find_file_location_ext( const char *filename, const int ext_num, const char **ext_list, int pathtype) { int cur_ext, i; uint ui; int cfs_slow_search = 0; - char longname[MAX_PATH_LEN]; + SCP_string longname; char filespec[MAX_FILENAME_LEN]; char *p = NULL; @@ -1399,7 +1365,6 @@ CFileLocationExt cf_find_file_location_ext( const char *filename, const int ext_ search_order[num_search_dirs++] = i; } - memset( longname, 0, sizeof(longname) ); memset( filespec, 0, sizeof(filespec) ); // strip any existing extension @@ -1442,11 +1407,11 @@ CFileLocationExt cf_find_file_location_ext( const char *filename, const int ext_ strcat_s( filespec, ext_list[cur_ext] ); - if ( !cf_create_default_path_string(longname, sizeof(longname)-1, search_order[ui], filespec, localize) ) { + if ( !cf_create_default_path_string(longname, search_order[ui], filespec) ) { continue; } - FILE *fp = fopen(longname, "rb" ); + FILE *fp = fopen(longname.c_str(), "rb" ); if (fp) { CFileLocationExt res(cur_ext); @@ -1537,42 +1502,6 @@ CFileLocationExt cf_find_file_location_ext( const char *filename, const int ext_ strcat_s( filespec, ext_list[cur_ext] ); - if (localize) { - // create localized filespec - strncpy(longname, filespec, MAX_PATH_LEN - 1); - - if ( lcl_add_dir_to_path_with_filename(longname, MAX_PATH_LEN - 1) ) { - if ( !stricmp(longname, f->name_ext.c_str()) ) { - CFileLocationExt res(cur_ext); - res.found = true; - res.size = static_cast(f->size); - res.offset = (size_t)f->pack_offset; - res.data_ptr = f->data; - res.name_ext = f->name_ext; - - if (f->data != nullptr) { - // This is an in-memory file so we just copy the pathtype name + file name - res.full_name = Pathtypes[f->pathtype_index].path; - res.full_name += DIR_SEPARATOR_STR; - res.full_name += f->name_ext; - } else if (f->pack_offset < 1) { - // This is a real file, return the actual file path - res.full_name = f->real_name; - } else { - // File is in a pack file - cf_root *r = cf_get_root(f->root_index); - - res.full_name = r->path; - } - - // found it, so cleanup and return - file_list_index.clear(); - - return res; - } - } - } - // file either not localized or localized version not found if ( !stricmp(filespec, f->name_ext.c_str()) ) { CFileLocationExt res(cur_ext); @@ -1586,6 +1515,7 @@ CFileLocationExt cf_find_file_location_ext( const char *filename, const int ext_ // This is an in-memory file so we just copy the pathtype name + file name res.full_name = Pathtypes[f->pathtype_index].path; res.full_name += DIR_SEPARATOR_STR; + res.full_name += f->sub_path; res.full_name += f->name_ext; } else if (f->pack_offset < 1) { // This is a real file, return the actual file path @@ -1739,7 +1669,7 @@ int cf_get_file_list(SCP_vector& list, int pathtype, const char* fil Get_file_list_child = nullptr; } - cf_create_default_path_string(filespec, pathtype, (char*)Get_file_list_child, false, location_flags); + cf_create_default_path_string(filespec, pathtype, (char*)Get_file_list_child, location_flags); SCP_vector<_file_list_t> files; @@ -1891,7 +1821,7 @@ int cf_get_file_list(int max, char** list, int pathtype, const char* filter, int Get_file_list_child = nullptr; } - cf_create_default_path_string(filespec, pathtype, (char*)Get_file_list_child, false, location_flags); + cf_create_default_path_string(filespec, pathtype, (char*)Get_file_list_child, location_flags); SCP_vector<_file_list_t> files; @@ -2068,7 +1998,7 @@ int cf_get_file_list_preallocated(int max, char arr[][MAX_FILENAME_LEN], char** } // Search the default directories - cf_create_default_path_string(filespec, pathtype, (char*)Get_file_list_child, false, location_flags); + cf_create_default_path_string(filespec, pathtype, (char*)Get_file_list_child, location_flags); SCP_vector<_file_list_t> files; @@ -2194,12 +2124,11 @@ int cf_get_file_list_preallocated(int max, char arr[][MAX_FILENAME_LEN], char** // filename - optional, if set, tacks the filename onto end of path. // Output: path - Fully qualified pathname. //Returns 0 if the result would be too long (invalid result) -int cf_create_default_path_string(char* path, uint path_max, int pathtype, const char* filename, bool localize, - uint32_t _location_flags) +int cf_create_default_path_string(char* path, uint path_max, int pathtype, const char* filename, uint32_t _location_flags) { SCP_string fullpath; - cf_create_default_path_string(fullpath, pathtype, filename, localize, _location_flags); + cf_create_default_path_string(fullpath, pathtype, filename, _location_flags); // if truncation would occur, return error if (fullpath.length() >= path_max) { @@ -2224,8 +2153,7 @@ int cf_create_default_path_string(char* path, uint path_max, int pathtype, const // filename - optional, if set, tacks the filename onto end of path. // Output: path - Fully qualified pathname. //Returns 0 if the result would be too long (invalid result) -int cf_create_default_path_string(SCP_string& path, int pathtype, const char* filename, bool /*localize*/, - uint32_t _location_flags) +int cf_create_default_path_string(SCP_string& path, int pathtype, const char* filename, uint32_t _location_flags) { uint32_t location_flags = _location_flags; diff --git a/code/cfile/cfilesystem.h b/code/cfile/cfilesystem.h index 551ad280733..b251d28f56f 100644 --- a/code/cfile/cfilesystem.h +++ b/code/cfile/cfilesystem.h @@ -42,8 +42,8 @@ bool cf_check_location_flags(uint32_t check_flags, uint32_t desired_flags); // Output: path - Fully qualified pathname. //Returns 0 if result would be too long (invalid result) int cf_create_default_path_string(char* path, uint path_max, int pathtype, const char* filename = nullptr, - bool localize = false, uint32_t location_flags = CF_LOCATION_ALL); -int cf_create_default_path_string(SCP_string& path, int pathtype, const char* filename = nullptr, bool localize = false, + uint32_t location_flags = CF_LOCATION_ALL); +int cf_create_default_path_string(SCP_string& path, int pathtype, const char* filename = nullptr, uint32_t location_flags = CF_LOCATION_ALL); #endif //_CFILESYSTEM_H diff --git a/code/graphics/generic.cpp b/code/graphics/generic.cpp index 078f8bf68d8..9e87ca37324 100644 --- a/code/graphics/generic.cpp +++ b/code/graphics/generic.cpp @@ -162,7 +162,7 @@ int generic_anim_stream(generic_anim *ga, const bool cache) ga->type = BM_TYPE_NONE; - auto res = cf_find_file_location_ext(ga->filename, BM_ANI_NUM_TYPES, bm_ani_ext_list, CF_TYPE_ANY, false); + auto res = cf_find_file_location_ext(ga->filename, BM_ANI_NUM_TYPES, bm_ani_ext_list, CF_TYPE_ANY); // could not be found, or is invalid for some reason if ( !res.found ) diff --git a/code/graphics/software/FontManager.cpp b/code/graphics/software/FontManager.cpp index a939b5c3953..80aa43f772a 100644 --- a/code/graphics/software/FontManager.cpp +++ b/code/graphics/software/FontManager.cpp @@ -10,6 +10,7 @@ #include "bmpman/bmpman.h" #include "cfile/cfile.h" +#include "localization/localize.h" namespace font { @@ -115,9 +116,20 @@ namespace font return data; } - bool localize = true; + // try localized version first + CFILE* fp = nullptr; + SCP_string typeface_lcl = typeface; + + lcl_add_dir_to_path_with_filename(typeface_lcl); + + fp = cfopen(typeface_lcl.c_str(), "rb", CFILE_NORMAL, CF_TYPE_ANY); + + // fallback if not found + if ( !fp ) + { + fp = cfopen(typeface.c_str(), "rb", CFILE_NORMAL, CF_TYPE_ANY); + } - CFILE* fp = cfopen(typeface.c_str(), "rb", CFILE_NORMAL, CF_TYPE_ANY, localize); if (fp == NULL) { mprintf(("Unable to find font file \"%s\"\n", typeface.c_str())); diff --git a/code/localization/localize.cpp b/code/localization/localize.cpp index c19f7b1d846..47bd6c9c4fe 100644 --- a/code/localization/localize.cpp +++ b/code/localization/localize.cpp @@ -644,6 +644,37 @@ int lcl_add_dir_to_path_with_filename(char *current_path, size_t path_max) return 1; } +int lcl_add_dir_to_path_with_filename(SCP_string ¤t_path) +{ + int lang = lcl_get_current_lang_index(); + + // if the disk extension is 0 length, don't add anything + if (strlen(Lcl_languages[lang].lang_ext) <= 0) { + return 1; + } + + SCP_string filename; + + auto sep = current_path.find_last_of(DIR_SEPARATOR_CHAR); + + if (sep == SCP_string::npos) { + filename = current_path; + current_path.clear(); + } else { + filename = current_path.substr(sep+1); + current_path.erase(sep+1); + } + + // add extension + current_path += Lcl_languages[lang].lang_ext; + current_path += DIR_SEPARATOR_STR; + + // copy rest of filename + current_path += filename; + + return 1; +} + // externalization of table/mission files ----------------------- diff --git a/code/localization/localize.h b/code/localization/localize.h index 4b207c699c0..01d7a652d49 100644 --- a/code/localization/localize.h +++ b/code/localization/localize.h @@ -103,6 +103,7 @@ ubyte lcl_get_font_index(int font_num); // maybe add localized directory to full path with file name when opening a localized file int lcl_add_dir_to_path_with_filename(char *current_path, size_t path_max); +int lcl_add_dir_to_path_with_filename(SCP_string ¤t_path); // Goober5000 void lcl_replace_stuff(char *text, size_t max_len, bool force = false); diff --git a/code/pilotfile/csg_convert.cpp b/code/pilotfile/csg_convert.cpp index 443962b694c..7be22b7d9ba 100644 --- a/code/pilotfile/csg_convert.cpp +++ b/code/pilotfile/csg_convert.cpp @@ -1166,7 +1166,7 @@ bool pilotfile_convert::csg_convert(const char *fname, bool inferno) filename.reserve(200); - cf_create_default_path_string(filename, CF_TYPE_SINGLE_PLAYERS, (inferno) ? "inferno" : nullptr, false, + cf_create_default_path_string(filename, CF_TYPE_SINGLE_PLAYERS, (inferno) ? "inferno" : nullptr, CF_LOCATION_ROOT_USER | CF_LOCATION_ROOT_GAME | CF_LOCATION_TYPE_ROOT); if (inferno) { diff --git a/code/pilotfile/plr_convert.cpp b/code/pilotfile/plr_convert.cpp index ef51a93e7da..fa43a52c9d7 100644 --- a/code/pilotfile/plr_convert.cpp +++ b/code/pilotfile/plr_convert.cpp @@ -863,7 +863,7 @@ bool pilotfile_convert::plr_convert(const char *fname, bool inferno) filename.reserve(200); - cf_create_default_path_string(filename, CF_TYPE_SINGLE_PLAYERS, (inferno) ? "inferno" : nullptr, false, + cf_create_default_path_string(filename, CF_TYPE_SINGLE_PLAYERS, (inferno) ? "inferno" : nullptr, CF_LOCATION_ROOT_USER | CF_LOCATION_ROOT_GAME | CF_LOCATION_TYPE_ROOT); if (inferno) { diff --git a/code/sound/ffmpeg/FFmpegWaveFile.cpp b/code/sound/ffmpeg/FFmpegWaveFile.cpp index 4941f3d5de2..4cade1bc6e9 100644 --- a/code/sound/ffmpeg/FFmpegWaveFile.cpp +++ b/code/sound/ffmpeg/FFmpegWaveFile.cpp @@ -208,7 +208,7 @@ bool FFmpegWaveFile::Open(const char* pszFilename, bool keep_ext) throw FFmpegException("Unknown file extension."); } - auto res = cf_find_file_location(pszFilename, CF_TYPE_ANY, false); + auto res = cf_find_file_location(pszFilename, CF_TYPE_ANY); if (!res.found) { throw FFmpegException("File not found."); @@ -217,7 +217,7 @@ bool FFmpegWaveFile::Open(const char* pszFilename, bool keep_ext) cfp = cfopen_special(res, "rb", CF_TYPE_ANY); } else { // ... otherwise we just find the best match - auto res = cf_find_file_location_ext(filename, NUM_AUDIO_EXT, audio_ext_list, CF_TYPE_ANY, false); + auto res = cf_find_file_location_ext(filename, NUM_AUDIO_EXT, audio_ext_list, CF_TYPE_ANY); if (!res.found) { throw FFmpegException("File not found with any known extension."); diff --git a/fred2/campaigneditordlg.cpp b/fred2/campaigneditordlg.cpp index cfa2ed4d964..ac5c4ed6af4 100644 --- a/fred2/campaigneditordlg.cpp +++ b/fred2/campaigneditordlg.cpp @@ -139,7 +139,7 @@ void campaign_editor::OnLoad() } } - auto res = cf_find_file_location(Campaign.missions[Cur_campaign_mission].name, CF_TYPE_MISSIONS, false); + auto res = cf_find_file_location(Campaign.missions[Cur_campaign_mission].name, CF_TYPE_MISSIONS); if (res.found) { FREDDoc_ptr->SetPathName(res.full_name.c_str()); From 26bd0f3633e79347052c49daa6bcd498f751054a Mon Sep 17 00:00:00 2001 From: Taylor Richards Date: Sat, 15 Oct 2022 22:43:49 -0400 Subject: [PATCH 13/21] add subfolder support to cf_find_file_location_ext() --- code/cfile/cfilesystem.cpp | 88 +++++++++++++++++++++----------------- 1 file changed, 48 insertions(+), 40 deletions(-) diff --git a/code/cfile/cfilesystem.cpp b/code/cfile/cfilesystem.cpp index ee43ba0be53..4114e7ff08d 100644 --- a/code/cfile/cfilesystem.cpp +++ b/code/cfile/cfilesystem.cpp @@ -1335,21 +1335,19 @@ extern char *stristr(char *str, const char *substr); * * @return A structure containing information about the found file */ -CFileLocationExt cf_find_file_location_ext( const char *filename, const int ext_num, const char **ext_list, int pathtype) +CFileLocationExt cf_find_file_location_ext(const char *filename, const int ext_num, const char **ext_list, int pathtype) { int cur_ext, i; uint ui; int cfs_slow_search = 0; SCP_string longname; - char filespec[MAX_FILENAME_LEN]; - char *p = NULL; - + Assert( (filename != NULL) && (strlen(filename) < MAX_FILENAME_LEN) ); Assert( (ext_list != NULL) && (ext_num > 1) ); // if we are searching for just one ext // then this is the wrong function to use // if we have a full path already then fail. this function if for searching via filter only! - if ( strchr(filename, DIR_SEPARATOR_CHAR) ) { // do we have a full path already? + if (is_absolute_path(filename)) { // do we have a full path already? Int3(); return CFileLocationExt(); } @@ -1365,10 +1363,38 @@ CFileLocationExt cf_find_file_location_ext( const char *filename, const int ext_ search_order[num_search_dirs++] = i; } - memset( filespec, 0, sizeof(filespec) ); + // fixup filename and sub directory path, if needed + SCP_string filespec = filename; + SCP_string sub_path; + + auto seperator = filespec.find_last_of("\\/"); + + if (seperator != SCP_string::npos) { + sub_path = filespec.substr(0, seperator); + sub_path += DIR_SEPARATOR_STR; + + filespec.erase(0, seperator+1); + + // fix separators in sub path + char bad_sep = '/'; + + if (bad_sep == DIR_SEPARATOR_CHAR) { + bad_sep = '\\'; + } + + std::replace(sub_path.begin(), sub_path.end(), bad_sep, DIR_SEPARATOR_CHAR); + } // strip any existing extension - strncpy(filespec, filename, MAX_FILENAME_LEN-1); + // (NOTE: to be fully retail compatible, we need to support multiple periods for something like *_1.5.wav, + // which means that we need to strip a length of >2 only, assuming that all valid ext are at least 2 chars) + auto dot = filespec.find_last_of("."); + + if ( (dot != SCP_string::npos) && ((filespec.length() - dot) > 2) ) { + filespec.erase(dot); + } + + SCP_string filespec_ext; for (ui = 0; ui < num_search_dirs; ui++) { cfs_slow_search = 0; @@ -1398,16 +1424,9 @@ CFileLocationExt cf_find_file_location_ext( const char *filename, const int ext_ continue; for (cur_ext = 0; cur_ext < ext_num; cur_ext++) { - // strip any extension and add the one we want to check for - // (NOTE: to be fully retail compatible, we need to support multiple periods for something like *_1.5.wav, - // which means that we need to strip a length of >2 only, assuming that all valid ext are at least 2 chars) - p = strrchr(filespec, '.'); - if ( p && (strlen(p) > 2) ) - (*p) = 0; - - strcat_s( filespec, ext_list[cur_ext] ); - - if ( !cf_create_default_path_string(longname, search_order[ui], filespec) ) { + filespec_ext = filespec + ext_list[cur_ext]; + + if ( !cf_create_default_path_string(longname, search_order[ui], filespec_ext.c_str()) ) { continue; } @@ -1422,7 +1441,7 @@ CFileLocationExt cf_find_file_location_ext( const char *filename, const int ext_ res.offset = 0; res.full_name = longname; - res.name_ext = filespec; + res.name_ext = filespec_ext; return res; } @@ -1431,19 +1450,9 @@ CFileLocationExt cf_find_file_location_ext( const char *filename, const int ext_ // Search the pak files and CD-ROM. - // first off, make sure that we don't have an extension - // (NOTE: to be fully retail compatible, we need to support multiple periods for something like *_1.5.wav, - // which means that we need to strip a length of >2 only, assuming that all valid ext are at least 2 chars) - p = strrchr(filespec, '.'); - if ( p && (strlen(p) > 2) ) - (*p) = 0; - - // go ahead and get our length, which is used to test with later - size_t filespec_len = strlen(filespec); - - // get total legnth, with extension, which is iused to test with later + // get total length, with extension, which is used to test with later // (FIXME: this assumes that everything in ext_list[] is the same length!) - size_t filespec_len_big = filespec_len + strlen(ext_list[0]); + size_t filespec_len_big = filespec.length() + strlen(ext_list[0]); SCP_vector< cf_file* > file_list_index; int last_root_index = -1; @@ -1459,12 +1468,17 @@ CFileLocationExt cf_find_file_location_ext( const char *filename, const int ext_ if ( (num_search_dirs == 1) && (pathtype != f->pathtype_index) ) continue; + // ... match subdirectories (if specified) + if ( !sub_path_match(sub_path, f->sub_path) ) { + continue; + } + // ... check that our names are the same length (accounting for the missing extension on our own name) if (f->name_ext.length() != filespec_len_big ) continue; // ... check that we match the base filename - if ( strnicmp(f->name_ext.c_str(), filespec, filespec_len) != 0 ) + if ( strnicmp(f->name_ext.c_str(), filespec.c_str(), filespec.length()) != 0 ) continue; // ... make sure that it's one of our supported types @@ -1499,11 +1513,11 @@ CFileLocationExt cf_find_file_location_ext( const char *filename, const int ext_ for (cur_ext = 0; cur_ext < ext_num; cur_ext++) { for (SCP_vector::iterator fli = file_list_index.begin(); fli != file_list_index.end(); ++fli) { cf_file *f = *fli; - - strcat_s( filespec, ext_list[cur_ext] ); + + filespec_ext = filespec + ext_list[cur_ext]; // file either not localized or localized version not found - if ( !stricmp(filespec, f->name_ext.c_str()) ) { + if ( !stricmp(filespec_ext.c_str(), f->name_ext.c_str()) ) { CFileLocationExt res(cur_ext); res.found = true; res.size = static_cast(f->size); @@ -1532,12 +1546,6 @@ CFileLocationExt cf_find_file_location_ext( const char *filename, const int ext_ return res; } - - // ok, we're still here, so strip off the extension again in order to - // prepare for the next run - p = strrchr(filespec, '.'); - if ( p && (strlen(p) > 2) ) - (*p) = 0; } } From ba70e20574160e8160fd884db9fb071ee641af75 Mon Sep 17 00:00:00 2001 From: Taylor Richards Date: Sat, 15 Oct 2022 23:44:19 -0400 Subject: [PATCH 14/21] add subfolder tests for cf_find_file_location*() --- test/src/cfile/cfile.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/src/cfile/cfile.cpp b/test/src/cfile/cfile.cpp index c7b5e683110..1f1fb3299aa 100644 --- a/test/src/cfile/cfile.cpp +++ b/test/src/cfile/cfile.cpp @@ -168,4 +168,16 @@ TEST_F(CFileTest, subfolders) // sub-subfolder check ASSERT_TRUE(cf_exists("sub/folder/file3.tbl", CF_TYPE_TABLES)); + + // check cf_find_file_location() with and without subfolders + auto loc = cf_find_file_location("file2.tbl", CF_TYPE_TABLES); + ASSERT_TRUE(loc.found); + + loc = cf_find_file_location("sub/file2.tbl", CF_TYPE_ANY); + ASSERT_TRUE(loc.found); + + // check that cf_find_file_location_ext() works with subfolders + const char *exts[] = { ".tbl", ".cfg" }; + loc = cf_find_file_location_ext("sub/folder/file3.tbl", 2, exts, CF_TYPE_TABLES); + ASSERT_TRUE(loc.found); } \ No newline at end of file From 1fa62b7f25e3603322cbb00bbfaae143794b57fb Mon Sep 17 00:00:00 2001 From: Taylor Richards Date: Sat, 22 Oct 2022 16:09:53 -0400 Subject: [PATCH 15/21] add shlwapi to needed win32 libs --- cmake/platform-win32.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/platform-win32.cmake b/cmake/platform-win32.cmake index 6e3345ef5a5..b20502fb9cb 100644 --- a/cmake/platform-win32.cmake +++ b/cmake/platform-win32.cmake @@ -10,6 +10,7 @@ SET(WIN32_LIBS winmm ws2_32 psapi + shlwapi ) IF (MINGW) From ef430183513098baf3941daf540f2fa45a9dbdb4 Mon Sep 17 00:00:00 2001 From: Taylor Richards Date: Sun, 23 Oct 2022 18:55:05 -0400 Subject: [PATCH 16/21] small bit of cleanup --- code/cfile/cfilesystem.cpp | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/code/cfile/cfilesystem.cpp b/code/cfile/cfilesystem.cpp index 4114e7ff08d..f8fed2ce504 100644 --- a/code/cfile/cfilesystem.cpp +++ b/code/cfile/cfilesystem.cpp @@ -553,14 +553,15 @@ void cf_build_pack_list( cf_root *root ) } } -static char normalize_directory_separator(char in) +static void normalize_directory_separators(SCP_string &str) { - if (in == '/') - { - return DIR_SEPARATOR_CHAR; + char bad_sep = '/'; + + if (bad_sep == DIR_SEPARATOR_CHAR) { + bad_sep = '\\'; } - return in; + std::replace(str.begin(), str.end(), bad_sep, DIR_SEPARATOR_CHAR); } static void cf_add_mod_roots(const char* rootDirectory, uint32_t basic_location) @@ -586,7 +587,7 @@ static void cf_add_mod_roots(const char* rootDirectory, uint32_t basic_location) } // normalize the path to the native path format - std::transform(rootPath.begin(), rootPath.end(), rootPath.begin(), normalize_directory_separator); + normalize_directory_separators(rootPath); cf_root* root = cf_create_root(); @@ -1256,13 +1257,7 @@ CFileLocation cf_find_file_location(const char* filespec, int pathtype, uint32_t filename.erase(0, seperator+1); // fix separators in sub path - char bad_sep = '/'; - - if (bad_sep == DIR_SEPARATOR_CHAR) { - bad_sep = '\\'; - } - - std::replace(sub_path.begin(), sub_path.end(), bad_sep, DIR_SEPARATOR_CHAR); + normalize_directory_separators(sub_path); } // Search the pak files and CD-ROM. @@ -1376,13 +1371,7 @@ CFileLocationExt cf_find_file_location_ext(const char *filename, const int ext_n filespec.erase(0, seperator+1); // fix separators in sub path - char bad_sep = '/'; - - if (bad_sep == DIR_SEPARATOR_CHAR) { - bad_sep = '\\'; - } - - std::replace(sub_path.begin(), sub_path.end(), bad_sep, DIR_SEPARATOR_CHAR); + normalize_directory_separators(sub_path); } // strip any existing extension From 0dafd6b35d369ba637e04a11abe4d22f24c7bbc9 Mon Sep 17 00:00:00 2001 From: Taylor Richards Date: Sun, 23 Oct 2022 18:57:01 -0400 Subject: [PATCH 17/21] add subfolder support to cf_get_file_list() --- code/cfile/cfilesystem.cpp | 237 ++++++++++++++++++++++++++----------- 1 file changed, 168 insertions(+), 69 deletions(-) diff --git a/code/cfile/cfilesystem.cpp b/code/cfile/cfilesystem.cpp index f8fed2ce504..f9a3be83b51 100644 --- a/code/cfile/cfilesystem.cpp +++ b/code/cfile/cfilesystem.cpp @@ -1248,16 +1248,15 @@ CFileLocation cf_find_file_location(const char* filespec, int pathtype, uint32_t SCP_string filename = filespec; SCP_string sub_path; - auto seperator = filename.find_last_of("\\/"); + normalize_directory_separators(filename); + + auto seperator = filename.rfind(DIR_SEPARATOR_CHAR); if (seperator != SCP_string::npos) { sub_path = filename.substr(0, seperator); - sub_path += DIR_SEPARATOR_STR; + sub_path += DIR_SEPARATOR_CHAR; filename.erase(0, seperator+1); - - // fix separators in sub path - normalize_directory_separators(sub_path); } // Search the pak files and CD-ROM. @@ -1362,16 +1361,15 @@ CFileLocationExt cf_find_file_location_ext(const char *filename, const int ext_n SCP_string filespec = filename; SCP_string sub_path; - auto seperator = filespec.find_last_of("\\/"); + normalize_directory_separators(filespec); + + auto seperator = filespec.rfind(DIR_SEPARATOR_CHAR); if (seperator != SCP_string::npos) { sub_path = filespec.substr(0, seperator); - sub_path += DIR_SEPARATOR_STR; + sub_path += DIR_SEPARATOR_CHAR; filespec.erase(0, seperator+1); - - // fix separators in sub path - normalize_directory_separators(sub_path); } // strip any existing extension @@ -1640,7 +1638,7 @@ static int cf_file_already_in_list( SCP_vector &list, const char *fi // Note that filesystem listing is always sorted by name *before* sorting by the // provided sort order. This isn't strictly needed on NTFS, which always provides the list // sorted by name. But on mac/linux it's always needed. -int cf_get_file_list(SCP_vector& list, int pathtype, const char* filter, int sort, +int cf_get_file_list(SCP_vector& list, int pathtype, const char* _filter, int sort, SCP_vector* info, uint32_t location_flags) { uint i; @@ -1668,24 +1666,54 @@ int cf_get_file_list(SCP_vector& list, int pathtype, const char* fil cf_create_default_path_string(filespec, pathtype, (char*)Get_file_list_child, location_flags); + // fixup filter and sub directory path, if needed + SCP_string filter = _filter; + SCP_string sub_path; + bool glob = false; + + normalize_directory_separators(filter); + + auto seperator = filter.rfind(DIR_SEPARATOR_CHAR); + + if (seperator != SCP_string::npos) { + sub_path = filter.substr(0, seperator); + + if (sub_path == "*") { + glob = true; + } else { + sub_path += DIR_SEPARATOR_CHAR; + } + + filter.erase(0, seperator+1); + } + + SCP_string fullname; SCP_vector<_file_list_t> files; - cf_get_list_of_files(filespec, files, filter); + cf_get_list_of_files(filespec, files, filter.c_str()); for (auto &file : files) { - if ( !Get_file_list_filter || (*Get_file_list_filter)(file.name.c_str()) ) { - if (check_duplicates && cf_file_already_in_list(list, file.name.c_str())) { - continue; - } + if ( !glob && !sub_path_match(sub_path, file.sub_path) ) { + continue; + } - SCP_string::size_type pos = file.name.find_last_of('.'); + if ( !Get_file_list_filter || (*Get_file_list_filter)(file.name.c_str()) ) { + auto pos = file.name.rfind('.'); - if (pos != SCP_string::npos) { - list.push_back(file.name.substr(0, pos)); + if ( !sub_path.empty() ) { + // prepend file name with sub directory path + // allows us to open the specific file and allows duplicate filenames in different paths + fullname = file.sub_path + file.name.substr(0, pos); } else { - list.push_back(file.name); + fullname = file.name.substr(0, pos); + } + + if (check_duplicates && cf_file_already_in_list(list, fullname.c_str())) { + continue; } + list.push_back(fullname); + if (info) { tinfo.write_time = file.m_time; info->push_back(tinfo); @@ -1705,6 +1733,16 @@ int cf_get_file_list(SCP_vector& list, int pathtype, const char* fil for (i=0; ipack_offset != 0) { + // If the packfile skip flag is set we skip files in VPs but still search in directories + continue; + } + + if (Skip_memory_files && f->data != nullptr) { + // If we want to skip memory files and this is a memory file then ignore it + continue; + } + // only search paths we're supposed to... if ( (pathtype != CF_TYPE_ANY) && (pathtype != f->pathtype_index) ) { continue; @@ -1720,30 +1758,32 @@ int cf_get_file_list(SCP_vector& list, int pathtype, const char* fil } } - if ( !cf_matches_spec(filter, f->name_ext.c_str()) ) { + if ( !glob && !sub_path_match(sub_path, f->sub_path) ) { continue; } - if ( cf_file_already_in_list(list, f->name_ext.c_str()) ) { + if ( !cf_matches_spec(filter.c_str(), f->name_ext.c_str()) ) { continue; } - if (Skip_packfile_search && f->pack_offset != 0) { - // If the packfile skip flag is set we skip files in VPs but still search in directories - continue; - } + if ( !Get_file_list_filter || (*Get_file_list_filter)(f->name_ext.c_str()) ) { + auto pos = f->name_ext.rfind('.'); - if (Skip_memory_files && f->data != nullptr) { - // If we want to skip memory files and this is a memory file then ignore it - continue; - } + if ( !sub_path.empty() ) { + // prepend file name with sub directory path + // allows us to open the specific file and allows duplicate filenames in different paths + fullname = f->sub_path + f->name_ext.substr(0, pos); + } else { + fullname = f->name_ext.substr(0, pos); + } - if ( !Get_file_list_filter || (*Get_file_list_filter)(f->name_ext.c_str()) ) { - //mprintf(( "Found '%s' in root %d path %d\n", f->name_ext, f->root_index, f->pathtype_index )); + if ( cf_file_already_in_list(list, fullname.c_str()) ) { + continue; + } - auto pos = f->name_ext.rfind('.'); + //mprintf(( "Found '%s' in root %d path %d\n", f->name_ext, f->root_index, f->pathtype_index )); - list.push_back( f->name_ext.substr(0, pos) ); + list.push_back(fullname); if (info) { tinfo.write_time = f->write_time; @@ -1792,12 +1832,11 @@ int cf_file_already_in_list( int num_files, char **list, const char *filename ) // This one has a 'type', which is a CF_TYPE_* value. Because this specifies the directory // location, 'filter' only needs to be the filter itself, with no path information. // See above descriptions of cf_get_file_list() for more information about how it all works. -int cf_get_file_list(int max, char** list, int pathtype, const char* filter, int sort, file_list_info* info, +int cf_get_file_list(int max, char** list, int pathtype, const char* _filter, int sort, file_list_info* info, uint32_t location_flags) { uint i; int num_files = 0, own_flag = 0; - size_t l; if (max < 1) { Get_file_list_filter = NULL; @@ -1820,26 +1859,56 @@ int cf_get_file_list(int max, char** list, int pathtype, const char* filter, int cf_create_default_path_string(filespec, pathtype, (char*)Get_file_list_child, location_flags); + // fixup filter and sub directory path, if needed + SCP_string filter = _filter; + SCP_string sub_path; + bool glob = false; + + normalize_directory_separators(filter); + + auto seperator = filter.rfind(DIR_SEPARATOR_CHAR); + + if (seperator != SCP_string::npos) { + sub_path = filter.substr(0, seperator); + + if (sub_path == "*") { + glob = true; + } else { + sub_path += DIR_SEPARATOR_CHAR; + } + + filter.erase(0, seperator+1); + } + + SCP_string fullname; SCP_vector<_file_list_t> files; - cf_get_list_of_files(filespec, files, filter); + cf_get_list_of_files(filespec, files, filter.c_str()); for (auto &file : files) { if (num_files >= max) { break; } + if ( !glob && !sub_path_match(sub_path, file.sub_path) ) { + continue; + } + if ( !Get_file_list_filter || (*Get_file_list_filter)(file.name.c_str()) ) { auto pos = file.name.rfind('.'); - if (pos != SCP_string::npos) { - l = pos; + if (glob) { + // prepend file name with sub directory path + // allows us to open the specific file and allows duplicate filenames in different paths + fullname = file.sub_path + file.name.substr(0, pos); } else { - l = file.name.length(); + fullname = file.name.substr(0, pos); } - list[num_files] = reinterpret_cast(vm_malloc(l + 1)); - SDL_strlcpy(list[num_files], file.name.substr(0, l).c_str(), l+1); + auto len = fullname.length(); + + list[num_files] = reinterpret_cast(vm_malloc(len + 1)); + SDL_strlcpy(list[num_files], fullname.c_str(), len+1); if (info) { info[num_files].write_time = file.m_time; @@ -1862,6 +1931,20 @@ int cf_get_file_list(int max, char** list, int pathtype, const char* filter, int for (i=0; i= max) { + break; + } + + if (Skip_packfile_search && f->pack_offset != 0) { + // If the packfile skip flag is set we skip files in VPs but still search in directories + continue; + } + + if (Skip_memory_files && f->data != nullptr) { + // If we want to skip memory files and this is a memory file then ignore it + continue; + } + // only search paths we're supposed to... if ( (pathtype != CF_TYPE_ANY) && (pathtype != f->pathtype_index) ) { continue; @@ -1877,42 +1960,36 @@ int cf_get_file_list(int max, char** list, int pathtype, const char* filter, int } } - if (num_files >= max) - break; - - if ( !cf_matches_spec(filter, f->name_ext.c_str())) { + if ( !glob && !sub_path_match(sub_path, f->sub_path) ) { continue; } - if ( cf_file_already_in_list(num_files, list, f->name_ext.c_str()) ) { + if ( !cf_matches_spec(filter.c_str(), f->name_ext.c_str())) { continue; } - if (Skip_packfile_search && f->pack_offset != 0) { - // If the packfile skip flag is set we skip files in VPs but still search in directories - continue; - } + if ( !Get_file_list_filter || (*Get_file_list_filter)(f->name_ext.c_str()) ) { + auto pos = f->name_ext.rfind('.'); - if (Skip_memory_files && f->data != nullptr) { - // If we want to skip memory files and this is a memory file then ignore it - continue; - } + if (glob) { + // prepend file name with sub directory path + // allows us to open the specific file and allows duplicate filenames in different paths + fullname = f->sub_path + f->name_ext.substr(0, pos); + } else { + fullname = f->name_ext.substr(0, pos); + } - if ( !Get_file_list_filter || (*Get_file_list_filter)(f->name_ext.c_str()) ) { + if ( cf_file_already_in_list(num_files, list, fullname.c_str()) ) { + continue; + } //mprintf(( "Found '%s' in root %d path %d\n", f->name_ext, f->root_index, f->pathtype_index )); - auto pos = f->name_ext.rfind('.'); - - if (pos != SCP_string::npos) { - l = pos; - } else { - l = f->name_ext.length(); - } + auto len = fullname.length(); - list[num_files] = (char *)vm_malloc(l + 1); - strncpy(list[num_files], f->name_ext.c_str(), l); - list[num_files][l] = 0; + list[num_files] = (char *)vm_malloc(len + 1); + strncpy(list[num_files], f->name_ext.c_str(), len); + list[num_files][len] = 0; if (info) { info[num_files].write_time = f->write_time; @@ -1962,7 +2039,7 @@ int cf_file_already_in_list_preallocated( int num_files, char arr[][MAX_FILENAME // This one has a 'type', which is a CF_TYPE_* value. Because this specifies the directory // location, 'filter' only needs to be the filter itself, with no path information. // See above descriptions of cf_get_file_list() for more information about how it all works. -int cf_get_file_list_preallocated(int max, char arr[][MAX_FILENAME_LEN], char** list, int pathtype, const char* filter, +int cf_get_file_list_preallocated(int max, char arr[][MAX_FILENAME_LEN], char** list, int pathtype, const char* _filter, int sort, file_list_info* info, uint32_t location_flags) { int num_files = 0, own_flag = 0; @@ -1997,9 +2074,31 @@ int cf_get_file_list_preallocated(int max, char arr[][MAX_FILENAME_LEN], char** // Search the default directories cf_create_default_path_string(filespec, pathtype, (char*)Get_file_list_child, location_flags); + // fixup filter and sub directory path, if needed + SCP_string filter = _filter; + SCP_string sub_path; + + normalize_directory_separators(filter); + + auto seperator = filter.rfind(DIR_SEPARATOR_CHAR); + + if (seperator != SCP_string::npos) { + sub_path = filter.substr(0, seperator); + + if (sub_path != "*") { + sub_path += DIR_SEPARATOR_CHAR; + } + + filter.erase(0, seperator+1); + + // due to the limited filename length we can't reliably add sub paths to it + // so we just strip off the sub path and do a regular search instead, after a warning + Warning(LOCATION, "Subdirectory searches aren't available for cf_get_file_list_preallocated()!"); + } + SCP_vector<_file_list_t> files; - cf_get_list_of_files(filespec, files, filter); + cf_get_list_of_files(filespec, files, filter.c_str()); for (auto &file : files) { if (num_files >= max) { @@ -2059,7 +2158,7 @@ int cf_get_file_list_preallocated(int max, char arr[][MAX_FILENAME_LEN], char** break; - if ( !cf_matches_spec(filter, f->name_ext.c_str()) ) { + if ( !cf_matches_spec(filter.c_str(), f->name_ext.c_str()) ) { continue; } From 9b6e05ac6bffe78e7fedf864c490ef6f7eefba33 Mon Sep 17 00:00:00 2001 From: Taylor Richards Date: Sun, 23 Oct 2022 18:57:56 -0400 Subject: [PATCH 18/21] add tests for cfile subfolder lists --- test/src/cfile/cfile.cpp | 31 +++++++++++++++++++ .../cfile/subfolder_list/data/tables/file.tbl | 0 .../data/tables/folder/file.tbl | 0 3 files changed, 31 insertions(+) create mode 100644 test/test_data/cfile/subfolder_list/data/tables/file.tbl create mode 100644 test/test_data/cfile/subfolder_list/data/tables/folder/file.tbl diff --git a/test/src/cfile/cfile.cpp b/test/src/cfile/cfile.cpp index 1f1fb3299aa..3ba5006a601 100644 --- a/test/src/cfile/cfile.cpp +++ b/test/src/cfile/cfile.cpp @@ -180,4 +180,35 @@ TEST_F(CFileTest, subfolders) const char *exts[] = { ".tbl", ".cfg" }; loc = cf_find_file_location_ext("sub/folder/file3.tbl", 2, exts, CF_TYPE_TABLES); ASSERT_TRUE(loc.found); +} + +TEST_F(CFileTest, subfolder_list) +{ + SCP_vector table_files; + extern bool Skip_memory_files; + + // For this test we need to skip the memory files to keep the results consistent + Skip_memory_files = true; + + // look for all tables (one result, shadowing second file) + ASSERT_EQ(1, cf_get_file_list(table_files, CF_TYPE_TABLES, "*.tbl", CF_SORT_NAME)); + + // look for file in subfolder only, with unix and windows directory separators + table_files.clear(); + ASSERT_EQ(1, cf_get_file_list(table_files, CF_TYPE_TABLES, "folder/*.tbl", CF_SORT_NAME)); + ASSERT_TRUE(table_files.front().substr(0, 6) == "folder"); + + table_files.clear(); + ASSERT_EQ(1, cf_get_file_list(table_files, CF_TYPE_TABLES, "folder\\*.tbl", CF_SORT_NAME)); + ASSERT_TRUE(table_files.front().substr(0, 6) == "folder"); + + // subfolder glob, with unix and windows directory separators + // should return two files, one with a sub path + table_files.clear(); + ASSERT_EQ(2, cf_get_file_list(table_files, CF_TYPE_TABLES, "*/*.tbl", CF_SORT_NAME)); + ASSERT_TRUE(table_files.back().substr(0, 6) == "folder"); + + table_files.clear(); + ASSERT_EQ(2, cf_get_file_list(table_files, CF_TYPE_TABLES, "*\\*.tbl", CF_SORT_NAME)); + ASSERT_TRUE(table_files.back().substr(0, 6) == "folder"); } \ No newline at end of file diff --git a/test/test_data/cfile/subfolder_list/data/tables/file.tbl b/test/test_data/cfile/subfolder_list/data/tables/file.tbl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/test_data/cfile/subfolder_list/data/tables/folder/file.tbl b/test/test_data/cfile/subfolder_list/data/tables/folder/file.tbl new file mode 100644 index 00000000000..e69de29bb2d From d83782e261c5151a74ee9b53fdce805a24546b4d Mon Sep 17 00:00:00 2001 From: Taylor Richards Date: Sat, 29 Oct 2022 17:32:10 -0400 Subject: [PATCH 19/21] report files that might be shadowed --- code/cfile/cfilesystem.cpp | 61 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/code/cfile/cfilesystem.cpp b/code/cfile/cfilesystem.cpp index f9a3be83b51..754987a0641 100644 --- a/code/cfile/cfilesystem.cpp +++ b/code/cfile/cfilesystem.cpp @@ -768,6 +768,63 @@ int is_ext_in_list( const char *ext_list, const char *ext ) return 0; } +// Run a basic test for indexed files that may be shadowed +#ifndef NDEBUG + #define ENABLE_SHADOW_CHECK 1 +#endif + +static void check_file_shadows(const int root_index __UNUSED, const int pathtype __UNUSED, const SCP_string &name __UNUSED, const SCP_string sub_path __UNUSED) +{ +#if ENABLE_SHADOW_CHECK + if ( !cf_should_scan_subdirs(pathtype) ) { + return; + } + + SCP_string curfile, newfile; + + const auto root = cf_get_root(root_index); + + newfile = root->path + ((root->roottype == CF_ROOTTYPE_PACK) ? "::" : ""); + newfile += cf_get_root_pathtype(root, pathtype) + DIR_SEPARATOR_CHAR; + newfile += sub_path + name; + + for (uint i = 0; i < Num_files; ++i) { + const auto f = cf_get_file(i); + const auto r = cf_get_root(f->root_index); + + // skip memory roots, no subdirs there + if (r->roottype == CF_ROOTTYPE_MEMORY) { + continue; + } + + // if basic path type doesn't match then skip + if (pathtype != f->pathtype_index) { + continue; + } + + if (name != f->name_ext) { + continue; + } + + // if the subpath matches then consider it an override rather than shadow + if (sub_path == f->sub_path) { + continue; + } + + curfile = r->path + ((r->roottype == CF_ROOTTYPE_PACK) ? "::" : ""); + curfile += cf_get_root_pathtype(r, pathtype) + DIR_SEPARATOR_CHAR; + curfile += f->sub_path + f->name_ext; + + // this log message occurs in the middle of an existing line, hence the extra new lines + mprintf(("\nWARNING! A file being indexed may be shadowed by an existing file!\n New:\n %s\n Existing:\n %s\n", + newfile.c_str(), curfile.c_str())); + + break; + } +#endif +} + + void cf_search_root_path(int root_index) { int i; @@ -830,6 +887,8 @@ void cf_search_root_path(int root_index) continue; } + check_file_shadows(root_index, i, file.name, file.sub_path); + cf_file *cfile = cf_create_file(); cfile->name_ext = file.name; @@ -872,6 +931,8 @@ static int cf_add_pack_files(const int root_index, SCP_vector<_file_list_t> &fil std::sort(files.begin(), files.end(), sort_file_list); for (auto &file : files) { + check_file_shadows(root_index, file.pathtype, file.name, file.sub_path); + cf_file *pf = cf_create_file(); pf->name_ext = file.name; From 8a576d0cdd5389c3f791a4fb2045e9e0231b370c Mon Sep 17 00:00:00 2001 From: Taylor Richards Date: Mon, 31 Oct 2022 00:57:53 -0400 Subject: [PATCH 20/21] fix issue with lists when a filter is used Also fixes discrepancy in older cf_get_file_list() when a sub path is specified (should be used in all cases, not just glob). --- code/cfile/cfilesystem.cpp | 90 +++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/code/cfile/cfilesystem.cpp b/code/cfile/cfilesystem.cpp index 754987a0641..8537aecb680 100644 --- a/code/cfile/cfilesystem.cpp +++ b/code/cfile/cfilesystem.cpp @@ -1758,21 +1758,21 @@ int cf_get_file_list(SCP_vector& list, int pathtype, const char* _fi continue; } - if ( !Get_file_list_filter || (*Get_file_list_filter)(file.name.c_str()) ) { - auto pos = file.name.rfind('.'); + auto pos = file.name.rfind('.'); - if ( !sub_path.empty() ) { - // prepend file name with sub directory path - // allows us to open the specific file and allows duplicate filenames in different paths - fullname = file.sub_path + file.name.substr(0, pos); - } else { - fullname = file.name.substr(0, pos); - } + if ( !sub_path.empty() ) { + // prepend file name with sub directory path + // allows us to open the specific file and allows duplicate filenames in different paths + fullname = file.sub_path + file.name.substr(0, pos); + } else { + fullname = file.name.substr(0, pos); + } - if (check_duplicates && cf_file_already_in_list(list, fullname.c_str())) { - continue; - } + if (check_duplicates && cf_file_already_in_list(list, fullname.c_str())) { + continue; + } + if ( !Get_file_list_filter || (*Get_file_list_filter)(file.name.c_str()) ) { list.push_back(fullname); if (info) { @@ -1827,21 +1827,21 @@ int cf_get_file_list(SCP_vector& list, int pathtype, const char* _fi continue; } - if ( !Get_file_list_filter || (*Get_file_list_filter)(f->name_ext.c_str()) ) { - auto pos = f->name_ext.rfind('.'); + auto pos = f->name_ext.rfind('.'); - if ( !sub_path.empty() ) { - // prepend file name with sub directory path - // allows us to open the specific file and allows duplicate filenames in different paths - fullname = f->sub_path + f->name_ext.substr(0, pos); - } else { - fullname = f->name_ext.substr(0, pos); - } + if ( !sub_path.empty() ) { + // prepend file name with sub directory path + // allows us to open the specific file and allows duplicate filenames in different paths + fullname = f->sub_path + f->name_ext.substr(0, pos); + } else { + fullname = f->name_ext.substr(0, pos); + } - if ( cf_file_already_in_list(list, fullname.c_str()) ) { - continue; - } + if ( cf_file_already_in_list(list, fullname.c_str()) ) { + continue; + } + if ( !Get_file_list_filter || (*Get_file_list_filter)(f->name_ext.c_str()) ) { //mprintf(( "Found '%s' in root %d path %d\n", f->name_ext, f->root_index, f->pathtype_index )); list.push_back(fullname); @@ -1955,17 +1955,17 @@ int cf_get_file_list(int max, char** list, int pathtype, const char* _filter, in continue; } - if ( !Get_file_list_filter || (*Get_file_list_filter)(file.name.c_str()) ) { - auto pos = file.name.rfind('.'); + auto pos = file.name.rfind('.'); - if (glob) { - // prepend file name with sub directory path - // allows us to open the specific file and allows duplicate filenames in different paths - fullname = file.sub_path + file.name.substr(0, pos); - } else { - fullname = file.name.substr(0, pos); - } + if ( !sub_path.empty() ) { + // prepend file name with sub directory path + // allows us to open the specific file and allows duplicate filenames in different paths + fullname = file.sub_path + file.name.substr(0, pos); + } else { + fullname = file.name.substr(0, pos); + } + if ( !Get_file_list_filter || (*Get_file_list_filter)(file.name.c_str()) ) { auto len = fullname.length(); list[num_files] = reinterpret_cast(vm_malloc(len + 1)); @@ -2029,21 +2029,21 @@ int cf_get_file_list(int max, char** list, int pathtype, const char* _filter, in continue; } - if ( !Get_file_list_filter || (*Get_file_list_filter)(f->name_ext.c_str()) ) { - auto pos = f->name_ext.rfind('.'); + auto pos = f->name_ext.rfind('.'); - if (glob) { - // prepend file name with sub directory path - // allows us to open the specific file and allows duplicate filenames in different paths - fullname = f->sub_path + f->name_ext.substr(0, pos); - } else { - fullname = f->name_ext.substr(0, pos); - } + if ( !sub_path.empty() ) { + // prepend file name with sub directory path + // allows us to open the specific file and allows duplicate filenames in different paths + fullname = f->sub_path + f->name_ext.substr(0, pos); + } else { + fullname = f->name_ext.substr(0, pos); + } - if ( cf_file_already_in_list(num_files, list, fullname.c_str()) ) { - continue; - } + if ( cf_file_already_in_list(num_files, list, fullname.c_str()) ) { + continue; + } + if ( !Get_file_list_filter || (*Get_file_list_filter)(f->name_ext.c_str()) ) { //mprintf(( "Found '%s' in root %d path %d\n", f->name_ext, f->root_index, f->pathtype_index )); auto len = fullname.length(); From 24ad8b1215adc51edc0311442aa049e73cda0d4d Mon Sep 17 00:00:00 2001 From: Taylor Richards Date: Tue, 8 Nov 2022 14:59:39 -0500 Subject: [PATCH 21/21] address review suggestions --- code/cfile/cfile.cpp | 9 ++------- code/cfile/cfilesystem.cpp | 17 +++++------------ 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/code/cfile/cfile.cpp b/code/cfile/cfile.cpp index 9aee12172ac..dc85a88b6e4 100644 --- a/code/cfile/cfile.cpp +++ b/code/cfile/cfile.cpp @@ -1857,13 +1857,8 @@ int cfile_get_path_type(const SCP_string& dir) } // Use official DIR_SEPARATOR_CHAR - char bad_sep = '/'; - - if (bad_sep == DIR_SEPARATOR_CHAR) { - bad_sep = '\\'; - } - - std::replace(buf.begin(), buf.end(), bad_sep, DIR_SEPARATOR_CHAR); + extern void normalize_directory_separators(SCP_string &str); + normalize_directory_separators(buf); // identify path type auto best_match = CF_TYPE_INVALID; diff --git a/code/cfile/cfilesystem.cpp b/code/cfile/cfilesystem.cpp index 8537aecb680..9daf44a4283 100644 --- a/code/cfile/cfilesystem.cpp +++ b/code/cfile/cfilesystem.cpp @@ -553,7 +553,7 @@ void cf_build_pack_list( cf_root *root ) } } -static void normalize_directory_separators(SCP_string &str) +void normalize_directory_separators(SCP_string &str) { char bad_sep = '/'; @@ -1011,7 +1011,7 @@ void cf_search_root_pack(int root_index) if ( find.size == 0 ) { if ( !stricmp(find.filename, "..")) { - auto end = search_path.find_last_of(DIR_SEPARATOR_CHAR); + auto end = search_path.rfind(DIR_SEPARATOR_CHAR); if (end != SCP_string::npos) { search_path.erase(end); @@ -1436,7 +1436,7 @@ CFileLocationExt cf_find_file_location_ext(const char *filename, const int ext_n // strip any existing extension // (NOTE: to be fully retail compatible, we need to support multiple periods for something like *_1.5.wav, // which means that we need to strip a length of >2 only, assuming that all valid ext are at least 2 chars) - auto dot = filespec.find_last_of("."); + auto dot = filespec.rfind('.'); if ( (dot != SCP_string::npos) && ((filespec.length() - dot) > 2) ) { filespec.erase(dot); @@ -1966,10 +1966,7 @@ int cf_get_file_list(int max, char** list, int pathtype, const char* _filter, in } if ( !Get_file_list_filter || (*Get_file_list_filter)(file.name.c_str()) ) { - auto len = fullname.length(); - - list[num_files] = reinterpret_cast(vm_malloc(len + 1)); - SDL_strlcpy(list[num_files], fullname.c_str(), len+1); + list[num_files] = vm_strdup(fullname.c_str()); if (info) { info[num_files].write_time = file.m_time; @@ -2046,11 +2043,7 @@ int cf_get_file_list(int max, char** list, int pathtype, const char* _filter, in if ( !Get_file_list_filter || (*Get_file_list_filter)(f->name_ext.c_str()) ) { //mprintf(( "Found '%s' in root %d path %d\n", f->name_ext, f->root_index, f->pathtype_index )); - auto len = fullname.length(); - - list[num_files] = (char *)vm_malloc(len + 1); - strncpy(list[num_files], f->name_ext.c_str(), len); - list[num_files][len] = 0; + list[num_files] = vm_strdup(fullname.c_str()); if (info) { info[num_files].write_time = f->write_time;