diff --git a/cmake/platform-win32.cmake b/cmake/platform-win32.cmake index 6e3345ef5a5..b20502fb9cb 100644 --- a/cmake/platform-win32.cmake +++ b/cmake/platform-win32.cmake @@ -10,6 +10,7 @@ SET(WIN32_LIBS winmm ws2_32 psapi + shlwapi ) IF (MINGW) diff --git a/code/bmpman/bmpman.cpp b/code/bmpman/bmpman.cpp index 5c9a13b4b2c..2d45cf25467 100644 --- a/code/bmpman/bmpman.cpp +++ b/code/bmpman/bmpman.cpp @@ -1835,7 +1835,7 @@ int bm_load_sub_fast(const char *real_filename, int *handle, int dir_type, bool } int bm_load_sub_slow(const char *real_filename, const int num_ext, const char **ext_list, CFILE **img_cfp, int dir_type) { - auto res = cf_find_file_location_ext(real_filename, num_ext, ext_list, dir_type, false); + auto res = cf_find_file_location_ext(real_filename, num_ext, ext_list, dir_type); // could not be found, or is invalid for some reason if (!res.found) diff --git a/code/cfile/cfile.cpp b/code/cfile/cfile.cpp index cf99770ee38..dc85a88b6e4 100644 --- a/code/cfile/cfile.cpp +++ b/code/cfile/cfile.cpp @@ -59,8 +59,6 @@ cf_pathtype Pathtypes[CF_MAX_PATH_TYPES] = { { CF_TYPE_MODELS, "data" DIR_SEPARATOR_STR "models", ".pof", CF_TYPE_DATA }, { CF_TYPE_TABLES, "data" DIR_SEPARATOR_STR "tables", ".tbl .tbm .lua", CF_TYPE_DATA }, { CF_TYPE_SOUNDS, "data" DIR_SEPARATOR_STR "sounds", ".wav .ogg", CF_TYPE_DATA }, - { CF_TYPE_SOUNDS_8B22K, "data" DIR_SEPARATOR_STR "sounds" DIR_SEPARATOR_STR "8b22k", ".wav .ogg", CF_TYPE_SOUNDS }, - { CF_TYPE_SOUNDS_16B11K, "data" DIR_SEPARATOR_STR "sounds" DIR_SEPARATOR_STR "16b11k", ".wav .ogg", CF_TYPE_SOUNDS }, { CF_TYPE_VOICE, "data" DIR_SEPARATOR_STR "voice", "", CF_TYPE_DATA }, { CF_TYPE_VOICE_BRIEFINGS, "data" DIR_SEPARATOR_STR "voice" DIR_SEPARATOR_STR "briefing", ".wav .ogg", CF_TYPE_VOICE }, { CF_TYPE_VOICE_CMD_BRIEF, "data" DIR_SEPARATOR_STR "voice" DIR_SEPARATOR_STR "command_briefings", ".wav .ogg", CF_TYPE_VOICE }, @@ -70,7 +68,7 @@ cf_pathtype Pathtypes[CF_MAX_PATH_TYPES] = { { CF_TYPE_VOICE_TRAINING, "data" DIR_SEPARATOR_STR "voice" DIR_SEPARATOR_STR "training", ".wav .ogg", CF_TYPE_VOICE }, { CF_TYPE_MUSIC, "data" DIR_SEPARATOR_STR "music", ".wav .ogg", CF_TYPE_DATA }, { CF_TYPE_MOVIES, "data" DIR_SEPARATOR_STR "movies", ".mve .msb .ogg .mp4 .srt .webm .png",CF_TYPE_DATA }, - { CF_TYPE_INTERFACE, "data" DIR_SEPARATOR_STR "interface", ".pcx .ani .dds .tga .eff .png .jpg", CF_TYPE_DATA }, + { CF_TYPE_INTERFACE, "data" DIR_SEPARATOR_STR "interface", ".pcx .ani .dds .tga .eff .png .jpg .rml .rcss", CF_TYPE_DATA }, { CF_TYPE_FONT, "data" DIR_SEPARATOR_STR "fonts", ".vf .ttf .otf", CF_TYPE_DATA }, { CF_TYPE_EFFECTS, "data" DIR_SEPARATOR_STR "effects", ".ani .eff .pcx .neb .tga .jpg .png .dds .sdr", CF_TYPE_DATA }, { CF_TYPE_HUD, "data" DIR_SEPARATOR_STR "hud", ".pcx .ani .eff .tga .jpg .png .dds", CF_TYPE_DATA }, @@ -79,6 +77,7 @@ cf_pathtype Pathtypes[CF_MAX_PATH_TYPES] = { { CF_TYPE_SQUAD_IMAGES, "data" DIR_SEPARATOR_STR "players" DIR_SEPARATOR_STR "squads", ".pcx .png .dds", CF_TYPE_PLAYERS }, { CF_TYPE_SINGLE_PLAYERS, "data" DIR_SEPARATOR_STR "players" DIR_SEPARATOR_STR "single", ".pl2 .cs2 .plr .csg .css .json", CF_TYPE_PLAYERS }, { CF_TYPE_MULTI_PLAYERS, "data" DIR_SEPARATOR_STR "players" DIR_SEPARATOR_STR "multi", ".plr .json", CF_TYPE_PLAYERS }, + { CF_TYPE_PLAYER_BINDS, "data" DIR_SEPARATOR_STR "players" DIR_SEPARATOR_STR "presets", ".json", CF_TYPE_PLAYERS }, { CF_TYPE_CACHE, "data" DIR_SEPARATOR_STR "cache", ".clr .tmp .bx", CF_TYPE_DATA }, //clr=cached color { CF_TYPE_MULTI_CACHE, "data" DIR_SEPARATOR_STR "multidata", ".pcx .png .jpg .dds .fs2 .txt", CF_TYPE_DATA }, { CF_TYPE_MISSIONS, "data" DIR_SEPARATOR_STR "missions", ".fs2 .fc2 .ntl .ssv", CF_TYPE_DATA }, @@ -88,10 +87,7 @@ cf_pathtype Pathtypes[CF_MAX_PATH_TYPES] = { { CF_TYPE_INTEL_ANIMS, "data" DIR_SEPARATOR_STR "intelanims", ".pcx .ani .eff .tga .jpg .png .dds", CF_TYPE_DATA }, { CF_TYPE_SCRIPTS, "data" DIR_SEPARATOR_STR "scripts", ".lua .lc .fnl", CF_TYPE_DATA }, { CF_TYPE_FICTION, "data" DIR_SEPARATOR_STR "fiction", ".txt", CF_TYPE_DATA }, - { CF_TYPE_FREDDOCS, "data" DIR_SEPARATOR_STR "freddocs", ".html", CF_TYPE_DATA }, - { CF_TYPE_INTERFACE_MARKUP, "data" DIR_SEPARATOR_STR "interface" DIR_SEPARATOR_STR "markup", ".rml", CF_TYPE_INTERFACE }, - { CF_TYPE_INTERFACE_CSS, "data" DIR_SEPARATOR_STR "interface" DIR_SEPARATOR_STR "css", ".rcss", CF_TYPE_INTERFACE }, - { CF_TYPE_PLAYER_BINDS, "data" DIR_SEPARATOR_STR "players" DIR_SEPARATOR_STR "presets", ".json", CF_TYPE_PLAYERS }, + { CF_TYPE_FREDDOCS, "data" DIR_SEPARATOR_STR "freddocs", ".html", CF_TYPE_DATA } }; // clang-format on @@ -473,7 +469,7 @@ int cf_delete(const char *filename, int path_type, uint32_t location_flags) Assert(CF_TYPE_SPECIFIED(path_type)); - cf_create_default_path_string(longname, path_type, filename, false, location_flags); + cf_create_default_path_string(longname, path_type, filename, location_flags); return (_unlink(longname.c_str()) != -1); } @@ -623,7 +619,7 @@ void cf_create_directory(int dir_type, uint32_t location_flags) int i; for (i=num_dirs-1; i>=0; i-- ) { - cf_create_default_path_string(longname, dir_tree[i], nullptr, false, location_flags); + cf_create_default_path_string(longname, dir_tree[i], nullptr, location_flags); if (stat(longname.c_str(), &statbuf) != 0) { mprintf(( "CFILE: Creating new directory '%s'\n", longname.c_str() )); mkdir_recursive(longname.c_str()); @@ -648,7 +644,7 @@ void cf_create_directory(int dir_type, uint32_t location_flags) // CFILE* _cfopen(const char* source, int line, const char* file_path, const char* mode, int type, int dir_type, - bool localize, uint32_t location_flags) + bool /* localize */, uint32_t location_flags) { /* Bobboau, what is this doing here? 31 is way too short... - Goober5000 if( strlen(file_path) > 31 ) @@ -689,7 +685,7 @@ CFILE* _cfopen(const char* source, int line, const char* file_path, const char* // Create the directory if necessary cf_create_directory(dir_type, location_flags); - cf_create_default_path_string(longname, dir_type, file_path, false, location_flags); + cf_create_default_path_string(longname, dir_type, file_path, location_flags); } Assert( !(type & CFILE_MEMORY_MAPPED) ); @@ -739,10 +735,8 @@ CFILE* _cfopen(const char* source, int line, const char* file_path, const char* //================================================ // Search for file on disk, on cdrom, or in a packfile - char copy_file_path[MAX_PATH_LEN]; // FIX change in memory from cf_find_file_location - strcpy_s(copy_file_path, file_path); + auto find_res = cf_find_file_location(file_path, dir_type, location_flags); - auto find_res = cf_find_file_location( copy_file_path, dir_type, localize, location_flags ); if ( find_res.found ) { // Fount it, now create a cfile out of it @@ -1848,28 +1842,51 @@ int cfile_get_path_type(const SCP_string& dir) { SCP_string buf = dir; - // Remove trailing slashes; avoid buffer overflow on 1-char strings - while (buf.size() > 0 && (buf[buf.size() - 1] == '\\' || buf[buf.size() - 1] == '/')) { - buf.resize(buf.size() - 1); + // remove leading and trailing slashes + if ( !buf.empty() ) { + auto start = buf.find_first_not_of("\\/"); + auto end = buf.find_last_not_of("\\/"); + + if ( (start > 0) || (end < buf.length()-1) ) { + buf = buf.substr(start, end-start+1); + } } - // Remove leading slashes - while (buf.size() > 0 && (buf[0] == '\\' || buf[0] == '/')) { - buf = buf.substr(1); + if (buf.empty()) { + return CF_TYPE_ROOT; } // Use official DIR_SEPARATOR_CHAR - for (char& c : buf) { - if (c == '\\' || c == '/') { - c = DIR_SEPARATOR_CHAR; - } - } + extern void normalize_directory_separators(SCP_string &str); + normalize_directory_separators(buf); + + // identify path type + auto best_match = CF_TYPE_INVALID; for (auto& Pathtype : Pathtypes) { - if (Pathtype.path != nullptr && buf == Pathtype.path) { - return Pathtype.index; + if ( !Pathtype.path ) { + continue; + } + + // skip root, it should have been detected before getting here + if (Pathtype.index == CF_TYPE_ROOT) { + continue; + } + + // don't allow unknown subdirs directly under data + if (Pathtype.index == CF_TYPE_DATA) { + if (buf == Pathtype.path) { + best_match = CF_TYPE_DATA; + } + + continue; + } + + // for everything else just find closest match, allowing for unknown subdirectories + if ( !buf.compare(0, strlen(Pathtype.path), Pathtype.path) ) { + best_match = Pathtype.index; } } - return CF_TYPE_INVALID; + return best_match; } diff --git a/code/cfile/cfile.h b/code/cfile/cfile.h index 17d32ff4a22..4754f65bea5 100644 --- a/code/cfile/cfile.h +++ b/code/cfile/cfile.h @@ -49,42 +49,38 @@ typedef struct { #define CF_TYPE_MODELS 5 #define CF_TYPE_TABLES 6 #define CF_TYPE_SOUNDS 7 -#define CF_TYPE_SOUNDS_8B22K 8 -#define CF_TYPE_SOUNDS_16B11K 9 -#define CF_TYPE_VOICE 10 -#define CF_TYPE_VOICE_BRIEFINGS 11 -#define CF_TYPE_VOICE_CMD_BRIEF 12 -#define CF_TYPE_VOICE_DEBRIEFINGS 13 -#define CF_TYPE_VOICE_PERSONAS 14 -#define CF_TYPE_VOICE_SPECIAL 15 -#define CF_TYPE_VOICE_TRAINING 16 -#define CF_TYPE_MUSIC 17 -#define CF_TYPE_MOVIES 18 -#define CF_TYPE_INTERFACE 19 -#define CF_TYPE_FONT 20 -#define CF_TYPE_EFFECTS 21 -#define CF_TYPE_HUD 22 -#define CF_TYPE_PLAYERS 23 -#define CF_TYPE_PLAYER_IMAGES 24 -#define CF_TYPE_SQUAD_IMAGES 25 -#define CF_TYPE_SINGLE_PLAYERS 26 -#define CF_TYPE_MULTI_PLAYERS 27 -#define CF_TYPE_CACHE 28 -#define CF_TYPE_MULTI_CACHE 29 -#define CF_TYPE_MISSIONS 30 -#define CF_TYPE_CONFIG 31 -#define CF_TYPE_DEMOS 32 -#define CF_TYPE_CBANIMS 33 -#define CF_TYPE_INTEL_ANIMS 34 -#define CF_TYPE_SCRIPTS 35 -#define CF_TYPE_FICTION 36 -#define CF_TYPE_FREDDOCS 37 -#define CF_TYPE_INTERFACE_MARKUP 38 -#define CF_TYPE_INTERFACE_CSS 39 -#define CF_TYPE_PLAYER_BINDS 40 +#define CF_TYPE_VOICE 8 +#define CF_TYPE_VOICE_BRIEFINGS 9 +#define CF_TYPE_VOICE_CMD_BRIEF 10 +#define CF_TYPE_VOICE_DEBRIEFINGS 11 +#define CF_TYPE_VOICE_PERSONAS 12 +#define CF_TYPE_VOICE_SPECIAL 13 +#define CF_TYPE_VOICE_TRAINING 14 +#define CF_TYPE_MUSIC 15 +#define CF_TYPE_MOVIES 16 +#define CF_TYPE_INTERFACE 17 +#define CF_TYPE_FONT 18 +#define CF_TYPE_EFFECTS 19 +#define CF_TYPE_HUD 20 +#define CF_TYPE_PLAYERS 21 +#define CF_TYPE_PLAYER_IMAGES 22 +#define CF_TYPE_SQUAD_IMAGES 23 +#define CF_TYPE_SINGLE_PLAYERS 24 +#define CF_TYPE_MULTI_PLAYERS 25 +#define CF_TYPE_PLAYER_BINDS 26 +#define CF_TYPE_CACHE 27 +#define CF_TYPE_MULTI_CACHE 28 +#define CF_TYPE_MISSIONS 29 +#define CF_TYPE_CONFIG 30 +#define CF_TYPE_DEMOS 31 +#define CF_TYPE_CBANIMS 32 +#define CF_TYPE_INTEL_ANIMS 33 +#define CF_TYPE_SCRIPTS 34 +#define CF_TYPE_FICTION 35 +#define CF_TYPE_FREDDOCS 36 #define CF_MAX_PATH_TYPES \ - 41 // Can be as high as you'd like //DTP; yeah but beware alot of things uses CF_MAX_PATH_TYPES + 37 // Can be as high as you'd like //DTP; yeah but beware alot of things uses CF_MAX_PATH_TYPES // TRUE if type is specified and valid #define CF_TYPE_SPECIFIED(path_type) (((path_type)>CF_TYPE_INVALID) && ((path_type) #include #include /* needed for memory mapping of file functions */ +#include #endif #ifdef SCP_UNIX @@ -37,7 +38,6 @@ #include "cmdline/cmdline.h" #include "globalincs/pstypes.h" #include "def_files/def_files.h" -#include "localization/localize.h" #include "osapi/osapi.h" #include "parse/parselo.h" @@ -96,6 +96,7 @@ typedef struct cf_file { int size; // How big it is in bytes int pack_offset; // For pack files, where it is at. 0 if not in a pack file. This can be used to tell if in a pack file. SCP_string real_name; // For real files, the full path + SCP_string sub_path; // subfolder path off of pathtype root (should include trailing directory seperator) const void* data; // For in-memory files, the data pointer cf_file() : root_index(0), pathtype_index(0), write_time(0), size(0), pack_offset(0), data(nullptr) {} @@ -188,41 +189,84 @@ cf_root *cf_create_root() struct _file_list_t { SCP_string name; + SCP_string sub_path; time_t m_time; size_t size; + int pathtype; + int offset; + + _file_list_t() : m_time(0), size(0), pathtype(CF_TYPE_INVALID), offset(0) {} }; static bool sort_file_list(const _file_list_t &a, const _file_list_t &b) { + if ( a.sub_path.empty() && !b.sub_path.empty() ) { + return stricmp(a.name.c_str(), b.sub_path.c_str()) < 0; + } + + if ( !a.sub_path.empty() && b.sub_path.empty() ) { + return stricmp(a.sub_path.c_str(), b.name.c_str()) < 0; + } + + if ( !a.sub_path.empty() && !b.sub_path.empty() ) { + int rc = stricmp(a.sub_path.c_str(), b.sub_path.c_str()); + + if (rc) { + return rc < 0; + } + } + return stricmp(a.name.c_str(), b.name.c_str()) < 0; } -static size_t cf_get_list_of_files(SCP_string &path, SCP_vector<_file_list_t> &files, const char *filter = nullptr) +static size_t cf_get_list_of_files(const SCP_string &in_path, SCP_vector<_file_list_t> &files, const char *filter = nullptr, bool recursive = false, const char *subpath = nullptr) { _file_list_t nfile; + SCP_string path = in_path; + + if (path.back() != DIR_SEPARATOR_CHAR) { + path += DIR_SEPARATOR_CHAR; + } + + if (subpath) { + path += subpath; + path += DIR_SEPARATOR_CHAR; + } #if defined _WIN32 - SCP_string tmppath = path; intptr_t find_handle; _finddata_t find; - if (filter) { - if (tmppath.back() != DIR_SEPARATOR_CHAR) { - tmppath += DIR_SEPARATOR_CHAR; - } + // make sure we return all entries by default + path += "*"; - tmppath += filter; - } - - find_handle = _findfirst(tmppath.c_str(), &find); + find_handle = _findfirst(path.c_str(), &find); if (find_handle == -1) { return 0; } do { + if (find.attrib & _A_SUBDIR) { + if ( recursive && strcmp(find.name, ".") && strcmp(find.name, "..") ) { + SCP_string sub; + + if (subpath) { + sub = subpath; + sub += DIR_SEPARATOR_CHAR; + } + + sub += find.name; + + cf_get_list_of_files(in_path, files, filter, recursive, sub.c_str()); + } + + continue; + } + + if (filter && !PathMatchSpec(find.name, filter)) { continue; } @@ -230,6 +274,11 @@ static size_t cf_get_list_of_files(SCP_string &path, SCP_vector<_file_list_t> &f nfile.m_time = find.time_write; nfile.size = find.size; + if (subpath) { + nfile.sub_path = subpath; + nfile.sub_path += DIR_SEPARATOR_CHAR; + } + files.push_back(nfile); } while ( !_findnext(find_handle, &find) ); @@ -248,21 +297,27 @@ static size_t cf_get_list_of_files(SCP_string &path, SCP_vector<_file_list_t> &f } while ((dir = readdir(dirp)) != nullptr) { - if (filter && fnmatch(filter, dir->d_name, 0)) { + filepath = path; + filepath += dir->d_name; + + struct stat buf; + + if (stat(filepath.c_str(), &buf) == -1) { continue; } - filepath = path; + if ( recursive && S_ISDIR(buf.st_mode) && strcmp(dir->d_name, ".") && strcmp(dir->d_name, "..") ) { + SCP_string sub; - if (filepath.back() != DIR_SEPARATOR_CHAR) { - filepath += DIR_SEPARATOR_CHAR; - } + if (subpath) { + sub = subpath; + sub += DIR_SEPARATOR_CHAR; + } - filepath += dir->d_name; + sub += dir->d_name; - struct stat buf; + cf_get_list_of_files(in_path, files, filter, recursive, sub.c_str()); - if (stat(filepath.c_str(), &buf) == -1) { continue; } @@ -270,10 +325,24 @@ static size_t cf_get_list_of_files(SCP_string &path, SCP_vector<_file_list_t> &f continue; } + // zero byte files shouldn't be indexed, but that breaks unit tests + // if ( !buf.st_size ) { + // continue; + // } + + if (filter && fnmatch(filter, dir->d_name, 0)) { + continue; + } + nfile.name = dir->d_name; nfile.m_time = buf.st_mtime; nfile.size = buf.st_size; + if (subpath) { + nfile.sub_path = subpath; + nfile.sub_path += DIR_SEPARATOR_CHAR; + } + files.push_back(nfile); } @@ -281,11 +350,36 @@ static size_t cf_get_list_of_files(SCP_string &path, SCP_vector<_file_list_t> &f #endif - std::sort(files.begin(), files.end(), sort_file_list); + if ( !subpath ) { + std::sort(files.begin(), files.end(), sort_file_list); + } return files.size(); } +static bool cf_should_scan_subdirs(int pathtype) +{ + if ((pathtype <= CF_TYPE_DATA) || (pathtype >= CF_MAX_PATH_TYPES)) { + return false; + } + + switch (pathtype) { + // all pilot file related directories, except for images, should be ignored + case CF_TYPE_PLAYERS: + case CF_TYPE_SINGLE_PLAYERS: + case CF_TYPE_MULTI_PLAYERS: + case CF_TYPE_PLAYER_BINDS: + // voice, missing extensions, has no automatic subfolder support + case CF_TYPE_VOICE: + return false; + + default: + break; + } + + return true; +} + static void cf_init_root_pathtypes(cf_root *root) { @@ -360,7 +454,7 @@ static void cf_init_root_pathtypes(cf_root *root) #endif } -static SCP_string cf_get_root_pathtype(const cf_root *root, const int type) +static SCP_string cf_get_root_pathtype(__UNUSED const cf_root *root, const int type) { #ifdef SCP_UNIX auto parentPathIter = root->pathTypeToRealPath.find(type); @@ -459,14 +553,15 @@ void cf_build_pack_list( cf_root *root ) } } -static char normalize_directory_separator(char in) +void normalize_directory_separators(SCP_string &str) { - if (in == '/') - { - return DIR_SEPARATOR_CHAR; + char bad_sep = '/'; + + if (bad_sep == DIR_SEPARATOR_CHAR) { + bad_sep = '\\'; } - return in; + std::replace(str.begin(), str.end(), bad_sep, DIR_SEPARATOR_CHAR); } static void cf_add_mod_roots(const char* rootDirectory, uint32_t basic_location) @@ -492,7 +587,7 @@ static void cf_add_mod_roots(const char* rootDirectory, uint32_t basic_location) } // normalize the path to the native path format - std::transform(rootPath.begin(), rootPath.end(), rootPath.begin(), normalize_directory_separator); + normalize_directory_separators(rootPath); cf_root* root = cf_create_root(); @@ -673,6 +768,63 @@ int is_ext_in_list( const char *ext_list, const char *ext ) return 0; } +// Run a basic test for indexed files that may be shadowed +#ifndef NDEBUG + #define ENABLE_SHADOW_CHECK 1 +#endif + +static void check_file_shadows(const int root_index __UNUSED, const int pathtype __UNUSED, const SCP_string &name __UNUSED, const SCP_string sub_path __UNUSED) +{ +#if ENABLE_SHADOW_CHECK + if ( !cf_should_scan_subdirs(pathtype) ) { + return; + } + + SCP_string curfile, newfile; + + const auto root = cf_get_root(root_index); + + newfile = root->path + ((root->roottype == CF_ROOTTYPE_PACK) ? "::" : ""); + newfile += cf_get_root_pathtype(root, pathtype) + DIR_SEPARATOR_CHAR; + newfile += sub_path + name; + + for (uint i = 0; i < Num_files; ++i) { + const auto f = cf_get_file(i); + const auto r = cf_get_root(f->root_index); + + // skip memory roots, no subdirs there + if (r->roottype == CF_ROOTTYPE_MEMORY) { + continue; + } + + // if basic path type doesn't match then skip + if (pathtype != f->pathtype_index) { + continue; + } + + if (name != f->name_ext) { + continue; + } + + // if the subpath matches then consider it an override rather than shadow + if (sub_path == f->sub_path) { + continue; + } + + curfile = r->path + ((r->roottype == CF_ROOTTYPE_PACK) ? "::" : ""); + curfile += cf_get_root_pathtype(r, pathtype) + DIR_SEPARATOR_CHAR; + curfile += f->sub_path + f->name_ext; + + // this log message occurs in the middle of an existing line, hence the extra new lines + mprintf(("\nWARNING! A file being indexed may be shadowed by an existing file!\n New:\n %s\n Existing:\n %s\n", + newfile.c_str(), curfile.c_str())); + + break; + } +#endif +} + + void cf_search_root_path(int root_index) { int i; @@ -726,7 +878,7 @@ void cf_search_root_path(int root_index) SCP_vector<_file_list_t> files; - cf_get_list_of_files(search_path, files, "*.*"); + cf_get_list_of_files(search_path, files, "*.*", cf_should_scan_subdirs(i)); for (auto &file : files) { auto ext_idx = file.name.rfind('.'); @@ -735,6 +887,8 @@ void cf_search_root_path(int root_index) continue; } + check_file_shadows(root_index, i, file.name, file.sub_path); + cf_file *cfile = cf_create_file(); cfile->name_ext = file.name; @@ -743,7 +897,8 @@ void cf_search_root_path(int root_index) cfile->write_time = file.m_time; cfile->size = static_cast(file.size); cfile->pack_offset = 0; - cfile->real_name = search_path + DIR_SEPARATOR_STR + file.name; + cfile->real_name = search_path + DIR_SEPARATOR_STR + file.sub_path + file.name; + cfile->sub_path = file.sub_path; ++num_files; } @@ -767,6 +922,31 @@ typedef struct VP_FILE { _fs_time_t write_time; } VP_FILE; +static int cf_add_pack_files(const int root_index, SCP_vector<_file_list_t> &files) +{ + if (files.empty()) { + return 0; + } + + std::sort(files.begin(), files.end(), sort_file_list); + + for (auto &file : files) { + check_file_shadows(root_index, file.pathtype, file.name, file.sub_path); + + cf_file *pf = cf_create_file(); + + pf->name_ext = file.name; + pf->root_index = root_index; + pf->pathtype_index = file.pathtype; + pf->write_time = file.m_time; + pf->size = static_cast(file.size); + pf->pack_offset = file.offset; // Mark as a packed file + pf->sub_path = file.sub_path; + } + + return static_cast(files.size()); +} + void cf_search_root_pack(int root_index) { int num_files = 0; @@ -805,17 +985,22 @@ void cf_search_root_pack(int root_index) // Read index info fseek(fp, VP_header.index_offset, SEEK_SET); - char search_path[CF_MAX_PATHNAME_LENGTH]; - strcpy_s( search_path, "" ); - + SCP_string search_path; + SCP_string sub_path; + int path_type = CF_TYPE_INVALID; + + SCP_vector<_file_list_t> files; + + files.reserve(256); // should be set to a good baseline of files per path + // Go through all the files int i; for (i=0; i search_path) && (*p != DIR_SEPARATOR_CHAR) ) { - p--; + if ( !stricmp(find.filename, "..")) { + auto end = search_path.rfind(DIR_SEPARATOR_CHAR); + + if (end != SCP_string::npos) { + search_path.erase(end); + } else { + search_path.clear(); } - *p = 0; } else { - if ( search_path_len && (search_path[search_path_len-1] != DIR_SEPARATOR_CHAR) ) { - strcat_s( search_path, DIR_SEPARATOR_STR ); + if ( !search_path.empty() && (search_path.back() != DIR_SEPARATOR_CHAR) ) { + search_path += DIR_SEPARATOR_CHAR; } - strcat_s( search_path, find.filename ); + + search_path += find.filename; + } + + auto rval = cfile_get_path_type(search_path); + + // if the pathtype root changed then add all of the files + if (rval != path_type) { + num_files += cf_add_pack_files(root_index, files); + files.clear(); } - //mprintf(( "Current dir = '%s'\n", search_path )); + path_type = rval; + + if ( (path_type > CF_TYPE_DATA) && Pathtypes[path_type].path && + (search_path.length() > strlen(Pathtypes[path_type].path)) ) + { + sub_path = search_path.substr(strlen(Pathtypes[path_type].path)+1) + DIR_SEPARATOR_STR; + } else { + sub_path.clear(); + } + + //mprintf(( "Current dir = '%s'\n", search_path.c_str() )); } else { - - int j; - - for (j=CF_TYPE_ROOT; jname_ext = find.filename; - file->root_index = root_index; - file->pathtype_index = j; - file->write_time = (time_t)find.write_time; - file->size = find.size; - file->pack_offset = find.offset; // Mark as a packed file - - num_files++; - //mprintf(( "Found pack file '%s'\n", file->name_ext )); - } + if (path_type != CF_TYPE_INVALID) { + char *ext = strrchr( find.filename, '.' ); + if ( ext ) { + if ( is_ext_in_list( Pathtypes[path_type].extensions, ext ) ) { + // Found a file!!!! + _file_list_t file; + + file.name = find.filename; + file.m_time = find.write_time; + file.size = find.size; + file.sub_path = sub_path; + file.pathtype = path_type; + file.offset = find.offset; + + files.push_back(file); + + //mprintf(( "Found pack file '%s'\n", find.filename )); } } } } } + // add final set of files + num_files += cf_add_pack_files(root_index, files); + files.clear(); + fclose(fp); mprintf(( "%i files\n", num_files )); @@ -965,6 +1169,34 @@ void cf_free_secondary_filelist() Num_files = 0; } +static bool is_absolute_path(const char *path) +{ + if ( !path || !strlen(path) ) { + return false; + } + +#ifdef WIN32 + return (PathIsRelative(path) == FALSE); +#else + return (*path == '/'); +#endif +} + +static bool sub_path_match(const SCP_string &search, const SCP_string &index) +{ + // if search path is empty then we only care about the file name + if (search.empty()) { + return true; + } + + // if we have search but no index then fail + if (index.empty()) { + return false; + } + + return !stricmp(search.c_str(), index.c_str()); +} + /** * Searches for a file. * @@ -972,17 +1204,16 @@ void cf_free_secondary_filelist() * * @param filespec Filename & extension * @param pathtype See CF_TYPE_ defines in CFILE.H - * @param localize Undertake localization * @param location_flags Specifies where to search for the specified flag * * @return A structure which describes the found file */ -CFileLocation cf_find_file_location(const char* filespec, int pathtype, bool localize, uint32_t location_flags) +CFileLocation cf_find_file_location(const char* filespec, int pathtype, uint32_t location_flags) { int i; uint ui; int cfs_slow_search = 0; - char longname[MAX_PATH_LEN]; + SCP_string longname; Assert( (filespec != NULL) && (strlen(filespec) > 0) ); //-V805 @@ -993,10 +1224,9 @@ CFileLocation cf_find_file_location(const char* filespec, int pathtype, bool loc // of the file // NOTE: full path should also include localization, if so desired - auto last_separator = strrchr(filespec, DIR_SEPARATOR_CHAR); // do we have a full path already? - if ( last_separator ) { + if (is_absolute_path(filespec)) { FILE *fp = fopen(filespec, "rb" ); if (fp) { CFileLocation res(true); @@ -1004,6 +1234,10 @@ CFileLocation cf_find_file_location(const char* filespec, int pathtype, bool loc fclose(fp); + auto last_separator = strrchr(filespec, DIR_SEPARATOR_CHAR); + + Assertion(last_separator != nullptr, "We have full path, but no separator!?"); + res.offset = 0; res.full_name = filespec; res.name_ext = last_separator + 1; @@ -1027,9 +1261,6 @@ CFileLocation cf_find_file_location(const char* filespec, int pathtype, bool loc } } - memset( longname, 0, sizeof(longname) ); - - for (ui=0; uiname_ext.c_str()) ) { - CFileLocation res(true); - res.size = static_cast(f->size); - res.offset = (size_t)f->pack_offset; - res.data_ptr = f->data; - res.name_ext = f->name_ext; - - if (f->data != nullptr) { - // This is an in-memory file so we just copy the pathtype name + file name - res.full_name = Pathtypes[f->pathtype_index].path; - res.full_name += DIR_SEPARATOR_STR; - res.full_name += f->name_ext; - } else if (f->pack_offset < 1) { - // This is a real file, return the actual file path - res.full_name = f->real_name; - } else { - // File is in a pack file - cf_root *r = cf_get_root(f->root_index); - - res.full_name = r->path; - } - - return res; - } - } + if ( !sub_path_match(sub_path, f->sub_path) ) { + continue; } // file either not localized or localized version not found - if ( !stricmp(filespec, f->name_ext.c_str()) ) { + if ( !stricmp(filename.c_str(), f->name_ext.c_str()) ) { CFileLocation res(true); res.size = static_cast(f->size); res.offset = (size_t)f->pack_offset; @@ -1136,6 +1354,7 @@ CFileLocation cf_find_file_location(const char* filespec, int pathtype, bool loc // This is an in-memory file so we just copy the pathtype name + file name res.full_name = Pathtypes[f->pathtype_index].path; res.full_name += DIR_SEPARATOR_STR; + res.full_name += f->sub_path; res.full_name += f->name_ext; } else if (f->pack_offset < 1) { // This is a real file, return the actual file path @@ -1168,25 +1387,22 @@ extern char *stristr(char *str, const char *substr); * @param ext_list Extension filter list * @param pathtype See CF_TYPE_ defines in CFILE.H * @param max_out Maximum string length that should be stuffed into pack_filename - * @param localize Undertake localization * * @return A structure containing information about the found file */ -CFileLocationExt cf_find_file_location_ext( const char *filename, const int ext_num, const char **ext_list, int pathtype, bool localize) +CFileLocationExt cf_find_file_location_ext(const char *filename, const int ext_num, const char **ext_list, int pathtype) { int cur_ext, i; uint ui; int cfs_slow_search = 0; - char longname[MAX_PATH_LEN]; - char filespec[MAX_FILENAME_LEN]; - char *p = NULL; - + SCP_string longname; + Assert( (filename != NULL) && (strlen(filename) < MAX_FILENAME_LEN) ); Assert( (ext_list != NULL) && (ext_num > 1) ); // if we are searching for just one ext // then this is the wrong function to use // if we have a full path already then fail. this function if for searching via filter only! - if ( strchr(filename, DIR_SEPARATOR_CHAR) ) { // do we have a full path already? + if (is_absolute_path(filename)) { // do we have a full path already? Int3(); return CFileLocationExt(); } @@ -1202,11 +1418,31 @@ CFileLocationExt cf_find_file_location_ext( const char *filename, const int ext_ search_order[num_search_dirs++] = i; } - memset( longname, 0, sizeof(longname) ); - memset( filespec, 0, sizeof(filespec) ); + // fixup filename and sub directory path, if needed + SCP_string filespec = filename; + SCP_string sub_path; + + normalize_directory_separators(filespec); + + auto seperator = filespec.rfind(DIR_SEPARATOR_CHAR); + + if (seperator != SCP_string::npos) { + sub_path = filespec.substr(0, seperator); + sub_path += DIR_SEPARATOR_CHAR; + + filespec.erase(0, seperator+1); + } // strip any existing extension - strncpy(filespec, filename, MAX_FILENAME_LEN-1); + // (NOTE: to be fully retail compatible, we need to support multiple periods for something like *_1.5.wav, + // which means that we need to strip a length of >2 only, assuming that all valid ext are at least 2 chars) + auto dot = filespec.rfind('.'); + + if ( (dot != SCP_string::npos) && ((filespec.length() - dot) > 2) ) { + filespec.erase(dot); + } + + SCP_string filespec_ext; for (ui = 0; ui < num_search_dirs; ui++) { cfs_slow_search = 0; @@ -1236,20 +1472,13 @@ CFileLocationExt cf_find_file_location_ext( const char *filename, const int ext_ continue; for (cur_ext = 0; cur_ext < ext_num; cur_ext++) { - // strip any extension and add the one we want to check for - // (NOTE: to be fully retail compatible, we need to support multiple periods for something like *_1.5.wav, - // which means that we need to strip a length of >2 only, assuming that all valid ext are at least 2 chars) - p = strrchr(filespec, '.'); - if ( p && (strlen(p) > 2) ) - (*p) = 0; - - strcat_s( filespec, ext_list[cur_ext] ); - - if ( !cf_create_default_path_string(longname, sizeof(longname)-1, search_order[ui], filespec, localize) ) { + filespec_ext = filespec + ext_list[cur_ext]; + + if ( !cf_create_default_path_string(longname, search_order[ui], filespec_ext.c_str()) ) { continue; } - FILE *fp = fopen(longname, "rb" ); + FILE *fp = fopen(longname.c_str(), "rb" ); if (fp) { CFileLocationExt res(cur_ext); @@ -1260,7 +1489,7 @@ CFileLocationExt cf_find_file_location_ext( const char *filename, const int ext_ res.offset = 0; res.full_name = longname; - res.name_ext = filespec; + res.name_ext = filespec_ext; return res; } @@ -1269,19 +1498,9 @@ CFileLocationExt cf_find_file_location_ext( const char *filename, const int ext_ // Search the pak files and CD-ROM. - // first off, make sure that we don't have an extension - // (NOTE: to be fully retail compatible, we need to support multiple periods for something like *_1.5.wav, - // which means that we need to strip a length of >2 only, assuming that all valid ext are at least 2 chars) - p = strrchr(filespec, '.'); - if ( p && (strlen(p) > 2) ) - (*p) = 0; - - // go ahead and get our length, which is used to test with later - size_t filespec_len = strlen(filespec); - - // get total legnth, with extension, which is iused to test with later + // get total length, with extension, which is used to test with later // (FIXME: this assumes that everything in ext_list[] is the same length!) - size_t filespec_len_big = filespec_len + strlen(ext_list[0]); + size_t filespec_len_big = filespec.length() + strlen(ext_list[0]); SCP_vector< cf_file* > file_list_index; int last_root_index = -1; @@ -1297,12 +1516,17 @@ CFileLocationExt cf_find_file_location_ext( const char *filename, const int ext_ if ( (num_search_dirs == 1) && (pathtype != f->pathtype_index) ) continue; + // ... match subdirectories (if specified) + if ( !sub_path_match(sub_path, f->sub_path) ) { + continue; + } + // ... check that our names are the same length (accounting for the missing extension on our own name) if (f->name_ext.length() != filespec_len_big ) continue; // ... check that we match the base filename - if ( strnicmp(f->name_ext.c_str(), filespec, filespec_len) != 0 ) + if ( strnicmp(f->name_ext.c_str(), filespec.c_str(), filespec.length()) != 0 ) continue; // ... make sure that it's one of our supported types @@ -1337,47 +1561,11 @@ CFileLocationExt cf_find_file_location_ext( const char *filename, const int ext_ for (cur_ext = 0; cur_ext < ext_num; cur_ext++) { for (SCP_vector::iterator fli = file_list_index.begin(); fli != file_list_index.end(); ++fli) { cf_file *f = *fli; - - strcat_s( filespec, ext_list[cur_ext] ); - - if (localize) { - // create localized filespec - strncpy(longname, filespec, MAX_PATH_LEN - 1); - - if ( lcl_add_dir_to_path_with_filename(longname, MAX_PATH_LEN - 1) ) { - if ( !stricmp(longname, f->name_ext.c_str()) ) { - CFileLocationExt res(cur_ext); - res.found = true; - res.size = static_cast(f->size); - res.offset = (size_t)f->pack_offset; - res.data_ptr = f->data; - res.name_ext = f->name_ext; - - if (f->data != nullptr) { - // This is an in-memory file so we just copy the pathtype name + file name - res.full_name = Pathtypes[f->pathtype_index].path; - res.full_name += DIR_SEPARATOR_STR; - res.full_name += f->name_ext; - } else if (f->pack_offset < 1) { - // This is a real file, return the actual file path - res.full_name = f->real_name; - } else { - // File is in a pack file - cf_root *r = cf_get_root(f->root_index); - - res.full_name = r->path; - } - - // found it, so cleanup and return - file_list_index.clear(); - - return res; - } - } - } + + filespec_ext = filespec + ext_list[cur_ext]; // file either not localized or localized version not found - if ( !stricmp(filespec, f->name_ext.c_str()) ) { + if ( !stricmp(filespec_ext.c_str(), f->name_ext.c_str()) ) { CFileLocationExt res(cur_ext); res.found = true; res.size = static_cast(f->size); @@ -1389,6 +1577,7 @@ CFileLocationExt cf_find_file_location_ext( const char *filename, const int ext_ // This is an in-memory file so we just copy the pathtype name + file name res.full_name = Pathtypes[f->pathtype_index].path; res.full_name += DIR_SEPARATOR_STR; + res.full_name += f->sub_path; res.full_name += f->name_ext; } else if (f->pack_offset < 1) { // This is a real file, return the actual file path @@ -1405,12 +1594,6 @@ CFileLocationExt cf_find_file_location_ext( const char *filename, const int ext_ return res; } - - // ok, we're still here, so strip off the extension again in order to - // prepare for the next run - p = strrchr(filespec, '.'); - if ( p && (strlen(p) > 2) ) - (*p) = 0; } } @@ -1516,7 +1699,7 @@ static int cf_file_already_in_list( SCP_vector &list, const char *fi // Note that filesystem listing is always sorted by name *before* sorting by the // provided sort order. This isn't strictly needed on NTFS, which always provides the list // sorted by name. But on mac/linux it's always needed. -int cf_get_file_list(SCP_vector& list, int pathtype, const char* filter, int sort, +int cf_get_file_list(SCP_vector& list, int pathtype, const char* _filter, int sort, SCP_vector* info, uint32_t location_flags) { uint i; @@ -1542,25 +1725,55 @@ int cf_get_file_list(SCP_vector& list, int pathtype, const char* fil Get_file_list_child = nullptr; } - cf_create_default_path_string(filespec, pathtype, (char*)Get_file_list_child, false, location_flags); + cf_create_default_path_string(filespec, pathtype, (char*)Get_file_list_child, location_flags); + + // fixup filter and sub directory path, if needed + SCP_string filter = _filter; + SCP_string sub_path; + bool glob = false; + + normalize_directory_separators(filter); + + auto seperator = filter.rfind(DIR_SEPARATOR_CHAR); + + if (seperator != SCP_string::npos) { + sub_path = filter.substr(0, seperator); + + if (sub_path == "*") { + glob = true; + } else { + sub_path += DIR_SEPARATOR_CHAR; + } + filter.erase(0, seperator+1); + } + + SCP_string fullname; SCP_vector<_file_list_t> files; - cf_get_list_of_files(filespec, files, filter); + cf_get_list_of_files(filespec, files, filter.c_str()); for (auto &file : files) { - if ( !Get_file_list_filter || (*Get_file_list_filter)(file.name.c_str()) ) { - if (check_duplicates && cf_file_already_in_list(list, file.name.c_str())) { - continue; - } + if ( !glob && !sub_path_match(sub_path, file.sub_path) ) { + continue; + } - SCP_string::size_type pos = file.name.find_last_of('.'); + auto pos = file.name.rfind('.'); - if (pos != SCP_string::npos) { - list.push_back(file.name.substr(0, pos)); - } else { - list.push_back(file.name); - } + if ( !sub_path.empty() ) { + // prepend file name with sub directory path + // allows us to open the specific file and allows duplicate filenames in different paths + fullname = file.sub_path + file.name.substr(0, pos); + } else { + fullname = file.name.substr(0, pos); + } + + if (check_duplicates && cf_file_already_in_list(list, fullname.c_str())) { + continue; + } + + if ( !Get_file_list_filter || (*Get_file_list_filter)(file.name.c_str()) ) { + list.push_back(fullname); if (info) { tinfo.write_time = file.m_time; @@ -1581,6 +1794,16 @@ int cf_get_file_list(SCP_vector& list, int pathtype, const char* fil for (i=0; ipack_offset != 0) { + // If the packfile skip flag is set we skip files in VPs but still search in directories + continue; + } + + if (Skip_memory_files && f->data != nullptr) { + // If we want to skip memory files and this is a memory file then ignore it + continue; + } + // only search paths we're supposed to... if ( (pathtype != CF_TYPE_ANY) && (pathtype != f->pathtype_index) ) { continue; @@ -1596,30 +1819,32 @@ int cf_get_file_list(SCP_vector& list, int pathtype, const char* fil } } - if ( !cf_matches_spec(filter, f->name_ext.c_str()) ) { + if ( !glob && !sub_path_match(sub_path, f->sub_path) ) { continue; } - if ( cf_file_already_in_list(list, f->name_ext.c_str()) ) { + if ( !cf_matches_spec(filter.c_str(), f->name_ext.c_str()) ) { continue; } - if (Skip_packfile_search && f->pack_offset != 0) { - // If the packfile skip flag is set we skip files in VPs but still search in directories - continue; + auto pos = f->name_ext.rfind('.'); + + if ( !sub_path.empty() ) { + // prepend file name with sub directory path + // allows us to open the specific file and allows duplicate filenames in different paths + fullname = f->sub_path + f->name_ext.substr(0, pos); + } else { + fullname = f->name_ext.substr(0, pos); } - if (Skip_memory_files && f->data != nullptr) { - // If we want to skip memory files and this is a memory file then ignore it + if ( cf_file_already_in_list(list, fullname.c_str()) ) { continue; } if ( !Get_file_list_filter || (*Get_file_list_filter)(f->name_ext.c_str()) ) { //mprintf(( "Found '%s' in root %d path %d\n", f->name_ext, f->root_index, f->pathtype_index )); - auto pos = f->name_ext.rfind('.'); - - list.push_back( f->name_ext.substr(0, pos) ); + list.push_back(fullname); if (info) { tinfo.write_time = f->write_time; @@ -1668,12 +1893,11 @@ int cf_file_already_in_list( int num_files, char **list, const char *filename ) // This one has a 'type', which is a CF_TYPE_* value. Because this specifies the directory // location, 'filter' only needs to be the filter itself, with no path information. // See above descriptions of cf_get_file_list() for more information about how it all works. -int cf_get_file_list(int max, char** list, int pathtype, const char* filter, int sort, file_list_info* info, +int cf_get_file_list(int max, char** list, int pathtype, const char* _filter, int sort, file_list_info* info, uint32_t location_flags) { uint i; int num_files = 0, own_flag = 0; - size_t l; if (max < 1) { Get_file_list_filter = NULL; @@ -1694,28 +1918,55 @@ int cf_get_file_list(int max, char** list, int pathtype, const char* filter, int Get_file_list_child = nullptr; } - cf_create_default_path_string(filespec, pathtype, (char*)Get_file_list_child, false, location_flags); + cf_create_default_path_string(filespec, pathtype, (char*)Get_file_list_child, location_flags); + + // fixup filter and sub directory path, if needed + SCP_string filter = _filter; + SCP_string sub_path; + bool glob = false; + normalize_directory_separators(filter); + + auto seperator = filter.rfind(DIR_SEPARATOR_CHAR); + + if (seperator != SCP_string::npos) { + sub_path = filter.substr(0, seperator); + + if (sub_path == "*") { + glob = true; + } else { + sub_path += DIR_SEPARATOR_CHAR; + } + + filter.erase(0, seperator+1); + } + + SCP_string fullname; SCP_vector<_file_list_t> files; - cf_get_list_of_files(filespec, files, filter); + cf_get_list_of_files(filespec, files, filter.c_str()); for (auto &file : files) { if (num_files >= max) { break; } - if ( !Get_file_list_filter || (*Get_file_list_filter)(file.name.c_str()) ) { - auto pos = file.name.rfind('.'); + if ( !glob && !sub_path_match(sub_path, file.sub_path) ) { + continue; + } - if (pos != SCP_string::npos) { - l = pos; - } else { - l = file.name.length(); - } + auto pos = file.name.rfind('.'); - list[num_files] = reinterpret_cast(vm_malloc(l + 1)); - SDL_strlcpy(list[num_files], file.name.substr(0, l).c_str(), l+1); + if ( !sub_path.empty() ) { + // prepend file name with sub directory path + // allows us to open the specific file and allows duplicate filenames in different paths + fullname = file.sub_path + file.name.substr(0, pos); + } else { + fullname = file.name.substr(0, pos); + } + + if ( !Get_file_list_filter || (*Get_file_list_filter)(file.name.c_str()) ) { + list[num_files] = vm_strdup(fullname.c_str()); if (info) { info[num_files].write_time = file.m_time; @@ -1738,6 +1989,20 @@ int cf_get_file_list(int max, char** list, int pathtype, const char* filter, int for (i=0; i= max) { + break; + } + + if (Skip_packfile_search && f->pack_offset != 0) { + // If the packfile skip flag is set we skip files in VPs but still search in directories + continue; + } + + if (Skip_memory_files && f->data != nullptr) { + // If we want to skip memory files and this is a memory file then ignore it + continue; + } + // only search paths we're supposed to... if ( (pathtype != CF_TYPE_ANY) && (pathtype != f->pathtype_index) ) { continue; @@ -1753,42 +2018,32 @@ int cf_get_file_list(int max, char** list, int pathtype, const char* filter, int } } - if (num_files >= max) - break; - - if ( !cf_matches_spec(filter, f->name_ext.c_str())) { + if ( !glob && !sub_path_match(sub_path, f->sub_path) ) { continue; } - if ( cf_file_already_in_list(num_files, list, f->name_ext.c_str()) ) { + if ( !cf_matches_spec(filter.c_str(), f->name_ext.c_str())) { continue; } - if (Skip_packfile_search && f->pack_offset != 0) { - // If the packfile skip flag is set we skip files in VPs but still search in directories - continue; + auto pos = f->name_ext.rfind('.'); + + if ( !sub_path.empty() ) { + // prepend file name with sub directory path + // allows us to open the specific file and allows duplicate filenames in different paths + fullname = f->sub_path + f->name_ext.substr(0, pos); + } else { + fullname = f->name_ext.substr(0, pos); } - if (Skip_memory_files && f->data != nullptr) { - // If we want to skip memory files and this is a memory file then ignore it + if ( cf_file_already_in_list(num_files, list, fullname.c_str()) ) { continue; } if ( !Get_file_list_filter || (*Get_file_list_filter)(f->name_ext.c_str()) ) { - //mprintf(( "Found '%s' in root %d path %d\n", f->name_ext, f->root_index, f->pathtype_index )); - auto pos = f->name_ext.rfind('.'); - - if (pos != SCP_string::npos) { - l = pos; - } else { - l = f->name_ext.length(); - } - - list[num_files] = (char *)vm_malloc(l + 1); - strncpy(list[num_files], f->name_ext.c_str(), l); - list[num_files][l] = 0; + list[num_files] = vm_strdup(fullname.c_str()); if (info) { info[num_files].write_time = f->write_time; @@ -1838,7 +2093,7 @@ int cf_file_already_in_list_preallocated( int num_files, char arr[][MAX_FILENAME // This one has a 'type', which is a CF_TYPE_* value. Because this specifies the directory // location, 'filter' only needs to be the filter itself, with no path information. // See above descriptions of cf_get_file_list() for more information about how it all works. -int cf_get_file_list_preallocated(int max, char arr[][MAX_FILENAME_LEN], char** list, int pathtype, const char* filter, +int cf_get_file_list_preallocated(int max, char arr[][MAX_FILENAME_LEN], char** list, int pathtype, const char* _filter, int sort, file_list_info* info, uint32_t location_flags) { int num_files = 0, own_flag = 0; @@ -1871,11 +2126,33 @@ int cf_get_file_list_preallocated(int max, char arr[][MAX_FILENAME_LEN], char** } // Search the default directories - cf_create_default_path_string(filespec, pathtype, (char*)Get_file_list_child, false, location_flags); + cf_create_default_path_string(filespec, pathtype, (char*)Get_file_list_child, location_flags); + + // fixup filter and sub directory path, if needed + SCP_string filter = _filter; + SCP_string sub_path; + + normalize_directory_separators(filter); + + auto seperator = filter.rfind(DIR_SEPARATOR_CHAR); + + if (seperator != SCP_string::npos) { + sub_path = filter.substr(0, seperator); + + if (sub_path != "*") { + sub_path += DIR_SEPARATOR_CHAR; + } + + filter.erase(0, seperator+1); + + // due to the limited filename length we can't reliably add sub paths to it + // so we just strip off the sub path and do a regular search instead, after a warning + Warning(LOCATION, "Subdirectory searches aren't available for cf_get_file_list_preallocated()!"); + } SCP_vector<_file_list_t> files; - cf_get_list_of_files(filespec, files, filter); + cf_get_list_of_files(filespec, files, filter.c_str()); for (auto &file : files) { if (num_files >= max) { @@ -1935,7 +2212,7 @@ int cf_get_file_list_preallocated(int max, char arr[][MAX_FILENAME_LEN], char** break; - if ( !cf_matches_spec(filter, f->name_ext.c_str()) ) { + if ( !cf_matches_spec(filter.c_str(), f->name_ext.c_str()) ) { continue; } @@ -1997,12 +2274,11 @@ int cf_get_file_list_preallocated(int max, char arr[][MAX_FILENAME_LEN], char** // filename - optional, if set, tacks the filename onto end of path. // Output: path - Fully qualified pathname. //Returns 0 if the result would be too long (invalid result) -int cf_create_default_path_string(char* path, uint path_max, int pathtype, const char* filename, bool localize, - uint32_t _location_flags) +int cf_create_default_path_string(char* path, uint path_max, int pathtype, const char* filename, uint32_t _location_flags) { SCP_string fullpath; - cf_create_default_path_string(fullpath, pathtype, filename, localize, _location_flags); + cf_create_default_path_string(fullpath, pathtype, filename, _location_flags); // if truncation would occur, return error if (fullpath.length() >= path_max) { @@ -2027,12 +2303,11 @@ int cf_create_default_path_string(char* path, uint path_max, int pathtype, const // filename - optional, if set, tacks the filename onto end of path. // Output: path - Fully qualified pathname. //Returns 0 if the result would be too long (invalid result) -int cf_create_default_path_string(SCP_string& path, int pathtype, const char* filename, bool /*localize*/, - uint32_t _location_flags) +int cf_create_default_path_string(SCP_string& path, int pathtype, const char* filename, uint32_t _location_flags) { uint32_t location_flags = _location_flags; - if ( filename && strchr(filename, DIR_SEPARATOR_CHAR) ) { + if ( filename && is_absolute_path(filename)) { // Already has full path path.assign(filename); @@ -2045,6 +2320,7 @@ int cf_create_default_path_string(SCP_string& path, int pathtype, const char* fi case CF_TYPE_PLAYERS: case CF_TYPE_MULTI_PLAYERS: case CF_TYPE_SINGLE_PLAYERS: + case CF_TYPE_PLAYER_BINDS: location_flags = CF_LOCATION_ROOT_USER | CF_LOCATION_ROOT_GAME | CF_LOCATION_TYPE_ROOT; break; } diff --git a/code/cfile/cfilesystem.h b/code/cfile/cfilesystem.h index 551ad280733..b251d28f56f 100644 --- a/code/cfile/cfilesystem.h +++ b/code/cfile/cfilesystem.h @@ -42,8 +42,8 @@ bool cf_check_location_flags(uint32_t check_flags, uint32_t desired_flags); // Output: path - Fully qualified pathname. //Returns 0 if result would be too long (invalid result) int cf_create_default_path_string(char* path, uint path_max, int pathtype, const char* filename = nullptr, - bool localize = false, uint32_t location_flags = CF_LOCATION_ALL); -int cf_create_default_path_string(SCP_string& path, int pathtype, const char* filename = nullptr, bool localize = false, + uint32_t location_flags = CF_LOCATION_ALL); +int cf_create_default_path_string(SCP_string& path, int pathtype, const char* filename = nullptr, uint32_t location_flags = CF_LOCATION_ALL); #endif //_CFILESYSTEM_H diff --git a/code/def_files/data/scripts/cfile_require.lua b/code/def_files/data/scripts/cfile_require.lua index 06629421e21..8a1a7f3f912 100644 --- a/code/def_files/data/scripts/cfile_require.lua +++ b/code/def_files/data/scripts/cfile_require.lua @@ -8,7 +8,7 @@ local function cfileLoader(name) local file = nil for i,v in ipairs(exts) do - local real_name = name .. v + local real_name = string.gsub(name, "%.", "/") .. v file = cf.openFile(real_name, "r", "data/scripts") if (file:isValid()) then diff --git a/code/graphics/generic.cpp b/code/graphics/generic.cpp index 078f8bf68d8..9e87ca37324 100644 --- a/code/graphics/generic.cpp +++ b/code/graphics/generic.cpp @@ -162,7 +162,7 @@ int generic_anim_stream(generic_anim *ga, const bool cache) ga->type = BM_TYPE_NONE; - auto res = cf_find_file_location_ext(ga->filename, BM_ANI_NUM_TYPES, bm_ani_ext_list, CF_TYPE_ANY, false); + auto res = cf_find_file_location_ext(ga->filename, BM_ANI_NUM_TYPES, bm_ani_ext_list, CF_TYPE_ANY); // could not be found, or is invalid for some reason if ( !res.found ) diff --git a/code/graphics/software/FontManager.cpp b/code/graphics/software/FontManager.cpp index a939b5c3953..80aa43f772a 100644 --- a/code/graphics/software/FontManager.cpp +++ b/code/graphics/software/FontManager.cpp @@ -10,6 +10,7 @@ #include "bmpman/bmpman.h" #include "cfile/cfile.h" +#include "localization/localize.h" namespace font { @@ -115,9 +116,20 @@ namespace font return data; } - bool localize = true; + // try localized version first + CFILE* fp = nullptr; + SCP_string typeface_lcl = typeface; + + lcl_add_dir_to_path_with_filename(typeface_lcl); + + fp = cfopen(typeface_lcl.c_str(), "rb", CFILE_NORMAL, CF_TYPE_ANY); + + // fallback if not found + if ( !fp ) + { + fp = cfopen(typeface.c_str(), "rb", CFILE_NORMAL, CF_TYPE_ANY); + } - CFILE* fp = cfopen(typeface.c_str(), "rb", CFILE_NORMAL, CF_TYPE_ANY, localize); if (fp == NULL) { mprintf(("Unable to find font file \"%s\"\n", typeface.c_str())); diff --git a/code/localization/localize.cpp b/code/localization/localize.cpp index c19f7b1d846..47bd6c9c4fe 100644 --- a/code/localization/localize.cpp +++ b/code/localization/localize.cpp @@ -644,6 +644,37 @@ int lcl_add_dir_to_path_with_filename(char *current_path, size_t path_max) return 1; } +int lcl_add_dir_to_path_with_filename(SCP_string ¤t_path) +{ + int lang = lcl_get_current_lang_index(); + + // if the disk extension is 0 length, don't add anything + if (strlen(Lcl_languages[lang].lang_ext) <= 0) { + return 1; + } + + SCP_string filename; + + auto sep = current_path.find_last_of(DIR_SEPARATOR_CHAR); + + if (sep == SCP_string::npos) { + filename = current_path; + current_path.clear(); + } else { + filename = current_path.substr(sep+1); + current_path.erase(sep+1); + } + + // add extension + current_path += Lcl_languages[lang].lang_ext; + current_path += DIR_SEPARATOR_STR; + + // copy rest of filename + current_path += filename; + + return 1; +} + // externalization of table/mission files ----------------------- diff --git a/code/localization/localize.h b/code/localization/localize.h index 4b207c699c0..01d7a652d49 100644 --- a/code/localization/localize.h +++ b/code/localization/localize.h @@ -103,6 +103,7 @@ ubyte lcl_get_font_index(int font_num); // maybe add localized directory to full path with file name when opening a localized file int lcl_add_dir_to_path_with_filename(char *current_path, size_t path_max); +int lcl_add_dir_to_path_with_filename(SCP_string ¤t_path); // Goober5000 void lcl_replace_stuff(char *text, size_t max_len, bool force = false); diff --git a/code/pilotfile/csg_convert.cpp b/code/pilotfile/csg_convert.cpp index 443962b694c..7be22b7d9ba 100644 --- a/code/pilotfile/csg_convert.cpp +++ b/code/pilotfile/csg_convert.cpp @@ -1166,7 +1166,7 @@ bool pilotfile_convert::csg_convert(const char *fname, bool inferno) filename.reserve(200); - cf_create_default_path_string(filename, CF_TYPE_SINGLE_PLAYERS, (inferno) ? "inferno" : nullptr, false, + cf_create_default_path_string(filename, CF_TYPE_SINGLE_PLAYERS, (inferno) ? "inferno" : nullptr, CF_LOCATION_ROOT_USER | CF_LOCATION_ROOT_GAME | CF_LOCATION_TYPE_ROOT); if (inferno) { diff --git a/code/pilotfile/plr_convert.cpp b/code/pilotfile/plr_convert.cpp index ef51a93e7da..fa43a52c9d7 100644 --- a/code/pilotfile/plr_convert.cpp +++ b/code/pilotfile/plr_convert.cpp @@ -863,7 +863,7 @@ bool pilotfile_convert::plr_convert(const char *fname, bool inferno) filename.reserve(200); - cf_create_default_path_string(filename, CF_TYPE_SINGLE_PLAYERS, (inferno) ? "inferno" : nullptr, false, + cf_create_default_path_string(filename, CF_TYPE_SINGLE_PLAYERS, (inferno) ? "inferno" : nullptr, CF_LOCATION_ROOT_USER | CF_LOCATION_ROOT_GAME | CF_LOCATION_TYPE_ROOT); if (inferno) { diff --git a/code/sound/ffmpeg/FFmpegWaveFile.cpp b/code/sound/ffmpeg/FFmpegWaveFile.cpp index 4941f3d5de2..4cade1bc6e9 100644 --- a/code/sound/ffmpeg/FFmpegWaveFile.cpp +++ b/code/sound/ffmpeg/FFmpegWaveFile.cpp @@ -208,7 +208,7 @@ bool FFmpegWaveFile::Open(const char* pszFilename, bool keep_ext) throw FFmpegException("Unknown file extension."); } - auto res = cf_find_file_location(pszFilename, CF_TYPE_ANY, false); + auto res = cf_find_file_location(pszFilename, CF_TYPE_ANY); if (!res.found) { throw FFmpegException("File not found."); @@ -217,7 +217,7 @@ bool FFmpegWaveFile::Open(const char* pszFilename, bool keep_ext) cfp = cfopen_special(res, "rb", CF_TYPE_ANY); } else { // ... otherwise we just find the best match - auto res = cf_find_file_location_ext(filename, NUM_AUDIO_EXT, audio_ext_list, CF_TYPE_ANY, false); + auto res = cf_find_file_location_ext(filename, NUM_AUDIO_EXT, audio_ext_list, CF_TYPE_ANY); if (!res.found) { throw FFmpegException("File not found with any known extension."); diff --git a/fred2/campaigneditordlg.cpp b/fred2/campaigneditordlg.cpp index cfa2ed4d964..ac5c4ed6af4 100644 --- a/fred2/campaigneditordlg.cpp +++ b/fred2/campaigneditordlg.cpp @@ -139,7 +139,7 @@ void campaign_editor::OnLoad() } } - auto res = cf_find_file_location(Campaign.missions[Cur_campaign_mission].name, CF_TYPE_MISSIONS, false); + auto res = cf_find_file_location(Campaign.missions[Cur_campaign_mission].name, CF_TYPE_MISSIONS); if (res.found) { FREDDoc_ptr->SetPathName(res.full_name.c_str()); diff --git a/test/src/cfile/cfile.cpp b/test/src/cfile/cfile.cpp index 1e11ae76650..3ba5006a601 100644 --- a/test/src/cfile/cfile.cpp +++ b/test/src/cfile/cfile.cpp @@ -141,14 +141,74 @@ TEST_F(CFileTest, test_get_path_type) ASSERT_EQ(CF_TYPE_DATA, cfile_get_path_type("////data\\\\\\")); // Test single subdirectory case - ASSERT_EQ(CF_TYPE_INTERFACE, cfile_get_path_type("data/interface")); - ASSERT_EQ(CF_TYPE_INTERFACE, cfile_get_path_type("///data/interface")); - ASSERT_EQ(CF_TYPE_INTERFACE, cfile_get_path_type("data/interface\\\\\\")); - ASSERT_EQ(CF_TYPE_INTERFACE, cfile_get_path_type("////data/interface\\\\\\")); + ASSERT_EQ(CF_TYPE_VOICE, cfile_get_path_type("data/voice")); + ASSERT_EQ(CF_TYPE_VOICE, cfile_get_path_type("///data/voice")); + ASSERT_EQ(CF_TYPE_VOICE, cfile_get_path_type("data/voice\\\\\\")); + ASSERT_EQ(CF_TYPE_VOICE, cfile_get_path_type("////data/voice\\\\\\")); // Test nested subdirectory case - ASSERT_EQ(CF_TYPE_INTERFACE_MARKUP, cfile_get_path_type("data/interface/markup")); - ASSERT_EQ(CF_TYPE_INTERFACE_MARKUP, cfile_get_path_type("///data/interface/markup")); - ASSERT_EQ(CF_TYPE_INTERFACE_MARKUP, cfile_get_path_type("data/interface/markup\\\\\\")); - ASSERT_EQ(CF_TYPE_INTERFACE_MARKUP, cfile_get_path_type("////data/interface/markup\\\\\\")); + ASSERT_EQ(CF_TYPE_VOICE_SPECIAL, cfile_get_path_type("data/voice/special")); + ASSERT_EQ(CF_TYPE_VOICE_SPECIAL, cfile_get_path_type("///data/voice/special")); + ASSERT_EQ(CF_TYPE_VOICE_SPECIAL, cfile_get_path_type("data/voice/special\\\\\\")); + ASSERT_EQ(CF_TYPE_VOICE_SPECIAL, cfile_get_path_type("////data/voice/special\\\\\\")); } + +TEST_F(CFileTest, subfolders) +{ + // base check for all files + ASSERT_TRUE(cf_exists("file.tbl", CF_TYPE_TABLES)); + ASSERT_TRUE(cf_exists("file2.tbl", CF_TYPE_TABLES)); + ASSERT_TRUE(cf_exists("file3.tbl", CF_TYPE_TABLES)); + + // good direct subfolder check + ASSERT_TRUE(cf_exists("sub/file2.tbl", CF_TYPE_TABLES)); + + // bad direct subfolder check + ASSERT_FALSE(cf_exists("sub/file3.tbl", CF_TYPE_TABLES)); + + // sub-subfolder check + ASSERT_TRUE(cf_exists("sub/folder/file3.tbl", CF_TYPE_TABLES)); + + // check cf_find_file_location() with and without subfolders + auto loc = cf_find_file_location("file2.tbl", CF_TYPE_TABLES); + ASSERT_TRUE(loc.found); + + loc = cf_find_file_location("sub/file2.tbl", CF_TYPE_ANY); + ASSERT_TRUE(loc.found); + + // check that cf_find_file_location_ext() works with subfolders + const char *exts[] = { ".tbl", ".cfg" }; + loc = cf_find_file_location_ext("sub/folder/file3.tbl", 2, exts, CF_TYPE_TABLES); + ASSERT_TRUE(loc.found); +} + +TEST_F(CFileTest, subfolder_list) +{ + SCP_vector table_files; + extern bool Skip_memory_files; + + // For this test we need to skip the memory files to keep the results consistent + Skip_memory_files = true; + + // look for all tables (one result, shadowing second file) + ASSERT_EQ(1, cf_get_file_list(table_files, CF_TYPE_TABLES, "*.tbl", CF_SORT_NAME)); + + // look for file in subfolder only, with unix and windows directory separators + table_files.clear(); + ASSERT_EQ(1, cf_get_file_list(table_files, CF_TYPE_TABLES, "folder/*.tbl", CF_SORT_NAME)); + ASSERT_TRUE(table_files.front().substr(0, 6) == "folder"); + + table_files.clear(); + ASSERT_EQ(1, cf_get_file_list(table_files, CF_TYPE_TABLES, "folder\\*.tbl", CF_SORT_NAME)); + ASSERT_TRUE(table_files.front().substr(0, 6) == "folder"); + + // subfolder glob, with unix and windows directory separators + // should return two files, one with a sub path + table_files.clear(); + ASSERT_EQ(2, cf_get_file_list(table_files, CF_TYPE_TABLES, "*/*.tbl", CF_SORT_NAME)); + ASSERT_TRUE(table_files.back().substr(0, 6) == "folder"); + + table_files.clear(); + ASSERT_EQ(2, cf_get_file_list(table_files, CF_TYPE_TABLES, "*\\*.tbl", CF_SORT_NAME)); + ASSERT_TRUE(table_files.back().substr(0, 6) == "folder"); +} \ No newline at end of file diff --git a/test/test_data/cfile/subfolder_list/data/tables/file.tbl b/test/test_data/cfile/subfolder_list/data/tables/file.tbl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/test_data/cfile/subfolder_list/data/tables/folder/file.tbl b/test/test_data/cfile/subfolder_list/data/tables/folder/file.tbl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/test_data/cfile/subfolders/data/tables/file.tbl b/test/test_data/cfile/subfolders/data/tables/file.tbl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/test_data/cfile/subfolders/data/tables/sub/file2.tbl b/test/test_data/cfile/subfolders/data/tables/sub/file2.tbl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/test_data/cfile/subfolders/data/tables/sub/folder/file3.tbl b/test/test_data/cfile/subfolders/data/tables/sub/folder/file3.tbl new file mode 100644 index 00000000000..e69de29bb2d