Utils.cpp

#include "Utils.h"
#include "UnicodeFileReader.h"
#include <fstream>
#include <iomanip>
#include <algorithm>
#include <fstream>
#include <locale>
#include <filesystem>
using namespace std; // Import entire std namespace.
namespace fs = std::filesystem; 
ofstream logFile;
ofstream traceFile;



vector<string> splitString(const string& str,int qoute_count, int& comment_count) {
	vector<string> result;
    string new_line="";			//Whatever is to the left of { or }
    string delimiter="";
	int char_count=-1;
	for (char c : str) {
        char_count++;
		if (c=='"' and comment_count%2==0) qoute_count++;
		if (qoute_count % 2 ==0){
			if (c=='/' and char_count < (int)str.size()-1 and comment_count%2==0) comment_count++;
			if (c=='/' and comment_count%2!=0 and char_count > 0) if (str[char_count-1]=='*'){
				comment_count--;
				continue;
			} 
			if (comment_count%2!=0) continue;	
			if ((c == '{') or (c == '}')) {		//if I find a { or } when I'm not between qoutes 	
				if (!new_line.empty()) result.push_back(new_line);
				delimiter=c;
				result.push_back(delimiter);										//and I push the { or } as another line	
				new_line.clear();
				continue;
			}
			new_line += c;										//clear new_line to continue with the following part of the string
		} else {
			if (c=='{' or c=='}')new_line += ' ';						//If I"m between qoutes and I find a { or } I have to add a space to make sure is not interpreted as a delimiter
			new_line += c;
		}
	}
	if (!new_line.empty()) result.push_back(new_line); 											//When I'm done, if there is somenthing left,is pushed.

	return result;
}

void deleteFilesInFolder(const string& folderName){
	if (!fs::exists(folderName)) return;
	for (const auto& entry : fs::directory_iterator(folderName)) {
    	if (entry.path().extension() == ".csv")
        	fs::remove(entry.path());
	}
}

string extractFileName(const string& fullPath) {
    /*size_t lastSlash = fullPath.find_last_of("/\\");
    if (lastSlash == string::npos) return fullPath;
    return fullPath.substr(lastSlash + 1);*/
    return fs::path(fullPath).filename().string();
    
}
string joinPath(const vector<string>& path) {
    string result;
    for (size_t i = 0;i<path.size(); i++) {
        result+= trim(path[i])+",";
    }
    result.pop_back();
    return result;
}


string trim(const string &str) {
    size_t first = str.find_first_not_of(" \t\r\n\0"); // Find first non-whitespace
    size_t last = str.find_last_not_of(" \t\r\n\0"); // Find last non-whitespace
    if (first == string::npos) {return "";} // String is all whitespace
	return str.substr(first, (last - first + 1));
}

string formatNumberWithSeparators(long long number) {
    ostringstream oss;
    oss.imbue(locale(oss.getloc(), new thousands_separator));
    oss << number;
    return oss.str();
}


string escapeCSV(const string& data) {
    if (data=="") return data;
	if (data.find(',') == string::npos && data.find('"') == string::npos) return data; // No commas or quotes, no escaping needed
    
    string escaped = "\"";
    for (char c : data) {
        if (c == '"') escaped += "\"\""; // Escape quotes by doubling
        else escaped += c;
    }
    escaped += "\"";
    return escaped;
}

std::vector<std::vector<std::string>> readCSVFile(string file_name){
    string line;
	string current;
	vector<vector<string>> result;
	vector<string> row;
	logMessage("EVENT","Reading configuration file: "+extractFileName(file_name));
	ifstream file(file_name);	
	if (!file.is_open()) return result;
		
	getline(file, line); //Skips the header line
	while (getline(file, line)) {
		row.clear();
		current="";
		for (char c : line){
			if (c==','){
				if (current=="") current=" ";
				row.push_back(current);
				current="";
			}
			else{
				current+=c;
			}
		}
		for (int i=row.size()-1;i>=0;i--){
			if (row[i]==" "){
				row.pop_back();
			}else{
				break;
			}
		}
		result.push_back(row);
	}
	file.close();
	return result;
}

map<string, TableConfig> loadTypeConfig(){
	map<string, TableConfig> config_info;
	string first_level_action;
	string first_level_table;
	string data_action;
	string data_table;
	vector<string> headers;
    string type;
    
	vector<vector<string>> csv_data =readCSVFile("Config/"+TYPE_CONFIG_FILE);

    int col_count;
    for (const auto& row : csv_data){
		col_count=0;
		for (const auto& cell : row){
			col_count++;
			switch (col_count){
			case 1:	type=cell; break;
			case 2:	config_info[type].header_count=stoi(cell); break;
			case 3:	config_info[type].first_level_action=cell; break;
			case 4:	config_info[type].first_level_table=cell; break;
			case 5:	config_info[type].data_action=cell; break;
			case 6:	config_info[type].data_table=cell; break;
			default: config_info[type].headers.push_back(cell);
			}
		}
	}
	return config_info;
}

map<string, TableConfig> loadTableConfig(){
	map<string, TableConfig> config_info;
	string first_level_action;
	string first_level_table;
	string data_action;
	string data_table;
	vector<string> headers;
    string type;
    
	vector<vector<string>> csv_data =readCSVFile("Config/"+TABLE_CONFIG_FILE);
    
    int col_count=0;
    for (const auto& row : csv_data){
		col_count=0;
		for (const auto& cell : row){
			col_count++;
			switch (col_count){
			case 1:	type=cell; break;
			default: config_info[type].headers.push_back(cell);
			}
		}
	}
	return config_info;
}


void updateConfig(const string& type,vector<string>& headers,int& header_count){
	bool file_exist=false;
	string text;
	
	//Update the Type_Config Table
	string file_path="Config/"+TYPE_CONFIG_FILE;
	ifstream file_check(file_path);
    if (file_check.good()) file_exist=true;
	file_check.close();
	ofstream file(file_path, ios::app | ios::binary); 
	if (!file_exist){
		file<<"_OBJ_TYPE,HEADER COUNT,FIRST LEVEL ACTION,FIRST LEVEL TABLE,DATA ACTION,DATA TABLE,Column1,Column2\n";	//If the file is not found, create the headers list of the config file
		logMessage("WARNING","Creating configuration file: "+TYPE_CONFIG_FILE);
	}
	file<<escapeCSV(type)<<","<<header_count<<",INDIVIDUAL,"<<escapeCSV(type)<<",INDIVIDUAL,"<<escapeCSV(type+"_data");
	for (const auto& header : headers){
		file<<","<<escapeCSV(header);
	}
	file<<"\n";
	file.close();
	
	//Update the Table Config Table
	file_exist=false;
	file_path="Config/"+TABLE_CONFIG_FILE;
	file_check.open(file_path);
    if (file_check.good()) file_exist=true;
	file_check.close();
	file.open(file_path, ios::app | ios::binary); 
	if (!file_exist){
		logMessage("WARNING","Creating configuration file: "+TABLE_CONFIG_FILE);
		file<<"TABLE,Column1,Column2,Column3\n";	//If the file is not found, create the headers list of the config file
	}
	file<<escapeCSV(type)<<",_OBJ_TYPE";								//First Level Table
	for (const auto& header : headers){
		file<<","<<escapeCSV(header);
	}
	file<<"\n";
	//Data Table
	
	text=escapeCSV(type+"_data")+",_OBJ_TYPE,";
	for (int i=0;i<3;i++){
		if (headers.size()>i) text+=escapeCSV(headers[i]);
		text+=",";
	}
	
	file<<text<<"ATTRIBUTE NAME,ATTRIBUTE VALUE,PARENT OBJECT 1,PARENT OBJECT 1 NAME,PARENT OBJECT 2,PARENT OBJECT 2 NAME,PARENT OBJECT 3,PARENT OBJECT 3 NAME,PARENT OBJECT 4,PARENT OBJECT 4 NAME,PARENT OBJECT 5,PARENT OBJECT 5 NAME\n";
	file.close();
}

string loadWordList(const string file_name){
	logMessage("EVENT","Reading configuration file: "+file_name);
	string loadWordList;
	string line;
	ifstream file("Config/"+file_name);
	if (!file.is_open()) return loadWordList;
    while (getline(file, line)) loadWordList+=" "+line;
    loadWordList+=" ";
	file.close();
    return loadWordList;
}

void createOutTable(const vector<string>& headers,string file_name,string folder_name){
	string text;
	string file_path=folder_name+file_name+".csv";
	ifstream file_check(file_path);
    if (file_check.good()) return;
    logMessage("EVENT","Creating data table: "+file_name);
	ofstream file(file_path, ios::app | ios::binary);   
	for (const auto& header : headers) text+=escapeCSV(header)+",";
	text.pop_back();
	file<<text<<"\n";
	file.close();
}

void createOutTables(const map<string, TableConfig>& table_config,const map<string, TableConfig>& type_config,const string& output_folder){
	//CREAR CARPETAS SI NO EXISTEN!
	fs::create_directories(output_folder);
	for (const auto& type : type_config){
		if (type.second.first_level_action!="SKIP"){
			if (table_config.find(type.second.first_level_table)==table_config.end()){
				logMessage("ERROR","Can't find table': "+type.second.first_level_table+" in TableConfig file");
				exit (300);
			}
			createOutTable(table_config.at(type.second.first_level_table).headers,type.second.first_level_table,output_folder);
		} 
		if (type.second.data_action!="SKIP"){
			if (table_config.find(type.second.data_table)==table_config.end()){
				logMessage("ERROR","Can't find table': "+type.second.data_table+" in TableConfig file");
				exit (300);
			}
			createOutTable(table_config.at(type.second.data_table).headers,type.second.data_table,output_folder);	
		} 
	} 
	
}

string getTimestamp() {
    auto now = time(nullptr);
    stringstream ss;
    ss << put_time(localtime(&now), "%Y-%m-%d %H:%M:%S");
    return ss.str();
}

void initLogFiles() {
    string timestamp = getTimestamp();

    replace(timestamp.begin(), timestamp.end(), ' ', '_');
    replace(timestamp.begin(), timestamp.end(), ':', '-');

    string logFilePath = "parser_events.log";//+ timestamp 
    string traceFilePath = "parser_trace.txt";

    logFile.open(logFilePath, ios::out);
    if (!logFile.is_open()) {
        cerr << "ERROR: Failed to open log file: " << logFilePath << endl;
    }
	logFile.imbue(locale(logFile.getloc(), new thousands_separator));
		
    traceFile.open(traceFilePath, ios::out);
    if (!traceFile.is_open()) {
        cerr << "ERROR: Failed to open trace file: " << traceFilePath << endl;
    }
	traceFile.imbue(locale(traceFile.getloc(), new thousands_separator));
	
    if (logFile.is_open()) {
        logFile << "TIME STAMP\tTYPE\tMESSAGE"<< endl;
        logFile.flush();
    }
}

// Function to log events, warnings, or errors
void logMessage(string severity,string message) {
    string entry = "[" + getTimestamp() + "]\t" + severity + "\t" + message;
    if (logFile.is_open()) {
        logFile << entry << endl;
        logFile.flush();
    }
    // Also output to console for debugging
    if (severity == "ERROR") cerr << entry << endl;
}

// Function to log a file line (trace)
void logTraceLine(long long lineNumber, const string& line,int level) {
    if (traceFile.is_open()) {
        traceFile << "Line " << lineNumber << "[" <<level<< "]: " << line << endl;
        traceFile.flush();
    }
}

long long countLines(const string& filename) {
	logMessage("EVENT","Counting lines");
	
	UnicodeFileReader file(filename);
    if (!file.is_open()) {
        logMessage("ERROR","Failed to open file: "+filename);
        return 100;
    }
    cout.imbue(locale(cout.getloc(), new thousands_separator));
	long long line_count = 0;
    string line;
	
    char frames[] = {'|', '/', '-', '\\'};
	int frame_count=0;
	
	while (file.readLine(line)) {
		line_count++;
		if (line_count % 20000==0){
			cout<<"\rCounting lines: "<<frames[frame_count]<<flush;
			frame_count++;
			if (frame_count>=sizeof(frames)) frame_count=0;
		}	
	}
    file.close();
	cout<<"\r";
	logMessage("EVENT","Completed line count: " + formatNumberWithSeparators(line_count));
	return line_count;
}

void updateProgress(long long current_line,long long total_lines, double update_rate,const chrono::steady_clock::time_point& start_time, chrono::steady_clock::time_point& last_update,bool silent_mode) {
	auto now = chrono::steady_clock::now();
    double elapsed_since_last = chrono::duration<double>(now - last_update).count(); 	
	if (elapsed_since_last < update_rate and total_lines-current_line>100) return;
	string eta_text;
	int barWidth = 40;
    double progress = static_cast<double>(current_line) / total_lines;
    int pos = barWidth * progress;
    double elapsed = chrono::duration<double>(now - start_time).count();
    double speed = current_line / elapsed;
    double remaining = total_lines - current_line;
    double eta = remaining / speed;    
    int eta_min = eta / 60;
    int eta_sec = (int)eta % 60;
	eta_text=to_string(eta_min)+"m "+to_string(eta_sec)+"s";
    if (progress<0.02) eta_text="?m ??s";
    if (silent_mode){
		cout<<(int)(progress * 100.0)<<"pct\r ETA:"<<eta_text<<" Line: "<<current_line<<" out of "<<total_lines<<"\r";
		cout.flush();
	}else{
		cout <<"Processing file: " << "ETA:"<<eta_text<<" Line: "<<current_line<<" out of "<<total_lines;
    	cout << " [";
    	for (int i = 0; i < barWidth; ++i) {
        	if (i < pos) cout << "=";
        	else if (i == pos) cout << ">";
        	else cout << " ";
    	}
    	cout << "] " << fixed << setprecision(1) << (progress * 100.0) << " %  \r";
		cout<<flush;
	}
    

    last_update = now;
}

chrono::steady_clock::time_point printCurrentTime(chrono::steady_clock::time_point start_time) {
	auto now = time(nullptr);
	if (start_time == chrono::steady_clock::time_point{}){
		cout << put_time(localtime(&now), "%H:%M:%S")<<"\n";
		return chrono::steady_clock::now();
	}else{
		auto duration = chrono::duration_cast<chrono::seconds>(chrono::steady_clock::now()-start_time);
		long long seconds =  duration.count();
    	long long minutes = seconds / 60;
    	seconds %= 60;
		cout << put_time(localtime(&now), "%H:%M:%S")<<"\n";
		logMessage("EVENT","Total time: "+ to_string(minutes) + " min " + to_string(seconds) + " sec");
		cout<<"Total Time: "<< minutes << " min " << seconds << " sec\n";
		return chrono::steady_clock::now();
	}
}