diff --git a/Project.toml b/Project.toml index 0406988d27..f38aee85eb 100644 --- a/Project.toml +++ b/Project.toml @@ -11,7 +11,9 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Distances = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" +DuckDB = "d2f5444f-75bc-4fdf-ac35-56f514c445e1" HiGHS = "87dc4568-4c63-4d18-b0c0-bb2238e4078b" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" JuMP = "4076af6c-e467-56ae-b986-b466b2749572" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" diff --git a/docs/src/User_Guide/model_configuration.md b/docs/src/User_Guide/model_configuration.md index 2c8ccdcded..d5ef271cea 100644 --- a/docs/src/User_Guide/model_configuration.md +++ b/docs/src/User_Guide/model_configuration.md @@ -123,6 +123,21 @@ The following tables summarize the model settings parameters and their default/p |OverwriteResults | Flag for overwriting the output results from the previous run.| ||1 = overwrite the results.| ||0 = do not overwrite the results.| +|ResultsFileType | File type to save the results files.| +||Default `auto_detect` = Detect the extension from the name of the results file. In the abscence of an extension `.csv` will be used| +||`.csv` = Save as uncompressed CSV.| +||`.csv.gz` = Save as compressed CSV.| +||`.json` = Save as uncompressed JSON.| +||`.json.csv` = Save as compressed JSON.| +||`.parquet` = Save as uncompressed parquet.| +||`-snappy.parquet` = Save as snappy parquet.| +||`-zstd.parquet` = Save as zstd parquet.| +|ResultsCompressionType | Compression type to save the results files.| +||Default `auto_detect` = Detect the compression from the name of the results file. In the abscence of a compression type, none will be used| +||`gzip` = GZIP compression for CSV or JSON files| +|| `snappy` = snappy compression for parquet files| +|| `zstd` = zstd compression for parquet files| +|| `none` = no compression| ## 6. Solver related diff --git a/docs/src/User_Guide/model_input.md b/docs/src/User_Guide/model_input.md index dc28ae19fe..b796893eac 100644 --- a/docs/src/User_Guide/model_input.md +++ b/docs/src/User_Guide/model_input.md @@ -1,11 +1,14 @@ # GenX Inputs -All input files are in CSV format. Running the GenX model requires a minimum of four **mandatory input files** and one folder, which consists of CSV files for generating resources: +!!! note "Note" + As of GenX version 0.4.2, all input and results files can be of filetypes CSV, JSON, or parquet, and can also have gzip, snappy, or zstd compression. Earlier versions of GenX require all files to be of format CSV. + +Running the GenX model requires a minimum of four **mandatory input files** and one folder, which consists of input files for generating resources. The following are the default names of the files, but users can change the names of the input files by including an input_settings.yml file in the `settings` folder. ```@raw html
    -
  1. Fuels_data.csv: specify fuel type, CO2 emissions intensity, and time-series of fuel prices.
    2. -
  2. Network.csv: specify network topology, transmission fixed costs, capacity and loss parameters.
    3. +
  3. Fuels_data: specify fuel type, CO2 emissions intensity, and time-series of fuel prices.
    4. +
  4. Network: specify network topology, transmission fixed costs, capacity and loss parameters.
``` @@ -14,34 +17,30 @@ All input files are in CSV format. Running the GenX model requires a minimum of ```@raw html
    -
  1. Demand_data.csv: specify time-series of demand profiles for each model zone, weights for each time step, demand shedding costs, and optional time domain reduction parameters.
    2. -
  2. Generators_variability.csv: specify time-series of capacity factor/availability for each resource.
    3. +
  3. Demand_data: specify time-series of demand profiles for each model zone, weights for each time step, demand shedding costs, and optional time domain reduction parameters.
    4. +
  4. Generators_variability: specify time-series of capacity factor/availability for each resource.
  5. Resources folder: specify cost and performance data for generation, storage and demand flexibility resources.
``` Additionally, the user may need to specify eight more **settings-specific** input files based on model configuration and type of scenarios of interest: -1. Operational\_reserves.csv: specify operational reserve requirements as a function of demand and renewables generation and penalty for not meeting these requirements. -2. Energy\_share\_requirement.csv: specify regional renewable portfolio standard and clean energy standard style policies requiring minimum energy generation from qualifying resources. +1. Operational\_reserves: specify operational reserve requirements as a function of demand and renewables generation and penalty for not meeting these requirements. +2. Energy\_share\_requirement: specify regional renewable portfolio standard and clean energy standard style policies requiring minimum energy generation from qualifying resources. 3. CO2\_cap.csv: specify regional CO2 emission limits. -4. Capacity\_reserve\_margin.csv: specify regional capacity reserve margin requirements. -5. Minimum\_capacity\_requirement.csv: specify regional minimum technology capacity deployment requirements. -6. Vre\_and\_stor\_data.csv: specify cost and performance data for co-located VRE and storage resources. -7. Vre\_and\_stor\_solar\_variability.csv: specify time-series of capacity factor/availability for each solar PV resource that exists for every co-located VRE and storage resource (in DC terms). -8. Vre\_and\_stor\_wind\_variability.csv: specify time-series of capacity factor/availability for each wind resource that exists for every co-located VRE and storage resource (in AC terms). -9. Hydrogen\_demand.csv: specify regional hydrogen production requirements. - - -!!! note "Note" - Names of the input files are case sensitive. +4. Capacity\_reserve\_margin: specify regional capacity reserve margin requirements. +5. Minimum\_capacity\_requirement: specify regional minimum technology capacity deployment requirements. +6. Vre\_and\_stor\_data: specify cost and performance data for co-located VRE and storage resources. +7. Vre\_and\_stor\_solar\_variability: specify time-series of capacity factor/availability for each solar PV resource that exists for every co-located VRE and storage resource (in DC terms). +8. Vre\_and\_stor\_wind\_variability: specify time-series of capacity factor/availability for each wind resource that exists for every co-located VRE and storage resource (in AC terms). +9. Hydrogen\_demand: specify regional hydrogen production requirements. ## 1 Mandatory input data -### 1.1 Fuels\_data.csv +### 1.1 Fuels\_data -• **First row:** names of all fuels used in the model instance which should match the labels used in `Fuel` column in one of the resource `.csv` file in the `resources` folder. For renewable resources or other resources that do not consume a fuel, the name of the fuel is `None`. +• **First row:** names of all fuels used in the model instance which should match the labels used in `Fuel` column in one of the resource file in the `resources` folder. For renewable resources or other resources that do not consume a fuel, the name of the fuel is `None`. • **Second row:** The second row specifies the CO2 emissions intensity of each fuel in tons/MMBtu (million British thermal units). Note that by convention, tons correspond to metric tonnes and not short tons (although as long as the user is internally consistent in their application of units, either can be used). @@ -50,14 +49,14 @@ Additionally, the user may need to specify eight more **settings-specific** inpu * ** First column:** The first column in this file denotes, Time\_index, represents the index of time steps in a model instance. -### 1.2 Network.csv +### 1.2 Network !!! note "Note" If running a single-zone model, this file is not mandatory. This input file contains input parameters related to: 1) definition of model zones (regions between which transmission flows are explicitly modeled) and 2) definition of transmission network topology, existing capacity, losses and reinforcement costs. The following table describe each of the mandatory parameter inputs need to be specified to run an instance of the model, along with comments for the model configurations when they are needed. -###### Table 3: Structure of the Network.csv file +###### Table 3: Structure of the Network file --- |**Column Name** | **Description**| | :------------ | :-----------| @@ -109,11 +108,11 @@ Note that in either case, positive flows indicate flow from start to end zone; negative flows indicate flow from end to start zone. -### 1.3 Demand\_data.csv (Load\_data.csv) +### 1.3 Demand\_data (Load\_data.csv) This file includes parameters to characterize model temporal resolution to approximate annual grid operations, electricity demand for each time step for each zone, and cost of load shedding. Note that GenX is designed to model hourly time steps. With some care and effort, finer (e.g. 15 minute) or courser (e.g. 2 hour) time steps can be modeled so long as all time-related parameters are scaled appropriately (e.g. time period weights, heat rates, ramp rates and minimum up and down times for generators, variable costs, etc). -###### Table 4: Structure of the Demand\_data.csv file +###### Table 4: Structure of the Demand\_data file --- |**Column Name** | **Description**| | :------------ | :-----------| @@ -131,14 +130,14 @@ This file includes parameters to characterize model temporal resolution to appro ### 1.4 Resources input files The `resources` folder contains the input files for each resource type. At the current version of GenX, the following resources are included in the model: -1) thermal generators, specified in the `Thermal.csv` file, -2) variable renewable energy resources (VRE), specified in the `VRE.csv` file, -3) reservoir hydro resources, specified in the `Hydro.csv` file, -4) storage resources, specified in the `Storage.csv` file, -5) flexible demand resources, specified in the `Flex_demand.csv` file, -6) must-run resources, specified in the `Must_run.csv` file, -7) electrolyzers, specified in the `Electrolyzer.csv` file, and -8) co-located VRE and storage resources, specified in the `Vre_stor.csv` file. +1) thermal generators, specified in the `Thermal` file, +2) variable renewable energy resources (VRE), specified in the `VRE` file, +3) reservoir hydro resources, specified in the `Hydro` file, +4) storage resources, specified in the `Storage` file, +5) flexible demand resources, specified in the `Flex_demand` file, +6) must-run resources, specified in the `Must_run` file, +7) electrolyzers, specified in the `Electrolyzer` file, and +8) co-located VRE and storage resources, specified in the `Vre_stor` file. Each file contains cost and performance parameters for various generators and other resources included in the model formulation. The following table describes the mandatory columns in each of these files. Note that the column names are case insensitive. @@ -166,7 +165,7 @@ Each file contains cost and performance parameters for various generators and ot |Var\_OM\_Cost\_per\_MWh | Variable operations and maintenance cost of a technology ($/MWh). Note that for co-located VRE-STOR resources, these costs apply to the AC generation sent to the grid from the entire site. | |**Technical performance parameters**| |Heat\_Rate\_MMBTU\_per\_MWh |Heat rate of a generator or MMBtu of fuel consumed per MWh of electricity generated for export (net of on-site consumption). The heat rate is the inverse of the efficiency: a lower heat rate is better. Should be consistent with fuel prices in terms of reporting on higher heating value (HHV) or lower heating value (LHV) basis. | -|Fuel |Fuel needed for a generator. The names should match with the ones in the `Fuels_data.csv`. | +|Fuel |Fuel needed for a generator. The names should match with the ones in the `Fuels_data`. | |**Required for writing outputs**| |region | Name of the model region| |cluster | Number of the cluster when representing multiple clusters of a given technology in a given region. | @@ -182,7 +181,7 @@ Each file contains cost and performance parameters for various generators and ot |Retrofit\_Id | Unique identifier to group retrofittable source technologies with retrofit options inside the same zone.| |Retrofit\_Efficiency | [0,1], Efficiency of the retrofit technology.| -##### Table 5b: Settings-specific columns in all resource .csv file +##### Table 5b: Settings-specific columns in all resource file --- |**Column Name** | **Description**| | :------------ | :-----------| @@ -204,7 +203,7 @@ Each file contains cost and performance parameters for various generators and ot ||Biomass = 1: Uses biomass as fuel.| |CCS\_Disposal\_Cost\_per\_Metric_Ton | Cost associated with CCS disposal ($/tCO2), including pipeline, injection and storage costs of CCS-equipped generators.| -##### Table 6a: Additional columns in the Thermal.csv file +##### Table 6a: Additional columns in the Thermal file |**Column Name** | **Description**| | :------------ | :-----------| |Model | {1, 2}, Flag to indicate membership in set of thermal resources (e.g. nuclear, combined heat and power, natural gas combined cycle, coal power plant)| @@ -236,7 +235,7 @@ Each file contains cost and performance parameters for various generators and ot |Fuel2\_Max\_Cofire\_Level |The maximum blendng level of 'Fuel2' in total heat inputs of a mulit-fuel generator (MULTI_FUELS = 1) during the normal generation process. | |Fuel2\_Max\_Cofire_Level\_Start |The maximum blendng level of 'Fuel2' in total heat inputs of a mulit-fuel generator (MULTI_FUELS = 1) during the start-up process. | -##### Table 6b: Settings-specific columns in the Thermal.csv file +##### Table 6b: Settings-specific columns in the Thermal file --- |**Column Name** | **Description**| | :------------ | :-----------| @@ -251,7 +250,7 @@ Each file contains cost and performance parameters for various generators and ot |Reg\_Max |[0,1], Fraction of nameplate capacity that can committed to provided regulation reserves. .| |Rsv\_Max |[0,1], Fraction of nameplate capacity that can committed to provided upwards spinning or contingency reserves.| -##### Table 7a: Additional columns in the Vre.csv file +##### Table 7a: Additional columns in the Vre file --- |**Column Name** | **Description**| | :------------ | :-----------| @@ -259,7 +258,7 @@ Each file contains cost and performance parameters for various generators and ot ||Num\_VRE\_bins = 1: using a single resource availability profile per technology per zone. 1 capacity investment decision variable and 1 generator RID tracking technology power output (and in each zone).| ||Num\_VRE\_bins > 1: using multiple resource availability profiles per technology per zone. Num\_VRE\_bins capacity investment decision variables and 1 generator RID used to define technology power output at each time step (and in each zone). Example: Suppose we are modeling 3 bins of wind profiles for each zone. Then include 3 rows with wind resource names as Wind\_1, Wind\_2, and Wind\_3 and a corresponding increasing sequence of RIDs. Set Num\_VRE\_bins for the generator with smallest RID, Wind\_1, to be 3 and set Num\_VRE\_bins for the other rows corresponding to Wind\_2 and Wind\_3, to be zero. By setting Num\_VRE\_bins for Wind\_2 and Wind\_3, the model eliminates the power outputs variables for these generators. The power output from the technology across all bins is reported in the power output variable for the first generator. This allows for multiple bins without significantly increasing number of model variables (adding each bin only adds one new capacity variable and no operational variables). See documentation for `curtailable_variable_renewable()` for more. | -##### Table 6b: Settings-specific columns in the Vre.csv file +##### Table 6b: Settings-specific columns in the Vre file --- |**Column Name** | **Description**| | :------------ | :-----------| @@ -269,11 +268,11 @@ Each file contains cost and performance parameters for various generators and ot |Reg\_Max |[0,1], Fraction of nameplate capacity that can committed to provided regulation reserves. .| |Rsv\_Max |[0,1], Fraction of nameplate capacity that can committed to provided upwards spinning or contingency reserves.| -##### Table 7a: Additional columns in the Hydro.csv file +##### Table 7a: Additional columns in the Hydro file --- |**Column Name** | **Description**| | :------------ | :-----------| -|Min\_Power |[0,1], The minimum generation level for a unit as a fraction of total capacity. This value cannot be higher than the smallest time-dependent CF value for a resource in `Generators_variability.csv`.| +|Min\_Power |[0,1], The minimum generation level for a unit as a fraction of total capacity. This value cannot be higher than the smallest time-dependent CF value for a resource in `Generators_variability`.| |Ramp\_Up\_Percentage |[0,1], Maximum increase in power output from between two periods (typically hours), reported as a fraction of nameplate capacity.| |Ramp\_Dn\_Percentage |[0,1], Maximum decrease in power output from between two periods (typically hours), reported as a fraction of nameplate capacity.| |Hydro\_Energy\_to\_Power\_Ratio |The rated number of hours of reservoir hydro storage at peak discharge power output. (hours). | @@ -281,7 +280,7 @@ Each file contains cost and performance parameters for various generators and ot ||LDS = 0: Not part of set (default) | ||LDS = 1: Long duration storage resources| -##### Table 7b: Settings-specific columns in the Hydro.csv file +##### Table 7b: Settings-specific columns in the Hydro file --- |**Column Name** | **Description**| | :------------ | :-----------| @@ -291,7 +290,7 @@ Each file contains cost and performance parameters for various generators and ot |Reg\_Max |[0,1], Fraction of nameplate capacity that can committed to provided regulation reserves. .| |Rsv\_Max |[0,1], Fraction of nameplate capacity that can committed to provided upwards spinning or contingency reserves.| -##### Table 8a: Additional columns in the Storage.csv file +##### Table 8a: Additional columns in the Storage file --- |**Column Name** | **Description**| | :------------ | :-----------| @@ -322,7 +321,7 @@ Each file contains cost and performance parameters for various generators and ot |Fixed\_OM\_Cost\_Charge\_per\_MWyr | Fixed operations and maintenance cost of the charging component of a storage technology of type `Model = 2`. | |Var\_OM\_Cost\_per\_MWhIn | Variable operations and maintenance cost of the charging aspect of a storage technology with `Model = 2`. Otherwise 0 ($/MWh).| -##### Table 8b: Settings-specific columns in the Storage.csv file +##### Table 8b: Settings-specific columns in the Storage file --- |**Column Name** | **Description**| | :------------ | :-----------| @@ -332,7 +331,7 @@ Each file contains cost and performance parameters for various generators and ot |Reg\_Max |[0,1], Fraction of nameplate capacity that can committed to provided regulation reserves. .| |Rsv\_Max |[0,1], Fraction of nameplate capacity that can committed to provided upwards spinning or contingency reserves.| -##### Table 9: Additional columns in the Flex_demand.csv file +##### Table 9: Additional columns in the Flex_demand file --- |**Column Name** | **Description**| | :------------ | :-----------| @@ -342,21 +341,21 @@ Each file contains cost and performance parameters for various generators and ot |**Cost parameters**| |Var\_OM\_Cost\_per\_MWhIn | Variable operations and maintenance costs associated with flexible demand deferral. Otherwise 0 ($/MWh). | -##### Table 10: Additional columns in the Electrolyzer.csv file +##### Table 10: Additional columns in the Electrolyzer file --- |**Column Name** | **Description**| | :------------ | :-----------| |Hydrogen\_MWh\_Per\_Tonne| Electrolyzer efficiency in megawatt-hours (MWh) of electricity per metric tonne of hydrogen produced (MWh/t)| |Electrolyzer\_Min\_kt| Minimum annual quantity of hydrogen that must be produced by electrolyzer in kilotonnes (kt)| |Hydrogen\_Price\_Per\_Tonne| Price (or value) of hydrogen per metric tonne ($/t)| -|Min\_Power |[0,1], The minimum generation level for a unit as a fraction of total capacity. This value cannot be higher than the smallest time-dependent CF value for a resource in `Generators_variability.csv`.| +|Min\_Power |[0,1], The minimum generation level for a unit as a fraction of total capacity. This value cannot be higher than the smallest time-dependent CF value for a resource in `Generators_variability`.| |Ramp\_Up\_Percentage |[0,1], Maximum increase in power output from between two periods (typically hours), reported as a fraction of nameplate capacity.| |Ramp\_Dn\_Percentage |[0,1], Maximum decrease in power output from between two periods (typically hours), reported as a fraction of nameplate capacity.| !!! note Check `Qualified_Hydrogen_Supply` column in table 5a if electrolyzers are included in the model. This column is used to indicate which resources are eligible to supply electrolyzers in the same zone (used for hourly clean supply constraint). Each co-located VRE, electrolyzer, and storage resource can be easily configured to contain either a co-located VRE-ELEC-storage resource, standalone VRE resource (either wind, solar PV, or both), standalone eletrolyzers, or standalone storage resource. -##### Table 11a: Additional columns in the Vre_stor.csv file +##### Table 11a: Additional columns in the Vre_stor file --- |**Column Name** | **Description**| | :------------ | :-----------| @@ -460,10 +459,10 @@ Each co-located VRE, electrolyzer, and storage resource can be easily configured |Eff\_Down\_AC |[0,1], Efficiency of AC discharging storage – applies to storage technologies (all STOR types). | |Ramp\_Up\_Percentage\_Elec |[0,1], Maximum increase in power output from between two periods (typically hours), reported as a fraction of nameplate capacity.| |Ramp\_Dn\_Percentage\_Elec |[0,1], Maximum decrease in power output from between two periods (typically hours), reported as a fraction of nameplate capacity.| -|Min\_Power\_Elec |[0,1], The minimum generation level for a unit as a fraction of total capacity. This value cannot be higher than the smallest time-dependent CF value for a resource in `Generators_variability.csv`.| +|Min\_Power\_Elec |[0,1], The minimum generation level for a unit as a fraction of total capacity. This value cannot be higher than the smallest time-dependent CF value for a resource in `Generators_variability`.| |Hydrogen\_MWh\_Per\_Tonne\_Elec| Electrolyzer efficiency in megawatt-hours (MWh) of electricity per metric tonne of hydrogen produced (MWh/t)| |Hydrogen\_Price\_Per\_Tonne\_Elec| Price (or value) of hydrogen per metric tonne ($/t)| -##### Table 11b: Settings-specific columns in the Vre_stor.csv file +##### Table 11b: Settings-specific columns in the Vre_stor file --- |**Column Name** | **Description**| | :------------ | :-----------| @@ -476,12 +475,12 @@ Each co-located VRE, electrolyzer, and storage resource can be easily configured ##### Policy-related columns for all resources In addition to the files described above, the `resources` folder contains a folder called `policy_assignments` (the filename can be changed in the settings file) with the following files that are used to specify policy-related parameters for specific resources: -1) `Resource_energy_share_requirement.csv` -2) `Resource_minimum_capacity_requirement.csv` -3) `Resource_maximum_capacity_requirement.csv` -4) `Resource_capacity_reserve_margin.csv` -5) `Resource_hydrogen_demand.csv` -6) `Resource_hourly_matching.csv` +1) `Resource_energy_share_requirement` +2) `Resource_minimum_capacity_requirement` +3) `Resource_maximum_capacity_requirement` +4) `Resource_capacity_reserve_margin` +5) `Resource_hydrogen_demand` +6) `Resource_hourly_matching` !!! note These files are optional and can be omitted if no policy-related settings are specified in the `genx_settings.yml` file. Also, not all the resources need to be included in these files, only those for which the policy applies. @@ -491,9 +490,9 @@ The following table describes the columns in each of these four files. !!! warning The first column of each file must contain the resource name corresponding to a resource in one of the resource data files described above. Note that the order of resources in the policy files is not important. -This policy is applied when if `EnergyShareRequirement > 0` in the settings file. \* corresponds to the ith row of the file `Energy_share_requirement.csv`. +This policy is applied when if `EnergyShareRequirement > 0` in the settings file. \* corresponds to the ith row of the file `Energy_share_requirement`. -##### Table 12: Energy share requirement policy parameters in Resource\_energy\_share\_requirement.csv +##### Table 12: Energy share requirement policy parameters in Resource\_energy\_share\_requirement --- |**Column Name** | **Description**| | :------------ | :-----------| @@ -508,7 +507,7 @@ This policy is applied when if `EnergyShareRequirement > 0` in the settings file This policy is applied when if `MinCapReq = 1` in the settings file. \* corresponds to the ith row of the file `Minimum_capacity_requirement.csv`. -##### Table 13: Minimum capacity requirement policy parameters in Resource\_minimum\_capacity\_requirement.csv +##### Table 13: Minimum capacity requirement policy parameters in Resource\_minimum\_capacity\_requirement --- |**Column Name** | **Description**| | :------------ | :-----------| @@ -521,7 +520,7 @@ This policy is applied when if `MinCapReq = 1` in the settings file. \* correspo This policy is applied when if `MaxCapReq = 1` in the settings file. \* corresponds to the ith row of the file `Maximum_capacity_requirement.csv`. -##### Table 14: Maximum capacity requirement policy parameters in Resource\_maximum\_capacity\_requirement.csv +##### Table 14: Maximum capacity requirement policy parameters in Resource\_maximum\_capacity\_requirement --- |**Column Name** | **Description**| | :------------ | :-----------| @@ -532,18 +531,18 @@ This policy is applied when if `MaxCapReq = 1` in the settings file. \* correspo |Max\_Cap\_Wind\_*| Eligibility of resources with a wind component to participate in Maximum Technology Carveout constraint (AC terms). |Max\_Cap\_Stor\_*| Eligibility of resources with a storage component to participate in Maximum Technology Carveout constraint (discharge capacity in AC terms).| -This policy is applied when if `CapacityReserveMargin > 0` in the settings file. \* corresponds to the ith row of the file `Capacity_reserve_margin.csv`. +This policy is applied when if `CapacityReserveMargin > 0` in the settings file. \* corresponds to the ith row of the file `Capacity_reserve_margin`. -##### Table 15: Capacity reserve margin policy parameters in Resource\_capacity\_reserve\_margin.csv +##### Table 15: Capacity reserve margin policy parameters in Resource\_capacity\_reserve\_margin --- |**Column Name** | **Description**| | :------------ | :-----------| |Resource| Resource name corresponding to a resource in one of the resource data files described above.| |Derating\_Factor\_*| Fraction of the resource capacity eligible for contributing to the capacity reserve margin constraint (e.g. derate factor).| -This policy is applied when if `HydrogenMinimumProduction = 1` in the settings file. \* corresponds to the ith row of the file `Hydrogen_demand.csv`. +This policy is applied when if `HydrogenMinimumProduction = 1` in the settings file. \* corresponds to the ith row of the file `Hydrogen_demand`. -##### Table 16: Hydrogen demand policy parameters in Resource\_hydrogen\_demand.csv +##### Table 16: Hydrogen demand policy parameters in Resource\_hydrogen\_demand --- |**Column Name** | **Description**| | :------------ | :-----------| @@ -552,7 +551,7 @@ This policy is applied when if `HydrogenMinimumProduction = 1` in the settings f This policy is applied when if `HourlyMatching = 1` in the settings file. -##### Table 17: Hourly matching policy parameters in Resource\_hourly\_matching.csv +##### Table 17: Hourly matching policy parameters in Resource\_hourly\_matching --- |**Column Name** | **Description**| | :------------ | :-----------| @@ -562,7 +561,7 @@ This policy is applied when if `HourlyMatching = 1` in the settings file. ##### Additional module-related columns for all resources In addition to the files described above, the `resources` folder can contain additional files that are used to specify attributes for specific resources and modules. Currently, the following files are supported: -`Resource_multistage_data.csv`: mandatory if `MultiStage = 1` in the settings file +`Resource_multistage_data`: mandatory if `MultiStage = 1` in the settings file !!! warning The first column of each additional module file must contain the resource name corresponding to a resource in one of the resource data files described above. Note that the order of resources in these files is not important. @@ -606,20 +605,20 @@ In addition to the files described above, the `resources` folder can contain add | WACC\_Discharge\_AC | The line-specific weighted average cost of capital for the discharging AC storage component with `STOR_AC_DISCHARGE = 2`. | | WACC\_Charge\_AC | The line-specific weighted average cost of capital for the charging AC storage component with `STOR_AC_CHARGE = 2`. | -#### 1.5 Generator\_variability.csv +#### 1.5 Generators\_variability -This file contains the time-series of capacity factors / availability of each resource included in the resource `.csv` file in the `resources` folder for each time step (e.g. hour) modeled. +This file contains the time-series of capacity factors / availability of each resource included in the resource file in the `resources` folder for each time step (e.g. hour) modeled. 1) First column: The first column contains the time index of each row (starting in the second row) from 1 to N. -2) Second column onwards: Resources are listed from the second column onward with headers matching each resource name in the resource `.csv` file in the `resources` folder in any order. The availability for each resource at each time step is defined as a fraction of installed capacity and should be between 0 and 1. Note that for this reason, resource names specified in the resource `.csv` file must be unique. Note that for Hydro reservoir resources (i.e. `Hydro.csv`), values in this file correspond to inflows (in MWhs) to the hydro reservoir as a fraction of installed power capacity, rather than hourly capacity factor. Note that for co-located VRE and storage resources, solar PV and wind resource profiles should not be located in this file but rather in separate variability files (these variabilities can be in the `Generators_variability.csv` if time domain reduction functionalities will be utilized because the time domain reduction functionalities will separate the files after the clustering is completed). +2) Second column onwards: Resources are listed from the second column onward with headers matching each resource name in the resource file in the `resources` folder in any order. The availability for each resource at each time step is defined as a fraction of installed capacity and should be between 0 and 1. Note that for this reason, resource names specified in the resource file must be unique. Note that for Hydro reservoir resources (i.e. `Hydro`), values in this file correspond to inflows (in MWhs) to the hydro reservoir as a fraction of installed power capacity, rather than hourly capacity factor. Note that for co-located VRE and storage resources, solar PV and wind resource profiles should not be located in this file but rather in separate variability files (these variabilities can be in the `Generators_variability` if time domain reduction functionalities will be utilized because the time domain reduction functionalities will separate the files after the clustering is completed). -###### Table 19: Structure of the Generator\_variability.csv file +###### Table 19: Structure of the Generators\_variability file --- |**Column Name** | **Description**| | :------------ | :-----------| |Resource| Resource name corresponding to a resource in one of the resource data files described above.| |Self\_Disch |[0,1], The power loss of storage technologies per hour (fraction loss per hour)- only applies to storage techs. Note that for co-located VRE-STOR resources, this value applies to the storage component of each resource.| -|Min\_Power |[0,1], The minimum generation level for a unit as a fraction of total capacity. This value cannot be higher than the smallest time-dependent CF value for a resource in `Generators_variability.csv`. Applies to thermal plants, and reservoir hydro resource (`HYDRO = 1`).| +|Min\_Power |[0,1], The minimum generation level for a unit as a fraction of total capacity. This value cannot be higher than the smallest time-dependent CF value for a resource in `Generators_variability`. Applies to thermal plants, and reservoir hydro resource (`HYDRO = 1`).| |Ramp\_Up\_Percentage |[0,1], Maximum increase in power output from between two periods (typically hours), reported as a fraction of nameplate capacity. Applies to thermal plants, and reservoir hydro resource (`HYDRO = 1`).| |Ramp\_Dn\_Percentage |[0,1], Maximum decrease in power output from between two periods (typically hours), reported as a fraction of nameplate capacity. Applies to thermal plants, and reservoir hydro resource (`HYDRO = 1`).| |Eff\_Up |[0,1], Efficiency of charging storage – applies to storage technologies (all STOR types except co-located storage resources).| @@ -630,29 +629,29 @@ This file contains the time-series of capacity factors / availability of each re |Max\_Flexible\_Demand\_Advance |Maximum number of hours that demand can be scheduled in advance of the original schedule. Applies to resources with FLEX type 1 (hours). | |Flexible\_Demand\_Energy\_Eff |[0,1], Energy efficiency associated with time shifting demand. Represents energy losses due to time shifting (or 'snap back' effect of higher consumption due to delay in use) that may apply to some forms of flexible demand. Applies to resources with FLEX type 1 (hours). For example, one may need to pre-cool a building more than normal to advance demand. | -#### 1.6 Vre\_and\_stor\_solar\_variability.csv +#### 1.6 Vre\_and\_stor\_solar\_variability -This file contains the time-series of capacity factors / availability of the solar PV component (DC capacity factors) of each co-located resource included in the `Vre_and_stor_data.csv` file for each time step (e.g. hour) modeled. +This file contains the time-series of capacity factors / availability of the solar PV component (DC capacity factors) of each co-located resource included in the `Vre_and_stor_data` file for each time step (e.g. hour) modeled. • first column: The first column contains the time index of each row (starting in the second row) from 1 to N. -• Second column onwards: Resources are listed from the second column onward with headers matching each resource name in the `Vre_stor.csv` files in any order. The availability for each resource at each time step is defined as a fraction of installed capacity and should be between 0 and 1. Note that for this reason, resource names specified in all the resource `.csv` files must be unique. +• Second column onwards: Resources are listed from the second column onward with headers matching each resource name in the `Vre_stor` files in any order. The availability for each resource at each time step is defined as a fraction of installed capacity and should be between 0 and 1. Note that for this reason, resource names specified in all the resource files must be unique. -#### 1.7 Vre\_and\_stor\_wind\_variability.csv +#### 1.7 Vre\_and\_stor\_wind\_variability -This file contains the time-series of capacity factors / availability of the wind component (AC capacity factors) of each co-located resource included in the `Vre_and_stor_data.csv` file for each time step (e.g. hour) modeled. +This file contains the time-series of capacity factors / availability of the wind component (AC capacity factors) of each co-located resource included in the `Vre_and_stor_data` file for each time step (e.g. hour) modeled. • First column: The first column contains the time index of each row (starting in the second row) from 1 to N. -• Second column onwards: Resources are listed from the second column onward with headers matching each resource name in the `Vre_stor.csv` files in any order. The availability for each resource at each time step is defined as a fraction of installed capacity and should be between 0 and 1. Note that for this reason, resource names specified in all the resource `.csv` files must be unique. +• Second column onwards: Resources are listed from the second column onward with headers matching each resource name in the `Vre_stor` files in any order. The availability for each resource at each time step is defined as a fraction of installed capacity and should be between 0 and 1. Note that for this reason, resource names specified in all the resource files must be unique. ## 2. Optional inputs files -### 2.1 Operational_reserves.csv +### 2.1 Operational_reserves This file includes parameter inputs needed to model time-dependent procurement of regulation and spinning reserves. This file is needed if `OperationalReserves` flag is activated in the YAML file `genx_settings.yml`. -###### Table 20: Structure of the Operational_reserves.csv file +###### Table 20: Structure of the Operational_reserves file --- |**Column Name** | **Description**| | :------------ | :-----------| @@ -668,27 +667,27 @@ This file includes parameter inputs needed to model time-dependent procurement o -### 2.2 Energy\_share\_requirement.csv +### 2.2 Energy\_share\_requirement This file contains inputs specifying minimum energy share requirement policies, such as Renewable Portfolio Standard (RPS) or Clean Energy Standard (CES) policies. This file is needed if parameter EnergyShareRequirement has a non-zero value in the YAML file `genx_settings.yml`. -Note: this file should use the same region name as specified in the the resource `.csv` file (inside the `Resource`). +Note: this file should use the same region name as specified in the the resource file (inside the `Resource`). -###### Table 21: Structure of the Energy\_share\_requirement.csv file +###### Table 21: Structure of the Energy\_share\_requirement file --- |**Column Name** | **Description**| | :------------ | :-----------| |Region\_description |Region name| |Network\_zones |zone number represented as z*| -|ESR\_* |[0,1], Energy share requirements as a share of zonal demand (calculated on an annual basis). * represents the number of the ESR constraint, given by the number of ESR\_* columns in the `Energy_share_requirement.csv` file.| +|ESR\_* |[0,1], Energy share requirements as a share of zonal demand (calculated on an annual basis). * represents the number of the ESR constraint, given by the number of ESR\_* columns in the `Energy_share_requirement` file.| -### 2.3 CO2\_cap.csv +### 2.3 CO2\_cap This file contains inputs specifying CO2 emission limits policies (e.g. emissions cap and permit trading programs). This file is needed if `CO2Cap` flag is activated in the YAML file `genx_settings.yml`. `CO2Cap` flag set to 1 represents mass-based (tCO2 ) emission target. `CO2Cap` flag set to 2 is specified when emission target is given in terms of rate (tCO2/MWh) and is based on total demand met. `CO2Cap` flag set to 3 is specified when emission target is given in terms of rate (tCO2 /MWh) and is based on total generation. -###### Table 22: Structure of the CO2\_cap.csv file +###### Table 22: Structure of the CO2\_cap file --- |**Column Name** | **Description**| | :------------ | :-----------| @@ -699,13 +698,13 @@ This file contains inputs specifying CO2 emission limits policies (e.g. emission |CO\_2\_Max\_Mtons_* |Emission limit in absolute values, in Million of tons | | | where in the above inputs, * represents the number of the emission limit constraints. For example, if the model has 2 emission limit constraints applied separately for 2 zones, the above CSV file will have 2 columns for specifying emission limit in terms on rate: CO\_2\_Max\_tons\_MWh\_1 and CO\_2\_Max\_tons\_MWh\_2.| -### 2.4 Capacity\_reserve\_margin.csv +### 2.4 Capacity\_reserve\_margin This file contains the regional capacity reserve margin requirements. This file is needed if parameter CapacityReserveMargin has a non-zero value in the YAML file `genx_settings.yml`. -Note: this file should use the same region name as specified in the resource `.csv` file (inside the `Resource`). +Note: this file should use the same region name as specified in the resource file (inside the `Resource`). -###### Table 23: Structure of the Capacity\_reserve\_margin.csv file +###### Table 23: Structure of the Capacity\_reserve\_margin file --- |**Column Name** | **Description**| | :------------ | :-----------| @@ -715,11 +714,11 @@ Note: this file should use the same region name as specified in the resource `.c -### 2.5 Minimum\_capacity\_requirement.csv +### 2.5 Minimum\_capacity\_requirement This file contains the minimum capacity carve-out requirement to be imposed (e.g. a storage capacity mandate or offshore wind capacity mandate). This file is needed if the `MinCapReq` flag has a non-zero value in the YAML file `genx_settings.yml`. -###### Table 24: Structure of the Minimum\_capacity\_requirement.csv file +###### Table 24: Structure of the Minimum\_capacity\_requirement file --- |**Column Name** | **Description**| | :------------ | :-----------| @@ -730,12 +729,12 @@ This file contains the minimum capacity carve-out requirement to be imposed (e.g Some of the columns specified in the input files in Section 2.2 and 2.1 are not used in the GenX model formulation. These columns are necessary for interpreting the model outputs and used in the output module of the GenX. -### 2.6 Maximum\_capacity\_requirement.csv +### 2.6 Maximum\_capacity\_requirement This contains the maximum capacity limits to be imposed (e.g. limits on total deployment of solar, wind, or batteries in the system as a whole or in certain collections of zones). It is required if the `MaxCapReq` flag has a non-zero value in `genx_settings.yml`. -###### Table 25: Structure of the Maximum\_capacity\_requirement.csv file +###### Table 25: Structure of the Maximum\_capacity\_requirement file --- |**Column Name** | **Description**| | :------------ | :-----------| @@ -746,11 +745,11 @@ It is required if the `MaxCapReq` flag has a non-zero value in `genx_settings.ym Some of the columns specified in the input files in Section 2.2 and 2.1 are not used in the GenX model formulation. These columns are necessary for interpreting the model outputs and used in the output module of the GenX. -### 2.7 Method\_of\_morris\_range.csv +### 2.7 Method\_of\_morris\_range This file contains the settings parameters required to run the Method of Morris algorithm in GenX. This file is needed if the `MethodofMorris` flag is ON in the YAML file `genx_settings.yml`. -###### Table 26: Structure of the Method\_of\_morris\_range.csv file +###### Table 26: Structure of the Method\_of\_morris\_range file --- |**Column Name** | **Description**| | :------------ | :-----------| @@ -758,7 +757,7 @@ This file contains the settings parameters required to run the Method of Morris |Zone | Integer representing zone number where the resource is located. | |Lower\_bound | Percentage lower deviation from the nominal value| |Upper\_bound| Percentage upper deviation from the nominal value| -|Parameter| Column from the resource `.csv` file (inside the `Resource`) containing uncertain parameters| +|Parameter| Column from the resource file (inside the `Resource`) containing uncertain parameters| |Group| Group the uncertain parameters that will be changed all at once while performing the sensitivity analysis. For example, if the fuel price of natural gas is uncertain, all generators consuming natural gas should be in the same group. Group name is user defined| |p_steps| Number of steps between upper and lower bound| |total\_num\_trajectory| Total number of trakectories through the design matrix| @@ -775,17 +774,76 @@ This file contains the settings parameters required to run the Method of Morris 6. num\_trajectory should be approximately equal to the total number of uncertain parameters 7. len\_design_mat should be 1.5 to 2 times the total number of uncertain parameters 8. Higher number of num\_trajectory and len_design_mat would lead to higher accuracy - 9. Upper and lower bounds should be specified for all the resources included in the resource `.csv` file (inside the `Resource`). If a parameter related to a particular resource is not uncertain, specify upper bound = lower bound = 0. + 9. Upper and lower bounds should be specified for all the resources included in the resource file (inside the `Resource`). If a parameter related to a particular resource is not uncertain, specify upper bound = lower bound = 0. -### 2.8 Hydrogen\_demand.csv +### 2.8 Hydrogen\_demand -This file contains inputs specifying regional hydrogen production requirements. This file is needed if `electrolyzer.csv` is included in the resources folder or there are electrolyzer components in `Vre_stor.csv`. +This file contains inputs specifying regional hydrogen production requirements. This file is needed if `electrolyzer` is included in the resources folder or there are electrolyzer components in `Vre_stor`. -###### Table 27: Structure of the Hydrogen\_demand.csv file +###### Table 27: Structure of the Hydrogen\_demand file --- |**Column Name** | **Description**| | :------------ | :-----------| |H2DemandConstraint| Index of the hydrogen demand constraint.| |Constraint\_Description| Names of hydrogen demand constraints; not to be read by model, but used as a helpful notation to the model user. | |Hydrogen\_Demand\_kt| Hydrogen production requirements in 1,000 tons| -|PriceCap| Price of hydrogen per metric ton ($/t)| \ No newline at end of file +|PriceCap| Price of hydrogen per metric ton ($/t)| + +## 3. Changing Input File Names + +As of GenX v0.4.2, a file called `input_settings.yml` can be included in `settings`. This file contains a dictionary where users can change the names of the input files. For example, if you wanted to run GenX twice with two different `Demand` files, you could call one `Demand1.csv` and one `Demand2.csv`. *Note: Adding this input file is not necessary. In the abscence of an input settings file, default names will be used.* For an example, see `1_three_zones/settings` or `example_systems/6_three_zones_w_multistage/settings`. + +To set your own input file names, please include the entire name including the extension. The extension can be of the following types: + +- .csv +- .csv.gz +- .json +- .json.gz +- .parquet +- -snappy.parquet +- -zstd.parquet + + Do not include the path of the file in the name. The path is a separate argument that can also be changed in `input_settings.yml`. + +The file `input_settings.yml` has the following structure: + +|**Key** | **Default Value**| +|:----------------------|:---------------| +|**system_location** | joinpath(case, "system")| +|demand | Demand_data.csv| +|fuel | Fuels_data.csv| +|generators | Generators_variability.csv| +|network | Network.csv| +|**resources_location** | joinpath(case, "resources")| +|storage| Storage.csv| +|thermal | Thermal.csv| +|vre | Vre.csv| +|vre_stor | Vre_stor.csv| +|vre_stor_solar_variability | Vre_and_stor_solar_variability.csv| +|vre_stor_wind_variability | Vre_and_stor_wind_variability.csv| +|hydro | Hydro.csv| +|demand | Demand_data.csv| +|flex_demand | Flex_demand.csv| +|must_run | Must_run.csv| +|electrolyzer | Electrolyzer.csv| +|resources_cap | Resource_capacity_reserve_margin.csv| +|resource_energy_share_requirement | Resource_energy_share_requirement.csv| +|resource_min | Resource_minimum_capacity_requirement.csv| +|resource_max | Resource_maximum_capacity_requirement.csv| +|resource_multistage_data | Resource_multistage_data.csv | +|resource_hydrogen_demand | Resource_hydrogen_demand.csv| +|resource_hourly_matching | Resource_hourly_matching.csv| +|**policies_location** | joinpath(case, "policies")| +|period_map | Period_map.csv| +|capacity | Capacity_reserve_margin.csv| +|CRM_slack| Capacity_reserve_margin_slack.csv| +|co2_cap | CO2_cap.csv| +|co2_cap_slack | CO2_cap_slack.csv| +|esr | Energy_share_requirement.csv| +|esr_slack | Energy_share_requirement_slack.csv| +|min_cap| Minimum_capacity_requirement.csv| +|max_cap | Maximum_capacity_requirement.csv| +|operational_reserves| Operational_reserves.csv| + +!!! note "Note" + For MultiStage, input file names and paths can be specified according to stage. An example of a multistage input YAML file can be found in `6_three_zones_w_multistage/settings`. \ No newline at end of file diff --git a/docs/src/User_Guide/model_output.md b/docs/src/User_Guide/model_output.md index 18cf1a18ca..7033cff964 100644 --- a/docs/src/User_Guide/model_output.md +++ b/docs/src/User_Guide/model_output.md @@ -153,3 +153,81 @@ This file includes the renewable/clean credit revenue earned by each generator l ### 2.8 SubsidyRevenue.csv This file includes subsidy revenue earned if a generator specified Min\_Cap is provided in the input file. GenX will print this file only the shadow price can be obtained form the solver. Do not confuse this with the Minimum Capacity Carveout constraint, which is for a subset of generators, and a separate revenue term will be calculated in other files. The unit is $. + +## 3 Output file names and type + +As of GenX v0.4.2, the names of all results files can be changed by including the file `results_settings.yml`. Inclusion of this file is not necessary; the default names described above will be used if the file is not present. + +Files are automatically saved as `.csv` in GenX. To change this, you can 1) change the extension of a file name in `results_settings.yml` (e.g. set `demand: "Demand.json"`), or 2) use `ResultsFileType` in `genx_settings.yml` to change the type of all the files. File names in `results_settings.yml` with preexisting extensions will override the type in `ResultsFileType`. For example, if `ResultsFileType = .csv`, but in `results_settings.yml` you have `demand: "Demand.json"`, the demand file will be saved as JSON, and all others will be saved as CSV. No error will be thrown. + +Files can also be saved with gzip, snappy, and zstd compression. To choose which files are compressed, add the compression to the extension in `results_settings.yml` (e.g. set `demand: "Demand-snappy.parquet"` or `fuels: "Fuels.csv.gz"`). To compress all files, specify the `ResultsCompressionType` in `genx_settings.yml`. If the compression type is specified in settings, it does not need to be present in the results names. The correct file extension will be appended to the filename. If the file type and compression type conflict (e.g. CSV with snappy compression), no compression will be used. + +For an example, see `1_three_zones/settings`. + +Both single and multistage results files are in the same file. The file `results_settings.yml` has the following structure: + +|**Key** | **Default Value**| +|:----------------------|:---------------| +|angles | angles| +|capacity_name | capacity| +|capacity_factor | capacityfactor| +|capacity_vaue | CapacityValue| +|capacities_charge_multi_stage | capacities_charge_multi_stage| +|capacities_multi_stage | capacities_multi_stage| +|capacities_energy_multi_stage | capacities_energy_multi_stage| +|captured_emissions_plant | captured_emissions_plant| +|charge | charge.csv| +|charging_cost | ChargingCost| +|co2_prices | CO2_prices_and_penalties| +|commit | commit| +|costs | costs| +|costs_multi_stage | costs_multi_stage | +|curtail | curtail| +|dStorage | dStorage| +|emissions_plant | emissions_plant| +|emissions | emissions| +|energy_revenue | EnergyRevenue| +|esr_prices_and_penalties | ESR_prices_and_penalties| +|esr_revenue | ESR_Revenue| +|flow | flow| +|fuel_cost_plant | Fuel_cost_plant| +|fuel_consumption_plant | FuelConsumption_plant_MMBTU| +|fuel_consumption_total | FuelConsumtion_total_MMBTU| +|hourly_matching_prices | hourly_matching_prices| +|hydrogen_prices | hydrogen_prices| +|mincap | MinCapReq_prices_and_penalties| +|maxcap | MaxCapReq_prices_and_penalties| +|maint_down | maint_down| +|revenue | NetRevenue| +|network_expansion | network_expansion| +|network_expansion_multi_stage | network_expansion_multi_stage| +|nse | nse| +|power_balance | power_balance| +|power | power| +|prices | prices| +|reg_subsidy_revenue | RegSubsidyRevenue| +|reserve_margin | ReserveMargin| +|reserve_margin_revenue | ReserveMarginRevenue| +|reserve_margin_prices_and_penalties | ReserveMargin_prices_and_penalties| +|reserve_margin_w | ReserveMargin_w.csv| +|reg | reg| +|reg_dn | reg_dn| +|reliability | reliability| +|shutdown | shutdown| +|start | start| +|status | status| +|storage | storage| +|storagebal_duals | storagebal_duals| +|storage_init | StorageInit| +|subsidy_revenue | SubsidyRevenue| +|time_weights | time_weights| +|tlosses | tlosses| +|virtual_discharge | virtual_discharge| +|vre_stor_dc_charge | vre_stor_dc_charge| +|vre_stor_ac_charge | vre_stor_ac_charge| +|vre_stor_dc_discharge | vre_stor_dc_discharge| +|vre_stor_ac_discharge | vre_stor_ac_discharge| +|vre_stor_elec_power_consumption | vre_stor_elec_power_consumption| +|vre_stor_wind_power | vre_stor_wind_power| +|vre_stor_solar_power | vre_stor_solar_power +|vre_stor_capacity | vre_stor_capacity| \ No newline at end of file diff --git a/example_systems/1_three_zones/settings/genx_settings.yml b/example_systems/1_three_zones/settings/genx_settings.yml index d48032b1d1..345b03bf66 100644 --- a/example_systems/1_three_zones/settings/genx_settings.yml +++ b/example_systems/1_three_zones/settings/genx_settings.yml @@ -10,4 +10,6 @@ ParameterScale: 1 # Turn on parameter scaling wherein demand, capacity and power WriteShadowPrices: 1 # Write shadow prices of LP or relaxed MILP; 0 = not active; 1 = active UCommit: 2 # Unit committment of thermal power plants; 0 = not active; 1 = active using integer clestering; 2 = active using linearized clustering TimeDomainReduction: 1 # Time domain reduce (i.e. cluster) inputs based on Demand_data.csv, Generators_variability.csv, and Fuels_data.csv; 0 = not active (use input data as provided); 0 = active (cluster input data, or use data that has already been clustered) -OutputFullTimeSeries: 1 \ No newline at end of file +OutputFullTimeSeries: 1 # Reconstuct all hours of the year and output in a folder called Full_TimeSeries +ResultsFileType: "auto_detect" # Automatically detect the type of the results files from the extension name. If no extension is present, files are saved as .csv. +ResultsCompressionType: "auto_detect" # Automatically detect the type of compression for the results files from the extension name. If no compression is present, saved as uncompressed. \ No newline at end of file diff --git a/example_systems/1_three_zones/settings/input_settings.yml b/example_systems/1_three_zones/settings/input_settings.yml new file mode 100644 index 0000000000..2e67d88c34 --- /dev/null +++ b/example_systems/1_three_zones/settings/input_settings.yml @@ -0,0 +1,15 @@ +#default_location: default/path/to/data +# system +#system_location: path/to/data +demand: "Demand_data.csv" +fuel: "Fuels_data.csv" +generators: "Generators_variability.csv" + +# policies +#policies_location: /path/to/data +#co2_cap: file_or_table_name +#minimum_capacity: file_or_table_name + +# resources + +# policy assignments \ No newline at end of file diff --git a/example_systems/1_three_zones/settings/results_settings.yml b/example_systems/1_three_zones/settings/results_settings.yml new file mode 100644 index 0000000000..d18d5fbb23 --- /dev/null +++ b/example_systems/1_three_zones/settings/results_settings.yml @@ -0,0 +1,14 @@ +capacity: "Capacity_test.csv" +capacity_factor: "capacityfactor.csv" +charge: "charge.csv" +charging_cost: "ChargingCost.csv" +co2_prices: "CO2_prices_and_penalties.csv" +commit: "commit.parquet" +costs: "costs.parquet" +curtail: "curtail.csv.gz" +emissions_plant: "emissions_plant" +nse: "nse.csv" +power_balance: "power_balance.csv" + + + diff --git a/example_systems/6_three_zones_w_multistage/settings/highs_settings.yml b/example_systems/6_three_zones_w_multistage/settings/highs_settings.yml index 7257ed4da6..07ce75381c 100644 --- a/example_systems/6_three_zones_w_multistage/settings/highs_settings.yml +++ b/example_systems/6_three_zones_w_multistage/settings/highs_settings.yml @@ -1,7 +1,7 @@ # HiGHS Solver Parameters # Common solver settings -Feasib_Tol: 1.0e-05 # Primal feasibility tolerance # [type: double, advanced: false, range: [1e-10, inf], default: 1e-07] -Optimal_Tol: 1.0e-05 # Dual feasibility tolerance # [type: double, advanced: false, range: [1e-10, inf], default: 1e-07] +Feasib_Tol: 1.0e-05 # Primal feasibility tolerance # [type: double, advanced: false, range: [1e-10, inf], default: 1e-07] +Optimal_Tol: 1.0e-05 # Dual feasibility tolerance # [type: double, advanced: false, range: [1e-10, inf], default: 1e-07] TimeLimit: 1.0e23 # Time limit # [type: double, advanced: false, range: [0, inf], default: inf] Pre_Solve: choose # Presolve option: "off", "choose" or "on" # [type: string, advanced: false, default: "choose"] Method: ipm #HiGHS-specific solver settings # Solver option: "simplex", "choose" or "ipm" # [type: string, advanced: false, default: "choose"] diff --git a/example_systems/6_three_zones_w_multistage/settings/input_settings.yml b/example_systems/6_three_zones_w_multistage/settings/input_settings.yml new file mode 100644 index 0000000000..5d80f76139 --- /dev/null +++ b/example_systems/6_three_zones_w_multistage/settings/input_settings.yml @@ -0,0 +1,41 @@ +inputs_p1: + resources_location: "inputs/inputs_p1/resources" + resources: "Resource_multistage_data.csv" + storage: "Storage.csv" + thermal: "Thermal.csv" + vre: "Vre.csv" + policies_location: "inputs/inputs_p1/policies" + co2_cap: "CO2_cap.csv" + system_location: "inputs/inputs_p1/system" + demand: "Demand_data.csv" + fuel: "Fuels_data.csv" + generators: "Generators_variability1.csv" + network: "Network1.csv" + +inputs_p2: + resources_location: "inputs/inputs_p1/resources" + resources: "Resource_multistage_data.csv" + storage: "Storage.csv" + thermal: "Thermal.csv" + vre: "Vre.csv" + policies_location: "inputs/inputs_p2/policies" + co2_cap: "CO2_cap.csv" + system_location: "inputs/inputs_p2/system" + demand: "Demand_data.csv" + fuel: "Fuels_data.csv" + generators: "Generators_variability.csv" + network: "Network.csv" + +inputs_p3: + resources_location: "inputs/inputs_p1/resources" + resources: "Resource_multistage_data.csv" + storage: "Storage.csv" + thermal: "Thermal.csv" + vre: "Vre.csv" + policies_location: "inputs/inputs_p3/policies" + co2_cap: "CO2_cap.csv" + system_location: "inputs/inputs_p3/system" + demand: "Demand_data.csv" + fuel: "Fuels_data.csv" + generators: "Generators_variability.csv" + network: "Network.csv" \ No newline at end of file diff --git a/precompile/case/settings/input_settings.yml b/precompile/case/settings/input_settings.yml new file mode 100644 index 0000000000..321b7410b6 --- /dev/null +++ b/precompile/case/settings/input_settings.yml @@ -0,0 +1,16 @@ +#default_location: default/path/to/data +# system +system_location: path/to/data +demand: "Demand_data.csv" +fuel: "Fuels_data.csv" +generators: "Generators_variability.csv" + +# policies +#policies_location: /path/to/data +#co2_name: file_or_table_name +#minimum_capacity_name: file_or_table_name + +# resources + + +# policy assignments \ No newline at end of file diff --git a/precompile/case/settings/results_settings.yml b/precompile/case/settings/results_settings.yml new file mode 100644 index 0000000000..d18d5fbb23 --- /dev/null +++ b/precompile/case/settings/results_settings.yml @@ -0,0 +1,14 @@ +capacity: "Capacity_test.csv" +capacity_factor: "capacityfactor.csv" +charge: "charge.csv" +charging_cost: "ChargingCost.csv" +co2_prices: "CO2_prices_and_penalties.csv" +commit: "commit.parquet" +costs: "costs.parquet" +curtail: "curtail.csv.gz" +emissions_plant: "emissions_plant" +nse: "nse.csv" +power_balance: "power_balance.csv" + + + diff --git a/src/GenX.jl b/src/GenX.jl index 3a5a399800..20a915618b 100644 --- a/src/GenX.jl +++ b/src/GenX.jl @@ -25,6 +25,7 @@ export run_timedomainreduction! using JuMP # used for mathematical programming using DataFrames #This package allows put together data into a matrix using CSV +using JSON using StatsBase using LinearAlgebra using YAML @@ -37,8 +38,8 @@ using RecursiveArrayTools using Statistics using HiGHS using Logging - using PrecompileTools: @compile_workload +using DuckDB # Global scaling factor used when ParameterScale is on to shift values from MW to GW # DO NOT CHANGE THIS (Unless you do so very carefully) diff --git a/src/additional_tools/method_of_morris.jl b/src/additional_tools/method_of_morris.jl index ed5cd5ddfa..99e8aa6738 100644 --- a/src/additional_tools/method_of_morris.jl +++ b/src/additional_tools/method_of_morris.jl @@ -262,6 +262,10 @@ function morris(EP::Model, #save the variance of effect of each uncertain variable on the objective function Morris_range[!, :variance] = DataFrame(m.variances', :auto)[!, :x1] - CSV.write(joinpath(outpath, "morris.csv"), Morris_range) + write_output_file(joinpath(outpath, + setup["WriteResultsNamesDict"]["morris"]), + Morris_range, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) return Morris_range end diff --git a/src/case_runners/case_runner.jl b/src/case_runners/case_runner.jl index afea227f29..30ab350434 100644 --- a/src/case_runners/case_runner.jl +++ b/src/case_runners/case_runner.jl @@ -32,8 +32,7 @@ function run_genx_case!(case::AbstractString, optimizer::Any = HiGHS.Optimizer) print_genx_version() # Log the GenX version genx_settings = get_settings_path(case, "genx_settings.yml") # Settings YAML file path writeoutput_settings = get_settings_path(case, "output_settings.yml") # Write-output settings YAML file path - mysetup = configure_settings(genx_settings, writeoutput_settings) # mysetup dictionary stores settings and GenX-specific parameters - + mysetup = configure_settings(genx_settings, writeoutput_settings, case) # mysetup dictionary stores settings and GenX-specific parameters if mysetup["MultiStage"] == 0 run_genx_case_simple!(case, mysetup, optimizer) else @@ -41,10 +40,10 @@ function run_genx_case!(case::AbstractString, optimizer::Any = HiGHS.Optimizer) end end -function time_domain_reduced_files_exist(tdrpath) - tdr_demand = file_exists(tdrpath, ["Demand_data.csv", "Load_data.csv"]) - tdr_genvar = isfile(joinpath(tdrpath, "Generators_variability.csv")) - tdr_fuels = isfile(joinpath(tdrpath, "Fuels_data.csv")) +function time_domain_reduced_files_exist(tdrpath, setup::Dict) + tdr_demand = isfile(joinpath(tdrpath, setup["demand"])) + tdr_genvar = isfile(joinpath(tdrpath, setup["generators"])) + tdr_fuels = isfile(joinpath(tdrpath, setup["fuel"])) return (tdr_demand && tdr_genvar && tdr_fuels) end @@ -55,8 +54,8 @@ function run_genx_case_simple!(case::AbstractString, mysetup::Dict, optimizer::A if mysetup["TimeDomainReduction"] == 1 TDRpath = joinpath(case, mysetup["TimeDomainReductionFolder"]) system_path = joinpath(case, mysetup["SystemFolder"]) - prevent_doubled_timedomainreduction(system_path) - if !time_domain_reduced_files_exist(TDRpath) + prevent_doubled_timedomainreduction(system_path, mysetup["WriteInputNamesDict"]) + if !time_domain_reduced_files_exist(TDRpath, mysetup["WriteInputNamesDict"]) println("Clustering Time Series Data (Grouped)...") cluster_inputs(case, settings_path, mysetup) else @@ -110,7 +109,7 @@ function run_genx_case_multistage!(case::AbstractString, mysetup::Dict, optimize settings_path = get_settings_path(case) multistage_settings = get_settings_path(case, "multi_stage_settings.yml") # Multi stage settings YAML file path # merge default settings with those specified in the YAML file - mysetup["MultiStageSettingsDict"] = configure_settings_multistage(multistage_settings) + mysetup["MultiStageSettingsDict"] = configure_settings_multistage(case, multistage_settings) ### Cluster time series inputs if necessary and if specified by the user if mysetup["TimeDomainReduction"] == 1 @@ -120,8 +119,11 @@ function run_genx_case_multistage!(case::AbstractString, mysetup::Dict, optimize first_stage_path = joinpath(case, "inputs", "inputs_p1") TDRpath = joinpath(first_stage_path, mysetup["TimeDomainReductionFolder"]) system_path = joinpath(first_stage_path, mysetup["SystemFolder"]) - prevent_doubled_timedomainreduction(system_path) - if !time_domain_reduced_files_exist(TDRpath) + + mysetup["MultiStageSettingsDict"]["CurStage"] = 1 # Define current stage for cluster_inputs to access input_names dictionary at stage 1 + + prevent_doubled_timedomainreduction(system_path, mysetup["WriteInputNamesDict"]["inputs_p1"]) + if !time_domain_reduced_files_exist(TDRpath, mysetup["WriteInputNamesDict"]["inputs_p1"]) if (mysetup["MultiStage"] == 1) && (TDRSettingsDict["MultiStageConcatenate"] == 0) println("Clustering Time Series Data (Individually)...") @@ -151,9 +153,7 @@ function run_genx_case_multistage!(case::AbstractString, mysetup::Dict, optimize mysetup["MultiStageSettingsDict"]["CurStage"] = t # Step 1) Load Inputs - inpath_sub = joinpath(case, "inputs", string("inputs_p", t)) - - inputs_dict[t] = load_inputs(mysetup, inpath_sub) + inputs_dict[t] = load_inputs(mysetup, case) inputs_dict[t] = configure_multi_stage_inputs(inputs_dict[t], mysetup["MultiStageSettingsDict"], mysetup["NetworkExpansion"]) diff --git a/src/configure_settings/configure_settings.jl b/src/configure_settings/configure_settings.jl index 9df418e334..fe6ad3914e 100644 --- a/src/configure_settings/configure_settings.jl +++ b/src/configure_settings/configure_settings.jl @@ -37,32 +37,55 @@ function default_settings() "ResourcePoliciesFolder" => "policy_assignments", "SystemFolder" => "system", "PoliciesFolder" => "policies", - "ObjScale" => 1) + "ObjScale" => 1, + "ResultsFileType" => "auto_detect", + "ResultsCompressionType" => "auto_detect") end @doc raw""" - configure_settings(settings_path::String, output_settings_path::String) + configure_settings(settings_path::String, output_settings_path::String, case::AbstractString) -Reads in the settings from the `genx_settings.yml` and `output_settings.yml` YAML files and +Reads in the settings from the `genx_settings.yml`, `output_settings.yml`, `input_settings.yml`, and `results_settings.yml` YAML files and merges them with the default settings. It then validates the settings and returns the settings dictionary. # Arguments - `settings_path::String`: The path to the settings YAML file. - `output_settings_path::String`: The path to the output settings YAML file. +- `case::AbstractString`: The case used for this instance of GenX. # Returns - `settings::Dict`: The settings dictionary. """ -function configure_settings(settings_path::String, output_settings_path::String) +function configure_settings(settings_path::String, output_settings_path::String, case::AbstractString) println("\nConfiguring Settings") model_settings = YAML.load(open(settings_path)) settings = default_settings() merge!(settings, model_settings) + if settings["ResultsFileType"] ∉ ["auto_detect",".csv.gz",".parquet",".json",".json.gz","csv","parquet","json","csv.gz","json.gz"] + throw("Results File Type in genx_settings.yml is not accepted. Acceptable types are csv, csv.gz, parquet, json, and json.gz.") + end + output_settings = configure_writeoutput(output_settings_path, settings) settings["WriteOutputsSettingsDict"] = output_settings + + if settings["MultiStage"] == 1 + multistage_settings = configure_settings_multistage(case,settings_path) + settings["WriteInputNamesDict"] = Dict{}() + for t in 1:multistage_settings["NumStages"] + subdict_name = string("inputs_p", t) + input_settings = configure_input_names(case,t=t) + settings["WriteInputNamesDict"][subdict_name] = input_settings + end + else + input_settings = configure_input_names(case) + settings["WriteInputNamesDict"] = input_settings + end + + results_settings = configure_results_names(case) + settings["WriteResultsNamesDict"] = results_settings validate_settings!(settings) return settings @@ -216,3 +239,163 @@ function validate_multistage_settings!(settings::Dict{Any, Any}) settings["WriteIntermittentOutputs"] = 0 end end + +function default_input_names(case::AbstractString) + Dict{Any, Any}("system_location" => joinpath(case, "system"), + "demand" => "Demand_data.csv", + "fuel" => "Fuels_data.csv", + "generators" => "Generators_variability.csv", + "network" => "Network.csv", + "resources_location" => joinpath(case, "resources"), + "storage" => "Storage.csv", + "thermal" => "Thermal.csv", + "vre" => "Vre.csv", + "vre_stor" => "Vre_stor.csv", + "vre_stor_solar_variability" => "Vre_and_stor_solar_variability.csv", + "vre_stor_wind_variability" => "Vre_and_stor_wind_variability.csv", + "hydro" => "Hydro.csv", + "flex_demand" => "Flex_demand.csv", + "must_run" => "Must_run.csv", + "electrolyzer" => "Electrolyzer.csv", + "resource_cap" => "Resource_capacity_reserve_margin.csv", + "resource_energy_share_requirement" => "Resource_energy_share_requirement.csv", + "resource_min" => "Resource_minimum_capacity_requirement.csv", + "resource_max" => "Resource_maximum_capacity_requirement.csv", + "resource_hydrogen_demand" => "Resource_hydrogen_demand.csv", + "resource_hourly_matching" => "Resource_hourly_matching.csv", + "resource_multistage_data" => "Resource_multistage_data.csv", + "policies_location" => joinpath(case, "policies"), + "period_map" => "Period_map.csv", + "capacity" => "Capacity_reserve_margin.csv", + "CRM_slack" => "Capacity_reserve_margin_slack.csv", + "co2_cap" => "CO2_cap.csv", + "co2_cap_slack" => "CO2_cap_slack.csv", + "esr" => "Energy_share_requirement.csv", + "esr_slack" => "Energy_share_requirement_slack.csv", + "min_cap" => "Minimum_capacity_requirement.csv", + "max_cap" => "Maximum_capacity_requirement.csv", + "operational_reserves" => "Operational_reserves.csv") +end + +@doc raw""" + configure_input_names(case::AbstractString) + +Reads in the settings from the `input_settings.yml` YAML file and +merges them with the default input settings. It then returns the +settings dictionary. + +# Arguments +- `case::AbstractString`: The case containing the settings file. + +# Returns +- `names::Dict`: The input names dictionary. +""" +function configure_input_names(case::AbstractString; t::Int64 = 0) + println("Configuring Input File and Path Names") + input_settings_path = get_settings_path(case, "input_settings.yml") + input_names = isfile(input_settings_path) ? YAML.load(open(input_settings_path)) : Dict{Any, Any}() + + if t > 0 + input_folder = string("inputs_p", t) + names = default_input_names(joinpath(case,"inputs",input_folder)) + merge!(names,input_names[input_folder]) + else + names = default_input_names(case) + merge!(names,input_names) + end + + return names +end + +function default_results_names() + Dict{Any, Any}("angles" => "angles", + "capacity" => "capacity", + "capacity_factor" => "capacityfactor", + "capacity_vaue" => "CapacityValue", + "capacities_charge_multi_stage" => "capacities_charge_multi_stage", + "capacities_multi_stage" => "capacities_multi_stage", + "capacities_energy_multi_stage" => "capacities_energy_multi_stage", + "captured_emissions_plant" => "captured_emissions_plant", + "charge" => "charge.csv", + "charging_cost" => "ChargingCost", + "co2_prices" => "CO2_prices_and_penalties", + "commit" => "commit", + "costs" => "costs", + "costs_multi_stage" => "costs_multi_stage", + "curtail" => "curtail", + "dStorage" => "dStorage", + "emissions_plant" => "emissions_plant", + "emissions" => "emissions", + "energy_revenue" => "EnergyRevenue", + "esr_prices_and_penalties" => "ESR_prices_and_penalties", + "esr_revenue" => "ESR_Revenue", + "flow" => "flow", + "fuel_cost_plant" => "Fuel_cost_plant", + "fuel_consumption_plant" => "FuelConsumption_plant_MMBTU", + "fuel_consumption_total" => "FuelConsumtion_total_MMBTU", + "hourly_matching_prices" => "hourly_matching_prices", + "hydrogen_prices" => "hydrogen_prices", + "mincap" => "MinCapReq_prices_and_penalties", + "maxcap" => "MaxCapReq_prices_and_penalties", + "maint_down" => "maint_down", + "morris" => "morris", + "revenue" => "NetRevenue", + "network_expansion" => "network_expansion", + "network_expansion_multi_stage" => "network_expansion_multi_stage", + "nse" => "nse", + "power_balance" => "power_balance", + "power" => "power", + "prices" => "prices", + "reg_subsidy_revenue" => "RegSubsidyRevenue", + "reserve_margin" => "ReserveMargin", + "reserve_margin_revenue" => "ReserveMarginRevenue", + "reserve_margin_prices_and_penalties" => "ReserveMargin_prices_and_penalties", + "reserve_margin_w" => "ReserveMargin_w.csv", + "reg" => "reg", + "reg_dn" => "reg_dn", + "reliability" => "reliability", + "shutdown" => "shutdown", + "start" => "start", + "status" => "status", + "storage" => "storage", + "storagebal_duals" => "storagebal_duals", + "storage_init" => "StorageInit", + "storage_evol" => "StorageEvol", + "subsidy_revenue" => "SubsidyRevenue", + "time_weights" => "time_weights", + "tlosses" => "tlosses", + "virtual_discharge" => "virtual_discharge", + "vre_stor_dc_charge" => "vre_stor_dc_charge", + "vre_stor_ac_charge" => "vre_stor_ac_charge", + "vre_stor_dc_discharge" => "vre_stor_dc_discharge", + "vre_stor_ac_discharge" => "vre_stor_ac_discharge", + "vre_stor_elec_power_consumption" => "vre_stor_elec_power_consumption", + "vre_stor_wind_power" => "vre_stor_wind_power", + "vre_stor_solar_power" => "vre_stor_solar_power", + "vre_stor_capacity" => "vre_stor_capacity") +end + +@doc raw""" + configure_results_names(case::AbstractString) + +Reads in the settings from the `results_settings.yml` YAML file and +merges them with the default results settings. It then returns the +settings dictionary. + +# Arguments +- `case::AbstractString`: The case containing the settings file. + +# Returns +- `names::Dict`: The results names dictionary. +""" +function configure_results_names(case::AbstractString) + println("Configuring Results File Names") + results_settings_path = get_settings_path(case, "results_settings.yml") + results_names = isfile(results_settings_path) ? YAML.load(open(results_settings_path)) : Dict{Any, Any}() + + names = default_results_names() + merge!(names,results_names) + + return names +end + diff --git a/src/load_inputs/load_cap_reserve_margin.jl b/src/load_inputs/load_cap_reserve_margin.jl index 0a652bc78f..22a4e0f57e 100644 --- a/src/load_inputs/load_cap_reserve_margin.jl +++ b/src/load_inputs/load_cap_reserve_margin.jl @@ -6,14 +6,14 @@ Read input parameters related to planning reserve margin constraints function load_cap_reserve_margin!(setup::Dict, path::AbstractString, inputs::Dict) scale_factor = setup["ParameterScale"] == 1 ? ModelScalingFactor : 1 - filename = "Capacity_reserve_margin_slack.csv" + filename = setup["WriteInputNamesDict"]["CRM_slack"] if isfile(joinpath(path, filename)) df = load_dataframe(joinpath(path, filename)) inputs["dfCapRes_slack"] = df inputs["dfCapRes_slack"][!, :PriceCap] ./= scale_factor # Million $/GW if scaled, $/MW if not scaled end - filename = "Capacity_reserve_margin.csv" + filename = setup["WriteInputNamesDict"]["capacity"] df = load_dataframe(joinpath(path, filename)) mat = extract_matrix_from_dataframe(df, "CapRes") diff --git a/src/load_inputs/load_co2_cap.jl b/src/load_inputs/load_co2_cap.jl index 08e6802a0a..b17dc5ed91 100644 --- a/src/load_inputs/load_co2_cap.jl +++ b/src/load_inputs/load_co2_cap.jl @@ -6,14 +6,20 @@ Read input parameters related to CO$_2$ emissions cap constraints function load_co2_cap!(setup::Dict, path::AbstractString, inputs::Dict) scale_factor = setup["ParameterScale"] == 1 ? ModelScalingFactor : 1 - filename = "CO2_cap_slack.csv" - if isfile(joinpath(path, filename)) - df = load_dataframe(joinpath(path, filename)) + if setup["MultiStage"] == 1 + stage = setup["MultiStageSettingsDict"]["CurStage"] + filename_slack = setup["WriteInputNamesDict"][string("inputs_p",stage)]["co2_cap_slack"] + filename = setup["WriteInputNamesDict"][string("inputs_p",stage)]["co2_cap"] + else + filename_slack = setup["WriteInputNamesDict"]["co2_cap_slack"] + filename = setup["WriteInputNamesDict"]["co2_cap"] + end + if isfile(joinpath(path, filename_slack)) + df = load_dataframe(joinpath(path, filename_slack)) inputs["dfCO2Cap_slack"] = df inputs["dfCO2Cap_slack"][!, :PriceCap] ./= scale_factor # Million $/kton if scaled, $/ton if not scaled end - - filename = "CO2_cap.csv" + df = load_dataframe(joinpath(path, filename)) inputs["dfCO2Cap"] = df diff --git a/src/load_inputs/load_dataframe.jl b/src/load_inputs/load_dataframe.jl index bd212b75ef..bfd714f20d 100644 --- a/src/load_inputs/load_dataframe.jl +++ b/src/load_inputs/load_dataframe.jl @@ -127,9 +127,13 @@ function check_for_duplicate_keys(path::AbstractString) end end -function load_dataframe_from_file(path)::DataFrame +function load_dataframe_from_file(path::AbstractString) check_for_duplicate_keys(path) - CSV.read(path, DataFrame, header = 1) + #CSV.read(path, DataFrame, header = 1) + + # Create a duckdb connection + con = DBInterface.connect(DuckDB.DB, ":memory:") + DBInterface.execute(con, "SELECT * FROM '$path'") |> DataFrames.DataFrame end function find_matrix_columns_in_dataframe(df::DataFrame, diff --git a/src/load_inputs/load_demand_data.jl b/src/load_inputs/load_demand_data.jl index 52c5bd7bf2..e51ab0b367 100644 --- a/src/load_inputs/load_demand_data.jl +++ b/src/load_inputs/load_demand_data.jl @@ -1,5 +1,5 @@ -function get_demand_dataframe(path) - filename = "Demand_data.csv" +function get_demand_dataframe(path::String, setup::Dict) + filename = setup["demand"] deprecated_synonym = "Load_data.csv" df = load_dataframe(path, [filename, deprecated_synonym]) # update column names @@ -23,19 +23,23 @@ DEMAND_COLUMN_PREFIX() = "Demand_MW_z" DEMAND_COLUMN_PREFIX_DEPRECATED() = "Load_MW_z" @doc raw""" - load_demand_data!(setup::Dict, path::AbstractString, inputs::Dict) + load_demand_data!(setup::Dict, path::AbstractString, inputs::Dict; stage::Int64 = 0) Read input parameters related to electricity demand (load) """ -function load_demand_data!(setup::Dict, path::AbstractString, inputs::Dict) - +function load_demand_data!(setup::Dict, path::AbstractString, inputs::Dict; stage::Int64 = 0) # Load related inputs TDR_directory = joinpath(path, setup["TimeDomainReductionFolder"]) # if TDR is used, my_dir = TDR_directory, else my_dir = "system" - my_dir = get_systemfiles_path(setup, TDR_directory, path) - - demand_in = get_demand_dataframe(my_dir) - + if setup["MultiStage"] == 1 + stage = setup["MultiStageSettingsDict"]["CurStage"] + my_dir = get_systemfiles_path(setup, TDR_directory, path) + demand_in = get_demand_dataframe(my_dir, setup["WriteInputNamesDict"][string("inputs_p",stage)]) + else + my_dir = get_systemfiles_path(setup, TDR_directory, path) + demand_in = get_demand_dataframe(my_dir, setup["WriteInputNamesDict"]) + end + as_vector(col::Symbol) = collect(skipmissing(demand_in[!, col])) # Number of time steps (periods) @@ -43,7 +47,7 @@ function load_demand_data!(setup::Dict, path::AbstractString, inputs::Dict) # Number of demand curtailment/lost load segments SEG = length(as_vector(:Demand_Segment)) - ## Set indices for internal use + ## Set indices for internal usex inputs["T"] = T inputs["SEG"] = SEG Z = inputs["Z"] # Number of zones @@ -163,8 +167,8 @@ end This function prevents TimeDomainReduction from running on a case which already has more than one Representative Period or has more than one Sub_Weight specified. """ -function prevent_doubled_timedomainreduction(path::AbstractString) - demand_in = get_demand_dataframe(path) +function prevent_doubled_timedomainreduction(path::AbstractString, setup::Dict) + demand_in = get_demand_dataframe(path, setup) as_vector(col::Symbol) = collect(skipmissing(demand_in[!, col])) representative_periods = convert(Int16, as_vector(:Rep_Periods)[1]) sub_weights = as_vector(:Sub_Weights) diff --git a/src/load_inputs/load_energy_share_requirement.jl b/src/load_inputs/load_energy_share_requirement.jl index e3205196b7..103431bdf0 100644 --- a/src/load_inputs/load_energy_share_requirement.jl +++ b/src/load_inputs/load_energy_share_requirement.jl @@ -7,14 +7,14 @@ Read input parameters related to minimum energy share requirement constraints function load_energy_share_requirement!(setup::Dict, path::AbstractString, inputs::Dict) scale_factor = setup["ParameterScale"] == 1 ? ModelScalingFactor : 1 - filename = "Energy_share_requirement_slack.csv" + filename = setup["WriteInputNamesDict"]["esr_slack"] if isfile(joinpath(path, filename)) df = load_dataframe(joinpath(path, filename)) inputs["dfESR_slack"] = df inputs["dfESR_slack"][!, :PriceCap] ./= scale_factor # million $/GWh if scaled, $/MWh if not scaled end - filename = "Energy_share_requirement.csv" + filename = setup["WriteInputNamesDict"]["esr"] df = load_dataframe(joinpath(path, filename)) mat = extract_matrix_from_dataframe(df, "ESR") inputs["dfESR"] = mat diff --git a/src/load_inputs/load_fuels_data.jl b/src/load_inputs/load_fuels_data.jl index 61b0ff2f0f..718ad79f56 100644 --- a/src/load_inputs/load_fuels_data.jl +++ b/src/load_inputs/load_fuels_data.jl @@ -8,9 +8,16 @@ function load_fuels_data!(setup::Dict, path::AbstractString, inputs::Dict) # Fuel related inputs - read in different files depending on if time domain reduction is activated or not TDR_directory = joinpath(path, setup["TimeDomainReductionFolder"]) # if TDR is used, my_dir = TDR_directory, else my_dir = "system" - my_dir = get_systemfiles_path(setup, TDR_directory, path) - - filename = "Fuels_data.csv" + if setup["MultiStage"] == 1 + stage = setup["MultiStageSettingsDict"]["CurStage"] + my_dir = get_systemfiles_path(setup, TDR_directory, path) + filename = setup["WriteInputNamesDict"][string("inputs_p",stage)]["fuel"] + else + my_dir = get_systemfiles_path(setup, TDR_directory, path) + filename = setup["WriteInputNamesDict"]["fuel"] + end + + println(filename) fuels_in = load_dataframe(joinpath(my_dir, filename)) for nonfuel in ("None",) diff --git a/src/load_inputs/load_generators_variability.jl b/src/load_inputs/load_generators_variability.jl index 99294bffed..af71362209 100644 --- a/src/load_inputs/load_generators_variability.jl +++ b/src/load_inputs/load_generators_variability.jl @@ -8,9 +8,15 @@ function load_generators_variability!(setup::Dict, path::AbstractString, inputs: # Hourly capacity factors TDR_directory = joinpath(path, setup["TimeDomainReductionFolder"]) # if TDR is used, my_dir = TDR_directory, else my_dir = "system" - my_dir = get_systemfiles_path(setup, TDR_directory, path) + if setup["MultiStage"] == 1 + stage = setup["MultiStageSettingsDict"]["CurStage"] + filename = setup["WriteInputNamesDict"][string("inputs_p",stage)]["generators"] + else + + filename = setup["WriteInputNamesDict"]["generators"] + end - filename = "Generators_variability.csv" + my_dir = get_systemfiles_path(setup, TDR_directory, path) gen_var = load_dataframe(joinpath(my_dir, filename)) all_resources = inputs["RESOURCE_NAMES"] diff --git a/src/load_inputs/load_inputs.jl b/src/load_inputs/load_inputs.jl index aa1ee28a0e..81ea58c4c6 100644 --- a/src/load_inputs/load_inputs.jl +++ b/src/load_inputs/load_inputs.jl @@ -10,31 +10,48 @@ path - string path to working directory returns: Dict (dictionary) object containing all data inputs """ function load_inputs(setup::Dict, path::AbstractString) - ## Read input files println("Reading Input CSV Files") ## input paths - system_path = joinpath(path, setup["SystemFolder"]) - resources_path = joinpath(path, setup["ResourcesFolder"]) - policies_path = joinpath(path, setup["PoliciesFolder"]) + ## Declare Dict (dictionary) object used to store parameters inputs = Dict() - # Read input data about power network topology, operating and expansion attributes - if isfile(joinpath(system_path, "Network.csv")) - network_var = load_network_data!(setup, system_path, inputs) + + if setup["MultiStage"] == 0 + system_path = setup["WriteInputNamesDict"]["system_location"] + resources_path = setup["WriteInputNamesDict"]["resources_location"] + policies_path = setup["WriteInputNamesDict"]["policies_location"] + + # Read input data about power network topology, operating and expansion attributes + if isfile(joinpath(system_path,setup["WriteInputNamesDict"]["network"])) + network_var = load_network_data!(setup, system_path, inputs) + else + inputs["Z"] = 1 + inputs["L"] = 0 + end else - inputs["Z"] = 1 - inputs["L"] = 0 + stage = setup["MultiStageSettingsDict"]["CurStage"] + system_path = joinpath(path,setup["WriteInputNamesDict"][string("inputs_p", stage)]["system_location"]) + resources_path = joinpath(path,setup["WriteInputNamesDict"][string("inputs_p", stage)]["resources_location"]) + policies_path = joinpath(path,setup["WriteInputNamesDict"][string("inputs_p", stage)]["policies_location"]) + # Read input data about power network topology, operating and expansion attributes + if isfile(joinpath(system_path,setup["WriteInputNamesDict"][string("inputs_p", stage)]["network"])) + network_var = load_network_data!(setup, system_path, inputs) + else + inputs["Z"] = 1 + inputs["L"] = 0 + end end - # Read temporal-resolved load data, and clustering information if relevant - load_demand_data!(setup, path, inputs) - # Read fuel cost data, including time-varying fuel costs - load_fuels_data!(setup, path, inputs) - # Read in generator/resource related inputs - load_resources_data!(inputs, setup, path, resources_path) - # Read in generator/resource availability profiles - load_generators_variability!(setup, path, inputs) + # Read temporal-resolved load data, and clustering information if relevant + load_demand_data!(setup, system_path, inputs) + # Read fuel cost data, including time-varying fuel costs + load_fuels_data!(setup, system_path, inputs) + # Read in generator/resource related inputs + load_resources_data!(inputs, setup, path, resources_path) + # Read in generator/resource availability profiles + load_generators_variability!(setup,system_path, inputs) + validatetimebasis(inputs) @@ -99,7 +116,7 @@ function is_period_map_necessary(inputs::Dict) end function is_period_map_exist(setup::Dict, path::AbstractString) - filename = "Period_map.csv" + filename = setup["WriteResultsNamesDict"]["period_map"] is_in_system_dir = isfile(joinpath(path, setup["SystemFolder"], filename)) is_in_TDR_dir = isfile(joinpath(path, setup["TimeDomainReductionFolder"], filename)) is_in_system_dir || is_in_TDR_dir @@ -124,11 +141,21 @@ Returns: function get_systemfiles_path(setup::Dict, TDR_directory::AbstractString, path::AbstractString) - if setup["TimeDomainReduction"] == 1 && time_domain_reduced_files_exist(TDR_directory) - return TDR_directory + if setup["MultiStage"] == 1 + stage = setup["MultiStageSettingsDict"]["CurStage"] + if setup["TimeDomainReduction"] == 1 && time_domain_reduced_files_exist(TDR_directory, setup["WriteInputNamesDict"][string("inputs_p",stage)]) + return TDR_directory + else + # If TDR is not used, then use the "system" directory specified in the setup + return path + end else - # If TDR is not used, then use the "system" directory specified in the setup - return joinpath(path, setup["SystemFolder"]) + if setup["TimeDomainReduction"] == 1 && time_domain_reduced_files_exist(TDR_directory, setup) + return TDR_directory + else + # If TDR is not used, then use the "system" directory specified in the setup + return path + end end end diff --git a/src/load_inputs/load_maximum_capacity_requirement.jl b/src/load_inputs/load_maximum_capacity_requirement.jl index 95b766be8e..e84624b79f 100644 --- a/src/load_inputs/load_maximum_capacity_requirement.jl +++ b/src/load_inputs/load_maximum_capacity_requirement.jl @@ -4,7 +4,7 @@ Read input parameters related to maximum capacity requirement constraints (e.g. technology specific deployment mandates) """ function load_maximum_capacity_requirement!(path::AbstractString, inputs::Dict, setup::Dict) - filename = "Maximum_capacity_requirement.csv" + filename = setup["WriteInputNamesDict"]["max_cap"] df = load_dataframe(joinpath(path, filename)) inputs["NumberOfMaxCapReqs"] = nrow(df) inputs["MaxCapReq"] = df[!, :Max_MW] diff --git a/src/load_inputs/load_minimum_capacity_requirement.jl b/src/load_inputs/load_minimum_capacity_requirement.jl index e0561b126d..a5ce034233 100644 --- a/src/load_inputs/load_minimum_capacity_requirement.jl +++ b/src/load_inputs/load_minimum_capacity_requirement.jl @@ -4,7 +4,7 @@ Read input parameters related to minimum capacity requirement constraints (e.g. technology specific deployment mandates) """ function load_minimum_capacity_requirement!(path::AbstractString, inputs::Dict, setup::Dict) - filename = "Minimum_capacity_requirement.csv" + filename = setup["WriteInputNamesDict"]["min_cap"] df = load_dataframe(joinpath(path, filename)) NumberOfMinCapReqs = length(df[!, :MinCapReqConstraint]) inputs["NumberOfMinCapReqs"] = NumberOfMinCapReqs diff --git a/src/load_inputs/load_network_data.jl b/src/load_inputs/load_network_data.jl index ac7f2b1c8c..0efd878f1e 100644 --- a/src/load_inputs/load_network_data.jl +++ b/src/load_inputs/load_network_data.jl @@ -1,12 +1,17 @@ @doc raw""" - load_network_data!(setup::Dict, path::AbstractString, inputs_nw::Dict) + load_network_data!(setup::Dict, path::AbstractString, inputs_nw::Dict; stage::Int64 = 0) Function for reading input parameters related to the electricity transmission network """ function load_network_data!(setup::Dict, path::AbstractString, inputs_nw::Dict) scale_factor = setup["ParameterScale"] == 1 ? ModelScalingFactor : 1 - filename = "Network.csv" + if setup["MultiStage"] == 1 + stage = setup["MultiStageSettingsDict"]["CurStage"] + filename = setup["WriteInputNamesDict"][string("inputs_p",stage)]["network"] + else + filename = setup["WriteInputNamesDict"]["network"] + end network_var = load_dataframe(joinpath(path, filename)) as_vector(col::Symbol) = collect(skipmissing(network_var[!, col])) diff --git a/src/load_inputs/load_operational_reserves.jl b/src/load_inputs/load_operational_reserves.jl index 6b6d67cb78..a1acb7deb1 100644 --- a/src/load_inputs/load_operational_reserves.jl +++ b/src/load_inputs/load_operational_reserves.jl @@ -4,7 +4,7 @@ Read input parameters related to frequency regulation and operating reserve requirements """ function load_operational_reserves!(setup::Dict, path::AbstractString, inputs::Dict) - filename = "Operational_reserves.csv" + filename = setup["WriteInputNamesDict"]s["operational_reserves"] deprecated_synonym = "Reserves.csv" res_in = load_dataframe(path, [filename, deprecated_synonym]) diff --git a/src/load_inputs/load_period_map.jl b/src/load_inputs/load_period_map.jl index 3966ea03b3..f756e19c24 100644 --- a/src/load_inputs/load_period_map.jl +++ b/src/load_inputs/load_period_map.jl @@ -4,7 +4,7 @@ Read input parameters related to mapping of representative time periods to full chronological time series """ function load_period_map!(setup::Dict, path::AbstractString, inputs::Dict) - period_map = "Period_map.csv" + period_map = setup["WriteResultsNamesDict"]["period_map"] data_directory = joinpath(path, setup["TimeDomainReductionFolder"]) if setup["TimeDomainReduction"] == 1 && isfile(joinpath(data_directory, period_map)) # Use Time Domain Reduced data for GenX my_dir = data_directory diff --git a/src/load_inputs/load_resources_data.jl b/src/load_inputs/load_resources_data.jl index 7966038a0a..2b63a9e0ff 100644 --- a/src/load_inputs/load_resources_data.jl +++ b/src/load_inputs/load_resources_data.jl @@ -1,5 +1,5 @@ """ - _get_resource_info() + _get_resource_info(names::Dict) Internal function to get resource information (filename and GenX type) for each type of resource available in GenX. @@ -7,20 +7,20 @@ Internal function to get resource information (filename and GenX type) for each resource_info (NamedTuple): A tuple containing resource information. """ -function _get_resource_info() - resource_info = (hydro = (filename = "Hydro.csv", type = Hydro), - thermal = (filename = "Thermal.csv", type = Thermal), - vre = (filename = "Vre.csv", type = Vre), - storage = (filename = "Storage.csv", type = Storage), - flex_demand = (filename = "Flex_demand.csv", type = FlexDemand), - must_run = (filename = "Must_run.csv", type = MustRun), - electrolyzer = (filename = "Electrolyzer.csv", type = Electrolyzer), - vre_stor = (filename = "Vre_stor.csv", type = VreStorage)) +function _get_resource_info(names::Dict) + resource_info = (hydro = (filename = names["hydro"], type = Hydro), + thermal = (filename = names["thermal"], type = Thermal), + vre = (filename = names["vre"], type = Vre), + storage = (filename = names["storage"], type = Storage), + flex_demand = (filename = names["flex_demand"], type = FlexDemand), + must_run = (filename = names["must_run"], type = MustRun), + electrolyzer = (filename = names["electrolyzer"], type = Electrolyzer), + vre_stor = (filename = names["vre_stor"], type = VreStorage)) return resource_info end """ - _get_policyfile_info() + _get_policyfile_info(names::Dict) Internal function to get policy file information. @@ -28,14 +28,14 @@ Internal function to get policy file information. policyfile_info (NamedTuple): A tuple containing policy file information. """ -function _get_policyfile_info() +function _get_policyfile_info(names::Dict) # filename for each type of policy available in GenX - esr_filenames = ["Resource_energy_share_requirement.csv"] - cap_res_filenames = ["Resource_capacity_reserve_margin.csv"] - min_cap_filenames = ["Resource_minimum_capacity_requirement.csv"] - max_cap_filenames = ["Resource_maximum_capacity_requirement.csv"] - h2_demand_filenames = ["Resource_hydrogen_demand.csv"] - hourly_matching_filenames = ["Resource_hourly_matching.csv"] + h2_demand_filenames = [names["resource_hydrogen_demand"]] + hourly_matching_filenames = [names["resource_hourly_matching"]] + esr_filenames = [names["resource_energy_share_requirement"]] + cap_res_filenames = [names["resource_cap"]] + min_cap_filenames = [names["resource_min"]] + max_cap_filenames = [names["resource_max"]] policyfile_info = ( esr = (filenames = esr_filenames, setup_param = "EnergyShareRequirement"), @@ -495,7 +495,7 @@ function check_qualified_hydrogen_supply(r::AbstractResource) resource_name(r), " has :qualified_hydrogen_supply = 1. However \n" * "the :qualified_hydrogen_supply attribute is deprecated and will be removed in a future version. \n" * - "Please use the :qualified_supply column in the `Resource_hourly_matching.csv` file instead, and remove \n" * + "Please use the :qualified_supply column in the `Resource_hourly_matching` file instead, and remove \n" * "the :qualified_hydrogen_supply attribute from the resource file. \n" * "Please see the documentation for more information.") push!(warning_strings, e) @@ -593,7 +593,13 @@ function create_resource_array(setup::Dict, resources_path::AbstractString) scale_factor = setup["ParameterScale"] == 1 ? ModelScalingFactor : 1.0 # get filename and GenX type for each type of resources available in GenX - resources_info = _get_resource_info() + if setup["MultiStage"] == 0 + input_names = setup["WriteInputNamesDict"] + else + stage = setup["MultiStageSettingsDict"]["CurStage"] + input_names = setup["WriteInputNamesDict"][string("inputs_p",stage)] + end + resources_info = _get_resource_info(input_names) # load each resource type, scale data and return array of resources resources = create_resource_array(resources_path, resources_info, scale_factor) @@ -616,7 +622,13 @@ Validate the policy files by checking if they exist in the specified folder and !isfile(joinpath(resource_policies_path, filename)) """ function validate_policy_files(resource_policies_path::AbstractString, setup::Dict) - policyfile_info = _get_policyfile_info() + if setup["MultiStage"] == 0 + input_names = setup["WriteInputNamesDict"] + else + stage = setup["MultiStageSettingsDict"]["CurStage"] + input_names = setup["WriteInputNamesDict"][string("inputs_p",stage)] + end + policyfile_info = _get_policyfile_info(input_names) for (filenames, setup_param) in values(policyfile_info) if setup[setup_param] == 1 && any(!isfile(joinpath(resource_policies_path, filename)) @@ -745,18 +757,19 @@ function add_policy_to_resources!(resources::Vector{<:AbstractResource}, end """ - add_policies_to_resources!(resources::Vector{<:AbstractResource}, resources_path::AbstractString) + add_policies_to_resources!(resources::Vector{<:AbstractResource}, resources_path::AbstractString, input_names::Dict) Reads policy files and adds policies-related attributes to resources in the model. # Arguments - `resources::Vector{<:AbstractResource}`: Vector of resources in the model. - `resources_path::AbstractString`: The path to the resources folder. +- `input_names::Dict`: The names of the input files specified from the user, usually part of the setup """ function add_policies_to_resources!(resources::Vector{<:AbstractResource}, - resource_policy_path::AbstractString) + resource_policy_path::AbstractString,input_names::Dict) # get filename for each type of policy available in GenX - policies_info = _get_policyfile_info() + policies_info = _get_policyfile_info(input_names) # loop over policy files for (filenames, _) in values(policies_info) for filename in filenames @@ -807,7 +820,8 @@ function add_modules_to_resources!(resources::Vector{<:AbstractResource}, ## Load all modules and add them to the list of modules to be added to resources # Add multistage if multistage is activated if setup["MultiStage"] == 1 - filename = joinpath(resources_path, "Resource_multistage_data.csv") + stage = setup["MultiStageSettingsDict"]["CurStage"] + filename = joinpath(resources_path, setup["WriteInputNamesDict"][string("inputs_p",stage)]["resource_multistage_data"]) multistage_in = load_multistage_dataframe(filename, scale_factor) push!(modules, multistage_in) @info "Multistage data successfully read." @@ -1175,7 +1189,7 @@ function add_resources_to_input_data!(inputs::Dict, if !has_all_options_contributing(gen[retrofit_res], gen) && !has_all_options_not_contributing(gen[retrofit_res], gen) msg = "Retrofit options in the same cluster either all have Contribute_Min_Retirement set to 1 or none of them do. \n" * - "Check column Contribute_Min_Retirement in the \"Resource_multistage_data.csv\" file for resource $(resource_name(gen[retrofit_res]))." + "Check column Contribute_Min_Retirement in the \"Resource_multistage_data\" file for resource $(resource_name(gen[retrofit_res]))." @error msg error("Invalid input detected for Contribute_Min_Retirement.") end @@ -1233,7 +1247,7 @@ function add_resources_to_input_data!(inputs::Dict, if isempty(inputs["QUALIFIED_SUPPLY"]) && !isempty(ids_with(gen, qualified_hydrogen_supply)) Base.depwarn("""The column name :qualified_hydrogen_supply is deprecated. - Please use the `Resource_hourly_matching.csv` instead. The resource attribute + Please use the `Resource_hourly_matching` instead. The resource attribute :qualified_hydrogen_supply will be removed in the future release.""", :add_resources_to_input_data!, force = true) inputs["QUALIFIED_SUPPLY"] = ids_with(gen, qualified_hydrogen_supply) @@ -1438,8 +1452,12 @@ function load_resources_data!(inputs::Dict, # read policy files and add policies-related attributes to resource dataframe resource_policies_path = joinpath(resources_path, setup["ResourcePoliciesFolder"]) validate_policy_files(resource_policies_path, setup) - add_policies_to_resources!(resources, resource_policies_path) - + if setup["MultiStage"] == 1 + stage = setup["MultiStageSettingsDict"]["CurStage"] + add_policies_to_resources!(resources, resource_policies_path, setup["WriteInputNamesDict"][string("inputs_p",stage)]) + else + add_policies_to_resources!(resources, resource_policies_path, setup["WriteInputNamesDict"]) + end # read module files add module-related attributes to resource dataframe add_modules_to_resources!(resources, setup, resources_path) diff --git a/src/load_inputs/load_vre_stor_variability.jl b/src/load_inputs/load_vre_stor_variability.jl index 51ccc61097..4ad74b6af6 100644 --- a/src/load_inputs/load_vre_stor_variability.jl +++ b/src/load_inputs/load_vre_stor_variability.jl @@ -10,21 +10,26 @@ function load_vre_stor_variability!(setup::Dict, path::AbstractString, inputs::D # Hourly capacity factors TDR_directory = joinpath(path, setup["TimeDomainReductionFolder"]) # if TDR is used, my_dir = TDR_directory, else my_dir = "system" - my_dir = get_systemfiles_path(setup, TDR_directory, path) + if setup["MultiStage"] == 1 + stage = setup["MultiStageSettingsDict"]["CurStage"] + my_dir = get_systemfiles_path(setup, TDR_directory, path) + else + my_dir = get_systemfiles_path(setup, TDR_directory, path) + end - # Resource names all_resources = inputs["RESOURCE_NAMES"] # SOLAR VARIABILITY if !isempty(inputs["VS_SOLAR"]) - filename = "Vre_and_stor_solar_variability.csv" + + filename = setup["WriteInputNamesDict"]["vre_stor_solar_variability"] filepath = joinpath(my_dir, filename) load_process_variability!(filepath, all_resources, inputs, "Solar") end # WIND VARIABILITY if !isempty(inputs["VS_WIND"]) - filename = "Vre_and_stor_wind_variability.csv" + filename = setup["WriteInputNamesDict"]["vre_stor_wind_variability"] filepath = joinpath(my_dir, filename) load_process_variability!(filepath, all_resources, inputs, "Wind") end diff --git a/src/model/core/co2.jl b/src/model/core/co2.jl index 7b03d7920a..e93a0f1eca 100644 --- a/src/model/core/co2.jl +++ b/src/model/core/co2.jl @@ -13,7 +13,7 @@ fraction, and whether the feedstock is biomass. Biomass is a factor in this equa biomass generators are assumed to generate zero net CO2 emissions, or negative net CO2 emissions in the case that the CO2 they emit is captured and sequestered underground. -If a user wishes to represent a generator that combusts biomass, then in the resource .csv files, +If a user wishes to represent a generator that combusts biomass, then in the resource input files, the "Biomass" column (boolean, 1 or 0), which represents if a generator $y$ uses biomass or not, should be set to 1. The CO2 emissions from such a generator will be assumed to be zero without CCS and negative with CCS. diff --git a/src/model/core/fuel.jl b/src/model/core/fuel.jl index eac573e02d..f897be98b9 100644 --- a/src/model/core/fuel.jl +++ b/src/model/core/fuel.jl @@ -36,7 +36,7 @@ vFuel_{y,t} >= vP_{y,t} * h_{y,x} + U_{g,t}* f_{y,x} Where $h_{y,x}$ represents the heat rate slope for generator $y$ in segment $x$ [MMBTU/MWh], $f_{y,x}$ represents the heat rate intercept (MMBTU) for a generator $y$ in segment $x$ [MMBTU], and $U_{y,t}$ represents the commitment status of a generator $y$ at time $t$. These parameters -are optional inputs to the resource .csv files. +are optional inputs to the resource input files. When Unit commitment is on, if a user provides slope and intercept, the standard heat rate (i.e., Heat\_Rate\_MMBTU\_per\_MWh) will not be used. When unit commitment is off, the model will always use the standard heat rate. diff --git a/src/model/core/operational_reserves.jl b/src/model/core/operational_reserves.jl index 1790db3bb1..0e57acad6d 100644 --- a/src/model/core/operational_reserves.jl +++ b/src/model/core/operational_reserves.jl @@ -27,7 +27,7 @@ This function establishes several different versions of contingency reserve requ Contingency operational reserves represent requirements for upward ramping capability within a specified time frame to compensated for forced outages or unplanned failures of generators or transmission lines (e.g. N-1 contingencies). There are three options for the $Contingency$ expression, depending on user settings: - 1. a static contingency, in which the contingency requirement is set based on a fixed value (in MW) specified in the '''Operational_reserves.csv''' input file; + 1. a static contingency, in which the contingency requirement is set based on a fixed value (in MW) specified in the '''Operational_reserves''' input file; 2. a dynamic contingency based on installed capacity decisions, in which the largest 'installed' generator is used to determine the contingency requirement for all time periods; and 3. dynamic unit commitment based contingency, in which the largest 'committed' generator in any time period is used to determine the contingency requirement in that time period. @@ -81,7 +81,7 @@ function operational_reserves_contingency!(EP::Model, inputs::Dict, setup::Dict) ### Variables ### - # NOTE: If Dynamic_Contingency == 0, then contingency is a fixed parameter equal the value specified in Operational_reserves.csv via pStatic_Contingency. + # NOTE: If Dynamic_Contingency == 0, then contingency is a fixed parameter equal the value specified in Operational_reserves via pStatic_Contingency. if UCommit == 1 && pDynamic_Contingency == 1 # Contingency = largest installed thermal unit @variable(EP, vLARGEST_CONTINGENCY>=0) diff --git a/src/model/policies/hourly_matching.jl b/src/model/policies/hourly_matching.jl index 283edfd1d6..0f72db5990 100644 --- a/src/model/policies/hourly_matching.jl +++ b/src/model/policies/hourly_matching.jl @@ -2,7 +2,7 @@ hourly_matching!(EP::Model, inputs::Dict) This module defines the hourly matching policy constraint. -This constraint can be enabled by setting `HourlyMatching==1` in `genx_settings.yml`) requires generation from qualified resources ($y \in \mathcal{Qualified}$, indicated by `Qualified_Supply==1` in the `Resource_hourly_matching.csv` files) to be >= hourly consumption from electrolyzers in the zone and any charging by qualified storage within the zone used to help increase electrolyzer utilization: +This constraint can be enabled by setting `HourlyMatching==1` in `genx_settings.yml`) requires generation from qualified resources ($y \in \mathcal{Qualified}$, indicated by `Qualified_Supply==1` in the `Resource_hourly_matching` files) to be >= hourly consumption from electrolyzers in the zone and any charging by qualified storage within the zone used to help increase electrolyzer utilization: ```math \begin{aligned} diff --git a/src/model/policies/hydrogen_demand.jl b/src/model/policies/hydrogen_demand.jl index 27efc42e36..1f136ac180 100644 --- a/src/model/policies/hydrogen_demand.jl +++ b/src/model/policies/hydrogen_demand.jl @@ -8,7 +8,7 @@ The hydrogen demand requirement can be defined as a zonal limit defined in terms **Minimum annual hydrogen production** **Zonal limit** -The sum of annual hydrogen production by each electrolyzer $y \in \mathcal{EL}$ must exceed a minimum quantity specified in inputs in "Hydrogen_demand.csv": +The sum of annual hydrogen production by each electrolyzer $y \in \mathcal{EL}$ must exceed a minimum quantity specified in inputs in "Hydrogen_demand": ```math \begin{aligned} diff --git a/src/model/resources/storage/storage.jl b/src/model/resources/storage/storage.jl index 8d19b8ae3d..db95c3fd78 100644 --- a/src/model/resources/storage/storage.jl +++ b/src/model/resources/storage/storage.jl @@ -77,7 +77,7 @@ When modeling the entire year as a single chronological period with total number Alternatively, when modeling the entire year with multiple representative periods, this constraint relates storage inventory in the first timestep of the representative period with the inventory at the last time step of the representative period, where each representative period is made of $\tau^{period}$ time steps. In this implementation, energy exchange between representative periods is not permitted. When modeling representative time periods, GenX enables modeling of long duration energy storage which tracks state of charge (and state of charge held in reserve, if a capacity reserve margin is being modeled) between representative periods enable energy to be moved throughout the year. -If there is more than one representative period and ```LDS``` has been enabled for resources in ```Generators.csv```, this function calls ```long_duration_storage()``` in ```long_duration_storage.jl``` to enable this feature. +If there is more than one representative period and ```LDS``` has been enabled for resources in ```Generators```, this function calls ```long_duration_storage()``` in ```long_duration_storage.jl``` to enable this feature. The next constraint limits the volume of energy stored at any time, $\Gamma_{o,z,t}$, to be less than the installed energy storage capacity, $\Delta^{total, energy}_{o,z}$. Finally, the maximum combined discharge and virtual discharge rate for storage resources, $\Pi_{o,z,t} + \Pi^{CRM}_{o,z,t}$, is constrained to be less than the discharge power capacity, $\Omega_{o,z,t}$ or the state of charge at the end of the last period, $\Gamma_{o,z,t-1}$, whichever is less. ```math diff --git a/src/model/resources/vre_stor/vre_stor.jl b/src/model/resources/vre_stor/vre_stor.jl index 2e93bbc024..96b4158846 100644 --- a/src/model/resources/vre_stor/vre_stor.jl +++ b/src/model/resources/vre_stor/vre_stor.jl @@ -916,7 +916,7 @@ The following two constraints track the state of charge of the storage resources this constraint relates storage inventory in the first timestep of the representative period with the inventory at the last time step of the representative period, where each representative period is made of $\tau^{period}$ time steps. In this implementation, energy exchange between representative periods is not permitted. When modeling representative time periods, GenX enables modeling of long duration energy storage which tracks state of charge between representative periods enable energy to be moved throughout the year. If there is more than one representative period and ```LDS_VRE_STOR=1``` has been enabled for - resources in ```Vre_and_stor_data.csv```, this function calls ```lds_vre_stor!()``` to enable this feature. The first of these two constraints enforces storage inventory balance for interior time + resources in ```Vre_and_stor_data```, this function calls ```lds_vre_stor!()``` to enable this feature. The first of these two constraints enforces storage inventory balance for interior time steps $(t \in \mathcal{T}^{interior})$, while the second enforces storage balance constraint for the initial time step $(t \in \mathcal{T}^{start})$: ```math \begin{aligned} @@ -1498,7 +1498,7 @@ end This function defines the decision variables, expressions, and constraints for any long duration energy storage component of each co-located VRE and storage generator ( - there is more than one representative period and ```LDS_VRE_STOR=1``` in the ```Vre_and_stor_data.csv```). + there is more than one representative period and ```LDS_VRE_STOR=1``` in the ```Vre_and_stor_data```). These constraints follow the same formulation that is outlined by the function ```long_duration_storage!()``` in the storage module. One constraint changes, which links the state of charge between the start of periods diff --git a/src/multi_stage/configure_multi_stage_inputs.jl b/src/multi_stage/configure_multi_stage_inputs.jl index a1c887e02a..2812eba4a0 100644 --- a/src/multi_stage/configure_multi_stage_inputs.jl +++ b/src/multi_stage/configure_multi_stage_inputs.jl @@ -9,7 +9,7 @@ For each resource $y \in \mathcal{G}$ with annualized investment cost $AIC_{y}$ & OCC_{y} = \sum^{min(CRP_{y},H)}_{i=1}\frac{AIC_{y}}{(1+WACC_{y})^{i}} \end{aligned} ``` -where $WACC_y$ is the technology-specific weighted average cost of capital (set by the "WACC" field in the Generators\_data.csv or Network.csv files), $H$ is the number of years remaining between the start of the current model stage and the model horizon (the end of the final model stage) and $CRP_y$ is the capital recovery period for technology $y$ (specified in Generators\_data.csv). +where $WACC_y$ is the technology-specific weighted average cost of capital (set by the "WACC" field in the Generators\_data or Network.csv files), $H$ is the number of years remaining between the start of the current model stage and the model horizon (the end of the final model stage) and $CRP_y$ is the capital recovery period for technology $y$ (specified in Generators\_data.csv). inputs: @@ -70,7 +70,7 @@ This function overwrites input parameters read in via the load\_inputs() method 3) Internal set representations of resources eligible for capacity retirements are overwritten to ensure compatability with multi-stage modeling. -4) When NetworkExpansion is active and there are multiple model zones, parameters related to transmission and network expansion are updated. First, annualized transmission reinforcement costs are converted into overnight capital costs. Next, the maximum allowable transmission line reinforcement parameter is overwritten by the model stage-specific value specified in the "Line\_Max\_Flow\_Possible\_MW" fields in the network\_multi\_stage.csv file. Finally, internal representations of lines eligible or not eligible for transmission expansion are overwritten based on the updated maximum allowable transmission line reinforcement parameters. +4) When NetworkExpansion is active and there are multiple model zones, parameters related to transmission and network expansion are updated. First, annualized transmission reinforcement costs are converted into overnight capital costs. Next, the maximum allowable transmission line reinforcement parameter is overwritten by the model stage-specific value specified in the "Line\_Max\_Flow\_Possible\_MW" fields in the network\_multi\_stage file. Finally, internal representations of lines eligible or not eligible for transmission expansion are overwritten based on the updated maximum allowable transmission line reinforcement parameters. inputs: diff --git a/src/multi_stage/write_multi_stage_capacities_charge.jl b/src/multi_stage/write_multi_stage_capacities_charge.jl index a9d7f4cf11..b4839ee27c 100644 --- a/src/multi_stage/write_multi_stage_capacities_charge.jl +++ b/src/multi_stage/write_multi_stage_capacities_charge.jl @@ -29,5 +29,7 @@ function write_multi_stage_capacities_charge(outpath::String, settings_d::Dict) df_cap[!, Symbol("EndChargeCap_p$p")] = capacities_d[p][!, :EndChargeCap] end - CSV.write(joinpath(outpath, "capacities_charge_multi_stage.csv"), df_cap) + CSV.write(joinpath(outpath, setup["WriteResultsNamesDict"]["capacities_charge_multi_stage"]), df_cap) + #write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["capacities_charge_multi_stage"]),df_cap, filetype = setup["ResultsFileType"], compression = setup["ResultsCompressionType"]) + end diff --git a/src/multi_stage/write_multi_stage_capacities_discharge.jl b/src/multi_stage/write_multi_stage_capacities_discharge.jl index 0da02b7002..f4c81a9b6b 100644 --- a/src/multi_stage/write_multi_stage_capacities_discharge.jl +++ b/src/multi_stage/write_multi_stage_capacities_discharge.jl @@ -29,5 +29,7 @@ function write_multi_stage_capacities_discharge(outpath::String, settings_d::Dic df_cap[!, Symbol("EndCap_p$p")] = capacities_d[p][!, :EndCap] end - CSV.write(joinpath(outpath, "capacities_multi_stage.csv"), df_cap) + CSV.write(joinpath(outpath, setup["WriteResultsNamesDict"]["capacities_multi_stage"]), df_cap) + #write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["capacities_multi_stage"]),df_cap, filetype = setup["ResultsFileType"], compression = setup["ResultsCompressionType"]) + end diff --git a/src/multi_stage/write_multi_stage_capacities_energy.jl b/src/multi_stage/write_multi_stage_capacities_energy.jl index 9c7a5c1567..5a1618fef9 100644 --- a/src/multi_stage/write_multi_stage_capacities_energy.jl +++ b/src/multi_stage/write_multi_stage_capacities_energy.jl @@ -29,5 +29,7 @@ function write_multi_stage_capacities_energy(outpath::String, settings_d::Dict) df_cap[!, Symbol("EndEnergyCap_p$p")] = capacities_d[p][!, :EndEnergyCap] end - CSV.write(joinpath(outpath, "capacities_energy_multi_stage.csv"), df_cap) + CSV.write(joinpath(outpath, setup["WriteResultsNamesDict"]["capacities_energy_multi_stage"]), df_cap) + #write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["capacities_energy_multi_stage"]),df_cap, filetype = setup["ResultsFileType"], compression = setup["ResultsCompressionType"]) + end diff --git a/src/multi_stage/write_multi_stage_costs.jl b/src/multi_stage/write_multi_stage_costs.jl index 92a8c9a71c..115aab6249 100644 --- a/src/multi_stage/write_multi_stage_costs.jl +++ b/src/multi_stage/write_multi_stage_costs.jl @@ -54,4 +54,6 @@ function write_multi_stage_costs(outpath::String, settings_d::Dict, inputs_dict: @warn("The cost calculation of the multi-stage GenX is approximate currently, and we will be refining it more in one of the future releases.") CSV.write(joinpath(outpath, "costs_multi_stage.csv"), df_costs) + #write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["costs_multi_stage"]),df_costs, filetype = setup["ResultsFileType"], compression = setup["ResultsCompressionType"]) + end diff --git a/src/multi_stage/write_multi_stage_network_expansion.jl b/src/multi_stage/write_multi_stage_network_expansion.jl index 1a6ddc7015..4c270bd626 100644 --- a/src/multi_stage/write_multi_stage_network_expansion.jl +++ b/src/multi_stage/write_multi_stage_network_expansion.jl @@ -27,5 +27,7 @@ function write_multi_stage_network_expansion(outpath::String, settings_d::Dict) :New_Trans_Capacity] end - CSV.write(joinpath(outpath, "network_expansion_multi_stage.csv"), df_trans_cap) + CSV.write(joinpath(outpath, setup["WriteResultsNamesDict"]["network_expansion_multi_stage"]), df_trans_cap) + #write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["network_expansion_multi_stage"]),df_trans_cap, filetype = setup["ResultsFileType"], compression = setup["ResultsCompressionType"]) + end diff --git a/src/startup/genx_startup.jl b/src/startup/genx_startup.jl index 76ff8fdbc2..ec05c35af9 100644 --- a/src/startup/genx_startup.jl +++ b/src/startup/genx_startup.jl @@ -71,7 +71,7 @@ end function _precompile_tdr(case) settings_path = get_settings_path(case) setup = configure_settings(get_settings_path(case, "genx_settings.yml"), - get_settings_path(case, "output_settings.yml")) + get_settings_path(case, "output_settings.yml"), case) setup["TimeDomainReduction"] = 1 # Enable TDR for precompilation cluster_inputs(case, settings_path, setup) isdir("precompile/case/TDR_results") && @@ -81,5 +81,5 @@ end # Precompile `run_genx_case!` and `TDR` unless the environment variable `GENX_PRECOMPILE` is set to `false` if get(ENV, "GENX_PRECOMPILE", "true") != "false" - _precompile() + #_precompile() end diff --git a/src/time_domain_reduction/full_time_series_reconstruction.jl b/src/time_domain_reduction/full_time_series_reconstruction.jl index 768f3ba0b1..b03984f7f3 100644 --- a/src/time_domain_reduction/full_time_series_reconstruction.jl +++ b/src/time_domain_reduction/full_time_series_reconstruction.jl @@ -24,7 +24,7 @@ function full_time_series_reconstruction( TDRpath = joinpath(case, setup["TimeDomainReductionFolder"]) end # Read Period map file Period_map.csv - Period_map = CSV.read(joinpath(TDRpath, "Period_map.csv"), DataFrame) + Period_map = CSV.read(joinpath(TDRpath, setup["WriteResultsNamesDict"]["period_map"]), DataFrame) # Read time domain reduction settings file time_domain_reduction_settings.yml myTDRsetup = YAML.load(open(joinpath( @@ -55,7 +55,7 @@ function full_time_series_reconstruction( end recon = [recon recon_col] end - reconDF = DataFrame(recon, :auto) + reconDF = DataFrame(recon, names(DF)) # Insert rows that were above "t1" in the original DataFrame (e.g. "Zone" and "AnnualSum") if present for i in range(1, t1 - 1) diff --git a/src/time_domain_reduction/precluster.jl b/src/time_domain_reduction/precluster.jl index 7e3db51988..3f49d776c7 100644 --- a/src/time_domain_reduction/precluster.jl +++ b/src/time_domain_reduction/precluster.jl @@ -30,7 +30,7 @@ function run_timedomainreduction_multistage!(case::AbstractString) mysetup = configure_settings(genx_settings) multistage_settings = get_settings_path(case, "multi_stage_settings.yml") - mysetup["MultiStageSettingsDict"] = configure_settings_multistage(multistage_settings) + mysetup["MultiStageSettingsDict"] = configure_settings_multistage(case,multistage_settings) tdr_settings = get_settings_path(case, "time_domain_reduction_settings.yml") TDRSettingsDict = YAML.load(open(tdr_settings)) diff --git a/src/time_domain_reduction/time_domain_reduction.jl b/src/time_domain_reduction/time_domain_reduction.jl index 36aa7ecb6d..376a306e8f 100644 --- a/src/time_domain_reduction/time_domain_reduction.jl +++ b/src/time_domain_reduction/time_domain_reduction.jl @@ -673,16 +673,22 @@ function cluster_inputs(inpath, MultiStage = mysetup["MultiStage"] if MultiStage == 1 + stage = mysetup["MultiStageSettingsDict"]["CurStage"] MultiStageConcatenate = myTDRsetup["MultiStageConcatenate"] NumStages = mysetup["MultiStageSettingsDict"]["NumStages"] + Demand_Outfile = joinpath(TimeDomainReductionFolder, mysetup["WriteInputNamesDict"][string("inputs_p",stage)]["demand"]) + GVar_Outfile = joinpath(TimeDomainReductionFolder, mysetup["WriteInputNamesDict"][string("inputs_p",stage)]["generators"]) + Fuel_Outfile = joinpath(TimeDomainReductionFolder, mysetup["WriteInputNamesDict"][string("inputs_p",stage)]["fuel"]) + PMap_Outfile = joinpath(TimeDomainReductionFolder, mysetup["WriteInputNamesDict"][string("inputs_p",stage)]["period_map"]) + YAML_Outfile = joinpath(TimeDomainReductionFolder, "time_domain_reduction_settings.yml") + else + Demand_Outfile = joinpath(TimeDomainReductionFolder, mysetup["WriteInputNamesDict"]["demand"]) + GVar_Outfile = joinpath(TimeDomainReductionFolder, mysetup["WriteInputNamesDict"]["generators"]) + Fuel_Outfile = joinpath(TimeDomainReductionFolder, mysetup["WriteInputNamesDict"]["fuel"]) + PMap_Outfile = joinpath(TimeDomainReductionFolder, mysetup["WriteInputNamesDict"]["period_map"]) + YAML_Outfile = joinpath(TimeDomainReductionFolder, "time_domain_reduction_settings.yml") end - Demand_Outfile = joinpath(TimeDomainReductionFolder, "Demand_data.csv") - GVar_Outfile = joinpath(TimeDomainReductionFolder, "Generators_variability.csv") - Fuel_Outfile = joinpath(TimeDomainReductionFolder, "Fuels_data.csv") - PMap_Outfile = joinpath(TimeDomainReductionFolder, "Period_map.csv") - YAML_Outfile = joinpath(TimeDomainReductionFolder, "time_domain_reduction_settings.yml") - # Define a local version of the setup so that you can modify the mysetup["ParameterScale"] value to be zero in case it is 1 mysetup_local = copy(mysetup) # If ParameterScale =1 then make it zero, since clustered inputs will be scaled prior to generating model @@ -703,14 +709,12 @@ function cluster_inputs(inpath, mysetup["MultiStageSettingsDict"]["CurStage"] = t # Step 1) Load Inputs - global inpath_sub = string("$inpath/inputs/inputs_p", t) # this prevents doubled time domain reduction in stages past # the first, even if the first stage is okay. - prevent_doubled_timedomainreduction(joinpath(inpath_sub, - mysetup["SystemFolder"])) + prevent_doubled_timedomainreduction(joinpath(inpath,mysetup["WriteInputNamesDict"][string("inputs_p",t)]["system_location"]),mysetup["WriteInputNamesDict"][string("inputs_p",t)]) - inputs_dict[t] = load_inputs(mysetup_MS, inpath_sub) + inputs_dict[t] = load_inputs(mysetup_MS, inpath) inputs_dict[t] = configure_multi_stage_inputs(inputs_dict[t], mysetup["MultiStageSettingsDict"], @@ -1217,9 +1221,9 @@ function cluster_inputs(inpath, Stage_PeriodMaps = Dict() Stage_Outfiles = Dict() SolarVar_Outfile = joinpath(TimeDomainReductionFolder, - "Vre_and_stor_solar_variability.csv") + mysetup["WriteInputNamesDict"]["vre_stor_solar_variability"]) WindVar_Outfile = joinpath(TimeDomainReductionFolder, - "Vre_and_stor_wind_variability.csv") + mysetup["WriteInputNamesDict"]["vre_stor_wind_variability"]) for per in 1:NumStages # Iterate over multi-stages mkpath(joinpath(inpath, "inputs", @@ -1253,7 +1257,8 @@ function cluster_inputs(inpath, ### TDR_Results/Demand_data_clustered.csv demand_in = get_demand_dataframe( joinpath(inpath, "inputs", "inputs_p$per"), - mysetup["SystemFolder"]) + mysetup["SystemFolder"] + ) demand_in[!, :Sub_Weights] = demand_in[!, :Sub_Weights] * 1.0 demand_in[1:length(Stage_Weights[per]), :Sub_Weights] .= Stage_Weights[per] demand_in[!, :Rep_Periods][1] = length(Stage_Weights[per]) @@ -1276,8 +1281,9 @@ function cluster_inputs(inpath, if v println("Writing demand file...") end - CSV.write(joinpath(inpath, "inputs", Stage_Outfiles[per]["Demand"]), - demand_in) + + write_output_file(joinpath(inpath, "inputs", Stage_Outfiles[per]["Demand"]), demand_in) + ### TDR_Results/Generators_variability.csv # Reset column ordering, add time index, and solve duplicate column name trouble with CSV.write's header kwarg @@ -1290,9 +1296,8 @@ function cluster_inputs(inpath, if v println("Writing resource file...") end - CSV.write(joinpath(inpath, "inputs", Stage_Outfiles[per]["GVar"]), - GVOutputData, - header = NewGVColNames) + write_output_file(joinpath(inpath, "inputs", Stage_Outfiles[per]["GVar"]),GVOutputData) + if !isempty(inputs_dict[per]["VRE_STOR"]) gen_var = load_dataframe(joinpath(inpath, @@ -1326,11 +1331,11 @@ function cluster_inputs(inpath, solar_var[!, :Time_Index] = 1:size(solar_var, 1) wind_var = gen_var[!, wind_col_names] wind_var[!, :Time_Index] = 1:size(wind_var, 1) + + write_output_file(joinpath(inpath, "inputs", Stage_Outfiles[per]["GSolar"]), solar_var) + write_output_file(joinpath(inpath, "inputs", Stage_Outfiles[per]["GWind"]), wind_var) + - CSV.write(joinpath(inpath, "inputs", Stage_Outfiles[per]["GSolar"]), - solar_var) - CSV.write(joinpath(inpath, "inputs", Stage_Outfiles[per]["GWind"]), - wind_var) end ### TDR_Results/Fuels_data.csv @@ -1338,8 +1343,9 @@ function cluster_inputs(inpath, "inputs", "inputs_p$per", mysetup["SystemFolder"], - "Fuels_data.csv")) + mysetup["WriteInputNamesDict"]["fuel"])) ensure_column!(fuel_in, "None", 0.0) + select!(fuel_in, Not(:Time_Index)) SepFirstRow = DataFrame(fuel_in[1, :]) NewFuelOutput = vcat(SepFirstRow, FPOutputData) @@ -1348,15 +1354,15 @@ function cluster_inputs(inpath, if v println("Writing fuel profiles...") end - CSV.write(joinpath(inpath, "inputs", Stage_Outfiles[per]["Fuel"]), - NewFuelOutput) + + write_output_file(joinpath(inpath, "inputs", Stage_Outfiles[per]["Fuel"]), NewFuelOutput) + ### TDR_Results/Period_map.csv if v println("Writing period map...") end - CSV.write(joinpath(inpath, "inputs", Stage_Outfiles[per]["PMap"]), - Stage_PeriodMaps[per]) + write_output_file(joinpath(inpath, "inputs", Stage_Outfiles[per]["PMap"]),Stage_PeriodMaps[per]) ### TDR_Results/time_domain_reduction_settings.yml if v @@ -1383,7 +1389,9 @@ function cluster_inputs(inpath, demand_in = get_demand_dataframe(joinpath(inpath, "inputs", input_stage_directory, - mysetup["SystemFolder"])) + mysetup["SystemFolder"]), + mysetup["WriteInputNamesDict"][string("inputs_p",stage_id)] + ) demand_in[!, :Sub_Weights] = demand_in[!, :Sub_Weights] * 1.0 demand_in[1:length(W), :Sub_Weights] .= W demand_in[!, :Rep_Periods][1] = length(W) @@ -1406,8 +1414,9 @@ function cluster_inputs(inpath, if v println("Writing demand file...") end - CSV.write(joinpath(inpath, "inputs", input_stage_directory, Demand_Outfile), - demand_in) + + write_output_file(joinpath(inpath, "inputs", input_stage_directory, Demand_Outfile),demand_in) + ### TDR_Results/Generators_variability.csv @@ -1421,9 +1430,9 @@ function cluster_inputs(inpath, if v println("Writing resource file...") end - CSV.write(joinpath(inpath, "inputs", input_stage_directory, GVar_Outfile), - GVOutputData, - header = NewGVColNames) + + write_output_file(joinpath(inpath, "inputs", input_stage_directory, GVar_Outfile),GVOutputData) + # Break up VRE-storage components if needed if !isempty(myinputs["VRE_STOR"]) @@ -1461,21 +1470,14 @@ function cluster_inputs(inpath, wind_var[!, :Time_Index] = 1:size(wind_var, 1) SolarVar_Outfile = joinpath(TimeDomainReductionFolder, - "Vre_and_stor_solar_variability.csv") + mysetup["WriteInputNamesDict"]["vre_stor_solar_variability"]) WindVar_Outfile = joinpath(TimeDomainReductionFolder, - "Vre_and_stor_wind_variability.csv") - CSV.write( - joinpath(inpath, - "inputs", - input_stage_directory, - SolarVar_Outfile), - solar_var) - CSV.write( - joinpath(inpath, - "inputs", - input_stage_directory, - WindVar_Outfile), - wind_var) + mysetup["WriteInputNamesDict"]["vre_stor_wind_variability"]) + + write_output_file(joinpath(inpath, "inputs", input_stage_directory, SolarVar_Outfile),solar_var) + write_output_file(joinpath(inpath, "inputs", input_stage_directory, WindVar_Outfile),wind_var) + + end ### TDR_Results/Fuels_data.csv @@ -1484,8 +1486,9 @@ function cluster_inputs(inpath, "inputs", input_stage_directory, mysetup["SystemFolder"], - "Fuels_data.csv")) + mysetup["WriteInputNamesDict"][string("inputs_p",stage_id)]["fuel"])) ensure_column!(fuel_in, "None", 0.0) + select!(fuel_in, Not(:Time_Index)) SepFirstRow = DataFrame(fuel_in[1, :]) NewFuelOutput = vcat(SepFirstRow, FPOutputData) @@ -1494,15 +1497,15 @@ function cluster_inputs(inpath, if v println("Writing fuel profiles...") end - CSV.write(joinpath(inpath, "inputs", input_stage_directory, Fuel_Outfile), - NewFuelOutput) + + write_output_file(joinpath(inpath, "inputs", input_stage_directory, Fuel_Outfile),NewFuelOutput) + ### Period_map.csv if v println("Writing period map...") end - CSV.write(joinpath(inpath, "inputs", input_stage_directory, PMap_Outfile), - PeriodMap) + write_output_file(joinpath(inpath, "inputs", input_stage_directory, PMap_Outfile),PeriodMap) ### time_domain_reduction_settings.yml if v @@ -1520,7 +1523,7 @@ function cluster_inputs(inpath, ### TDR_Results/Demand_data.csv system_path = joinpath(inpath, mysetup["SystemFolder"]) - demand_in = get_demand_dataframe(system_path) + demand_in = get_demand_dataframe(system_path,mysetup) demand_in[!, :Sub_Weights] = demand_in[!, :Sub_Weights] * 1.0 demand_in[1:length(W), :Sub_Weights] .= W demand_in[!, :Rep_Periods][1] = length(W) @@ -1542,7 +1545,8 @@ function cluster_inputs(inpath, if v println("Writing demand file...") end - CSV.write(joinpath(inpath, Demand_Outfile), demand_in) + write_output_file(joinpath(inpath, Demand_Outfile),demand_in) + ### TDR_Results/Generators_variability.csv @@ -1556,8 +1560,9 @@ function cluster_inputs(inpath, if v println("Writing resource file...") end - CSV.write(joinpath(inpath, GVar_Outfile), GVOutputData, header = NewGVColNames) - + path = joinpath(inpath, GVar_Outfile) + file_extension = splitext(path) + write_output_file(joinpath(inpath, GVar_Outfile), GVOutputData) # Break up VRE-storage components if needed if !isempty(myinputs["VRE_STOR"]) gen_var = load_dataframe(joinpath(inpath, GVar_Outfile)) @@ -1591,16 +1596,17 @@ function cluster_inputs(inpath, wind_var[!, :Time_Index] = 1:size(wind_var, 1) SolarVar_Outfile = joinpath(TimeDomainReductionFolder, - "Vre_and_stor_solar_variability.csv") + mysetup["WriteInputNamesDict"]["vre_stor_solar_variability"]) WindVar_Outfile = joinpath(TimeDomainReductionFolder, - "Vre_and_stor_wind_variability.csv") - CSV.write(joinpath(inpath, SolarVar_Outfile), solar_var) - CSV.write(joinpath(inpath, WindVar_Outfile), wind_var) + mysetup["WriteInputNamesDict"]["vre_stor_wind_variability"]) + write_output_file(joinpath(inpath, SolarVar_Outfile), solar_var) + write_output_file(joinpath(inpath, WindVar_Outfile), wind_var) + end ### TDR_Results/Fuels_data.csv system_path = joinpath(inpath, mysetup["SystemFolder"]) - fuel_in = load_dataframe(joinpath(system_path, "Fuels_data.csv")) + fuel_in = load_dataframe(joinpath(system_path, mysetup["WriteInputNamesDict"]["fuel"])) ensure_column!(fuel_in, "None", 0.0) select!(fuel_in, Not(:Time_Index)) SepFirstRow = DataFrame(fuel_in[1, :]) @@ -1610,13 +1616,14 @@ function cluster_inputs(inpath, if v println("Writing fuel profiles...") end - CSV.write(joinpath(inpath, Fuel_Outfile), NewFuelOutput) + path = joinpath(inpath, Fuel_Outfile) + write_output_file(joinpath(inpath, Fuel_Outfile), NewFuelOutput) ### TDR_Results/Period_map.csv if v println("Writing period map...") end - CSV.write(joinpath(inpath, PMap_Outfile), PeriodMap) + write_output_file(joinpath(inpath, PMap_Outfile), PeriodMap) ### TDR_Results/time_domain_reduction_settings.yml if v diff --git a/src/write_outputs/capacity_reserve_margin/write_capacity_value.jl b/src/write_outputs/capacity_reserve_margin/write_capacity_value.jl index b1aa431bb6..5ecaf8587b 100644 --- a/src/write_outputs/capacity_reserve_margin/write_capacity_value.jl +++ b/src/write_outputs/capacity_reserve_margin/write_capacity_value.jl @@ -13,7 +13,7 @@ This is the value of the derated capacities of different types of resources mult - EP::Model: EnergyModel object. # Results -- A CSV file named "CapacityValue.csv" is written to the directory specified by `path`. +- A file named "CapacityValue" is written to the directory specified by `path`. """ function write_capacity_value(path::AbstractString, inputs::Dict, setup::Dict, EP::Model) gen = inputs["RESOURCES"] @@ -147,7 +147,10 @@ function write_capacity_value(path::AbstractString, inputs::Dict, setup::Dict, E rename!(temp_dfCapValue, auxNew_Names) append!(dfCapValue, temp_dfCapValue) end - write_simple_csv(joinpath(path, "CapacityValue.csv"), dfCapValue) + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["capacity_value"]), + dfCapValue, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) end @doc raw""" diff --git a/src/write_outputs/capacity_reserve_margin/write_reserve_margin.jl b/src/write_outputs/capacity_reserve_margin/write_reserve_margin.jl index 1eeca0ef0e..461be087da 100644 --- a/src/write_outputs/capacity_reserve_margin/write_reserve_margin.jl +++ b/src/write_outputs/capacity_reserve_margin/write_reserve_margin.jl @@ -4,6 +4,9 @@ function write_reserve_margin(path::AbstractString, setup::Dict, EP::Model) temp_ResMar = temp_ResMar * ModelScalingFactor # Convert from MillionUS$/GWh to US$/MWh end dfResMar = DataFrame(temp_ResMar, :auto) - CSV.write(joinpath(path, "ReserveMargin.csv"), dfResMar) + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["reserve_margin"]), + dftranspose(dfResMar,false), + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) return nothing end diff --git a/src/write_outputs/capacity_reserve_margin/write_reserve_margin_revenue.jl b/src/write_outputs/capacity_reserve_margin/write_reserve_margin_revenue.jl index 418b14eeab..f007a33f6b 100644 --- a/src/write_outputs/capacity_reserve_margin/write_reserve_margin_revenue.jl +++ b/src/write_outputs/capacity_reserve_margin/write_reserve_margin_revenue.jl @@ -91,6 +91,10 @@ function write_reserve_margin_revenue(path::AbstractString, dfResRevenue = hcat(dfResRevenue, DataFrame([tempresrev], [Symbol("CapRes_$i")])) end dfResRevenue.AnnualSum = annual_sum - CSV.write(joinpath(path, "ReserveMarginRevenue.csv"), dfResRevenue) + + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["reserve_margin_revenue"]), + dfResRevenue, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) return dfResRevenue end diff --git a/src/write_outputs/capacity_reserve_margin/write_reserve_margin_slack.jl b/src/write_outputs/capacity_reserve_margin/write_reserve_margin_slack.jl index f6d71e4fb4..103531287f 100644 --- a/src/write_outputs/capacity_reserve_margin/write_reserve_margin_slack.jl +++ b/src/write_outputs/capacity_reserve_margin/write_reserve_margin_slack.jl @@ -14,7 +14,10 @@ function write_reserve_margin_slack(path::AbstractString, end if setup["WriteOutputs"] == "annual" - CSV.write(joinpath(path, "ReserveMargin_prices_and_penalties.csv"), dfResMar_slack) + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["reserve_margin_prices_and_penalties"]), + dfResMar_slack, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) else # setup["WriteOutputs"] == "full" temp_ResMar_slack = value.(EP[:vCapResSlack]) if setup["ParameterScale"] == 1 @@ -22,9 +25,10 @@ function write_reserve_margin_slack(path::AbstractString, end dfResMar_slack = hcat(dfResMar_slack, DataFrame(temp_ResMar_slack, [Symbol("t$t") for t in 1:T])) - CSV.write(joinpath(path, "ReserveMargin_prices_and_penalties.csv"), + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["reserve_margin_prices_and_penalties"]), dftranspose(dfResMar_slack, false), - writeheader = false) + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) end return nothing end diff --git a/src/write_outputs/capacity_reserve_margin/write_reserve_margin_w.jl b/src/write_outputs/capacity_reserve_margin/write_reserve_margin_w.jl index 74b4efa7fe..70d04b7b5c 100644 --- a/src/write_outputs/capacity_reserve_margin/write_reserve_margin_w.jl +++ b/src/write_outputs/capacity_reserve_margin/write_reserve_margin_w.jl @@ -1,7 +1,8 @@ function write_reserve_margin_w(path::AbstractString, inputs::Dict, setup::Dict, EP::Model) T = inputs["T"] # Number of time steps (hours) #dfResMar dataframe with weights included for calculations - dfResMar_w = DataFrame(Constraint = [Symbol("t$t") for t in 1:T]) + # Maya: make t column type String for DuckDB + dfResMar_w = DataFrame(Constraint = [String("t$t") for t in 1:T]) temp_ResMar_w = transpose(dual.(EP[:cCapacityResMargin])) ./ inputs["omega"] if setup["ParameterScale"] == 1 temp_ResMar_w = temp_ResMar_w * ModelScalingFactor # Convert from MillionUS$/GWh to US$/MWh @@ -10,5 +11,8 @@ function write_reserve_margin_w(path::AbstractString, inputs::Dict, setup::Dict, auxNew_Names_res = [Symbol("Constraint"); [Symbol("CapRes_$i") for i in 1:inputs["NCapacityReserveMargin"]]] rename!(dfResMar_w, auxNew_Names_res) - CSV.write(joinpath(path, "ReserveMargin_w.csv"), dfResMar_w) + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["reserve_margin_w"]), + dfResMar_w, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) end diff --git a/src/write_outputs/capacity_reserve_margin/write_virtual_discharge.jl b/src/write_outputs/capacity_reserve_margin/write_virtual_discharge.jl index 1aa52623de..4978d0af4e 100644 --- a/src/write_outputs/capacity_reserve_margin/write_virtual_discharge.jl +++ b/src/write_outputs/capacity_reserve_margin/write_virtual_discharge.jl @@ -19,11 +19,11 @@ function write_virtual_discharge(path::AbstractString, inputs::Dict, setup::Dict dfVirtualDischarge = DataFrame(Resource = resources, Zone = zones) dfVirtualDischarge.AnnualSum .= virtual_discharge * inputs["omega"] - filepath = joinpath(path, "virtual_discharge.csv") + filepath = joinpath(path, setup["WriteResultsNamesDict"]["virtual_discharge"]) if setup["WriteOutputs"] == "annual" - write_annual(filepath, dfVirtualDischarge) + write_annual(filepath, dfVirtualDischarge, setup) else # setup["WriteOutputs"] == "full" - write_fulltimeseries(filepath, virtual_discharge, dfVirtualDischarge) + write_fulltimeseries(filepath, virtual_discharge, dfVirtualDischarge, setup) end return nothing end diff --git a/src/write_outputs/co2_cap/write_co2_cap.jl b/src/write_outputs/co2_cap/write_co2_cap.jl index 19cba87d71..fdeaec0180 100644 --- a/src/write_outputs/co2_cap/write_co2_cap.jl +++ b/src/write_outputs/co2_cap/write_co2_cap.jl @@ -5,8 +5,9 @@ Function for reporting carbon price associated with carbon cap constraints. """ function write_co2_cap(path::AbstractString, inputs::Dict, setup::Dict, EP::Model) + # Maya: Changed Symbol to string for CO2 Cap labels dfCO2Price = DataFrame( - CO2_Cap = [Symbol("CO2_Cap_$cap") for cap in 1:inputs["NCO2Cap"]], + CO2_Cap = [String("CO2_Cap_$cap") for cap in 1:inputs["NCO2Cap"]], CO2_Price = (-1) * (dual.(EP[:cCO2Emissions_systemwide]))) if setup["ParameterScale"] == 1 dfCO2Price.CO2_Price .*= ModelScalingFactor # Convert Million$/kton to $/ton @@ -20,7 +21,10 @@ function write_co2_cap(path::AbstractString, inputs::Dict, setup::Dict, EP::Mode end end - CSV.write(joinpath(path, "CO2_prices_and_penalties.csv"), dfCO2Price) + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["co2_prices"]), + dfCO2Price, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) return nothing end diff --git a/src/write_outputs/dftranspose.jl b/src/write_outputs/dftranspose.jl index ec9d1a5f39..014b4702e9 100644 --- a/src/write_outputs/dftranspose.jl +++ b/src/write_outputs/dftranspose.jl @@ -18,9 +18,10 @@ Returns a transpose of a Dataframe. function dftranspose(df::DataFrame, withhead::Bool) if withhead colnames = cat(:Row, Symbol.(df[!, 1]), dims = 1) - return DataFrame([[names(df)]; collect.(eachrow(df))], colnames) + df_new = df[:,2:end] + return DataFrame([[names(df_new)]; collect.(eachrow(df_new))], colnames) else return DataFrame([[names(df)]; collect.(eachrow(df))], - [:Row; Symbol.("x", axes(df, 1))]) + [:Row1; Symbol.("x", axes(df, 1))]) end end # End dftranpose() diff --git a/src/write_outputs/energy_share_requirement/write_esr_prices.jl b/src/write_outputs/energy_share_requirement/write_esr_prices.jl index e9cccc46ae..077f1ee4fe 100644 --- a/src/write_outputs/energy_share_requirement/write_esr_prices.jl +++ b/src/write_outputs/energy_share_requirement/write_esr_prices.jl @@ -12,6 +12,10 @@ function write_esr_prices(path::AbstractString, inputs::Dict, setup::Dict, EP::M dfESR[!, :ESR_AnnualPenalty] *= (ModelScalingFactor^2) # Converting MillionUSD to USD end end - CSV.write(joinpath(path, "ESR_prices_and_penalties.csv"), dfESR) + + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["esr_prices_and_penalties"]), + dfESR, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) return dfESR end diff --git a/src/write_outputs/energy_share_requirement/write_esr_revenue.jl b/src/write_outputs/energy_share_requirement/write_esr_revenue.jl index 580ee45837..ce5ff57aff 100644 --- a/src/write_outputs/energy_share_requirement/write_esr_revenue.jl +++ b/src/write_outputs/energy_share_requirement/write_esr_revenue.jl @@ -65,6 +65,9 @@ function write_esr_revenue(path::AbstractString, end end dfESRRev.Total = sum(eachcol(dfESRRev[:, 6:(nESR + 5)])) - CSV.write(joinpath(path, "ESR_Revenue.csv"), dfESRRev) + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["esr_revenue"]), + dfESRRev, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) return dfESRRev end diff --git a/src/write_outputs/hydrogen/write_hourly_matching_prices.jl b/src/write_outputs/hydrogen/write_hourly_matching_prices.jl index 92fe82c0b6..e427e08a6c 100644 --- a/src/write_outputs/hydrogen/write_hourly_matching_prices.jl +++ b/src/write_outputs/hydrogen/write_hourly_matching_prices.jl @@ -16,8 +16,10 @@ function write_hourly_matching_prices(path::AbstractString, auxNew_Names = [Symbol("Zone"); [Symbol("t$t") for t in 1:T]] rename!(dfHourlyMatchPrices, auxNew_Names) - CSV.write(joinpath(path, "hourly_matching_prices.csv"), - dftranspose(dfHourlyMatchPrices, false), writeheader = false) + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["hourly_matching_prices"]), + dftranspose(dfHourlyMatchPrices, false), + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) return nothing end diff --git a/src/write_outputs/hydrogen/write_hydrogen_prices.jl b/src/write_outputs/hydrogen/write_hydrogen_prices.jl index 6701931967..cfc23f800d 100644 --- a/src/write_outputs/hydrogen/write_hydrogen_prices.jl +++ b/src/write_outputs/hydrogen/write_hydrogen_prices.jl @@ -6,7 +6,10 @@ function write_hydrogen_prices(path::AbstractString, inputs::Dict, setup::Dict, H2_Demand = [Symbol("H2_Demand_$h2demand") for h2demand in 1:NumberOfH2DemandReqs], Hydrogen_Price_Per_Tonne = convert( Array{Float64}, dual.(EP[:cZoneH2DemandReq]) * scale_factor)) - CSV.write(joinpath(path, "hydrogen_prices.csv"), dfHydrogenPrice) + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["hydrogen_prices"]), + dfHydrogenPrice, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) return nothing end diff --git a/src/write_outputs/long_duration_storage/write_opwrap_lds_dstor.jl b/src/write_outputs/long_duration_storage/write_opwrap_lds_dstor.jl index 875d8e6f86..afb81e84d7 100644 --- a/src/write_outputs/long_duration_storage/write_opwrap_lds_dstor.jl +++ b/src/write_outputs/long_duration_storage/write_opwrap_lds_dstor.jl @@ -26,7 +26,8 @@ function write_opwrap_lds_dstor(path::AbstractString, inputs::Dict, setup::Dict, dfdStorage = hcat(dfdStorage, DataFrame(dsoc, :auto)) auxNew_Names = [Symbol("Resource"); Symbol("Zone"); [Symbol("w$t") for t in 1:W]] rename!(dfdStorage, auxNew_Names) - CSV.write(joinpath(path, "dStorage.csv"), + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["dStorage"]), dftranspose(dfdStorage, false), - header = false) + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) end diff --git a/src/write_outputs/long_duration_storage/write_opwrap_lds_stor_init.jl b/src/write_outputs/long_duration_storage/write_opwrap_lds_stor_init.jl index e0a731d177..8ce72a8d98 100644 --- a/src/write_outputs/long_duration_storage/write_opwrap_lds_stor_init.jl +++ b/src/write_outputs/long_duration_storage/write_opwrap_lds_stor_init.jl @@ -29,9 +29,10 @@ function write_opwrap_lds_stor_init(path::AbstractString, dfStorageInit = hcat(dfStorageInit, DataFrame(socw, :auto)) auxNew_Names = [Symbol("Resource"); Symbol("Zone"); [Symbol("n$t") for t in 1:NPeriods]] rename!(dfStorageInit, auxNew_Names) - CSV.write(joinpath(path, "StorageInit.csv"), + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["storage_init"]), dftranspose(dfStorageInit, false), - header = false) + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) # Write storage evolution over full time horizon hours_per_subperiod = inputs["hours_per_subperiod"]; @@ -80,6 +81,7 @@ function write_opwrap_lds_stor_init(path::AbstractString, df_SOC_t = hcat(df_SOC_t, DataFrame(SOC_t, :auto)) auxNew_Names = [Symbol("Resource"); Symbol("Zone"); [Symbol("n$t") for t in 1:T_hor]] rename!(df_SOC_t,auxNew_Names) - CSV.write(joinpath(path, "StorageEvol.csv"), dftranspose(df_SOC_t, false), writeheader=false) - + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["storage_evol"]), + dftranspose(df_SOC_t, false), + filetype = setup["ResultsFileType"], compression = setup["ResultsCompressionType"]) end diff --git a/src/write_outputs/min_max_capacity_requirement/write_maximum_capacity_requirement.jl b/src/write_outputs/min_max_capacity_requirement/write_maximum_capacity_requirement.jl index 8d1b3450ee..a45087cf51 100644 --- a/src/write_outputs/min_max_capacity_requirement/write_maximum_capacity_requirement.jl +++ b/src/write_outputs/min_max_capacity_requirement/write_maximum_capacity_requirement.jl @@ -3,8 +3,9 @@ function write_maximum_capacity_requirement(path::AbstractString, setup::Dict, EP::Model) NumberOfMaxCapReqs = inputs["NumberOfMaxCapReqs"] + # Maya: Changed Symbol to string for CO2 Cap labels dfMaxCapPrice = DataFrame( - Constraint = [Symbol("MaxCapReq_$maxcap") + Constraint = [String("MaxCapReq_$maxcap") for maxcap in 1:NumberOfMaxCapReqs], Price = -dual.(EP[:cZoneMaxCapReq])) @@ -18,5 +19,9 @@ function write_maximum_capacity_requirement(path::AbstractString, dfMaxCapPrice.Slack *= scale_factor # Convert GW to MW dfMaxCapPrice.Penalty *= scale_factor^2 # Convert Million $ to $ end - CSV.write(joinpath(path, "MaxCapReq_prices_and_penalties.csv"), dfMaxCapPrice) + + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["maxcap"]), + dfMaxCapPrice, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) end diff --git a/src/write_outputs/min_max_capacity_requirement/write_minimum_capacity_requirement.jl b/src/write_outputs/min_max_capacity_requirement/write_minimum_capacity_requirement.jl index bae7d17ee9..d258e17cbd 100644 --- a/src/write_outputs/min_max_capacity_requirement/write_minimum_capacity_requirement.jl +++ b/src/write_outputs/min_max_capacity_requirement/write_minimum_capacity_requirement.jl @@ -3,8 +3,9 @@ function write_minimum_capacity_requirement(path::AbstractString, setup::Dict, EP::Model) NumberOfMinCapReqs = inputs["NumberOfMinCapReqs"] + # Maya: Changed Symbol to string for CO2 Cap labels dfMinCapPrice = DataFrame( - Constraint = [Symbol("MinCapReq_$mincap") + Constraint = [String("MinCapReq_$mincap") for mincap in 1:NumberOfMinCapReqs], Price = dual.(EP[:cZoneMinCapReq])) @@ -18,5 +19,9 @@ function write_minimum_capacity_requirement(path::AbstractString, dfMinCapPrice.Slack *= scale_factor # Convert GW to MW dfMinCapPrice.Penalty *= scale_factor^2 # Convert Million $ to $ end - CSV.write(joinpath(path, "MinCapReq_prices_and_penalties.csv"), dfMinCapPrice) + + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["mincap"]), + dfMinCapPrice, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) end diff --git a/src/write_outputs/reserves/write_operating_reserve_price_revenue.jl b/src/write_outputs/reserves/write_operating_reserve_price_revenue.jl index 79ab9b8cbe..9a79b1e94b 100644 --- a/src/write_outputs/reserves/write_operating_reserve_price_revenue.jl +++ b/src/write_outputs/reserves/write_operating_reserve_price_revenue.jl @@ -45,8 +45,14 @@ function write_operating_reserve_regulation_revenue(path::AbstractString, dfOpRsvRevenue.AnnualSum .= rsvrevenue * inputs["omega"] dfOpRegRevenue.AnnualSum .= regrevenue * inputs["omega"] - write_simple_csv(joinpath(path, "OperatingReserveRevenue.csv"), dfOpRsvRevenue) - write_simple_csv(joinpath(path, "OperatingRegulationRevenue.csv"), dfOpRegRevenue) + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["op_reserve_revenue"]), + dfOpRsvRevenue, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["op_regulation_revenue"]), + dfOpRegRevenue, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) return dfOpRegRevenue, dfOpRsvRevenue end diff --git a/src/write_outputs/reserves/write_reg.jl b/src/write_outputs/reserves/write_reg.jl index 7d7ca1efd6..16e8f09334 100644 --- a/src/write_outputs/reserves/write_reg.jl +++ b/src/write_outputs/reserves/write_reg.jl @@ -10,11 +10,11 @@ function write_reg(path::AbstractString, inputs::Dict, setup::Dict, EP::Model) dfReg = DataFrame(Resource = resources, Zone = zones) dfReg.AnnualSum = reg * inputs["omega"] - filepath = joinpath(path, "reg.csv") + filepath = joinpath(path,setup["WriteResultsNamesDict"]["reg"]) if setup["WriteOutputs"] == "annual" - write_annual(filepath, dfReg) + write_annual(filepath, dfReg, setup) else # setup["WriteOutputs"] == "full" - write_fulltimeseries(filepath, reg, dfReg) + write_fulltimeseries(filepath, reg, dfReg, setup) end return nothing end diff --git a/src/write_outputs/reserves/write_rsv.jl b/src/write_outputs/reserves/write_rsv.jl index 19ed31aadc..dc7fb8eeaa 100644 --- a/src/write_outputs/reserves/write_rsv.jl +++ b/src/write_outputs/reserves/write_rsv.jl @@ -12,7 +12,7 @@ function write_rsv(path::AbstractString, inputs::Dict, setup::Dict, EP::Model) dfRsv.AnnualSum = rsv * inputs["omega"] if setup["WriteOutputs"] == "annual" - write_annual(joinpath(path, "reserves.csv"), dfRsv) + write_annual(joinpath(path, setup["WriteResultsNamesDict"]["reserves"]), dfRsv, setup) else # setup["WriteOutputs"] == "full" unmet_vec = value.(EP[:vUNMET_RSV]) * scale_factor total_unmet = sum(unmet_vec) @@ -30,8 +30,9 @@ function write_rsv(path::AbstractString, inputs::Dict, setup::Dict, EP::Model) rename!(total, auxNew_Names) rename!(unmet, auxNew_Names) dfRsv = vcat(dfRsv, unmet, total) - CSV.write(joinpath(path, "reserves.csv"), + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["reserves"]), dftranspose(dfRsv, false), - writeheader = false) + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) end end diff --git a/src/write_outputs/transmission/write_nw_expansion.jl b/src/write_outputs/transmission/write_nw_expansion.jl index f89e1bfe1f..e970b5dd1c 100644 --- a/src/write_outputs/transmission/write_nw_expansion.jl +++ b/src/write_outputs/transmission/write_nw_expansion.jl @@ -19,5 +19,8 @@ function write_nw_expansion(path::AbstractString, inputs::Dict, setup::Dict, EP: dfTransCap.Cost_Trans_Capacity *= ModelScalingFactor^2 # MUSD to USD end - CSV.write(joinpath(path, "network_expansion.csv"), dfTransCap) + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["network_expansion"]), + dfTransCap, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) end diff --git a/src/write_outputs/transmission/write_transmission_flows.jl b/src/write_outputs/transmission/write_transmission_flows.jl index 5290d71afe..c29d8d108c 100644 --- a/src/write_outputs/transmission/write_transmission_flows.jl +++ b/src/write_outputs/transmission/write_transmission_flows.jl @@ -12,20 +12,27 @@ function write_transmission_flows(path::AbstractString, flow *= ModelScalingFactor end - filepath = joinpath(path, "flow.csv") + filepath = joinpath(path, setup["WriteResultsNamesDict"]["flow"]) if setup["WriteOutputs"] == "annual" dfFlow.AnnualSum = flow * inputs["omega"] total = DataFrame(["Total" sum(dfFlow.AnnualSum)], [:Line, :AnnualSum]) dfFlow = vcat(dfFlow, total) - CSV.write(filepath, dfFlow) + write_output_file(filepath, + dfFlow, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) else # setup["WriteOutputs"] == "full" dfFlow = hcat(dfFlow, DataFrame(flow, :auto)) auxNew_Names = [Symbol("Line"); [Symbol("t$t") for t in 1:T]] rename!(dfFlow, auxNew_Names) - CSV.write(filepath, dftranspose(dfFlow, false), writeheader = false) + + write_output_file(filepath, + dftranspose(dfFlow, true), + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1 - write_full_time_series_reconstruction(path, setup, dfFlow, "flow") + write_full_time_series_reconstruction(path, setup, dftranspose(dfFlow, true), setup["WriteResultsNamesDict"]["flow"]) @info("Writing Full Time Series for Transmission Flows") end end diff --git a/src/write_outputs/transmission/write_transmission_losses.jl b/src/write_outputs/transmission/write_transmission_losses.jl index a76bca1180..9eca72fc41 100644 --- a/src/write_outputs/transmission/write_transmission_losses.jl +++ b/src/write_outputs/transmission/write_transmission_losses.jl @@ -18,7 +18,10 @@ function write_transmission_losses(path::AbstractString, if setup["WriteOutputs"] == "annual" total = DataFrame(["Total" sum(dfTLosses.AnnualSum)], [:Line, :AnnualSum]) dfTLosses = vcat(dfTLosses, total) - CSV.write(joinpath(path, "tlosses.csv"), dfTLosses) + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["tlosses"]), + dfTLosses, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) else dfTLosses = hcat(dfTLosses, DataFrame(tlosses, :auto)) auxNew_Names = [Symbol("Line"); Symbol("AnnualSum"); [Symbol("t$t") for t in 1:T]] @@ -27,12 +30,14 @@ function write_transmission_losses(path::AbstractString, auxNew_Names) total[:, 3:(T + 2)] .= sum(tlosses, dims = 1) dfTLosses = vcat(dfTLosses, total) - CSV.write(joinpath(path, "tlosses.csv"), - dftranspose(dfTLosses, false), - writeheader = false) + + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["tlosses"]), + dftranspose(dfTLosses, true), + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1 - write_full_time_series_reconstruction(path, setup, dfTLosses, "tlosses") + write_full_time_series_reconstruction(path, setup, dftranspose(dfTLosses, true), setup["WriteResultsNamesDict"]["tlosses"]) @info("Writing Full Time Series for Time Losses") end end diff --git a/src/write_outputs/ucommit/write_commit.jl b/src/write_outputs/ucommit/write_commit.jl new file mode 100644 index 0000000000..b175a24d55 --- /dev/null +++ b/src/write_outputs/ucommit/write_commit.jl @@ -0,0 +1,27 @@ +function write_commit(path::AbstractString, inputs::Dict, setup::Dict, EP::Model) + COMMIT = inputs["COMMIT"] + T = inputs["T"] + + # Commitment state for each resource in each time step + resources = inputs["RESOURCE_NAMES"][COMMIT] + zones = inputs["R_ZONES"][COMMIT] + zones = convert.(Float64,zones) + commit = value.(EP[:vCOMMIT][COMMIT, :].data) + dfCommit = DataFrame(Resource = resources, Zone = zones) + dfCommit = hcat(dfCommit, DataFrame(commit, :auto)) + auxNew_Names = [Symbol("Resource"); Symbol("Zone"); [Symbol("t$t") for t in 1:T]] + rename!(dfCommit, auxNew_Names) + + dfCommit = dftranspose(dfCommit,true) + + write_output_file(joinpath(path, + setup["WriteResultsNamesDict"]["commit"]), + dfCommit, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) + + if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1 + write_full_time_series_reconstruction(path, setup, dfCommit, setup["WriteResultsNamesDict"]["commit"]) + @info("Writing Full Time Series for Commitment") + end +end diff --git a/src/write_outputs/ucommit/write_shutdown.jl b/src/write_outputs/ucommit/write_shutdown.jl new file mode 100644 index 0000000000..68e650edbe --- /dev/null +++ b/src/write_outputs/ucommit/write_shutdown.jl @@ -0,0 +1,24 @@ +function write_shutdown(path::AbstractString, inputs::Dict, setup::Dict, EP::Model) + # Operational decision variable states + COMMIT = inputs["COMMIT"] + zones = inputs["R_ZONES"][COMMIT] + zones = convert.(Float64,zones) + # Shutdown state for each resource in each time step + shut = value.(EP[:vSHUT][COMMIT, :].data) + resources = inputs["RESOURCE_NAMES"][COMMIT] + + dfShutdown = DataFrame(Resource = resources, Zone = zones) + dfShutdown.AnnualSum = shut * inputs["omega"] + + filepath = joinpath(path,setup["WriteResultsNamesDict"]["shutdown"]) + if setup["WriteOutputs"] == "annual" + write_annual(filepath, dfShutdown, setup) + else # setup["WriteOutputs"] == "full" + df_Shutdown = write_fulltimeseries(filepath, shut, dfShutdown, setup) + if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1 + write_full_time_series_reconstruction(path, setup, df_Shutdown, setup["WriteResultsNamesDict"]["shutdown"]) + @info("Writing Full Time Series for Shutdown") + end + end + return nothing +end diff --git a/src/write_outputs/ucommit/write_start.jl b/src/write_outputs/ucommit/write_start.jl new file mode 100644 index 0000000000..28d9812368 --- /dev/null +++ b/src/write_outputs/ucommit/write_start.jl @@ -0,0 +1,23 @@ +function write_start(path::AbstractString, inputs::Dict, setup::Dict, EP::Model) + COMMIT = inputs["COMMIT"] + # Startup state for each resource in each time step + resources = inputs["RESOURCE_NAMES"][COMMIT] + zones = inputs["R_ZONES"][COMMIT] + zones = convert.(Float64,zones) + + dfStart = DataFrame(Resource = resources, Zone = zones) + start = value.(EP[:vSTART][COMMIT, :].data) + dfStart.AnnualSum = start * inputs["omega"] + + filepath = joinpath(path, setup["WriteResultsNamesDict"]["start"]) + if setup["WriteOutputs"] == "annual" + write_annual(filepath, dfStart, setup) + else # setup["WriteOutputs"] == "full" + df_Start = write_fulltimeseries(filepath, start, dfStart, setup) + if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1 + write_full_time_series_reconstruction(path, setup, df_Start, setup["WriteResultsNamesDict"]["start"]) + @info("Writing Full Time Series for Startup") + end + end + return nothing +end diff --git a/src/write_outputs/write_angles.jl b/src/write_outputs/write_angles.jl index b93870354f..37d6fa865a 100644 --- a/src/write_outputs/write_angles.jl +++ b/src/write_outputs/write_angles.jl @@ -15,8 +15,9 @@ function write_angles(path::AbstractString, inputs::Dict, setup::Dict, EP::Model rename!(dfAngles, auxNew_Names) ## Linear configuration final output - CSV.write(joinpath(path, "angles.csv"), - dftranspose(dfAngles, false), - writeheader = false) + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["angles"]), + dftranspose(dfAngles, false), + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) return nothing end diff --git a/src/write_outputs/write_capacity.jl b/src/write_outputs/write_capacity.jl index 99e4797ecc..9b40aa72d8 100755 --- a/src/write_outputs/write_capacity.jl +++ b/src/write_outputs/write_capacity.jl @@ -112,11 +112,11 @@ function write_capacity(path::AbstractString, inputs::Dict, setup::Dict, EP::Mod dfCap.NewChargeCap = dfCap.NewChargeCap * ModelScalingFactor dfCap.EndChargeCap = dfCap.EndChargeCap * ModelScalingFactor end - total = DataFrame(Resource = "Total", Zone = "n/a", Retrofit_Id = "n/a", + total = DataFrame(Resource = "Total", Zone = missing, Retrofit_Id = missing, StartCap = sum(dfCap[!, :StartCap]), RetCap = sum(dfCap[!, :RetCap]), NewCap = sum(dfCap[!, :NewCap]), EndCap = sum(dfCap[!, :EndCap]), RetroCap = sum(dfCap[!, :RetroCap]), - CapacityConstraintDual = "n/a", + CapacityConstraintDual = missing, StartEnergyCap = sum(dfCap[!, :StartEnergyCap]), RetEnergyCap = sum(dfCap[!, :RetEnergyCap]), NewEnergyCap = sum(dfCap[!, :NewEnergyCap]), @@ -127,6 +127,6 @@ function write_capacity(path::AbstractString, inputs::Dict, setup::Dict, EP::Mod EndChargeCap = sum(dfCap[!, :EndChargeCap])) dfCap = vcat(dfCap, total) - CSV.write(joinpath(path, "capacity.csv"), dfCap) + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["capacity"]), dfCap, filetype = setup["ResultsFileType"], compression = setup["ResultsCompressionType"]) return dfCap end diff --git a/src/write_outputs/write_capacityfactor.jl b/src/write_outputs/write_capacityfactor.jl index d7e5f13c6b..b4b2d06276 100644 --- a/src/write_outputs/write_capacityfactor.jl +++ b/src/write_outputs/write_capacityfactor.jl @@ -74,7 +74,7 @@ function write_capacityfactor(path::AbstractString, inputs::Dict, setup::Dict, E sum(weight) end - CSV.write(joinpath(path, "capacityfactor.csv"), df) + #write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["capacity_factor"]), df.CapacityFactor, filetype = setup["ResultsFileType"], compression = setup["ResultsCompressionType"]) return nothing end diff --git a/src/write_outputs/write_charge.jl b/src/write_outputs/write_charge.jl index 89d95fbb2e..5709a705ea 100644 --- a/src/write_outputs/write_charge.jl +++ b/src/write_outputs/write_charge.jl @@ -7,6 +7,7 @@ function write_charge(path::AbstractString, inputs::Dict, setup::Dict, EP::Model gen = inputs["RESOURCES"] # Resources (objects) resources = inputs["RESOURCE_NAMES"] # Resource names zones = zone_id.(gen) + zones = convert.(Float64,zones) T = inputs["T"] # Number of time steps (hours) STOR_ALL = inputs["STOR_ALL"] @@ -51,6 +52,6 @@ function write_charge(path::AbstractString, inputs::Dict, setup::Dict, EP::Model Zone = zones[charge_ids]) df.AnnualSum = charge * weight - write_temporal_data(df, charge, path, setup, "charge") + write_temporal_data(df, charge, path, setup, setup["WriteResultsNamesDict"]["charge"]) return nothing end diff --git a/src/write_outputs/write_charging_cost.jl b/src/write_outputs/write_charging_cost.jl index cabc4db135..dab27c784b 100644 --- a/src/write_outputs/write_charging_cost.jl +++ b/src/write_outputs/write_charging_cost.jl @@ -50,6 +50,9 @@ function write_charging_cost(path::AbstractString, inputs::Dict, setup::Dict, EP AnnualSum = Array{Float64}(undef, G)) dfChargingcost.AnnualSum .= chargecost * weight - write_simple_csv(joinpath(path, "ChargingCost.csv"), dfChargingcost) + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["charging_cost"]), + dfChargingcost, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) return dfChargingcost end diff --git a/src/write_outputs/write_co2.jl b/src/write_outputs/write_co2.jl index 5a2860d197..360f65e161 100644 --- a/src/write_outputs/write_co2.jl +++ b/src/write_outputs/write_co2.jl @@ -17,6 +17,7 @@ function write_co2_emissions_plant(path::AbstractString, gen = inputs["RESOURCES"] # Resources (objects) resources = inputs["RESOURCE_NAMES"] # Resource names zones = zone_id.(gen) + zones = convert.(Float64,zones) G = inputs["G"] # Number of resources (generators, storage, DR, and DERs) @@ -31,7 +32,7 @@ function write_co2_emissions_plant(path::AbstractString, AnnualSum = zeros(G)) df.AnnualSum .= emissions_plant * weight - write_temporal_data(df, emissions_plant, path, setup, "emissions_plant") + write_temporal_data(df, emissions_plant, path, setup, setup["WriteResultsNamesDict"]["emissions"]) return nothing end @@ -53,9 +54,9 @@ function write_co2_capture_plant(path::AbstractString, inputs::Dict, setup::Dict emissions_captured_plant *= scale_factor df.AnnualSum .= emissions_captured_plant * weight - write_temporal_data( - df, emissions_captured_plant, path, setup, "captured_emissions_plant") + df, emissions_captured_plant, path, setup, setup["WriteResultsNamesDict"]["captured_emissions_plant"]) + end return nothing end diff --git a/src/write_outputs/write_costs.jl b/src/write_outputs/write_costs.jl index 9193865911..295535c64e 100644 --- a/src/write_outputs/write_costs.jl +++ b/src/write_outputs/write_costs.jl @@ -314,13 +314,13 @@ function write_costs(path::AbstractString, inputs::Dict, setup::Dict, EP::Model) tempCFuel, tempCNSE, tempCStart, - "-", - "-", - "-", + missing, + missing, + missing, tempCCO2 ] if !isempty(VRE_STOR) - push!(temp_cost_list, "-") + push!(temp_cost_list, missing) end if !isempty(ELECTROLYZER_ALL) push!(temp_cost_list, tempHydrogenValue) @@ -328,5 +328,6 @@ function write_costs(path::AbstractString, inputs::Dict, setup::Dict, EP::Model) dfCost[!, Symbol("Zone$z")] = temp_cost_list end - CSV.write(joinpath(path, "costs.csv"), dfCost) + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["costs"]), dfCost, filetype = setup["ResultsFileType"], compression = setup["ResultsCompressionType"]) + end diff --git a/src/write_outputs/write_curtailment.jl b/src/write_outputs/write_curtailment.jl index 4d41289750..0dc70fa950 100644 --- a/src/write_outputs/write_curtailment.jl +++ b/src/write_outputs/write_curtailment.jl @@ -8,6 +8,7 @@ function write_curtailment(path::AbstractString, inputs::Dict, setup::Dict, EP:: gen = inputs["RESOURCES"] # Resources (objects) resources = inputs["RESOURCE_NAMES"] # Resource names zones = zone_id.(gen) + zones = convert.(Float64,zones) G = inputs["G"] # Number of resources (generators, storage, DR, and DERs) T = inputs["T"] # Number of time steps (hours) @@ -55,6 +56,6 @@ function write_curtailment(path::AbstractString, inputs::Dict, setup::Dict, EP:: AnnualSum = zeros(G)) df.AnnualSum = curtailment * weight - write_temporal_data(df, curtailment, path, setup, "curtailment") + write_temporal_data(df, curtailment, path, setup, setup["WriteResultsNamesDict"]["curtail"]) return nothing end diff --git a/src/write_outputs/write_emissions.jl b/src/write_outputs/write_emissions.jl index 2e0c011f68..ca64a2c0e9 100644 --- a/src/write_outputs/write_emissions.jl +++ b/src/write_outputs/write_emissions.jl @@ -35,6 +35,7 @@ function write_emissions(path::AbstractString, inputs::Dict, setup::Dict, EP::Mo [Symbol("CO2_Price_$cap") for cap in 1:inputs["NCO2Cap"]]; Symbol("AnnualSum")] rename!(dfEmissions, auxNew_Names) + dfEmissions[!,1] = convert.(Float64,dfEmissions[!,1]) else dfEmissions = DataFrame(Zone = 1:Z, AnnualSum = Array{Float64}(undef, Z)) end @@ -55,7 +56,11 @@ function write_emissions(path::AbstractString, inputs::Dict, setup::Dict, EP::Mo :AnnualSum]) end dfEmissions = vcat(dfEmissions, total) - CSV.write(joinpath(path, "emissions.csv"), dfEmissions) + write_output_file(joinpath(path, + setup["WriteResultsNamesDict"]["emissions"]), + dfEmissions, filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) + else# setup["WriteOutputs"] == "full" dfEmissions = hcat(dfEmissions, DataFrame(emissions_by_zone * scale_factor, :auto)) @@ -88,9 +93,14 @@ function write_emissions(path::AbstractString, inputs::Dict, setup::Dict, EP::Mo end rename!(total, auxNew_Names) dfEmissions = vcat(dfEmissions, total) - CSV.write(joinpath(path, "emissions.csv"), - dftranspose(dfEmissions, false), - writeheader = false) + # Maya: + dfEmissions = dftranspose(dfEmissions, true) + + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["emissions"]), + dfEmissions, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) + end ## Aaron - Combined elseif setup["Dual_MIP"]==1 block with the first block since they were identical. Why do we have this third case? What is different about it? else @@ -106,7 +116,11 @@ function write_emissions(path::AbstractString, inputs::Dict, setup::Dict, EP::Mo if setup["WriteOutputs"] == "annual" total = DataFrame(["Total" sum(dfEmissions.AnnualSum)], [:Zone; :AnnualSum]) dfEmissions = vcat(dfEmissions, total) - CSV.write(joinpath(path, "emissions.csv"), dfEmissions) + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["emissions"]), + dfEmissions, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) + else# setup["WriteOutputs"] == "full" dfEmissions = hcat(dfEmissions, DataFrame(emissions_by_zone * scale_factor, :auto)) @@ -121,12 +135,17 @@ function write_emissions(path::AbstractString, inputs::Dict, setup::Dict, EP::Mo end rename!(total, auxNew_Names) dfEmissions = vcat(dfEmissions, total) - CSV.write(joinpath(path, "emissions.csv"), - dftranspose(dfEmissions, false), - writeheader = false) + + dfEmissions = dftranspose(dfEmissions, true) + + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["emissions"]), + dfEmissions, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) + if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1 - write_full_time_series_reconstruction(path, setup, dfEmissions, "emissions") + write_full_time_series_reconstruction(path, setup, dfEmissions, setup["WriteResultsNamesDict"]["emissions"]) @info("Writing Full Time Series for Emissions") end end diff --git a/src/write_outputs/write_energy_revenue.jl b/src/write_outputs/write_energy_revenue.jl index 3e0834bd1e..7a9aa25760 100644 --- a/src/write_outputs/write_energy_revenue.jl +++ b/src/write_outputs/write_energy_revenue.jl @@ -30,6 +30,10 @@ function write_energy_revenue(path::AbstractString, inputs::Dict, setup::Dict, E energyrevenue *= ModelScalingFactor end dfEnergyRevenue.AnnualSum .= energyrevenue * inputs["omega"] - write_simple_csv(joinpath(path, "EnergyRevenue.csv"), dfEnergyRevenue) + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["energy_revenue"]), + dfEnergyRevenue, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) + return dfEnergyRevenue end diff --git a/src/write_outputs/write_fuel_consumption.jl b/src/write_outputs/write_fuel_consumption.jl index 8385e3e5ce..fb63cf6d5e 100644 --- a/src/write_outputs/write_fuel_consumption.jl +++ b/src/write_outputs/write_fuel_consumption.jl @@ -65,7 +65,8 @@ function write_fuel_consumption_plant(path::AbstractString, tempannualsum *= ModelScalingFactor^2 # end dfPlantFuel.AnnualSumCosts .+= tempannualsum - CSV.write(joinpath(path, "Fuel_cost_plant.csv"), dfPlantFuel) + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["fuel_cost_plant"]), dfPlantFuel, filetype = setup["ResultsFileType"], compression = setup["ResultsCompressionType"]) + end function write_fuel_consumption_ts(path::AbstractString, @@ -83,12 +84,15 @@ function write_fuel_consumption_ts(path::AbstractString, end dfPlantFuel_TS = hcat(dfPlantFuel_TS, DataFrame(tempts, [Symbol("t$t") for t in 1:T])) - CSV.write(joinpath(path, "FuelConsumption_plant_MMBTU.csv"), - dftranspose(dfPlantFuel_TS, false), header = false) + + write_output_file(joinpath(path, + setup["WriteResultsNamesDict"]["fuel_consumption_plant"]), + dftranspose(dfPlantFuel_TS, true), filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1 write_full_time_series_reconstruction( - path, setup, dfPlantFuel_TS, "FuelConsumption_plant_MMBTU") + path, setup, dftranspose(dfPlantFuel_TS, true),setup["WriteResultsNamesDict"]["fuel_consumption_plant"]) @info("Writing Full Time Series for Fuel Consumption") end end @@ -107,5 +111,6 @@ function write_fuel_consumption_tot(path::AbstractString, tempannualsum *= ModelScalingFactor # billion MMBTU to MMBTU end dfFuel.AnnualSum .+= tempannualsum - CSV.write(joinpath(path, "FuelConsumption_total_MMBTU.csv"), dfFuel) + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["fuel_consumption_total"]),dfFuel, filetype = setup["ResultsFileType"], compression = setup["ResultsCompressionType"]) + end diff --git a/src/write_outputs/write_maintenance.jl b/src/write_outputs/write_maintenance.jl index 8a6ba1fa5e..b4ddbd21f2 100644 --- a/src/write_outputs/write_maintenance.jl +++ b/src/write_outputs/write_maintenance.jl @@ -42,5 +42,5 @@ end function write_maintenance(path::AbstractString, inputs::Dict, setup::Dict, EP::Model) set, data = prepare_maintenance_downvars_matrix(EP, inputs) df = _create_annualsum_df(inputs, set, data) - write_temporal_data(df, data, path, setup, "maint_down") -end + write_temporal_data(df, data, path, setup, setup["WriteResultsNamesDict"]["maint_down"]) +end \ No newline at end of file diff --git a/src/write_outputs/write_net_revenue.jl b/src/write_outputs/write_net_revenue.jl index 2311c8d1a4..48fdf864c6 100644 --- a/src/write_outputs/write_net_revenue.jl +++ b/src/write_outputs/write_net_revenue.jl @@ -270,5 +270,6 @@ function write_net_revenue(path::AbstractString, dfNetRevenue.CO2SequestrationCost) dfNetRevenue.Profit = dfNetRevenue.Revenue .- dfNetRevenue.Cost - CSV.write(joinpath(path, "NetRevenue.csv"), dfNetRevenue) + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["revenue"]),dfNetRevenue, filetype = setup["ResultsFileType"], compression = setup["ResultsCompressionType"]) + end diff --git a/src/write_outputs/write_nse.jl b/src/write_outputs/write_nse.jl index 9d1c73e835..d38edd2e51 100644 --- a/src/write_outputs/write_nse.jl +++ b/src/write_outputs/write_nse.jl @@ -22,7 +22,11 @@ function write_nse(path::AbstractString, inputs::Dict, setup::Dict, EP::Model) total = DataFrame(["Total" 0 sum(dfNse[!, :AnnualSum])], [:Segment, :Zone, :AnnualSum]) dfNse = vcat(dfNse, total) - CSV.write(joinpath(path, "nse.csv"), dfNse) + #CSV.write(joinpath(path, setup["WriteResultsNamesDict"]["nse"]), dfNse) + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["nse"]), + dfNse, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) else # setup["WriteOutputs"] == "full" dfNse = hcat(dfNse, DataFrame(nse, :auto)) auxNew_Names = [Symbol("Segment"); @@ -35,13 +39,25 @@ function write_nse(path::AbstractString, inputs::Dict, setup::Dict, EP::Model) total[:, 4:(T + 3)] .= sum(nse, dims = 1) rename!(total, auxNew_Names) dfNse = vcat(dfNse, total) + #= + # Maya: Cast zones as floats + dfNse[!,:Zone] = convert.(Float64,dfNse[!,:Zone]) - CSV.write(joinpath(path, "nse.csv"), dftranspose(dfNse, false), writeheader = false) + dfNse = dftranspose(dfNse, false) + rename!(dfNse, Symbol.(Vector(dfNse[1,:]))) + dfNse = dfNse[2:end,:] + dfNse[!,2:end] = convert.(Float64,dfNse[!,2:end])=# - if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1 + CSV.write(joinpath(path, setup["WriteResultsNamesDict"]["nse"]), dftranspose(dfNse, false), writeheader = false) + #=write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["nse"]), + dftranspose(dfNse, false), + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"])=# + + #= if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1 write_full_time_series_reconstruction(path, setup, dfNse, "nse") @info("Writing Full Time Series for NSE") - end + end=# end return nothing end diff --git a/src/write_outputs/write_outputs.jl b/src/write_outputs/write_outputs.jl index 3b4e0707b3..844848d7a6 100644 --- a/src/write_outputs/write_outputs.jl +++ b/src/write_outputs/write_outputs.jl @@ -313,7 +313,7 @@ function write_outputs(EP::Model, path::AbstractString, setup::Dict, inputs::Dic end if output_settings_d["WriteTimeWeights"] - elapsed_time_time_weights = @elapsed write_time_weights(path, inputs) + elapsed_time_time_weights = @elapsed write_time_weights(path, inputs, setup) println("Time elapsed for writing time weights is") println(elapsed_time_time_weights) end @@ -489,20 +489,21 @@ end # END output() Internal function for writing annual outputs. """ -function write_annual(fullpath::AbstractString, dfOut::DataFrame) +function write_annual(fullpath::AbstractString, dfOut::DataFrame, setup::Dict) push!(dfOut, ["Total" 0 sum(dfOut[!, :AnnualSum], init = 0.0)]) - CSV.write(fullpath, dfOut) + write_output_file(fullpath, dfOut, filetype = setup["ResultsFileType"], compression = setup["ResultsCompressionType"]) return nothing end """ - write_fulltimeseries(fullpath::AbstractString, dataOut::Matrix{Float64}, dfOut::DataFrame) + write_fulltimeseries(fullpath::AbstractString, dataOut::Matrix{Float64}, dfOut::DataFrame, setup::Dict) Internal function for writing full time series outputs. This function wraps the instructions for creating the full time series output files. """ function write_fulltimeseries(fullpath::AbstractString, dataOut::Matrix{Float64}, - dfOut::DataFrame) + dfOut::DataFrame, + setup::Dict) T = size(dataOut, 2) dfOut = hcat(dfOut, DataFrame(dataOut, :auto)) auxNew_Names = [Symbol("Resource"); @@ -511,11 +512,13 @@ function write_fulltimeseries(fullpath::AbstractString, [Symbol("t$t") for t in 1:T]] rename!(dfOut, auxNew_Names) total = DataFrame( - ["Total" 0 sum(dfOut[!, :AnnualSum], init = 0.0) fill(0.0, (1, T))], auxNew_Names) + ["Total" missing sum(dfOut[!, :AnnualSum], init = 0.0) fill(0.0, (1, T))], auxNew_Names) total[!, 4:(T + 3)] .= sum(dataOut, dims = 1, init = 0.0) dfOut = vcat(dfOut, total) - - CSV.write(fullpath, dftranspose(dfOut, false), writeheader = false) + dfOut = dftranspose(dfOut, true) + write_output_file(fullpath, dfOut, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) return dfOut end @@ -572,12 +575,12 @@ end function write_temporal_data( df_annual, data, path::AbstractString, setup::Dict, filename::AbstractString) - filepath = joinpath(path, filename * ".csv") + filepath = joinpath(path, filename) if setup["WriteOutputs"] == "annual" # df_annual is expected to have an AnnualSum column. - write_annual(filepath, df_annual) + write_annual(filepath, df_annual, setup) else # setup["WriteOutputs"] == "full" - df_full = write_fulltimeseries(filepath, data, df_annual) + df_full = write_fulltimeseries(filepath, data, df_annual, setup) if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1 write_full_time_series_reconstruction(path, setup, df_full, filename) @info("Writing Full Time Series for "*filename) @@ -592,7 +595,7 @@ end name::String) Create a DataFrame with all 8,760 hours of the year from the reduced output. -This function calls `full_time_series_reconstruction()``, which uses Period_map.csv to create a new DataFrame with 8,760 time steps, as well as other pre-existing rows such as "Zone". +This function calls `full_time_series_reconstruction()``, which uses the file `Period_map`` to create a new DataFrame with 8,760 time steps, as well as other pre-existing rows such as "Zone". For each 52 weeks of the year, the corresponding representative week is taken from the input DataFrame and copied into the new DataFrame. Representative periods that represent more than one week will appear multiple times in the output. @@ -612,7 +615,231 @@ function write_full_time_series_reconstruction( path::AbstractString, setup::Dict, DF::DataFrame, name::String) FullTimeSeriesFolder = setup["OutputFullTimeSeriesFolder"] output_path = joinpath(path, FullTimeSeriesFolder) - dfOut_full = full_time_series_reconstruction(path, setup, dftranspose(DF, false)) - CSV.write(joinpath(output_path, "$name.csv"), dfOut_full, header = false) + dfOut_full = full_time_series_reconstruction(path, setup, DF) + write_output_file(joinpath(output_path, "$name"), + dfOut_full, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) return nothing end + +@doc raw"""write_output_file(path::String, + file::DataFrame; + filetype::String = "auto_detect", + compression::String = "auto_detect") + This internal function takes a dataframe and saves it according to the type specified in `ResultsFileType` in `genx_settings.yml`. Acceptable file types are .csv, .json, and .parqet. + It also has the option to compress files according to the compression type specified in `ResultsCompressionType` in `genx_settings.yml`. Acceptable compression types are gzip for CSV and JSON files, + and snappy and zstd for parquet files. It compresses and saves the files using DuckDB. + + This function has the ability to automatically detect the correct file extension from the file name, if one exists, by setting `ResultsFileType = "auto_detect"`. If a filename has an extension that clashes with the extension provided in + `ResultsFileType`, the extension already present in the name is used. For example, if a file is called "capacity.csv" in `results_settings.yml`, but `ResultsFileType = ".parquet"`, the file will be saved as a CSV. + If no extension is present, and `ResultsFileType` is set to `auto_detect`, then .csv is automatically used. + + Compression type can also be automatically detected by setting `ResultsCompressionType = "auto_detect"`. This will automatically detect if `.gz` is present in the filename for CSV and JSON files, + and for parquet files will automatically detect if "-snappy" or "-zstd" is present in the file name. If `auto_detect` is on, but no compression is present, the files will be saved uncompressed. + If a file extension contains `.gz`, but `ResultsCompressionType = "none"`, the file will still be compressed as a gzip. + + The keyword arguments `filetype` and `compression` are optional and are both set to `auto_detect` by default. + + # Arguments + - `path::AbstractString`: The path including the file name. This can include the file extension (e.g. .csv) but does not have to. + - `file::DataFrame`: The DataFrame being saved to the input path. All columns in the DataFrame must have a type (cannot be type "Any") in order for DuckDB to work. + - `filetype::String`: The file type, as specified in `ResultsFileType` in `genx_settings.yml`. Accepted inputs are `.csv`,`.csv.gz` `.parquet`, `.json`, `.json.gz`, and `auto_detect` (default). + - `compression::String`: The compression type, as specified in `ResultsCompressionType` in `genx_settings.yml`. Accepted inputs are `gzip`, `snappy`, `zstd`, `none`, and `auto_detect` (default). +""" +function write_output_file(path::AbstractString, file::DataFrame; filetype::String = "auto_detect", compression::String = "auto_detect") + # 1) Check if an extension is already in the file name, if not, add it based on filetype + if occursin(".", path) + if occursin(".", splitext(path)[1]) # If two extensions are present (eg .csv.gz, or .json.gz, only the first will be added to the filetype as .gz will be autodetected by DuckDB later) + if filetype == "auto_detect" # If auto-detect is on for the extension type, change the filetype to the extension detected using splitext + filetype = splitext(splitext(path)[1])[2] + elseif filetype != splitext(splitext(path)[1])[2] # If the extension in the file name is different than the filetype key, override the filetype key and throw a warning. + filetype = splitext(splitext(path)[1])[2] + @warn("File extension conflicts with filetype in genx_settings.yml. Saving file as $filetype") + end + else + if filetype == "auto_detect" # If auto-detect is on for the extension type, change the filetype to the extension detected using splitext + filetype = splitext(path)[2] + elseif filetype != splitext(path)[2] # If the extension in the file name is different than the filetype key, override the filetype key and throw a warning. + filetype = splitext(path)[2] + @warn("File extension conflicts with filetype in genx_settings.yml. Saving file as $filetype") + end + if splitext(path)[2] == ".csv" && isgzip(compression) + path = path * ".gz" # If the file only ends in ".csv", but compression is set to gzip, add ".gz" to the end of the file + elseif splitext(path)[2] == ".json" && isgzip(compression) + path *= ".gz" + end + end + elseif filetype == "auto_detect" # If no extension is detected in the file name, but auto-detect is on, .csv will automatically be added + filetype = ".csv" + path *= ".csv" + elseif filetype == ".csv" # If no extension is present, but filetype is set to .csv, .csv will be appended to the path name. + if compression == "none" + path *= ".csv" + elseif isgzip(compression) # If no extension is present, and compression is set to gzip, add .gz to the end of the file name. + path *= ".csv.gz" + elseif compression == "auto_detect" # If no extension is present, but compression is set to auto_detect, no compression is added + path *= ".csv" + else + @warn("Compression type '$compression' not supported with .csv. Saving as uncompressed csv.") + path *= ".csv" + end + elseif filetype == ".json" # If no extension is present, but filetype is set to .csv, .csv will be appended to the path name + if compression == "none" + path *= ".json" + elseif isgzip(compression) + path *= ".json.gz" + elseif compression == "auto_detect" + path *= ".json" + else + @warn("Compression type '$compression' not supported with .json. Saving as uncompressed json.") + path *= ".json" + end + elseif filetype == ".parquet" + if compression == "none" + path *= ".parquet" + elseif compression == "snappy" || compression == "-snappy" + path *= "-snappy.parqet" + elseif compression == "zstd" || compression == "-zstd" + path *= "-zstd.parquet" + elseif compression == "auto_detect" + path *= ".parquet" + else + @warn("Compression type '$compression' not supported with .parquet. Saving as uncompressed parquet.") + path *= ".parquet" + end + else + @error "Filetype '$filetype' not accepted. Accepted formats are .csv, .gz, .parquet, and .json." + end + + # 2) Save file according to compression type: auto_detect, gzip, snappy, zstd, or none + if compression == "auto_detect" + if filetype == ".csv" || filetype == ".csv.gz" + save_with_duckdb(file,path,"csv","none") # DuckDB will automatically detect if the file should be compressed or not + elseif filetype == ".parquet" + if occursin("-", path) # Parquet files can be saved with compression types in the name e.g. "capacity-snappy.parquet" + filename = splitext(path)[1] + compression_type = filename[findlast('-', filename):end] + if compression_type == "-snappy" + save_with_duckdb(file,path,"parquet","snappy") + elseif compression_type == "-zstd" + save_with_duckdb(file,path,"parquet","zstd") + elseif compression_type == "-uncompressed" + save_with_duckdb(file,path,"parquet","uncompressed") + else + @warn "Unable to auto-detect compression type of parquet file. Saving as uncompressed parquet." + save_with_duckdb(file,path,"parquet","uncompressed") + end + else + save_with_duckdb(file,path,"parquet","uncompressed") # If no "-" is present, file is saved uncompressed. + end + elseif filetype == ".json" + save_with_duckdb(file,path,"json","none") + else + @error "Filetype '$filetype' not accepted. Accepted formats are .csv, .parquet, and .json." + end + elseif isgzip(compression) + if filetype == ".csv" + if splitext(path)[2] == ".gz" + save_with_duckdb(file,path,"csv","gzip") + else + path *= ".gz" + save_with_duckdb(file,path,"csv","gzip") + end + elseif filetype == ".json" + if splitext(path)[2] == ".gz" + save_with_duckdb(file,path,"json","auto_detect") + else + path *= ".gz" + save_with_duckdb(file,path,"json","auto_detect") + end + elseif filetype == ".parquet" + @warn(".parquet cannot be compressed as gzip. Saving as uncompressed parquet") + save_with_duckdb(file,path,"parquet","uncompressed") + else + @error("Filetype '$filetype' not accepted. Accepted formats are .csv, .csv.gz, .parquet, .json, and .json.gz.") + end + elseif compression == "snappy" || compression == "-snappy" + if filetype == ".parquet" + save_with_duckdb(file,path,"parquet","snappy") + elseif filetype == ".csv" + @warn("Filetype .csv cannot be saved with snappy compression. Saving as uncompressed csv.") + save_with_duckdb(file,path,"csv","none") + elseif filetype == ".json" + @warn("Filetype .json cannot be saved with snappy compression. Saving as uncompressed json.") + save_with_duckdb(file,path,"json","auto_detect") + end + elseif compression == "zstd" || compression == "-zstd" + if filetype == ".parquet" + save_with_duckdb(file,path,"parquet","zstd") + elseif filetype == ".csv" + @warn("Filetype .csv cannot be saved with zstd compression. Saving as uncompressed csv.") + save_with_duckdb(file,path,"csv","none") + elseif filetype == ".json" + save_with_duckdb(file,path,"json","zstd") + else + @error "Filetype '$filetype' not accepted. Accepted formats are .csv, .csv.gz, .parquet, .json, and .json.gz." + end + else + if compression != "none" + @warn("Compression type '$compression' is not accepted. Saving without file compression.") + end + if filetype == ".csv" + save_with_duckdb(file,path,"csv","none") + elseif filetype == "csv.gz" # If compression type is listed as none, but filetype has .gz in it, compression type is overridden and .gz is used. + @warn("Gzip compression detected in file name. Saving with gzip compression.") + save_with_duckdb(file,path,"csv","gzip") + elseif filetype == ".parquet" + save_with_duckdb(file,path,"parquet","uncompressed") + elseif filetype == ".json" + save_with_duckdb(file,path,"json","none") + elseif filetype == ".json.gz" + @warn("Gzip compression detected in file name. Saving with gzip compression.") + save_with_duckdb(file,path,"json","gzip") + else + @error "Filetype '$filetype' not accepted. Accepted formats are .csv, .csv.gz, .parquet, .json, and .json.gz." + end + end +end + +@doc raw"""isgzip(compression::String) + This internal function determines if the compression is of type gzip. It's purporse is to prevent a misspelling of gzip, since you can write "gz" or "gzip". + + # Arguments + - `compression::String`: - compression type, from the genxsettings YAML file or default dictionary + + # Output + - `true` or `false` if the file has a compression type of gzip. +""" +function isgzip(compression::String) + if compression == "gzip" || compression == ".gz" || compression == "gz" || compression == ".gzip" + return true + end + return false +end + +@doc raw"""save_with_duckdb(compression::String) + This internal function saves a DataFrame using the package DuckDB. + + # Arguments + - `file::DataFrame`: Dataframe of information to be saved + - `path::AbstractString`: path of the directory to save the file in + - `filetype::String`: file type, from the genxsettings YAML file or default dictionary. Can be `csv`, `json`, or `parquet` + - `compression::String`: - compression type, from the genxsettings YAML file or default dictionary. Can be `gzip`, `snappy`, `zstd`, `none`, or `uncompressed`. +""" +function save_with_duckdb(file::DataFrame,path::AbstractString,filetype::String,compression::String) + con = DBInterface.connect(DuckDB.DB) + DuckDB.register_data_frame(con, file, "temp_df") + if filetype == "csv" + DBInterface.execute(con, "COPY temp_df TO '$path'") # DuckDB will auto detect the prescence of gzip + elseif filetype == "parquet" + DBInterface.execute(con, "COPY temp_df TO '$path' (FORMAT 'parquet', CODEC '$compression');") + elseif filetype == "json" + if compression == "auto_detect" + DBInterface.execute(con, "COPY temp_df TO '$path' (FORMAT JSON, AUTO_DETECT true);") + else + DBInterface.execute(con, "COPY temp_df TO '$path' (FORMAT JSON, COMPRESSION '$compression');") + end + end + DBInterface.close(con) +end \ No newline at end of file diff --git a/src/write_outputs/write_power.jl b/src/write_outputs/write_power.jl index 995a2d941b..811c9dfeec 100644 --- a/src/write_outputs/write_power.jl +++ b/src/write_outputs/write_power.jl @@ -7,6 +7,7 @@ function write_power(path::AbstractString, inputs::Dict, setup::Dict, EP::Model) gen = inputs["RESOURCES"] # Resources (objects) resources = inputs["RESOURCE_NAMES"] # Resource names zones = zone_id.(gen) + zones = convert.(Float64,zones) G = inputs["G"] # Number of resources (generators, storage, DR, and DERs) T = inputs["T"] # Number of time steps (hours) @@ -17,12 +18,11 @@ function write_power(path::AbstractString, inputs::Dict, setup::Dict, EP::Model) # Power injected by each resource in each time step power = value.(EP[:vP]) power *= scale_factor - df = DataFrame(Resource = resources, Zone = zones, AnnualSum = zeros(G)) df.AnnualSum .= power * weight - write_temporal_data(df, power, path, setup, "power") + write_temporal_data(df, power, path, setup, setup["WriteResultsNamesDict"]["power"]) return df end diff --git a/src/write_outputs/write_power_balance.jl b/src/write_outputs/write_power_balance.jl index 72bfee22d1..bbbdcc1b31 100644 --- a/src/write_outputs/write_power_balance.jl +++ b/src/write_outputs/write_power_balance.jl @@ -103,7 +103,8 @@ function write_power_balance(path::AbstractString, inputs::Dict, setup::Dict, EP dfPowerBalance.AnnualSum .= powerbalance * inputs["omega"] if setup["WriteOutputs"] == "annual" - CSV.write(joinpath(path, "power_balance.csv"), dfPowerBalance) + CSV.write(joinpath(path, setup["WriteResultsNamesDict"]["power_balance"]), dfPowerBalance) + #write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["power_balance"]),dfPowerBalance, filetype = setup["ResultsFileType"], compression = setup["ResultsCompressionType"]) else # setup["WriteOutputs"] == "full" dfPowerBalance = hcat(dfPowerBalance, DataFrame(powerbalance, :auto)) auxNew_Names = [Symbol("BalanceComponent"); @@ -111,15 +112,29 @@ function write_power_balance(path::AbstractString, inputs::Dict, setup::Dict, EP Symbol("AnnualSum"); [Symbol("t$t") for t in 1:T]] rename!(dfPowerBalance, auxNew_Names) - CSV.write(joinpath(path, "power_balance.csv"), + + dfPowerBalance[!,:Zone] = convert.(Float64,dfPowerBalance[!,:Zone]) + CSV.write(joinpath(path, setup["WriteResultsNamesDict"]["power_balance"]), dftranspose(dfPowerBalance, false), writeheader = false) - if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1 + #= Maya: + dfPowerBalance = dftranspose(dfPowerBalance, false) + + rename!(dfPowerBalance, Symbol.(Vector(dfPowerBalance[1,:]))) + dfPowerBalance = dfPowerBalance[2:end,:] + dfPowerBalance[!,2:end] = convert.(Float64,dfPowerBalance[!,2:end]) + + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["power_balance"]), + dfPowerBalance, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) =# + + #=if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1 write_full_time_series_reconstruction( path, setup, dfPowerBalance, "power_balance") @info("Writing Full Time Series for Power Balance") - end + end=# end return nothing end diff --git a/src/write_outputs/write_price.jl b/src/write_outputs/write_price.jl index 79afb5b428..419083b721 100644 --- a/src/write_outputs/write_price.jl +++ b/src/write_outputs/write_price.jl @@ -18,12 +18,13 @@ function write_price(path::AbstractString, inputs::Dict, setup::Dict, EP::Model) rename!(dfPrice, auxNew_Names) ## Linear configuration final output - CSV.write(joinpath(path, "prices.csv"), - dftranspose(dfPrice, false), - writeheader = false) + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["prices"]), + dftranspose(dfPrice, true), + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1 - write_full_time_series_reconstruction(path, setup, dfPrice, "prices") + write_full_time_series_reconstruction(path, setup, dftranspose(dfPrice, true), setup["WriteResultsNamesDict"]["prices"]) @info("Writing Full Time Series for Price") end return nothing diff --git a/src/write_outputs/write_reliability.jl b/src/write_outputs/write_reliability.jl index afb3a3284c..dfaa5aa288 100644 --- a/src/write_outputs/write_reliability.jl +++ b/src/write_outputs/write_reliability.jl @@ -17,12 +17,14 @@ function write_reliability(path::AbstractString, inputs::Dict, setup::Dict, EP:: auxNew_Names = [Symbol("Zone"); [Symbol("t$t") for t in 1:T]] rename!(dfReliability, auxNew_Names) - CSV.write(joinpath(path, "reliability.csv"), - dftranspose(dfReliability, false), - header = false) + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["reliability"]), + dftranspose(dfReliability, true), + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) + if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1 - write_full_time_series_reconstruction(path, setup, dfReliability, "reliability") + write_full_time_series_reconstruction(path, setup, dftranspose(dfReliability, true), setup["WriteResultsNamesDict"]["reliability"]) @info("Writing Full Time Series for Reliability") end end diff --git a/src/write_outputs/write_status.jl b/src/write_outputs/write_status.jl index 8558a21a50..0a4cbe548c 100644 --- a/src/write_outputs/write_status.jl +++ b/src/write_outputs/write_status.jl @@ -7,6 +7,7 @@ function write_status(path::AbstractString, inputs::Dict, setup::Dict, EP::Model # https://jump.dev/MathOptInterface.jl/v0.9.10/apireference/#MathOptInterface.TerminationStatusCode status = termination_status(EP) + status = string(status) # Note: Gurobi excludes constants from solver reported objective function value - MIPGap calculated may be erroneous if (setup["UCommit"] == 0 || setup["UCommit"] == 2) @@ -17,5 +18,9 @@ function write_status(path::AbstractString, inputs::Dict, setup::Dict, EP::Model Objval = objective_value(EP), Objbound = objective_bound(EP), FinalMIPGap = (objective_value(EP) - objective_bound(EP)) / objective_value(EP)) end - CSV.write(joinpath(path, "status.csv"), dfStatus) + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["status"]), + dfStatus, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) + end diff --git a/src/write_outputs/write_storage.jl b/src/write_outputs/write_storage.jl index 4e64c0f9bd..c30e340c22 100644 --- a/src/write_outputs/write_storage.jl +++ b/src/write_outputs/write_storage.jl @@ -7,6 +7,7 @@ function write_storage(path::AbstractString, inputs::Dict, setup::Dict, EP::Mode gen = inputs["RESOURCES"] # Resources (objects) resources = inputs["RESOURCE_NAMES"] # Resource names zones = zone_id.(gen) + zones = convert.(Float64,zones) T = inputs["T"] # Number of time steps (hours) G = inputs["G"] @@ -40,5 +41,5 @@ function write_storage(path::AbstractString, inputs::Dict, setup::Dict, EP::Mode Zone = zones[stored_ids]) df.AnnualSum = stored * weight - write_temporal_data(df, stored, path, setup, "storage") + write_temporal_data(df, stored, path, setup, setup["WriteResultsNamesDict"]["storage"]) end diff --git a/src/write_outputs/write_storagedual.jl b/src/write_outputs/write_storagedual.jl index 90eae94ff2..f1cffdf533 100644 --- a/src/write_outputs/write_storagedual.jl +++ b/src/write_outputs/write_storagedual.jl @@ -6,7 +6,7 @@ Function for reporting dual of storage level (state of charge) balance of each r function write_storagedual(path::AbstractString, inputs::Dict, setup::Dict, EP::Model) gen = inputs["RESOURCES"] zones = zone_id.(gen) - + zones = convert.(Float64,zones) G = inputs["G"] # Number of resources (generators, storage, DR, and DERs) T = inputs["T"] # Number of time steps (hours) @@ -74,15 +74,16 @@ function write_storagedual(path::AbstractString, inputs::Dict, setup::Dict, EP:: dfStorageDual = hcat(dfStorageDual, DataFrame(dual_values, :auto)) rename!(dfStorageDual, - [Symbol("Resource"); Symbol("Zone"); [Symbol("t$t") for t in 1:T]]) + ["Resource"; "Zone"; [String("t$t") for t in 1:T]]) - CSV.write(joinpath(path, "storagebal_duals.csv"), - dftranspose(dfStorageDual, false), - header = false) + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["storagebal_duals"]), + dftranspose(dfStorageDual, true), + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) if setup["OutputFullTimeSeries"] == 1 && setup["TimeDomainReduction"] == 1 write_full_time_series_reconstruction( - path, setup, dfStorageDual, "storagebal_duals") + path, setup, dftranspose(dfStorageDual, true), setup["WriteResultsNamesDict"]["storagebal_duals"]) @info("Writing Full Time Series for Storage Duals") end end diff --git a/src/write_outputs/write_subsidy_revenue.jl b/src/write_outputs/write_subsidy_revenue.jl index 3262ec94d7..3c3a52fcc3 100644 --- a/src/write_outputs/write_subsidy_revenue.jl +++ b/src/write_outputs/write_subsidy_revenue.jl @@ -98,7 +98,14 @@ function write_subsidy_revenue(path::AbstractString, inputs::Dict, setup::Dict, dfRegSubRevenue.SubsidyRevenue *= ModelScalingFactor^2 #convert from Million US$ to US$ end - CSV.write(joinpath(path, "SubsidyRevenue.csv"), dfSubRevenue) - CSV.write(joinpath(path, "RegSubsidyRevenue.csv"), dfRegSubRevenue) + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["subsidy_revenue"]), + dfSubRevenue, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["reg_subsidy_revenue"]), + dfRegSubRevenue, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) + return dfSubRevenue, dfRegSubRevenue end diff --git a/src/write_outputs/write_time_weights.jl b/src/write_outputs/write_time_weights.jl index b29bbdcb2f..34c924659e 100644 --- a/src/write_outputs/write_time_weights.jl +++ b/src/write_outputs/write_time_weights.jl @@ -1,6 +1,11 @@ -function write_time_weights(path::AbstractString, inputs::Dict) +function write_time_weights(path::AbstractString, inputs::Dict, setup::Dict) T = inputs["T"] # Number of time steps (hours) # Save array of weights for each time period (when using time sampling) dfTimeWeights = DataFrame(Time = 1:T, Weight = inputs["omega"]) - CSV.write(joinpath(path, "time_weights.csv"), dfTimeWeights) + + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["time_weights"]), + dfTimeWeights, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) + end diff --git a/src/write_outputs/write_vre_stor.jl b/src/write_outputs/write_vre_stor.jl index a50687c957..cd44a462a3 100644 --- a/src/write_outputs/write_vre_stor.jl +++ b/src/write_outputs/write_vre_stor.jl @@ -293,8 +293,8 @@ function write_vre_stor_capacity(path::AbstractString, inputs::Dict, setup::Dict dfCap[!, columns_to_scale] .*= ModelScalingFactor end - total = DataFrame(Resource = "Total", Zone = "n/a", Resource_Type = "Total", - Cluster = "n/a", + total = DataFrame(Resource = "Total", Zone = missing, Resource_Type = "Total", + Cluster = missing, StartCapSolar = sum(dfCap[!, :StartCapSolar]), RetCapSolar = sum(dfCap[!, :RetCapSolar]), NewCapSolar = sum(dfCap[!, :NewCapSolar]), @@ -331,7 +331,12 @@ function write_vre_stor_capacity(path::AbstractString, inputs::Dict, setup::Dict EndDischargeACCap = sum(dfCap[!, :EndDischargeACCap])) dfCap = vcat(dfCap, total) - CSV.write(joinpath(path, "vre_stor_capacity.csv"), dfCap) + + write_output_file(joinpath(path, setup["WriteResultsNamesDict"]["vre_stor_capacity"]), + dfCap, + filetype = setup["ResultsFileType"], + compression = setup["ResultsCompressionType"]) + return dfCap end @@ -358,11 +363,11 @@ function write_vre_stor_charge(path::AbstractString, inputs::Dict, setup::Dict, (setup["ParameterScale"] == 1 ? ModelScalingFactor : 1) dfCharge_DC.AnnualSum .= charge_dc * inputs["omega"] - filepath = joinpath(path, "vre_stor_dc_charge.csv") + filepath = joinpath(path, setup["WriteResultsNamesDict"]["vre_stor_dc_charge"]) if setup["WriteOutputs"] == "annual" - write_annual(filepath, dfCharge_DC) + write_annual(filepath, dfCharge_DC, setup) else # setup["WriteOutputs"] == "full" - write_fulltimeseries(filepath, charge_dc, dfCharge_DC) + write_fulltimeseries(filepath, charge_dc, dfCharge_DC, setup) end end @@ -376,11 +381,11 @@ function write_vre_stor_charge(path::AbstractString, inputs::Dict, setup::Dict, (setup["ParameterScale"] == 1 ? ModelScalingFactor : 1) dfCharge_AC.AnnualSum .= charge_ac * inputs["omega"] - filepath = joinpath(path, "vre_stor_ac_charge.csv") + filepath = joinpath(path, setup["WriteResultsNamesDict"]["vre_stor_ac_charge"]) if setup["WriteOutputs"] == "annual" - write_annual(filepath, dfCharge_AC) + write_annual(filepath, dfCharge_AC, setup) else # setup["WriteOutputs"] == "full" - write_fulltimeseries(filepath, charge_ac, dfCharge_AC) + write_fulltimeseries(filepath, charge_ac, dfCharge_AC, setup) end end return nothing @@ -416,11 +421,11 @@ function write_vre_stor_discharge(path::AbstractString, end dfDischarge_DC.AnnualSum .= power_vre_stor * inputs["omega"] - filepath = joinpath(path, "vre_stor_dc_discharge.csv") + filepath = joinpath(path, setup["WriteResultsNamesDict"]["vre_stor_dc_discharge"]) if setup["WriteOutputs"] == "annual" - write_annual(filepath, dfDischarge_DC) + write_annual(filepath, dfDischarge_DC, setup) else # setup["WriteOutputs"] == "full" - write_fulltimeseries(filepath, power_vre_stor, dfDischarge_DC) + write_fulltimeseries(filepath, power_vre_stor, dfDischarge_DC, setup) end end @@ -435,11 +440,11 @@ function write_vre_stor_discharge(path::AbstractString, end dfDischarge_AC.AnnualSum .= power_vre_stor * inputs["omega"] - filepath = joinpath(path, "vre_stor_ac_discharge.csv") + filepath = joinpath(path, setup["WriteResultsNamesDict"]["vre_stor_ac_discharge"]) if setup["WriteOutputs"] == "annual" - write_annual(filepath, dfDischarge_AC) + write_annual(filepath, dfDischarge_AC, setup) else # setup["WriteOutputs"] == "full" - write_fulltimeseries(filepath, power_vre_stor, dfDischarge_AC) + write_fulltimeseries(filepath, power_vre_stor, dfDischarge_AC, setup) end end @@ -454,11 +459,11 @@ function write_vre_stor_discharge(path::AbstractString, end dfVP_VRE_STOR.AnnualSum .= vre_vre_stor * inputs["omega"] - filepath = joinpath(path, "vre_stor_wind_power.csv") + filepath = joinpath(path, setup["WriteResultsNamesDict"]["vre_stor_wind_power"]) if setup["WriteOutputs"] == "annual" - write_annual(filepath, dfVP_VRE_STOR) + write_annual(filepath, dfVP_VRE_STOR, setup) else # setup["WriteOutputs"] == "full" - write_fulltimeseries(filepath, vre_vre_stor, dfVP_VRE_STOR) + write_fulltimeseries(filepath, vre_vre_stor, dfVP_VRE_STOR, setup) end end @@ -473,9 +478,9 @@ function write_vre_stor_discharge(path::AbstractString, end dfVP_VRE_STOR.AnnualSum .= elec_vre_stor * inputs["omega"] - filepath = joinpath(path, "vre_stor_elec_power_consumption.csv") + filepath = joinpath(path, setup["WriteResultsNamesDict"]["vre_stor_elec_power_consumption"]) if setup["WriteOutputs"] == "annual" - write_annual(filepath, dfVP_VRE_STOR) + write_annual(filepath, dfVP_VRE_STOR,setup) else # setup["WriteOutputs"] == "full" write_fulltimeseries(filepath, elec_vre_stor, dfVP_VRE_STOR) end @@ -492,11 +497,11 @@ function write_vre_stor_discharge(path::AbstractString, end dfVP_VRE_STOR.AnnualSum .= vre_vre_stor * inputs["omega"] - filepath = joinpath(path, "vre_stor_solar_power.csv") + filepath = joinpath(path, setup["WriteResultsNamesDict"]["vre_stor_solar_power"]) if setup["WriteOutputs"] == "annual" - write_annual(filepath, dfVP_VRE_STOR) + write_annual(filepath, dfVP_VRE_STOR, setup) else # setup["WriteOutputs"] == "full" - write_fulltimeseries(filepath, vre_vre_stor, dfVP_VRE_STOR) + write_fulltimeseries(filepath, vre_vre_stor, dfVP_VRE_STOR, setup) end end return nothing