Merge pull request #87 from holgerteichgraeber/dev

holgerteichgraeber · web-flow · commit aa493afc3e18 · 2019-05-15T17:38:40.000-07:00
v0.4.0. Updated ClustResult struct and related functions. See NEWS.md for details.
diff --git a/.travis.yml b/.travis.yml
@@ -19,7 +19,7 @@ jobs:
       script:
         - julia --project=test/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
         - julia --project=test/ test/clustering.jl
-        - julia --project=test/ test/capacityexpansion.jl
+        - julia --project=test/ test/test_workflow_introduction.jl
       after_success:
     - stage: "Documentation"
       julia: 1.0
diff --git a/NEWS.md b/NEWS.md
@@ -0,0 +1,12 @@
+ClustForOpt release notes
+=========================
+
+Version 0.4.0
+-------------
+
+Breaking changes
+
+- The `ClustResult` struct has been renamed to `AbstractClustResult`.
+- The `ClustResultBest` struct has been renamed to `ClustResult`.
+- The structs `ClustResult` and `ClustResultAll` have had several field names renamed: `best_results` to `clust_data`, `best_cost` to `cost`, `clust_config` to `config`. The fields `data_type` and `best_ids` have been removed, because they are already contained explicitly (`k_ids`) or implicitly(call `data_type(data::ClustData)`) in `ClustData`. 
+- The field names `centers, weights, clustids, cost, iter` in `ClustResultAll` have been renamed, all have now the ending `_all` to indicate that these are the results for all random initializations of the clustering algorithm.
diff --git a/Project.toml b/Project.toml
@@ -4,7 +4,7 @@ keywords = ["clustering", "JuMP", "optimization"]
 license = "MIT"
 desc = "julia implementation of using different clustering methods for finding representative periods for the optimization of energy systems"
 author = ["Holger Teichgraeber"]
-version = "0.3.4"
+version = "0.4.0"
 
 [deps]
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
diff --git a/README.md b/README.md
@@ -5,7 +5,7 @@
 [![License](http://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat)](LICENSE)
 [![Build Status](https://travis-ci.com/holgerteichgraeber/ClustForOpt.jl.svg?token=HRFemjSxM1NBCsbHGNDG&branch=master)](https://travis-ci.com/holgerteichgraeber/ClustForOpt.jl)
 
-ClustForOpt is a [julia](www.juliaopt.com) implementation of clustering methods for finding representative periods for the optimization of energy systems. The package furthermore provides a multi-node capacity expansion model.
+ClustForOpt is a [julia](www.juliaopt.com) implementation of clustering methods for finding representative periods for the optimization of energy systems. The package can be used in conjunction with the multi-node capacity expansion model [CapacityExpansion](https://github.com/YoungFaithful/CapacityExpansion.jl). 
 
 The package has two main purposes: 1) Provide a simple process of clustering time-series input data, with clustered data output in a generalized type system 2) provide an interface between clustered data and optimization problem.
 
@@ -38,6 +38,8 @@ add ClustForOpt
 ```
 where `]` opens the julia package manager.
 
+**See [NEWS](NEWS.md) for significant breaking changes when updating from one version of ClustForOpt to another.**
+
 ## Documentation
 [Stable](https://holgerteichgraeber.github.io/ClustForOpt.jl/stable)
 
@@ -50,6 +52,8 @@ Generally, the workflow requires three steps:
 - clustering
 - optimization
 
+An example workflow with examples on how to use the different functions can be found in [`examples/workflow_introduction.jl`](examples/workflow_introduction.jl)
+
 ```julia
 using ClustForOpt
 
@@ -102,4 +106,4 @@ For use of DTW barycenter averaging (DBA) and k-shape clustering on single-attri
 ### Optimization
 The function `run_opt()` runs the optimization problem and gives as an output a struct that contains optimal objective function value, decision variables, and additional info. The `run_opt()` function infers the optimization problem type from the input data. See the examples folder for further details.
 
-A Capacity Expansion Optimization Problem that utilizes `ClustForOpt` can be found in the package [CEP](https://github.com/YoungFaithful/CEP.jl). 
+A Capacity Expansion Optimization Problem that utilizes `ClustForOpt` can be found in the package [CapacityExpansion](https://github.com/YoungFaithful/CapacityExpansion.jl). 
diff --git a/data/TS_GER_18/solar.csv b/data/TS_GER_18/solar.csv
diff --git a/data/TS_GER_18/wind.csv b/data/TS_GER_18/wind.csv
diff --git a/docs/src/clust.md b/docs/src/clust.md
@@ -29,16 +29,15 @@ FullInputData
 ClustData
 ClustDataMerged
 ClustResultAll
-ClustResultBest
-ClustResultSimple
+ClustResult
 ```
 
 ## Example running clustering
 ```@example
 using ClustForOpt
 # load ts-input-data
 ts_input_data = load_timeseries_data(normpath(joinpath(@__DIR__,"..","..","data","TS_GER_1")); T=24, years=[2016])
-ts_clust_data = run_clust(ts_input_data).best_results
+ts_clust_data = run_clust(ts_input_data).clust_data
 using Plots
 plot(ts_clust_data.data["solar-germany"], legend=false, linestyle=:solid, width=3, xlabel="Time [h]", ylabel="Solar availability factor [%]")
 savefig("clust.svg")
diff --git a/examples/workflow_example_bat.jl b/examples/workflow_example_bat.jl
@@ -17,15 +17,15 @@ end
 
  # optimization
 
-opt_res = run_opt("battery",clust_res_ar[2].best_results)
- #opt_res = run_opt("gas_turbine",clust_res.best_results[5])
+opt_res = run_opt("battery",clust_res_ar[2].clust_data)
+ #opt_res = run_opt("gas_turbine",clust_res.clust_data[5])
 
  ###
  # run optimization for all k=1:9
 opt_res_all = []
 obj=[]
 for i=1:2
-  push!(opt_res_all,run_opt("battery", clust_res_ar[i].best_results))
+  push!(opt_res_all,run_opt("battery", clust_res_ar[i].clust_data))
   push!(obj,opt_res_all[i].obj)
 end
  # run reference case
diff --git a/examples/workflow_example_extr.jl b/examples/workflow_example_extr.jl
@@ -18,4 +18,4 @@ ts_input_data = load_timeseries_data(data_path; T=24, years=[2015])
 ts_clust_res = run_clust(ts_input_data_mod;method="kmeans",representation="centroid",n_init=10,n_clust=5) # default k-means
 
 # representation modification
-ts_clust_extr = representation_modification(extr_vals,ts_clust_res.best_results)
+ts_clust_extr = representation_modification(extr_vals,ts_clust_res.clust_data)
diff --git a/examples/workflow_introduction.jl b/examples/workflow_introduction.jl
@@ -66,7 +66,7 @@ end
 #############
 # Quick example and investigation of the best result:
 ts_clust_result = run_clust(ts_input_data; method="kmeans", representation="centroid", n_init=5, n_clust=5) # note that you should use n_init=1000 at least for kmeans.
-ts_clust_data = ts_clust_result.best_results
+ts_clust_data = ts_clust_result.clust_data
 # And some plotting:
 plot_comb_solar=plot!(plot_input_solar, ts_clust_data.data["solar-germany"], linestyle=:solid, width=3)
 plot_clust_soar=plot(ts_clust_data.data["el_demand-germany"], legend=false, linestyle=:solid, width=3, xlabel="Time [h]", ylabel="Solar availability factor [%]")
diff --git a/src/ClustForOpt.jl b/src/ClustForOpt.jl
@@ -22,10 +22,9 @@ module ClustForOpt
           FullInputData,
           ClustData,
           ClustDataMerged,
-          ClustResult,
-          ClustResultSimple,
+          AbstractClustResult,
           ClustResultAll,
-          ClustResultBest,
+          ClustResult,
           SimpleExtremeValueDescr,
           load_timeseries_data,
           combine_timeseries_weather_data,
@@ -35,6 +34,7 @@ module ClustForOpt
           get_sup_kw_args,
           run_clust,
           run_opt,
+          data_type,
           get_EUR_to_USD, #TODO Check which of the following should really be exported
           z_normalize,
           undo_z_normalize,
diff --git a/src/clustering/extreme_vals.jl b/src/clustering/extreme_vals.jl
@@ -140,7 +140,7 @@ function input_data_modification(data::ClustData,extr_val_idcs::Array{Int,1})
   #just modify the k_ids that are also represented within this clust-data (don't reduce 0 to -1...)
   k_ids_dn[findall(data.k_ids.!=0)]=k_ids_dn_data
   #return the new Clust Data
-  return ClustData(data.region,data.years,K_dn,data.T,data_dn,weights_dn,deltas_dn,k_ids_dn;mean=data.mean,sdv=data.sdv)
+  return ClustData(data.region,data.years,K_dn,data.T,data_dn,weights_dn,k_ids_dn;delta_t=deltas_dn,mean=data.mean,sdv=data.sdv)
 end
 
 """
@@ -186,7 +186,7 @@ function extreme_val_output(data::ClustData,
     @error("rep_mod_method - "*rep_mod_method*" - does not exist")
   end
   delta_t_ed=data.delta_t[:,unique_extr_val_idcs]
-  extr_vals = ClustData(data.region,data.years,K_ed,data.T,data_ed,weights_ed,delta_t_ed,k_ids_ed;mean=data.mean,sdv=data.sdv)
+  extr_vals = ClustData(data.region,data.years,K_ed,data.T,data_ed,weights_ed,k_ids_ed;delta_t=delta_t_ed,mean=data.mean,sdv=data.sdv)
   return extr_vals
 end
 
@@ -219,7 +219,7 @@ function representation_modification(extr_vals::ClustData,
   k_ids_mod=deepcopy(clust_data.k_ids)
   # if this particular original time series period is though represented in the extreme values, the new period number of the extreme value (clust_data.K+old number) is assigned to this original time series period - in case of feasibility they are all zero and nothing is changed
   k_ids_mod[findall(extr_vals.k_ids.!=0)]=extr_vals.k_ids[findall(extr_vals.k_ids.!=0)].+clust_data.K
-  return ClustData(clust_data.region,clust_data.years,K_mod,clust_data.T,data_mod,weights_mod,delta_t_mod,k_ids_mod;mean=clust_data.mean,sdv=clust_data.sdv)
+  return ClustData(clust_data.region,clust_data.years,K_mod,clust_data.T,data_mod,weights_mod,k_ids_mod;delta_t=delta_t_mod,mean=clust_data.mean,sdv=clust_data.sdv)
 end
 
 """
diff --git a/src/clustering/run_clust.jl b/src/clustering/run_clust.jl
@@ -24,42 +24,40 @@ function run_clust(data::ClustData;
     # When adding new methods: add combination of clust+rep to sup_kw_args
     check_kw_args(norm_op,norm_scope,method,representation)
 
-    # normalize
-    # TODO: implement 0-1 normalization and add as a choice to runclust
-    data_norm = z_normalize(data;scope=norm_scope)
-    if !isempty(attribute_weights)
-      data_norm = attribute_weighting(data_norm,attribute_weights)
-    end
-    data_norm_merged = ClustDataMerged(data_norm)
-
     #clustering
-    b_merged, cost, cost_best, iter =run_clust(data_norm_merged, data; method=method, representation=representation, n_clust=n_clust, n_init=n_init, iterations=iterations, orig_k_ids=deepcopy(data.k_ids), kwargs...)
-
-     if n_seg!=b_merged.T &&  n_seg!=0
-       b_merged=intraperiod_segmentation(b_merged;n_seg=n_seg,norm_scope=norm_scope,iterations=iterations)
-     else
-       n_seg=b_merged.T
+    clust_data, cost, centers_all, weights_all, clustids_all, cost_all, iter_all =run_clust_method(data;norm_op=norm_op, norm_scope=norm_scope, method=method, representation=representation, n_clust=n_clust, n_init=n_init, iterations=iterations, attribute_weights=attribute_weights, orig_k_ids=deepcopy(data.k_ids), kwargs...)
+      
+     # inter period segmentation (reduce the number of time steps per cluster - not fully implemented yet)
+       if n_seg!=data.T &&  n_seg!=0
+       clust_data_merged = ClustDataMerged(clust_data) 
+       segmented_merged=intraperiod_segmentation(clust_data_merged;n_seg=n_seg,norm_scope=norm_scope,iterations=iterations)
+       clust_data = ClustData(segmented_merged)
+     else # if interperiod segmentation is not used
+       n_seg=clust_data.T
      end
 
-    # transfer into ClustData format
-    best_results = ClustData(b_merged)
+    # set configuration file
     clust_config = set_clust_config(;norm_op=norm_op, norm_scope=norm_scope, method=method, representation=representation, n_clust=n_clust, n_seg=n_seg, n_init=n_init, iterations=iterations, attribute_weights=attribute_weights)
-    # save all locally converged solutions and the best into a struct
-
+    
     if get_all_clust_results
-      clust_result = ClustResultAll(best_results,b_merged.k_ids,cost_best,data_norm_merged.data_type,clust_config,b_merged.centers,b_merged.weights,b_merged.k_ids,cost,iter)
+      # save all locally converged solutions and the best into a struct
+      clust_result = ClustResultAll(clust_data,cost,clust_config,centers_all,weights_all,clustids_all,cost_all,iter_all)
     else
-      clust_result =  ClustResultBest(best_results,b_merged.k_ids,cost_best,data_norm_merged.data_type,clust_config)
+      # save best locally converged solution into a struct
+      clust_result =  ClustResult(clust_data,cost,clust_config)
     end
-    #TODO save in save file
+    #TODO save in save file  save_clust_result()
     return clust_result
 end
 
 """
-function run_clust(data_norm_merged::ClustDataMerged;
+    run_clust_method(data::ClustData;
+                  norm_op::String="zscore",
+                  norm_scope::String="full",
                   method::String="kmeans",
                   representation::String="centroid",
                   n_clust::Int=5,
+                  n_seg::Int=data.T,
                   n_init::Int=100,
                   iterations::Int=300,
                   orig_k_ids::Array{Int,1}=Array{Int,1}(),
@@ -68,16 +66,27 @@ function run_clust(data_norm_merged::ClustDataMerged;
 method: "kmeans","kmedoids","kmedoids_exact","hierarchical"
 representation: "centroid","medoid"
 """
-function run_clust(data_norm_merged::ClustDataMerged,
-                  data::ClustData;
+function run_clust_method(data::ClustData;
+                  norm_op::String="zscore",
+                  norm_scope::String="full",
                   method::String="kmeans",
                   representation::String="centroid",
                   n_clust::Int=5,
+                  n_seg::Int=data.T,
                   n_init::Int=100,
                   iterations::Int=300,
+                  attribute_weights::Dict{String,Float64}=Dict{String,Float64}(),
                   orig_k_ids::Array{Int,1}=Array{Int,1}(),
                   kwargs...)
-    # initialize data arrays
+    # normalize
+    # TODO: implement 0-1 normalization and add as a choice to runclust
+    data_norm = z_normalize(data;scope=norm_scope)
+    if !isempty(attribute_weights)
+      data_norm = attribute_weighting(data_norm,attribute_weights)
+    end
+    data_norm_merged = ClustDataMerged(data_norm)
+   
+    # initialize data arrays (all initial starting points)
     centers = Array{Array{Float64},1}(undef,n_init)
     clustids = Array{Array{Int,1},1}(undef,n_init)
     weights = Array{Array{Float64},1}(undef,n_init)
@@ -97,16 +106,18 @@ function run_clust(data_norm_merged::ClustDataMerged,
         centers[i] = resize_medoids(data,centers[i],weights[i])
       end
     end
-    # find best
-    # TODO: write as function
+    # find best. TODO: write as function
     cost_best,ind_mincost = findmin(cost)  # along dimension 2, only store indice
 
     k_ids=orig_k_ids
     k_ids[findall(orig_k_ids.!=0)]=clustids[ind_mincost]
     # save in merged format as array
+    
     # NOTE if you need clustered data more precise than 8 digits change the following line accordingly
-     n_digits_data_round=8 # Gurobi throws warning when rounding errors on order~1e-13 are passed in. Rounding errors occur in clustering of many zeros (e.g. solar).
-     return ClustDataMerged(data_norm_merged.region,data_norm_merged.years,n_clust,data_norm_merged.T,round.(centers[ind_mincost]; digits=n_digits_data_round),data_norm_merged.data_type,weights[ind_mincost],k_ids), cost, cost_best, iter
+    n_digits_data_round=8 # Gurobi throws warning when rounding errors on order~1e-13 are passed in. Rounding errors occur in clustering of many zeros (e.g. solar).
+    clust_data_merged = ClustDataMerged(data.region,data.years,n_clust,data.T,round.(centers[ind_mincost]; digits=n_digits_data_round),data_type(data),weights[ind_mincost],k_ids)
+    clust_data = ClustData(clust_data_merged)
+        return clust_data, cost_best, centers, weights, clustids, cost, iter
  end
 
 """
@@ -129,7 +140,7 @@ function run_clust(
       save::String="",
       kwargs...
     )
-    results_ar = Array{ClustResult,1}(undef,length(n_clust_ar))
+    results_ar = Array{AbstractClustResult,1}(undef,length(n_clust_ar))
     for i=1:length(n_clust_ar)
       results_ar[i] = run_clust(data;norm_op=norm_op,norm_scope=norm_scope,method=method,representation=representation,n_init=n_init,n_clust=n_clust_ar[i],iterations=iterations,save=save,kwargs...)
     end
diff --git a/src/utils/datastructs.jl b/src/utils/datastructs.jl
@@ -2,7 +2,7 @@
 abstract type InputData end
 abstract type TSData <:InputData end
 abstract type OptData <: InputData end
-abstract type ClustResult end
+abstract type AbstractClustResult end
 
 "FullInputData"
 struct FullInputData <: TSData
@@ -53,34 +53,23 @@ struct ClustDataMerged <: TSData
 end
 
 "ClustResultAll"
-struct ClustResultAll <: ClustResult
- best_results::ClustData
- best_ids::Array{Int,1}
- best_cost::Float64
- data_type::Array{String}
- clust_config::Dict{String,Any}
- centers::Array{Array{Float64},1}
- weights::Array{Array{Float64},1}
- clustids::Array{Array{Int,1},1}
- cost::Array{Float64,1}
- iter::Array{Int,1}
+struct ClustResultAll <: AbstractClustResult
+ clust_data::ClustData
+ cost::Float64
+ config::Dict{String,Any}
+ centers_all::Array{Array{Float64},1}
+ weights_all::Array{Array{Float64},1}
+ clustids_all::Array{Array{Int,1},1}
+ cost_all::Array{Float64,1}
+ iter_all::Array{Int,1}
 end
 
 # TODO: not used yet, but maybe best to implement this one later for users who just want to use clustering but do not care about the locally converged solutions
-"ClustResultBest"
-struct ClustResultBest <: ClustResult
- best_results::ClustData
- best_ids::Array{Int,1}
- best_cost::Float64
- data_type::Array{String}
- clust_config::Dict{String,Any}
-end
-
-"ClustResultSimple"
-struct ClustResultSimple <: ClustResult
- best_results::ClustData
- #TODO: clust_data::ClustData
- clust_config::Dict{String,Any}
+"ClustResult"
+struct ClustResult <: AbstractClustResult
+ clust_data::ClustData
+ cost::Float64
+ config::Dict{String,Any}
 end
 
 "SimpleExtremeValueDescr"
@@ -226,8 +215,8 @@ function ClustData(region::String,
                        T::Int,
                        data::Dict{String,Array},
                        weights::Array{Float64},
-                       delta_t::Array{Float64,2},
                        k_ids::Array{Int,1};
+                       delta_t::Array{Float64,2}=ones(T,K),
                        mean::Dict{String,Array}=Dict{String,Array}(),
                        sdv::Dict{String,Array}=Dict{String,Array}()
                        )
@@ -268,7 +257,7 @@ function ClustData(data::FullInputData,
   for (k,v) in data.data
      data_reshape[k] =  reshape(v,T,K)
   end
-  return ClustData(data.region,data.years,K,T,data_reshape,ones(K),ones(T,K),collect(1:K))
+  return ClustData(data.region,data.years,K,T,data_reshape,ones(K),collect(1:K))
 end
 
 """
diff --git a/src/utils/utils.jl b/src/utils/utils.jl
diff --git a/test/capacityexpansion.jl b/test/capacityexpansion.jl
diff --git a/test/test_workflow_introduction.jl b/test/test_workflow_introduction.jl
