Update ClustResult, AbstractClustResult, ClustResultAll. Update corresponding constructors and all methods that use these.

Author: holgerteichgraeber

docs/src/clust.md

@@ -29,16 +29,15 @@ FullInputData
 ClustData
 ClustDataMerged
 ClustResultAll
-ClustResultBest
-ClustResultSimple
+ClustResult
 ```
 
 ## Example running clustering
 ```@example
 using ClustForOpt
 # load ts-input-data
 ts_input_data = load_timeseries_data(normpath(joinpath(@__DIR__,"..","..","data","TS_GER_1")); T=24, years=[2016])
-ts_clust_data = run_clust(ts_input_data).best_results
+ts_clust_data = run_clust(ts_input_data).clust_data
 using Plots
 plot(ts_clust_data.data["solar-germany"], legend=false, linestyle=:solid, width=3, xlabel="Time [h]", ylabel="Solar availability factor [%]")
 savefig("clust.svg")

examples/workflow_example_bat.jl

@@ -17,15 +17,15 @@ end
 
  # optimization
 
-opt_res = run_opt("battery",clust_res_ar[2].best_results)
- #opt_res = run_opt("gas_turbine",clust_res.best_results[5])
+opt_res = run_opt("battery",clust_res_ar[2].clust_data)
+ #opt_res = run_opt("gas_turbine",clust_res.clust_data[5])
 
  ###
  # run optimization for all k=1:9
 opt_res_all = []
 obj=[]
 for i=1:2
-  push!(opt_res_all,run_opt("battery", clust_res_ar[i].best_results))
+  push!(opt_res_all,run_opt("battery", clust_res_ar[i].clust_data))
   push!(obj,opt_res_all[i].obj)
 end
  # run reference case

examples/workflow_example_extr.jl

@@ -18,4 +18,4 @@ ts_input_data = load_timeseries_data(data_path; T=24, years=[2015])
 ts_clust_res = run_clust(ts_input_data_mod;method="kmeans",representation="centroid",n_init=10,n_clust=5) # default k-means
 
 # representation modification
-ts_clust_extr = representation_modification(extr_vals,ts_clust_res.best_results)
+ts_clust_extr = representation_modification(extr_vals,ts_clust_res.clust_data)

examples/workflow_introduction.jl

@@ -66,7 +66,7 @@ end
 #############
 # Quick example and investigation of the best result:
 ts_clust_result = run_clust(ts_input_data; method="kmeans", representation="centroid", n_init=5, n_clust=5) # note that you should use n_init=1000 at least for kmeans.
-ts_clust_data = ts_clust_result.best_results
+ts_clust_data = ts_clust_result.clust_data
 # And some plotting:
 plot_comb_solar=plot!(plot_input_solar, ts_clust_data.data["solar-germany"], linestyle=:solid, width=3)
 plot_clust_soar=plot(ts_clust_data.data["el_demand-germany"], legend=false, linestyle=:solid, width=3, xlabel="Time [h]", ylabel="Solar availability factor [%]")

src/ClustForOpt.jl

@@ -22,10 +22,9 @@ module ClustForOpt
           FullInputData,
           ClustData,
           ClustDataMerged,
-          ClustResult,
-          ClustResultSimple,
+          AbstractClustResult,
           ClustResultAll,
-          ClustResultBest,
+          ClustResult,
           SimpleExtremeValueDescr,
           load_timeseries_data,
           combine_timeseries_weather_data,
@@ -35,6 +34,7 @@ module ClustForOpt
           get_sup_kw_args,
           run_clust,
           run_opt,
+          data_type,
           get_EUR_to_USD, #TODO Check which of the following should really be exported
           z_normalize,
           undo_z_normalize,

src/clustering/extreme_vals.jl

@@ -140,7 +140,7 @@ function input_data_modification(data::ClustData,extr_val_idcs::Array{Int,1})
   #just modify the k_ids that are also represented within this clust-data (don't reduce 0 to -1...)
   k_ids_dn[findall(data.k_ids.!=0)]=k_ids_dn_data
   #return the new Clust Data
-  return ClustData(data.region,data.years,K_dn,data.T,data_dn,weights_dn,deltas_dn,k_ids_dn;mean=data.mean,sdv=data.sdv)
+  return ClustData(data.region,data.years,K_dn,data.T,data_dn,weights_dn,k_ids_dn;delta_t=deltas_dn,mean=data.mean,sdv=data.sdv)
 end
 
 """
@@ -186,7 +186,7 @@ function extreme_val_output(data::ClustData,
     @error("rep_mod_method - "*rep_mod_method*" - does not exist")
   end
   delta_t_ed=data.delta_t[:,unique_extr_val_idcs]
-  extr_vals = ClustData(data.region,data.years,K_ed,data.T,data_ed,weights_ed,delta_t_ed,k_ids_ed;mean=data.mean,sdv=data.sdv)
+  extr_vals = ClustData(data.region,data.years,K_ed,data.T,data_ed,weights_ed,k_ids_ed;delta_t=delta_t_ed,mean=data.mean,sdv=data.sdv)
   return extr_vals
 end
 
@@ -219,7 +219,7 @@ function representation_modification(extr_vals::ClustData,
   k_ids_mod=deepcopy(clust_data.k_ids)
   # if this particular original time series period is though represented in the extreme values, the new period number of the extreme value (clust_data.K+old number) is assigned to this original time series period - in case of feasibility they are all zero and nothing is changed
   k_ids_mod[findall(extr_vals.k_ids.!=0)]=extr_vals.k_ids[findall(extr_vals.k_ids.!=0)].+clust_data.K
-  return ClustData(clust_data.region,clust_data.years,K_mod,clust_data.T,data_mod,weights_mod,delta_t_mod,k_ids_mod;mean=clust_data.mean,sdv=clust_data.sdv)
+  return ClustData(clust_data.region,clust_data.years,K_mod,clust_data.T,data_mod,weights_mod,k_ids_mod;delta_t=delta_t_mod,mean=clust_data.mean,sdv=clust_data.sdv)
 end
 
 """

src/clustering/run_clust.jl

@@ -24,42 +24,40 @@ function run_clust(data::ClustData;
     # When adding new methods: add combination of clust+rep to sup_kw_args
     check_kw_args(norm_op,norm_scope,method,representation)
 
-    # normalize
-    # TODO: implement 0-1 normalization and add as a choice to runclust
-    data_norm = z_normalize(data;scope=norm_scope)
-    if !isempty(attribute_weights)
-      data_norm = attribute_weighting(data_norm,attribute_weights)
-    end
-    data_norm_merged = ClustDataMerged(data_norm)
-
     #clustering
-    b_merged, cost, cost_best, iter =run_clust(data_norm_merged, data; method=method, representation=representation, n_clust=n_clust, n_init=n_init, iterations=iterations, orig_k_ids=deepcopy(data.k_ids), kwargs...)
-
-     if n_seg!=b_merged.T &&  n_seg!=0
-       b_merged=intraperiod_segmentation(b_merged;n_seg=n_seg,norm_scope=norm_scope,iterations=iterations)
-     else
-       n_seg=b_merged.T
+    clust_data, cost, centers_all, weights_all, clustids_all, cost_all, iter_all =run_clust_method(data;norm_op=norm_op, norm_scope=norm_scope, method=method, representation=representation, n_clust=n_clust, n_init=n_init, iterations=iterations, attribute_weights=attribute_weights, orig_k_ids=deepcopy(data.k_ids), kwargs...)
+      
+     # inter period segmentation (reduce the number of time steps per cluster - not fully implemented yet)
+       if n_seg!=data.T &&  n_seg!=0
+       clust_data_merged = ClustDataMerged(clust_data) 
+       segmented_merged=intraperiod_segmentation(clust_data_merged;n_seg=n_seg,norm_scope=norm_scope,iterations=iterations)
+       clust_data = ClustData(segmented_merged)
+     else # if interperiod segmentation is not used
+       n_seg=clust_data.T
      end
 
-    # transfer into ClustData format
-    best_results = ClustData(b_merged)
+    # set configuration file
     clust_config = set_clust_config(;norm_op=norm_op, norm_scope=norm_scope, method=method, representation=representation, n_clust=n_clust, n_seg=n_seg, n_init=n_init, iterations=iterations, attribute_weights=attribute_weights)
-    # save all locally converged solutions and the best into a struct
-
+    
     if get_all_clust_results
-      clust_result = ClustResultAll(best_results,b_merged.k_ids,cost_best,data_norm_merged.data_type,clust_config,b_merged.centers,b_merged.weights,b_merged.k_ids,cost,iter)
+      # save all locally converged solutions and the best into a struct
+      clust_result = ClustResultAll(clust_data,cost,clust_config,centers_all,weights_all,clustids_all,cost_all,iter_all)
     else
-      clust_result =  ClustResultBest(best_results,b_merged.k_ids,cost_best,data_norm_merged.data_type,clust_config)
+      # save best locally converged solution into a struct
+      clust_result =  ClustResult(clust_data,cost,clust_config)
     end
-    #TODO save in save file
+    #TODO save in save file  save_clust_result()
     return clust_result
 end
 
 """
-function run_clust(data_norm_merged::ClustDataMerged;
+    run_clust_method(data::ClustData;
+                  norm_op::String="zscore",
+                  norm_scope::String="full",
                   method::String="kmeans",
                   representation::String="centroid",
                   n_clust::Int=5,
+                  n_seg::Int=data.T,
                   n_init::Int=100,
                   iterations::Int=300,
                   orig_k_ids::Array{Int,1}=Array{Int,1}(),
@@ -68,16 +66,27 @@ function run_clust(data_norm_merged::ClustDataMerged;
 method: "kmeans","kmedoids","kmedoids_exact","hierarchical"
 representation: "centroid","medoid"
 """
-function run_clust(data_norm_merged::ClustDataMerged,
-                  data::ClustData;
+function run_clust_method(data::ClustData;
+                  norm_op::String="zscore",
+                  norm_scope::String="full",
                   method::String="kmeans",
                   representation::String="centroid",
                   n_clust::Int=5,
+                  n_seg::Int=data.T,
                   n_init::Int=100,
                   iterations::Int=300,
+                  attribute_weights::Dict{String,Float64}=Dict{String,Float64}(),
                   orig_k_ids::Array{Int,1}=Array{Int,1}(),
                   kwargs...)
-    # initialize data arrays
+    # normalize
+    # TODO: implement 0-1 normalization and add as a choice to runclust
+    data_norm = z_normalize(data;scope=norm_scope)
+    if !isempty(attribute_weights)
+      data_norm = attribute_weighting(data_norm,attribute_weights)
+    end
+    data_norm_merged = ClustDataMerged(data_norm)
+   
+    # initialize data arrays (all initial starting points)
     centers = Array{Array{Float64},1}(undef,n_init)
     clustids = Array{Array{Int,1},1}(undef,n_init)
     weights = Array{Array{Float64},1}(undef,n_init)
@@ -97,16 +106,18 @@ function run_clust(data_norm_merged::ClustDataMerged,
         centers[i] = resize_medoids(data,centers[i],weights[i])
       end
     end
-    # find best
-    # TODO: write as function
+    # find best. TODO: write as function
     cost_best,ind_mincost = findmin(cost)  # along dimension 2, only store indice
 
     k_ids=orig_k_ids
     k_ids[findall(orig_k_ids.!=0)]=clustids[ind_mincost]
     # save in merged format as array
+    
     # NOTE if you need clustered data more precise than 8 digits change the following line accordingly
-     n_digits_data_round=8 # Gurobi throws warning when rounding errors on order~1e-13 are passed in. Rounding errors occur in clustering of many zeros (e.g. solar).
-     return ClustDataMerged(data_norm_merged.region,data_norm_merged.years,n_clust,data_norm_merged.T,round.(centers[ind_mincost]; digits=n_digits_data_round),data_norm_merged.data_type,weights[ind_mincost],k_ids), cost, cost_best, iter
+    n_digits_data_round=8 # Gurobi throws warning when rounding errors on order~1e-13 are passed in. Rounding errors occur in clustering of many zeros (e.g. solar).
+    clust_data_merged = ClustDataMerged(data.region,data.years,n_clust,data.T,round.(centers[ind_mincost]; digits=n_digits_data_round),data_type(data),weights[ind_mincost],k_ids)
+    clust_data = ClustData(clust_data_merged)
+        return clust_data, cost_best, centers, weights, clustids, cost, iter
  end
 
 """
@@ -129,7 +140,7 @@ function run_clust(
       save::String="",
       kwargs...
     )
-    results_ar = Array{ClustResult,1}(undef,length(n_clust_ar))
+    results_ar = Array{AbstractClustResult,1}(undef,length(n_clust_ar))
     for i=1:length(n_clust_ar)
       results_ar[i] = run_clust(data;norm_op=norm_op,norm_scope=norm_scope,method=method,representation=representation,n_init=n_init,n_clust=n_clust_ar[i],iterations=iterations,save=save,kwargs...)
     end

src/utils/datastructs.jl

@@ -2,7 +2,7 @@
 abstract type InputData end
 abstract type TSData <:InputData end
 abstract type OptData <: InputData end
-abstract type ClustResult end
+abstract type AbstractClustResult end
 
 "FullInputData"
 struct FullInputData <: TSData
@@ -53,34 +53,23 @@ struct ClustDataMerged <: TSData
 end
 
 "ClustResultAll"
-struct ClustResultAll <: ClustResult
- best_results::ClustData
- best_ids::Array{Int,1}
- best_cost::Float64
- data_type::Array{String}
- clust_config::Dict{String,Any}
- centers::Array{Array{Float64},1}
- weights::Array{Array{Float64},1}
- clustids::Array{Array{Int,1},1}
- cost::Array{Float64,1}
- iter::Array{Int,1}
+struct ClustResultAll <: AbstractClustResult
+ clust_data::ClustData
+ cost::Float64
+ config::Dict{String,Any}
+ centers_all::Array{Array{Float64},1}
+ weights_all::Array{Array{Float64},1}
+ clustids_all::Array{Array{Int,1},1}
+ cost_all::Array{Float64,1}
+ iter_all::Array{Int,1}
 end
 
 # TODO: not used yet, but maybe best to implement this one later for users who just want to use clustering but do not care about the locally converged solutions
-"ClustResultBest"
-struct ClustResultBest <: ClustResult
- best_results::ClustData
- best_ids::Array{Int,1}
- best_cost::Float64
- data_type::Array{String}
- clust_config::Dict{String,Any}
-end
-
-"ClustResultSimple"
-struct ClustResultSimple <: ClustResult
- best_results::ClustData
- #TODO: clust_data::ClustData
- clust_config::Dict{String,Any}
+"ClustResult"
+struct ClustResult <: AbstractClustResult
+ clust_data::ClustData
+ cost::Float64
+ config::Dict{String,Any}
 end
 
 "SimpleExtremeValueDescr"
@@ -226,8 +215,8 @@ function ClustData(region::String,
                        T::Int,
                        data::Dict{String,Array},
                        weights::Array{Float64},
-                       delta_t::Array{Float64,2},
                        k_ids::Array{Int,1};
+                       delta_t::Array{Float64,2}=ones(T,K),
                        mean::Dict{String,Array}=Dict{String,Array}(),
                        sdv::Dict{String,Array}=Dict{String,Array}()
                        )
@@ -268,7 +257,7 @@ function ClustData(data::FullInputData,
   for (k,v) in data.data
      data_reshape[k] =  reshape(v,T,K)
   end
-  return ClustData(data.region,data.years,K,T,data_reshape,ones(K),ones(T,K),collect(1:K))
+  return ClustData(data.region,data.years,K,T,data_reshape,ones(K),collect(1:K))
 end
 
 """

src/utils/utils.jl

@@ -40,7 +40,7 @@ function z_normalize(data::ClustData;
  for (k,v) in data.data
    data_norm[k],mean[k],sdv[k] = z_normalize(v,scope=scope)
  end
- return ClustData(data.region,data.years,data.K,data.T,data_norm,data.weights,data.delta_t,data.k_ids;mean=mean,sdv=sdv)
+ return ClustData(data.region,data.years,data.K,data.T,data_norm,data.weights,data.k_ids;delta_t=data.delta_t,mean=mean,sdv=sdv)
 end
 
 """
@@ -327,7 +327,7 @@ function run_pure_clust(data::ClustData;
                             get_all_clust_results::Bool=false,
                             kwargs...)
   clust_result=run_clust(data;norm_op=norm_op,norm_scope=norm_scope,method=method,representation=representation,n_clust=n_clust,n_init=n_init,iterations=iterations,attribute_weights=attribute_weights)
-  clust_data=clust_result.best_results
+  clust_data=clust_result.clust_data
   mod_data=deepcopy(data.data)
   for i in 1:clust_data.K
     index=findall(clust_data.k_ids.==i)
@@ -338,5 +338,19 @@ function run_pure_clust(data::ClustData;
       end
     end
   end
-  return ClustResultSimple(ClustData(data.region, data.years, data.K, data.T, mod_data, data.weights, data.delta_t, data.k_ids), clust_result.clust_config)
+  return ClustResult(ClustData(data.region, data.years, data.K, data.T, mod_data, data.weights, data.k_ids;delta_t=data.delta_t),clust_result.cost, clust_result.config)
 end
+
+"""
+    data_type(clust_data::ClustData)
+Get data_type from a struct ClustData
+"""
+function data_type(data::ClustData)
+  n_datasets = length(keys(data.data))
+  data_type=String[]
+  for (k,v) in data.data
+    push!(data_type,k)
+  end
+  return data_type
+end
+

test/capacityexpansion.jl

@@ -10,7 +10,7 @@ using Clp
         # run clustering
         ts_clust_res = run_clust(ts_input_data;method="kmeans",representation="centroid",n_init=1,n_clust=365) # default k-means
         # run optimization
-        model = run_opt(ts_clust_res.best_results,cep_input_data,Clp.Optimizer)
+        model = run_opt(ts_clust_res.clust_data,cep_input_data,Clp.Optimizer)
         # compare to exact result
         exact_res=[70540.26439790576;0.0;8498.278397905757;0.0;80132.88454450261]
         @test exact_res ≈ model.variables["CAP"].data[:,1,1] atol=1
@@ -26,13 +26,13 @@ using Clp
         ts_full_data = run_clust(ts_input_data;method="hierarchical",representation="centroid",n_init=1,n_clust=30)
         ## OPTIMIZATION ##
         optimizer=Clp.Optimizer
-        scenarios["$state-$years-co2"] = run_opt(ts_clust_data.best_results,cep_data,optimizer;descriptor="co2",co2_limit=1000)
-        scenarios["$state-$years-slack"] = run_opt(ts_clust_data.best_results,cep_data,optimizer;descriptor="slack",lost_el_load_cost=1e6, lost_CO2_emission_cost=700)
-        scenarios["$state-$years-ex"] = run_opt(ts_clust_data.best_results,cep_data,optimizer;descriptor="ex",existing_infrastructure=true)
-        scenarios["$state-$years-simple"] = run_opt(ts_clust_data.best_results,cep_data,optimizer;descriptor="simple storage",storage="simple")
-        scenarios["$state-$years-seasonal"] = run_opt(ts_clust_data.best_results,cep_data,optimizer;descriptor="seasonal storage",storage="seasonal")
-        design_result=run_opt(ts_clust_data.best_results,cep_data,optimizer;descriptor="des&op")
-        scenarios["$state-$years-des&op"] = run_opt(ts_full_data.best_results,cep_data,design_result.opt_config,get_cep_design_variables(design_result),optimizer;lost_el_load_cost=1e6,lost_CO2_emission_cost=700)
+        scenarios["$state-$years-co2"] = run_opt(ts_clust_data.clust_data,cep_data,optimizer;descriptor="co2",co2_limit=1000)
+        scenarios["$state-$years-slack"] = run_opt(ts_clust_data.clust_data,cep_data,optimizer;descriptor="slack",lost_el_load_cost=1e6, lost_CO2_emission_cost=700)
+        scenarios["$state-$years-ex"] = run_opt(ts_clust_data.clust_data,cep_data,optimizer;descriptor="ex",existing_infrastructure=true)
+        scenarios["$state-$years-simple"] = run_opt(ts_clust_data.clust_data,cep_data,optimizer;descriptor="simple storage",storage="simple")
+        scenarios["$state-$years-seasonal"] = run_opt(ts_clust_data.clust_data,cep_data,optimizer;descriptor="seasonal storage",storage="seasonal")
+        design_result=run_opt(ts_clust_data.clust_data,cep_data,optimizer;descriptor="des&op")
+        scenarios["$state-$years-des&op"] = run_opt(ts_full_data.clust_data,cep_data,design_result.opt_config,get_cep_design_variables(design_result),optimizer;lost_el_load_cost=1e6,lost_CO2_emission_cost=700)
         end
     end
     #Test transmission for a multi-node scenario
@@ -44,7 +44,7 @@ using Clp
            ts_clust_data = run_clust(ts_input_data;method="hierarchical",representation="centroid",n_init=1,n_clust=3)
            ## OPTIMIZATION ##
            optimizer=Clp.Optimizer
-           scenarios["$state-$years-trans"] = run_opt(ts_clust_data.best_results,cep_data,optimizer;descriptor="trans",transmission=true)
+           scenarios["$state-$years-trans"] = run_opt(ts_clust_data.clust_data,cep_data,optimizer;descriptor="trans",transmission=true)
        end
     end
     #Test exact values for each of the previously calculated scenarios by comparison with exact scenarios