Merge pull request #80 from holgerteichgraeber/Float64

holgerteichgraeber · web-flow · commit 486f3690dfff · 2019-04-30T17:57:02.000-07:00
Float64 instead of AbstractFloat and testing added
diff --git a/.travis.yml b/.travis.yml
@@ -13,18 +13,19 @@ after_success:
 
 jobs:
   include:
-    - stage: "Documentation"
+    - stage: "Testing"
       julia: 1.0
       os: linux
       script:
-        - julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
-        - julia --project=docs/ docs/make.jl
+        - julia --project=test/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
+        - julia --project=test/ test/clustering.jl
+        - julia --project=test/ test/capacityexpansion.jl
       after_success:
-    - stage: "Testing"
+    - stage: "Documentation"
       julia: 1.0
       os: linux
       script:
-        - julia --project=test/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
-        - julia --project=test/ test/clustering.jl
+        - julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
+        - julia --project=docs/ docs/make.jl
       after_success:
 services: docker
diff --git a/Project.toml b/Project.toml
@@ -4,7 +4,7 @@ keywords = ["clustering", "JuMP", "optimization"]
 license = "MIT"
 desc = "julia implementation of using different clustering methods for finding representative periods for the optimization of energy systems"
 author = ["Holger Teichgraeber"]
-version = "0.3.3"
+version = "0.3.4"
 
 [deps]
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
diff --git a/examples/workflow_introduction.jl b/examples/workflow_introduction.jl
@@ -14,7 +14,7 @@ ts_input_data = load_timeseries_data(data_path; T=24, years=[2016])
 
 #= ClustData
 How the struct is setup:
-    ClustData{region::String,K::Int,T::Int,data::Dict{String,Array},weights::Array{AbstractFloat},mean::Dict{String,Array},sdv::Dict{String,Array}} <: TSData
+    ClustData{region::String,K::Int,T::Int,data::Dict{String,Array},weights::Array{Float64},mean::Dict{String,Array},sdv::Dict{String,Array}} <: TSData
 -region: specifies region data belongs to
 -K: number of periods
 -T: time steps per period
diff --git a/src/clustering/attribute_weighting.jl b/src/clustering/attribute_weighting.jl
@@ -1,10 +1,10 @@
 """
-function attribute_weighting(data::ClustData,attribute_weights::Dict{String,AbstractFloat})
+function attribute_weighting(data::ClustData,attribute_weights::Dict{String,Float64})
 
 apply the different attribute weights based on the dictionary entry for each tech or exact name
 """
 function attribute_weighting(data::ClustData,
-                              attribute_weights::Dict{String,AbstractFloat}
+                              attribute_weights::Dict{String,Float64}
                               )
   for name in keys(data.data)
     tech=split(name,"-")[1]
diff --git a/src/clustering/exact_kmedoids.jl b/src/clustering/exact_kmedoids.jl
@@ -2,15 +2,15 @@
 
 "Holds results of kmedoids run"
 mutable struct kmedoidsResult
-    medoids::Array{AbstractFloat}
+    medoids::Array{Float64}
     assignments::Array{Int}
-    totalcost::AbstractFloat
+    totalcost::Float64
 end
 
 
 """
     kmedoids_exact(
-     data::Array{AbstractFloat},
+     data::Array{Float64},
      nclust::Int,
      _dist::SemiMetric = SqEuclidean(),
      env::Any;
@@ -21,7 +21,7 @@ Performs the exact kmedoids algorithm as in Kotzur et al, 2017
 optimizer=Gurobi.Optimizer
 """
 function kmedoids_exact(
-   data::Array{AbstractFloat},
+   data::Array{Float64},
    nclust::Int,
    optimizer::DataType;
    _dist::SemiMetric = SqEuclidean(),
diff --git a/src/clustering/extreme_vals.jl b/src/clustering/extreme_vals.jl
@@ -67,7 +67,7 @@ function simple_extr_val_ident(data::ClustData,
 end
 
 """
-    simple_extr_val_ident(data::Array{AbstractFloat};extremum="max",peak_def="absolute")
+    simple_extr_val_ident(data::Array{Float64};extremum="max",peak_def="absolute")
 identifies a single simple extreme period from the data and returns column index of extreme period
 - `data_type`: any attribute from the attributes contained within *data*
 - `extremum`: "min" or "max"
diff --git a/src/clustering/intraperiod_segmentation.jl b/src/clustering/intraperiod_segmentation.jl
@@ -28,10 +28,10 @@ function intraperiod_segmentation(data_merged::ClustDataMerged;
 end
 
 """
-      run_clust_segmentation(period::Array{AbstractFloat,2};n_seg::Int=24,iterations::Int=300,norm_scope::String="full")
+      run_clust_segmentation(period::Array{Float64,2};n_seg::Int=24,iterations::Int=300,norm_scope::String="full")
 !!! Not yet proven implementation of segmentation introduced by Bahl et al. 2018
 """
-function run_clust_segmentation(period::Array{AbstractFloat,2};
+function run_clust_segmentation(period::Array{Float64,2};
                 n_seg::Int=24,
                 iterations::Int=300,
                 norm_scope::String="full")
diff --git a/src/clustering/run_clust.jl b/src/clustering/run_clust.jl
@@ -1,6 +1,6 @@
 
 """
-    run_clust(data::ClustData;norm_op::String="zscore",norm_scope::String="full",method::String="kmeans",representation::String="centroid",n_clust::Int=5,n_init::Int=100,iterations::Int=300,save::String="",attribute_weights::Dict{String,AbstractFloat}=Dict{String,AbstractFloat}(),get_all_clust_results::Bool=false,kwargs...)
+    run_clust(data::ClustData;norm_op::String="zscore",norm_scope::String="full",method::String="kmeans",representation::String="centroid",n_clust::Int=5,n_init::Int=100,iterations::Int=300,save::String="",attribute_weights::Dict{String,Float64}=Dict{String,Float64}(),get_all_clust_results::Bool=false,kwargs...)
 norm_op: "zscore", "01"(not implemented yet)
 norm_scope: "full","sequence","hourly"
 method: "kmeans","kmedoids","kmedoids_exact","hierarchical"
@@ -15,7 +15,7 @@ function run_clust(data::ClustData;
       n_seg::Int=data.T,
       n_init::Int=100,
       iterations::Int=300,
-      attribute_weights::Dict{String,AbstractFloat}=Dict{String,AbstractFloat}(),
+      attribute_weights::Dict{String,Float64}=Dict{String,Float64}(),
       save::String="",#QUESTION dead?
       get_all_clust_results::Bool=false,
       kwargs...
@@ -78,10 +78,10 @@ function run_clust(data_norm_merged::ClustDataMerged,
                   orig_k_ids::Array{Int,1}=Array{Int,1}(),
                   kwargs...)
     # initialize data arrays
-    centers = Array{Array{AbstractFloat},1}(undef,n_init)
+    centers = Array{Array{Float64},1}(undef,n_init)
     clustids = Array{Array{Int,1},1}(undef,n_init)
-    weights = Array{Array{AbstractFloat},1}(undef,n_init)
-    cost = Array{AbstractFloat,1}(undef,n_init)
+    weights = Array{Array{Float64},1}(undef,n_init)
+    cost = Array{Float64,1}(undef,n_init)
     iter = Array{Int,1}(undef,n_init)
 
     # clustering
@@ -346,7 +346,7 @@ end
 Helper function to run run_clust_hierarchical_centroids and run_clust_hierarchical_medoid
 """
 function run_clust_hierarchical(
-    data::Array{AbstractFloat,2},
+    data::Array{Float64,2},
     n_clust::Int,
     iterations::Int;
     _dist::SemiMetric = SqEuclidean()
diff --git a/src/clustering/shape_based/cluster_gen_dbaclust_parallel.jl b/src/clustering/shape_based/cluster_gen_dbaclust_parallel.jl
@@ -64,7 +64,7 @@ writetable(joinpath("outfiles",string("parameters_dtw_",region,".txt")),df)
 
  # Function that can be an input to pmap
 
- @everywhere function dbac_par_sc(n_clust::Int,i::Int,rad_sc::Int,seq::Array{AbstractFloat,2},n_init::Int,iterations::Int,inner_iterations::Int) # function to use with pmap to parallelize sc band calculation
+ @everywhere function dbac_par_sc(n_clust::Int,i::Int,rad_sc::Int,seq::Array{Float64,2},n_init::Int,iterations::Int,inner_iterations::Int) # function to use with pmap to parallelize sc band calculation
 
   rmin,rmax=sakoe_chiba_band(rad_sc,24)
 
diff --git a/src/utils/datastructs.jl b/src/utils/datastructs.jl
@@ -13,7 +13,7 @@ struct FullInputData <: TSData
 end
 
 """
-      ClustData{region::String,K::Int,T::Int,data::Dict{String,Array},weights::Array{AbstractFloat,2},mean::Dict{String,Array},sdv::Dict{String,Array},delta_t::Array{AbstractFloat,2},k_ids::Array{Int}} <: TSData
+      ClustData{region::String,K::Int,T::Int,data::Dict{String,Array},weights::Array{Float64,2},mean::Dict{String,Array},sdv::Dict{String,Array},delta_t::Array{Float64,2},k_ids::Array{Int}} <: TSData
 - region: optional information to specify the region data belongs to
 - K: number of periods
 - T: time steps per period
@@ -30,10 +30,10 @@ struct ClustData <: TSData
  K::Int
  T::Int
  data::Dict{String,Array}
- weights::Array{AbstractFloat}
+ weights::Array{Float64}
  mean::Dict{String,Array}
  sdv::Dict{String,Array}
- delta_t::Array{AbstractFloat,2}
+ delta_t::Array{Float64,2}
  k_ids::Array{Int}
 end
 
@@ -45,24 +45,24 @@ struct ClustDataMerged <: TSData
  T::Int
  data::Array
  data_type::Array{String}
- weights::Array{AbstractFloat}
+ weights::Array{Float64}
  mean::Dict{String,Array}
  sdv::Dict{String,Array}
- delta_t::Array{AbstractFloat,2}
+ delta_t::Array{Float64,2}
  k_ids::Array{Int}
 end
 
 "ClustResultAll"
 struct ClustResultAll <: ClustResult
  best_results::ClustData
  best_ids::Array{Int,1}
- best_cost::AbstractFloat
+ best_cost::Float64
  data_type::Array{String}
  clust_config::Dict{String,Any}
- centers::Array{Array{AbstractFloat},1}
- weights::Array{Array{AbstractFloat},1}
+ centers::Array{Array{Float64},1}
+ weights::Array{Array{Float64},1}
  clustids::Array{Array{Int,1},1}
- cost::Array{AbstractFloat,1}
+ cost::Array{Float64,1}
  iter::Array{Int,1}
 end
 
@@ -71,7 +71,7 @@ end
 struct ClustResultBest <: ClustResult
  best_results::ClustData
  best_ids::Array{Int,1}
- best_cost::AbstractFloat
+ best_cost::Float64
  data_type::Array{String}
  clust_config::Dict{String,Any}
 end
@@ -149,10 +149,10 @@ end
                          el_demand::Array=[],
                          solar::Array=[],
                          wind::Array=[],
-                         weights::Array{AbstractFloat}=ones(K),
+                         weights::Array{Float64}=ones(K),
                          mean::Dict{String,Array}=Dict{String,Array}(),
                          sdv::Dict{String,Array}=Dict{String,Array}(),
-                         delta_t::Array{AbstractFloat,2}=ones(T,K),
+                         delta_t::Array{Float64,2}=ones(T,K),
                          k_ids::Array{Int,1}=collect(1:K)
                          )
 constructor 1 for ClustData: provide data individually
@@ -165,10 +165,10 @@ function ClustData(region::String,
                          el_demand::Array=[],
                          solar::Array=[],
                          wind::Array=[],
-                         weights::Array{AbstractFloat}=ones(K),
+                         weights::Array{Float64}=ones(K),
                          mean::Dict{String,Array}=Dict{String,Array}(),
                          sdv::Dict{String,Array}=Dict{String,Array}(),
-                         delta_t::Array{AbstractFloat,2}=ones(T,K),
+                         delta_t::Array{Float64,2}=ones(T,K),
                          k_ids::Array{Int,1}=collect(1:K)
                          )
    dt = Dict{String,Array}()
@@ -212,8 +212,8 @@ end
                       K::Int,
                       T::Int,
                       data::Dict{String,Array},
-                      weights::Array{AbstractFloat},
-                      delta_t::Array{AbstractFloat,2},
+                      weights::Array{Float64},
+                      delta_t::Array{Float64,2},
                       k_ids::Array{Int,1};
                       mean::Dict{String,Array}=Dict{String,Array}(),
                       sdv::Dict{String,Array}=Dict{String,Array}()
@@ -225,8 +225,8 @@ function ClustData(region::String,
                        K::Int,
                        T::Int,
                        data::Dict{String,Array},
-                       weights::Array{AbstractFloat},
-                       delta_t::Array{AbstractFloat,2},
+                       weights::Array{Float64},
+                       delta_t::Array{Float64,2},
                        k_ids::Array{Int,1};
                        mean::Dict{String,Array}=Dict{String,Array}(),
                        sdv::Dict{String,Array}=Dict{String,Array}()
@@ -278,9 +278,9 @@ end
                         T::Int,
                         data::Array,
                         data_type::Array{String},
-                        weights::Array{AbstractFloat},
+                        weights::Array{Float64},
                         k_ids::Array{Int,1};
-                        delta_t::Array{AbstractFloat,2}=ones(T,K),
+                        delta_t::Array{Float64,2}=ones(T,K),
                         mean::Dict{String,Array}=Dict{String,Array}(),
                         sdv::Dict{String,Array}=Dict{String,Array}()
                         )
@@ -292,9 +292,9 @@ function ClustDataMerged(region::String,
                        T::Int,
                        data::Array,
                        data_type::Array{String},
-                       weights::Array{AbstractFloat},
+                       weights::Array{Float64},
                        k_ids::Array{Int,1};
-                       delta_t::Array{AbstractFloat,2}=ones(T,K),
+                       delta_t::Array{Float64,2}=ones(T,K),
                        mean::Dict{String,Array}=Dict{String,Array}(),
                        sdv::Dict{String,Array}=Dict{String,Array}()
                        )
diff --git a/src/utils/load_data.jl b/src/utils/load_data.jl
@@ -81,7 +81,7 @@ function add_timeseries_data!(dt::Dict{String,Array},
             else
                 K=K_calc
             end
-            dt[data_name*"-"*string(column[1])]=AbstractFloat.(column[2][1:(Int(T*K))])
+            dt[data_name*"-"*string(column[1])]=Float64.(column[2][1:(Int(T*K))])
         end
     end
     return K
diff --git a/src/utils/utils.jl b/src/utils/utils.jl
@@ -212,7 +212,7 @@ function calc_medoids(data::Array,assignments::Array)
   K=maximum(assignments) #number of clusters
   n_per_period=size(data,1)
   n_periods =size(data,2)
-  SSE=AbstractFloat[]
+  SSE=Float64[]
   for i=1:K
     push!(SSE,Inf)
   end
@@ -310,7 +310,7 @@ function set_clust_config(;kwargs...)
 end
 
 """
-    run_pure_clust(data::ClustData; norm_op::String="zscore", norm_scope::String="full", method::String="kmeans", representation::String="centroid", n_clust_1::Int=5, n_clust_2::Int=3, n_seg::Int=data.T, n_init::Int=100, iterations::Int=300, attribute_weights::Dict{String,AbstractFloat}=Dict{String,AbstractFloat}(), clust::Array{String,1}=Array{String,1}(), get_all_clust_results::Bool=false, kwargs...)
+    run_pure_clust(data::ClustData; norm_op::String="zscore", norm_scope::String="full", method::String="kmeans", representation::String="centroid", n_clust_1::Int=5, n_clust_2::Int=3, n_seg::Int=data.T, n_init::Int=100, iterations::Int=300, attribute_weights::Dict{String,Float64}=Dict{String,Float64}(), clust::Array{String,1}=Array{String,1}(), get_all_clust_results::Bool=false, kwargs...)
 Replace the original timeseries of the attributes in clust with their clustered value
 """
 function run_pure_clust(data::ClustData;
@@ -322,7 +322,7 @@ function run_pure_clust(data::ClustData;
                             n_seg::Int=data.T,
                             n_init::Int=100,
                             iterations::Int=300,
-                            attribute_weights::Dict{String,AbstractFloat}=Dict{String,AbstractFloat}(),
+                            attribute_weights::Dict{String,Float64}=Dict{String,Float64}(),
                             clust::Array{String,1}=Array{String,1}(),
                             get_all_clust_results::Bool=false,
                             kwargs...)
diff --git a/test/Project.toml b/test/Project.toml
@@ -2,4 +2,4 @@
 Clp = "e2554f3b-3117-50c0-817c-e040a3ddf72d"
 StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
-
+CapacityExpansion = "c5f7ca31-19be-526c-ac1d-e0dcac83edb8"
diff --git a/test/capacityexpansion.jl b/test/capacityexpansion.jl
@@ -0,0 +1,60 @@
+using Test
+using CapacityExpansion
+using Clp
+
+@testset "CEP" begin
+    @testset "TX_1" begin #Compare to Merrick Testcase
+        # load data
+        ts_input_data = load_timeseries_data_provided("TX_1"; T=24, years=[2008])
+        cep_input_data=load_cep_data_provided("TX_1")
+        # run clustering
+        ts_clust_res = run_clust(ts_input_data;method="kmeans",representation="centroid",n_init=1,n_clust=365) # default k-means
+        # run optimization
+        model = run_opt(ts_clust_res.best_results,cep_input_data,Clp.Optimizer)
+        # compare to exact result
+        exact_res=[70540.26439790576;0.0;8498.278397905757;0.0;80132.88454450261]
+        @test exact_res ≈ model.variables["CAP"].data[:,1,1] atol=1
+    end
+    scenarios=Dict{String,OptResult}()
+    #Test the workflow for single node scenarios for Germany and California
+    @testset "workflow $state" for (state, years) in [["GER_1", [2016]],["CA_1", [2016]]] begin
+        # laod data
+        ts_input_data = load_timeseries_data_provided(state; T=24, years=years) #CEP
+        cep_data = load_cep_data_provided(state)
+        ## CLUSTERING ##
+        ts_clust_data = run_clust(ts_input_data;method="hierarchical",representation="centroid",n_init=1,n_clust=3)
+        ts_full_data = run_clust(ts_input_data;method="hierarchical",representation="centroid",n_init=1,n_clust=30)
+        ## OPTIMIZATION ##
+        optimizer=Clp.Optimizer
+        scenarios["$state-$years-co2"] = run_opt(ts_clust_data.best_results,cep_data,optimizer;descriptor="co2",co2_limit=1000)
+        scenarios["$state-$years-slack"] = run_opt(ts_clust_data.best_results,cep_data,optimizer;descriptor="slack",lost_el_load_cost=1e6, lost_CO2_emission_cost=700)
+        scenarios["$state-$years-ex"] = run_opt(ts_clust_data.best_results,cep_data,optimizer;descriptor="ex",existing_infrastructure=true)
+        scenarios["$state-$years-simple"] = run_opt(ts_clust_data.best_results,cep_data,optimizer;descriptor="simple storage",storage="simple")
+        scenarios["$state-$years-seasonal"] = run_opt(ts_clust_data.best_results,cep_data,optimizer;descriptor="seasonal storage",storage="seasonal")
+        design_result=run_opt(ts_clust_data.best_results,cep_data,optimizer;descriptor="des&op")
+        scenarios["$state-$years-des&op"] = run_opt(ts_full_data.best_results,cep_data,design_result.opt_config,get_cep_design_variables(design_result),optimizer;lost_el_load_cost=1e6,lost_CO2_emission_cost=700)
+        end
+    end
+    #Test transmission for a multi-node scenario
+    @testset "workflow $state" for (state, years) in [["GER_18", [2016]],["CA_14", [2016]]] begin
+           # laod data
+           ts_input_data = load_timeseries_data_provided(state; T=24, years=years) #CEP
+           cep_data = load_cep_data_provided(state)
+           ## CLUSTERING ##
+           ts_clust_data = run_clust(ts_input_data;method="hierarchical",representation="centroid",n_init=1,n_clust=3)
+           ## OPTIMIZATION ##
+           optimizer=Clp.Optimizer
+           scenarios["$state-$years-trans"] = run_opt(ts_clust_data.best_results,cep_data,optimizer;descriptor="trans",transmission=true)
+       end
+    end
+    #Test exact values for each of the previously calculated scenarios by comparison with exact scenarios
+    @load normpath(joinpath(dirname(@__FILE__),"cep_exact_data.jld2")) exact_scenarios
+    @testset "exact $k" for (k,v) in scenarios begin
+            #Test for each variable within this scenario
+            @testset "variable $kv" for (kv,vv) in v.variables begin
+                    @test sum((round.(exact_scenarios[k].variables[kv].data;sigdigits=4).==round.(vv.data;sigdigits=4)).==false)==0
+                end
+            end
+        end
+    end
+end
diff --git a/test/cep_exact_data.jld2 b/test/cep_exact_data.jld2
