Optimize drop_singletons!() (#260)

* Optimize drop_singletons!()

Multi-thread singleton-detection pass.
Stop obs counters at 2 to store them in UInt8.

* drop debug.jl

* Delete Manifest.toml

Author: droodman

.gitignore

@@ -1,2 +1,3 @@
 
 benchmark/.sublime2Terminal.jl
+.vscode/settings.json

src/fit.jl

@@ -165,7 +165,7 @@ function StatsAPI.fit(::Type{FixedEffectModel},
 
     n_singletons = 0
     if drop_singletons
-        n_singletons = drop_singletons!(esample, fes)
+        n_singletons = drop_singletons!(esample, fes, nthreads)
     end
 
     nobs = sum(esample)

src/utils/fixedeffects.jl

@@ -4,35 +4,57 @@
 ##
 ##############################################################################
 
-function drop_singletons!(esample, fes::Vector{<:FixedEffect})
-    ns = Int[]
-    for fe in Iterators.cycle(fes)
-        # break loop if number of singletons did not change since the last time fe was iterated on
-        if length(ns) >= length(fes) && sum(view(ns, (length(ns)-length(fes)+1):length(ns))) == ns[end-length(fes)+1]
-            break
+function drop_singletons!(esample, fes::Vector{<:FixedEffect}, nthreads)
+# Main._esample = copy(esample); Main._fes=copy(fes); Main._nthreads=nthreads
+    nsingletons = 0
+    dirtypasses = length(fes)
+    
+    bounds = ceil.(Int, (0:nthreads) * (length(esample) / nthreads))
+    chunks = [bounds[t]+1:bounds[t+1] for t ∈ 1:nthreads]
+  
+    caches = [Vector{UInt8}(undef, fes[i].n) for i ∈ eachindex(fes)]
+    for (fe,cache) ∈ Iterators.cycle(zip(fes,caches))  # if 1 set of FE, only need 1 pass
+        n = drop_singletons!(esample, fe, cache, chunks)
+        if iszero(n)
+            dirtypasses -= 1
+        else
+            nsingletons += n
+            dirtypasses = length(fes)  # restart counter
         end
-        push!(ns, drop_singletons!(esample, fe))
+        dirtypasses <= 1 && break  # done if there are N FE groups and at least last N-1 have been clean (includes case N=1)
     end
-    return sum(ns)
+  
+    nsingletons
 end
 
-function drop_singletons!(esample, fe::FixedEffect)
-    n = 0
-    cache = zeros(Int, fe.n)
-    @inbounds for i in eachindex(esample)
+function drop_singletons!(esample, fe::FixedEffect, cache, chunks)::Int
+# Main._cache = cache; Main._chunks = copy(chunks)
+    refs = fe.refs
+      
+    fill!(cache, zero(UInt8))
+    @inbounds for i ∈ eachindex(esample,refs)  # count obs in each FE group
         if esample[i]
-            cache[fe.refs[i]] += 1
+            cache[refs[i]] = min(0x02, cache[refs[i]] + 0x01)  # stop counting obs in a group at 2, to keep counters in 8-bit integers
         end
     end
-    @inbounds for i in eachindex(esample)
-        if esample[i] && (cache[fe.refs[i]] == 1)
-            esample[i] = false
-            n += 1
+
+    tasks = map(chunks) do chunk  # drop newly found singletons
+        Threads.@spawn begin
+            n = 0
+            @inbounds for i ∈ chunk
+                if esample[i] && isone(cache[refs[i]])
+                    esample[i] = false
+                    n += 1
+                end
+            end
+            n
         end
     end
-    return n
+
+    sum(fetch.(tasks))
 end
 
+
 ##############################################################################
 ##
 ## Number of distinct values (only ever call when fe without missing values)