Fix loading of non memmappable classical binary embeddings
Change-Id: I1c357e5f025a456528fad34685e2ad357294679f
diff --git a/src/DerekoVecs.jl b/src/DerekoVecs.jl
index e368413..b31eb2d 100644
--- a/src/DerekoVecs.jl
+++ b/src/DerekoVecs.jl
@@ -97,6 +97,34 @@
if (occursin(r".vecs$", modelfn) && isfile(vecsfn))
M = Mmap.mmap(vecsfn, Matrix{Float32}, (d, n))
d2vmodel(M, d, n, vocabdict, vocab, freqs, total, cdb, modelfn, basefn)
+ elseif (occursin(r".vecs$", modelfn) && isfile(modelfn))
+ M = Matrix{Float32}(undef, d, n)
+ open(modelfn) do io
+ readline(io) # skip header
+ float_buf = Vector{Float32}(undef, d)
+ for i in 1:n
+ # Skip word
+ while true
+ b = read(io, UInt8)
+ if b == UInt8(' ')
+ break
+ end
+ end
+ # Read floats
+ read!(io, float_buf)
+ nrm = norm(float_buf)
+ if nrm > 0
+ float_buf ./= nrm
+ end
+ M[:, i] .= float_buf
+ # Skip trailing newline if present
+ b = peek(io)
+ if b == UInt8('\n')
+ read(io, UInt8)
+ end
+ end
+ end
+ d2vmodel(M, d, n, vocabdict, vocab, freqs, total, cdb, modelfn, basefn)
else
d2vmodel(Matrix{Float32}(undef, 2, 0), 0, n, vocabdict, vocab, freqs, total, cdb, modelfn, basefn)
end