减小 glmer 模型大小

Question

我是 R 新手，我正在使用 glmer 来拟合几个二项式模型，我只需要它们来调用 predict使用得到的概率。但是，我有一个非常大的数据集，即使只有一个模型的大小也会变得非常大:

> library(pryr)
> object_size(mod)
701 MB

相比之下，模型的系数大小相形见绌:

> object_size(coef(mod))
1.16 MB

拟合值的大小也是如此:

> object_size(fitted(mod))
25.6 MB

首先，我不明白为什么模型的对象尺寸这么大。它似乎包含用于拟合模型的原始数据框，但即使这样也没有考虑到大小。为什么这么大？

其次，是否可以将模型剥离为仅需要调用预测的部分？如果是这样，我将如何去做？我找到了一篇为 glm 完成的帖子在 http://blog.yhathq.com/posts/reducing-your-r-memory-footprint-by-7000x.html但似乎 glmer 模型的访问方式不同并且具有不同的组件。

任何帮助将不胜感激。

编辑:

深入了解模型的内部结构:

> object_size(getME(mod, "X"))
205 MB
> object_size(getME(mod, "Z"))
36.9 MB
> object_size(getME(mod, "Zt"))
38.4 MB
> object_size(getME(mod, "Ztlist"))
41.6 MB
> object_size(getME(mod, "mmList"))
38.4 MB
> object_size(getME(mod, "y"))
3.2 MB
> object_size(getME(mod, "mu"))
3.2 MB
> object_size(getME(mod, "u"))
18.4 kB
> object_size(getME(mod, "b"))
19.5 kB
> object_size(getME(mod, "Gp"))
56 B
> object_size(getME(mod, "Tp"))
472 B
> object_size(getME(mod, "L"))
15.5 MB
> object_size(getME(mod, "Lambda"))
38.1 kB
> object_size(getME(mod, "Lambdat"))
38.1 kB
> object_size(getME(mod, "Lind"))
9.22 kB
> object_size(getME(mod, "Tlist"))
936 B
> object_size(getME(mod, "A"))
38.4 MB
> object_size(getME(mod, "RX"))
30.3 kB
> object_size(getME(mod, "RZX"))
1.05 MB
> object_size(getME(mod, "sigma"))
48 B
> object_size(getME(mod, "flist"))
4.89 MB
> object_size(getME(mod, "fixef"))
4.5 kB
> object_size(getME(mod, "beta"))
496 B
> object_size(getME(mod, "theta"))
472 B
> object_size(getME(mod, "ST"))
936 B
> object_size(getME(mod, "REML"))
48 B
> object_size(getME(mod, "is_REML"))
48 B
> object_size(getME(mod, "n_rtrms"))
48 B
> object_size(getME(mod, "n_rfacs"))
48 B
> object_size(getME(mod, "N"))
256 B
> object_size(getME(mod, "n"))
256 B
> object_size(getME(mod, "p"))
256 B
> object_size(getME(mod, "q"))
256 B
> object_size(getME(mod, "p_i"))
408 B
> object_size(getME(mod, "l_i"))
408 B
> object_size(getME(mod, "q_i"))
408 B
> object_size(getME(mod, "mod"))
48 B
> object_size(getME(mod, "m_i"))
424 B
> object_size(getME(mod, "m"))
48 B
> object_size(getME(mod, "cnms"))
624 B
> object_size(getME(mod, "devcomp"))
2.21 kB
> object_size(getME(mod, "offset"))
3.2 MB

> get_obj_size(mod@resp, "RC")
                       [,1]
family            673355488
initialize        673355488
initialize#lmResp 673355488
ptr               673355488
resDev            673355488
updateMu          673355488
updateWts         673355488
wrss              673355488
eta                 3196024
mu                  3196024
n                   3196024
offset              3196024
sqrtrwt             3196024
sqrtXwt             3196024
weights             3196024
wtres               3196024
y                   3196024
Ptr                      40
> get_obj_size(mod@pp, "RC")
                   [,1]
beta          449419408
initialize    449419408
initializePtr 449419408
ldL2          449419408
ldRX2         449419408
linPred       449419408
ptr           449419408
setTheta      449419408
sqrL          449419408
u             449419408
X             204549128
V             182171288
Ut             38448168
Zt             38448168
LamtUt         38353248
Xwts            3196024
RZX             1047176
Lambdat           38136
VtV               26192
delu              18408
u0                18408
Utr               18408
Lind               9224
beta0               496
delb                496
Vtr                 496
theta                72
Ptr                  40

Answer 1

现在发布为不完整的答案:

library("lme4")
gm1 <- glmer(cbind(incidence, size - incidence) ~ period + (1 | herd),
               data = cbpp, family = binomial)
library("pryr") 
object_size(gm1)  ## 505 kB

按照 Steve Walker 的 S3/S4/Reference 类字典列出和提取字段:

get_obj_size <- function(obj,type="S4") {
    fields <- switch(type,
                     S4=slotNames(obj),
                     RC=ls(obj))
    get_field <- switch(type,
                     S4=function(x) slot(obj,x),
                     RC=function(x) obj[[x]])
    field_list <- setNames(lapply(fields,get_field),fields)
    cbind(sort(sapply(field_list,object_size),decreasing=TRUE))
}
get_obj_size(gm1)
##           [,1]
## resp    356620  ## 'response module'
## pp      355420  ## 'predictor module'
## frame     6640
## optinfo   1748
## devcomp   1424
## call      1244
## flist     1232
## cnms       224
## u          152
## beta        56
## Gp          32
## lower       32
## theta       32

值得进一步深入研究响应和预测模块，看看有什么/有什么大的，有一些信息将存储在这些组件的环境中的警告/复杂性

例如，我认为下面所有名义上相同大小的组件实际上并不是独立的，而是具有相同的环境......

get_obj_size(gm1@resp,"RC")
##                     [,1]
## initialize        356620
## initialize#lmResp 356620
## ptr               356620
## resDev            356620
## setOffset         356620
## updateMu          356620
## updateWts         356620
## wrss              356620
## family             26016
## eta                  472
## mu                   472
## n                    472
## offset               472
## sqrtrwt              472
## sqrtXwt              472
## weights              472
## wtres                472
## y                    472
## Ptr                   20

查看存储了哪些组件的另一种方法是使用 object_size(getME(model,component))并遍历通过 eval(formals(getME)$name) 列出的组件;这与内部存储信息的方式不太精确，但会让您了解保存(例如)固定效应或随机效应模型矩阵需要多少空间。

我在这方面做了更多工作并得到了部分解决方案，但仍有很多我似乎无法正确找到/修剪掉的内容( 注意 这需要最新版本的 lme4 Github:我不得不稍微修改 predict函数以削弱对内部结构的依赖)。

glmer_chop <- function(object) {
    newobj <- object
    newobj@frame <- model.frame(object)[0,]
    newobj@pp <- with(object@pp,
                   new("merPredD",
                       Lambdat=Lambdat,
                       Lind=Lind,
                       theta=theta,
                       u=u,u0=u0,
                       n=nrow(X),
                       X=matrix(1,nrow=nrow(X)),
                       Zt=Zt)) ## .sparseDiagonal(n,shape="g")))
    newobj@resp <- new("glmResp",family=binomial(),y=numeric(0))
    return(newobj)
}
get_obj_size(environment(fm2@pp$initialize),"RC")
fm1 <- glmer(use ~ urban+age+livch+(1|district), Contraception, binomial)
object_size(Contraception)  ## 133 kB
object_size(fm1)  ## 1.05 MB
object_size(fm2 <- glmer_chop(fm1))  ## 699 kB
get_obj_size(fm2)          ## 'pp' is 547200 bytes
get_obj_size(fm2@pp,"RC")  ## 'initialize' object is 547200
saveRDS(fm2,file="tmp.rds")
fm2 <- readRDS("tmp.rds")
object_size(fm2)  ## 796 kB
rm(fm1)
pp <- predict(fm2,newdata=Contraception)
object_size(fm2) ## still 796K; no sharing

最后请注意 compare_size(fm2)确认这里的大部分信息都存储在环境中，而不是对象本身(但我不知道 compare_size/ object.size 如何处理引用类...)