r - 无法连接 data.table 结果来自 R 中的 foreach 循环

Question

我在嵌套的 foreach 循环中使用 data.table 对象，但我无法按照我喜欢的方式创建结果对象。

基本上这个想法是生成长度(intersect(set1，set2))。我还想生成 length(union(set1, set2)) 和其他几个指标。

下面的代码和示例数据:

library(iterators)
library(data.table)
library(foreach)

#generate dummy data
set.seed(1212)
sample1 <- data.frame(parentid=round((runif(50000, min=1, max=50000))), childid=round(runif(100000, min=1, max=100000)))
length(unique(sample1$parentid))

#get unique parents
sample1uniq <- as.data.frame(unique(sample1$parentid))
names(sample1uniq) <- "parentid"

#convert original dataset to data.table
sample1 <- data.table(sample1)
setkey(sample1,parentid)

#convert unique ids to data.table
sample1uniq <- data.table(sample1uniq)
setkey(sample1uniq,parentid)

#a random sample of 5K to users to scan against
sample2uniq_idx <- sample(1:nrow(sample1uniq), size=5000)
sample2uniq <- sample1uniq[sample2uniq_idx]
sample2uniq <- data.table(sample2uniq)
setkey(sample2uniq,parentid)

#construct iterators
sample1uniq_iter <- iter(sample1uniq)
sample2uniq_iter <- iter(sample2uniq)

编辑 12/5/2013 使我的问题更清楚:

outerresults <- foreach (x = sample1uniq_iter, .combine=rbind, .packages=c('foreach','doParallel', 'data.table')) %dopar% {
  b <- sample1[J(x)]                          #ith parent
  b2 <- as.data.frame(b)[,2]  #ith parent's children

  foreach (y = sample2uniq_iter, .combine=rbind) %dopar% {
    c <- sample1[J(y)]                          #jth parent
    c2 <- as.data.frame(c)[,2]  #jth parent's children

    common <- length(intersect(b2, c2))

    results <- list(u1=x, u2=y, inter=common)        
  }  
}

我期待结果是这样的(编造的):

u1 u2 inter
1  2  10
1  3  4
1  4  7
1  5  6
2  3  10
2  4  4
3  5  7
4  5  6

相反，它以列表形式出现，其中 u1 和 u2 作为前 2 个元素，inter 作为 SUM(length(intersect(set1, set2)))。

感谢任何想法...

Answer 1

解决方案

您的主要问题是迭代器。请记住，数据表被许多事物(例如迭代器)视为列表，因此您生成的两个迭代器将分别迭代一个项目，即每个数据表中的单个列。如果您仔细查看结果:

> str(outerresults)
List of 3
 $ u1   : num [1:31602] 2 3 5 6 7 8 10 11 12 14 ...
 $ u2   : num [1:5000] 14 26 27 31 34 61 68 81 99 106 ...
 $ inter: int 14778

u1 基本上只是 sample1unique，u2 是 sample2unique，inter 是:

> length(intersect(sample1[J(sample1uniq)][,childid], sample1[J(sample2uniq)][,childid]))
[1] 14778

换句话说，您实际上根本没有遍历任何内容。

您遇到的另一个问题是这种方法(一旦解决了上述问题)非常慢。您正在通过 rbinding lists 大约 160MM 次来增长一个非常大的对象。这是个坏消息。我修复了它(更改了迭代器)并以更小的尺寸运行它以给您一个想法(100 x 20，或原始尺寸的 1/8000):

#generate dummy data
set.seed(1212)
sample1 <- data.frame(parentid=round((runif(50, min=1, max=50))), childid=round(runif(100, min=1, max=100)))
length(unique(sample1$parentid))

#get unique parents
sample1uniq <- as.data.frame(unique(sample1$parentid))
names(sample1uniq) <- "parentid"

#convert original dataset to data.table
sample1 <- data.table(sample1)
setkey(sample1,parentid)

#convert unique ids to data.table
sample1uniq <- data.table(sample1uniq)
setkey(sample1uniq,parentid)

#a random sample of 5K to users to scan against
sample2uniq_idx <- sample(1:nrow(sample1uniq), size=20)
sample2uniq <- sample1uniq[sample2uniq_idx]
sample2uniq <- data.table(sample2uniq)
setkey(sample2uniq,parentid)

# Notice how we don't use iterator objects

outerresults <- foreach (x = sample1uniq$parentid, .combine=rbind, .packages=c('foreach','doParallel', 'data.table')) %dopar% {
  b <- sample1[J(x)]                          #ith parent
  b2 <- as.data.frame(b)[,2]  #ith parent's children

  results <- foreach (y = sample2uniq$parentid, .combine=rbind) %dopar% {
    c <- sample1[J(y)]                          #jth parent
    c2 <- as.data.frame(c)[,2]  #jth parent's children

    common <- length(intersect(b2, c2))

    results <- list(u1=x, u2=y, inter=common)
    results
  }
}
#  user  system elapsed 
#  1.57    0.00    1.60 
head(outerresults)
#         u1 u2 inter
# result.1 2  2  4    
# result.2 2  4  0    
# result.3 2  7  0    
# result.4 2  7  0    
# result.5 2  8  2    
# result.6 2  8  2        

如果所有内容都正确缩放，则全尺寸将花费 3 个多小时。

优化

我认为你最好完全放弃循环并在周围使用 data.table:

# Prepare data in two data tables

vec.samp1 <- par.ids             # exact copy of what we generated earlier
vec.samp1.child <- child.ids     # exact copy of what we generated earlier
dt.s1 <- data.table(sample1=vec.samp1, sample1.child=vec.samp1.child, key="sample1")

vec.samp2 <- sample2.ids         # exact copy of what we generated earlier
dt.s2 <- dt.s1[data.table(sample2=vec.samp2)]
setnames(dt.s2, c("sample2", "sample2.child"))

# Create the cartesian join of our data sets and then
# join to get the child values

combinations <- CJ(sample1=vec.samp1, sample2=vec.samp2)
setkey(combinations, "sample1")
combinations <- combinations[dt.s1, allow.cartesian=T]
setkey(combinations, "sample2")
combinations <- combinations[dt.s2, allow.cartesian=T]

# Compute intersect and union

combinations[order(sample1, sample2), 
  list(
    intersect=length(intersect(sample1.child, sample2.child)),
    union=length(union(sample1.child, sample2.child))
  ),
  by=list(sample1, sample2)
]
#    user  system elapsed 
#    0.06    0.00    0.06 

#    sample1 sample2 intersect union
# 1:       2       2         4     4
# 2:       2       4         0     6
# 3:       2       7         0    10
# 4:       2       8         2    10
# 5:       2       9         0     6

结果相同，但速度提高了 25 倍(但请注意，data.table 版本仅报告 sample1-sample2 的独特组合)。