gpt4 book ai didi

r - 使用 "data.table"包合并数据帧时出错

转载 作者:行者123 更新时间:2023-12-03 18:26:04 27 4
gpt4 key购买 nike

以下是可重现的示例 我遇到并坚持的情况(这是我用来评估 合并数据集 以用于我的论文研究的各种方法的测试客户端)。

testData <- "https://github.com/abnova/test/blob/master/mergeTestData.zip?raw=true"

tmpFile <- tempfile()
tmpDir <- tempdir()

download.file(testData, tmpFile, method = 'curl',
extra = '-L', quiet = TRUE)
testFiles <- unzip(tmpFile, exdir = tmpDir)

# To enable desired merge option, uncomment corresponding line

#MERGE_OPTION <- "lapply_merge"
#MERGE_OPTION <- "lapply_merge2"
#MERGE_OPTION <- "reduce_merge"
#MERGE_OPTION <- "reduce_merge2"
#MERGE_OPTION <- "reshape"
#MERGE_OPTION <- "plyr"
#MERGE_OPTION <- "dplyr"
MERGE_OPTION <- "data.table"
#MERGE_OPTION <- "data.table2"

loadData <- function (dataFile) {

if (file.exists(dataFile)) {
data <- readRDS(dataFile)
}
else { # error() undefined - replaced for stop() for now
stop("Data file \'", dataFile, "\' not found! Run 'make' first.")
}
return (data)
}

loadDataSets <- function (dataDir) {

dataSets <- list()

dataFiles <- dir(dataDir, pattern='\\.rds$')
dataSets <- lapply(seq_along(dataFiles),
function(i) {
nameSplit <- strsplit(dataFiles[i], "\\.")
dataset <- nameSplit[[1]][1]
assign(dataset,
loadData(file.path(dataDir, dataFiles[i])))
return (get(dataset))
})
return (dataSets)
}

# load the datasets of transformed data
dataSets <- loadDataSets(tmpDir)

if (MERGE_OPTION == "lapply_merge") { # Option 1

flossData <- data.frame(dataSets[[1]][1])

# merge all loaded datasets by common column ("Project ID")
silent <- lapply(seq(2, length(dataSets)),
function(i) {merge(flossData, dataSets[[1]][i],
by = "Project ID",
all = TRUE)})
}

if (MERGE_OPTION == "lapply_merge2") { # Option 1

pids <- which(sapply(dataSets,
FUN=function(x) {'Project ID' %in% names(x)}))

flossData <- dataSets[[pids[1]]]

for (id in pids[2:length(pids)]) {
flossData <- merge(flossData, dataSets[[id]],
by='Project ID', all = TRUE)
}
}

if (MERGE_OPTION == "reduce_merge") { # Option 2

flossData <- Reduce(function(...)
merge(..., by.x = "row.names", by.y = "Project ID", all = TRUE),
dataSets)
}

# http://r.789695.n4.nabble.com/merge-multiple-data-frames-tt4331089.html#a4333772
if (MERGE_OPTION == "reduce_merge2") { # Option 2

mergeAll <- function(..., by = "Project ID", all = TRUE) {
dotArgs <- list(...)
dotNames <- lapply(dotArgs, names)
repNames <- Reduce(intersect, dotNames)
repNames <- repNames[repNames != by]
for(i in seq_along(dotArgs)){
wn <- which( (names(dotArgs[[i]]) %in% repNames) &
(names(dotArgs[[i]]) != by))
names(dotArgs[[i]])[wn] <- paste(names(dotArgs[[i]])[wn],
names(dotArgs)[[i]], sep = ".")
}
Reduce(function(x, y) merge(x, y, by = by, all = all), dotArgs)
}

flossData <- mergeAll(dataSets)
}

if (MERGE_OPTION == "reshape") { # Option 3

if (!suppressMessages(require(reshape))) install.packages('reshape')
library(reshape)
flossData <- reshape::merge_all(dataSets)
}

if (MERGE_OPTION == "plyr") { # Option 4

if (!suppressMessages(require(plyr))) install.packages('plyr')
library(plyr)
flossData <- plyr::join_all(dataSets)
}

if (MERGE_OPTION == "dplyr") { # Option 5

if (!suppressMessages(require(dplyr))) install.packages('dplyr')
library(dplyr)

flossData <- dataSets[[1]][1]
flossData <- lapply(dataSets[[1]][-1],
function(x) {dplyr::left_join(x, flossData)})
}

if (MERGE_OPTION == "data.table") { # Option 6

if (!suppressMessages(require(data.table)))
install.packages('data.table')
library(data.table)

flossData <- data.table(dataSets[[1]], key="Project ID")

for (id in 2:length(dataSets)) {
flossData <- merge(flossData, data.table(dataSets[[id]]),
by='Project ID', all.x = TRUE, all.y = FALSE)
}
}

# http://stackoverflow.com/a/17458887/2872891
if (MERGE_OPTION == "data.table2") { # Option 6

if (!suppressMessages(require(data.table)))
install.packages('data.table')
library(data.table)

DT <- data.table(dataSets[[1]], key="Project ID")
flossData <- lapply(dataSets[[1]][-1], function(x) DT[.(x)])
}

# Additional Transformations (see TODO above)

# convert presence of Repo URL to integer
flossData[["Repo URL"]] <- as.integer(flossData[["Repo URL"]] != "")

# convert License Restrictiveness' factor levels to integers
#flossData[["License Restrictiveness"]] <-
# as.integer(flossData[["License Restrictiveness"]])

# convert User Community Size from character to integer
flossData[["User Community Size"]] <-
as.integer(flossData[["User Community Size"]])

# remove NAs
#flossData <- flossData[complete.cases(flossData[,3]),]
rowsNA <- apply(flossData, 1, function(x) {any(is.na(x))})
flossData <- flossData[!rowsNA,]

print(str(flossData))

错误信息 如下:
Starting bmerge ...done in 0.001 secs
Starting bmerge ...done in 0.002 secs
Error in vecseq(f__, len__, if (allow.cartesian) NULL else as.integer(max(nrow(x), :

Join results in 121229 rows; more than 100000 = max(nrow(x),nrow(i)). Check for duplicate key values in i, each of which join to the same group in x over and over again. If that's ok, try including j and dropping by (by-without-by) so that j runs for each group to avoid the large allocation. If you are sure you wish to proceed, rerun with allow.cartesian=TRUE. Otherwise, please search for this error message in the FAQ, Wiki, Stack Overflow and datatable-help for advice.



当前问题 已启用 data.table选项,但是,因为它是同一个包,我也很感激下一个选项的建议,该选项使用 替代 data.table 语法 用于合并(尽管我觉得它太困惑了,但为了知识的完整性)。先感谢您!

最佳答案

我会以这种方式处理这个问题:

首先,有一条错误消息。它说什么?

Join results in 121229 rows; more than 100000 = max(nrow(x),nrow(i)). Check for duplicate key values in i, each of which join to the same group in x over and over again. If that's ok, try including j and dropping by (by-without-by) so that j runs for each group to avoid the large allocation. If you are sure you wish to proceed, rerun with allow.cartesian=TRUE. Otherwise, please search for this error message in the FAQ, Wiki, Stack Overflow and datatable-help for advice.



伟大的!但是我有很多我正在使用的数据集,很多包和很多功能。我必须将范围缩小到产生此错误的数据集。

一一测试:
ans1 = merge(as.data.table(dataSets[[1]]), as.data.table(dataSets[[2]]), 
all.x=TRUE, all.y=FALSE, by="Project ID")
## works fine.

ans2 = merge(as.data.table(dataSets[[1]]), as.data.table(dataSets[[3]]),
all.x=TRUE, all.y=FALSE, by="Project ID")
## same error

啊哈,同样的错误。

阅读错误消息的第二行:

所以, dataSets[[3]] 似乎发生了一些事情。 .它说要检查 i 中的重复键值.让我们这样做:
dim(dataSets[[3]])
# [1] 81487 3
dim(unique(as.data.table(dataSets[[3]]), by="Project ID"))
# [1] 49999 3

所以, dataSets[[3]]具有重复的“项目 ID”值,因此对于每个重复的值,来自 dataSets[[1]] 的所有匹配行返回 - 这是第二行的第二部分解释的内容: each of which join to the same group in x over and over again .

试用中 allow.cartesian=TRUE :

我知道有重复的 key ,但仍希望继续。但是错误消息提到了我们如何继续,添加“allow.cartesian=TRUE”。
ans2 = merge(as.data.table(dataSets[[1]]), as.data.table(dataSets[[3]]), 
all.x=TRUE, all.y=FALSE, by="Project ID", allow.cartesian=TRUE)

啊哈,现在它工作正常!那么 allow.cartesian = TRUE有什么用做?或者为什么要添加?错误消息说要在 stackoverflow 上搜索消息(在其他事情中)。

搜索 allow.cartesian=TRUE关于 SO:

搜索使我进入了这个 Why is allow.cartesian required at times when when joining data.tables with duplicate keys?问题,它解释了目的,并且在评论下还包含来自@Roland 的另一个链接: Merging data.tables uses more than 10 GB RAM这指向了所有开始它的最初问题。现在让我阅读那些帖子。

base::merge给出不同的结果?

现在, base::merge 是否返回不同的结果(100,000 行)?
dim(merge(dataSets[[1]], dataSets[[3]], all.x=TRUE, all.y=FALSE, by="Project ID"))
# [1] 121229 4

并不真地。它提供与使用 data.table 时相同的尺寸,但它并不关心是否有重复的键,而 data.table警告您合并结果的潜在爆炸性,并允许您做出明智的决定。

关于r - 使用 "data.table"包合并数据帧时出错,我们在Stack Overflow上找到一个类似的问题: https://stackoverflow.com/questions/25204859/

27 4 0
Copyright 2021 - 2024 cfsdn All Rights Reserved 蜀ICP备2022000587号
广告合作:1813099741@qq.com 6ren.com