From 8ed10f6bf34671ebdb81a565af5890eafeae18a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kirill=20M=C3=BCller?= <kirill@cynkra.com>
Date: Mon, 23 Sep 2024 01:14:03 +0200
Subject: [PATCH 01/10] Copy from dplyr with search/replace

---
 duckplyr/VERSION                |   1 +
 duckplyr/groupby-duckplyr.R     | 170 ++++++++++++++++++++++++++++++++
 duckplyr/groupby2014-duckplyr.R | 100 +++++++++++++++++++
 duckplyr/join-duckplyr.R        | 112 +++++++++++++++++++++
 duckplyr/read-duckplyr.R        |  61 ++++++++++++
 duckplyr/setup-duckplyr.sh      |   6 ++
 duckplyr/sort-duckplyr.R        |  48 +++++++++
 duckplyr/upg-duckplyr.sh        |   6 ++
 duckplyr/ver-duckplyr.sh        |   4 +
 9 files changed, 508 insertions(+)
 create mode 100644 duckplyr/VERSION
 create mode 100755 duckplyr/groupby-duckplyr.R
 create mode 100755 duckplyr/groupby2014-duckplyr.R
 create mode 100755 duckplyr/join-duckplyr.R
 create mode 100755 duckplyr/read-duckplyr.R
 create mode 100755 duckplyr/setup-duckplyr.sh
 create mode 100755 duckplyr/sort-duckplyr.R
 create mode 100755 duckplyr/upg-duckplyr.sh
 create mode 100755 duckplyr/ver-duckplyr.sh

diff --git a/duckplyr/VERSION b/duckplyr/VERSION
new file mode 100644
index 00000000..267577d4
--- /dev/null
+++ b/duckplyr/VERSION
@@ -0,0 +1 @@
+0.4.1
diff --git a/duckplyr/groupby-duckplyr.R b/duckplyr/groupby-duckplyr.R
new file mode 100755
index 00000000..1a52f90a
--- /dev/null
+++ b/duckplyr/groupby-duckplyr.R
@@ -0,0 +1,170 @@
+#!/usr/bin/env Rscript
+
+cat("# groupby-duckplyr.R\n")
+
+source("./_helpers/helpers.R")
+
+stopifnot(requireNamespace(c("bit64","data.table"), quietly=TRUE)) # used in chk to sum numeric columns and data loading
+.libPaths("./duckplyr/r-duckplyr") # tidyverse/duckplyr#4641
+suppressPackageStartupMessages(library("duckplyr", lib.loc="./duckplyr/r-duckplyr", warn.conflicts=FALSE))
+ver = packageVersion("duckplyr")
+git = "" # uses stable version now #124
+task = "groupby"
+solution = "duckplyr"
+fun = "group_by"
+cache = TRUE
+on_disk = FALSE
+
+data_name = Sys.getenv("SRC_DATANAME")
+src_grp = file.path("data", paste(data_name, "csv", sep="."))
+cat(sprintf("loading dataset %s\n", data_name))
+
+x = as_tibble(data.table::fread(src_grp, showProgress=FALSE, stringsAsFactors=TRUE, na.strings="", data.table=FALSE))
+print(nrow(x))
+
+task_init = proc.time()[["elapsed"]]
+cat("grouping...\n")
+
+question = "sum v1 by id1" # q1
+t = system.time(print(dim(ans<-x %>% group_by(id1) %>% summarise(v1=sum(v1, na.rm=TRUE)))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ungroup(ans), v1=sum(bit64::as.integer64(v1))))[["elapsed"]]
+write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+rm(ans)
+t = system.time(print(dim(ans<-x %>% group_by(id1) %>% summarise(v1=sum(v1, na.rm=TRUE)))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ungroup(ans), v1=sum(bit64::as.integer64(v1))))[["elapsed"]]
+write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+print(head(ans, 3))
+print(tail(ans, 3))
+rm(ans)
+
+question = "sum v1 by id1:id2" # q2
+t = system.time(print(dim(ans<-x %>% group_by(id1, id2) %>% summarise(v1=sum(v1, na.rm=TRUE)))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ungroup(ans), v1=sum(bit64::as.integer64(v1))))[["elapsed"]]
+write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+rm(ans)
+t = system.time(print(dim(ans<-x %>% group_by(id1, id2) %>% summarise(v1=sum(v1, na.rm=TRUE)))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ungroup(ans), v1=sum(bit64::as.integer64(v1))))[["elapsed"]]
+write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+print(head(ans, 3))
+print(tail(ans, 3))
+rm(ans)
+
+question = "sum v1 mean v3 by id3" # q3
+t = system.time(print(dim(ans<-x %>% group_by(id3) %>% summarise(v1=sum(v1, na.rm=TRUE), v3=mean(v3, na.rm=TRUE)))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ungroup(ans), v1=sum(bit64::as.integer64(v1)), v3=sum(v3)))[["elapsed"]]
+write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+rm(ans)
+t = system.time(print(dim(ans<-x %>% group_by(id3) %>% summarise(v1=sum(v1, na.rm=TRUE), v3=mean(v3, na.rm=TRUE)))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ungroup(ans), v1=sum(bit64::as.integer64(v1)), v3=sum(v3)))[["elapsed"]]
+write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+print(head(ans, 3))
+print(tail(ans, 3))
+rm(ans)
+
+question = "mean v1:v3 by id4" # q4
+t = system.time(print(dim(ans<-x %>% group_by(id4) %>% summarise_at(.funs="mean", .vars=c("v1","v2","v3"), na.rm=TRUE))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ungroup(ans), v1=sum(v1), v2=sum(v2), v3=sum(v3)))[["elapsed"]]
+write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+rm(ans)
+t = system.time(print(dim(ans<-x %>% group_by(id4) %>% summarise_at(.funs="mean", .vars=c("v1","v2","v3"), na.rm=TRUE))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ungroup(ans), v1=sum(v1), v2=sum(v2), v3=sum(v3)))[["elapsed"]]
+write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+print(head(ans, 3))
+print(tail(ans, 3))
+rm(ans)
+
+question = "sum v1:v3 by id6" # q5
+t = system.time(print(dim(ans<-x %>% group_by(id6) %>% summarise_at(.funs="sum", .vars=c("v1","v2","v3"), na.rm=TRUE))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ungroup(ans), v1=sum(bit64::as.integer64(v1)), v2=sum(bit64::as.integer64(v2)), v3=sum(v3)))[["elapsed"]]
+write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+rm(ans)
+t = system.time(print(dim(ans<-x %>% group_by(id6) %>% summarise_at(.funs="sum", .vars=c("v1","v2","v3"), na.rm=TRUE))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ungroup(ans), v1=sum(bit64::as.integer64(v1)), v2=sum(bit64::as.integer64(v2)), v3=sum(v3)))[["elapsed"]]
+write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+print(head(ans, 3))
+print(tail(ans, 3))
+rm(ans)
+
+question = "median v3 sd v3 by id4 id5" # q6
+t = system.time(print(dim(ans<-x %>% group_by(id4, id5) %>% summarise(median_v3=median(v3, na.rm=TRUE), sd_v3=sd(v3, na.rm=TRUE)))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ungroup(ans), median_v3=sum(median_v3), sd_v3=sum(sd_v3)))[["elapsed"]]
+write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+rm(ans)
+t = system.time(print(dim(ans<-x %>% group_by(id4, id5) %>% summarise(median_v3=median(v3, na.rm=TRUE), sd_v3=sd(v3, na.rm=TRUE)))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ungroup(ans), median_v3=sum(median_v3), sd_v3=sum(sd_v3)))[["elapsed"]]
+write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+print(head(ans, 3))
+print(tail(ans, 3))
+rm(ans)
+
+question = "max v1 - min v2 by id3" # q7
+t = system.time(print(dim(ans<-x %>% group_by(id3) %>% summarise(range_v1_v2=max(v1, na.rm=TRUE)-min(v2, na.rm=TRUE)))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ungroup(ans), range_v1_v2=sum(bit64::as.integer64(range_v1_v2))))[["elapsed"]]
+write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+rm(ans)
+t = system.time(print(dim(ans<-x %>% group_by(id3) %>% summarise(range_v1_v2=max(v1, na.rm=TRUE)-min(v2, na.rm=TRUE)))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ungroup(ans), range_v1_v2=sum(bit64::as.integer64(range_v1_v2))))[["elapsed"]]
+write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+print(head(ans, 3))
+print(tail(ans, 3))
+rm(ans)
+
+question = "largest two v3 by id6" # q8
+t = system.time(print(dim(ans<-x %>% select(id6, largest2_v3=v3) %>% filter(!is.na(largest2_v3)) %>% arrange(desc(largest2_v3)) %>% group_by(id6) %>% filter(row_number() <= 2L))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ungroup(ans), largest2_v3=sum(largest2_v3)))[["elapsed"]]
+write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+rm(ans)
+t = system.time(print(dim(ans<-x %>% select(id6, largest2_v3=v3) %>% filter(!is.na(largest2_v3)) %>% arrange(desc(largest2_v3)) %>% group_by(id6) %>% filter(row_number() <= 2L))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ungroup(ans), largest2_v3=sum(largest2_v3)))[["elapsed"]]
+write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+print(head(ans, 3))
+print(tail(ans, 3))
+rm(ans)
+
+question = "regression v1 v2 by id2 id4" # q9
+t = system.time(print(dim(ans<-x %>% group_by(id2, id4) %>% summarise(r2=cor(v1, v2, use="na.or.complete")^2))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ungroup(ans), r2=sum(r2)))[["elapsed"]]
+write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+rm(ans)
+t = system.time(print(dim(ans<-x %>% group_by(id2, id4) %>% summarise(r2=cor(v1, v2, use="na.or.complete")^2))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ungroup(ans), r2=sum(r2)))[["elapsed"]]
+write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+print(head(ans, 3))
+print(tail(ans, 3))
+rm(ans)
+
+question = "sum v3 count by id1:id6" # q10
+t = system.time(print(dim(ans<-x %>% group_by(id1, id2, id3, id4, id5, id6) %>% summarise(v3=sum(v3, na.rm=TRUE), count=n()))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ungroup(ans), v3=sum(v3), count=sum(bit64::as.integer64(count))))[["elapsed"]]
+write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+rm(ans)
+t = system.time(print(dim(ans<-x %>% group_by(id1, id2, id3, id4, id5, id6) %>% summarise(v3=sum(v3, na.rm=TRUE), count=n()))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ungroup(ans), v3=sum(v3), count=sum(bit64::as.integer64(count))))[["elapsed"]]
+write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+print(head(ans, 3))
+print(tail(ans, 3))
+rm(ans)
+
+cat(sprintf("grouping finished, took %.0fs\n", proc.time()[["elapsed"]]-task_init))
+
+if( !interactive() ) q("no", status=0)
diff --git a/duckplyr/groupby2014-duckplyr.R b/duckplyr/groupby2014-duckplyr.R
new file mode 100755
index 00000000..e4a5ab12
--- /dev/null
+++ b/duckplyr/groupby2014-duckplyr.R
@@ -0,0 +1,100 @@
+#!/usr/bin/env Rscript
+
+cat("# groupby2014-duckplyr.R\n")
+
+source("./_helpers/helpers.R")
+
+stopifnot(requireNamespace(c("bit64","data.table"), quietly=TRUE)) # used in chk to sum numeric columns and data loading
+.libPaths("./duckplyr/r-duckplyr") # tidyverse/duckplyr#4641
+suppressPackageStartupMessages(library("duckplyr", lib.loc="./duckplyr/r-duckplyr", warn.conflicts=FALSE))
+ver = packageVersion("duckplyr")
+git = "" # uses stable version now #124
+task = "groupby2014"
+solution = "duckplyr"
+fun = "group_by"
+cache = TRUE
+on_disk = FALSE
+
+data_name = Sys.getenv("SRC_DATANAME")
+src_grp = file.path("data", paste(data_name, "csv", sep="."))
+cat(sprintf("loading dataset %s\n", data_name))
+
+x = as_tibble(data.table::fread(src_grp, showProgress=FALSE, data.table=FALSE))
+print(nrow(x))
+
+task_init = proc.time()[["elapsed"]]
+cat("grouping...\n")
+
+question = "sum v1 by id1" # q1
+t = system.time(print(dim(ans<-x %>% group_by(id1) %>% summarise(sum(v1)))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ans, v1=sum(bit64::as.integer64(`sum(v1)`))))[["elapsed"]]
+write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+rm(ans)
+t = system.time(print(dim(ans<-x %>% group_by(id1) %>% summarise(sum(v1)))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ans, v1=sum(bit64::as.integer64(`sum(v1)`))))[["elapsed"]]
+write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+print(head(ans, 3))
+print(tail(ans, 3))
+rm(ans)
+
+question = "sum v1 by id1:id2" # q2
+t = system.time(print(dim(ans<-x %>% group_by(id1,id2) %>% summarise(sum(v1)))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ungroup(ans), v1=sum(bit64::as.integer64(`sum(v1)`))))[["elapsed"]]
+write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+rm(ans)
+t = system.time(print(dim(ans<-x %>% group_by(id1,id2) %>% summarise(sum(v1)))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ungroup(ans), v1=sum(bit64::as.integer64(`sum(v1)`))))[["elapsed"]]
+write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+print(head(ans, 3))
+print(tail(ans, 3))
+rm(ans)
+
+question = "sum v1 mean v3 by id3" # q3
+t = system.time(print(dim(ans<-x %>% group_by(id3) %>% summarise(sum(v1), mean(v3)))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ans, v1=sum(bit64::as.integer64(`sum(v1)`)), v3=sum(`mean(v3)`)))[["elapsed"]]
+write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+rm(ans)
+t = system.time(print(dim(ans<-x %>% group_by(id3) %>% summarise(sum(v1), mean(v3)))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ans, v1=sum(bit64::as.integer64(`sum(v1)`)), v3=sum(`mean(v3)`)))[["elapsed"]]
+write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+print(head(ans, 3))
+print(tail(ans, 3))
+rm(ans)
+
+question = "mean v1:v3 by id4" # q4
+t = system.time(print(dim(ans<-x %>% group_by(id4) %>% summarise(across(v1:v3, mean)))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ans, v1=sum(v1), v2=sum(v2), v3=sum(v3)))[["elapsed"]]
+write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+rm(ans)
+t = system.time(print(dim(ans<-x %>% group_by(id4) %>% summarise(across(v1:v3, mean)))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ans, v1=sum(v1), v2=sum(v2), v3=sum(v3)))[["elapsed"]]
+write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+print(head(ans, 3))
+print(tail(ans, 3))
+rm(ans)
+
+question = "sum v1:v3 by id6" # q5
+t = system.time(print(dim(ans<-x %>% group_by(id6) %>% summarise(across(v1:v3, sum)))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ans, v1=sum(bit64::as.integer64(v1)), v2=sum(bit64::as.integer64(v2)), v3=sum(v3)))[["elapsed"]]
+write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+rm(ans)
+t = system.time(print(dim(ans<-x %>% group_by(id6) %>% summarise(across(v1:v3, sum)))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ans, v1=sum(bit64::as.integer64(v1)), v2=sum(bit64::as.integer64(v2)), v3=sum(v3)))[["elapsed"]]
+write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+print(head(ans, 3))
+print(tail(ans, 3))
+rm(ans)
+
+cat(sprintf("grouping finished, took %.0fs\n", proc.time()[["elapsed"]]-task_init))
+
+if( !interactive() ) q("no", status=0)
diff --git a/duckplyr/join-duckplyr.R b/duckplyr/join-duckplyr.R
new file mode 100755
index 00000000..870eadd6
--- /dev/null
+++ b/duckplyr/join-duckplyr.R
@@ -0,0 +1,112 @@
+#!/usr/bin/env Rscript
+
+cat("# join-duckplyr.R\n")
+
+source("./_helpers/helpers.R")
+
+stopifnot(requireNamespace(c("data.table"), quietly=TRUE)) # used for data loading
+.libPaths("./duckplyr/r-duckplyr") # tidyverse/duckplyr#4641
+suppressPackageStartupMessages(library("duckplyr", lib.loc="./duckplyr/r-duckplyr", warn.conflicts=FALSE))
+ver = packageVersion("duckplyr")
+git = "" # uses stable version now #124
+task = "join"
+solution = "duckplyr"
+cache = TRUE
+on_disk = FALSE
+
+data_name = Sys.getenv("SRC_DATANAME")
+src_jn_x = file.path("data", paste(data_name, "csv", sep="."))
+y_data_name = join_to_tbls(data_name)
+src_jn_y = setNames(file.path("data", paste(y_data_name, "csv", sep=".")), names(y_data_name))
+stopifnot(length(src_jn_y)==3L)
+cat(sprintf("loading datasets %s\n", paste(c(data_name, y_data_name), collapse=", ")))
+
+x = as_tibble(data.table::fread(src_jn_x, showProgress=FALSE, stringsAsFactors=TRUE, data.table=FALSE, na.strings=""))
+JN = lapply(sapply(simplify=FALSE, src_jn_y, data.table::fread, showProgress=FALSE, stringsAsFactors=TRUE, data.table=FALSE, na.strings=""), as_tibble)
+print(nrow(x))
+sapply(sapply(JN, nrow), print) -> nul
+small = JN$small
+medium = JN$medium
+big = JN$big
+
+task_init = proc.time()[["elapsed"]]
+cat("joining...\n")
+
+question = "small inner on int" # q1
+fun = "inner_join"
+t = system.time(print(dim(ans<-inner_join(x, small, by="id1"))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ans, sum(v1, na.rm=TRUE), sum(v2, na.rm=TRUE)))[["elapsed"]]
+write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+rm(ans)
+t = system.time(print(dim(ans<-inner_join(x, small, by="id1"))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ans, sum(v1, na.rm=TRUE), sum(v2, na.rm=TRUE)))[["elapsed"]]
+write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+print(head(ans, 3))
+print(tail(ans, 3))
+rm(ans)
+
+question = "medium inner on int" # q2
+fun = "inner_join"
+t = system.time(print(dim(ans<-inner_join(x, medium, by="id2"))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ans, sum(v1, na.rm=TRUE), sum(v2, na.rm=TRUE)))[["elapsed"]]
+write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+rm(ans)
+t = system.time(print(dim(ans<-inner_join(x, medium, by="id2"))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ans, sum(v1, na.rm=TRUE), sum(v2, na.rm=TRUE)))[["elapsed"]]
+write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+print(head(ans, 3))
+print(tail(ans, 3))
+rm(ans)
+
+question = "medium outer on int" # q3
+fun = "left_join"
+t = system.time(print(dim(ans<-left_join(x, medium, by="id2"))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ans, sum(v1, na.rm=TRUE), sum(v2, na.rm=TRUE)))[["elapsed"]]
+write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+rm(ans)
+t = system.time(print(dim(ans<-left_join(x, medium, by="id2"))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ans, sum(v1, na.rm=TRUE), sum(v2, na.rm=TRUE)))[["elapsed"]]
+write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+print(head(ans, 3))
+print(tail(ans, 3))
+rm(ans)
+
+question = "medium inner on factor" # q4
+fun = "inner_join"
+t = system.time(print(dim(ans<-inner_join(x, medium, by="id5"))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ans, sum(v1, na.rm=TRUE), sum(v2, na.rm=TRUE)))[["elapsed"]]
+write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+rm(ans)
+t = system.time(print(dim(ans<-inner_join(x, medium, by="id5"))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ans, sum(v1, na.rm=TRUE), sum(v2, na.rm=TRUE)))[["elapsed"]]
+write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+print(head(ans, 3))
+print(tail(ans, 3))
+rm(ans)
+
+question = "big inner on int" # q5
+fun = "inner_join"
+t = system.time(print(dim(ans<-inner_join(x, big, by="id3"))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ans, sum(v1, na.rm=TRUE), sum(v2, na.rm=TRUE)))[["elapsed"]]
+write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+rm(ans)
+t = system.time(print(dim(ans<-inner_join(x, big, by="id3"))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ans, sum(v1, na.rm=TRUE), sum(v2, na.rm=TRUE)))[["elapsed"]]
+write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
+print(head(ans, 3))
+print(tail(ans, 3))
+rm(ans)
+
+cat(sprintf("joining finished, took %.0fs\n", proc.time()[["elapsed"]]-task_init))
+
+if( !interactive() ) q("no", status=0)
diff --git a/duckplyr/read-duckplyr.R b/duckplyr/read-duckplyr.R
new file mode 100755
index 00000000..332682ce
--- /dev/null
+++ b/duckplyr/read-duckplyr.R
@@ -0,0 +1,61 @@
+#!/usr/bin/env Rscript
+
+cat("# read-duckplyr\n")
+
+source("./helpers.R")
+source("./duckplyr/helpers-duckplyr.R")
+
+suppressPackageStartupMessages({
+  library(readr, warn.conflicts=FALSE)
+  library(duckplyr, warn.conflicts=FALSE)
+})
+ver = NA_character_ #packageVersion("duckplyr")
+git = NA_character_ #duckplyr.git()
+task = "read"
+solution = "duckplyr"
+fun = "readr::read_csv"
+cache = TRUE
+
+src_grp = Sys.getenv("SRC_GRP_LOCAL")
+data_name = basename(src_grp)
+options("readr.show_progress"=FALSE)
+
+in_rows = as.numeric(strsplit(system(sprintf("wc -l %s", data_name), intern=TRUE), " ", fixed=TRUE)[[1L]][1L])-1
+
+cat("reading...\n")
+
+question = "all rows" #1
+t = system.time(print(dim(ans<-read_csv(data_name, col_types="ccciiiiid"))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ungroup(ans), v3=sum(v1)))[["elapsed"]]
+write.log(run=1L, task=task, data=data_name, in_rows=in_rows, question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt)
+rm(ans)
+t = system.time(print(dim(ans<-read_csv(data_name, col_types="ccciiiiid"))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ungroup(ans), v3=sum(v1)))[["elapsed"]]
+write.log(run=2L, task=task, data=data_name, in_rows=in_rows, question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt)
+rm(ans)
+t = system.time(print(dim(ans<-read_csv(data_name, col_types="ccciiiiid"))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ungroup(ans), v3=sum(v1)))[["elapsed"]]
+write.log(run=3L, task=task, data=data_name, in_rows=in_rows, question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt)
+rm(ans)
+
+question = "top 100 rows" #2
+t = system.time(print(dim(ans<-read_csv(data_name, n_max=100, col_types="ccciiiiid"))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ungroup(ans), v3=sum(v1)))[["elapsed"]]
+write.log(run=1L, task=task, data=data_name, in_rows=in_rows, question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt)
+rm(ans)
+t = system.time(print(dim(ans<-read_csv(data_name, n_max=100, col_types="ccciiiiid"))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ungroup(ans), v3=sum(v1)))[["elapsed"]]
+write.log(run=2L, task=task, data=data_name, in_rows=in_rows, question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt)
+rm(ans)
+t = system.time(print(dim(ans<-read_csv(data_name, n_max=100, col_types="ccciiiiid"))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ungroup(ans), v3=sum(v1)))[["elapsed"]]
+write.log(run=3L, task=task, data=data_name, in_rows=in_rows, question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt)
+rm(ans)
+
+if( !interactive() ) q("no", status=0)
diff --git a/duckplyr/setup-duckplyr.sh b/duckplyr/setup-duckplyr.sh
new file mode 100755
index 00000000..98c4ec8e
--- /dev/null
+++ b/duckplyr/setup-duckplyr.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+# install stable duckplyr
+mkdir -p ./duckplyr/r-duckplyr
+Rscript -e 'install.packages("duckplyr", lib="./duckplyr/r-duckplyr", repos = "http://cloud.r-project.org")'
diff --git a/duckplyr/sort-duckplyr.R b/duckplyr/sort-duckplyr.R
new file mode 100755
index 00000000..c4247b78
--- /dev/null
+++ b/duckplyr/sort-duckplyr.R
@@ -0,0 +1,48 @@
+#!/usr/bin/env Rscript
+
+cat("# sort-duckplyr\n")
+
+source("./helpers.R")
+source("./duckplyr/helpers-duckplyr.R")
+
+src_x = Sys.getenv("SRC_X", NA_character_)
+
+# if (get.nrow(src_x) > 1e9L) {
+#   cat("# sort with duckplyr skipped due data volume cap for single machine set to total 1e9 rows")
+#   quit("no", status=0) # datasets > 1e9 too big to try load on single machine
+# }
+
+stopifnot(requireNamespace("bit64", quietly=TRUE)) # used in chk to sum numeric columns
+suppressPackageStartupMessages(library(duckplyr, warn.conflicts=FALSE))
+ver = packageVersion("duckplyr")
+git = duckplyr.git()
+data_name = basename(src_x)
+task = "sort"
+solution = "duckplyr"
+fun = "arrange"
+question = "by int KEY"
+cache = TRUE
+
+cat("loading dataset...\n")
+X = data.table::fread(if(file.exists(basename(src_x))) basename(src_x) else sprintf("hadoop fs -cat %s", src_x), data.table=FALSE) # csv can be provided in local dir for faster import
+
+cat("sorting...\n")
+t = system.time(print(dim(ans<-arrange(X, KEY))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ans, sum(bit64::as.integer64(X2))))[["elapsed"]]
+write.log(run=1L, task=task, data=data_name, in_rows=nrow(X), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt)
+rm(ans)
+
+t = system.time(print(dim(ans<-arrange(X, KEY))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ans, sum(bit64::as.integer64(X2))))[["elapsed"]]
+write.log(run=2L, task=task, data=data_name, in_rows=nrow(X), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt)
+rm(ans)
+
+t = system.time(print(dim(ans<-arrange(X, KEY))))[["elapsed"]]
+m = memory_usage()
+chkt = system.time(chk<-summarise(ans, sum(bit64::as.integer64(X2))))[["elapsed"]]
+write.log(run=3L, task=task, data=data_name, in_rows=nrow(X), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt)
+rm(ans)
+
+if( !interactive() ) q("no", status=0)
diff --git a/duckplyr/upg-duckplyr.sh b/duckplyr/upg-duckplyr.sh
new file mode 100755
index 00000000..e9277ffb
--- /dev/null
+++ b/duckplyr/upg-duckplyr.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+# upgrade all packages in duckplyr library only if new duckplyr is out
+echo 'upgrading duckplyr...'
+Rscript -e 'ap=available.packages(); install.packages(c("duckplyr", "data.table"), lib="./duckplyr/r-duckplyr", ask=FALSE, checkBuilt=TRUE, quiet=TRUE)'
diff --git a/duckplyr/ver-duckplyr.sh b/duckplyr/ver-duckplyr.sh
new file mode 100755
index 00000000..5a8d9ad5
--- /dev/null
+++ b/duckplyr/ver-duckplyr.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+
+Rscript -e 'v=read.dcf(system.file(package="duckplyr", lib.loc="./duckplyr/r-duckplyr", "DESCRIPTION"), fields=c("Version","RemoteSha")); colnames(v)[colnames(v)=="RemoteSha"]="Revision"; cnafill=function(x) {x=c(x); x[is.na(x)]=""; x}; fw=function(f, v) writeLines(v, file.path("duckplyr", f)); invisible(mapply(fw, toupper(colnames(v)), cnafill(v)))'

From 7967d83ad6c65d06f2e62e991bf91ccc199cfe64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kirill=20M=C3=BCller?= <kirill@cynkra.com>
Date: Mon, 23 Sep 2024 01:16:14 +0200
Subject: [PATCH 02/10] as_duckplyr_tibble()

---
 duckplyr/groupby-duckplyr.R     | 2 +-
 duckplyr/groupby2014-duckplyr.R | 2 +-
 duckplyr/join-duckplyr.R        | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/duckplyr/groupby-duckplyr.R b/duckplyr/groupby-duckplyr.R
index 1a52f90a..bfefdd53 100755
--- a/duckplyr/groupby-duckplyr.R
+++ b/duckplyr/groupby-duckplyr.R
@@ -19,7 +19,7 @@ data_name = Sys.getenv("SRC_DATANAME")
 src_grp = file.path("data", paste(data_name, "csv", sep="."))
 cat(sprintf("loading dataset %s\n", data_name))
 
-x = as_tibble(data.table::fread(src_grp, showProgress=FALSE, stringsAsFactors=TRUE, na.strings="", data.table=FALSE))
+x = as_duckplyr_tibble(data.table::fread(src_grp, showProgress=FALSE, stringsAsFactors=TRUE, na.strings="", data.table=FALSE))
 print(nrow(x))
 
 task_init = proc.time()[["elapsed"]]
diff --git a/duckplyr/groupby2014-duckplyr.R b/duckplyr/groupby2014-duckplyr.R
index e4a5ab12..d6d5ff14 100755
--- a/duckplyr/groupby2014-duckplyr.R
+++ b/duckplyr/groupby2014-duckplyr.R
@@ -19,7 +19,7 @@ data_name = Sys.getenv("SRC_DATANAME")
 src_grp = file.path("data", paste(data_name, "csv", sep="."))
 cat(sprintf("loading dataset %s\n", data_name))
 
-x = as_tibble(data.table::fread(src_grp, showProgress=FALSE, data.table=FALSE))
+x = as_duckplyr_tibble(data.table::fread(src_grp, showProgress=FALSE, data.table=FALSE))
 print(nrow(x))
 
 task_init = proc.time()[["elapsed"]]
diff --git a/duckplyr/join-duckplyr.R b/duckplyr/join-duckplyr.R
index 870eadd6..fdcd6aab 100755
--- a/duckplyr/join-duckplyr.R
+++ b/duckplyr/join-duckplyr.R
@@ -21,8 +21,8 @@ src_jn_y = setNames(file.path("data", paste(y_data_name, "csv", sep=".")), names
 stopifnot(length(src_jn_y)==3L)
 cat(sprintf("loading datasets %s\n", paste(c(data_name, y_data_name), collapse=", ")))
 
-x = as_tibble(data.table::fread(src_jn_x, showProgress=FALSE, stringsAsFactors=TRUE, data.table=FALSE, na.strings=""))
-JN = lapply(sapply(simplify=FALSE, src_jn_y, data.table::fread, showProgress=FALSE, stringsAsFactors=TRUE, data.table=FALSE, na.strings=""), as_tibble)
+x = as_duckplyr_tibble(data.table::fread(src_jn_x, showProgress=FALSE, stringsAsFactors=TRUE, data.table=FALSE, na.strings=""))
+JN = lapply(sapply(simplify=FALSE, src_jn_y, data.table::fread, showProgress=FALSE, stringsAsFactors=TRUE, data.table=FALSE, na.strings=""), as_duckplyr_tibble)
 print(nrow(x))
 sapply(sapply(JN, nrow), print) -> nul
 small = JN$small

From 9199b52482a8ecad1bdcd0f890f1aa00d6d13447 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kirill=20M=C3=BCller?= <kirill@cynkra.com>
Date: Mon, 23 Sep 2024 01:52:47 +0200
Subject: [PATCH 03/10] duckplyr-ify

---
 duckplyr/groupby-duckplyr.R | 80 ++++++++++++++++++-------------------
 duckplyr/join-duckplyr.R    | 20 +++++-----
 2 files changed, 50 insertions(+), 50 deletions(-)

diff --git a/duckplyr/groupby-duckplyr.R b/duckplyr/groupby-duckplyr.R
index bfefdd53..a2221a53 100755
--- a/duckplyr/groupby-duckplyr.R
+++ b/duckplyr/groupby-duckplyr.R
@@ -26,140 +26,140 @@ task_init = proc.time()[["elapsed"]]
 cat("grouping...\n")
 
 question = "sum v1 by id1" # q1
-t = system.time(print(dim(ans<-x %>% group_by(id1) %>% summarise(v1=sum(v1, na.rm=TRUE)))))[["elapsed"]]
+t = system.time(print(dim(ans<-x %>% summarise(.by = id1, v1=sum(v1)))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ungroup(ans), v1=sum(bit64::as.integer64(v1))))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, v1=sum(v1)))[["elapsed"]]
 write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 rm(ans)
-t = system.time(print(dim(ans<-x %>% group_by(id1) %>% summarise(v1=sum(v1, na.rm=TRUE)))))[["elapsed"]]
+t = system.time(print(dim(ans<-x %>% summarise(.by = id1, v1=sum(v1)))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ungroup(ans), v1=sum(bit64::as.integer64(v1))))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, v1=sum(v1)))[["elapsed"]]
 write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 print(head(ans, 3))
 print(tail(ans, 3))
 rm(ans)
 
 question = "sum v1 by id1:id2" # q2
-t = system.time(print(dim(ans<-x %>% group_by(id1, id2) %>% summarise(v1=sum(v1, na.rm=TRUE)))))[["elapsed"]]
+t = system.time(print(dim(ans<-x %>% summarise(.by = c(id1, id2), v1=sum(v1)))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ungroup(ans), v1=sum(bit64::as.integer64(v1))))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, v1=sum(v1)))[["elapsed"]]
 write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 rm(ans)
-t = system.time(print(dim(ans<-x %>% group_by(id1, id2) %>% summarise(v1=sum(v1, na.rm=TRUE)))))[["elapsed"]]
+t = system.time(print(dim(ans<-x %>% summarise(.by = c(id1, id2), v1=sum(v1)))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ungroup(ans), v1=sum(bit64::as.integer64(v1))))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, v1=sum(v1)))[["elapsed"]]
 write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 print(head(ans, 3))
 print(tail(ans, 3))
 rm(ans)
 
 question = "sum v1 mean v3 by id3" # q3
-t = system.time(print(dim(ans<-x %>% group_by(id3) %>% summarise(v1=sum(v1, na.rm=TRUE), v3=mean(v3, na.rm=TRUE)))))[["elapsed"]]
+t = system.time(print(dim(ans<-x %>% summarise(.by = id3, v1=sum(v1), v3=mean(v3)))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ungroup(ans), v1=sum(bit64::as.integer64(v1)), v3=sum(v3)))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, v1=sum(v1), v3=sum(v3)))[["elapsed"]]
 write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 rm(ans)
-t = system.time(print(dim(ans<-x %>% group_by(id3) %>% summarise(v1=sum(v1, na.rm=TRUE), v3=mean(v3, na.rm=TRUE)))))[["elapsed"]]
+t = system.time(print(dim(ans<-x %>% summarise(.by = id3, v1=sum(v1), v3=mean(v3)))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ungroup(ans), v1=sum(bit64::as.integer64(v1)), v3=sum(v3)))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, v1=sum(v1), v3=sum(v3)))[["elapsed"]]
 write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 print(head(ans, 3))
 print(tail(ans, 3))
 rm(ans)
 
 question = "mean v1:v3 by id4" # q4
-t = system.time(print(dim(ans<-x %>% group_by(id4) %>% summarise_at(.funs="mean", .vars=c("v1","v2","v3"), na.rm=TRUE))))[["elapsed"]]
+t = system.time(print(dim(ans<-x %>% summarise(.by = id4, v1 = mean(v1), v2 = mean(v2), v3 = mean(v3)))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ungroup(ans), v1=sum(v1), v2=sum(v2), v3=sum(v3)))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, v1=sum(v1), v2=sum(v2), v3=sum(v3)))[["elapsed"]]
 write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 rm(ans)
-t = system.time(print(dim(ans<-x %>% group_by(id4) %>% summarise_at(.funs="mean", .vars=c("v1","v2","v3"), na.rm=TRUE))))[["elapsed"]]
+t = system.time(print(dim(ans<-x %>% summarise(.by = id4, v1 = mean(v1), v2 = mean(v2), v3 = mean(v3)))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ungroup(ans), v1=sum(v1), v2=sum(v2), v3=sum(v3)))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, v1=sum(v1), v2=sum(v2), v3=sum(v3)))[["elapsed"]]
 write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 print(head(ans, 3))
 print(tail(ans, 3))
 rm(ans)
 
 question = "sum v1:v3 by id6" # q5
-t = system.time(print(dim(ans<-x %>% group_by(id6) %>% summarise_at(.funs="sum", .vars=c("v1","v2","v3"), na.rm=TRUE))))[["elapsed"]]
+t = system.time(print(dim(ans<-x %>% summarise(.by = id6, v1 = sum(v1), v2 = sum(v2), v3 = sum(v3)))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ungroup(ans), v1=sum(bit64::as.integer64(v1)), v2=sum(bit64::as.integer64(v2)), v3=sum(v3)))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans), v1=sum(v1), v2=sum(v2), v3=sum(v3)))[["elapsed"]]
 write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 rm(ans)
-t = system.time(print(dim(ans<-x %>% group_by(id6) %>% summarise_at(.funs="sum", .vars=c("v1","v2","v3"), na.rm=TRUE))))[["elapsed"]]
+t = system.time(print(dim(ans<-x %>% summarise(.by = id6, v1 = sum(v1), v2 = sum(v2), v3 = sum(v3)))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ungroup(ans), v1=sum(bit64::as.integer64(v1)), v2=sum(bit64::as.integer64(v2)), v3=sum(v3)))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans), v1=sum(v1), v2=sum(v2), v3=sum(v3)))[["elapsed"]]
 write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 print(head(ans, 3))
 print(tail(ans, 3))
 rm(ans)
 
 question = "median v3 sd v3 by id4 id5" # q6
-t = system.time(print(dim(ans<-x %>% group_by(id4, id5) %>% summarise(median_v3=median(v3, na.rm=TRUE), sd_v3=sd(v3, na.rm=TRUE)))))[["elapsed"]]
+t = system.time(print(dim(ans<-x %>% summarise(.by = c(id4, id5), median_v3=median(v3), sd_v3=sd(v3)))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ungroup(ans), median_v3=sum(median_v3), sd_v3=sum(sd_v3)))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, median_v3=sum(median_v3), sd_v3=sum(sd_v3)))[["elapsed"]]
 write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 rm(ans)
-t = system.time(print(dim(ans<-x %>% group_by(id4, id5) %>% summarise(median_v3=median(v3, na.rm=TRUE), sd_v3=sd(v3, na.rm=TRUE)))))[["elapsed"]]
+t = system.time(print(dim(ans<-x %>% summarise(.by = c(id4, id5), median_v3=median(v3), sd_v3=sd(v3)))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ungroup(ans), median_v3=sum(median_v3), sd_v3=sum(sd_v3)))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, median_v3=sum(median_v3), sd_v3=sum(sd_v3)))[["elapsed"]]
 write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 print(head(ans, 3))
 print(tail(ans, 3))
 rm(ans)
 
 question = "max v1 - min v2 by id3" # q7
-t = system.time(print(dim(ans<-x %>% group_by(id3) %>% summarise(range_v1_v2=max(v1, na.rm=TRUE)-min(v2, na.rm=TRUE)))))[["elapsed"]]
+t = system.time(print(dim(ans<-x %>% summarise(.by = id3, range_v1_v2=max(v1)-min(v2)))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ungroup(ans), range_v1_v2=sum(bit64::as.integer64(range_v1_v2))))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, range_v1_v2=sum(range_v1_v2)))[["elapsed"]]
 write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 rm(ans)
-t = system.time(print(dim(ans<-x %>% group_by(id3) %>% summarise(range_v1_v2=max(v1, na.rm=TRUE)-min(v2, na.rm=TRUE)))))[["elapsed"]]
+t = system.time(print(dim(ans<-x %>% summarise(.by = id3, range_v1_v2=max(v1)-min(v2)))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ungroup(ans), range_v1_v2=sum(bit64::as.integer64(range_v1_v2))))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, range_v1_v2=sum(range_v1_v2)))[["elapsed"]]
 write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 print(head(ans, 3))
 print(tail(ans, 3))
 rm(ans)
 
 question = "largest two v3 by id6" # q8
-t = system.time(print(dim(ans<-x %>% select(id6, largest2_v3=v3) %>% filter(!is.na(largest2_v3)) %>% arrange(desc(largest2_v3)) %>% group_by(id6) %>% filter(row_number() <= 2L))))[["elapsed"]]
+t = system.time(print(dim(ans<-x %>% select(id6, largest2_v3=v3) %>% filter(!is.na(largest2_v3)) %>% arrange(desc(largest2_v3)) %>% filter(.by = id6, row_number() <= 2L))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ungroup(ans), largest2_v3=sum(largest2_v3)))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, largest2_v3=sum(largest2_v3)))[["elapsed"]]
 write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 rm(ans)
-t = system.time(print(dim(ans<-x %>% select(id6, largest2_v3=v3) %>% filter(!is.na(largest2_v3)) %>% arrange(desc(largest2_v3)) %>% group_by(id6) %>% filter(row_number() <= 2L))))[["elapsed"]]
+t = system.time(print(dim(ans<-x %>% select(id6, largest2_v3=v3) %>% filter(!is.na(largest2_v3)) %>% arrange(desc(largest2_v3)) %>% filter(.by = id6, row_number() <= 2L))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ungroup(ans), largest2_v3=sum(largest2_v3)))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, largest2_v3=sum(largest2_v3)))[["elapsed"]]
 write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 print(head(ans, 3))
 print(tail(ans, 3))
 rm(ans)
 
 question = "regression v1 v2 by id2 id4" # q9
-t = system.time(print(dim(ans<-x %>% group_by(id2, id4) %>% summarise(r2=cor(v1, v2, use="na.or.complete")^2))))[["elapsed"]]
+t = system.time(print(dim(ans<-x %>% summarise(.by = c(id2, id4), r2=cor(v1, v2, use="na.or.complete")^2))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ungroup(ans), r2=sum(r2)))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, r2=sum(r2)))[["elapsed"]]
 write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 rm(ans)
-t = system.time(print(dim(ans<-x %>% group_by(id2, id4) %>% summarise(r2=cor(v1, v2, use="na.or.complete")^2))))[["elapsed"]]
+t = system.time(print(dim(ans<-x %>% summarise(.by = c(id2, id4), r2=cor(v1, v2, use="na.or.complete")^2))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ungroup(ans), r2=sum(r2)))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, r2=sum(r2)))[["elapsed"]]
 write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 print(head(ans, 3))
 print(tail(ans, 3))
 rm(ans)
 
 question = "sum v3 count by id1:id6" # q10
-t = system.time(print(dim(ans<-x %>% group_by(id1, id2, id3, id4, id5, id6) %>% summarise(v3=sum(v3, na.rm=TRUE), count=n()))))[["elapsed"]]
+t = system.time(print(dim(ans<-x %>% summarise(.by = c(id1, id2, id3, id4, id5, id6), v3=sum(v3), count=n()))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ungroup(ans), v3=sum(v3), count=sum(bit64::as.integer64(count))))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, v3=sum(v3), count=sum(count)))[["elapsed"]]
 write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 rm(ans)
-t = system.time(print(dim(ans<-x %>% group_by(id1, id2, id3, id4, id5, id6) %>% summarise(v3=sum(v3, na.rm=TRUE), count=n()))))[["elapsed"]]
+t = system.time(print(dim(ans<-x %>% summarise(.by = c(id1, id2, id3, id4, id5, id6), v3=sum(v3), count=n()))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ungroup(ans), v3=sum(v3), count=sum(bit64::as.integer64(count))))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, v3=sum(v3), count=sum(count)))[["elapsed"]]
 write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 print(head(ans, 3))
 print(tail(ans, 3))
diff --git a/duckplyr/join-duckplyr.R b/duckplyr/join-duckplyr.R
index fdcd6aab..2a254b3f 100755
--- a/duckplyr/join-duckplyr.R
+++ b/duckplyr/join-duckplyr.R
@@ -36,12 +36,12 @@ question = "small inner on int" # q1
 fun = "inner_join"
 t = system.time(print(dim(ans<-inner_join(x, small, by="id1"))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ans, sum(v1, na.rm=TRUE), sum(v2, na.rm=TRUE)))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, sum(v1), sum(v2)))[["elapsed"]]
 write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 rm(ans)
 t = system.time(print(dim(ans<-inner_join(x, small, by="id1"))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ans, sum(v1, na.rm=TRUE), sum(v2, na.rm=TRUE)))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, sum(v1), sum(v2)))[["elapsed"]]
 write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 print(head(ans, 3))
 print(tail(ans, 3))
@@ -51,12 +51,12 @@ question = "medium inner on int" # q2
 fun = "inner_join"
 t = system.time(print(dim(ans<-inner_join(x, medium, by="id2"))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ans, sum(v1, na.rm=TRUE), sum(v2, na.rm=TRUE)))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, sum(v1), sum(v2)))[["elapsed"]]
 write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 rm(ans)
 t = system.time(print(dim(ans<-inner_join(x, medium, by="id2"))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ans, sum(v1, na.rm=TRUE), sum(v2, na.rm=TRUE)))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, sum(v1), sum(v2)))[["elapsed"]]
 write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 print(head(ans, 3))
 print(tail(ans, 3))
@@ -66,12 +66,12 @@ question = "medium outer on int" # q3
 fun = "left_join"
 t = system.time(print(dim(ans<-left_join(x, medium, by="id2"))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ans, sum(v1, na.rm=TRUE), sum(v2, na.rm=TRUE)))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, sum(v1), sum(v2)))[["elapsed"]]
 write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 rm(ans)
 t = system.time(print(dim(ans<-left_join(x, medium, by="id2"))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ans, sum(v1, na.rm=TRUE), sum(v2, na.rm=TRUE)))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, sum(v1), sum(v2)))[["elapsed"]]
 write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 print(head(ans, 3))
 print(tail(ans, 3))
@@ -81,12 +81,12 @@ question = "medium inner on factor" # q4
 fun = "inner_join"
 t = system.time(print(dim(ans<-inner_join(x, medium, by="id5"))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ans, sum(v1, na.rm=TRUE), sum(v2, na.rm=TRUE)))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, sum(v1), sum(v2)))[["elapsed"]]
 write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 rm(ans)
 t = system.time(print(dim(ans<-inner_join(x, medium, by="id5"))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ans, sum(v1, na.rm=TRUE), sum(v2, na.rm=TRUE)))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, sum(v1), sum(v2)))[["elapsed"]]
 write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 print(head(ans, 3))
 print(tail(ans, 3))
@@ -96,12 +96,12 @@ question = "big inner on int" # q5
 fun = "inner_join"
 t = system.time(print(dim(ans<-inner_join(x, big, by="id3"))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ans, sum(v1, na.rm=TRUE), sum(v2, na.rm=TRUE)))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, sum(v1), sum(v2)))[["elapsed"]]
 write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 rm(ans)
 t = system.time(print(dim(ans<-inner_join(x, big, by="id3"))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ans, sum(v1, na.rm=TRUE), sum(v2, na.rm=TRUE)))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, sum(v1), sum(v2)))[["elapsed"]]
 write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 print(head(ans, 3))
 print(tail(ans, 3))

From 2ea70b0ee6b8b6bea966a02894edfe0fbcba5bc1 Mon Sep 17 00:00:00 2001
From: Tom Ebergen <Tmonster@users.noreply.github.com>
Date: Wed, 25 Sep 2024 07:45:22 +0200
Subject: [PATCH 04/10] Update solutions.csv to include duckplyr

---
 _control/solutions.csv | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/_control/solutions.csv b/_control/solutions.csv
index 89009a06..ddeab48a 100644
--- a/_control/solutions.csv
+++ b/_control/solutions.csv
@@ -8,6 +8,8 @@ data.table,groupby2014
 dplyr,groupby
 dplyr,join
 dplyr,groupby2014
+duckplyr,groupby
+duckplyr,join
 pandas,groupby
 pandas,join
 pandas,groupby2014

From 115c6c08e6813601a3dea374b779d098868e2bb6 Mon Sep 17 00:00:00 2001
From: Tom Ebergen <Tmonster@users.noreply.github.com>
Date: Wed, 25 Sep 2024 07:47:22 +0200
Subject: [PATCH 05/10] add duckplyr to regression-test-runner matrix solutions

---
 .github/workflows/regression.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml
index 386a810b..1638b38e 100644
--- a/.github/workflows/regression.yml
+++ b/.github/workflows/regression.yml
@@ -17,7 +17,7 @@ jobs:
   strategy:
     fail-fast: false
     matrix:
-      solution: [data.table, collapse, dplyr, pandas, pydatatable, spark, juliadf, juliads, polars, R-arrow, duckdb, datafusion, dask, clickhouse]
+      solution: [data.table, collapse, dplyr, duckplyr, pandas, pydatatable, spark, juliadf, juliads, polars, R-arrow, duckdb, datafusion, dask, clickhouse]
   name: Regression Tests solo solutions
   runs-on: ubuntu-20.04
   env:

From 56357c5f0b89f9db2f3c3587f104d91af24ecff5 Mon Sep 17 00:00:00 2001
From: Tmonster <tom@ebergen.com>
Date: Wed, 25 Sep 2024 12:45:43 +0200
Subject: [PATCH 06/10] more duckplyr changes to run benchmark

---
 _benchplot/benchplot-dict.R | 22 ++++++++++++++++++++++
 _launcher/launcher.R        |  2 +-
 _launcher/solution.R        |  2 +-
 run.conf                    |  2 +-
 4 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/_benchplot/benchplot-dict.R b/_benchplot/benchplot-dict.R
index 9d650875..7419f380 100644
--- a/_benchplot/benchplot-dict.R
+++ b/_benchplot/benchplot-dict.R
@@ -34,6 +34,7 @@ solution.dict = {list(
   "collapse" = list(name=c(short="collapse", long="collapse"), color=c(strong="darkturquoise", light="turquoise")),
   "data.table" = list(name=c(short="data.table", long="data.table"), color=c(strong="blue", light="#7777FF")),
   "dplyr" = list(name=c(short="dplyr", long="dplyr"), color=c(strong="red", light="#FF7777")),
+  "duckplyr" = list(name=c(short="duckplyr", long="duckplyr"), color=c(strong="#ddcd07", light="#fff100")),
   "pandas" = list(name=c(short="pandas", long="pandas"), color=c(strong="green4", light="#77FF77")),
   "pydatatable" = list(name=c(short="pydatatable", long="(py)datatable"), color=c(strong="darkorange", light="orange")),
   "spark" = list(name=c(short="spark", long="spark"), color=c(strong="#8000FFFF", light="#CC66FF")),
@@ -103,6 +104,18 @@ groupby.syntax.dict = {list(
     "regression v1 v2 by id2 id4" = "DF %>% group_by(id2, id4) %>% summarise(r2=cor(v1, v2, use=\"na.or.complete\")^2)",
     "sum v3 count by id1:id6" = "DF %>% group_by(id1, id2, id3, id4, id5, id6) %>% summarise(v3=sum(v3, na.rm=TRUE), count=n())"
   )},
+  "duckplyr" = {c(
+    "sum v1 by id1" = "DF %>% group_by(id1) %>% summarise(v1=sum(v1, na.rm=TRUE))",
+    "sum v1 by id1:id2" = "DF %>% group_by(id1, id2) %>% summarise(v1=sum(v1, na.rm=TRUE))",
+    "sum v1 mean v3 by id3" = "DF %>% group_by(id3) %>% summarise(v1=sum(v1, na.rm=TRUE), v3=mean(v3, na.rm=TRUE))",
+    "mean v1:v3 by id4" = "DF %>% group_by(id4) %>% summarise_at(.funs=\"mean\", .vars=c(\"v1\",\"v2\",\"v3\"), na.rm=TRUE)",
+    "sum v1:v3 by id6" = "DF %>% group_by(id6) %>% summarise_at(.funs=\"sum\", .vars=c(\"v1\",\"v2\",\"v3\"), na.rm=TRUE)",
+    "median v3 sd v3 by id4 id5" = "DF %>% group_by(id4, id5) %>% summarise(median_v3=median(v3, na.rm=TRUE), sd_v3=sd(v3, na.rm=TRUE))",
+    "max v1 - min v2 by id3" = "DF %>% group_by(id3) %>% summarise(range_v1_v2=max(v1, na.rm=TRUE)-min(v2, na.rm=TRUE))",
+    "largest two v3 by id6" = "DF %>% select(id6, largest2_v3=v3) %>% filter(!is.na(largest2_v3)) %>% arrange(desc(largest2_v3)) %>% group_by(id6) %>% filter(row_number() <= 2L)",
+    "regression v1 v2 by id2 id4" = "DF %>% group_by(id2, id4) %>% summarise(r2=cor(v1, v2, use=\"na.or.complete\")^2)",
+    "sum v3 count by id1:id6" = "DF %>% group_by(id1, id2, id3, id4, id5, id6) %>% summarise(v3=sum(v3, na.rm=TRUE), count=n())"
+  )},
   "pandas" = {c(
     "sum v1 by id1" = "DF.groupby('id1', as_index=False, sort=False, observed=True, dropna=False).agg({'v1':'sum'})",
     "sum v1 by id1:id2" = "DF.groupby(['id1','id2'], as_index=False, sort=False, observed=True, dropna=False).agg({'v1':'sum'})",
@@ -252,6 +265,7 @@ groupby.syntax.dict = {list(
   "collapse" =    list(),
   "data.table" =  list(),
   "dplyr" =       list(),
+  "duckplyr" =    list(),
   "pandas" =      list(),
   "pydatatable" = list(),
   "spark" =       list("not yet implemented: SPARK-26589" = "median v3 sd v3 by id4 id5"),
@@ -364,6 +378,13 @@ join.syntax.dict = {list(
     "medium inner on factor" = "inner_join(DF, medium, by='id5')",
     "big inner on int" = "inner_join(DF, big, by='id3')"
   )},
+  "duckplyr" = {c(
+    "small inner on int" = "inner_join(DF, small, by='id1')",
+    "medium inner on int" = "inner_join(DF, medium, by='id2')",
+    "medium outer on int" = "left_join(DF, medium, by='id2')",
+    "medium inner on factor" = "inner_join(DF, medium, by='id5')",
+    "big inner on int" = "inner_join(DF, big, by='id3')"
+  )},
   "juliadf" = {c(
     "small inner on int" = "innerjoin(DF, small, on = :id1, makeunique=true, matchmissing=:equal)",
     "medium inner on int" = "innerjoin(DF, medium, on = :id2, makeunique=true, matchmissing=:equal)",
@@ -446,6 +467,7 @@ join.query.exceptions = {list(
   "collapse" =    list(),
   "data.table" =  list(),
   "dplyr" =       list(),
+  "duckplyr" =    list(),
   "pandas" =      list(),
   "pydatatable" = list(),
   "spark" =       list(),
diff --git a/_launcher/launcher.R b/_launcher/launcher.R
index d0079ba4..dc49c127 100644
--- a/_launcher/launcher.R
+++ b/_launcher/launcher.R
@@ -14,7 +14,7 @@ readret = function(x) {
 file.ext = function(x) {
   ans = switch(
     x,
-    "collapse"=, "data.table"=, "dplyr"=, "h2o"=, "R-arrow"=, "duckdb"="R", "duckdb-latest"="R",
+    "collapse"=, "data.table"=, "dplyr"=, "duckplyr"=, "h2o"=, "R-arrow"=, "duckdb"="R", "duckdb-latest"="R",
     "pandas"=, "spark"=, "pydatatable"=, "modin"=, "dask"=, "datafusion"=, "polars"="py",
     "clickhouse"="sh", "juliadf"="jl", "juliads"="jl",
   )
diff --git a/_launcher/solution.R b/_launcher/solution.R
index 0d498c2d..7caad174 100755
--- a/_launcher/solution.R
+++ b/_launcher/solution.R
@@ -110,7 +110,7 @@ if ("quiet" %in% names(args)) {
 file.ext = function(x) {
   ans = switch(
     x,
-    "collapse"=, "data.table"=, "dplyr"=, "h2o"=, "R-arrow"=, "duckdb"="R", "duckdb-latest"="R",
+    "collapse"=, "data.table"=, "dplyr"=, "duckplyr", "h2o"=, "R-arrow"=, "duckdb"="R", "duckdb-latest"="R",
     "pandas"="py", "spark"=, "pydatatable"=, "modin"=, "dask"=, "datafusion"=, "polars"="py",
     "clickhouse"="sh", "juliadf"="jl", "juliads"="jl"
   )
diff --git a/run.conf b/run.conf
index c019b15f..4bd5f229 100644
--- a/run.conf
+++ b/run.conf
@@ -1,7 +1,7 @@
 # task, used in init-setup-iteration.R
 export RUN_TASKS="groupby join"
 # solution, used in init-setup-iteration.R
-export RUN_SOLUTIONS="collapse data.table juliads juliadf dplyr pandas pydatatable spark dask clickhouse polars R-arrow duckdb duckdb-latest datafusion"
+export RUN_SOLUTIONS="collapse data.table juliads juliadf dplyr duckplyr pandas pydatatable spark dask clickhouse polars R-arrow duckdb duckdb-latest datafusion"
 
 # flag to upgrade tools, used in run.sh on init
 export DO_UPGRADE=false

From b26b8237cdb51fbe01b3bcb48937911bd82b6b94 Mon Sep 17 00:00:00 2001
From: Tmonster <tom@ebergen.com>
Date: Wed, 25 Sep 2024 13:18:41 +0200
Subject: [PATCH 07/10] fix group by q5

---
 duckplyr/groupby-duckplyr.R | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/duckplyr/groupby-duckplyr.R b/duckplyr/groupby-duckplyr.R
index a2221a53..71b3d531 100755
--- a/duckplyr/groupby-duckplyr.R
+++ b/duckplyr/groupby-duckplyr.R
@@ -84,12 +84,12 @@ rm(ans)
 question = "sum v1:v3 by id6" # q5
 t = system.time(print(dim(ans<-x %>% summarise(.by = id6, v1 = sum(v1), v2 = sum(v2), v3 = sum(v3)))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ans), v1=sum(v1), v2=sum(v2), v3=sum(v3)))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, v1=sum(v1), v2=sum(v2), v3=sum(v3)))[["elapsed"]]
 write.log(run=1L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 rm(ans)
 t = system.time(print(dim(ans<-x %>% summarise(.by = id6, v1 = sum(v1), v2 = sum(v2), v3 = sum(v3)))))[["elapsed"]]
 m = memory_usage()
-chkt = system.time(chk<-summarise(ans), v1=sum(v1), v2=sum(v2), v3=sum(v3)))[["elapsed"]]
+chkt = system.time(chk<-summarise(ans, v1=sum(v1), v2=sum(v2), v3=sum(v3)))[["elapsed"]]
 write.log(run=2L, task=task, data=data_name, in_rows=nrow(x), question=question, out_rows=nrow(ans), out_cols=ncol(ans), solution=solution, version=ver, git=git, fun=fun, time_sec=t, mem_gb=m, cache=cache, chk=make_chk(chk), chk_time_sec=chkt, on_disk=on_disk)
 print(head(ans, 3))
 print(tail(ans, 3))

From 3a3cb84a10b6acc6cd99dfc90ca5d1ac758e8dda Mon Sep 17 00:00:00 2001
From: Tmonster <tom@ebergen.com>
Date: Wed, 25 Sep 2024 15:10:43 +0200
Subject: [PATCH 08/10] fix soltion.R

---
 _launcher/solution.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_launcher/solution.R b/_launcher/solution.R
index 7caad174..aa672be2 100755
--- a/_launcher/solution.R
+++ b/_launcher/solution.R
@@ -110,7 +110,7 @@ if ("quiet" %in% names(args)) {
 file.ext = function(x) {
   ans = switch(
     x,
-    "collapse"=, "data.table"=, "dplyr"=, "duckplyr", "h2o"=, "R-arrow"=, "duckdb"="R", "duckdb-latest"="R",
+    "collapse"=, "data.table"=, "dplyr"=, "duckplyr"=, "h2o"=, "R-arrow"=, "duckdb"="R", "duckdb-latest"="R",
     "pandas"="py", "spark"=, "pydatatable"=, "modin"=, "dask"=, "datafusion"=, "polars"="py",
     "clickhouse"="sh", "juliadf"="jl", "juliads"="jl"
   )

From 5d09d6dd9c566214b294e7095af05e20d8b59b99 Mon Sep 17 00:00:00 2001
From: Tmonster <tom@ebergen.com>
Date: Wed, 25 Sep 2024 15:52:35 +0200
Subject: [PATCH 09/10] cannot treat string as factors, otherwise there is a
 lot of dplyr fallback

---
 duckplyr/groupby-duckplyr.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/duckplyr/groupby-duckplyr.R b/duckplyr/groupby-duckplyr.R
index 71b3d531..9c4af517 100755
--- a/duckplyr/groupby-duckplyr.R
+++ b/duckplyr/groupby-duckplyr.R
@@ -19,7 +19,7 @@ data_name = Sys.getenv("SRC_DATANAME")
 src_grp = file.path("data", paste(data_name, "csv", sep="."))
 cat(sprintf("loading dataset %s\n", data_name))
 
-x = as_duckplyr_tibble(data.table::fread(src_grp, showProgress=FALSE, stringsAsFactors=TRUE, na.strings="", data.table=FALSE))
+x = as_duckplyr_tibble(data.table::fread(src_grp, showProgress=FALSE, na.strings="", data.table=FALSE))
 print(nrow(x))
 
 task_init = proc.time()[["elapsed"]]

From 2e369659c13da72e9364cbd4eaac6686013fa7dc Mon Sep 17 00:00:00 2001
From: Tmonster <tom@ebergen.com>
Date: Fri, 4 Oct 2024 11:27:12 +0200
Subject: [PATCH 10/10] add duckplyr to report.R solutions

---
 _report/report.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_report/report.R b/_report/report.R
index 01b27dd0..65749ec9 100644
--- a/_report/report.R
+++ b/_report/report.R
@@ -6,7 +6,7 @@ get_report_status_file = function(path=getwd()) {
   file.path(path, "report-done")
 }
 get_report_solutions = function() {
-  c("collapse", "data.table", "dplyr", "pandas", "pydatatable", "spark", "dask", "juliadf", "juliads", "clickhouse", "cudf", "polars", "duckdb", "datafusion", "arrow", "R-arrow")
+  c("collapse", "data.table", "dplyr", "duckplyr", "pandas", "pydatatable", "spark", "dask", "juliadf", "juliads", "clickhouse", "cudf", "polars", "duckdb", "datafusion", "arrow", "R-arrow")
 }
 get_data_levels = function() {
   ## groupby