Skip to content

Commit f286520

Browse files
committed
README: add benchmark
1 parent 1b328af commit f286520

2 files changed

Lines changed: 65 additions & 1 deletion

File tree

README.md

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -544,7 +544,7 @@ unique_ints: stream[int] = (
544544
assert list(unique_ints) == [0, 1]
545545
```
546546

547-
## vs `builtins.map/filter`
547+
## performances
548548

549549
There is zero overhead during iteration compared to `builtins.map` and `builtins.filter`:
550550

@@ -558,6 +558,31 @@ odd_int_chars = stream(range(N)).filter(lambda n: n % 2).map(str)
558558
map(str, filter(lambda n: n % 2, range(N)))
559559
```
560560

561+
Let's get an idea of the throughput of operations compared to `builtins.map`:
562+
563+
|operation|times slower than `builtins.map(lambda _: _, range(N))`|
564+
|--|--|
565+
|`stream(range(N)).map(lambda _: _)`|1.00x|
566+
|`stream(range(N)).filter(lambda _: _)`|1.05x|
567+
|`stream(range(N)).do(lambda _: _)`|1.9x|
568+
|`stream(range(N)).skip(N)`|1.9x|
569+
|`stream(range(N)).catch(ValueError)`|2.0x|
570+
|`stream(range(N)).group(5)`|2.4x|
571+
|`stream(range(N)).take(N)`|3.0x|
572+
|`stream(range(N)).observe('ints', do=bool)`|4.2x|
573+
|`stream(range(N)).observe('ints', do=bool, every=N)`|4.4x|
574+
|`stream(range(N)).observe('ints', do=bool, every=timedelta(...))`|5.5x|
575+
|`stream(range(N)).throttle(N, per=timedelta(...))`|5.8x|
576+
|`stream(range(N)).group(by=bool, up_to=5)`|7.3x|
577+
|`stream((i,) for i in range(N)).flatten()`|17x|
578+
|`stream(range(N)).buffer(N)`|50x|
579+
|`stream(range(N)).map(lambda _: _, concurrency=2)`|330x|
580+
|`stream(range(N)).map(lambda _: _, concurrency=2, as_completed=True)`|340x|
581+
|`stream(range(N)).group(within=timedelta(...))`|350x|
582+
|`stream((i,) for i in range(N)).flatten(concurrency=2)`|850x|
583+
584+
(source: `pytest -s tests/benchmark.py`)
585+
561586
## e.g. ETL via [`dlt`](https://github.com/dlt-hub/dlt)
562587

563588
A `stream` is an expressive way to declare a `dlt.resource`:

tests/benchmark.py

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
from collections.abc import Iterable
2+
from datetime import timedelta
3+
import timeit
4+
from streamable import stream
5+
6+
N = 1_000_000
7+
ints = stream(range(N))
8+
9+
10+
def consume(s: Iterable):
11+
for _ in s:
12+
pass
13+
14+
15+
baseline = timeit.timeit(lambda: consume(map(lambda _: _, ints)), number=10) / 10
16+
17+
for times, s in (
18+
(50, ints.map(lambda _: _)),
19+
(50, ints.filter(lambda _: _)),
20+
(50, ints.do(lambda _: _)),
21+
(50, ints.skip(N)),
22+
(50, ints.catch(ValueError)),
23+
(50, ints.group(5)),
24+
(50, ints.take(N)),
25+
(10, ints.observe("ints", do=bool)),
26+
(10, ints.observe("ints", do=bool, every=N)),
27+
(10, ints.observe("ints", do=bool, every=timedelta(seconds=1))),
28+
(10, ints.throttle(N, per=timedelta(seconds=1))),
29+
(10, ints.group(5, by=bool)),
30+
(1, stream((i,) for i in range(N)).flatten()),
31+
(1, ints.buffer(N)),
32+
(1, ints.map(lambda _: _, concurrency=2)),
33+
(1, ints.map(lambda _: _, concurrency=2, as_completed=True)),
34+
(1, ints.group(within=timedelta(seconds=1))),
35+
(1, stream((i,) for i in range(N)).flatten(concurrency=2)),
36+
):
37+
duration = timeit.timeit(lambda: consume(s), number=times) / times
38+
print(s)
39+
print(f"is {duration / baseline:.2f}x slower than builtins.map")

0 commit comments

Comments
 (0)