Skip to content

Commit ee2f490

Browse files
ganehagganehag
committed
feat: implement std.regexp with NFA backtracking engine
1 parent fde2653 commit ee2f490

8 files changed

Lines changed: 787 additions & 0 deletions

File tree

‎examples/spec/140_regexp.gengo‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
std := import("std")
2+
3+
// regexp.match
4+
assert(std.regexp.match("hello", "hello world"))
5+
assert(!std.regexp.match("hello", "world"))
6+
assert(std.regexp.match("^hello", "hello world"))
7+
assert(!std.regexp.match("^world", "hello world"))
8+
assert(std.regexp.match("world$", "hello world"))
9+
assert(!std.regexp.match("hello$", "hello world"))
10+
assert(std.regexp.match("h.llo", "hello"))
11+
assert(!std.regexp.match("h.llo", "hllo"))
12+
assert(std.regexp.match("he*llo", "hllo"))
13+
assert(std.regexp.match("he*llo", "hello"))
14+
assert(std.regexp.match("he*llo", "heeeello"))
15+
assert(std.regexp.match("he+llo", "hello"))
16+
assert(!std.regexp.match("he+llo", "hllo"))
17+
assert(std.regexp.match("he?llo", "hello"))
18+
assert(std.regexp.match("he?llo", "hllo"))
19+
assert(!std.regexp.match("he?llo", "heello"))
20+
21+
// character classes
22+
assert(std.regexp.match("[abc]", "a"))
23+
assert(!std.regexp.match("[abc]", "d"))
24+
assert(std.regexp.match("[a-z]", "m"))
25+
assert(!std.regexp.match("[a-z]", "3"))
26+
assert(std.regexp.match("[a-z0-9]", "5"))
27+
assert(!std.regexp.match("[^abc]", "a"))
28+
assert(std.regexp.match("[^abc]", "d"))
29+
assert(std.regexp.match("[a-z]+", "hello"))
30+
31+
// shorthand classes
32+
assert(std.regexp.match("\\d+", "123"))
33+
assert(!std.regexp.match("\\d+", "abc"))
34+
assert(std.regexp.match("\\w+", "hello_123"))
35+
assert(std.regexp.match("\\s", " "))
36+
assert(!std.regexp.match("\\D", "1"))
37+
assert(std.regexp.match("\\D", "a"))
38+
assert(std.regexp.match("\\W", "!"))
39+
40+
// alternation
41+
assert(std.regexp.match("a|b", "a"))
42+
assert(std.regexp.match("a|b", "b"))
43+
assert(!std.regexp.match("a|b", "c"))
44+
assert(std.regexp.match("cat|dog", "dog"))
45+
assert(!std.regexp.match("cat|dog", "bird"))
46+
assert(std.regexp.match("(a|b)c", "ac"))
47+
assert(std.regexp.match("(a|b)c", "bc"))
48+
assert(!std.regexp.match("(a|b)c", "cc"))
49+
50+
// groups
51+
assert(std.regexp.match("(ab)+", "abab"))
52+
assert(std.regexp.match("(ab)+", "aba"))
53+
54+
// find
55+
found := std.regexp.find("world", "hello world")
56+
assert(std.core.is_string(found))
57+
assert(found == "world")
58+
assert(std.regexp.find("xyz", "hello world") == null)
59+
60+
all := std.regexp.find_all("a", "banana")
61+
assert(std.core.len(all) == 3)
62+
assert(all[0] == "a")
63+
assert(all[1] == "a")
64+
assert(all[2] == "a")
65+
66+
// replace
67+
r := std.regexp.replace("world", "hello world", "there")
68+
assert(r == "hello there")
69+
r = std.regexp.replace("a", "banana", "o")
70+
assert(r == "bonono")
71+
r = std.regexp.replace("xyz", "hello", "!")
72+
assert(r == "hello")
73+
74+
// split
75+
parts := std.regexp.split(",", "a,b,c")
76+
assert(std.core.len(parts) == 3)
77+
78+
parts = std.regexp.split(" ", "hello world")
79+
assert(std.core.len(parts) == 2)
80+
assert(parts[0] == "hello")
81+
assert(parts[1] == "world")
82+
83+
// compile and use object
84+
re := std.regexp.compile("l+")
85+
assert(std.regexp.match(re, "hello"))
86+
m := std.regexp.find(re, "hello")
87+
assert(m == "ll")
88+
89+
std.io.println("ok")

‎examples/spec/140_regexp.out‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
ok

‎examples/spec/fail/121_regexp_invalid_pattern.err‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
UnterminatedClass

‎examples/spec/fail/121_regexp_invalid_pattern.gengo‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
std := import("std")
2+
std.regexp.match("[invalid", "test")

‎examples/spec/fail/122_regexp_unterm_group.err‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
UnterminatedGroup

‎examples/spec/fail/122_regexp_unterm_group.gengo‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
std := import("std")
2+
std.regexp.match("(abc", "test")

‎src/lang/native/main.zig‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,12 @@ const string_mod = @import("string.zig");
2929
const io_mod = @import("io.zig");
3030
const json_mod = @import("json.zig");
3131
const template_mod = @import("template.zig");
32+
const regexp_mod = @import("regexp.zig");
3233
const host_abi_mod = @import("host_abi.zig");
3334

3435
const TemplateTypeQualifiedName = "@std.template.obj";
3536
const TimeTypeQualifiedName = "@std.time.obj";
37+
const RegexpTypeQualifiedName = "@std.regexp.obj";
3638

3739
const NativeFnId = enum(u8) {
3840
io_println = 1,
@@ -160,6 +162,17 @@ const NativeFnId = enum(u8) {
160162
time_since = 123,
161163
time_until = 124,
162164
time_add_date = 125,
165+
re_match = 126,
166+
re_find = 127,
167+
re_find_all = 128,
168+
re_replace = 129,
169+
re_split = 130,
170+
re_compile = 131,
171+
re_obj_match = 132,
172+
re_obj_find = 133,
173+
re_obj_find_all = 134,
174+
re_obj_replace = 135,
175+
re_obj_split = 136,
163176
};
164177
const MaxNativeArgs = 255;
165178
const NamespaceEntry = struct {
@@ -403,6 +416,23 @@ pub fn buildStdModule() !*Object {
403416
try vms.pushTempRoot(.{ .object = base64_obj });
404417
defer vms.popTempRoot();
405418

419+
const regexp_type_obj = try regexp_mod.reGetType();
420+
try vms.pushTempRoot(.{ .object = regexp_type_obj });
421+
defer vms.popTempRoot();
422+
423+
const regexp_entries = [_]NamespaceEntry{
424+
.{ .name = "match", .value = try makeNative(.re_match, 2) },
425+
.{ .name = "find", .value = try makeNative(.re_find, 2) },
426+
.{ .name = "find_all", .value = try makeNative(.re_find_all, 2) },
427+
.{ .name = "replace", .value = try makeNative(.re_replace, 3) },
428+
.{ .name = "split", .value = try makeNative(.re_split, 2) },
429+
.{ .name = "compile", .value = try makeNative(.re_compile, 1) },
430+
.{ .name = "__type", .value = .{ .object = regexp_type_obj } },
431+
};
432+
const regexp_obj = try makeNamespace("regexp", "@module_type:std.regexp", &regexp_entries);
433+
try vms.pushTempRoot(.{ .object = regexp_obj });
434+
defer vms.popTempRoot();
435+
406436
const std_entries = [_]NamespaceEntry{
407437
.{ .name = "io", .value = .{ .object = io_obj } },
408438
.{ .name = "core", .value = .{ .object = core_obj } },
@@ -415,7 +445,9 @@ pub fn buildStdModule() !*Object {
415445
.{ .name = "time", .value = .{ .object = time_obj } },
416446
.{ .name = "hex", .value = .{ .object = hex_obj } },
417447
.{ .name = "base64", .value = .{ .object = base64_obj } },
448+
.{ .name = "regexp", .value = .{ .object = regexp_obj } },
418449
.{ .name = "Time", .value = .{ .object = time_type_obj } },
450+
.{ .name = "Regexp", .value = .{ .object = regexp_type_obj } },
419451
};
420452
const std_obj = try makeNamespace("std", "@module_type:std", &std_entries);
421453
vms.vmState().std_module = std_obj;
@@ -471,6 +503,26 @@ pub fn installStdGlobal() !void {
471503
}
472504
}
473505
}
506+
{
507+
const regexp_methods = [_]struct { name: []const u8, id: NativeFnId, arity: u8 }{
508+
.{ .name = "match", .id = .re_obj_match, .arity = 2 },
509+
.{ .name = "find", .id = .re_obj_find, .arity = 2 },
510+
.{ .name = "find_all", .id = .re_obj_find_all, .arity = 2 },
511+
.{ .name = "replace", .id = .re_obj_replace, .arity = 3 },
512+
.{ .name = "split", .id = .re_obj_split, .arity = 2 },
513+
};
514+
for (regexp_methods) |m| {
515+
const needed = RegexpTypeQualifiedName.len + 1 + m.name.len;
516+
const kbuf = (heap.bump(u8, needed) orelse return)[0..needed];
517+
@memcpy(kbuf[0..RegexpTypeQualifiedName.len], RegexpTypeQualifiedName);
518+
kbuf[RegexpTypeQualifiedName.len] = '.';
519+
@memcpy(kbuf[RegexpTypeQualifiedName.len + 1 .. needed], m.name);
520+
if (!globals.has(kbuf)) {
521+
const n = try makeNative(m.id, m.arity);
522+
try globals.def(kbuf, n);
523+
}
524+
}
525+
}
474526
}
475527

476528
pub fn callNative(nf: NativeFuncObj, argc: u8) !void {
@@ -1556,5 +1608,109 @@ pub fn callNative(nf: NativeFuncObj, argc: u8) !void {
15561608
_ = try vms.vmPop(); _ = try vms.vmPop(); _ = try vms.vmPop(); _ = try vms.vmPop(); _ = try vms.vmPop();
15571609
try vms.vmPush(out);
15581610
},
1611+
.re_match => {
1612+
if (argc != nf.arity) return error.ArityMismatch;
1613+
const top = vms.vmState().stack_top;
1614+
const pattern_val = vms.vmState().stack[top - 2];
1615+
const s_val = vms.vmState().stack[top - 1];
1616+
const result = try regexp_mod.nativeReMatch(pattern_val, s_val);
1617+
_ = try vms.vmPop(); _ = try vms.vmPop(); _ = try vms.vmPop();
1618+
try vms.vmPush(result);
1619+
},
1620+
.re_find => {
1621+
if (argc != nf.arity) return error.ArityMismatch;
1622+
const top = vms.vmState().stack_top;
1623+
const pattern_val = vms.vmState().stack[top - 2];
1624+
const s_val = vms.vmState().stack[top - 1];
1625+
const result = try regexp_mod.nativeReFind(pattern_val, s_val);
1626+
_ = try vms.vmPop(); _ = try vms.vmPop(); _ = try vms.vmPop();
1627+
try vms.vmPush(result);
1628+
},
1629+
.re_find_all => {
1630+
if (argc != nf.arity) return error.ArityMismatch;
1631+
const top = vms.vmState().stack_top;
1632+
const pattern_val = vms.vmState().stack[top - 2];
1633+
const s_val = vms.vmState().stack[top - 1];
1634+
const result = try regexp_mod.nativeReFindAll(pattern_val, s_val);
1635+
_ = try vms.vmPop(); _ = try vms.vmPop(); _ = try vms.vmPop();
1636+
try vms.vmPush(result);
1637+
},
1638+
.re_replace => {
1639+
if (argc != nf.arity) return error.ArityMismatch;
1640+
const top = vms.vmState().stack_top;
1641+
const pattern_val = vms.vmState().stack[top - 3];
1642+
const s_val = vms.vmState().stack[top - 2];
1643+
const repl_val = vms.vmState().stack[top - 1];
1644+
const result = try regexp_mod.nativeReReplace(pattern_val, s_val, repl_val);
1645+
_ = try vms.vmPop(); _ = try vms.vmPop(); _ = try vms.vmPop(); _ = try vms.vmPop();
1646+
try vms.vmPush(result);
1647+
},
1648+
.re_split => {
1649+
if (argc != nf.arity) return error.ArityMismatch;
1650+
const top = vms.vmState().stack_top;
1651+
const pattern_val = vms.vmState().stack[top - 2];
1652+
const s_val = vms.vmState().stack[top - 1];
1653+
const result = try regexp_mod.nativeReSplit(pattern_val, s_val);
1654+
_ = try vms.vmPop(); _ = try vms.vmPop(); _ = try vms.vmPop();
1655+
try vms.vmPush(result);
1656+
},
1657+
.re_compile => {
1658+
if (argc != nf.arity) return error.ArityMismatch;
1659+
const pattern_val = vms.vmState().stack[vms.vmState().stack_top - 1];
1660+
const result = try regexp_mod.nativeReCompile(pattern_val);
1661+
_ = try vms.vmPop(); _ = try vms.vmPop();
1662+
try vms.vmPush(result);
1663+
},
1664+
.re_obj_match => {
1665+
if (argc != nf.arity) return error.ArityMismatch;
1666+
const top = vms.vmState().stack_top;
1667+
const recv = vms.vmState().stack[top - 2];
1668+
const s_val = vms.vmState().stack[top - 1];
1669+
const pattern = try regexp_mod.reGetPattern(recv);
1670+
const result = try regexp_mod.nativeReMatch(.{ .string = pattern }, s_val);
1671+
_ = try vms.vmPop(); _ = try vms.vmPop(); _ = try vms.vmPop();
1672+
try vms.vmPush(result);
1673+
},
1674+
.re_obj_find => {
1675+
if (argc != nf.arity) return error.ArityMismatch;
1676+
const top = vms.vmState().stack_top;
1677+
const recv = vms.vmState().stack[top - 2];
1678+
const s_val = vms.vmState().stack[top - 1];
1679+
const pattern = try regexp_mod.reGetPattern(recv);
1680+
const result = try regexp_mod.nativeReFind(.{ .string = pattern }, s_val);
1681+
_ = try vms.vmPop(); _ = try vms.vmPop(); _ = try vms.vmPop();
1682+
try vms.vmPush(result);
1683+
},
1684+
.re_obj_find_all => {
1685+
if (argc != nf.arity) return error.ArityMismatch;
1686+
const top = vms.vmState().stack_top;
1687+
const recv = vms.vmState().stack[top - 2];
1688+
const s_val = vms.vmState().stack[top - 1];
1689+
const pattern = try regexp_mod.reGetPattern(recv);
1690+
const result = try regexp_mod.nativeReFindAll(.{ .string = pattern }, s_val);
1691+
_ = try vms.vmPop(); _ = try vms.vmPop(); _ = try vms.vmPop();
1692+
try vms.vmPush(result);
1693+
},
1694+
.re_obj_replace => {
1695+
if (argc != nf.arity) return error.ArityMismatch;
1696+
const top = vms.vmState().stack_top;
1697+
const recv = vms.vmState().stack[top - 3];
1698+
const s_val = vms.vmState().stack[top - 2];
1699+
const repl_val = vms.vmState().stack[top - 1];
1700+
const pattern = try regexp_mod.reGetPattern(recv);
1701+
const result = try regexp_mod.nativeReReplace(.{ .string = pattern }, s_val, repl_val);
1702+
_ = try vms.vmPop(); _ = try vms.vmPop(); _ = try vms.vmPop(); _ = try vms.vmPop();
1703+
try vms.vmPush(result);
1704+
},
1705+
.re_obj_split => {
1706+
if (argc != nf.arity) return error.ArityMismatch;
1707+
const top = vms.vmState().stack_top;
1708+
const recv = vms.vmState().stack[top - 2];
1709+
const s_val = vms.vmState().stack[top - 1];
1710+
const pattern = try regexp_mod.reGetPattern(recv);
1711+
const result = try regexp_mod.nativeReSplit(.{ .string = pattern }, s_val);
1712+
_ = try vms.vmPop(); _ = try vms.vmPop(); _ = try vms.vmPop();
1713+
try vms.vmPush(result);
1714+
},
15591715
}
15601716
}

0 commit comments

Comments
 (0)