From f69c1cb5007295940b9743e780b7227324cfd995 Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Mon, 27 Oct 2025 15:19:00 +0100 Subject: [PATCH 01/13] copy wrap --- inc/ti/fn/fncopy.h | 4 +- inc/ti/version.h | 4 +- inc/ti/wrap.h | 3 ++ itest/test_wrap.py | 28 +++++++++++ src/ti/wrap.c | 119 +++++++++++++++------------------------------ 5 files changed, 76 insertions(+), 82 deletions(-) diff --git a/inc/ti/fn/fncopy.h b/inc/ti/fn/fncopy.h index 0f02634d..97e9c5b9 100644 --- a/inc/ti/fn/fncopy.h +++ b/inc/ti/fn/fncopy.h @@ -27,7 +27,9 @@ static int do__f_copy(ti_query_t * query, cleri_node_t * nd, ex_t * e) query->rval = val; } - if (ti_val_copy(&query->rval, NULL, NULL, deep)) + if (ti_val_is_wrap(query->rval) && ti_wrap_cp(query, deep, e)) + return e->nr; /* quit without fail0, e is set*/ + else if (ti_val_copy(&query->rval, NULL, NULL, deep)) ex_set_mem(e); return e->nr; diff --git a/inc/ti/version.h b/inc/ti/version.h index 055af1e7..39587f4e 100644 --- a/inc/ti/version.h +++ b/inc/ti/version.h @@ -5,8 +5,8 @@ #define TI_VERSION_H_ #define TI_VERSION_MAJOR 1 -#define TI_VERSION_MINOR 7 -#define TI_VERSION_PATCH 7 +#define TI_VERSION_MINOR 8 +#define TI_VERSION_PATCH 0 /* The syntax version is used to test compatibility with functions * using the `ti_nodes_check_syntax()` function */ diff --git a/inc/ti/wrap.h b/inc/ti/wrap.h index 31630b21..b1a17b73 100644 --- a/inc/ti/wrap.h +++ b/inc/ti/wrap.h @@ -8,8 +8,10 @@ typedef struct ti_wrap_s ti_wrap_t; #include #include +#include #include #include +#include ti_wrap_t * ti_wrap_create(ti_thing_t * thing, uint16_t type_id); void ti_wrap_destroy(ti_wrap_t * wrap); @@ -19,6 +21,7 @@ int ti__wrap_field_thing( uint16_t spec, int deep, int flags); +int ti_wrap_cp(ti_query_t * query, uint8_t deep, ex_t * e); int ti_wrap_copy(ti_wrap_t ** wrap, uint8_t deep); int ti_wrap_dup(ti_wrap_t ** wrap, uint8_t deep); diff --git a/itest/test_wrap.py b/itest/test_wrap.py index 645f96fb..39ddf9d0 100755 --- a/itest/test_wrap.py +++ b/itest/test_wrap.py @@ -406,6 +406,34 @@ async def test_skip_false(self, client): {'name': 'd'} ]) + async def test_wrap_copy(self, client): + await client.query("""//ti + set_type('T', { + name: 'str', + upper: |this| this.name.upper(), + }); + """) + + res = await client.query("""//ti + .t = T{name: "Iris"}; + .t.wrap().copy(); + """) + self.assertEqual(res, {"name": "Iris", "upper": "IRIS"}) + + res = await client.query("""//ti + .o = { + t: .t.wrap() + }; + return .o.copy(2), 2; + """) + self.assertEqual(res, {'t': {'name': 'Iris', "upper": "IRIS"}}) + + res = await client.query("""//ti + .t = T{name: "Iris"}; + .t.wrap().copy().upper; + """) + self.assertEqual(res, "IRIS") + if __name__ == '__main__': run_test(TestWrap()) diff --git a/src/ti/wrap.c b/src/ti/wrap.c index eba69e5c..8e2be024 100644 --- a/src/ti/wrap.c +++ b/src/ti/wrap.c @@ -563,102 +563,63 @@ int ti__wrap_field_thing( return -1; } -int ti_wrap_copy(ti_wrap_t ** wrap, uint8_t deep) +int ti_wrap_cp(ti_query_t * query, uint8_t deep, ex_t * e) { - assert(deep); + ti_val_t * val; + mp_unp_t up; + msgpack_sbuffer buffer; + ti_vp_t vp = { + .query=query, + }; + ti_vup_t vup = { + .isclient = true, + .collection = query->collection, + .up = &up, + }; - ti_thing_t * other, * thing = (*wrap)->thing; - ti_type_t * type = ti_types_by_id( - thing->collection->types, - (*wrap)->type_id); + assert(ti_val_is_wrap(query->rval)); - if (!type) + if (mp_sbuffer_alloc_init(&buffer, ti_val_alloc_size(query->rval), 0)) { - ti_incref(thing); - - if (ti_thing_copy(&thing, deep)) - { - ti_decref(thing); - return -1; - } - - ti_val_unsafe_drop((ti_val_t *) *wrap); - *wrap = (ti_wrap_t *) thing; - return 0; + ex_set_mem(e); + goto fail0; } - --deep; + msgpack_packer_init(&vp.pk, &buffer, msgpack_sbuffer_write); - if (ti_thing_is_object(thing)) + if (ti_val_to_client_pk(query->rval, &vp, deep, TI_FLAGS_NO_IDS)) { - size_t n = ti_min(type->fields->n, ti_thing_n(thing)); - other = ti_thing_o_create(0, n , thing->collection); - if (!other) - return -1; - - for (vec_each(type->fields, ti_field_t, t_field)) - { - ti_prop_t * prop, * p; - prop = ti_thing_o_prop_weak_get(thing, t_field->name); - if (!prop || !ti_field_maps_to_val(t_field, prop->val)) - continue; - - p = ti_prop_dup(prop); - - if (!p || ti_val_copy(&p->val, other, p->name, deep)) - { - ti_prop_unassign_destroy(p); - goto fail; - } - VEC_push(other->items.vec, p); - }; + ex_set_mem(e); + goto fail1; } - else - { - ti_map_t * map = ti_type_map(type, thing->via.type); - if (!map) - return -1; - - other = ti_thing_o_create(0, map->mappings->n, thing->collection); - if (!other) - return -1; - for (vec_each(map->mappings, ti_mapping_t, mapping)) - { - ti_prop_t * p; - ti_val_t * val = VEC_get(thing->items.vec, mapping->f_field->idx); - - if (( - (mapping->t_field->flags & TI_FIELD_FLAG_SKIP_FALSE) && - !ti_val_is_bool(val)) || ( - (mapping->t_field->flags & TI_FIELD_FLAG_SKIP_NIL) && - ti_val_is_nil(val))) - continue; + mp_unp_init(&up, buffer.data, buffer.size); - p = ti_prop_create(mapping->f_field->name, val); - if (!p) - goto fail; + val = ti_val_from_vup_e(&vup, e); + ti_val_unsafe_drop(query->rval); + query->rval = val; - ti_incref(p->name); - ti_incref(p->val); +fail1: + msgpack_sbuffer_destroy(&buffer); +fail0: + return e->nr; +} - if (ti_val_copy(&p->val, other, p->name, deep)) - { - ti_prop_unassign_destroy(p); - goto fail; - } +int ti_wrap_copy(ti_wrap_t ** wrap, uint8_t deep) +{ + ti_thing_t * thing = (*wrap)->thing; + ti_wrap_t * _wrap = ti_wrap_create(thing, (*wrap)->type_id); + if (!_wrap) + return -1; - VEC_push(other->items.vec, p); - } + if (ti_thing_copy(&_wrap->thing, deep)) + { + ti_val_unsafe_drop((ti_val_t *) _wrap); + return -1; } - ti_val_unsafe_drop((ti_val_t *) *wrap); - *wrap = (ti_wrap_t *) other; + *wrap = _wrap; return 0; - -fail: - ti_val_unsafe_drop((ti_val_t *) other); - return -1; } int ti_wrap_dup(ti_wrap_t ** wrap, uint8_t deep) From 803301e419c04cab57543ea80b898b7457a79f49 Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Tue, 28 Oct 2025 09:49:24 +0100 Subject: [PATCH 02/13] . --- CHANGELOG.md | 3 ++- inc/ti/spec.t.h | 2 ++ inc/ti/types.t.h | 3 ++- src/ti/type.c | 12 ++++-------- src/ti/types.c | 6 +++++- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 603d8090..d99360cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ -# v1.7.7-alpha0 +# v1.8.0-alpha0 * Added `TI_PROTO_CLIENT_REQ_EMIT_PEER` protocol, pr #414. +* Changed `copy()` behavior of wrapped type, pr #415. # v1.7.6 diff --git a/inc/ti/spec.t.h b/inc/ti/spec.t.h index 469ff23a..f05ecfeb 100644 --- a/inc/ti/spec.t.h +++ b/inc/ti/spec.t.h @@ -45,6 +45,8 @@ typedef enum TI_SPEC_FLOAT_RANGE, /* `float<:> */ TI_SPEC_STR_RANGE, /* `str<:> */ TI_SPEC_UTF8_RANGE, /* `utf8<:> */ + TI_SPEC_TYPE, /* `{...} */ + TI_SPEC_ARR_TYPE, /* `[{...}] */ } ti_spec_enum_t; typedef enum diff --git a/inc/ti/types.t.h b/inc/ti/types.t.h index faf24fea..ce3aeeed 100644 --- a/inc/ti/types.t.h +++ b/inc/ti/types.t.h @@ -19,8 +19,9 @@ struct ti_types_s imap_t * imap; smap_t * smap; smap_t * removed; /* map with type id's which are removed */ - uint16_t next_id; ti_collection_t * collection; + uint32_t next_unnamed; + uint16_t next_id; }; #endif /* TI_TYPES_T_H_ */ diff --git a/src/ti/type.c b/src/ti/type.c index e96b6cc3..cf7e53d0 100644 --- a/src/ti/type.c +++ b/src/ti/type.c @@ -82,16 +82,10 @@ static int type__map_cleanup(ti_type_t * t_haystack, ti_type_t * t_needle) return 0; } -/* used as a callback function and destroys all type mappings */ -static void type__map_free(void * map) -{ - ti_map_destroy(map); -} - void ti_type_map_cleanup(ti_type_t * type) { (void) imap_walk(type->types->imap, (imap_cb) type__map_cleanup, type); - imap_clear(type->t_mappings, type__map_free); + imap_clear(type->t_mappings, (imap_destroy_cb) ti_map_destroy); } void ti_type_drop(ti_type_t * type) @@ -145,7 +139,7 @@ void ti_type_destroy(ti_type_t * type) vec_destroy(type->fields, (vec_destroy_cb) ti_field_destroy); vec_destroy(type->methods, (vec_destroy_cb) ti_method_destroy); - imap_destroy(type->t_mappings, type__map_free); + imap_destroy(type->t_mappings, (imap_destroy_cb) ti_map_destroy); ti_val_drop((ti_val_t *) type->rname); ti_val_drop((ti_val_t *) type->rwname); ti_val_drop((ti_val_t *) type->idname); @@ -325,6 +319,8 @@ static inline int type__assign( if (ti_val_is_closure(val)) return ti_type_add_method(type, name, (ti_closure_t *) val, e); + + ex_set(e, EX_TYPE_ERROR, "expecting a method of type `"TI_VAL_CLOSURE_S"` " "or a definition of type `"TI_VAL_STR_S"` " diff --git a/src/ti/types.c b/src/ti/types.c index d2d4653a..65f498f7 100644 --- a/src/ti/types.c +++ b/src/ti/types.c @@ -21,6 +21,7 @@ ti_types_t * ti_types_create(ti_collection_t * collection) types->removed = smap_create(); types->collection = collection; types->next_id = 0; + types->next_unnamed = 0x10000; // outside spec range if (!types->imap || !types->smap || !types->removed) { @@ -84,7 +85,10 @@ int types__spec_flags_pos(const unsigned char * x) x[i] == '&' || x[i] == '-' || x[i] == '+' || - x[i] == '*') + x[i] == '*' || + x[i] == '?' || + x[i] == '!' +) i++; return i; } From 400f1a9faf380fe0eb7f20c52ffa2852cb3524ac Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Tue, 28 Oct 2025 22:06:44 +0100 Subject: [PATCH 03/13] work on wrap-only nested... --- inc/ti/condition.t.h | 6 +- inc/ti/fn/fnjsondump.h | 1 + inc/ti/fn/fnmodtype.h | 92 ++++++++++-------- inc/ti/fn/fnsettype.h | 2 +- inc/ti/raw.inline.h | 5 + inc/ti/type.h | 18 +++- inc/ti/types.h | 2 + inc/ti/val.h | 1 + inc/ti/val.inline.h | 5 + inc/ti/vp.t.h | 1 + itest/test_wrap.py | 3 + src/ti/condition.c | 61 ++++++++++++ src/ti/ctask.c | 22 ++++- src/ti/enum.c | 5 +- src/ti/field.c | 40 ++++++++ src/ti/module.c | 5 +- src/ti/query.c | 7 +- src/ti/room.c | 1 + src/ti/spec.c | 10 ++ src/ti/store/storetypes.c | 23 +++-- src/ti/task.c | 10 +- src/ti/thing.c | 3 +- src/ti/type.c | 157 +++++++++++++++++++++++++++--- src/ti/types.c | 12 +++ src/ti/val.c | 5 +- src/ti/wrap.c | 195 ++++++++++++++++++++++++++++---------- 26 files changed, 565 insertions(+), 127 deletions(-) diff --git a/inc/ti/condition.t.h b/inc/ti/condition.t.h index 114995b7..db69688f 100644 --- a/inc/ti/condition.t.h +++ b/inc/ti/condition.t.h @@ -14,6 +14,7 @@ typedef struct ti_condition_rel_s ti_condition_rel_t; union ti_condition_via_u { + ti_type_t * type; /* nested type (wrap-only) */ ti_condition_re_t * re; /* str, utf8 */ ti_condition_srange_t * srange; /* str, utf8 */ ti_condition_irange_t * irange; /* int, float */ @@ -24,11 +25,12 @@ union ti_condition_via_u #include #include +#include #include #include -#include -#include #include +#include +#include typedef void (*ti_condition_rel_cb) (ti_field_t *, ti_thing_t *, ti_thing_t *); diff --git a/inc/ti/fn/fnjsondump.h b/inc/ti/fn/fnjsondump.h index 412f43d7..97eba082 100644 --- a/inc/ti/fn/fnjsondump.h +++ b/inc/ti/fn/fnjsondump.h @@ -44,6 +44,7 @@ static int do__f_json_dump(ti_query_t * query, cleri_node_t * nd, ex_t * e) size_t total_n; ti_vp_t vp = { .query=query, + .size_limit=ti.cfg->result_size_limit, }; json_dumps__options_t options = { .deep = query->qbind.deep, diff --git a/inc/ti/fn/fnmodtype.h b/inc/ti/fn/fnmodtype.h index d14b5c5b..a9de9f12 100644 --- a/inc/ti/fn/fnmodtype.h +++ b/inc/ti/fn/fnmodtype.h @@ -409,56 +409,66 @@ static void type__add( return; } - else if (!ti_val_is_str(query->rval)) - { - ex_set(e, EX_TYPE_ERROR, - "function `%s` expects argument 4 to be of " - "type `"TI_VAL_STR_S"` or type `"TI_VAL_CLOSURE_S"` " - "but got type `%s` instead"DOC_MOD_TYPE_ADD, - fnname, ti_val_str(query->rval)); - return; - } - spec_raw = (ti_raw_t *) query->rval; - if (spec_raw->n == 1 && spec_raw->data[0] == '#') + spec_raw = ti_type_nested_from_val(type, query->rval, e) + if (e-nr) + return e->nr; + + if (spec_raw) + ti_val_unsafe_gc_drop(query->rval); + else { - if (nargs == 5) + if (!ti_val_is_str(query->rval)) { - ex_set(e, EX_NUM_ARGUMENTS, - "function `%s` takes at most 4 arguments when adding " - "an Id ('#') definition"DOC_MOD_TYPE_ADD, - fnname); + ex_set(e, EX_TYPE_ERROR, + "function `%s` expects argument 4 to be of " + "type `"TI_VAL_STR_S"` or type `"TI_VAL_CLOSURE_S"` " + "but got type `%s` instead"DOC_MOD_TYPE_ADD, + fnname, ti_val_str(query->rval)); return; } + spec_raw = (ti_raw_t *) query->rval; - if (type->idname) + if (spec_raw->n == 1 && spec_raw->data[0] == '#') { - ex_set(e, EX_LOOKUP_ERROR, - "multiple Id ('#') definitions on type `%s`", - type->name); - return; - } + if (nargs == 5) + { + ex_set(e, EX_NUM_ARGUMENTS, + "function `%s` takes at most 4 arguments when adding " + "an Id ('#') definition"DOC_MOD_TYPE_ADD, + fnname); + return; + } - task = ti_task_get_task(query->change, query->collection->root); - if (!task) - { - ex_set_mem(e); - return; - } + if (type->idname) + { + ex_set(e, EX_LOOKUP_ERROR, + "multiple Id ('#') definitions on type `%s`", + type->name); + return; + } - /* update modified time-stamp */ - type->modified_at = util_now_usec(); - type->idname = name; - ti_incref(name); + task = ti_task_get_task(query->change, query->collection->root); + if (!task) + { + ex_set_mem(e); + return; + } - if (ti_task_add_mod_type_add_idname(task, type)) - { - /* modified is wrong; the rest we can revert */ - type->idname = NULL; - ti_decref(name); - ex_set_mem(e); + /* update modified time-stamp */ + type->modified_at = util_now_usec(); + type->idname = name; + ti_incref(name); + + if (ti_task_add_mod_type_add_idname(task, type)) + { + /* modified is wrong; the rest we can revert */ + type->idname = NULL; + ti_decref(name); + ex_set_mem(e); + } + return; } - return; } query->rval = NULL; @@ -1004,6 +1014,8 @@ static void type__mod( return; } + // TODO: WO-NESTED: handle arr and thing + ex_set(e, EX_TYPE_ERROR, "function `%s` expects argument 4 to be of " "type `"TI_VAL_STR_S"` or type `"TI_VAL_CLOSURE_S"` " @@ -1453,7 +1465,7 @@ static void type__wpo( if (wrap_only && ti_type_required_by_non_wpo(type, e)) return; - if (!wrap_only && ti_type_uses_wpo(type, e)) + if (!wrap_only && ti_type_requires_wpo(type, e)) return; task = ti_task_get_task(query->change, query->collection->root); diff --git a/inc/ti/fn/fnsettype.h b/inc/ti/fn/fnsettype.h index e7aa7ddc..335ade99 100644 --- a/inc/ti/fn/fnsettype.h +++ b/inc/ti/fn/fnsettype.h @@ -56,7 +56,7 @@ static int do__f_set_type(ti_query_t * query, cleri_node_t * nd, ex_t * e) type = ti_type_create( query->collection->types, type_id, - TI_TYPE_FLAG_WRAP_ONLY, /* prevents looking for instances */ + TI_TYPE_FLAG_WRAP_ONLY, (const char *) rname->data, rname->n, ts_now, diff --git a/inc/ti/raw.inline.h b/inc/ti/raw.inline.h index e0baaa0d..026da5c1 100644 --- a/inc/ti/raw.inline.h +++ b/inc/ti/raw.inline.h @@ -32,6 +32,11 @@ static inline _Bool ti_raw_is_name(ti_raw_t * raw) return raw->tp == TI_VAL_NAME; } +static inline _Bool ti_raw_is_mpdata(ti_raw_t * raw) +{ + return raw->tp == TI_VAL_MPDATA; +} + static inline _Bool ti_raw_is_reserved_key(ti_raw_t * raw) { return raw->n == 1 && (*raw->data >> 4 == 2); diff --git a/inc/ti/type.h b/inc/ti/type.h index 38fc911e..d8ff880a 100644 --- a/inc/ti/type.h +++ b/inc/ti/type.h @@ -8,9 +8,10 @@ #include #include #include +#include #include #include -#include +#include #include #include #include @@ -26,7 +27,9 @@ ti_type_t * ti_type_create( size_t name_n, uint64_t created_at, uint64_t modified_at); +ti_type_t * ti_type_create_unnamed(ti_types_t * types); void ti_type_drop(ti_type_t * type); +void ti_type_drop_unnamed(ti_type_t * type); void ti_type_del(ti_type_t * type, vec_t * vars); void ti_type_destroy(ti_type_t * type); void ti_type_map_cleanup(ti_type_t * type); @@ -67,8 +70,19 @@ int ti_type_methods_info_to_pk( msgpack_packer * pk, _Bool with_definition); int ti_type_required_by_non_wpo(ti_type_t * type, ex_t * e); -int ti_type_uses_wpo(ti_type_t * type, ex_t * e); +int ti_type_requires_wpo(ti_type_t * type, ex_t * e); int ti_type_rename(ti_type_t * type, ti_raw_t * nname); +ti_raw_t * ti__type_nested_from_val(ti_type_t * type, ti_val_t * val, ex_t * e); + +static inline ti_raw_t * ti_type_nested_from_val( + ti_type_t * type, + ti_val_t * val, + ex_t * e) +{ + return (ti_val_is_thing(val) || ti_val_is_array(val)) + ? ti__type_nested_from_val(type, val, e) + : NULL; +} static inline int ti_type_use(ti_type_t * type, ex_t * e) { diff --git a/inc/ti/types.h b/inc/ti/types.h index 674dcc3c..743c74af 100644 --- a/inc/ti/types.h +++ b/inc/ti/types.h @@ -18,11 +18,13 @@ ti_types_t * ti_types_create(ti_collection_t * collection); void ti_types_destroy(ti_types_t * types); int ti_types_add(ti_types_t * types, ti_type_t * type); void ti_types_del(ti_types_t * types, ti_type_t * type); +void ti_types_del_unnamed(ti_types_t * types, ti_type_t * type); int ti_types_ren_spec( ti_types_t * types, uint16_t type_or_enum_id, ti_raw_t * nname); int ti_types_ren_member_spec(ti_types_t * types, ti_member_t * member); +int ti_types_add_unnamed(ti_types_t * types, ti_type_t * type); uint16_t ti_types_get_new_id(ti_types_t * types, ti_raw_t * rname, ex_t * e); ti_varr_t * ti_types_info(ti_types_t * types, _Bool with_definition); int ti_types_to_pk(ti_types_t * types, msgpack_packer * pk); diff --git a/inc/ti/val.h b/inc/ti/val.h index af6d02fd..ff808acf 100644 --- a/inc/ti/val.h +++ b/inc/ti/val.h @@ -31,6 +31,7 @@ extern ti_val_t * val__parent_name; extern ti_val_t * val__parent_type_name; extern ti_val_t * val__key_name; extern ti_val_t * val__key_type_name; +extern ti_val_t * val__unnamed_name; /* string */ extern ti_val_t * val__sany; diff --git a/inc/ti/val.inline.h b/inc/ti/val.inline.h index 48b1f8ec..b1b00ccd 100644 --- a/inc/ti/val.inline.h +++ b/inc/ti/val.inline.h @@ -914,6 +914,11 @@ static inline ti_val_t * ti_val_gmt_offset_name(void) return ti_incref(val__gmt_offset_name), val__gmt_offset_name; } +static inline ti_val_t * ti_val_unnamed_name(void) +{ + return ti_incref(val__unnamed_name), val__unnamed_name; +} + static inline ti_val_t * ti_val_borrow_async_name(void) { return val__async_name; diff --git a/inc/ti/vp.t.h b/inc/ti/vp.t.h index 363668e9..c5bc15f6 100644 --- a/inc/ti/vp.t.h +++ b/inc/ti/vp.t.h @@ -19,6 +19,7 @@ struct ti_vp_s * but in this case `methods` will not * be calculated. */ + size_t size_limit; }; #endif /* TI_VP_T_H_ */ diff --git a/itest/test_wrap.py b/itest/test_wrap.py index 39ddf9d0..79ee521e 100755 --- a/itest/test_wrap.py +++ b/itest/test_wrap.py @@ -434,6 +434,9 @@ async def test_wrap_copy(self, client): """) self.assertEqual(res, "IRIS") + async def test_wrap_nested(self, client): + pass + # TODO: WO-NESTED: test if can call self in nested declaration if __name__ == '__main__': run_test(TestWrap()) diff --git a/src/ti/condition.c b/src/ti/condition.c index 8f74c3a7..c816edec 100644 --- a/src/ti/condition.c +++ b/src/ti/condition.c @@ -738,6 +738,63 @@ int ti_condition_field_rel_init( return e->nr; } +int ti_condition_nested_type(ti_field_t * field, ex_t * e) +{ + ti_val_t * val; + ti_thing_t * thing; + mp_unp_t up; + ti_vup_t vup = { + .isclient = true, + .collection = field->type->types->collection, + .up = &up, + }; + mp_unp_init(&up, field->spec_raw->data, field->spec_raw->n); + + val = ti_val_from_vup_e(&vup, e); + if (!val) + return e->nr; + + if (ti_val_is_array(val)) + { + vec_t * vec = VARR(val); + field->nested_spec = TI_SPEC_TYPE; + field->spec = TI_SPEC_ARR_TYPE; + thing = vec_first(vec); + if (!thing || !ti_val_is_thing((ti_val_t *) thing)) + { + ex_set_internal(e); /* only with corrupt internal data */ + goto fail0; + } + } + else if (ti_val_is_thing(val)) + { + /* val is thing */ + field->spec = TI_SPEC_TYPE; + thing = val; + } + else + { + ex_set_internal(e); /* only with corrupt internal data */ + goto fail0; + } + + type = ti_type_create_unnamed(field->type->types); + if (!type) + { + ex_set_mem(e); + goto fail0; + } + + if (ti_type_init_from_thing(type, thing, e)) + ti_type_drop_unnamed(type); + else + field->condition.type = type; + +fail0: + ti_val_unsafe_drop(val); + return e->nr; +} + void ti_condition_destroy(ti_condition_via_t condition, uint16_t spec) { if (!condition.none) @@ -747,6 +804,10 @@ void ti_condition_destroy(ti_condition_via_t condition, uint16_t spec) switch((ti_spec_enum_t) spec) { + case TI_SPEC_TYPE: + case TI_SPEC_ARR_TYPE: + ti_type_drop_unnamed(condition.type); + return; case TI_SPEC_ANY: return; /* a field may be set to ANY while using mod_type in which case a condition should be left alone */ diff --git a/src/ti/ctask.c b/src/ti/ctask.c index 4d6cd951..2a025b68 100644 --- a/src/ti/ctask.c +++ b/src/ti/ctask.c @@ -1425,7 +1425,7 @@ static int ctask__mod_type_add(ti_thing_t * thing, mp_unp_t * up) if (mp_str_eq(&mp_target, "spec")) { - if (mp_next(up, &mp_spec) != MP_STR) + if (mp_next(up, &mp_spec) != MP_STR && mp_spec.tp != MP_BIN) { log_critical( "task `mod_type_add` for "TI_COLLECTION_ID" is invalid", @@ -1475,7 +1475,9 @@ static int ctask__mod_type_add(ti_thing_t * thing, mp_unp_t * up) goto fail0; } - if (mp_spec.via.str.n == 1 && mp_spec.via.str.data[0] == '#') + if (mp_spec.tp == MP_STR && + mp_spec.via.str.n == 1 && + mp_spec.via.str.data[0] == '#') { if (type->idname) { @@ -1511,7 +1513,9 @@ static int ctask__mod_type_add(ti_thing_t * thing, mp_unp_t * up) } } - spec_raw = ti_str_create(mp_spec.via.str.data, mp_spec.via.str.n); + spec_raw = mp_spec.tp == MP_STR + ? ti_str_create(mp_spec.via.str.data, mp_spec.via.str.n) + : ti_mp_create(mp_spec.via.bin.data, mp_spec.via.bin.n); if (!spec_raw) { log_critical(EX_MEMORY_S); @@ -1703,7 +1707,16 @@ static int ctask__mod_type_mod(ti_thing_t * thing, mp_unp_t * up) return -1; } - if (mp_next(up, &mp_spec) != MP_STR) + if (mp_next(up, &mp_spec) == MP_STR) + { + spec_raw = ti_str_create(mp_spec.via.str.data, mp_spec.via.str.n); + + } + else if (mp_spec.tp == MP_BIN) + { + spec_raw = ti_mp_create(mp_spec.via.bin.data, mp_spec.via.bin.n); + } + else { log_critical( "task `mod_type_mod` for "TI_COLLECTION_ID" is invalid", @@ -1711,7 +1724,6 @@ static int ctask__mod_type_mod(ti_thing_t * thing, mp_unp_t * up) return -1; } - spec_raw = ti_str_create(mp_spec.via.str.data, mp_spec.via.str.n); if (!spec_raw) { log_critical(EX_MEMORY_S); diff --git a/src/ti/enum.c b/src/ti/enum.c index 2d6728f5..82fc24c3 100644 --- a/src/ti/enum.c +++ b/src/ti/enum.c @@ -797,7 +797,10 @@ ti_member_t * ti_enum_member_by_val_e( ti_val_t * ti_enum_as_mpval(ti_enum_t * enum_) { ti_raw_t * raw; - ti_vp_t vp; + ti_vp_t vp = { + .query=NULL, + .size_limit=ti.cfg->result_size_limit, + }; msgpack_sbuffer buffer; mp_sbuffer_alloc_init(&buffer, sizeof(ti_raw_t), sizeof(ti_raw_t)); diff --git a/src/ti/field.c b/src/ti/field.c index 33634108..ba1d36a1 100644 --- a/src/ti/field.c +++ b/src/ti/field.c @@ -456,6 +456,13 @@ static int field__init(ti_field_t * field, ex_t * e) return e->nr; } + if (ti_raw_is_mpdata(field->spec_raw)) + { + LOGC("WO-NESTED"); + return e-nr; + } + // TODO: WO-NESTED: Test and process nested fields + do { /* WARNING: do not forget to update types.c when modifying the flags in @@ -1126,6 +1133,7 @@ int ti_field_mod( assert(0); nillable: + /* not reached for wrap-only type, therefore spec_raw is always str type */ ex_set(e, EX_OPERATION, "cannot apply type declaration `%.*s` to `%s` on type `%s` without a " "closure to migrate existing instances; the old declaration " @@ -1136,6 +1144,7 @@ int ti_field_mod( goto undo_dep; incompatible: + /* not reached for wrap-only type, therefore spec_raw is always str type */ ex_set(e, EX_OPERATION, "cannot apply type declaration `%.*s` to `%s` on type `%s` without a " "closure to migrate existing instances; the old declaration `%.*s` " @@ -1688,6 +1697,14 @@ static _Bool field__maps_arr_to_arr(ti_field_t * field, ti_varr_t * varr) return true; } +static _Bool field__maps_arr_to_type(ti_field_t * field, ti_varr_t * varr) +{ + for (vec_each(varr->vec, ti_val_t, val)) + if (!ti_val_is_thing(val)) + return false; + return true; +} + static int field__map_restrict_cb(ti_prop_t * prop, ti_field_t * field) { return !ti_spec_maps_to_nested_val(field->nested_spec, prop->val); @@ -1980,6 +1997,9 @@ int ti_field_make_assignable( ((ti_raw_t *) *val)->n > field->condition.srange->ma) goto srange_error; return 0; + case TI_SPEC_TYPE: + case TI_SPEC_ARR_TYPE: + assert(0); /* both are only for wrap-only type */ } assert(spec >= TI_ENUM_ID_FLAG); @@ -2266,6 +2286,13 @@ _Bool ti_field_maps_to_val(ti_field_t * field, ti_val_t * val) ((ti_raw_t *) val)->n) && ((ti_raw_t *) val)->n >= field->condition.srange->mi && ((ti_raw_t *) val)->n <= field->condition.srange->ma); + case TI_SPEC_TYPE: + return ti_val_is_thing(val); + case TI_SPEC_ARR_TYPE: + return (( + ti_val_is_array(val) && + field__maps_arr_to_type(field, (ti_varr_t *) val) + ) || ti_val_is_set(val)); } /* any *thing* can be mapped */ @@ -2357,6 +2384,8 @@ static _Bool field__maps_to_nested(ti_field_t * t_field, ti_field_t * f_field) case TI_SPEC_FLOAT_RANGE: case TI_SPEC_STR_RANGE: case TI_SPEC_UTF8_RANGE: + case TI_SPEC_TYPE: + case TI_SPEC_ARR_TYPE: return false; } @@ -2504,6 +2533,17 @@ _Bool ti_field_maps_to_field(ti_field_t * t_field, ti_field_t * f_field) f_spec == TI_SPEC_UTF8_RANGE && f_field->condition.srange->mi >= t_field->condition.srange->mi && f_field->condition.srange->ma <= t_field->condition.srange->ma); + case TI_SPEC_TYPE: + return f_spec < TI_SPEC_ANY || f_spec == TI_SPEC_OBJECT; + case TI_SPEC_ARR_TYPE: + return ( + f_spec == TI_SPEC_SET || ( + f_spec == TI_SPEC_ARR && ( + f_field->nested_spec < TI_SPEC_ANY || + f_field->nested_spec == TI_SPEC_OBJECT + ) + ) + ); } assert(t_spec < TI_SPEC_ANY); diff --git a/src/ti/module.c b/src/ti/module.c index 86cc9ad9..5c20fe24 100644 --- a/src/ti/module.c +++ b/src/ti/module.c @@ -144,6 +144,7 @@ static void module__cb(ti_future_t * future) ti_thing_t * thing = VEC_get(future->args, 0); ti_vp_t vp = { .query=future->query, /* bug # #351 */ + .size_limit=ti.cfg->result_size_limit, }; msgpack_sbuffer buffer; size_t alloc_sz = 1024; @@ -220,6 +221,7 @@ ti_pkg_t * ti_module_conf_pkg(ti_val_t * val, ti_query_t * query) ti_pkg_t * pkg; ti_vp_t vp = { .query=query, + .size_limit=ti.cfg->result_size_limit, }; msgpack_sbuffer buffer; size_t alloc_sz = 1024; @@ -1447,7 +1449,8 @@ ti_val_t * ti_module_as_mpval(ti_module_t * module, int flags) ti_raw_t * raw; msgpack_sbuffer buffer; ti_vp_t vp = { - .query = NULL + .query=NULL, + .size_limit=ti.cfg->result_size_limit, }; mp_sbuffer_alloc_init(&buffer, sizeof(ti_raw_t), sizeof(ti_raw_t)); diff --git a/src/ti/query.c b/src/ti/query.c index f146f206..e9ec8d3d 100644 --- a/src/ti/query.c +++ b/src/ti/query.c @@ -1113,7 +1113,8 @@ static inline int query__pack_response( ex_t * e) { ti_vp_t vp = { - .query=query + .query=query, + .size_limit=ti.cfg->result_size_limit, }; msgpack_packer_init(&vp.pk, buffer, msgpack_sbuffer_write); @@ -1123,7 +1124,7 @@ static inline int query__pack_response( (int) query->qbind.deep, (int) query->flags & TI_FLAGS_NO_IDS)) { - if (buffer->size > ti.cfg->result_size_limit) + if (buffer->size > vp.size_limit) ex_set(e, EX_RESULT_TOO_LARGE, "too much data to return; " "try to use a lower `deep` value and/or `wrap` things to " @@ -1138,6 +1139,8 @@ static inline int query__pack_response( if (buffer->size > ti.counters->largest_result_size) ti.counters->largest_result_size = buffer->size; + ti_mp_create + return 0; } diff --git a/src/ti/room.c b/src/ti/room.c index ce8b7832..c544b574 100644 --- a/src/ti/room.c +++ b/src/ti/room.c @@ -175,6 +175,7 @@ int ti_room_emit( ti_rpkg_t * node_rpkg, * client_rpkg; ti_vp_t vp = { .query=query, /* required for wrap type, may be NULL from API */ + .size_limit=ti.cfg->result_size_limit, }; if (mp_sbuffer_alloc_init(&buffer, alloc, sizeof(ti_pkg_t))) diff --git a/src/ti/spec.c b/src/ti/spec.c index 129aeb29..cd4711d3 100644 --- a/src/ti/spec.c +++ b/src/ti/spec.c @@ -410,6 +410,8 @@ ti_spec_rval_enum ti__spec_check_nested_val(uint16_t spec, ti_val_t * val) case TI_SPEC_FLOAT_RANGE: case TI_SPEC_STR_RANGE: case TI_SPEC_UTF8_RANGE: + case TI_SPEC_TYPE: + case TI_SPEC_ARR_TYPE: assert(0); /* not supported on nested definition */ return false; } @@ -504,6 +506,8 @@ _Bool ti__spec_maps_to_nested_val(uint16_t spec, ti_val_t * val) case TI_SPEC_FLOAT_RANGE: case TI_SPEC_STR_RANGE: case TI_SPEC_UTF8_RANGE: + case TI_SPEC_TYPE: + case TI_SPEC_ARR_TYPE: assert(0); /* only nested so conditions are not possible */ return false; } @@ -549,6 +553,9 @@ const char * ti_spec_approx_type_str(uint16_t spec) case TI_SPEC_EMAIL: return "email"; case TI_SPEC_URL: return "url"; case TI_SPEC_TEL: return "tel"; + case TI_SPEC_TYPE: + case TI_SPEC_ARR_TYPE: + assert(0); /* not possible for wrap-only */ } return spec < TI_SPEC_ANY ? "thing" : "enum"; } @@ -689,6 +696,9 @@ ti_spec_mod_enum ti_spec_check_mod( ocondition.srange->mi >= ncondition.srange->mi && ocondition.srange->ma <= ncondition.srange->ma ) ? TI_SPEC_MOD_SUCCESS : TI_SPEC_MOD_ERR; + case TI_SPEC_TYPE: + case TI_SPEC_ARR_TYPE: + assert(0); /* both are only for wrap-only type */ } return ospec == nspec ? TI_SPEC_MOD_SUCCESS : TI_SPEC_MOD_ERR; diff --git a/src/ti/store/storetypes.c b/src/ti/store/storetypes.c index ed4e35fb..adc6e902 100644 --- a/src/ti/store/storetypes.c +++ b/src/ti/store/storetypes.c @@ -51,7 +51,10 @@ static int mktype_cb(ti_type_t * type, msgpack_packer * pk) { p = (uintptr_t) field->name; if (msgpack_pack_uint64(pk, p) || - mp_pack_strn(pk, field->spec_raw->data, field->spec_raw->n) + (ti_raw_is_mpdata(field->spec_raw) + ? mp_pack_bin(pk, field->spec_raw->data, field->spec_raw->n) + : mp_pack_strn(pk, field->spec_raw->data, field->spec_raw->n) + ) ) return -1; } @@ -173,7 +176,7 @@ int ti_store_types_restore(ti_types_t * types, imap_t * names, const char * fn) size_t i, ii; uint16_t type_id; uintptr_t utype_id; - ti_raw_t * spec; + ti_raw_t * spec_raw; mp_obj_t obj, mp_id, mp_name, mp_wpo, mp_hid, mp_spec, mp_created, mp_modified; mp_unp_t up; @@ -350,14 +353,16 @@ int ti_store_types_restore(ti_types_t * types, imap_t * names, const char * fn) for (ii = obj.via.sz; ii--;) { if (mp_next(&up, &mp_id) != MP_U64 || - mp_next(&up, &mp_spec) != MP_STR + (mp_next(&up, &mp_spec) != MP_STR && mp_spec.tp != MP_BIN) ) goto fail1; name = imap_get(names, mp_id.via.u64); if (!name) goto fail1; - if (mp_spec.via.str.n == 1 && mp_spec.via.str.data[0] == '#') + if (mp_spec.tp == MP_STR && + mp_spec.via.str.n == 1 && + mp_spec.via.str.data[0] == '#') { if (type->idname) goto fail1; @@ -366,17 +371,19 @@ int ti_store_types_restore(ti_types_t * types, imap_t * names, const char * fn) continue; } - spec = ti_str_create(mp_spec.via.str.data, mp_spec.via.str.n); - if (!spec) + spec_raw = mp_spec.tp == MP_STR + ? ti_str_create(mp_spec.via.str.data, mp_spec.via.str.n) + : ti_mp_create(mp_spec.via.bin.data, mp_spec.via.bin.n); + if (!spec_raw) goto fail1; - if (!ti_field_create(name, spec, type, &e)) + if (!ti_field_create(name, spec_raw, type, &e)) { log_critical(e.msg); goto fail1; } - ti_decref(spec); + ti_decref(spec_raw); } if (!with_methods) diff --git a/src/ti/task.c b/src/ti/task.c index f9308d5e..60c76ae8 100644 --- a/src/ti/task.c +++ b/src/ti/task.c @@ -1643,7 +1643,10 @@ int ti_task_add_mod_type_add_field( mp_pack_strn(&pk, field->name->str, field->name->n); mp_pack_str(&pk, "spec"); - mp_pack_strn(&pk, field->spec_raw->data, field->spec_raw->n); + if (ti_raw_is_mpdata(field->spec_raw)) + mp_pack_bin(&pk, field->spec_raw->data, field->spec_raw->n) + else + mp_pack_strn(&pk, field->spec_raw->data, field->spec_raw->n) if (dval) { @@ -1920,7 +1923,10 @@ int ti_task_add_mod_type_mod_field(ti_task_t * task, ti_field_t * field) mp_pack_strn(&pk, field->name->str, field->name->n); mp_pack_str(&pk, "spec"); - mp_pack_strn(&pk, field->spec_raw->data, field->spec_raw->n); + if (ti_raw_is_mpdata(field->spec_raw)) + mp_pack_bin(&pk, field->spec_raw->data, field->spec_raw->n) + else + mp_pack_strn(&pk, field->spec_raw->data, field->spec_raw->n) data = (ti_data_t *) buffer.data; ti_data_init(data, buffer.size); diff --git a/src/ti/thing.c b/src/ti/thing.c index 788d0f63..6915feb5 100644 --- a/src/ti/thing.c +++ b/src/ti/thing.c @@ -1295,8 +1295,7 @@ int ti_thing__to_client_pk( * The correct error is not set here, but instead the size should be * checked again to set either a `memory` or `too_much_data` error. */ - if (((msgpack_sbuffer *) vp->pk.data)->size > - ti.cfg->result_size_limit) + if (((msgpack_sbuffer *) vp->pk.data)->size > vp->size_limit) return -1; --deep; diff --git a/src/ti/type.c b/src/ti/type.c index cf7e53d0..a7ebd58d 100644 --- a/src/ti/type.c +++ b/src/ti/type.c @@ -66,7 +66,43 @@ ti_type_t * ti_type_create( type->methods = vec_new(0); if (!type->name || !type->wname || !type->dependencies || !type->fields || - !type->t_mappings || ti_types_add(types, type)) + !type->rname || !type->rwname || !type->t_mappings || !type->methods || + ti_types_add(types, type)) + { + ti_type_destroy(type); + return NULL; + } + + return type; +} + +ti_type_t * ti_type_create_unnamed(ti_types_t * types) +{ + ti_type_t * type = malloc(sizeof(ti_type_t)); + if (!type) + return NULL; + + type->refcount = 0; /* only incremented when this type + is used by another type */ + type->selfref = 0; /* increment on self references */ + type->type_id = TI_SPEC_TYPE; + type->flags = TI_TYPE_FLAG_WRAP_ONLY; + type->rname = ti_val_unnamed_name(); + type->name = strndup(type->rname->data, type->rname->n); + type->rwname = ti_val_unnamed_name(); + type->wname = strndup(type->rwname->data, type->rwname->n); + type->idname = NULL; + type->dependencies = vec_new(0); + type->fields = vec_new(0); + type->types = types; + type->t_mappings = imap_create(); + type->created_at = 0; + type->modified_at = 0; + type->methods = vec_new(0); + + if (!type->name || !type->wname || !type->dependencies || !type->fields || + !type->rname || !type->rwname || !type->t_mappings || !type->methods || + ti_types_add(types, type)) { ti_type_destroy(type); return NULL; @@ -108,6 +144,18 @@ void ti_type_drop(ti_type_t * type) ti_type_destroy(type); } +void ti_type_drop_unnamed(ti_type_t * type) +{ + if (!type) + return; + + for (vec_each(type->dependencies, ti_type_t, dep)) + --dep->refcount; + + ti_types_del_unnamed(type->types, type); + ti_type_destroy(type); +} + static int type__del(ti_thing_t * thing, uint16_t * type_id) { if (thing->type_id == *type_id) @@ -288,16 +336,96 @@ int ti_type_set_idname( return e->nr; } +ti_raw_t * ti__type_nested_from_val(ti_type_t * type, ti_val_t * val, ex_t * e) +{ + ti_raw_t * spec_raw = NULL; + msgpack_sbuffer buffer; + ti_vp_t vp = { + .query=query, + .size_limit=0x1000, /* we do not expect much but since we use deep + nesting, set a low size limit */ + }; + + if (ti_val_is_array(val)) + { + ti_varr_t * varr = (ti_varr_t *) val; + if (varr->vec->n != 1) + { + ex_set(e, EX_VALUE_ERROR, + "array with nested type must have exactly one element but " + "found %zu on type `%s`", + varr->n + type->name); + return e->nr; + } + if (!ti_val_is_thing(VEC_first(varr->vec))) + { + ex_set(e, EX_VALUE_ERROR, + "element in array must be of type `"TI_VAL_THING_S"` " + "but got type `%s` instead", + ti_val_str(VEC_first(varr->vec))); + return e->nr; + } + } + + if (~type->flags & TI_TYPE_FLAG_WRAP_ONLY) + { + ex_set(e, EX_VALUE_ERROR, + "defining nested types is only allowed ' + 'for wrap-only type; define type `%s` as wrap-only", + type->name); + return e->nr; + } + + if (mp_sbuffer_alloc_init(&buffer, vp.size_limit, 0)) + { + ex_set_mem(e); + return e->nr; + } + + msgpack_packer_init(&vp.pk, &buffer, msgpack_sbuffer_write); + + if (ti_val_to_client_pk(val, &vp, TI_MAX_DEEP, TI_FLAGS_NO_IDS)) + { + if (buffer.size > vp.size_limit) + ex_set(e, EX_VALUE_ERROR, + "nested type definition on type `%s` too large", + type->name); + else + ex_set_mem(e); + goto fail0; + } + + spec_raw = ti_mp_create(buffer.data, buffer.size); + if (!spec_raw) + ex_set_mem(e); + +fail0: + msgpack_sbuffer_destroy(&buffer); + return spec_raw; +} + static inline int type__assign( ti_type_t * type, ti_name_t * name, ti_val_t * val, ex_t * e) { + ti_raw_t * spec_raw = ti_type_nested_from_val(type, val, e) + if (spec_raw) + { + (void) ti_field_create(name, spec_raw, type, e); + ti_val_unsafe_drop((ti_val_t *) spec_raw); + return e->nr; + } + + if (e-nr) + return e->nr; + if (ti_val_is_str(val)) { - register ti_raw_t * raw = (ti_raw_t *) val; - if (raw->n == 1 && raw->data[0] == '#') + spec_raw = (ti_raw_t *) val; + if (spec_raw->n == 1 && spec_raw->data[0] == '#') { if (type->idname) { @@ -312,15 +440,13 @@ static inline int type__assign( return 0; } - (void) ti_field_create(name, raw, type, e); + (void) ti_field_create(name, spec_raw, type, e); return e->nr; } if (ti_val_is_closure(val)) return ti_type_add_method(type, name, (ti_closure_t *) val, e); - - ex_set(e, EX_TYPE_ERROR, "expecting a method of type `"TI_VAL_CLOSURE_S"` " "or a definition of type `"TI_VAL_STR_S"` " @@ -709,7 +835,7 @@ int ti_type_init_from_unp( { if (mp_next(up, &obj) != MP_ARR || obj.via.sz != 2 || mp_next(up, &mp_name) != MP_STR || - mp_next(up, &mp_spec) != MP_STR) + (mp_next(up, &mp_spec) != MP_STR && mp_spec.tp != MP_BIN) { ex_set(e, EX_BAD_DATA, "failed unpacking fields for type `%s`; " @@ -731,7 +857,9 @@ int ti_type_init_from_unp( if (!name) return e->nr; - if (mp_spec.via.str.n == 1 && mp_spec.via.str.data[0] == '#') + if (mp_spec.tp == MP_STR && + mp_spec.via.str.n == 1 && + mp_spec.via.str.data[0] == '#') { if (type->idname) { @@ -745,7 +873,9 @@ int ti_type_init_from_unp( continue; } - spec_raw = ti_str_create(mp_spec.via.str.data, mp_spec.via.str.n); + spec_raw = mp_spec.tp == MP_STR + ? ti_str_create(mp_spec.via.str.data, mp_spec.via.str.n) + : ti_mp_create(mp_spec.via.bin.data, mp_spec.via.bin.n); if (!spec_raw || !ti_field_create(name, spec_raw, type, e)) goto failed; @@ -839,7 +969,10 @@ int ti_type_fields_to_pk(ti_type_t * type, msgpack_packer * pk) { if (msgpack_pack_array(pk, 2) || mp_pack_strn(pk, field->name->str, field->name->n) || - mp_pack_strn(pk, field->spec_raw->data, field->spec_raw->n)) + (ti_raw_is_mpdata(field->spec_raw) + ? mp_pack_bin(pk, field->spec_raw->data, field->spec_raw->n) + : mp_pack_strn(pk, field->spec_raw->data, field->spec_raw->n) + )) return -1; } @@ -891,6 +1024,7 @@ int ti_type_relations_to_pk(ti_type_t * type, msgpack_packer * pk) mp_pack_strn(pk, ofield->name->str, ofield->name->n) || mp_pack_str(pk, "definition") || + // TODO: WO-NESTED: output nested mp_pack_strn(pk, ofield->spec_raw->data, ofield->spec_raw->n)) return -1; } @@ -1298,7 +1432,7 @@ int ti_type_required_by_non_wpo(ti_type_t * type, ex_t * e) return e->nr; } -int ti_type_uses_wpo(ti_type_t * type, ex_t * e) +int ti_type_requires_wpo(ti_type_t * type, ex_t * e) { for (vec_each(type->fields, ti_field_t, field)) { @@ -1315,6 +1449,7 @@ int ti_type_uses_wpo(ti_type_t * type, ex_t * e) return e->nr; } } + // TODO: WO-NESTED: check if field is nested type } return 0; } diff --git a/src/ti/types.c b/src/ti/types.c index 65f498f7..ab77cd66 100644 --- a/src/ti/types.c +++ b/src/ti/types.c @@ -60,6 +60,11 @@ int ti_types_add(ti_types_t * types, ti_type_t * type) return 0; } +int ti_types_add_unnamed(ti_types_t * types, ti_type_t * type) +{ + return imap_add(types->imap, types->next_unnamed++, type); +} + /* * Do not use this function directly; Call ti_type_del(..) so * existing things using this type will be converted to objects and @@ -72,6 +77,11 @@ void ti_types_del(ti_types_t * types, ti_type_t * type) (void) smap_pop(types->smap, type->name); } +void ti_types_del_unnamed(ti_types_t * types, ti_type_t * type) +{ + (void) imap_pop(types->imap, type->type_id); +} + typedef struct { uint16_t id; @@ -233,6 +243,8 @@ int ti_types_ren_member_spec(ti_types_t * types, ti_member_t * member) return imap_walk(types->imap, (imap_cb) types__ren_member_cb, member); } + + uint16_t ti_types_get_new_id(ti_types_t * types, ti_raw_t * rname, ex_t * e) { uintptr_t utype; diff --git a/src/ti/val.c b/src/ti/val.c index 832375da..0e7ad685 100644 --- a/src/ti/val.c +++ b/src/ti/val.c @@ -92,6 +92,7 @@ ti_val_t * val__parent_name; ti_val_t * val__parent_type_name; ti_val_t * val__key_name; ti_val_t * val__key_type_name; +ti_val_t * val__unnamed_name; /* string */ ti_val_t * val__sany; @@ -774,6 +775,7 @@ int ti_val_init_common(void) val__parent_type_name = (ti_val_t *) ti_names_from_str("parent_type"); val__key_name = (ti_val_t *) ti_names_from_str("key"); val__key_type_name = (ti_val_t *) ti_names_from_str("key_type"); + val__unnamed_name = (ti_val_t *) ti_names_from_str("__unnamed__"); val__re_email = (ti_val_t *) ti_regex_from_str("/^[a-zA-Z0-9.!#$%&’*+/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\\.[a-zA-Z0-9-]+)*$/"); val__re_url = (ti_val_t *) ti_regex_from_str("/^(https?|ftp):\\/\\/[-a-zA-Z0-9@:%._\\+~#=]{1,256}\\b([-a-zA-Z0-9@:%_\\+.~#?&//=]*)$/"); val__re_tel = (ti_val_t *) ti_regex_from_str("/^[\\+]?(\\([0-9]{1,4}\\)[-\\s\\.]?){0,2}([0-9]{1,4}[-\\s]?){3,5}$/"); @@ -792,7 +794,8 @@ int ti_val_init_common(void) !val__beautify_name || !val__parent_name || !val__parent_type_name || !val__key_name || !val__key_type_name || !val__flags_name || !val__data_name || !val__time_name || !val__re_email || - !val__smodule || !val__re_url || !val__re_tel || !val__async_name) + !val__smodule || !val__re_url || !val__re_tel || !val__async_name || + !val__unnamed_name) { return -1; } diff --git a/src/ti/wrap.c b/src/ti/wrap.c index 8e2be024..ecbe0c01 100644 --- a/src/ti/wrap.c +++ b/src/ti/wrap.c @@ -26,6 +26,14 @@ #include #include +static int wrap__field_thing_type( + ti_thing_t * thing, + ti_vp_t * vp, + ti_type_t * t_type, + int deep, + int flags); + + ti_wrap_t * ti_wrap_create(ti_thing_t * thing, uint16_t type_id) { ti_wrap_t * wrap = malloc(sizeof(ti_wrap_t)); @@ -62,6 +70,20 @@ static int wrap__walk(ti_thing_t * thing, wrap__walk_t * w) return ti__wrap_field_thing(thing, w->vp, w->spec, w->deep, w->flags); } +typedef struct +{ + ti_vp_t * vp; + ti_type_t * t_type; + long:48; + int deep; + int flags; +} wrap__walk_with_type_t; + +static int wrap__walk_with_type(ti_thing_t * thing, wrap__walk_with_type_t * w) +{ + return wrap__field_thing_type(thing, w->vp, w->t_type, w->deep, w->flags); +} + static int wrap__set( ti_vset_t * vset, ti_vp_t * vp, @@ -76,6 +98,29 @@ static int wrap__set( .flags = flags, }; + if (vset->imap->n > 1 && + vp->query && + vp->query->collection) + { + /* optimization for set's with multiple values */ + ti_type_t * t_type = ti_types_by_id(vp->query->collection->types, spec); + if (t_type) + { + wrap__walk_with_type_t wwt = { + .vp = vp, + .t_type = t_type, + .deep = deep, + .flags = flags, + }; + + return ( + msgpack_pack_array(&vp->pk, vset->imap->n) || + imap_walk(vset->imap, (imap_cb) wrap__walk_with_type, &wwt) + ); + } + /* fallback to no type */ + } + return ( msgpack_pack_array(&vp->pk, vset->imap->n) || imap_walk(vset->imap, (imap_cb) wrap__walk, &w) @@ -110,14 +155,28 @@ static int wrap__field_val( case TI_VAL_REGEX: return ti_regex_to_client_pk((ti_regex_t *) val, &vp->pk); case TI_VAL_THING: - return ti__wrap_field_thing( + return (*spec == TI_SPEC_TYPE) + ? wrap__field_thing_type( + (ti_thing_t *) val, + vp, + t_field->condition.type, + deep, + flags) + : ti__wrap_field_thing( (ti_thing_t *) val, vp, *spec, deep, flags); case TI_VAL_WRAP: - return ti__wrap_field_thing( + return (*spec == TI_SPEC_TYPE) + ? wrap__field_thing_type( + (ti_thing_t *) val, + vp, + t_field->condition.type, + deep, + flags) + : ti__wrap_field_thing( ((ti_wrap_t *) val)->thing, vp, *spec, @@ -323,59 +382,14 @@ int ti__wrap_methods_to_pk( return rc; } -/* - * Do not use directly, use ti_wrap_to_pk() instead - */ -int ti__wrap_field_thing( +static int wrap__field_thing( ti_thing_t * thing, ti_vp_t * vp, - uint16_t spec, + ti_type_t * t_type, int deep, int flags) { size_t nm; - ti_type_t * t_type; - spec &= TI_SPEC_MASK_NILLABLE; - - assert(thing->tp == TI_VAL_THING); - - /* - * Just return the ID when locked or if `deep` has reached zero. - */ - if ((thing->flags & TI_VFLAG_LOCK) || !deep) - { - if (!thing->id || (flags & TI_FLAGS_NO_IDS)) - return ti_thing_empty_to_client_pk(&vp->pk); - - if (spec < TI_SPEC_ANY) - { - t_type = ti_types_by_id(thing->collection->types, spec); - if (t_type) - { - register const ti_name_t * name = t_type->idname; - return -( - msgpack_pack_map(&vp->pk, 1) || (name - ? mp_pack_strn(&vp->pk, name->str, name->n) - : mp_pack_strn(&vp->pk, TI_KIND_S_THING, 1)) || - msgpack_pack_uint64(&vp->pk, thing->id) - ); - } - } - - return -( - msgpack_pack_map(&vp->pk, 1) || - mp_pack_strn(&vp->pk, TI_KIND_S_THING, 1) || - msgpack_pack_uint64(&vp->pk, thing->id) - ); - } - - /* - * If `spec` is not a type or a none existing type (thus ANY or OBJECT), - * then pack the thing as normal. - */ - if (spec >= TI_SPEC_ANY || /* TI_SPEC_ANY || TI_SPEC_OBJECT || ENUM */ - !(t_type = ti_types_by_id(thing->collection->types, spec))) - return ti_thing__to_client_pk(thing, vp, deep, flags); /* decrement `deep` by one */ --deep; @@ -563,6 +577,88 @@ int ti__wrap_field_thing( return -1; } + +static int wrap__field_thing_type( + ti_thing_t * thing, + ti_vp_t * vp, + ti_type_t * t_type, + int deep, + int flags) +{ + /* + * Just return the ID when locked or if `deep` has reached zero. + */ + if ((thing->flags & TI_VFLAG_LOCK) || !deep) + { + register const ti_name_t * name = t_type->idname; + + if (!thing->id || (flags & TI_FLAGS_NO_IDS)) + return ti_thing_empty_to_client_pk(&vp->pk); + + return -( + msgpack_pack_map(&vp->pk, 1) || (name + ? mp_pack_strn(&vp->pk, name->str, name->n) + : mp_pack_strn(&vp->pk, TI_KIND_S_THING, 1)) || + msgpack_pack_uint64(&vp->pk, thing->id) + ); + } + + return wrap__field_thing(thing, vp, t_type, deep, flags); +} +/* + * Do not use directly, use ti_wrap_to_pk() instead + */ +int ti__wrap_field_thing( + ti_thing_t * thing, + ti_vp_t * vp, + uint16_t spec, + int deep, + int flags) +{ + ti_type_t * t_type; + spec &= TI_SPEC_MASK_NILLABLE; + + /* + * Just return the ID when locked or if `deep` has reached zero. + */ + if ((thing->flags & TI_VFLAG_LOCK) || !deep) + { + if (!thing->id || (flags & TI_FLAGS_NO_IDS)) + return ti_thing_empty_to_client_pk(&vp->pk); + + if (spec < TI_SPEC_ANY) + { + t_type = ti_types_by_id(thing->collection->types, spec); + if (t_type) + { + register const ti_name_t * name = t_type->idname; + return -( + msgpack_pack_map(&vp->pk, 1) || (name + ? mp_pack_strn(&vp->pk, name->str, name->n) + : mp_pack_strn(&vp->pk, TI_KIND_S_THING, 1)) || + msgpack_pack_uint64(&vp->pk, thing->id) + ); + } + } + + return -( + msgpack_pack_map(&vp->pk, 1) || + mp_pack_strn(&vp->pk, TI_KIND_S_THING, 1) || + msgpack_pack_uint64(&vp->pk, thing->id) + ); + } + + /* + * If `spec` is not a type or a none existing type (thus ANY or OBJECT), + * then pack the thing as normal. + */ + if (spec >= TI_SPEC_ANY || /* TI_SPEC_ANY || TI_SPEC_OBJECT || ENUM */ + !(t_type = ti_types_by_id(thing->collection->types, spec))) + return ti_thing__to_client_pk(thing, vp, deep, flags); + + return wrap__field_thing(thing, vp, t_type, deep, flags); +} + int ti_wrap_cp(ti_query_t * query, uint8_t deep, ex_t * e) { ti_val_t * val; @@ -570,6 +666,7 @@ int ti_wrap_cp(ti_query_t * query, uint8_t deep, ex_t * e) msgpack_sbuffer buffer; ti_vp_t vp = { .query=query, + .size_limit=ti.cfg->result_size_limit, }; ti_vup_t vup = { .isclient = true, From 313346d384f1207dbfb243a591fdce8cf077151f Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Wed, 29 Oct 2025 15:09:55 +0100 Subject: [PATCH 04/13] with add to types --- inc/ti/condition.h | 1 + inc/ti/condition.t.h | 27 ++++--- inc/ti/fn/fnmodtype.h | 19 +++-- inc/ti/type.h | 55 ++----------- inc/ti/types.h | 7 +- inc/ti/types.inline.h | 10 +++ inc/ti/types.t.h | 2 +- inc/ti/vp.t.h | 5 +- itest/lib/node.py | 1 + itest/test_wrap.py | 3 - src/ti/condition.c | 70 +++++++++++++--- src/ti/enum.c | 1 + src/ti/field.c | 14 ++-- src/ti/query.c | 2 - src/ti/store/storetypes.c | 30 +++++-- src/ti/task.c | 8 +- src/ti/type.c | 165 ++++++++++++++++++++++++++++++-------- src/ti/types.c | 28 +++---- src/ti/wrap.c | 26 ++++-- 19 files changed, 307 insertions(+), 167 deletions(-) diff --git a/inc/ti/condition.h b/inc/ti/condition.h index 4ce527a0..3acd5610 100644 --- a/inc/ti/condition.h +++ b/inc/ti/condition.h @@ -26,6 +26,7 @@ int ti_condition_field_rel_init( ti_field_t * ofield, ex_t * e); int ti_condition_init_enum(ti_field_t * field, ti_member_t * member, ex_t * e); +int ti_condition_init_type(ti_field_t * field, ex_t * e); void ti_condition_destroy(ti_condition_via_t condition, uint16_t spec); diff --git a/inc/ti/condition.t.h b/inc/ti/condition.t.h index db69688f..623e20e0 100644 --- a/inc/ti/condition.t.h +++ b/inc/ti/condition.t.h @@ -11,10 +11,20 @@ typedef struct ti_condition_srange_s ti_condition_srange_t; typedef struct ti_condition_irange_s ti_condition_irange_t; typedef struct ti_condition_drange_s ti_condition_drange_t; typedef struct ti_condition_rel_s ti_condition_rel_t; +typedef struct ti_condition_type_s ti_condition_type_t; + +#include +#include +#include +#include +#include +#include +#include +#include union ti_condition_via_u { - ti_type_t * type; /* nested type (wrap-only) */ + ti_condition_type_t * type; /* nested type (wrap-only) */ ti_condition_re_t * re; /* str, utf8 */ ti_condition_srange_t * srange; /* str, utf8 */ ti_condition_irange_t * irange; /* int, float */ @@ -23,15 +33,6 @@ union ti_condition_via_u ti_condition_t * none; /* NULL */ }; -#include -#include -#include -#include -#include -#include -#include -#include - typedef void (*ti_condition_rel_cb) (ti_field_t *, ti_thing_t *, ti_thing_t *); struct ti_condition_s @@ -73,4 +74,10 @@ struct ti_condition_rel_s ti_condition_rel_cb add_cb; }; +struct ti_condition_type_s +{ + uint32_t type_id; + ti_type_t * type; +}; + #endif /* TI_CONDITION_T_H_ */ diff --git a/inc/ti/fn/fnmodtype.h b/inc/ti/fn/fnmodtype.h index a9de9f12..c5af1ca8 100644 --- a/inc/ti/fn/fnmodtype.h +++ b/inc/ti/fn/fnmodtype.h @@ -410,9 +410,9 @@ static void type__add( return; } - spec_raw = ti_type_nested_from_val(type, query->rval, e) - if (e-nr) - return e->nr; + spec_raw = ti_type_nested_from_val(type, query->rval, e); + if (e->nr) + return; if (spec_raw) ti_val_unsafe_gc_drop(query->rval); @@ -867,7 +867,7 @@ static void type__mod( const int nargs = fn_get_nargs(nd); ti_field_t * field = ti_field_by_name(type, name); ti_method_t * method = field ? NULL : ti_type_get_method(type, name); - + ti_raw_t * spec_raw = NULL; cleri_node_t * child; if (fn_nargs_range(fnname, DOC_MOD_TYPE_MOD, 4, 5, nargs, e)) @@ -945,10 +945,12 @@ static void type__mod( return; } + spec_raw = ti_type_nested_from_val(type, query->rval, e); + if (e->nr) + return; + if (ti_val_is_str(query->rval)) { - ti_raw_t * spec_raw; - if (!field) { ex_set(e, EX_TYPE_ERROR, @@ -966,7 +968,10 @@ static void type__mod( } query->rval = NULL; + } + if (spec_raw) + { if (nargs == 4) { ti_task_t * task; @@ -1014,8 +1019,6 @@ static void type__mod( return; } - // TODO: WO-NESTED: handle arr and thing - ex_set(e, EX_TYPE_ERROR, "function `%s` expects argument 4 to be of " "type `"TI_VAL_STR_S"` or type `"TI_VAL_CLOSURE_S"` " diff --git a/inc/ti/type.h b/inc/ti/type.h index d8ff880a..b3c49e09 100644 --- a/inc/ti/type.h +++ b/inc/ti/type.h @@ -27,9 +27,12 @@ ti_type_t * ti_type_create( size_t name_n, uint64_t created_at, uint64_t modified_at); -ti_type_t * ti_type_create_unnamed(ti_types_t * types); +ti_type_t * ti_type_create_unnamed( + ti_types_t * types, + uint32_t * type_id, + uint8_t flags); void ti_type_drop(ti_type_t * type); -void ti_type_drop_unnamed(ti_type_t * type); +void ti_type_drop_unnamed(ti_type_t * type, uint32_t type_id); void ti_type_del(ti_type_t * type, vec_t * vars); void ti_type_destroy(ti_type_t * type); void ti_type_map_cleanup(ti_type_t * type); @@ -74,16 +77,6 @@ int ti_type_requires_wpo(ti_type_t * type, ex_t * e); int ti_type_rename(ti_type_t * type, ti_raw_t * nname); ti_raw_t * ti__type_nested_from_val(ti_type_t * type, ti_val_t * val, ex_t * e); -static inline ti_raw_t * ti_type_nested_from_val( - ti_type_t * type, - ti_val_t * val, - ex_t * e) -{ - return (ti_val_is_thing(val) || ti_val_is_array(val)) - ? ti__type_nested_from_val(type, val, e) - : NULL; -} - static inline int ti_type_use(ti_type_t * type, ex_t * e) { if (type->flags & TI_TYPE_FLAG_LOCK) @@ -134,44 +127,6 @@ static inline _Bool ti_type_hide_id(ti_type_t * type) return type->flags & TI_TYPE_FLAG_HIDE_ID; } -static inline int ti_type_to_pk( - ti_type_t * type, - msgpack_packer * pk, - _Bool with_definition) -{ - return ( - msgpack_pack_map(pk, 9) || - mp_pack_str(pk, "type_id") || - msgpack_pack_uint16(pk, type->type_id) || - - mp_pack_str(pk, "name") || - mp_pack_strn(pk, type->rname->data, type->rname->n) || - - mp_pack_str(pk, "wrap_only") || - mp_pack_bool(pk, ti_type_is_wrap_only(type)) || - - mp_pack_str(pk, "hide_id") || - mp_pack_bool(pk, ti_type_hide_id(type)) || - - mp_pack_str(pk, "created_at") || - msgpack_pack_uint64(pk, type->created_at) || - - mp_pack_str(pk, "modified_at") || - (type->modified_at - ? msgpack_pack_uint64(pk, type->modified_at) - : msgpack_pack_nil(pk)) || - - mp_pack_str(pk, "fields") || - ti_type_fields_to_pk(type, pk) || - - mp_pack_str(pk, "methods") || - ti_type_methods_info_to_pk(type, pk, with_definition) || - - mp_pack_str(pk, "relations") || - ti_type_relations_to_pk(type, pk) - ); -} - static inline void ti_type_set_wrap_only_mode(ti_type_t * type, _Bool wpo) { if (wpo) diff --git a/inc/ti/types.h b/inc/ti/types.h index 743c74af..cc1ce61f 100644 --- a/inc/ti/types.h +++ b/inc/ti/types.h @@ -18,13 +18,16 @@ ti_types_t * ti_types_create(ti_collection_t * collection); void ti_types_destroy(ti_types_t * types); int ti_types_add(ti_types_t * types, ti_type_t * type); void ti_types_del(ti_types_t * types, ti_type_t * type); -void ti_types_del_unnamed(ti_types_t * types, ti_type_t * type); +void ti_types_del_unnamed(ti_types_t * types, uint32_t type_id); int ti_types_ren_spec( ti_types_t * types, uint16_t type_or_enum_id, ti_raw_t * nname); int ti_types_ren_member_spec(ti_types_t * types, ti_member_t * member); -int ti_types_add_unnamed(ti_types_t * types, ti_type_t * type); +int ti_types_add_unnamed( + ti_types_t * types, + ti_type_t * type, + uint32_t * type_id) uint16_t ti_types_get_new_id(ti_types_t * types, ti_raw_t * rname, ex_t * e); ti_varr_t * ti_types_info(ti_types_t * types, _Bool with_definition); int ti_types_to_pk(ti_types_t * types, msgpack_packer * pk); diff --git a/inc/ti/types.inline.h b/inc/ti/types.inline.h index e1abf9b8..1de9965a 100644 --- a/inc/ti/types.inline.h +++ b/inc/ti/types.inline.h @@ -8,7 +8,17 @@ #include #include #include +#include +static inline ti_raw_t * ti_type_nested_from_val( + ti_type_t * type, + ti_val_t * val, + ex_t * e) +{ + return (ti_val_is_thing(val) || ti_val_is_array(val)) + ? ti__type_nested_from_val(type, val, e) + : NULL; +} static inline ti_type_t * ti_types_by_strn(ti_types_t * types, const char * str, size_t n) { diff --git a/inc/ti/types.t.h b/inc/ti/types.t.h index ce3aeeed..9182db02 100644 --- a/inc/ti/types.t.h +++ b/inc/ti/types.t.h @@ -20,8 +20,8 @@ struct ti_types_s smap_t * smap; smap_t * removed; /* map with type id's which are removed */ ti_collection_t * collection; - uint32_t next_unnamed; uint16_t next_id; + uint16_t locked; /* 0 or 1 */ }; #endif /* TI_TYPES_T_H_ */ diff --git a/inc/ti/vp.t.h b/inc/ti/vp.t.h index c5bc15f6..a0ade6ab 100644 --- a/inc/ti/vp.t.h +++ b/inc/ti/vp.t.h @@ -12,10 +12,7 @@ typedef struct ti_vp_s ti_vp_t; struct ti_vp_s { msgpack_packer pk; - ti_query_t * query; /* May be undefined; - * Query must be set when the value - * packer is used with `options` > 0. - * Note that setting to NULL is fine + ti_query_t * query; /* Note that setting to NULL is fine * but in this case `methods` will not * be calculated. */ diff --git a/itest/lib/node.py b/itest/lib/node.py index 857050fa..d5e84b24 100644 --- a/itest/lib/node.py +++ b/itest/lib/node.py @@ -143,6 +143,7 @@ async def join_until_ready(self, client, secret='my_secret', timeout=60): def _handle_output(node, r): """Runs in another thread.""" r = os.fdopen(r, 'r') + print(r) for line in r: node.queue.put_nowait(line) if THINGSDB_NODE_OUTPUT is True or ( diff --git a/itest/test_wrap.py b/itest/test_wrap.py index 79ee521e..39ddf9d0 100755 --- a/itest/test_wrap.py +++ b/itest/test_wrap.py @@ -434,9 +434,6 @@ async def test_wrap_copy(self, client): """) self.assertEqual(res, "IRIS") - async def test_wrap_nested(self, client): - pass - # TODO: WO-NESTED: test if can call self in nested declaration if __name__ == '__main__': run_test(TestWrap()) diff --git a/src/ti/condition.c b/src/ti/condition.c index c816edec..10014012 100644 --- a/src/ti/condition.c +++ b/src/ti/condition.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -738,8 +739,11 @@ int ti_condition_field_rel_init( return e->nr; } -int ti_condition_nested_type(ti_field_t * field, ex_t * e) +int ti_condition_init_type(ti_field_t * field, ex_t * e) { + uint8_t flags = TI_TYPE_FLAG_WRAP_ONLY | TI_TYPE_FLAG_HIDE_ID; + uint32_t type_id; + ti_type_t * type; ti_val_t * val; ti_thing_t * thing; mp_unp_t up; @@ -758,19 +762,19 @@ int ti_condition_nested_type(ti_field_t * field, ex_t * e) { vec_t * vec = VARR(val); field->nested_spec = TI_SPEC_TYPE; - field->spec = TI_SPEC_ARR_TYPE; + field->spec = TI_SPEC_ARR_TYPE | TI_SPEC_NILLABLE; thing = vec_first(vec); - if (!thing || !ti_val_is_thing((ti_val_t *) thing)) + if (!thing || !ti_val_is_object((ti_val_t *) thing)) { ex_set_internal(e); /* only with corrupt internal data */ goto fail0; } } - else if (ti_val_is_thing(val)) + else if (ti_val_is_object(val)) { /* val is thing */ - field->spec = TI_SPEC_TYPE; - thing = val; + field->spec = TI_SPEC_TYPE | TI_SPEC_NILLABLE; + thing = (ti_thing_t *) val; } else { @@ -778,7 +782,37 @@ int ti_condition_nested_type(ti_field_t * field, ex_t * e) goto fail0; } - type = ti_type_create_unnamed(field->type->types); + field->flags = TI_FIELD_FLAG_SAME_DEEP | TI_FIELD_FLAG_SKIP_NIL; + + for (vec_each(thing->items.vec, ti_prop_t, prop)) + { + ti_raw_t * spec_raw = (ti_raw_t *) prop->val; + if (ti_val_is_raw(prop->val) && spec_raw->n) + { + if (spec_raw->data[0] == '#') + flags &= ~TI_TYPE_FLAG_HIDE_ID; + + if (spec_raw->data[0] == '|') + { + ti_qbind_t syntax = { + .immutable_n = 0, + .flags = TI_QBIND_FLAG_COLLECTION, + }; + ti_closure_t * closure = ti_closure_from_strn( + &syntax, + (const char *) spec_raw->data, + spec_raw->n, + e); + if (!closure) + goto fail0; + + ti_val_unsafe_drop(prop->val); + prop->val = (ti_val_t *) closure; + } + } + } + + type = ti_type_create_unnamed(field->type->types, &type_id, flags); if (!type) { ex_set_mem(e); @@ -786,11 +820,24 @@ int ti_condition_nested_type(ti_field_t * field, ex_t * e) } if (ti_type_init_from_thing(type, thing, e)) - ti_type_drop_unnamed(type); - else - field->condition.type = type; + goto fail1; + field->condition.type = malloc(sizeof(ti_condition_type_t)); + if (!field->condition.type) + { + ex_set_mem(e); + goto fail1; + } + + field->condition.type->type = type; + field->condition.type->type_id = type_id; + + goto done; + +fail1: + ti_type_drop_unnamed(type, type_id); fail0: +done: ti_val_unsafe_drop(val); return e->nr; } @@ -806,7 +853,8 @@ void ti_condition_destroy(ti_condition_via_t condition, uint16_t spec) { case TI_SPEC_TYPE: case TI_SPEC_ARR_TYPE: - ti_type_drop_unnamed(condition.type); + ti_type_drop_unnamed(condition.type->type, condition.type->type_id); + free(condition.type); return; case TI_SPEC_ANY: return; /* a field may be set to ANY while using mod_type in which diff --git a/src/ti/enum.c b/src/ti/enum.c index 82fc24c3..548ebf06 100644 --- a/src/ti/enum.c +++ b/src/ti/enum.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include diff --git a/src/ti/field.c b/src/ti/field.c index ba1d36a1..89864f84 100644 --- a/src/ti/field.c +++ b/src/ti/field.c @@ -457,16 +457,15 @@ static int field__init(ti_field_t * field, ex_t * e) } if (ti_raw_is_mpdata(field->spec_raw)) - { - LOGC("WO-NESTED"); - return e-nr; - } - // TODO: WO-NESTED: Test and process nested fields + return ti_condition_init_type(field, e); do { /* WARNING: do not forget to update types.c when modifying the flags in * this code; (types__spec_flags_pos) + * WARNING: never implement | as a flag or start of definition, this + * is used to read closures (methods) when storing nested types for + * wrap-only types. (see as example condition.c) */ switch(*str) { @@ -505,6 +504,7 @@ static int field__init(ti_field_t * field, ex_t * e) goto duplicate_flag; field->flags |= TI_FIELD_FLAG_SKIP_FALSE; break; + /* Read warnings above before adding new flags */ default: goto done_flags; } @@ -1697,7 +1697,7 @@ static _Bool field__maps_arr_to_arr(ti_field_t * field, ti_varr_t * varr) return true; } -static _Bool field__maps_arr_to_type(ti_field_t * field, ti_varr_t * varr) +static _Bool field__maps_arr_to_type(ti_varr_t * varr) { for (vec_each(varr->vec, ti_val_t, val)) if (!ti_val_is_thing(val)) @@ -2291,7 +2291,7 @@ _Bool ti_field_maps_to_val(ti_field_t * field, ti_val_t * val) case TI_SPEC_ARR_TYPE: return (( ti_val_is_array(val) && - field__maps_arr_to_type(field, (ti_varr_t *) val) + field__maps_arr_to_type((ti_varr_t *) val) ) || ti_val_is_set(val)); } diff --git a/src/ti/query.c b/src/ti/query.c index e9ec8d3d..62233935 100644 --- a/src/ti/query.c +++ b/src/ti/query.c @@ -1139,8 +1139,6 @@ static inline int query__pack_response( if (buffer->size > ti.counters->largest_result_size) ti.counters->largest_result_size = buffer->size; - ti_mp_create - return 0; } diff --git a/src/ti/store/storetypes.c b/src/ti/store/storetypes.c index adc6e902..dd9b8db2 100644 --- a/src/ti/store/storetypes.c +++ b/src/ti/store/storetypes.c @@ -30,6 +30,9 @@ static int rmtype_cb( static int mktype_cb(ti_type_t * type, msgpack_packer * pk) { uintptr_t p; + if (type->type_id == TI_SPEC_TYPE) + return 0; + if (msgpack_pack_array(pk, 8) || msgpack_pack_uint16(pk, type->type_id) || mp_pack_bool(pk, type->flags & TI_TYPE_FLAG_WRAP_ONLY) || @@ -72,8 +75,20 @@ static int mktype_cb(ti_type_t * type, msgpack_packer * pk) return 0; } -static int count_relations_cb(ti_type_t * type, size_t * n) +typedef struct { + size_t n; /* number of relations */ + size_t m; /* number of unnamed */ +} count_walk_t; + +static int count_cb(ti_type_t * type, count_walk_t * w) { + if (ti_type_is_wrap_only(type)) + { + if (type->type_id == TI_SPEC_TYPE) + (w->m)++; + return 0; + } + for (vec_each(type->fields, ti_field_t, field)) { if (ti_field_has_relation(field)) @@ -83,7 +98,7 @@ static int count_relations_cb(ti_type_t * type, size_t * n) field->type->type_id < other->type->type_id || ( field->type->type_id == other->type->type_id && field->idx < other->idx)) - (*n)++; + (w->n)++; } } return 0; @@ -118,7 +133,10 @@ int ti_store_types_store(ti_types_t * types, const char * fn) msgpack_packer pk; char namebuf[TI_NAME_MAX]; FILE * f = fopen(fn, "w"); - size_t n = 0; + count_walk_t w = { + .n = 0, + .m = 0, + }; if (!f) { @@ -127,7 +145,7 @@ int ti_store_types_store(ti_types_t * types, const char * fn) } /* count the number of relations */ - (void) imap_walk(types->imap, (imap_cb) count_relations_cb, &n); + (void) imap_walk(types->imap, (imap_cb) count_cb, &w); msgpack_packer_init(&pk, f, msgpack_fbuffer_write); @@ -138,11 +156,11 @@ int ti_store_types_store(ti_types_t * types, const char * fn) smap_items(types->removed, namebuf, (smap_item_cb) rmtype_cb, &pk) || /* active types */ mp_pack_str(&pk, "types") || - msgpack_pack_array(&pk, types->imap->n) || + msgpack_pack_array(&pk, types->imap->n - w.m) || imap_walk(types->imap, (imap_cb) mktype_cb, &pk) || /* relations */ mp_pack_str(&pk, "relations") || - msgpack_pack_array(&pk, n) || + msgpack_pack_array(&pk, w.n) || imap_walk(types->imap, (imap_cb) rltype_cb, &pk) ) goto fail; diff --git a/src/ti/task.c b/src/ti/task.c index 60c76ae8..a4d979f5 100644 --- a/src/ti/task.c +++ b/src/ti/task.c @@ -1644,9 +1644,9 @@ int ti_task_add_mod_type_add_field( mp_pack_str(&pk, "spec"); if (ti_raw_is_mpdata(field->spec_raw)) - mp_pack_bin(&pk, field->spec_raw->data, field->spec_raw->n) + mp_pack_bin(&pk, field->spec_raw->data, field->spec_raw->n); else - mp_pack_strn(&pk, field->spec_raw->data, field->spec_raw->n) + mp_pack_strn(&pk, field->spec_raw->data, field->spec_raw->n); if (dval) { @@ -1924,9 +1924,9 @@ int ti_task_add_mod_type_mod_field(ti_task_t * task, ti_field_t * field) mp_pack_str(&pk, "spec"); if (ti_raw_is_mpdata(field->spec_raw)) - mp_pack_bin(&pk, field->spec_raw->data, field->spec_raw->n) + mp_pack_bin(&pk, field->spec_raw->data, field->spec_raw->n); else - mp_pack_strn(&pk, field->spec_raw->data, field->spec_raw->n) + mp_pack_strn(&pk, field->spec_raw->data, field->spec_raw->n); data = (ti_data_t *) buffer.data; ti_data_init(data, buffer.size); diff --git a/src/ti/type.c b/src/ti/type.c index a7ebd58d..d442065a 100644 --- a/src/ti/type.c +++ b/src/ti/type.c @@ -76,21 +76,23 @@ ti_type_t * ti_type_create( return type; } -ti_type_t * ti_type_create_unnamed(ti_types_t * types) +ti_type_t * ti_type_create_unnamed( + ti_types_t * types, + uint32_t * type_id, + uint8_t flags) { ti_type_t * type = malloc(sizeof(ti_type_t)); if (!type) return NULL; - type->refcount = 0; /* only incremented when this type is used by another type */ type->selfref = 0; /* increment on self references */ type->type_id = TI_SPEC_TYPE; - type->flags = TI_TYPE_FLAG_WRAP_ONLY; - type->rname = ti_val_unnamed_name(); - type->name = strndup(type->rname->data, type->rname->n); - type->rwname = ti_val_unnamed_name(); - type->wname = strndup(type->rwname->data, type->rwname->n); + type->flags = TI_TYPE_FLAG_WRAP_ONLY|flags; + type->rname = (ti_raw_t *) ti_val_unnamed_name(); + type->name = strndup((const char *) type->rname->data, type->rname->n); + type->rwname = (ti_raw_t *) ti_val_unnamed_name(); + type->wname = strndup((const char *) type->rwname->data, type->rwname->n); type->idname = NULL; type->dependencies = vec_new(0); type->fields = vec_new(0); @@ -102,7 +104,7 @@ ti_type_t * ti_type_create_unnamed(ti_types_t * types) if (!type->name || !type->wname || !type->dependencies || !type->fields || !type->rname || !type->rwname || !type->t_mappings || !type->methods || - ti_types_add(types, type)) + ti_types_add_unnamed(types, type, type_id)) { ti_type_destroy(type); return NULL; @@ -144,15 +146,15 @@ void ti_type_drop(ti_type_t * type) ti_type_destroy(type); } -void ti_type_drop_unnamed(ti_type_t * type) +void ti_type_drop_unnamed(ti_type_t * type, uint32_t type_id) { - if (!type) + if (!type || type->types->locked) return; for (vec_each(type->dependencies, ti_type_t, dep)) --dep->refcount; - ti_types_del_unnamed(type->types, type); + ti_types_del_unnamed(type->types, type_id); ti_type_destroy(type); } @@ -336,13 +338,22 @@ int ti_type_set_idname( return e->nr; } +static int type__init_type_cb(ti_item_t * item, ex_t * e) +{ + if (!ti_raw_is_name(item->key)) + ex_set(e, EX_VALUE_ERROR, + "type keys must follow the naming rules"DOC_NAMES); + return e->nr; +} + ti_raw_t * ti__type_nested_from_val(ti_type_t * type, ti_val_t * val, ex_t * e) { + ti_thing_t * thing; ti_raw_t * spec_raw = NULL; msgpack_sbuffer buffer; ti_vp_t vp = { - .query=query, - .size_limit=0x1000, /* we do not expect much but since we use deep + .query=NULL, + .size_limit=0x4000, /* we do not expect much and since we use deep nesting, set a low size limit */ }; @@ -354,33 +365,43 @@ ti_raw_t * ti__type_nested_from_val(ti_type_t * type, ti_val_t * val, ex_t * e) ex_set(e, EX_VALUE_ERROR, "array with nested type must have exactly one element but " "found %zu on type `%s`", - varr->n + varr->vec->n, type->name); - return e->nr; + return NULL; } + if (!ti_val_is_thing(VEC_first(varr->vec))) { - ex_set(e, EX_VALUE_ERROR, + ex_set(e, EX_TYPE_ERROR, "element in array must be of type `"TI_VAL_THING_S"` " - "but got type `%s` instead", - ti_val_str(VEC_first(varr->vec))); - return e->nr; + "but got type `%s` instead; (happened on type `%s`)", + ti_val_str(VEC_first(varr->vec)), + type->name); + return NULL; } + + thing = VEC_first(varr->vec); } + else + thing = (ti_thing_t *) val; if (~type->flags & TI_TYPE_FLAG_WRAP_ONLY) { ex_set(e, EX_VALUE_ERROR, - "defining nested types is only allowed ' - 'for wrap-only type; define type `%s` as wrap-only", + "defining nested types is only allowed " + "for wrap-only type; (happened on type `%s`)", type->name); - return e->nr; + return NULL; } + if (ti_thing_is_dict(thing) && + smap_values(thing->items.smap, (smap_val_cb) type__init_type_cb, e)) + return NULL; + if (mp_sbuffer_alloc_init(&buffer, vp.size_limit, 0)) { ex_set_mem(e); - return e->nr; + return NULL; } msgpack_packer_init(&vp.pk, &buffer, msgpack_sbuffer_write); @@ -396,7 +417,7 @@ ti_raw_t * ti__type_nested_from_val(ti_type_t * type, ti_val_t * val, ex_t * e) goto fail0; } - spec_raw = ti_mp_create(buffer.data, buffer.size); + spec_raw = ti_mp_create((const unsigned char *) buffer.data, buffer.size); if (!spec_raw) ex_set_mem(e); @@ -411,7 +432,7 @@ static inline int type__assign( ti_val_t * val, ex_t * e) { - ti_raw_t * spec_raw = ti_type_nested_from_val(type, val, e) + ti_raw_t * spec_raw = ti_type_nested_from_val(type, val, e); if (spec_raw) { (void) ti_field_create(name, spec_raw, type, e); @@ -419,7 +440,7 @@ static inline int type__assign( return e->nr; } - if (e-nr) + if (e->nr) return e->nr; if (ti_val_is_str(val)) @@ -447,11 +468,20 @@ static inline int type__assign( if (ti_val_is_closure(val)) return ti_type_add_method(type, name, (ti_closure_t *) val, e); - ex_set(e, EX_TYPE_ERROR, - "expecting a method of type `"TI_VAL_CLOSURE_S"` " - "or a definition of type `"TI_VAL_STR_S"` " - "but got type `%s` instead"DOC_T_TYPED, - ti_val_str(val)); + if (ti_type_is_wrap_only(type)) + ex_set(e, EX_TYPE_ERROR, + "expecting a nested structure " + "(`"TI_VAL_LIST_S"` or `"TI_VAL_THING_S"`), " + "a method of type `"TI_VAL_CLOSURE_S"` " + "or a definition of type `"TI_VAL_STR_S"` " + "but got type `%s` instead"DOC_T_TYPED, + ti_val_str(val)); + else + ex_set(e, EX_TYPE_ERROR, + "expecting a method of type `"TI_VAL_CLOSURE_S"` " + "or a definition of type `"TI_VAL_STR_S"` " + "but got type `%s` instead"DOC_T_TYPED, + ti_val_str(val)); return e->nr; } @@ -835,7 +865,7 @@ int ti_type_init_from_unp( { if (mp_next(up, &obj) != MP_ARR || obj.via.sz != 2 || mp_next(up, &mp_name) != MP_STR || - (mp_next(up, &mp_spec) != MP_STR && mp_spec.tp != MP_BIN) + (mp_next(up, &mp_spec) != MP_STR && mp_spec.tp != MP_BIN)) { ex_set(e, EX_BAD_DATA, "failed unpacking fields for type `%s`; " @@ -979,6 +1009,35 @@ int ti_type_fields_to_pk(ti_type_t * type, msgpack_packer * pk) return 0; } +static int type__fields_to_pk(ti_type_t * type, msgpack_packer * pk) +{ + if (msgpack_pack_array(pk, type->fields->n + !!type->idname)) + return -1; + + if (type->idname) + { + if (msgpack_pack_array(pk, 2) || + mp_pack_strn(pk, type->idname->str, type->idname->n) || + mp_pack_strn(pk, "#", 1)) + return -1; + } + + for (vec_each(type->fields, ti_field_t, field)) + { + if (msgpack_pack_array(pk, 2) || + mp_pack_strn(pk, field->name->str, field->name->n) || + (ti_raw_is_mpdata(field->spec_raw) + ? mp_pack_append(pk, field->spec_raw->data, field->spec_raw->n) + : mp_pack_strn(pk, field->spec_raw->data, field->spec_raw->n) + )) + return -1; + } + + return 0; +} + + + /* adds a map with key/value pairs */ int ti_type_methods_to_pk(ti_type_t * type, msgpack_packer * pk) { @@ -1024,7 +1083,6 @@ int ti_type_relations_to_pk(ti_type_t * type, msgpack_packer * pk) mp_pack_strn(pk, ofield->name->str, ofield->name->n) || mp_pack_str(pk, "definition") || - // TODO: WO-NESTED: output nested mp_pack_strn(pk, ofield->spec_raw->data, ofield->spec_raw->n)) return -1; } @@ -1082,7 +1140,35 @@ ti_val_t * ti_type_as_mpval(ti_type_t * type, _Bool with_definition) mp_sbuffer_alloc_init(&buffer, sizeof(ti_raw_t), sizeof(ti_raw_t)); msgpack_packer_init(&pk, &buffer, msgpack_sbuffer_write); - if (ti_type_to_pk(type, &pk, with_definition)) + if (msgpack_pack_map(&pk, 9) || + mp_pack_str(&pk, "type_id") || + msgpack_pack_uint16(&pk, type->type_id) || + + mp_pack_str(&pk, "name") || + mp_pack_strn(&pk, type->rname->data, type->rname->n) || + + mp_pack_str(&pk, "wrap_only") || + mp_pack_bool(&pk, ti_type_is_wrap_only(type)) || + + mp_pack_str(&pk, "hide_id") || + mp_pack_bool(&pk, ti_type_hide_id(type)) || + + mp_pack_str(&pk, "created_at") || + msgpack_pack_uint64(&pk, type->created_at) || + + mp_pack_str(&pk, "modified_at") || + (type->modified_at + ? msgpack_pack_uint64(&pk, type->modified_at) + : msgpack_pack_nil(&pk)) || + + mp_pack_str(&pk, "fields") || + type__fields_to_pk(type, &pk) || + + mp_pack_str(&pk, "methods") || + ti_type_methods_info_to_pk(type, &pk, with_definition) || + + mp_pack_str(&pk, "relations") || + ti_type_relations_to_pk(type, &pk)) { msgpack_sbuffer_destroy(&buffer); return NULL; @@ -1436,6 +1522,16 @@ int ti_type_requires_wpo(ti_type_t * type, ex_t * e) { for (vec_each(type->fields, ti_field_t, field)) { + if ((field->spec & TI_SPEC_MASK_NILLABLE) == TI_SPEC_TYPE || + (field->spec & TI_SPEC_MASK_NILLABLE) == TI_SPEC_ARR_TYPE) + { + ex_set(e, EX_OPERATION, + "type `%s` contains a nested structure which requires " + "`wrap-only` mode to be enabled", + type->name); + return e->nr; + } + if (field->spec < TI_SPEC_ANY) { ti_type_t * dep = ti_types_by_id(type->types, field->spec); @@ -1449,7 +1545,6 @@ int ti_type_requires_wpo(ti_type_t * type, ex_t * e) return e->nr; } } - // TODO: WO-NESTED: check if field is nested type } return 0; } diff --git a/src/ti/types.c b/src/ti/types.c index ab77cd66..e00840b6 100644 --- a/src/ti/types.c +++ b/src/ti/types.c @@ -22,6 +22,7 @@ ti_types_t * ti_types_create(ti_collection_t * collection) types->collection = collection; types->next_id = 0; types->next_unnamed = 0x10000; // outside spec range + types->locked = 0; if (!types->imap || !types->smap || !types->removed) { @@ -36,7 +37,7 @@ void ti_types_destroy(ti_types_t * types) { if (!types) return; - + types->locked = 1; smap_destroy(types->smap, NULL); smap_destroy(types->removed, NULL); imap_destroy(types->imap, (imap_destroy_cb) ti_type_destroy); @@ -60,9 +61,14 @@ int ti_types_add(ti_types_t * types, ti_type_t * type) return 0; } -int ti_types_add_unnamed(ti_types_t * types, ti_type_t * type) +int ti_types_add_unnamed( + ti_types_t * types, + ti_type_t * type, + uint32_t * type_id) { - return imap_add(types->imap, types->next_unnamed++, type); + *type_id = types->next_unnamed++; + printf("Type: %p\n", type); + return 0; // imap_add(types->imap, *type_id, type); } /* @@ -77,9 +83,9 @@ void ti_types_del(ti_types_t * types, ti_type_t * type) (void) smap_pop(types->smap, type->name); } -void ti_types_del_unnamed(ti_types_t * types, ti_type_t * type) +void ti_types_del_unnamed(ti_types_t * types, uint32_t type_id) { - (void) imap_pop(types->imap, type->type_id); + (void) imap_pop(types->imap, type_id); } typedef struct @@ -306,15 +312,3 @@ ti_varr_t * ti_types_info(ti_types_t * types, _Bool with_definition) return varr; } -static inline int types__to_pk_cb(ti_type_t * type, msgpack_packer * pk) -{ - return ti_type_to_pk(type, pk, false); -} - -int ti_types_to_pk(ti_types_t * types, msgpack_packer * pk) -{ - return ( - msgpack_pack_array(pk, types->imap->n) || - imap_walk(types->imap, (imap_cb) types__to_pk_cb, pk) - ); -} diff --git a/src/ti/wrap.c b/src/ti/wrap.c index ecbe0c01..4cd2304b 100644 --- a/src/ti/wrap.c +++ b/src/ti/wrap.c @@ -155,11 +155,11 @@ static int wrap__field_val( case TI_VAL_REGEX: return ti_regex_to_client_pk((ti_regex_t *) val, &vp->pk); case TI_VAL_THING: - return (*spec == TI_SPEC_TYPE) + return ((*spec & TI_SPEC_MASK_NILLABLE) == TI_SPEC_TYPE) ? wrap__field_thing_type( (ti_thing_t *) val, vp, - t_field->condition.type, + t_field->condition.type->type, deep, flags) : ti__wrap_field_thing( @@ -169,11 +169,11 @@ static int wrap__field_val( deep, flags); case TI_VAL_WRAP: - return (*spec == TI_SPEC_TYPE) + return ((*spec & TI_SPEC_MASK_NILLABLE) == TI_SPEC_TYPE) ? wrap__field_thing_type( (ti_thing_t *) val, vp, - t_field->condition.type, + t_field->condition.type->type, deep, flags) : ti__wrap_field_thing( @@ -592,7 +592,11 @@ static int wrap__field_thing_type( { register const ti_name_t * name = t_type->idname; - if (!thing->id || (flags & TI_FLAGS_NO_IDS)) + /* here, we ignore TI_TYPE_FLAG_HIDE_ID intentionally as the behavior + * is defined as to return the Id when no other info is returned */ + + if (!thing->id || + (flags & TI_FLAGS_NO_IDS)) return ti_thing_empty_to_client_pk(&vp->pk); return -( @@ -632,6 +636,11 @@ int ti__wrap_field_thing( if (t_type) { register const ti_name_t * name = t_type->idname; + + /* here, we ignore TI_TYPE_FLAG_HIDE_ID intentionally as the + * behavior is defined as to return the Id when no other info + * is returned */ + return -( msgpack_pack_map(&vp->pk, 1) || (name ? mp_pack_strn(&vp->pk, name->str, name->n) @@ -666,7 +675,7 @@ int ti_wrap_cp(ti_query_t * query, uint8_t deep, ex_t * e) msgpack_sbuffer buffer; ti_vp_t vp = { .query=query, - .size_limit=ti.cfg->result_size_limit, + .size_limit=0x4000, /* we can increase this value */ }; ti_vup_t vup = { .isclient = true, @@ -678,7 +687,10 @@ int ti_wrap_cp(ti_query_t * query, uint8_t deep, ex_t * e) if (mp_sbuffer_alloc_init(&buffer, ti_val_alloc_size(query->rval), 0)) { - ex_set_mem(e); + if (buffer.size > vp.size_limit) + ex_set(e, EX_VALUE_ERROR, "wrap copy() result too large"); + else + ex_set_mem(e); goto fail0; } From 4572a7000e7ca038fb72f067b7dac5e2264b9125 Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Wed, 29 Oct 2025 15:09:59 +0100 Subject: [PATCH 05/13] with add to types --- itest/test_wrap_tree.py | 180 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100755 itest/test_wrap_tree.py diff --git a/itest/test_wrap_tree.py b/itest/test_wrap_tree.py new file mode 100755 index 00000000..26d9feab --- /dev/null +++ b/itest/test_wrap_tree.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python +import asyncio +import pickle +import time +from lib import run_test +from lib import default_test_setup +from lib.testbase import TestBase +from lib.client import get_client +from thingsdb.exceptions import AssertionError +from thingsdb.exceptions import ValueError +from thingsdb.exceptions import TypeError +from thingsdb.exceptions import NumArgumentsError +from thingsdb.exceptions import BadDataError +from thingsdb.exceptions import LookupError +from thingsdb.exceptions import OverflowError +from thingsdb.exceptions import ZeroDivisionError +from thingsdb.exceptions import OperationError + + +class TestWrapTree(TestBase): + + title = 'Test wrap with defined tree structure' + + @default_test_setup(num_nodes=1, seed=1, threshold_full_storage=10) + async def async_run(self): + + await self.node0.init_and_run() + + client = await get_client(self.node0) + client.set_default_scope('//stuff') + + await self.run_tests(client) + + client.close() + await client.wait_closed() + + async def test_define_err(self, client): + + with self.assertRaisesRegex( + ValueError, + r'defining nested types is only allowed for wrap-only type; ' + r'\(happened on type `W`\)'): + await client.query("""//ti + set_type('W', {nested: {x: '#'}}); + """) + + with self.assertRaisesRegex( + ValueError, + r'type keys must follow the naming rules;'): + await client.query("""//ti + nested = {}; + nested['a b'] = '#'; + set_type('W', {nested:,}, true); + """) + + with self.assertRaisesRegex( + TypeError, + r'type `W` has wrap-only mode enabled'): + await client.query("""//ti + set_type('W', {nested: {x: W{}}}, true); + """) + + with self.assertRaisesRegex( + ValueError, + r'array with nested type must have exactly one ' + r'element but found 0 on type `W`'): + await client.query("""//ti + set_type('W', {nested: []}, true); + """) + + with self.assertRaisesRegex( + TypeError, + r'element in array must be of type `thing` but got ' + r'type `int` instead; ' + r'\(happened on type `W`\)'): + await client.query("""//ti + set_type('W', {nested: [123]}, true); + """) + + with self.assertRaisesRegex( + TypeError, + r'expecting a nested structure \(`list` or `thing`\), ' + r'a method of type `closure` or a definition of type `str` ' + r'but got type `nil` instead;'): + await client.query("""//ti + set_type('W', {nested: nil}, true); + """) + + with self.assertRaisesRegex( + ValueError, + r'nested type definition on type `W` too large'): + await client.query("""//ti + nested = {}; + range(2000).map(|x| {nested[`key{x}`] = {x: 'int'}}); + set_type('W', {nested:,}, true); + """) + + with self.assertRaisesRegex( + OperationError, + r'type `W` contains a nested structure which ' + r'requires `wrap-only` mode to be enabled'): + await client.query("""//ti + set_type('W', {nested: {}}, true); + mod_type('W', 'wpo', false); + """) + + async def test_define(self, client): + await client.query("""//ti + set_type('W', { + nested: { + x: '#' + }, + other: { + y: 'int' + }, + arr: [{ + o: { + id: '#', + key: |this| `key{this.id()}`, + } + }] + }, true); + """) + + res = await client.query("""//ti + .x = {nested: {}, other: {}, arr: [{o: {}}]}; + .x.wrap('W'); + """) + self.assertTrue(isinstance(res['nested']['x'], int)) + self.assertEqual(len(res['other']), 0) + self.assertEqual(len(res['arr']), 1) + self.assertEqual(len(res['arr'][0]), 1) + self.assertEqual(len(res['arr'][0]['o']), 2) + self.assertEqual( + f"key{res['arr'][0]['o']['id']}", + res['arr'][0]['o']['key']) + + res = await client.query("""//ti + type_info('W'); + """) + self.assertEqual(sorted(res['fields']), sorted([ + ['nested', {'x': '#'}], + ['other', {'y': 'int'}], + ['arr', [{'o': { + 'id': '#', + 'key': '|this|`key{this.id()}`' + }}]], + ])) + + async def test_mod_type(self, client): + await client.query("""//ti + new_type('O'); + new_type('T'); + set_type('T', {name: 'str', t: 'T?', o: 'O'}); + set_type('O', {arr: '[thing]'}); + new_type('W', true); + set_type('W', { + name: 'str', + arr: [{ + t: 'W', + o: 'O', + t: 'T' + }] + }); + """) + + await client.query("""//ti + rename_type('O', 'OO'); + rename_type('T', 'TT'); + rename_type('W', 'WW'); + """) + + res = await client.query("""//ti + type_info('WW'); + """) + self.assertEqual(res['fields'], []) + + +if __name__ == '__main__': + run_test(TestWrapTree()) From 4cd03999bdebc1bcf3f0d99053f5d4f79bb0790c Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Thu, 30 Oct 2025 17:47:38 +0100 Subject: [PATCH 06/13] work on wrap nested --- inc/ti/condition.t.h | 9 +-- inc/ti/field.h | 28 ++++++--- inc/ti/fmt.h | 2 +- inc/ti/type.h | 8 +-- inc/ti/types.h | 5 -- inc/ti/types.t.h | 2 +- itest/lib/node.py | 1 - itest/test_enum.py | 9 ++- itest/test_wrap_tree.py | 15 ++++- src/ti/closure.c | 2 +- src/ti/condition.c | 29 +++------ src/ti/enum.c | 11 ++-- src/ti/export.c | 2 +- src/ti/field.c | 67 +++++++++++++++++++- src/ti/method.c | 2 +- src/ti/store/storetypes.c | 27 ++------ src/ti/type.c | 41 ++++++------ src/ti/types.c | 129 ++++++++++++++++++++++++++++---------- src/ti/wrap.c | 4 +- 19 files changed, 252 insertions(+), 141 deletions(-) diff --git a/inc/ti/condition.t.h b/inc/ti/condition.t.h index 623e20e0..b0f1a2cd 100644 --- a/inc/ti/condition.t.h +++ b/inc/ti/condition.t.h @@ -11,7 +11,6 @@ typedef struct ti_condition_srange_s ti_condition_srange_t; typedef struct ti_condition_irange_s ti_condition_irange_t; typedef struct ti_condition_drange_s ti_condition_drange_t; typedef struct ti_condition_rel_s ti_condition_rel_t; -typedef struct ti_condition_type_s ti_condition_type_t; #include #include @@ -24,7 +23,7 @@ typedef struct ti_condition_type_s ti_condition_type_t; union ti_condition_via_u { - ti_condition_type_t * type; /* nested type (wrap-only) */ + ti_type_t * type; /* nested type (wrap-only) */ ti_condition_re_t * re; /* str, utf8 */ ti_condition_srange_t * srange; /* str, utf8 */ ti_condition_irange_t * irange; /* int, float */ @@ -74,10 +73,4 @@ struct ti_condition_rel_s ti_condition_rel_cb add_cb; }; -struct ti_condition_type_s -{ - uint32_t type_id; - ti_type_t * type; -}; - #endif /* TI_CONDITION_T_H_ */ diff --git a/inc/ti/field.h b/inc/ti/field.h index 6c8432a7..a194ff39 100644 --- a/inc/ti/field.h +++ b/inc/ti/field.h @@ -46,6 +46,7 @@ int ti_field_make_assignable( ex_t * e); _Bool ti_field_maps_to_val(ti_field_t * field, ti_val_t * val); _Bool ti_field_maps_to_field(ti_field_t * t_field, ti_field_t * f_field); +ti_raw_t * ti_field_nested_to_mpdata(ti_field_t * field); ti_field_t * ti_field_by_strn_e( ti_type_t * type, const char * str, @@ -83,24 +84,31 @@ static inline _Bool ti_field_has_relation(ti_field_t * field) * is a non-nillable set or a nillable type. */ return field->condition.none && ( - field->spec == TI_SPEC_SET || - (field->spec & TI_SPEC_MASK_NILLABLE) < TI_SPEC_ANY); + field->spec == TI_SPEC_SET || + (field->spec & TI_SPEC_MASK_NILLABLE) < TI_SPEC_ANY); } static inline uint8_t ti_field_deep(ti_field_t * field, uint8_t deep) { - return ( - (field->flags & TI_FIELD_FLAG_DEEP) - ? deep - : (field->flags & TI_FIELD_FLAG_MAX_DEEP) - ? TI_MAX_DEEP-1 - : 0 - ) + (field->flags & TI_FIELD_FLAG_SAME_DEEP); + return ( + (field->flags & TI_FIELD_FLAG_DEEP) + ? deep + : (field->flags & TI_FIELD_FLAG_MAX_DEEP) + ? TI_MAX_DEEP-1 + : 0 + ) + (field->flags & TI_FIELD_FLAG_SAME_DEEP); } static inline int ti_field_ret_flags(ti_field_t * field) { - return field->flags & TI_FIELD_FLAG_NO_IDS; + return field->flags & TI_FIELD_FLAG_NO_IDS; +} + +static inline _Bool ti_field_is_nested(ti_field_t * field) +{ + return ( + (field->spec & TI_SPEC_MASK_NILLABLE) == TI_SPEC_TYPE || + (field->spec & TI_SPEC_MASK_NILLABLE) == TI_SPEC_ARR_TYPE); } diff --git a/inc/ti/fmt.h b/inc/ti/fmt.h index b1220aad..e0ba50c7 100644 --- a/inc/ti/fmt.h +++ b/inc/ti/fmt.h @@ -11,7 +11,7 @@ typedef struct ti_fmt_s ti_fmt_t; #include /* When 0, use TAB, else the number of spaces */ -#define TI_FMT_SPACES 0 +#define TI_FMT_TAB 0 struct ti_fmt_s { diff --git a/inc/ti/type.h b/inc/ti/type.h index b3c49e09..3feb3f6b 100644 --- a/inc/ti/type.h +++ b/inc/ti/type.h @@ -28,11 +28,11 @@ ti_type_t * ti_type_create( uint64_t created_at, uint64_t modified_at); ti_type_t * ti_type_create_unnamed( - ti_types_t * types, - uint32_t * type_id, - uint8_t flags); + ti_types_t * types, + ti_raw_t * name, + uint8_t flags); void ti_type_drop(ti_type_t * type); -void ti_type_drop_unnamed(ti_type_t * type, uint32_t type_id); +void ti_type_drop_unnamed(ti_type_t * type); void ti_type_del(ti_type_t * type, vec_t * vars); void ti_type_destroy(ti_type_t * type); void ti_type_map_cleanup(ti_type_t * type); diff --git a/inc/ti/types.h b/inc/ti/types.h index cc1ce61f..674dcc3c 100644 --- a/inc/ti/types.h +++ b/inc/ti/types.h @@ -18,16 +18,11 @@ ti_types_t * ti_types_create(ti_collection_t * collection); void ti_types_destroy(ti_types_t * types); int ti_types_add(ti_types_t * types, ti_type_t * type); void ti_types_del(ti_types_t * types, ti_type_t * type); -void ti_types_del_unnamed(ti_types_t * types, uint32_t type_id); int ti_types_ren_spec( ti_types_t * types, uint16_t type_or_enum_id, ti_raw_t * nname); int ti_types_ren_member_spec(ti_types_t * types, ti_member_t * member); -int ti_types_add_unnamed( - ti_types_t * types, - ti_type_t * type, - uint32_t * type_id) uint16_t ti_types_get_new_id(ti_types_t * types, ti_raw_t * rname, ex_t * e); ti_varr_t * ti_types_info(ti_types_t * types, _Bool with_definition); int ti_types_to_pk(ti_types_t * types, msgpack_packer * pk); diff --git a/inc/ti/types.t.h b/inc/ti/types.t.h index 9182db02..5ad9a712 100644 --- a/inc/ti/types.t.h +++ b/inc/ti/types.t.h @@ -21,7 +21,7 @@ struct ti_types_s smap_t * removed; /* map with type id's which are removed */ ti_collection_t * collection; uint16_t next_id; - uint16_t locked; /* 0 or 1 */ + uint16_t locked; }; #endif /* TI_TYPES_T_H_ */ diff --git a/itest/lib/node.py b/itest/lib/node.py index d5e84b24..857050fa 100644 --- a/itest/lib/node.py +++ b/itest/lib/node.py @@ -143,7 +143,6 @@ async def join_until_ready(self, client, secret='my_secret', timeout=60): def _handle_output(node, r): """Runs in another thread.""" r = os.fdopen(r, 'r') - print(r) for line in r: node.queue.put_nowait(line) if THINGSDB_NODE_OUTPUT is True or ( diff --git a/itest/test_enum.py b/itest/test_enum.py index bf571c84..b470fd95 100755 --- a/itest/test_enum.py +++ b/itest/test_enum.py @@ -928,6 +928,7 @@ async def test_def_enum_definition(self, client): C2: 'Colors{Yellow}?', C3: 'Colors', C4: 'Colors?', + C5: '?[Colors?]?' }) """) @@ -1041,6 +1042,7 @@ async def test_def_enum_definition(self, client): ['C2', 'Color{Orange}?'], ['C3', 'Color'], ['C4', 'Color?'], + ['C5', '?[Color?]?'], ]) res = await q("T{};") @@ -1048,7 +1050,8 @@ async def test_def_enum_definition(self, client): 'C1': 66, 'C2': 77, 'C3': 55, - 'C4': None + 'C4': None, + 'C5': None, }) res = await q("""//ti @@ -1057,13 +1060,15 @@ async def test_def_enum_definition(self, client): C2: Color{Blue}, C3: Color{Blue}, C4: Color{Blue}, + C5: [Color{Blue}], }; """) self.assertEqual(res, { 'C1': 55, 'C2': 55, 'C3': 55, - 'C4': 55 + 'C4': 55, + 'C5': [55], }) with self.assertRaisesRegex( diff --git a/itest/test_wrap_tree.py b/itest/test_wrap_tree.py index 26d9feab..ce3b3248 100755 --- a/itest/test_wrap_tree.py +++ b/itest/test_wrap_tree.py @@ -151,13 +151,15 @@ async def test_mod_type(self, client): await client.query("""//ti new_type('O'); new_type('T'); + set_enum('E', {A: 'a'}); set_type('T', {name: 'str', t: 'T?', o: 'O'}); set_type('O', {arr: '[thing]'}); new_type('W', true); set_type('W', { name: 'str', arr: [{ - t: 'W', + e: 'E', + w: 'W', o: 'O', t: 'T' }] @@ -168,12 +170,21 @@ async def test_mod_type(self, client): rename_type('O', 'OO'); rename_type('T', 'TT'); rename_type('W', 'WW'); + rename_enum('E', 'EE'); """) res = await client.query("""//ti type_info('WW'); """) - self.assertEqual(res['fields'], []) + self.assertEqual(res['fields'], [ + ['name', 'str'], + ['arr', [{ + 'e': 'EE', + 'o': 'OO', + 't': 'TT', + 'w': 'OO' + }]] + ]) if __name__ == '__main__': diff --git a/src/ti/closure.c b/src/ti/closure.c index 043a298f..32834e45 100644 --- a/src/ti/closure.c +++ b/src/ti/closure.c @@ -762,7 +762,7 @@ ti_raw_t * ti_closure_def(ti_closure_t * closure) { ti_raw_t * def; ti_fmt_t fmt; - ti_fmt_init(&fmt, TI_FMT_SPACES); + ti_fmt_init(&fmt, TI_FMT_TAB); if (ti_fmt_nd(&fmt, closure->node)) return NULL; diff --git a/src/ti/condition.c b/src/ti/condition.c index 10014012..7d764a1f 100644 --- a/src/ti/condition.c +++ b/src/ti/condition.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -742,7 +743,6 @@ int ti_condition_field_rel_init( int ti_condition_init_type(ti_field_t * field, ex_t * e) { uint8_t flags = TI_TYPE_FLAG_WRAP_ONLY | TI_TYPE_FLAG_HIDE_ID; - uint32_t type_id; ti_type_t * type; ti_val_t * val; ti_thing_t * thing; @@ -812,7 +812,10 @@ int ti_condition_init_type(ti_field_t * field, ex_t * e) } } - type = ti_type_create_unnamed(field->type->types, &type_id, flags); + type = ti_type_create_unnamed( + field->type->types, + field->type->rname, + flags); if (!type) { ex_set_mem(e); @@ -820,24 +823,11 @@ int ti_condition_init_type(ti_field_t * field, ex_t * e) } if (ti_type_init_from_thing(type, thing, e)) - goto fail1; - - field->condition.type = malloc(sizeof(ti_condition_type_t)); - if (!field->condition.type) - { - ex_set_mem(e); - goto fail1; - } - - field->condition.type->type = type; - field->condition.type->type_id = type_id; - - goto done; + ti_type_drop_unnamed(type); + else + field->condition.type = type; -fail1: - ti_type_drop_unnamed(type, type_id); fail0: -done: ti_val_unsafe_drop(val); return e->nr; } @@ -853,8 +843,7 @@ void ti_condition_destroy(ti_condition_via_t condition, uint16_t spec) { case TI_SPEC_TYPE: case TI_SPEC_ARR_TYPE: - ti_type_drop_unnamed(condition.type->type, condition.type->type_id); - free(condition.type); + ti_type_drop_unnamed(condition.type); return; case TI_SPEC_ANY: return; /* a field may be set to ANY while using mod_type in which diff --git a/src/ti/enum.c b/src/ti/enum.c index 548ebf06..72782595 100644 --- a/src/ti/enum.c +++ b/src/ti/enum.c @@ -827,19 +827,18 @@ static int enum__methods_info_to_pk(ti_enum_t * enum_, msgpack_packer * pk) for (vec_each(enum_->methods, ti_method_t, method)) { ti_raw_t * doc = ti_method_doc(method); - ti_raw_t * def; + ti_raw_t * def = ti_method_def(method); - if (mp_pack_strn(pk, method->name->str, method->name->n) || + if (!def || + mp_pack_strn(pk, method->name->str, method->name->n) || msgpack_pack_map(pk, 4) || mp_pack_str(pk, "doc") || mp_pack_strn(pk, doc->data, doc->n) || - ((def = ti_method_def(method)) && ( - mp_pack_str(pk, "definition") || - mp_pack_strn(pk, def->data, def->n) - )) || + mp_pack_str(pk, "definition") || + mp_pack_strn(pk, def->data, def->n) || mp_pack_str(pk, "with_side_effects") || mp_pack_bool(pk, method->closure->flags & TI_CLOSURE_FLAG_WSE) || diff --git a/src/ti/export.c b/src/ti/export.c index 0dd35e8f..aa720034 100644 --- a/src/ti/export.c +++ b/src/ti/export.c @@ -315,7 +315,7 @@ ti_raw_t * ti_export_collection(ti_collection_t * collection) { ti_raw_t * str; ti_fmt_t fmt; - ti_fmt_init(&fmt, TI_FMT_SPACES); + ti_fmt_init(&fmt, TI_FMT_TAB); if (export__collection(&fmt, collection)) return NULL; diff --git a/src/ti/field.c b/src/ti/field.c index 89864f84..da332f2a 100644 --- a/src/ti/field.c +++ b/src/ti/field.c @@ -762,6 +762,8 @@ static int field__init(ti_field_t * field, ex_t * e) if (field__cmp(str, n, field->type->name)) { + if (field->type->type_id == TI_SPEC_TYPE) + goto found; *spec |= field->type->type_id; if (&field->spec == spec && (~field->spec & TI_SPEC_NILLABLE)) goto circular_dep; @@ -906,7 +908,8 @@ static int field__init(ti_field_t * field, ex_t * e) "skip nil flags for property which cannot contain nil"DOC_T_TYPE, field->name->str, field->type->name); } - assert(field->dval_cb); /* callback must have been set */ + /* assert(field->dval_cb); callback must have been set except for fields + on unnamed type */ return e->nr; invalid: @@ -2551,6 +2554,68 @@ _Bool ti_field_maps_to_field(ti_field_t * t_field, ti_field_t * f_field) return f_spec < TI_SPEC_ANY || f_spec == TI_SPEC_OBJECT; } +static int field__nested_to_pk(ti_field_t * field, msgpack_packer * pk) +{ + ti_type_t * type = field->condition.type; + size_t size = type->fields->n + type->methods->n + !!type->idname; + + if ((field->spec & TI_SPEC_MASK_NILLABLE) == TI_SPEC_ARR_TYPE && + msgpack_pack_array(pk, 1)) + return -1; + + if (msgpack_pack_map(pk, size)) + return -1; + + if (type->idname && ( + mp_pack_strn(pk, type->idname->str, type->idname->n) || + mp_pack_strn(pk, "#", 1))) + return -1; + + for (vec_each(type->fields, ti_field_t, f)) + { + if (mp_pack_strn(pk, f->name->str, f->name->n)) + return -1; + + if ((f->spec & TI_SPEC_MASK_NILLABLE) == TI_SPEC_TYPE || + (f->spec & TI_SPEC_MASK_NILLABLE) == TI_SPEC_ARR_TYPE) + return field__nested_to_pk(f, pk); + + if (mp_pack_strn(pk, f->spec_raw->data, f->spec_raw->n)) + return -1; + } + + for (vec_each(type->methods, ti_method_t, m)) + { + ti_raw_t * def = ti_method_def(m); + if (!def || + mp_pack_strn(pk, m->name->str, m->name->n) || + mp_pack_strn(pk, def->data, def->n)) + return -1; + } + return 0; +} + +ti_raw_t * ti_field_nested_to_mpdata(ti_field_t * field) +{ + ti_raw_t * raw; + msgpack_packer pk; + msgpack_sbuffer buffer; + + mp_sbuffer_alloc_init(&buffer, sizeof(ti_raw_t), sizeof(ti_raw_t)); + msgpack_packer_init(&pk, &buffer, msgpack_sbuffer_write); + + if (field__nested_to_pk(field, &pk)) + { + msgpack_sbuffer_destroy(&buffer); + return NULL; + } + + raw = (ti_raw_t *) buffer.data; + ti_raw_init(raw, TI_VAL_MPDATA, buffer.size); + + return raw; +} + ti_field_t * ti_field_by_strn_e( ti_type_t * type, const char * str, diff --git a/src/ti/method.c b/src/ti/method.c index c026bf9c..8310fcfd 100644 --- a/src/ti/method.c +++ b/src/ti/method.c @@ -48,7 +48,7 @@ ti_raw_t * ti_method_doc(ti_method_t * method) return method->doc; } -/* may return an empty string but never NULL */ +/* may return NULL */ ti_raw_t * ti_method_def(ti_method_t * method) { if (!method->def) diff --git a/src/ti/store/storetypes.c b/src/ti/store/storetypes.c index dd9b8db2..8d524695 100644 --- a/src/ti/store/storetypes.c +++ b/src/ti/store/storetypes.c @@ -30,9 +30,6 @@ static int rmtype_cb( static int mktype_cb(ti_type_t * type, msgpack_packer * pk) { uintptr_t p; - if (type->type_id == TI_SPEC_TYPE) - return 0; - if (msgpack_pack_array(pk, 8) || msgpack_pack_uint16(pk, type->type_id) || mp_pack_bool(pk, type->flags & TI_TYPE_FLAG_WRAP_ONLY) || @@ -75,19 +72,10 @@ static int mktype_cb(ti_type_t * type, msgpack_packer * pk) return 0; } -typedef struct { - size_t n; /* number of relations */ - size_t m; /* number of unnamed */ -} count_walk_t; - -static int count_cb(ti_type_t * type, count_walk_t * w) +static int count_cb(ti_type_t * type, size_t * n) { if (ti_type_is_wrap_only(type)) - { - if (type->type_id == TI_SPEC_TYPE) - (w->m)++; return 0; - } for (vec_each(type->fields, ti_field_t, field)) { @@ -98,7 +86,7 @@ static int count_cb(ti_type_t * type, count_walk_t * w) field->type->type_id < other->type->type_id || ( field->type->type_id == other->type->type_id && field->idx < other->idx)) - (w->n)++; + (*n)++; } } return 0; @@ -133,10 +121,7 @@ int ti_store_types_store(ti_types_t * types, const char * fn) msgpack_packer pk; char namebuf[TI_NAME_MAX]; FILE * f = fopen(fn, "w"); - count_walk_t w = { - .n = 0, - .m = 0, - }; + size_t n = 0; if (!f) { @@ -145,7 +130,7 @@ int ti_store_types_store(ti_types_t * types, const char * fn) } /* count the number of relations */ - (void) imap_walk(types->imap, (imap_cb) count_cb, &w); + (void) imap_walk(types->imap, (imap_cb) count_cb, &n); msgpack_packer_init(&pk, f, msgpack_fbuffer_write); @@ -156,11 +141,11 @@ int ti_store_types_store(ti_types_t * types, const char * fn) smap_items(types->removed, namebuf, (smap_item_cb) rmtype_cb, &pk) || /* active types */ mp_pack_str(&pk, "types") || - msgpack_pack_array(&pk, types->imap->n - w.m) || + msgpack_pack_array(&pk, types->imap->n) || imap_walk(types->imap, (imap_cb) mktype_cb, &pk) || /* relations */ mp_pack_str(&pk, "relations") || - msgpack_pack_array(&pk, w.n) || + msgpack_pack_array(&pk, n) || imap_walk(types->imap, (imap_cb) rltype_cb, &pk) ) goto fail; diff --git a/src/ti/type.c b/src/ti/type.c index d442065a..e0a0dcfd 100644 --- a/src/ti/type.c +++ b/src/ti/type.c @@ -77,9 +77,9 @@ ti_type_t * ti_type_create( } ti_type_t * ti_type_create_unnamed( - ti_types_t * types, - uint32_t * type_id, - uint8_t flags) + ti_types_t * types, + ti_raw_t * name, + uint8_t flags) { ti_type_t * type = malloc(sizeof(ti_type_t)); if (!type) @@ -89,10 +89,10 @@ ti_type_t * ti_type_create_unnamed( type->selfref = 0; /* increment on self references */ type->type_id = TI_SPEC_TYPE; type->flags = TI_TYPE_FLAG_WRAP_ONLY|flags; - type->rname = (ti_raw_t *) ti_val_unnamed_name(); - type->name = strndup((const char *) type->rname->data, type->rname->n); - type->rwname = (ti_raw_t *) ti_val_unnamed_name(); - type->wname = strndup((const char *) type->rwname->data, type->rwname->n); + type->name = strndup((const char *) name->data, name->n); + type->rname = ti_grab(name); + type->wname = NULL; + type->rwname = NULL; type->idname = NULL; type->dependencies = vec_new(0); type->fields = vec_new(0); @@ -102,9 +102,8 @@ ti_type_t * ti_type_create_unnamed( type->modified_at = 0; type->methods = vec_new(0); - if (!type->name || !type->wname || !type->dependencies || !type->fields || - !type->rname || !type->rwname || !type->t_mappings || !type->methods || - ti_types_add_unnamed(types, type, type_id)) + if (!type->name || !type->dependencies || !type->fields || + !type->rname || !type->t_mappings || !type->methods) { ti_type_destroy(type); return NULL; @@ -146,15 +145,15 @@ void ti_type_drop(ti_type_t * type) ti_type_destroy(type); } -void ti_type_drop_unnamed(ti_type_t * type, uint32_t type_id) +void ti_type_drop_unnamed(ti_type_t * type) { - if (!type || type->types->locked) + if (!type) return; - for (vec_each(type->dependencies, ti_type_t, dep)) - --dep->refcount; + if (!type->types->locked) + for (vec_each(type->dependencies, ti_type_t, dep)) + --dep->refcount; - ti_types_del_unnamed(type->types, type_id); ti_type_destroy(type); } @@ -1103,16 +1102,19 @@ int ti_type_methods_info_to_pk( for (vec_each(type->methods, ti_method_t, method)) { ti_raw_t * doc = ti_method_doc(method); - ti_raw_t * def; + ti_raw_t * def = ti_method_def(method); /* def calculates only once + so not a problem to do + if not with_definition */ - if (mp_pack_strn(pk, method->name->str, method->name->n) || + if (!def || + mp_pack_strn(pk, method->name->str, method->name->n) || msgpack_pack_map(pk, 3 + !!with_definition) || mp_pack_str(pk, "doc") || mp_pack_strn(pk, doc->data, doc->n) || - (with_definition && (def = ti_method_def(method)) && ( + (with_definition && ( mp_pack_str(pk, "definition") || mp_pack_strn(pk, def->data, def->n) )) || @@ -1522,8 +1524,7 @@ int ti_type_requires_wpo(ti_type_t * type, ex_t * e) { for (vec_each(type->fields, ti_field_t, field)) { - if ((field->spec & TI_SPEC_MASK_NILLABLE) == TI_SPEC_TYPE || - (field->spec & TI_SPEC_MASK_NILLABLE) == TI_SPEC_ARR_TYPE) + if (ti_field_is_nested(field)) { ex_set(e, EX_OPERATION, "type `%s` contains a nested structure which requires " diff --git a/src/ti/types.c b/src/ti/types.c index e00840b6..27e2eb3f 100644 --- a/src/ti/types.c +++ b/src/ti/types.c @@ -3,6 +3,7 @@ */ #include #include +#include #include #include #include @@ -21,7 +22,6 @@ ti_types_t * ti_types_create(ti_collection_t * collection) types->removed = smap_create(); types->collection = collection; types->next_id = 0; - types->next_unnamed = 0x10000; // outside spec range types->locked = 0; if (!types->imap || !types->smap || !types->removed) @@ -61,16 +61,6 @@ int ti_types_add(ti_types_t * types, ti_type_t * type) return 0; } -int ti_types_add_unnamed( - ti_types_t * types, - ti_type_t * type, - uint32_t * type_id) -{ - *type_id = types->next_unnamed++; - printf("Type: %p\n", type); - return 0; // imap_add(types->imap, *type_id, type); -} - /* * Do not use this function directly; Call ti_type_del(..) so * existing things using this type will be converted to objects and @@ -83,14 +73,10 @@ void ti_types_del(ti_types_t * types, ti_type_t * type) (void) smap_pop(types->smap, type->name); } -void ti_types_del_unnamed(ti_types_t * types, uint32_t type_id) -{ - (void) imap_pop(types->imap, type_id); -} - typedef struct { uint16_t id; + uint32_t nchanges; ti_raw_t * nname; } types__ren_t; @@ -109,30 +95,83 @@ int types__spec_flags_pos(const unsigned char * x) return i; } -static int types__ren_member_cb(ti_type_t * type, ti_member_t * member) +static int types__ren_member(ti_field_t * field, ti_member_t * member) +{ + /* enum with default value rename */ + int flags_pos = types__spec_flags_pos(field->spec_raw->data); + ti_raw_t * spec_raw = ti_str_from_fmt( + "%.*s%s{%s}%s", + flags_pos, + (const char *) field->spec_raw->data, + member->enum_->name, + member->name->str, + (field->spec & TI_SPEC_NILLABLE) ? "?": ""); + if (!spec_raw) + return -1; + + ti_val_unsafe_drop((ti_val_t *) field->spec_raw); + field->spec_raw = spec_raw; + + return 0; +} + +static int types__ren_member_nested( + ti_type_t * type, + ti_member_t * member, + int * has_changed) { for (vec_each(type->fields, ti_field_t, field)) { + if (ti_field_is_nested(field)) + { + if (types__ren_member_nested( + field->condition.type, + member, + has_changed)) + return -1; + continue; + } + if (field->condition.none && - field->condition.none->dval == (ti_val_t *) member) + field->condition.none->dval == (ti_val_t *) member) { - /* enum with default value rename */ - int flags_pos = types__spec_flags_pos(field->spec_raw->data); - ti_member_t * member = \ - (ti_member_t *) field->condition.none->dval; - ti_raw_t * spec_raw = ti_str_from_fmt( - "%.*s%s{%s}%s", - flags_pos, - (const char *) field->spec_raw->data, - member->enum_->name, - member->name->str, - (field->spec & TI_SPEC_NILLABLE) ? "?": ""); - if (!spec_raw) + if (types__ren_member(field, member)) return -1; + *has_changed = 1; + } + } + return 0; +} - ti_val_unsafe_drop((ti_val_t *) field->spec_raw); - field->spec_raw = spec_raw; +static int types__ren_member_cb(ti_type_t * type, ti_member_t * member) +{ + for (vec_each(type->fields, ti_field_t, field)) + { + if (ti_field_is_nested(field)) + { + int has_changed = 0; + if (types__ren_member_nested( + field->condition.type, + member, + &has_changed)) + return -1; + + if (has_changed) + { + ti_raw_t * spec_raw = ti_field_nested_to_mpdata(field); + if (!spec_raw) + return -1; + + ti_val_unsafe_drop((ti_val_t *) field->spec_raw); + field->spec_raw = spec_raw; + } + continue; } + + if (field->condition.none && + field->condition.none->dval == (ti_val_t *) member && + types__ren_member(field, member)) + return -1; } return 0; } @@ -141,6 +180,25 @@ static int types__ren_cb(ti_type_t * type, types__ren_t * w) { for (vec_each(type->fields, ti_field_t, field)) { + if (ti_field_is_nested(field)) + { + uint32_t prev = w->nchanges; + if (types__ren_cb(field->condition.type, w)) + return -1; + + if (type->type_id != TI_SPEC_TYPE && w->nchanges > prev) + { + ti_raw_t * spec_raw = ti_field_nested_to_mpdata(field); + if (!spec_raw) + return -1; + + ti_val_unsafe_drop((ti_val_t *) field->spec_raw); + field->spec_raw = spec_raw; + } + + continue; + } + if ((field->spec & TI_SPEC_MASK_NILLABLE) == w->id) { int flags_pos = types__spec_flags_pos(field->spec_raw->data); @@ -162,12 +220,14 @@ static int types__ren_cb(ti_type_t * type, types__ren_t * w) ti_val_unsafe_drop((ti_val_t *) field->spec_raw); field->spec_raw = spec_raw; + w->nchanges++; } else if (field->spec == w->id && !flags_pos) { ti_val_unsafe_drop((ti_val_t *) field->spec_raw); field->spec_raw = w->nname; ti_incref(field->spec_raw); + w->nchanges++; } else { @@ -183,6 +243,7 @@ static int types__ren_cb(ti_type_t * type, types__ren_t * w) ti_val_unsafe_drop((ti_val_t *) field->spec_raw); field->spec_raw = spec_raw; + w->nchanges++; } } else if ((field->nested_spec & TI_SPEC_MASK_NILLABLE) == w->id) @@ -226,6 +287,7 @@ static int types__ren_cb(ti_type_t * type, types__ren_t * w) ti_val_unsafe_drop((ti_val_t *) field->spec_raw); field->spec_raw = spec_raw; + w->nchanges++; } } return 0; @@ -239,6 +301,7 @@ int ti_types_ren_spec( types__ren_t w = { .id = type_or_enum_id, .nname = nname, + .nchanges = 0, }; return imap_walk(types->imap, (imap_cb) types__ren_cb, &w); @@ -249,8 +312,6 @@ int ti_types_ren_member_spec(ti_types_t * types, ti_member_t * member) return imap_walk(types->imap, (imap_cb) types__ren_member_cb, member); } - - uint16_t ti_types_get_new_id(ti_types_t * types, ti_raw_t * rname, ex_t * e) { uintptr_t utype; diff --git a/src/ti/wrap.c b/src/ti/wrap.c index 4cd2304b..9116ee6d 100644 --- a/src/ti/wrap.c +++ b/src/ti/wrap.c @@ -159,7 +159,7 @@ static int wrap__field_val( ? wrap__field_thing_type( (ti_thing_t *) val, vp, - t_field->condition.type->type, + t_field->condition.type, deep, flags) : ti__wrap_field_thing( @@ -173,7 +173,7 @@ static int wrap__field_val( ? wrap__field_thing_type( (ti_thing_t *) val, vp, - t_field->condition.type->type, + t_field->condition.type, deep, flags) : ti__wrap_field_thing( From 6d3bd4ca96c1639e0d5dc6852f2f95c1463c7cbc Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Thu, 30 Oct 2025 19:37:43 +0100 Subject: [PATCH 07/13] changelog, fix set wrap --- CHANGELOG.md | 1 + itest/test_wrap_tree.py | 44 +++++++++++++++++++++++++++++++++++++++++ src/ti/wrap.c | 26 ++++++++++++++++++++---- 3 files changed, 67 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d99360cd..ed17b0e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ * Added `TI_PROTO_CLIENT_REQ_EMIT_PEER` protocol, pr #414. * Changed `copy()` behavior of wrapped type, pr #415. +* Allow defining nested structure in wrap type, pr #416. # v1.7.6 diff --git a/itest/test_wrap_tree.py b/itest/test_wrap_tree.py index ce3b3248..a5e4c142 100755 --- a/itest/test_wrap_tree.py +++ b/itest/test_wrap_tree.py @@ -186,6 +186,50 @@ async def test_mod_type(self, client): }]] ]) + async def test_pr_example(self, client): + await client.query("""//ti + set_type('M1', { + id: '#', + num: 'int', + }, true); + + set_type('T1', { + metrics: '&[M1]', + }, true, true); + + set_type('C1', { + x: '#', + name: 'str', + type: '&T1', + }, true); + + set_type('C2', { + x: '#', + name: 'str', + type: { + metrics: [{ + id: '#', + num: 'int', + }], + }, + }, true); + """) + + C1, C2 = await client.query("""//ti + .x = { + name: 'example', + type: { + metrics: set({ + num: 1 + }, { + num: 2 + }) + } + }; + [.x.wrap('C1'), .x.wrap('C2')]; + """) + self.assertEqual(C1, C2) + if __name__ == '__main__': run_test(TestWrapTree()) diff --git a/src/ti/wrap.c b/src/ti/wrap.c index 9116ee6d..c2049d93 100644 --- a/src/ti/wrap.c +++ b/src/ti/wrap.c @@ -87,23 +87,41 @@ static int wrap__walk_with_type(ti_thing_t * thing, wrap__walk_with_type_t * w) static int wrap__set( ti_vset_t * vset, ti_vp_t * vp, - uint16_t spec, + ti_field_t * t_field, int deep, int flags) { wrap__walk_t w = { .vp = vp, - .spec = spec, + .spec = t_field->nested_spec, .deep = deep, .flags = flags, }; + if ((t_field->nested_spec & TI_SPEC_MASK_NILLABLE) == TI_SPEC_TYPE) + { + wrap__walk_with_type_t wwt = { + .vp = vp, + .t_type = t_field->condition.type, + .deep = deep, + .flags = flags, + }; + + return ( + msgpack_pack_array(&vp->pk, vset->imap->n) || + imap_walk(vset->imap, (imap_cb) wrap__walk_with_type, &wwt) + ); + } + if (vset->imap->n > 1 && vp->query && vp->query->collection) { /* optimization for set's with multiple values */ - ti_type_t * t_type = ti_types_by_id(vp->query->collection->types, spec); + ti_type_t * t_type = ti_types_by_id( + vp->query->collection->types, + t_field->nested_spec); + if (t_type) { wrap__walk_with_type_t wwt = { @@ -208,7 +226,7 @@ static int wrap__field_val( return wrap__set( (ti_vset_t *) val, vp, - t_field->nested_spec, + t_field, deep, flags); case TI_VAL_ERROR: From f578774538e6ffba91eb0a7bfd5fb12d28f90a39 Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Thu, 30 Oct 2025 20:41:14 +0100 Subject: [PATCH 08/13] more tests --- itest/test_wrap_tree.py | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/itest/test_wrap_tree.py b/itest/test_wrap_tree.py index a5e4c142..ba521d28 100755 --- a/itest/test_wrap_tree.py +++ b/itest/test_wrap_tree.py @@ -104,6 +104,22 @@ async def test_define_err(self, client): mod_type('W', 'wpo', false); """) + with self.assertRaisesRegex( + TypeError, + r'cannot convert a property into a method; '): + await client.query("""//ti + mod_type('W', 'mod', 'nested', ||nil); + """) + + with self.assertRaisesRegex( + NumArgumentsError, + r'function `mod_type` with task `mod` takes at ' + r'most 4 arguments when used on a type with wrap-only ' + r'mode enabled;'): + await client.query("""//ti + mod_type('W', 'mod', 'nested', 'str', ||nil); + """) + async def test_define(self, client): await client.query("""//ti set_type('W', { @@ -182,10 +198,32 @@ async def test_mod_type(self, client): 'e': 'EE', 'o': 'OO', 't': 'TT', - 'w': 'OO' + 'w': 'OO', }]] ]) + await client.query("""//ti + mod_type('WW', 'ren', 'arr', 'obj'); + mod_type('WW', 'mod', 'obj', { + e: 'EE', + o: 'OO', + t: 'TT', + w: 'OO', + }); + """) + res = await client.query("""//ti + type_info('WW'); + """) + self.assertEqual(res['fields'], [ + ['name', 'str'], + ['obj', { + 'e': 'EE', + 'o': 'OO', + 't': 'TT', + 'w': 'OO' + }] + ]) + async def test_pr_example(self, client): await client.query("""//ti set_type('M1', { From 4006011eed7c793be833a3b086c5666137189c82 Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Thu, 30 Oct 2025 20:48:01 +0100 Subject: [PATCH 09/13] comment --- src/ti/type.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ti/type.c b/src/ti/type.c index e0a0dcfd..540f8e0d 100644 --- a/src/ti/type.c +++ b/src/ti/type.c @@ -90,7 +90,9 @@ ti_type_t * ti_type_create_unnamed( type->type_id = TI_SPEC_TYPE; type->flags = TI_TYPE_FLAG_WRAP_ONLY|flags; type->name = strndup((const char *) name->data, name->n); - type->rname = ti_grab(name); + type->rname = ti_grab(name); /* name must be equal to the master type; + in fields.c, this is compared for + circular references */ type->wname = NULL; type->rwname = NULL; type->idname = NULL; From fbfc7f2620aa91f3ef562576c4176080ad635c6e Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Thu, 30 Oct 2025 22:20:57 +0100 Subject: [PATCH 10/13] more test --- itest/test_wrap_tree.py | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/itest/test_wrap_tree.py b/itest/test_wrap_tree.py index ba521d28..bc6de0ae 100755 --- a/itest/test_wrap_tree.py +++ b/itest/test_wrap_tree.py @@ -77,6 +77,14 @@ async def test_define_err(self, client): set_type('W', {nested: [123]}, true); """) + with self.assertRaisesRegex( + TypeError, + r'invalid declaration for `a` on type `W`; unknown ' + r'type `UNKNOWN` in declaration;'): + await client.query("""//ti + set_type('W', {nested: [{a: 'UNKNOWN'}]}, true); + """) + with self.assertRaisesRegex( TypeError, r'expecting a nested structure \(`list` or `thing`\), ' @@ -170,7 +178,6 @@ async def test_mod_type(self, client): set_enum('E', {A: 'a'}); set_type('T', {name: 'str', t: 'T?', o: 'O'}); set_type('O', {arr: '[thing]'}); - new_type('W', true); set_type('W', { name: 'str', arr: [{ @@ -179,7 +186,7 @@ async def test_mod_type(self, client): o: 'O', t: 'T' }] - }); + }, true); """) await client.query("""//ti @@ -224,6 +231,30 @@ async def test_mod_type(self, client): }] ]) + await client.query("""//ti + mod_type('WW', 'del', 'obj'); + mod_type('WW', 'add', 'o', { + i: '#', + e: { + v: 'EE?' + } + }); + """) + + res = await client.query("""//ti + type_info('WW'); + """) + + self.assertEqual(res['fields'], [ + ['name', 'str'], + ['o', { + 'i': '#', + 'e': { + 'v': 'EE?' + } + }] + ]) + async def test_pr_example(self, client): await client.query("""//ti set_type('M1', { From 1b4a4c3bf8e9019984f3d7b73a25608c77f68d77 Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Fri, 31 Oct 2025 07:50:43 +0100 Subject: [PATCH 11/13] Added new test --- itest/run_all_tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/itest/run_all_tests.py b/itest/run_all_tests.py index cd7153ef..5f76fec6 100755 --- a/itest/run_all_tests.py +++ b/itest/run_all_tests.py @@ -39,6 +39,7 @@ from test_variable import TestVariable from test_whitelist import TestWhitelist from test_wrap import TestWrap +from test_wrap_tree import TestWrapTree from test_ws import TestWS from test_wss import TestWSS @@ -113,3 +114,4 @@ def no_mem_test(test_class): run_test(TestWS(), hide_version=hide_version()) run_test(TestWSS(), hide_version=hide_version()) run_test(TestWrap(), hide_version=hide_version()) + run_test(TestWrapTree(), hide_version=hide_version()) From 7066a533fb2cac47eb8b3c9ea40e3426a09ab0cc Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Fri, 31 Oct 2025 07:56:37 +0100 Subject: [PATCH 12/13] modules --- itest/run_all_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/itest/run_all_tests.py b/itest/run_all_tests.py index 5f76fec6..aeed45e1 100755 --- a/itest/run_all_tests.py +++ b/itest/run_all_tests.py @@ -90,7 +90,7 @@ def no_mem_test(test_class): run_test(TestIndexSlice(), hide_version=hide_version()) run_test(TestMath(), hide_version=hide_version()) if args.doc_modules is True: - run_test(TestModules(), hide_version=hide_version()) + no_mem_test(TestModules) # libcurl leaks mem run_test(TestMultiNode(), hide_version=hide_version()) run_test(TestNested(), hide_version=hide_version()) run_test(TestNodeFunctions(), hide_version=hide_version()) From 9791f8d3a5dc863d1c8a679fe57475a3724ca8a7 Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Fri, 31 Oct 2025 11:05:16 +0100 Subject: [PATCH 13/13] ver and changelog --- CHANGELOG.md | 4 ++-- inc/ti/version.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50a31c03..816bda9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ -# v1.8.0-alpha0 +# v1.8.0-alpha2 * Added `TI_PROTO_CLIENT_REQ_EMIT_PEER` protocol, pr #414. -* Changed `copy()` behavior of wrapped type, pr #415. +* Changed `copy()` behavior for wrapped type, pr #415. * Fixed missing wrap-prefix flags after renaming type, issue #416. * Allow defining nested structure in wrap type, pr #417. diff --git a/inc/ti/version.h b/inc/ti/version.h index af4cf1ab..6939859d 100644 --- a/inc/ti/version.h +++ b/inc/ti/version.h @@ -25,7 +25,7 @@ * "-rc0" * "" */ -#define TI_VERSION_PRE_RELEASE "-alpha1" +#define TI_VERSION_PRE_RELEASE "-alpha2" #define TI_MAINTAINER \ "Jeroen van der Heijden "