From a1935e43c2dc169bef6782063ed13592a6260526 Mon Sep 17 00:00:00 2001 From: Kan Li Date: Tue, 23 Feb 2021 16:32:03 -0500 Subject: [PATCH 1/5] fixed #69 & #72 --- flatten_json/__init__.py | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/flatten_json/__init__.py b/flatten_json/__init__.py index 9ab02ec..4cca5aa 100644 --- a/flatten_json/__init__.py +++ b/flatten_json/__init__.py @@ -201,7 +201,7 @@ def _flatten(object_, key): else: flattened_dict[key] = object_ - def _flatten_low_entropy(object_, key, cur_depth, max_depth_inner): + def _flatten_low_entropy(object_, key, cur_depth, cur_list_depth, max_depth_inner): """ For dict, list and set objects_ calls itself on the elements and for other types assigns the object_ to @@ -217,7 +217,7 @@ def _flatten_low_entropy(object_, key, cur_depth, max_depth_inner): # write latest child as value if max_depth exceeded if cur_depth > max_depth_inner: global_max_record = int(max(list( - list_prebuilt_flattened_dict.keys()))) + list_prebuilt_flattened_dict.keys()), key=int)) for d in list_prebuilt_flattened_dict[str(global_max_record)]: d[key] = object_ @@ -225,7 +225,7 @@ def _flatten_low_entropy(object_, key, cur_depth, max_depth_inner): # Empty object can't be iterated, take as is if not object_: global_max_record = int(max(list( - list_prebuilt_flattened_dict.keys()))) + list_prebuilt_flattened_dict.keys()), key=int)) for d in list_prebuilt_flattened_dict[str(global_max_record)]: d[key] = object_ @@ -239,7 +239,7 @@ def _flatten_low_entropy(object_, key, cur_depth, max_depth_inner): and not (isinstance(object_[first_key], dict) or isinstance(object_[first_key], list)): global_max_record = int(max(list( - list_prebuilt_flattened_dict.keys()))) + list_prebuilt_flattened_dict.keys()), key=int)) for d in list_prebuilt_flattened_dict[ str(global_max_record) @@ -265,6 +265,8 @@ def _flatten_low_entropy(object_, key, cur_depth, max_depth_inner): # lists could go into rows, like in a relational database elif isinstance(object_, list) or isinstance(object_, set): + cur_list_depth = cur_list_depth + 1 + if debug: print("\nparent key of list:", key, "| length: ", @@ -273,7 +275,7 @@ def _flatten_low_entropy(object_, key, cur_depth, max_depth_inner): # need to remember global list state when we entered # this recursion global_max_record_start = int(max(list( - list_prebuilt_flattened_dict.keys()))) + list_prebuilt_flattened_dict.keys()), key=int)) entry = copy.deepcopy(list_prebuilt_flattened_dict[ str(global_max_record_start) ]) @@ -294,27 +296,29 @@ def _flatten_low_entropy(object_, key, cur_depth, max_depth_inner): # start from second element, 1st element is like column if index > 0: global_max_record = int(max(list( - list_prebuilt_flattened_dict.keys()))) + list_prebuilt_flattened_dict.keys()), key=int)) list_prebuilt_flattened_dict[ str(global_max_record + 1) ] = copy.deepcopy(entry) - _flatten_low_entropy(item, key, cur_depth, + _flatten_low_entropy(item, key, cur_depth, cur_list_depth, max_depth_inner) else: pass - list_prebuilt_flattened_dict['0'] = \ - [subel for k, v in - sorted(list_prebuilt_flattened_dict.items()) - for idx, subel in enumerate(v)] + cur_list_depth = cur_list_depth - 1 + if cur_list_depth == 0: + list_prebuilt_flattened_dict['0'] = \ + [subel for k, v in + sorted(list_prebuilt_flattened_dict.items()) + for idx, subel in enumerate(v)] - for key in list(sorted(list_prebuilt_flattened_dict.keys())): - if key != '0': - del list_prebuilt_flattened_dict[key] - if debug: - print("collapsed global list") + for key in list(sorted(list_prebuilt_flattened_dict.keys())): + if key != '0': + del list_prebuilt_flattened_dict[key] + if debug: + print("collapsed global list") # Anything left take as is, assuming you hit the end of the line. else: @@ -322,7 +326,7 @@ def _flatten_low_entropy(object_, key, cur_depth, max_depth_inner): # a list of prebuilt_flattened_dict by now # so need to update them all. global_max_record = int(max(list( - list_prebuilt_flattened_dict.keys()))) + list_prebuilt_flattened_dict.keys()), key=int)) for d in list_prebuilt_flattened_dict[str(global_max_record)]: d[key] = object_ From d0145ce3ef3bbc341b9f87bcf51aced4bea24d4d Mon Sep 17 00:00:00 2001 From: Kan Li Date: Tue, 23 Feb 2021 19:00:18 -0500 Subject: [PATCH 2/5] fixed #69 & #72 --- flatten_json/__init__.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/flatten_json/__init__.py b/flatten_json/__init__.py index 4cca5aa..c58267e 100644 --- a/flatten_json/__init__.py +++ b/flatten_json/__init__.py @@ -201,7 +201,11 @@ def _flatten(object_, key): else: flattened_dict[key] = object_ - def _flatten_low_entropy(object_, key, cur_depth, cur_list_depth, max_depth_inner): + def _flatten_low_entropy(object_, + key, + cur_depth, + cur_list_depth, + max_depth_inner): """ For dict, list and set objects_ calls itself on the elements and for other types assigns the object_ to @@ -261,6 +265,7 @@ def _flatten_low_entropy(object_, key, cur_depth, cur_list_depth, max_depth_inne object_key, replace_separators=replace_separators), cur_depth, + cur_list_depth, max_depth_inner) # lists could go into rows, like in a relational database @@ -302,7 +307,10 @@ def _flatten_low_entropy(object_, key, cur_depth, cur_list_depth, max_depth_inne str(global_max_record + 1) ] = copy.deepcopy(entry) - _flatten_low_entropy(item, key, cur_depth, cur_list_depth, + _flatten_low_entropy(item, + key, + cur_depth, + cur_list_depth, max_depth_inner) else: pass @@ -352,7 +360,10 @@ def _flatten_low_entropy(object_, key, cur_depth, cur_list_depth, max_depth_inne # initialize global record list list_prebuilt_flattened_dict = {'0': [prebuilt_flattened_dict]} - _flatten_low_entropy(nested_dict, None, cur_depth=0, + _flatten_low_entropy(nested_dict, + None, + cur_depth=0, + cur_list_depth=0, max_depth_inner=max_depth) return list_prebuilt_flattened_dict['0'] From 63ce9d371437496605d1eedcbbb14a3e3707265d Mon Sep 17 00:00:00 2001 From: Kan Li Date: Wed, 24 Feb 2021 09:07:14 -0500 Subject: [PATCH 3/5] fixed #69 & #72 --- test_flatten.py | 129 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/test_flatten.py b/test_flatten.py index cbb24cb..c47535d 100644 --- a/test_flatten.py +++ b/test_flatten.py @@ -2170,6 +2170,135 @@ def test_flatten_preserve_lists_issue43(self): actual = flatten_preserve_lists(dic, max_list_index=50, max_depth=10) self.assertEqual(expected, actual) + + def test_flatten_preserve_lists_issue69(self): + """https://github.com/amirziai/flatten/issues/69""" + dic = { + 'a': 'a', + 'b': [ + {'b': 1, 'c': 1}, + {'b': 2, 'c': 2}, + {'b': 3, 'c': 3}, + {'b': 4, 'c': 4}, + {'b': 5, 'c': 5}, + {'b': 6, 'c': 6}, + {'b': 7, 'c': 7}, + {'b': 8, 'c': 8}, + {'b': 9, 'c': 9}, + {'b': 10, 'c': 10}, + {'b': 11, 'c': 11}, + {'b': 12, 'c': 12}, + {'b': 13, 'c': 13}, + {'b': 14, 'c': 14}, + {'b': 15, 'c': 15}, + {'b': 16, 'c': 16}, + {'b': 17, 'c': 17}, + {'b': 15, 'c': 18}, + {'b': 16, 'c': 19}, + {'b': 17, 'c': 20} + ] + } + + expected = [ + {'b.c': 1, + 'b.b': 1, + 'a': 'a'}, + {'b.c': 2, + 'b.b': 2, + 'a': 'a'}, + {'b.c': 11, + 'b.b': 11, + 'a': 'a'}, + {'b.c': 12, + 'b.b': 12, + 'a': 'a'}, + {'b.c': 13, + 'b.b': 13, + 'a': 'a'}, + {'b.c': 14, + 'b.b': 14, + 'a': 'a'}, + {'b.c': 15, + 'b.b': 15, + 'a': 'a'}, + {'b.c': 16, + 'b.b': 16, + 'a': 'a'}, + {'b.c': 17, + 'b.b': 17, + 'a': 'a'}, + {'b.c': 18, + 'b.b': 15, + 'a': 'a'}, + {'b.c': 19, + 'b.b': 16, + 'a': 'a'}, + {'b.c': 20, + 'b.b': 17, + 'a': 'a'}, + {'b.c': 3, + 'b.b': 3, + 'a': 'a'}, + {'b.c': 4, + 'b.b': 4, + 'a': 'a'}, + {'b.c': 5, + 'b.b': 5, + 'a': 'a'}, + {'b.c': 6, + 'b.b': 6, + 'a': 'a'}, + {'b.c': 7, + 'b.b': 7, + 'a': 'a'}, + {'b.c': 8, + 'b.b': 8, + 'a': 'a'}, + {'b.c': 9, + 'b.b': 9, + 'a': 'a'}, + {'b.c': 10, + 'b.b': 10, + 'a': 'a'}] + + actual = flatten_preserve_lists(dic, separator='.', max_list_index=100, max_depth=5) + self.assertEqual(expected, actual) + + def test_flatten_preserve_lists_issue72(self): + """https://github.com/amirziai/flatten/issues/72""" + dic = {'a': 0, + 'b':[ + {'c': 1, + 'd': [{'e': 1}]}, + {'c': 2, + 'd': [{'e': 2}]}, + {'c': 3, + 'd': [{'e': 3}]}, + {'c': 4, + 'd': [{'e': 4}]}, + {'c': 5, + 'd': [{'e': 5}]}]} + + expected = [ + {'b.c': 1, + 'a': 0, + 'b.d': 1}, + {'b.c': 2, + 'a': 0, + 'b.d': 2}, + {'b.c': 3, + 'a': 0, + 'b.d': 3}, + {'b.c': 4, + 'a': 0, + 'b.d': 4}, + {'b.c': 5, + 'a': 0, + 'b.d': 5}] + + actual = flatten_preserve_lists(dic, separator='.', max_list_index=10, max_depth=5) + self.assertEqual(expected, actual) + def test_unflatten_with_list_deep(self): dic = {'a': [ {'b': [{'c': [{'a': 5, 'b': {'a': [1, 2, 3]}, 'c': {'x': 3}}]}]}]} From d80a94fcfe57b8adaf570d060de07cbe5203efc6 Mon Sep 17 00:00:00 2001 From: Kan Li Date: Wed, 24 Feb 2021 09:08:40 -0500 Subject: [PATCH 4/5] fixed #69 & #72 --- test_flatten.py | 71 ++++++++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 36 deletions(-) diff --git a/test_flatten.py b/test_flatten.py index c47535d..5d18328 100644 --- a/test_flatten.py +++ b/test_flatten.py @@ -2170,33 +2170,32 @@ def test_flatten_preserve_lists_issue43(self): actual = flatten_preserve_lists(dic, max_list_index=50, max_depth=10) self.assertEqual(expected, actual) - def test_flatten_preserve_lists_issue69(self): """https://github.com/amirziai/flatten/issues/69""" dic = { - 'a': 'a', - 'b': [ - {'b': 1, 'c': 1}, - {'b': 2, 'c': 2}, - {'b': 3, 'c': 3}, - {'b': 4, 'c': 4}, - {'b': 5, 'c': 5}, - {'b': 6, 'c': 6}, - {'b': 7, 'c': 7}, - {'b': 8, 'c': 8}, - {'b': 9, 'c': 9}, - {'b': 10, 'c': 10}, - {'b': 11, 'c': 11}, - {'b': 12, 'c': 12}, - {'b': 13, 'c': 13}, - {'b': 14, 'c': 14}, - {'b': 15, 'c': 15}, - {'b': 16, 'c': 16}, - {'b': 17, 'c': 17}, - {'b': 15, 'c': 18}, - {'b': 16, 'c': 19}, - {'b': 17, 'c': 20} - ] + 'a': 'a', + 'b': [ + {'b': 1, 'c': 1}, + {'b': 2, 'c': 2}, + {'b': 3, 'c': 3}, + {'b': 4, 'c': 4}, + {'b': 5, 'c': 5}, + {'b': 6, 'c': 6}, + {'b': 7, 'c': 7}, + {'b': 8, 'c': 8}, + {'b': 9, 'c': 9}, + {'b': 10, 'c': 10}, + {'b': 11, 'c': 11}, + {'b': 12, 'c': 12}, + {'b': 13, 'c': 13}, + {'b': 14, 'c': 14}, + {'b': 15, 'c': 15}, + {'b': 16, 'c': 16}, + {'b': 17, 'c': 17}, + {'b': 15, 'c': 18}, + {'b': 16, 'c': 19}, + {'b': 17, 'c': 20} + ] } expected = [ @@ -2266,18 +2265,18 @@ def test_flatten_preserve_lists_issue69(self): def test_flatten_preserve_lists_issue72(self): """https://github.com/amirziai/flatten/issues/72""" - dic = {'a': 0, - 'b':[ - {'c': 1, - 'd': [{'e': 1}]}, - {'c': 2, - 'd': [{'e': 2}]}, - {'c': 3, - 'd': [{'e': 3}]}, - {'c': 4, - 'd': [{'e': 4}]}, - {'c': 5, - 'd': [{'e': 5}]}]} + dic = {'a': 0, + 'b': [ + {'c': 1, + 'd': [{'e': 1}]}, + {'c': 2, + 'd': [{'e': 2}]}, + {'c': 3, + 'd': [{'e': 3}]}, + {'c': 4, + 'd': [{'e': 4}]}, + {'c': 5, + 'd': [{'e': 5}]}]} expected = [ {'b.c': 1, From 7f434c4d17061bf0e5fcba6ed27dc7604afcf1ee Mon Sep 17 00:00:00 2001 From: Kan Li Date: Wed, 24 Feb 2021 09:11:17 -0500 Subject: [PATCH 5/5] fixed #69 & #72 --- test_flatten.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test_flatten.py b/test_flatten.py index 5d18328..291393a 100644 --- a/test_flatten.py +++ b/test_flatten.py @@ -2260,7 +2260,10 @@ def test_flatten_preserve_lists_issue69(self): 'b.b': 10, 'a': 'a'}] - actual = flatten_preserve_lists(dic, separator='.', max_list_index=100, max_depth=5) + actual = flatten_preserve_lists(dic, + separator='.', + max_list_index=100, + max_depth=5) self.assertEqual(expected, actual) def test_flatten_preserve_lists_issue72(self): @@ -2295,7 +2298,10 @@ def test_flatten_preserve_lists_issue72(self): 'a': 0, 'b.d': 5}] - actual = flatten_preserve_lists(dic, separator='.', max_list_index=10, max_depth=5) + actual = flatten_preserve_lists(dic, + separator='.', + max_list_index=10, + max_depth=5) self.assertEqual(expected, actual) def test_unflatten_with_list_deep(self):