diff --git a/flatten_json/__init__.py b/flatten_json/__init__.py index 9ab02ec..c58267e 100644 --- a/flatten_json/__init__.py +++ b/flatten_json/__init__.py @@ -201,7 +201,11 @@ def _flatten(object_, key): else: flattened_dict[key] = object_ - def _flatten_low_entropy(object_, key, cur_depth, max_depth_inner): + def _flatten_low_entropy(object_, + key, + cur_depth, + cur_list_depth, + max_depth_inner): """ For dict, list and set objects_ calls itself on the elements and for other types assigns the object_ to @@ -217,7 +221,7 @@ def _flatten_low_entropy(object_, key, cur_depth, max_depth_inner): # write latest child as value if max_depth exceeded if cur_depth > max_depth_inner: global_max_record = int(max(list( - list_prebuilt_flattened_dict.keys()))) + list_prebuilt_flattened_dict.keys()), key=int)) for d in list_prebuilt_flattened_dict[str(global_max_record)]: d[key] = object_ @@ -225,7 +229,7 @@ def _flatten_low_entropy(object_, key, cur_depth, max_depth_inner): # Empty object can't be iterated, take as is if not object_: global_max_record = int(max(list( - list_prebuilt_flattened_dict.keys()))) + list_prebuilt_flattened_dict.keys()), key=int)) for d in list_prebuilt_flattened_dict[str(global_max_record)]: d[key] = object_ @@ -239,7 +243,7 @@ def _flatten_low_entropy(object_, key, cur_depth, max_depth_inner): and not (isinstance(object_[first_key], dict) or isinstance(object_[first_key], list)): global_max_record = int(max(list( - list_prebuilt_flattened_dict.keys()))) + list_prebuilt_flattened_dict.keys()), key=int)) for d in list_prebuilt_flattened_dict[ str(global_max_record) @@ -261,10 +265,13 @@ def _flatten_low_entropy(object_, key, cur_depth, max_depth_inner): object_key, replace_separators=replace_separators), cur_depth, + cur_list_depth, max_depth_inner) # lists could go into rows, like in a relational database elif isinstance(object_, list) or isinstance(object_, set): + cur_list_depth = cur_list_depth + 1 + if debug: print("\nparent key of list:", key, "| length: ", @@ -273,7 +280,7 @@ def _flatten_low_entropy(object_, key, cur_depth, max_depth_inner): # need to remember global list state when we entered # this recursion global_max_record_start = int(max(list( - list_prebuilt_flattened_dict.keys()))) + list_prebuilt_flattened_dict.keys()), key=int)) entry = copy.deepcopy(list_prebuilt_flattened_dict[ str(global_max_record_start) ]) @@ -294,27 +301,32 @@ def _flatten_low_entropy(object_, key, cur_depth, max_depth_inner): # start from second element, 1st element is like column if index > 0: global_max_record = int(max(list( - list_prebuilt_flattened_dict.keys()))) + list_prebuilt_flattened_dict.keys()), key=int)) list_prebuilt_flattened_dict[ str(global_max_record + 1) ] = copy.deepcopy(entry) - _flatten_low_entropy(item, key, cur_depth, + _flatten_low_entropy(item, + key, + cur_depth, + cur_list_depth, max_depth_inner) else: pass - list_prebuilt_flattened_dict['0'] = \ - [subel for k, v in - sorted(list_prebuilt_flattened_dict.items()) - for idx, subel in enumerate(v)] + cur_list_depth = cur_list_depth - 1 + if cur_list_depth == 0: + list_prebuilt_flattened_dict['0'] = \ + [subel for k, v in + sorted(list_prebuilt_flattened_dict.items()) + for idx, subel in enumerate(v)] - for key in list(sorted(list_prebuilt_flattened_dict.keys())): - if key != '0': - del list_prebuilt_flattened_dict[key] - if debug: - print("collapsed global list") + for key in list(sorted(list_prebuilt_flattened_dict.keys())): + if key != '0': + del list_prebuilt_flattened_dict[key] + if debug: + print("collapsed global list") # Anything left take as is, assuming you hit the end of the line. else: @@ -322,7 +334,7 @@ def _flatten_low_entropy(object_, key, cur_depth, max_depth_inner): # a list of prebuilt_flattened_dict by now # so need to update them all. global_max_record = int(max(list( - list_prebuilt_flattened_dict.keys()))) + list_prebuilt_flattened_dict.keys()), key=int)) for d in list_prebuilt_flattened_dict[str(global_max_record)]: d[key] = object_ @@ -348,7 +360,10 @@ def _flatten_low_entropy(object_, key, cur_depth, max_depth_inner): # initialize global record list list_prebuilt_flattened_dict = {'0': [prebuilt_flattened_dict]} - _flatten_low_entropy(nested_dict, None, cur_depth=0, + _flatten_low_entropy(nested_dict, + None, + cur_depth=0, + cur_list_depth=0, max_depth_inner=max_depth) return list_prebuilt_flattened_dict['0'] diff --git a/test_flatten.py b/test_flatten.py index cbb24cb..291393a 100644 --- a/test_flatten.py +++ b/test_flatten.py @@ -2170,6 +2170,140 @@ def test_flatten_preserve_lists_issue43(self): actual = flatten_preserve_lists(dic, max_list_index=50, max_depth=10) self.assertEqual(expected, actual) + def test_flatten_preserve_lists_issue69(self): + """https://github.com/amirziai/flatten/issues/69""" + dic = { + 'a': 'a', + 'b': [ + {'b': 1, 'c': 1}, + {'b': 2, 'c': 2}, + {'b': 3, 'c': 3}, + {'b': 4, 'c': 4}, + {'b': 5, 'c': 5}, + {'b': 6, 'c': 6}, + {'b': 7, 'c': 7}, + {'b': 8, 'c': 8}, + {'b': 9, 'c': 9}, + {'b': 10, 'c': 10}, + {'b': 11, 'c': 11}, + {'b': 12, 'c': 12}, + {'b': 13, 'c': 13}, + {'b': 14, 'c': 14}, + {'b': 15, 'c': 15}, + {'b': 16, 'c': 16}, + {'b': 17, 'c': 17}, + {'b': 15, 'c': 18}, + {'b': 16, 'c': 19}, + {'b': 17, 'c': 20} + ] + } + + expected = [ + {'b.c': 1, + 'b.b': 1, + 'a': 'a'}, + {'b.c': 2, + 'b.b': 2, + 'a': 'a'}, + {'b.c': 11, + 'b.b': 11, + 'a': 'a'}, + {'b.c': 12, + 'b.b': 12, + 'a': 'a'}, + {'b.c': 13, + 'b.b': 13, + 'a': 'a'}, + {'b.c': 14, + 'b.b': 14, + 'a': 'a'}, + {'b.c': 15, + 'b.b': 15, + 'a': 'a'}, + {'b.c': 16, + 'b.b': 16, + 'a': 'a'}, + {'b.c': 17, + 'b.b': 17, + 'a': 'a'}, + {'b.c': 18, + 'b.b': 15, + 'a': 'a'}, + {'b.c': 19, + 'b.b': 16, + 'a': 'a'}, + {'b.c': 20, + 'b.b': 17, + 'a': 'a'}, + {'b.c': 3, + 'b.b': 3, + 'a': 'a'}, + {'b.c': 4, + 'b.b': 4, + 'a': 'a'}, + {'b.c': 5, + 'b.b': 5, + 'a': 'a'}, + {'b.c': 6, + 'b.b': 6, + 'a': 'a'}, + {'b.c': 7, + 'b.b': 7, + 'a': 'a'}, + {'b.c': 8, + 'b.b': 8, + 'a': 'a'}, + {'b.c': 9, + 'b.b': 9, + 'a': 'a'}, + {'b.c': 10, + 'b.b': 10, + 'a': 'a'}] + + actual = flatten_preserve_lists(dic, + separator='.', + max_list_index=100, + max_depth=5) + self.assertEqual(expected, actual) + + def test_flatten_preserve_lists_issue72(self): + """https://github.com/amirziai/flatten/issues/72""" + dic = {'a': 0, + 'b': [ + {'c': 1, + 'd': [{'e': 1}]}, + {'c': 2, + 'd': [{'e': 2}]}, + {'c': 3, + 'd': [{'e': 3}]}, + {'c': 4, + 'd': [{'e': 4}]}, + {'c': 5, + 'd': [{'e': 5}]}]} + + expected = [ + {'b.c': 1, + 'a': 0, + 'b.d': 1}, + {'b.c': 2, + 'a': 0, + 'b.d': 2}, + {'b.c': 3, + 'a': 0, + 'b.d': 3}, + {'b.c': 4, + 'a': 0, + 'b.d': 4}, + {'b.c': 5, + 'a': 0, + 'b.d': 5}] + + actual = flatten_preserve_lists(dic, + separator='.', + max_list_index=10, + max_depth=5) + self.assertEqual(expected, actual) + def test_unflatten_with_list_deep(self): dic = {'a': [ {'b': [{'c': [{'a': 5, 'b': {'a': [1, 2, 3]}, 'c': {'x': 3}}]}]}]}