High level data model to support full model editing

[spyoungtech]

Right now, it is only possible to edit comments by using the low-level model objects directly.

What we want to support is a high-level round-trip loader/dumper so that round-trip preservation/editing of comments/whitespace along with actual data modifications is made as simple as possible.

[poolfeast]

For my own usage I've made an object to handle the connection between AST nodes and python dict representations.

AKA it allows you to find the nodes that relate to a specific piece of data. I use this interface to manage the whitespace/comments around specific nodes.

I haven't included a complete executable example but hopefully this is enough to help people doing the similar whitespace/comment editing.

The AST stuff was super useful, thanks for your work!

import json5
import copy

class Browser:
    """Navigate through the AST with a dict-like interface
    Methods have been added to make this object "dict-like" enough that it can be passed into some libraries expecting dictionaries. Like jsonpath_ng.
    """

    def __init__(self, node, parent=None):
        # "return values" - return values rather than Browser objects when we reach a leaf in the tree.
        self.return_values = parent.return_values if parent else True
        self.container_ast_node = node
        self.parent = parent

        if isinstance(node, json5.model.JSONText):
            self.data_ast_node = node.value  # can still be useful
            self.associated_ast_nodes = []
        elif isinstance(node, json5.model.KeyValuePair):
            self.data_ast_node = node.value
            # Would otherwise not be linked to any dict representation
            self.associated_ast_nodes = [node.key]
        else:
            # There is no "label" node so data and container are the same thing
            self.data_ast_node = node
            self.associated_ast_nodes = []

    def set(self, value):
        current_node = self.data_ast_node
        new_node = json5.dumper.modelize(value)
        # ast_node is a linked "reference" to the same instance in the original AST tree
        # set each member separately, because setting the whole ast_node would not update the OG AST object.
        assert isinstance(new_node, type(
            current_node)), "You can't change the json type of what you're setting, set one level up to change type"
        for field in set(current_node._fields).difference(current_node.excluded_names):
            setattr(current_node, field, getattr(new_node, field))

    def __setitem__(self, key, value):
        self.get(key).set(value)

    # This causes the print representation to be a dict. Makes REPL usage better.
    def __repr__(self): return self.user_rep().__repr__()

    # Methods for jsonpath_ng compatibility
    def get(self, key, default=None):  # kw arg necessary to maintain standard dict interface
        if type(key) is int:
            # supports JSON arrays, allows key to be an integer
            return Browser(self.data_ast_node.values[key], self)
        else:
            return Browser(self.data_ast_node.key_value_pairs[self.keys().index(key)], self)

    def __getitem__(self, key):
        obj = self.get(key)
        # The assumption that length 1 means value node is not always true/correct
        if self.return_values and len(obj) == 1:
            # Automatically return node values when not operating in internal mode
            return obj.user_rep()
        # Otherwise return another instance
        return obj

    def keys(self): return [key.characters for key in self.data_ast_node.keys]

    def __len__(self):
        if hasattr(self.data_ast_node, "values"):
            return len(self.data_ast_node.values)
        else:
            # Length of 1 indicates that this is a "value", a leaf on the tree
            return 1

    # Methods for jsonschema compatibility
    def __contains__(self, key): return True if key in self.keys() else False

    def __iter__(self):
        # Return lookup value, ie, typical dict behaviour
        if isinstance(self.data_ast_node, json5.model.JSONObject):
            yield from self.keys()
        elif isinstance(self.data_ast_node, json5.model.JSONArray):
            yield from range(len(self))

    # Otherwise cookas accidentally calling len() on Value nodes
    def __bool__(self): return True

    def items(self):  # copies dict method
        for key in self:
            yield (key, self[key])

    def user_rep(self):
        """Return as python native types, IE floats, etc. Its often not necessary to call this manually"""
        return json5.loader.DefaultLoader().load(self.data_ast_node)

    def node_mode(self):
        """Return a new Browser that won't automatically switch to user_rep"""
        new = copy.copy(
            self)  # or return_values flag will be shared between both
        new.return_values = False
        return new

    def root(self):
        """Used to write updates the json, as the whole tree has to be written from the root"""
        node = self
        while node.parent is not None:
            node = node.parent
        return node

    def iter_child_nodes(self):
        """Designed to list every ast node that could affect characters "inside" our element"""
        # if node.container_ast_node is not node.data_ast_node: yield node.container_ast_node
        yield from self.associated_ast_nodes
        yield self.data_ast_node
        for key in self.node_mode():
            yield from self[key].iter_child_nodes()

Usage

data_file = open(data_file, "r+")
ast = json5.loader.load(data_file, loader=json5.loader.ModelLoader())
doc_root = Browser(ast)

# If this is a leaf, Browser will return the actual value here.
current_value = doc_root["hello"][5]["hello"]

# Setting the value will erase the comments only at this particular location
doc_root["hello"][5]["hello"] = {"More":"Data"}

# Write back the new data
data_file.seek(0)
json5.dump(doc_root.container_ast_node, data_file, dumper=json5.dumper.ModelDumper())
data_file.truncate()