fix: emit block line numbers when with_meta is set (#291)

CHANGELOG.md

@@ -7,7 +7,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## \[Unreleased\]
 
-- Nothing yet.
+### Added
+
+- `BlockView.start_line` and `BlockView.end_line`, so a block's span can be read from the query API without serializing it. `with_meta` puts the numbers in the output dict, which meant reaching them through the label nesting, or through the rule's private `_meta`. Both are `None` for a tree built by the deserializer, which carries no positions. `hq` picks them up through its property accessors: `hq 'resource[*] | .start_line' main.tf`.
+
+### Fixed
+
+- `with_meta` emits `__start_line__` and `__end_line__` again. The option, the `hcl2tojson --with-meta` flag and the migration guide's promise that the v7 keys are "still available" all survived the v8 rewrite; the code that produced the keys did not, leaving the option read nowhere in the package. Blocks are annotated with the same spans 7.3.1 produced for the same input. ([#291](https://github.com/amplify-education/python-hcl2/issues/291))
 
 ## \[8.1.3\] - 2026-08-26
 

docs/01_getting_started.md

@@ -70,7 +70,7 @@ data = loads(text, serialization_options=SerializationOptions(
 | Field | Type | Default | Description                                                                                                                                     |
 |---|---|---|-------------------------------------------------------------------------------------------------------------------------------------------------|
 | `with_comments` | `bool` | `True` | Include comments as `__comments__` and `__inline_comments__` keys (see [Comment Format](#comment-format))                                       |
-| `with_meta` | `bool` | `False` | Add `__start_line__` / `__end_line__` metadata                                                                                                  |
+| `with_meta` | `bool` | `False` | Add `__start_line__` / `__end_line__` metadata to each block, alongside its attributes. Attributes carry no metadata of their own.               |
 | `wrap_objects` | `bool` | `False` | Wrap object values as inline HCL2 strings                                                                                                       |
 | `wrap_tuples` | `bool` | `False` | Wrap tuple values as inline HCL2 strings                                                                                                        |
 | `explicit_blocks` | `bool` | `True` | Add `__is_block__: True` markers to blocks. **Mandatory for JSON->HCL2 deserialization and reconstruction.**                                    |

docs/02_querying.md

@@ -56,6 +56,8 @@ block.block_type    # "resource"
 block.labels        # ["resource", "aws_instance", "main"]
 block.name_labels   # ["aws_instance", "main"]
 block.body          # BodyView
+block.start_line    # 1
+block.end_line      # 12
 ```
 
 | Property / Method | Returns | Description |
@@ -64,6 +66,8 @@ block.body          # BodyView
 | `labels` | `List[str]` | All labels as plain strings |
 | `name_labels` | `List[str]` | Labels after the block type (`labels[1:]`) |
 | `body` | `BodyView` | The block body |
+| `start_line` | `int \| None` | Line the block opens on; `None` for a tree with no positions |
+| `end_line` | `int \| None` | Line the block closes on; `None` for a tree with no positions |
 | `blocks(...)` | `List[BlockView]` | Nested blocks (delegates to body) |
 | `attributes(...)` | `List[AttributeView]` | Nested attributes (delegates to body) |
 | `attribute(name)` | `AttributeView \| None` | Single nested attribute |

docs/04_hq.md

@@ -98,7 +98,7 @@ hq 'x | length' file.tf --value
 
 | View Type | Available Properties |
 |---|---|
-| `BlockView` | `.block_type` (e.g. `"resource"`), `.labels` (all labels including type), `.name_labels` (labels after the block type, e.g. `["aws_instance", "main"]`) |
+| `BlockView` | `.block_type` (e.g. `"resource"`), `.labels` (all labels including type), `.name_labels` (labels after the block type, e.g. `["aws_instance", "main"]`), `.start_line` / `.end_line` (the block's span in the file) |
 | `AttributeView` | `.name` (attribute name), `.value` (serialized value) |
 | `FunctionCallView` | `.name` (function name), `.args` (argument list), `.has_ellipsis` |
 | `ForTupleView` | `.iterator_name`, `.second_iterator_name`, `.iterable`, `.value_expr`, `.has_condition`, `.condition` |

hcl2/const.py

@@ -3,3 +3,5 @@
 IS_BLOCK = "__is_block__"
 COMMENTS_KEY = "__comments__"
 INLINE_COMMENTS_KEY = "__inline_comments__"
+START_LINE = "__start_line__"
+END_LINE = "__end_line__"

hcl2/deserializer.py

@@ -9,7 +9,7 @@
 
 from regex import regex
 
-from hcl2.const import COMMENTS_KEY, INLINE_COMMENTS_KEY, IS_BLOCK
+from hcl2.const import COMMENTS_KEY, END_LINE, INLINE_COMMENTS_KEY, IS_BLOCK, START_LINE
 from hcl2.parser import parser as _get_parser
 from hcl2.rules.abstract import LarkElement, LarkRule
 from hcl2.rules.base import (
@@ -369,7 +369,7 @@ def _deserialize_object_elem(self, key: Any, value: Any) -> ObjectElemRule:
 
     def _is_reserved_key(self, key: str) -> bool:
         """Check if a key is a reserved metadata key that should be skipped during deserialization."""
-        return key in (IS_BLOCK, COMMENTS_KEY, INLINE_COMMENTS_KEY)
+        return key in (IS_BLOCK, COMMENTS_KEY, INLINE_COMMENTS_KEY, START_LINE, END_LINE)
 
     def _is_expression(self, value: Any) -> bool:
         return isinstance(value, str) and value.startswith("${") and value.endswith("}")

hcl2/query/blocks.py

@@ -11,6 +11,19 @@
 from hcl2.utils import SerializationOptions
 
 
+def _meta_line(node: BlockRule, attribute: str) -> Optional[int]:
+    """Read a line number off a block's lark ``Meta``, or None when it has none.
+
+    A tree built by the deserializer rather than the parser carries an empty
+    ``Meta``, whose line attributes do not exist at all.
+    """
+    meta = node._meta  # pylint: disable=protected-access
+    if meta.empty:
+        return None
+    line: int = getattr(meta, attribute)
+    return line
+
+
 def _label_to_str(label) -> str:
     """Convert a block label (IdentifierRule or StringRule) to a plain string."""
     if isinstance(label, IdentifierRule):
@@ -53,6 +66,22 @@ def name_labels(self) -> List[str]:
         """Return labels after the block type (labels[1:]) as plain strings."""
         return self.labels[1:]
 
+    @property
+    def start_line(self) -> Optional[int]:
+        """Return the line the block opens on, or None if it has no position.
+
+        The same number ``with_meta`` reports as ``__start_line__``, without
+        serializing the block to get at it.
+        """
+        node: BlockRule = self._node  # type: ignore[assignment]
+        return _meta_line(node, "line")
+
+    @property
+    def end_line(self) -> Optional[int]:
+        """Return the line the block closes on, or None if it has no position."""
+        node: BlockRule = self._node  # type: ignore[assignment]
+        return _meta_line(node, "end_line")
+
     @property
     def body(self) -> "NodeView":
         """Return the block body as a BodyView."""

hcl2/rules/base.py

@@ -5,7 +5,7 @@
 
 from lark.tree import Meta
 
-from hcl2.const import INLINE_COMMENTS_KEY, IS_BLOCK
+from hcl2.const import END_LINE, INLINE_COMMENTS_KEY, IS_BLOCK, START_LINE
 from hcl2.rules.abstract import LarkRule, LarkToken
 from hcl2.rules.expressions import ExprTermRule
 from hcl2.rules.literal_rules import IdentifierRule
@@ -152,6 +152,13 @@ def serialize(self, options=SerializationOptions(), context=SerializationContext
         result = self._body.serialize(options)
         if options.explicit_blocks:
             result.update({IS_BLOCK: True})
+        if options.with_meta:
+            # Alongside the body, not wrapping it: the keys land on the same
+            # innermost dict the labels nest around, which is where v7 put them.
+            # A tree built by the deserializer carries no positions, so an empty
+            # Meta means "no line numbers to report" rather than line zero.
+            if not self._meta.empty:
+                result.update({START_LINE: self._meta.line, END_LINE: self._meta.end_line})
 
         labels = self._labels
         for label in reversed(labels[1:]):

hcl2/utils.py

@@ -20,7 +20,9 @@ class SerializationOptions:
 
     # Include __comments__ and __inline_comments__ keys in the output.
     with_comments: bool = True
-    # Add __start_line__ and __end_line__ metadata to each block/attribute.
+    # Add __start_line__ and __end_line__ metadata to each block. Attributes get
+    # none: an attribute serializes to its own {name: value} pair, which has
+    # nowhere to hang the keys without changing the shape of the value.
     with_meta: bool = False
     # Serialize nested objects as inline HCL strings (e.g. "${{key = value}}")
     # instead of Python dicts.

test/unit/cli/test_hcl_to_json.py

@@ -120,6 +120,9 @@ def test_with_meta_flag(self):
 
             result = json.loads(stdout.getvalue())
             self.assertIn("resource", result)
+            body = result["resource"][0]['"a"']['"b"']
+            self.assertEqual(body["__start_line__"], 1)
+            self.assertEqual(body["__end_line__"], 3)
 
     def test_no_comments_flag(self):
         hcl_with_comment = "# a comment\nx = 1\n"

test/unit/query/test_blocks.py

@@ -118,3 +118,43 @@ def test_no_adjacent_comments(self):
         block = doc.blocks("resource")[0]
         result = block.to_dict(options=self._OPTS)
         self.assertNotIn("__comments__", result)
+
+
+class TestBlockViewLines(TestCase):
+    """`start_line` / `end_line` report the span `with_meta` serializes.
+
+    The line numbers were reachable only by serializing the block with
+    `with_meta=True` and digging past the label nesting, or by reading the
+    rule's private `_meta`. Both are what these properties replace.
+    """
+
+    NESTED = 'resource "aws_instance" "web" {\n  ami = "ami-1"\n\n  network_interface {\n    x = 0\n  }\n}\n'
+
+    def test_span_of_a_top_level_block(self):
+        block = DocumentView.parse(self.NESTED).blocks("resource")[0]
+        self.assertEqual((block.start_line, block.end_line), (1, 7))
+
+    def test_span_of_a_nested_block(self):
+        block = DocumentView.parse(self.NESTED).blocks("resource")[0]
+        nested = block.blocks("network_interface")[0]
+        self.assertEqual((nested.start_line, nested.end_line), (4, 6))
+
+    def test_an_empty_block_spans_one_line(self):
+        block = DocumentView.parse('variable "x" {}\n').blocks("variable")[0]
+        self.assertEqual((block.start_line, block.end_line), (1, 1))
+
+    def test_agrees_with_with_meta(self):
+        block = DocumentView.parse(self.NESTED).blocks("resource")[0]
+        body = block.to_dict(options=SerializationOptions(with_meta=True))['"aws_instance"']['"web"']
+        self.assertEqual(block.start_line, body["__start_line__"])
+        self.assertEqual(block.end_line, body["__end_line__"])
+
+    def test_a_block_without_a_position_reports_none(self):
+        # A tree built by the deserializer carries an empty Meta.
+        from hcl2.api import from_dict
+        from hcl2.query.body import DocumentView as Doc
+
+        tree = from_dict({"resource": [{"aws_instance": {"web": {"__is_block__": True}}}]})
+        block = Doc(tree).blocks("resource")[0]
+        self.assertIsNone(block.start_line)
+        self.assertIsNone(block.end_line)

test/unit/test_api.py

@@ -48,8 +48,11 @@ def test_with_serialization_options(self):
     def test_with_meta_option(self):
         result = loads(BLOCK_HCL, serialization_options=SerializationOptions(with_meta=True))
         self.assertIn("resource", result)
-        # Verify the option is accepted and produces a dict with expected content
-        self.assertIsInstance(result, dict)
+        # Assert on the metadata itself, not just that the option is accepted:
+        # this test passed throughout #291, when the option emitted nothing.
+        body = result["resource"][0]['"aws_instance"']['"example"']
+        self.assertEqual(body["__start_line__"], 1)
+        self.assertEqual(body["__end_line__"], 3)
 
     def test_block_parsing(self):
         result = loads(BLOCK_HCL)

test/unit/test_with_meta.py

@@ -0,0 +1,133 @@
+# pylint: disable=C0103,C0114,C0115,C0116
+"""Regression tests for GH issue #291: `with_meta` produced no metadata.
+
+`SerializationOptions.with_meta` documents `__start_line__` and `__end_line__`
+keys, `hcl2tojson` exposes it as `--with-meta`, and the v8 migration guide says
+the v7 keys "are still available". None of that was true: v7 emitted the keys
+from `RuleTransformer.block`, and the v8 rewrite moved block serialization to
+`BlockRule.serialize` without carrying them over, leaving the option read
+nowhere in the package.
+
+Line numbers below were checked against python-hcl2 7.3.1 on the same input,
+so the values are v7's, not merely self-consistent.
+"""
+
+from unittest import TestCase
+
+from hcl2.api import dumps, from_dict, loads, serialize
+from hcl2.const import COMMENTS_KEY, END_LINE, INLINE_COMMENTS_KEY, IS_BLOCK, START_LINE
+from hcl2.utils import SerializationOptions
+
+_META = SerializationOptions(with_meta=True)
+
+NESTED_HCL = """resource "aws_instance" "web" {
+  ami = "ami-1"
+
+  network_interface {
+    device_index = 0
+  }
+}
+
+variable "x" {}
+"""
+
+
+class TestWithMetaEmitsLineNumbers(TestCase):
+    def test_block_carries_its_line_span(self):
+        result = loads(NESTED_HCL, serialization_options=_META)
+        body = result["resource"][0]['"aws_instance"']['"web"']
+        self.assertEqual(body[START_LINE], 1)
+        self.assertEqual(body[END_LINE], 7)
+
+    def test_nested_block_carries_its_own_span(self):
+        result = loads(NESTED_HCL, serialization_options=_META)
+        body = result["resource"][0]['"aws_instance"']['"web"']
+        interface = body["network_interface"][0]
+        self.assertEqual(interface[START_LINE], 4)
+        self.assertEqual(interface[END_LINE], 6)
+
+    def test_empty_block_spans_one_line(self):
+        result = loads(NESTED_HCL, serialization_options=_META)
+        body = result["variable"][0]['"x"']
+        self.assertEqual(body[START_LINE], 9)
+        self.assertEqual(body[END_LINE], 9)
+
+    def test_off_by_default(self):
+        result = loads(NESTED_HCL)
+        body = result["resource"][0]['"aws_instance"']['"web"']
+        self.assertNotIn(START_LINE, body)
+        self.assertNotIn(END_LINE, body)
+
+    def test_attributes_get_no_metadata(self):
+        # An attribute serializes to its own {name: value} pair, so there is
+        # nowhere to put the keys. v7 did not annotate attributes either.
+        result = loads("x = 1\n", serialization_options=_META)
+        self.assertEqual(result, {"x": 1})
+
+    def test_independent_of_explicit_blocks(self):
+        options = SerializationOptions(with_meta=True, explicit_blocks=False)
+        result = loads(NESTED_HCL, serialization_options=options)
+        body = result["resource"][0]['"aws_instance"']['"web"']
+        self.assertNotIn(IS_BLOCK, body)
+        self.assertEqual(body[START_LINE], 1)
+
+
+class TestWithMetaRoundTrip(TestCase):
+    """The keys are metadata, so `dumps()` must not write them back as HCL."""
+
+    def test_metadata_keys_are_not_emitted_as_attributes(self):
+        data = loads(NESTED_HCL, serialization_options=_META)
+        hcl = dumps(data)
+        self.assertNotIn(START_LINE, hcl)
+        self.assertNotIn(END_LINE, hcl)
+
+    def test_round_trip_matches_output_without_metadata(self):
+        with_meta = dumps(loads(NESTED_HCL, serialization_options=_META))
+        without = dumps(loads(NESTED_HCL))
+        self.assertEqual(with_meta, without)
+
+    def test_a_tree_without_positions_reports_no_lines(self):
+        # A tree built by the deserializer carries an empty Meta. Asking for
+        # metadata there must skip the keys rather than raise or invent zeros.
+        tree = from_dict({"resource": [{"aws_instance": {"web": {IS_BLOCK: True}}}]})
+        result = serialize(tree, serialization_options=_META)
+        body = result["resource"][0]["aws_instance"]["web"]
+        self.assertNotIn(START_LINE, body)
+        self.assertNotIn(END_LINE, body)
+
+
+class TestUserAttributesNamedLikeMetadata(TestCase):
+    """An attribute genuinely named `__start_line__` collides with the metadata.
+
+    The keys are carried in-band, in the same dict as the block's attributes,
+    which is where v7 put them and what the migration guide promises. That has
+    a cost: the deserializer cannot tell a metadata key it wrote from an
+    attribute the document really declared, so it drops both -- exactly as it
+    already dropped `__is_block__` and `__comments__` before these two keys
+    existed. `with_meta` additionally overwrites such an attribute.
+
+    These pin the behaviour rather than bless it. Anything that made the
+    metadata unambiguous would have to move all five keys out of band, which is
+    a breaking change to the serialized shape, not a fix to this option.
+    """
+
+    RESERVED = (START_LINE, END_LINE, IS_BLOCK, COMMENTS_KEY, INLINE_COMMENTS_KEY)
+
+    def test_a_reserved_name_does_not_survive_a_round_trip(self):
+        for key in self.RESERVED:
+            with self.subTest(key=key):
+                hcl = f'block "a" {{\n  {key} = 99\n  keep = 1\n}}\n'
+                written = dumps(loads(hcl))
+                self.assertNotIn(key, written)
+                self.assertIn("keep", written)
+
+    def test_with_meta_overwrites_an_attribute_of_the_same_name(self):
+        hcl = 'block "a" {\n  __start_line__ = 99\n}\n'
+        body = loads(hcl, serialization_options=_META)["block"][0]['"a"']
+        self.assertEqual(body[START_LINE], 1)
+
+    def test_an_ordinary_dunder_attribute_is_untouched(self):
+        # Only the five names are reserved; nothing about the leading
+        # underscores makes an attribute metadata.
+        hcl = 'block "a" {\n  __line__ = 99\n}\n'
+        self.assertIn("__line__", dumps(loads(hcl)))