Skip to content
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 12 additions & 3 deletions mosaic/libmosaic/utils/data_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import enum
import logging
from collections import defaultdict
from dataclasses import dataclass, field
from dataclasses import dataclass, field, fields as dataclass_fields
from typing import Any, List, Optional, Union

NUM_GPUS_PER_HOST = 8
Expand Down Expand Up @@ -154,10 +154,19 @@ def from_raw(
)

del raw_modified["frames"]
raw_modified.pop("user_metadata", None)

# Filter to only known TraceEvent fields to prevent future breakage
# from new fields added to PyTorch snapshots (e.g. pool_id, user_metadata)
known_fields = {f.name for f in dataclass_fields(cls)}
explicitly_set = {"classification", "custom_category", "annotation"}
filtered = {
k: v
for k, v in raw_modified.items()
if k in known_fields and k not in explicitly_set
}

return cls(
**raw_modified,
**filtered,
classification=classification,
custom_category=custom_category or "unknown",
annotation=annotation,
Expand Down
Loading