Skip to content

Commit cd81530

Browse files
committed
use resolve_version in get_dataframe
1 parent 092404f commit cd81530

1 file changed

Lines changed: 6 additions & 8 deletions

File tree

‎cfa/dataops/catalog.py‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -505,14 +505,16 @@ def get_dataframe(
505505
version_blobs, _ = self._get_version_blobs(
506506
version_spec=version_spec, selection=selection, print_version=print_version
507507
)
508+
version_meta = self.resolve_version(
509+
version_spec=version_spec, selection=selection
510+
)
508511
if not version_blobs:
509512
raise ValueError(
510513
f"No blobs found for version '{version_spec}' in container '{self.container}'."
511514
)
512-
name = version_blobs[0]["name"]
513-
file_ext = PurePosixPath(name).suffix.lstrip(".").lower()
514-
path = str(PurePosixPath(name).parent / f"*.{file_ext}")
515-
fullpath = f"az://{self.container}/{path}"
515+
516+
file_ext = version_meta.blob_url.split(".")[-1].lower()
517+
fullpath = version_meta.blob_url
516518
if output in ["pl_lazy", "lazy"]:
517519
if file_ext in ["parquet", "parq"]:
518520
df = pl.scan_parquet(
@@ -525,8 +527,6 @@ def get_dataframe(
525527
# self.ledger_entry(action="read")
526528
return df
527529
elif file_ext == "csv":
528-
path = str(PurePosixPath(name).parent / f"*.{file_ext}")
529-
fullpath = f"az://{self.container}/{path}"
530530
df = pl.scan_csv(
531531
fullpath,
532532
infer_schema_length=None,
@@ -538,8 +538,6 @@ def get_dataframe(
538538
##self.ledger_entry(action="read")
539539
return df
540540
elif file_ext == "ndjson" or file_ext == "jsonl":
541-
path = str(PurePosixPath(name).parent / f"*.{file_ext}")
542-
fullpath = f"az://{self.container}/{path}"
543541
df = pl.scan_ndjson(
544542
fullpath,
545543
infer_schema_length=None,

0 commit comments

Comments
 (0)