diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/checkpointing/HDFSMetadataLog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/checkpointing/HDFSMetadataLog.scala index fac502f75f3a2..b06a90de44c48 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/checkpointing/HDFSMetadataLog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/checkpointing/HDFSMetadataLog.scala @@ -266,10 +266,10 @@ class HDFSMetadataLog[T <: AnyRef : ClassTag]( } /** Return the latest batch id without reading the file. */ - def getLatestBatchId(): Option[Long] = listBatches.sorted.lastOption + def getLatestBatchId(): Option[Long] = listBatches.maxOption override def getLatest(): Option[(Long, T)] = { - listBatches.sorted.lastOption.map { batchId => + listBatches.maxOption.map { batchId => logInfo(log"Getting latest batch ${MDC(BATCH_ID, batchId)}") (batchId, getExistingBatch(batchId)) }