Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,10 @@ import pekko.util.ByteString
ByteString(endFileWord)

def isStartingByteString(b: ByteString): Boolean =
b.utf8String.startsWith(startFileWord)
b.size >= 7 && b.slice(0, 7).utf8String == startFileWord

def isEndingByteString(b: ByteString): Boolean =
b.utf8String == endFileWord
b.size == 5 && b.utf8String == endFileWord

def getPathFromStartingByteString(b: ByteString): String = {
val splitted = b.utf8String.split(separator)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,18 @@ import pekko.util.{ ByteString, ByteStringBuilder }
* INTERNAL API
*/
@InternalApi private[file] final class ZipArchiveFlowStage(
val shape: FlowShape[ByteString, ByteString]) extends GraphStageLogic(shape) {
val shape: FlowShape[ByteString, ByteString],
deflateCompression: Option[Int] = None) extends GraphStageLogic(shape) {

private def in = shape.in
private def out = shape.out

private val builder = new ByteStringBuilder()
private val zip = new ZipOutputStream(builder.asOutputStream)

override def preStart(): Unit =
deflateCompression.foreach(l => zip.setLevel(l))

private var emptyStream = true

setHandler(
Expand Down Expand Up @@ -88,7 +93,8 @@ import pekko.util.{ ByteString, ByteStringBuilder }
/**
* INTERNAL API
*/
@InternalApi private[file] final class ZipArchiveFlow extends GraphStage[FlowShape[ByteString, ByteString]] {
@InternalApi private[file] final class ZipArchiveFlow(deflateCompression: Option[Int] = None)
extends GraphStage[FlowShape[ByteString, ByteString]] {

val in: Inlet[ByteString] = Inlet(Logging.simpleName(this) + ".in")
val out: Outlet[ByteString] = Outlet(Logging.simpleName(this) + ".out")
Expand All @@ -99,5 +105,5 @@ import pekko.util.{ ByteString, ByteStringBuilder }
override val shape: FlowShape[ByteString, ByteString] = FlowShape(in, out)

override def createLogic(inheritedAttributes: Attributes): GraphStageLogic =
new ZipArchiveFlowStage(shape)
new ZipArchiveFlowStage(shape, deflateCompression)
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,9 @@ import pekko.util.ByteString
*/
@InternalApi private[file] object ZipArchiveManager {

def zipFlow(): Flow[(ArchiveMetadata, Source[ByteString, Any]), ByteString, NotUsed] = {
val archiveZipFlow = new ZipArchiveFlow()
def zipFlow(
deflateCompression: Option[Int] = None): Flow[(ArchiveMetadata, Source[ByteString, Any]), ByteString, NotUsed] = {
val archiveZipFlow = new ZipArchiveFlow(deflateCompression)
Flow[(ArchiveMetadata, Source[ByteString, Any])]
.flatMapConcat {
case (metadata, stream) =>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,22 @@ object Archive {

/**
* Flow for compressing multiple files into one ZIP file.
* @param deflateCompression optional compression level, 0-9, where 0 is no compression and 9 is maximum compression.
* If not specified, the default compression level of the underlying library will be used.
* @since 2.0.0
*/
def zip(): Flow[Pair[ArchiveMetadata, Source[ByteString, NotUsed]], ByteString, NotUsed] =
def zip(
deflateCompression: Option[Int]): Flow[Pair[ArchiveMetadata, Source[ByteString, NotUsed]], ByteString, NotUsed] =
Flow
.create[Pair[ArchiveMetadata, Source[ByteString, NotUsed]]]()
.map(func(pair => (pair.first, pair.second.asScala)))
.via(scaladsl.Archive.zip().asJava)
.via(scaladsl.Archive.zip(deflateCompression).asJava)

/**
* Flow for compressing multiple files into one ZIP file.
*/
def zip(): Flow[Pair[ArchiveMetadata, Source[ByteString, NotUsed]], ByteString, NotUsed] =
zip(None)

/**
* Flow for reading ZIP files.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,21 @@ import java.nio.charset.{ Charset, StandardCharsets }
*/
object Archive {

/**
* Flow for compressing multiple files into one ZIP file.
* @param deflateCompression optional compression level, 0-9, where 0 is no compression and 9 is maximum compression.
* If not specified, the default compression level of the underlying library will be used.
* @since 2.0.0
*/
def zip(
deflateCompression: Option[Int]): Flow[(ArchiveMetadata, Source[ByteString, Any]), ByteString, NotUsed] =
ZipArchiveManager.zipFlow(deflateCompression)

/**
* Flow for compressing multiple files into one ZIP file.
*/
def zip(): Flow[(ArchiveMetadata, Source[ByteString, Any]), ByteString, NotUsed] =
ZipArchiveManager.zipFlow()
zip(None)

/**
* Flow for reading ZIP files.
Expand Down
14 changes: 14 additions & 0 deletions file/src/test/scala/docs/scaladsl/ArchiveSpec.scala
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ package docs.scaladsl

import java.io._
import java.nio.file.{ Files, Path, Paths }
import java.util.zip.Deflater
import org.apache.pekko
import pekko.actor.ActorSystem
import pekko.stream.connectors.file.ArchiveMetadata
Expand Down Expand Up @@ -121,6 +122,19 @@ class ArchiveSpec
archiveHelper.unzip(pekkoZipped.futureValue).asScala shouldBe inputFiles
}

"archive files with compression flag" in {
val inputFiles = generateInputFiles(5, 100)
val inputStream = filesToStream(inputFiles)
val zipFlow = Archive.zip(Some(Deflater.NO_COMPRESSION))

val pekkoZipped: Future[ByteString] =
inputStream
.via(zipFlow)
.runWith(Sink.fold(ByteString.empty)(_ ++ _))

archiveHelper.unzip(pekkoZipped.futureValue).asScala shouldBe inputFiles
}

"unarchive files" in {
val inputFiles = generateInputFiles(5, 100)
val inputStream = filesToStream(inputFiles)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import pekko.stream.scaladsl.Keep
import pekko.stream.testkit.scaladsl.{ TestSink, TestSource }
import pekko.testkit.TestKit
import pekko.util.ByteString
import java.util.zip.Deflater
import org.scalatest.BeforeAndAfterAll
import org.scalatest.wordspec.AnyWordSpecLike

Expand Down Expand Up @@ -50,6 +51,27 @@ class ZipArchiveFlowTest
downstream.expectComplete()
}
}

"compression flag given and stream ends" should {
"emit element only when downstream requests" in {
val (upstream, downstream) =
TestSource[ByteString]()
.via(new ZipArchiveFlow(Some(Deflater.NO_COMPRESSION)))
.toMat(TestSink())(Keep.both)
.run()

upstream.sendNext(FileByteStringSeparators.createStartingByteString("test"))
upstream.sendNext(ByteString(1))
upstream.sendNext(FileByteStringSeparators.createEndingByteString())
upstream.sendComplete()

downstream.request(2)
downstream.expectNextN(2)
downstream.request(1)
downstream.expectNextN(1)
downstream.expectComplete()
}
}
}

override def afterAll(): Unit = {
Expand Down