Stream JSON report I/O

invert-gradle-plugin/src/main/kotlin/com/squareup/invert/internal/report/InvertJsonStreams.kt

@@ -0,0 +1,74 @@
+package com.squareup.invert.internal.report
+
+import com.squareup.invert.models.InvertSerialization.InvertJson
+import kotlinx.serialization.DeserializationStrategy
+import kotlinx.serialization.ExperimentalSerializationApi
+import kotlinx.serialization.SerializationStrategy
+import kotlinx.serialization.json.decodeFromStream
+import kotlinx.serialization.json.encodeToStream
+import okio.buffer
+import okio.sink
+import okio.source
+import java.io.File
+import java.io.OutputStream
+
+internal object InvertJsonStreams {
+
+  @OptIn(ExperimentalSerializationApi::class)
+  fun <T> write(
+    file: File,
+    serializer: SerializationStrategy<T>,
+    value: T,
+  ) {
+    file.sink().buffer().use { sink ->
+      write(sink.outputStream(), serializer, value)
+    }
+  }
+
+  @OptIn(ExperimentalSerializationApi::class)
+  fun <T> write(
+    stream: OutputStream,
+    serializer: SerializationStrategy<T>,
+    value: T,
+  ) {
+    InvertJson.encodeToStream(
+      serializer = serializer,
+      value = value,
+      stream = stream,
+    )
+  }
+
+  @OptIn(ExperimentalSerializationApi::class)
+  fun <T> read(
+    file: File,
+    deserializer: DeserializationStrategy<T>,
+  ): T = file.source().buffer().use { source ->
+    InvertJson.decodeFromStream(
+      deserializer = deserializer,
+      stream = source.inputStream(),
+    )
+  }
+
+  @OptIn(ExperimentalSerializationApi::class)
+  fun <T> encodedSizeInBytes(
+    serializer: SerializationStrategy<T>,
+    value: T,
+  ): Long {
+    val output = CountingOutputStream()
+    write(output, serializer, value)
+    return output.bytesWritten
+  }
+
+  private class CountingOutputStream : OutputStream() {
+    var bytesWritten = 0L
+      private set
+
+    override fun write(value: Int) {
+      bytesWritten++
+    }
+
+    override fun write(bytes: ByteArray, offset: Int, length: Int) {
+      bytesWritten += length
+    }
+  }
+}

invert-gradle-plugin/src/main/kotlin/com/squareup/invert/internal/report/InvertReportFileUtils.kt

@@ -5,7 +5,7 @@ import com.squareup.invert.internal.models.CollectedDependenciesForProject
 import com.squareup.invert.internal.models.CollectedOwnershipForProject
 import com.squareup.invert.internal.models.CollectedPluginsForProject
 import com.squareup.invert.internal.models.CollectedStatsForProject
-import com.squareup.invert.models.InvertSerialization.InvertJson
+import kotlinx.serialization.DeserializationStrategy
 import java.io.File
 
 object InvertReportFileUtils {
@@ -15,57 +15,45 @@ object InvertReportFileUtils {
    */
   fun readCollectedStatsForAllProjectsFromDisk(
     collectedStatFile: File
-  ): CollectedStatsForProject? {
-    return if (collectedStatFile.exists() && collectedStatFile.nameWithoutExtension.isNotBlank()) {
-      InvertJson.decodeFromString(CollectedStatsForProject.serializer(), collectedStatFile.readText())
-    } else {
-      null
-    }
-  }
+  ): CollectedStatsForProject? = readIfPresent(
+    file = collectedStatFile,
+    deserializer = CollectedStatsForProject.serializer(),
+  )
 
   fun readCollectedOwnershipForAllProjectsFromDisk(
     ownersFile: File
-  ): CollectedOwnershipForProject? {
-    return if (ownersFile.exists() && ownersFile.nameWithoutExtension.isNotBlank()) {
-      InvertJson.decodeFromString(
-        deserializer = CollectedOwnershipForProject.serializer(),
-        string = ownersFile.readText()
-      )
-    } else {
-      null
-    }
-  }
+  ): CollectedOwnershipForProject? = readIfPresent(
+    file = ownersFile,
+    deserializer = CollectedOwnershipForProject.serializer(),
+  )
 
-  fun readCollectedPluginsForAllModules(pluginsFile: File): CollectedPluginsForProject? {
-    return if (pluginsFile.exists() && pluginsFile.nameWithoutExtension.isNotBlank()) {
-      InvertJson.decodeFromString(
-        deserializer = CollectedPluginsForProject.serializer(),
-        string = pluginsFile.readText()
-      )
-    } else {
-      null
-    }
-  }
+  fun readCollectedPluginsForAllModules(
+    pluginsFile: File
+  ): CollectedPluginsForProject? = readIfPresent(
+    file = pluginsFile,
+    deserializer = CollectedPluginsForProject.serializer(),
+  )
 
-  fun buildModuleToFeaturesMap(dependenciesFile: File): CollectedDependenciesForProject? {
-    return if (dependenciesFile.exists() && dependenciesFile.nameWithoutExtension.isNotBlank()) {
-      InvertJson.decodeFromString(
-        deserializer = CollectedDependenciesForProject.serializer(),
-        string = dependenciesFile.readText()
-      )
-    } else {
-      null
-    }
-  }
+  fun buildModuleToFeaturesMap(
+    dependenciesFile: File
+  ): CollectedDependenciesForProject? = readIfPresent(
+    file = dependenciesFile,
+    deserializer = CollectedDependenciesForProject.serializer(),
+  )
 
   fun readCollectedConfigurationsForAllModules(
     collectedConfigurationsFile: File
-  ): CollectedConfigurationsForProject? {
-    return if (collectedConfigurationsFile.exists() && collectedConfigurationsFile.nameWithoutExtension.isNotBlank()) {
-      InvertJson.decodeFromString(
-        deserializer = CollectedConfigurationsForProject.serializer(),
-        string = collectedConfigurationsFile.readText()
-      )
+  ): CollectedConfigurationsForProject? = readIfPresent(
+    file = collectedConfigurationsFile,
+    deserializer = CollectedConfigurationsForProject.serializer(),
+  )
+
+  private fun <T> readIfPresent(
+    file: File,
+    deserializer: DeserializationStrategy<T>,
+  ): T? {
+    return if (file.exists() && file.nameWithoutExtension.isNotBlank()) {
+      InvertJsonStreams.read(file, deserializer)
     } else {
       null
     }

invert-gradle-plugin/src/main/kotlin/com/squareup/invert/internal/report/js/InvertJsReportWriter.kt

@@ -4,8 +4,8 @@ import com.squareup.invert.internal.InvertFileUtils
 import com.squareup.invert.internal.models.CollectedConfigurationsForProject
 import com.squareup.invert.internal.models.CollectedDependenciesForProject
 import com.squareup.invert.internal.models.CollectedPluginsForProject
+import com.squareup.invert.internal.report.InvertJsonStreams
 import com.squareup.invert.logging.InvertLogger
-import com.squareup.invert.models.InvertSerialization.InvertJson
 import com.squareup.invert.models.ModulePath
 import com.squareup.invert.models.Stat
 import com.squareup.invert.models.js.ChunkManifest
@@ -51,15 +51,15 @@ class InvertJsReportWriter(
     val statsMapData: Set<StatJsReportModel> = InvertJsReportUtils.buildStatJsReportModelList(allProjectsStatsData)
     statsMapData.forEach { statModel ->
       val fileKey = JsReportFileKey.STAT.key + "_" + statModel.statInfo.key
-      val serializedJson = InvertJson.encodeToString(StatJsReportModel.serializer(), statModel)
-      writeJsFileInDir(
+      val legacyStatFile = writeJsFileInDir(
         fileKey = fileKey,
-        serializedJson = serializedJson,
+        serializer = StatJsReportModel.serializer(),
+        value = statModel,
       )
       writeChunkedStatIfNeeded(
         fileKey = fileKey,
         statModel = statModel,
-        serializedJsonLength = serializedJson.length,
+        serializedJsonSizeBytes = serializedJsonSizeInBytes(legacyStatFile, fileKey),
       )
     }
 
@@ -204,9 +204,12 @@ class InvertJsReportWriter(
   internal fun writeChunkedStatIfNeeded(
     fileKey: String,
     statModel: StatJsReportModel,
-    serializedJsonLength: Int = InvertJson.encodeToString(StatJsReportModel.serializer(), statModel).length,
+    serializedJsonSizeBytes: Long = InvertJsonStreams.encodedSizeInBytes(
+      StatJsReportModel.serializer(),
+      statModel,
+    ),
   ) {
-    if (serializedJsonLength < CHUNK_THRESHOLD_BYTES) return
+    if (serializedJsonSizeBytes < CHUNK_THRESHOLD_BYTES) return
 
     val sortedModuleKeys = statModel.statsByModule.keys.sorted()
     if (sortedModuleKeys.isEmpty()) return
@@ -223,9 +226,7 @@ class InvertJsReportWriter(
         directory = rootBuildHtmlReportDir,
         filename = chunkFileNames[index]
       )
-      chunkFile.sink().buffer().use { sink ->
-        sink.writeUtf8(InvertJson.encodeToString(StatJsReportModel.serializer(), chunkModel))
-      }
+      InvertJsonStreams.write(chunkFile, StatJsReportModel.serializer(), chunkModel)
       logger.lifecycle("Writing chunk ${index + 1}/${chunks.size} for $fileKey to file://${chunkFile.canonicalPath}")
     }
 
@@ -239,9 +240,7 @@ class InvertJsReportWriter(
       directory = rootBuildHtmlReportDir,
       filename = "$fileKey.manifest.json"
     )
-    manifestFile.sink().buffer().use { sink ->
-      sink.writeUtf8(InvertJson.encodeToString(ChunkManifest.serializer(), manifest))
-    }
+    InvertJsonStreams.write(manifestFile, ChunkManifest.serializer(), manifest)
     logger.lifecycle("Writing chunk manifest for $fileKey (${chunks.size} chunks) to file://${manifestFile.canonicalPath}")
   }
 
@@ -272,10 +271,10 @@ class InvertJsReportWriter(
 
       // Pre-compute the fixed overhead of a chunk (statInfo + JSON structure) so we only
       // measure the incremental cost of each module entry in the loop.
-      val emptyChunkSize = InvertJson.encodeToString(
+      val emptyChunkSize = InvertJsonStreams.encodedSizeInBytes(
         StatJsReportModel.serializer(),
         StatJsReportModel(statInfo = statModel.statInfo, statsByModule = emptyMap()),
-      ).length.toLong()
+      )
 
       for (moduleKey in sortedModuleKeys) {
         val stat = statModel.statsByModule[moduleKey] ?: continue
@@ -284,7 +283,10 @@ class InvertJsReportWriter(
           statInfo = statModel.statInfo,
           statsByModule = mapOf(moduleKey to stat),
         )
-        val entrySize = InvertJson.encodeToString(StatJsReportModel.serializer(), singleEntryModel).length.toLong() - emptyChunkSize
+        val entrySize = InvertJsonStreams.encodedSizeInBytes(
+          StatJsReportModel.serializer(),
+          singleEntryModel,
+        ) - emptyChunkSize
 
         if (currentModules.isNotEmpty() && currentSize + entrySize > TARGET_CHUNK_SIZE_BYTES) {
           chunks.add(StatJsReportModel(statInfo = statModel.statInfo, statsByModule = currentModules))
@@ -329,17 +331,20 @@ class InvertJsReportWriter(
         parentFile.mkdirs()
       }
       sink().buffer().use { sink ->
-        sink.writeUtf8(
-          invertJsGlobalVariableAssignment(
-            fileKey = fileKey,
-            value = InvertJson.encodeToString(
-              serializer = serializer,
-              value = value
-            )
-          )
-        )
+        sink.writeUtf8(invertJsGlobalVariablePrefix(fileKey))
+        InvertJsonStreams.write(sink.outputStream(), serializer, value)
+        sink.writeByte('\n'.code)
       }
       logger.lifecycle("Writing JavaScript $fileKey to file://$canonicalPath")
     }
+
+    private fun invertJsGlobalVariablePrefix(fileKey: String): String {
+      return "window.invert_report[\"${fileKey}\"]=\n"
+    }
+
+    private fun serializedJsonSizeInBytes(jsFile: File, fileKey: String): Long {
+      val wrapperSize = invertJsGlobalVariablePrefix(fileKey).toByteArray().size + 1
+      return jsFile.length() - wrapperSize
+    }
   }
 }

invert-gradle-plugin/src/main/kotlin/com/squareup/invert/internal/report/json/InvertJsonReportWriter.kt

@@ -6,9 +6,9 @@ import com.squareup.invert.internal.models.CollectedDependenciesForProject
 import com.squareup.invert.internal.models.CollectedOwnershipForProject
 import com.squareup.invert.internal.models.CollectedPluginsForProject
 import com.squareup.invert.internal.models.InvertPluginFileKey
+import com.squareup.invert.internal.report.InvertJsonStreams
 import com.squareup.invert.logging.InvertLogger
 import com.squareup.invert.logging.SystemOutInvertLogger
-import com.squareup.invert.models.InvertSerialization.InvertJson
 import com.squareup.invert.models.StatKey
 import com.squareup.invert.models.js.CollectedStatTotalsJsReportModel
 import com.squareup.invert.models.js.HistoricalData
@@ -17,8 +17,6 @@ import com.squareup.invert.models.js.StatTotalAndMetadata
 import com.squareup.invert.models.js.StatsJsReportModel
 import kotlinx.serialization.KSerializer
 import kotlinx.serialization.builtins.SetSerializer
-import okio.buffer
-import okio.sink
 import java.io.File
 
 class InvertJsonReportWriter(
@@ -100,14 +98,7 @@ class InvertJsonReportWriter(
       serializer: KSerializer<T>,
       value: T,
     ) = jsonOutputFile.apply {
-      sink().buffer().use { sink ->
-        sink.writeUtf8(
-          InvertJson.encodeToString(
-            serializer = serializer,
-            value = value
-          )
-        )
-      }
+      InvertJsonStreams.write(this, serializer, value)
       logger.lifecycle("Writing JSON ${jsonFileKey.description} to file://$canonicalPath")
     }
 
@@ -118,14 +109,7 @@ class InvertJsonReportWriter(
       value: T,
       logger: InvertLogger = SystemOutInvertLogger,
     ) = jsonOutputFile.apply {
-      sink().buffer().use { sink ->
-        sink.writeUtf8(
-          InvertJson.encodeToString(
-            serializer = serializer,
-            value = value
-          )
-        )
-      }
+      InvertJsonStreams.write(this, serializer, value)
       logger.lifecycle("Writing JSON ${description} to file://$canonicalPath")
     }
   }

invert-gradle-plugin/src/main/kotlin/com/squareup/invert/internal/tasks/InvertTask.kt

@@ -12,14 +12,14 @@ import com.squareup.invert.internal.isRootProject
 import com.squareup.invert.internal.models.CollectedStatsForProject
 import com.squareup.invert.internal.models.InvertCombinedCollectedData
 import com.squareup.invert.internal.report.GradleProjectAnalysisCombiner
+import com.squareup.invert.internal.report.InvertJsonStreams
 import com.squareup.invert.internal.report.InvertReportWriter
 import com.squareup.invert.internal.report.json.InvertJsonReportWriter
 import com.squareup.invert.internal.report.sarif.InvertSarifReportWriter
 import com.squareup.invert.logging.GradleInvertLogger
 import com.squareup.invert.logging.InvertLogger
 import com.squareup.invert.models.ExtraDataType
 import com.squareup.invert.models.ExtraMetadata
-import com.squareup.invert.models.InvertSerialization.InvertJson
 import com.squareup.invert.models.ModulePath
 import com.squareup.invert.models.OwnerName
 import com.squareup.invert.models.Stat
@@ -136,8 +136,10 @@ abstract class InvertTask : DefaultTask() {
       val historicalData: Set<HistoricalData> =
         if (historicalDataFile?.isFile == true && historicalDataFile.length() > 0) {
           try {
-            val fileContents = historicalDataFile.readText()
-            InvertJson.decodeFromString(ListSerializer(HistoricalData.serializer()), fileContents)
+            InvertJsonStreams.read(
+              file = historicalDataFile,
+              deserializer = ListSerializer(HistoricalData.serializer()),
+            )
           } catch (e: Exception) {
             invertLogger().warn("Failed to read historical data file: $e")
             listOf()