`TastyFormatException: Unexpected term tag (APPLIEDtype|TYPEREF)` with Scala 3.8.2 standard library

[mrdziuban]

I'm not 100% sure if this is an issue with tasty-query or with the standard library itself, but I figured I'd post here first. Let me know if you think I should open an issue at https://github.com/scala/scala3 as well @sjrd.

If you take the following code and save it in a file:

//> using scala 3.8.2
//> using dep ch.epfl.scala::tasty-query:1.7.0
//> using dep io.get-coursier:coursier_2.13:2.1.24

import coursier.{Fetch, Repositories}
import coursier.core.{Dependency, Module, ModuleName, Organization}
import java.net.URI
import java.nio.file.{Files, FileSystems, Paths}
import scala.jdk.CollectionConverters.*
import tastyquery.Contexts.Context
import tastyquery.jdk.ClasspathLoaders

@main
def test = {
  val scalaLibraryDep = Dependency(Module(Organization("org.scala-lang"), ModuleName("scala-library"), Map()), "3.8.2")
  val scalaLibraryFiles = Fetch().withRepositories(Seq(Repositories.central)).addDependencies(scalaLibraryDep).run()
  val javaModules = Files.list(FileSystems.getFileSystem(URI.create("jrt:/")).getPath("modules")).iterator.asScala.toList
  val ctx = Context.initialize(ClasspathLoaders.read(javaModules ++ scalaLibraryFiles.map(_.toPath)))

  println(ctx.findStaticType("scala.collection.immutable.Map"))
}

Then run it with scala-cli:

scala-cli /path/to/the/file.scala

You'll see an error:

Exception in thread "main" tastyquery.Exceptions$TastyFormatException: Unexpected term tag TYPEREF at address Addr(1211) in file /Users/matt/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/3.8.2/scala-library-3.8.2.jar:scala/collection/immutable/Map.class
	at tastyquery.reader.tasties.TreeUnpickler.doReadTerm(TreeUnpickler.scala:1178)
	at tastyquery.reader.tasties.TreeUnpickler.readTerm(TreeUnpickler.scala:922)
	at tastyquery.reader.tasties.TreeUnpickler.doReadTerm(TreeUnpickler.scala:1172)
	at tastyquery.reader.tasties.TreeUnpickler.readTerm(TreeUnpickler.scala:922)
	at tastyquery.reader.tasties.TreeUnpickler.readTypeMappable(TreeUnpickler.scala:1338)
	at tastyquery.reader.tasties.TreeUnpickler.readTermType(TreeUnpickler.scala:1247)
	at tastyquery.reader.tasties.TreeUnpickler.doReadPattern(TreeUnpickler.scala:841)
	at tastyquery.reader.tasties.TreeUnpickler.readPattern(TreeUnpickler.scala:834)
	at tastyquery.reader.tasties.TreeUnpickler.doReadCaseDef(TreeUnpickler.scala:1214)
	at tastyquery.reader.tasties.TreeUnpickler.readCaseDef(TreeUnpickler.scala:1206)
	at tastyquery.reader.tasties.TreeUnpickler.readCases$$anonfun$2(TreeUnpickler.scala:1202)
	at tastyquery.reader.tasties.TastyReader.collectWhile(TastyReader.scala:154)
	at tastyquery.reader.tasties.TreeUnpickler.readCases(TreeUnpickler.scala:1203)
	at tastyquery.reader.tasties.TreeUnpickler.doReadTerm(TreeUnpickler.scala:1069)
	at tastyquery.reader.tasties.TreeUnpickler.readTerm(TreeUnpickler.scala:922)
	at tastyquery.reader.tasties.TreeUnpickler.doReadValOrDefDef(TreeUnpickler.scala:765)
	at tastyquery.reader.tasties.TreeUnpickler.readValOrDefDef(TreeUnpickler.scala:750)
	at tastyquery.reader.tasties.TreeUnpickler.doReadStat(TreeUnpickler.scala:528)
	at tastyquery.reader.tasties.TreeUnpickler.readStat(TreeUnpickler.scala:473)
	at tastyquery.reader.tasties.TreeUnpickler.readStats$$anonfun$1(TreeUnpickler.scala:470)
	at tastyquery.reader.tasties.TastyReader.until(TastyReader.scala:141)
	at tastyquery.reader.tasties.TreeUnpickler.readStats(TreeUnpickler.scala:470)
	at tastyquery.reader.tasties.TreeUnpickler.readTemplate(TreeUnpickler.scala:707)
	at tastyquery.reader.tasties.TreeUnpickler.doReadStat(TreeUnpickler.scala:512)
	at tastyquery.reader.tasties.TreeUnpickler.readStat(TreeUnpickler.scala:473)
	at tastyquery.reader.tasties.TreeUnpickler.doReadTopLevelStat(TreeUnpickler.scala:466)
	at tastyquery.reader.tasties.TreeUnpickler.readTopLevelStat(TreeUnpickler.scala:454)
	at tastyquery.reader.tasties.TreeUnpickler.doReadTopLevelStat$$anonfun$1(TreeUnpickler.scala:465)
	at tastyquery.reader.tasties.TastyReader.until(TastyReader.scala:141)
	at tastyquery.reader.tasties.TreeUnpickler.doReadTopLevelStat(TreeUnpickler.scala:465)
	at tastyquery.reader.tasties.TreeUnpickler.readTopLevelStat(TreeUnpickler.scala:454)
	at tastyquery.reader.tasties.TreeUnpickler.doReadTopLevelStat$$anonfun$1(TreeUnpickler.scala:465)
	at tastyquery.reader.tasties.TastyReader.until(TastyReader.scala:141)
	at tastyquery.reader.tasties.TreeUnpickler.doReadTopLevelStat(TreeUnpickler.scala:465)
	at tastyquery.reader.tasties.TreeUnpickler.readTopLevelStat(TreeUnpickler.scala:454)
	at tastyquery.reader.tasties.TreeUnpickler.doReadTopLevelStat$$anonfun$1(TreeUnpickler.scala:465)
	at tastyquery.reader.tasties.TastyReader.until(TastyReader.scala:141)
	at tastyquery.reader.tasties.TreeUnpickler.doReadTopLevelStat(TreeUnpickler.scala:465)
	at tastyquery.reader.tasties.TreeUnpickler.readTopLevelStat(TreeUnpickler.scala:454)
	at tastyquery.reader.tasties.TreeUnpickler.read$1(TreeUnpickler.scala:50)
	at tastyquery.reader.tasties.TreeUnpickler.unpickle(TreeUnpickler.scala:55)
	at tastyquery.reader.Loaders$PackageLoadingInfo.doLoadTasty(Loaders.scala:181)
	at tastyquery.reader.Loaders$PackageLoadingInfo.tryLoadRoot(Loaders.scala:107)
	at tastyquery.reader.Loaders$PackageLoadingInfo.loadOneRoot$$anonfun$1(Loaders.scala:75)
	at scala.runtime.function.JProcedure1.apply(JProcedure1.java:15)
	at scala.runtime.function.JProcedure1.apply(JProcedure1.java:10)
	at tastyquery.reader.Loaders$PackageLoadingInfo.loadingRoots(Loaders.scala:92)
	at tastyquery.reader.Loaders$PackageLoadingInfo.loadOneRoot(Loaders.scala:73)
	at tastyquery.Symbols$.tastyquery$Symbols$PackageSymbol$$_$getDecl$$anonfun$3$$anonfun$1(Symbols.scala:1900)
	at scala.runtime.function.JProcedure1.apply(JProcedure1.java:15)
	at scala.runtime.function.JProcedure1.apply(JProcedure1.java:10)
	at tastyquery.Symbols$PackageSymbol.loadingNewRoots(Symbols.scala:1876)
	at tastyquery.Symbols$PackageSymbol.getDecl$$anonfun$3(Symbols.scala:1900)
	at scala.Option.orElse(Option.scala:479)
	at tastyquery.Symbols$PackageSymbol.getDecl(Symbols.scala:1902)
	at tastyquery.Symbols$PackageSymbol.getDecl(Symbols.scala:1908)
	at tastyquery.Contexts$Context.findStaticType(Contexts.scala:175)
	at Scala382TastyQuery$package$.test(Scala382TastyQuery.scala:20)
	at test.main(Scala382TastyQuery.scala:13)

[mrdziuban]

This is actually not just limited to scala.collection.immutable.Map, I've found a bunch of other types that cause similar errors. Updated code:

//> using scala 3.8.2
//> using dep ch.epfl.scala::tasty-query:1.7.0
//> using dep io.get-coursier:coursier_2.13:2.1.24

import coursier.{Fetch, Repositories}
import coursier.core.{Dependency, Module, ModuleName, Organization}
import java.net.URI
import java.nio.file.{Files, FileSystems, Paths}
import scala.jdk.CollectionConverters.*
import tastyquery.Contexts.Context
import tastyquery.jdk.ClasspathLoaders

def getType(name: String)(using ctx: Context): Unit =
  scala.util.Try(ctx.findStaticType(name)).fold(e => println(e.getMessage), t => sys.error(s"Unexpected success: $t"))

@main
def test = {
  val scalaLibraryDep = Dependency(Module(Organization("org.scala-lang"), ModuleName("scala-library"), Map()), "3.8.2")
  val scalaLibraryFiles = Fetch().withRepositories(Seq(Repositories.central)).addDependencies(scalaLibraryDep).run()
  val javaModules = Files.list(FileSystems.getFileSystem(URI.create("jrt:/")).getPath("modules")).iterator.asScala.toList
  given ctx: Context = Context.initialize(ClasspathLoaders.read(javaModules ++ scalaLibraryFiles.map(_.toPath)))

  getType("scala.Array")
  getType("scala.collection.mutable.ListBuffer")
  getType("scala.collection.MapFactory")
  getType("scala.collection.StringOps")
  getType("scala.collection.immutable.ListMap")
  getType("scala.collection.immutable.List")
  getType("scala.collection.EvidenceIterableFactory")
  getType("scala.PartialFunction")
  getType("scala.collection.Iterable")
  getType("scala.collection.immutable.Map")
  getType("scala.collection.IterableOnce")
  getType("scala.collection.immutable.IndexedSeq")
  getType("scala.collection.immutable.Vector")
  getType("scala.collection.immutable.Seq")
  getType("scala.collection.immutable.SortedSet")
  getType("scala.collection.immutable.Set")
  getType("scala.collection.mutable.StringBuilder")
  getType("scala.collection.immutable.SortedMap")
}

And the output from it:

Unexpected term tag TYPEREF at address Addr(814) in file /Users/matt/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/3.8.2/scala-library-3.8.2.jar:scala/Array.class
Unexpected term tag APPLIEDtype at address Addr(1342) in file /Users/matt/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/3.8.2/scala-library-3.8.2.jar:scala/collection/mutable/ListBuffer.class
Unexpected term tag APPLIEDtype at address Addr(1515) in file /Users/matt/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/3.8.2/scala-library-3.8.2.jar:scala/collection/MapFactory.class
Unexpected term tag APPLIEDtype at address Addr(18857) in file /Users/matt/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/3.8.2/scala-library-3.8.2.jar:scala/collection/StringOps.class
Unexpected term tag APPLIEDtype at address Addr(4656) in file /Users/matt/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/3.8.2/scala-library-3.8.2.jar:scala/collection/immutable/ListMap.class
Unexpected term tag APPLIEDtype at address Addr(542) in file /Users/matt/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/3.8.2/scala-library-3.8.2.jar:scala/collection/immutable/List.class
Unexpected term tag APPLIEDtype at address Addr(555) in file /Users/matt/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/3.8.2/scala-library-3.8.2.jar:scala/collection/EvidenceIterableFactory.class
Unexpected term tag APPLIEDtype at address Addr(582) in file /Users/matt/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/3.8.2/scala-library-3.8.2.jar:scala/PartialFunction.class
Unexpected term tag APPLIEDtype at address Addr(868) in file /Users/matt/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/3.8.2/scala-library-3.8.2.jar:scala/collection/Iterable.class
Unexpected term tag TYPEREF at address Addr(1211) in file /Users/matt/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/3.8.2/scala-library-3.8.2.jar:scala/collection/immutable/Map.class
Unexpected term tag TYPEREF at address Addr(162) in file /Users/matt/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/3.8.2/scala-library-3.8.2.jar:scala/collection/IterableOnce.class
Unexpected term tag TYPEREF at address Addr(199) in file /Users/matt/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/3.8.2/scala-library-3.8.2.jar:scala/collection/immutable/IndexedSeq.class
Unexpected term tag TYPEREF at address Addr(378) in file /Users/matt/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/3.8.2/scala-library-3.8.2.jar:scala/collection/immutable/Vector.class
Unexpected term tag TYPEREF at address Addr(383) in file /Users/matt/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/3.8.2/scala-library-3.8.2.jar:scala/collection/immutable/Seq.class
Unexpected term tag TYPEREF at address Addr(404) in file /Users/matt/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/3.8.2/scala-library-3.8.2.jar:scala/collection/immutable/SortedSet.class
Unexpected term tag TYPEREF at address Addr(431) in file /Users/matt/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/3.8.2/scala-library-3.8.2.jar:scala/collection/immutable/Set.class
Unexpected term tag TYPEREF at address Addr(533) in file /Users/matt/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/3.8.2/scala-library-3.8.2.jar:scala/collection/mutable/StringBuilder.class
Unexpected term tag TYPEREF at address Addr(710) in file /Users/matt/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/3.8.2/scala-library-3.8.2.jar:scala/collection/immutable/SortedMap.class

[bishabosha]

just speculation, but unless this is a regreesion then probably its a legacy of the genesis of the whole unpickler which didnt follow spec of what was permitted to appear and instead restricted to what was tested to appear

[mrdziuban]

I'll wait for 3.8.3 to confirm with 100% certainty but I'm pretty sure scala/scala3#25406 was the cause of this. It doesn't happen with the latest nightly release.

[mrdziuban]

Confirmed this is fixed in Scala 3.8.3

[sjrd]

Confirmed this is fixed in Scala 3.8.3

Is it? I have trouble upgrading to 3.8.3 in #466, with apparently the same crash.

[mrdziuban]

Interesting, what I tested and confirmed was:

• tasty-query 1.7.0 compiled with 3.8.1
• find-unused compiled with 3.8.3
• My private codebase compiled with 3.8.3

What wasn't working when I opened this issue was using 3.8.2 in find-unused and my codebase.