diff --git a/shared/src/main/scala/io/kaitai/struct/Main.scala b/shared/src/main/scala/io/kaitai/struct/Main.scala index 7ac34ef6e..83bdaa854 100644 --- a/shared/src/main/scala/io/kaitai/struct/Main.scala +++ b/shared/src/main/scala/io/kaitai/struct/Main.scala @@ -39,7 +39,7 @@ object Main { * @return a list of compilation problems encountered during precompilation steps */ def precompile(specs: ClassSpecs, conf: RuntimeConfig): Iterable[CompilationProblem] = { - new MarkupClassNames(specs).run() + new CalculateFullNamesAndSetSurroundingType(specs).run() val resolveTypeProblems = specs.flatMap { case (_, topClass) => val config = updateConfigFromMeta(conf, topClass.meta) new ResolveTypes(specs, topClass, config.opaqueTypes).run() @@ -51,16 +51,17 @@ object Main { return resolveTypeProblems } - new ParentTypes(specs).run() - new SpecsValueTypeDerive(specs).run() - new CalculateSeqSizes(specs).run() - val typeValidatorProblems = new TypeValidator(specs).run() + var passes = Seq( + new ParentTypes(specs), + new DeriveValueInstanceTypes(specs), + new CalculateSeqSizes(specs), + new TypeValidator(specs), + // Warnings + new StyleCheckIds(specs), + new CanonicalizeEncodingNames(specs), + ) - // Warnings - val styleWarnings = new StyleCheckIds(specs).run() - val encodingProblems = new CanonicalizeEncodingNames(specs).run() - - resolveTypeProblems ++ typeValidatorProblems ++ styleWarnings ++ encodingProblems + resolveTypeProblems ++ passes.flatMap(_.run()) } /** diff --git a/shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala index 6a3cb0135..010516e00 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala @@ -58,6 +58,8 @@ case class ClassSpec( * Full absolute name of the class (including all names of classes that * it's nested into, as a namespace). Derived either from `meta`/`id` * (for top-level classes), or from keys in `types` (for nested classes). + * + * This name is calculated by the `CalculateFullNamesAndSetSurroundingType` pass. */ var name = List[String]() @@ -76,6 +78,8 @@ case class ClassSpec( /** * The class specification that this class is nested into, if it exists. * For top-level classes, it's None. + * + * This class is calculated by the `CalculateFullNamesAndSetSurroundingType` pass. */ var upClass: Option[ClassSpec] = None diff --git a/shared/src/main/scala/io/kaitai/struct/format/EnumSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/EnumSpec.scala index 6fcaf791e..5c41442f5 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/EnumSpec.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/EnumSpec.scala @@ -6,6 +6,12 @@ import scala.collection.immutable.SortedMap import scala.collection.mutable case class EnumSpec(path: List[String], map: SortedMap[Long, EnumValueSpec]) extends YAMLPath { + /** + * Absolute name of the enum (includes the names of all classes inside which + * it is defined). Derived either from keys in `enums`. + * + * This name is calculated by the `CalculateFullNamesAndSetSurroundingType` pass. + */ var name = List[String]() /** diff --git a/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala index 4a66307ed..db9b8169d 100644 --- a/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala +++ b/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala @@ -13,6 +13,10 @@ case class ValueInstanceSpec( path: List[String], value: Ast.expr, ifExpr: Option[Ast.expr] = None, + /** + * Type of the `value`. Calculated by the [[DeriveValueInstanceTypes]] pass. + * `None` means "un-calculated yet". + */ var dataTypeOpt: Option[DataType] = None, val _doc: DocSpec = DocSpec.EMPTY, ) extends InstanceSpec(_doc) { diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/MarkupClassNames.scala b/shared/src/main/scala/io/kaitai/struct/precompile/CalculateFullNamesAndSetSurroundingType.scala similarity index 55% rename from shared/src/main/scala/io/kaitai/struct/precompile/MarkupClassNames.scala rename to shared/src/main/scala/io/kaitai/struct/precompile/CalculateFullNamesAndSetSurroundingType.scala index 083aec982..087a82f44 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/MarkupClassNames.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/CalculateFullNamesAndSetSurroundingType.scala @@ -3,13 +3,17 @@ package io.kaitai.struct.precompile import io.kaitai.struct.format.{ClassSpec, ClassSpecs} import io.kaitai.struct.problems.CompilationProblem -class MarkupClassNames(classSpecs: ClassSpecs) extends PrecompileStep { +/** + * Assigns a full name to the `.name` property to each KS type and enum. + * Also for each [[ClassSpec]] assigns [[ClassSpec.upClass]] to a surrounding type. + */ +class CalculateFullNamesAndSetSurroundingType(classSpecs: ClassSpecs) extends PrecompileStep { override def run(): Iterable[CompilationProblem] = { - classSpecs.foreach { case (_, curClass) => markupClassNames(curClass) } + classSpecs.foreach { case (_, curClass) => calculate(curClass) } None } - def markupClassNames(curClass: ClassSpec): Unit = { + private def calculate(curClass: ClassSpec): Unit = { curClass.enums.foreach { case (enumName, enumSpec) => enumSpec.name = curClass.name ::: List(enumName) } @@ -17,7 +21,7 @@ class MarkupClassNames(classSpecs: ClassSpecs) extends PrecompileStep { curClass.types.foreach { case (nestedName: String, nestedClass) => nestedClass.name = curClass.name ::: List(nestedName) nestedClass.upClass = Some(curClass) - markupClassNames(nestedClass) + calculate(nestedClass) } } } diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala index 4d28c263b..688259fa0 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/CalculateSeqSizes.scala @@ -5,10 +5,12 @@ import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType._ import io.kaitai.struct.exprlang.Ast import io.kaitai.struct.format._ +import io.kaitai.struct.problems.CompilationProblem -class CalculateSeqSizes(specs: ClassSpecs) { - def run(): Unit = { +class CalculateSeqSizes(specs: ClassSpecs) extends PrecompileStep { + override def run(): Iterable[CompilationProblem] = { specs.forEachRec(CalculateSeqSizes.getSeqSize) + None } } diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/CanonicalizeEncodingNames.scala b/shared/src/main/scala/io/kaitai/struct/precompile/CanonicalizeEncodingNames.scala index 80ff95fe2..3635cbb3f 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/CanonicalizeEncodingNames.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/CanonicalizeEncodingNames.scala @@ -32,16 +32,23 @@ object CanonicalizeEncodingNames { } def canonicalizeMember(member: MemberSpec): Iterable[CompilationProblem] = { - (member.dataType match { - case strType: StrFromBytesType => - val (newEncoding, problem1) = canonicalizeName(strType.encoding) - strType.encoding = newEncoding - // Do not report problem if encoding was derived from `meta/encoding` key - if (strType.isEncodingDerived) None else problem1 - case _ => - // not a string type = no problem - None - }).map(problem => problem.localizedInPath(member.path ++ List("encoding"))) + try { + (member.dataType match { + case strType: StrFromBytesType => + val (newEncoding, problem1) = canonicalizeName(strType.encoding) + strType.encoding = newEncoding + // Do not report problem if encoding was derived from `meta/encoding` key + if (strType.isEncodingDerived) None else problem1 + case _ => + // not a string type = no problem + None + }).map(problem => problem.localizedInPath(member.path ++ List("encoding"))) + } catch { + // This pass can be called on model with errors, in particular, types of + // value instances could not be calculated. In that case just ignore that + // instance + case _: ExpressionError => None + } } def canonicalizeName(original: String): (String, Option[CompilationProblem with PathLocalizable]) = { diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala new file mode 100644 index 000000000..de42d4b4b --- /dev/null +++ b/shared/src/main/scala/io/kaitai/struct/precompile/DeriveValueInstanceTypes.scala @@ -0,0 +1,80 @@ +package io.kaitai.struct.precompile + +import io.kaitai.struct.{ClassTypeProvider, Log} +import io.kaitai.struct.format.{ClassSpec, ClassSpecs, ValueInstanceSpec} +import io.kaitai.struct.problems.{CompilationProblem, ErrorInInput} +import io.kaitai.struct.translators.TypeDetector + +/** + * Assign types to value instances by deriving them from their expressions. + * + * Calculates value of the [[ValueInstanceSpec.dataTypeOpt]] field, which is + * a type of value instance. + */ +class DeriveValueInstanceTypes(specs: ClassSpecs) extends PrecompileStep { + override def run(): Iterable[CompilationProblem] = { + var iterNum = 1 + var hasChanged = false + do { + hasChanged = false + Log.typeProcValue.info(() => s"### DeriveValueInstanceTypes: iteration #$iterNum") + specs.foreach { case (specName, spec) => + Log.typeProcValue.info(() => s"#### $specName") + + val provider = new ClassTypeProvider(specs, spec) + val detector = new TypeDetector(provider) + + val thisChanged = deriveValueType(spec, provider, detector) + Log.typeProcValue.info(() => ".... => " + (if (thisChanged) "changed" else "no changes")) + hasChanged |= thisChanged + } + iterNum += 1 + } while (hasChanged) + Log.typeProcValue.info(() => s"## value type deriving finished in ${iterNum - 1} iteration(s)") + None + } + + private def deriveValueType( + curClass: ClassSpec, + provider: ClassTypeProvider, + detector: TypeDetector + ): Boolean = { + Log.typeProcValue.info(() => s"deriveValueType(${curClass.nameAsStr})") + var hasChanged = false + + provider.nowClass = curClass; + curClass.instances.foreach { + case (instName, inst) => + inst match { + case vi: ValueInstanceSpec => + vi.dataTypeOpt match { + case None => + try { + val viType = detector.detectType(vi.value) + vi.dataTypeOpt = Some(viType) + Log.typeProcValue.info(() => s"${instName.name} derived type: $viType") + hasChanged = true + } catch { + case tue: TypeUndecidedError => + Log.typeProcValue.info(() => s"${instName.name} type undecided: ${tue.getMessage}") + // just ignore, we're not there yet, probably we'll get it on next iteration + case err: ExpressionError => + // Ignore all errors, the validation will be performed in TypeValidator pass later + } + case Some(_) => + // already derived, do nothing + } + case _ => + // do nothing + } + } + + // Continue with all nested types + curClass.types.foreach { + case (_, classSpec) => + hasChanged ||= deriveValueType(classSpec, provider, detector) + } + + hasChanged + } +} diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/ParentTypes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/ParentTypes.scala index 2929f4e5f..4a1389cd5 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/ParentTypes.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/ParentTypes.scala @@ -4,18 +4,20 @@ import io.kaitai.struct.{ClassTypeProvider, Log} import io.kaitai.struct.datatype.DataType import io.kaitai.struct.datatype.DataType.{ArrayTypeInStream, SwitchType, UserType} import io.kaitai.struct.format._ +import io.kaitai.struct.problems.CompilationProblem import io.kaitai.struct.translators.TypeDetector /** * Precompile step that calculates actual parent types of KSY-defined types * (the type of the `_parent` built-in property). */ -class ParentTypes(classSpecs: ClassSpecs) { - def run(): Unit = { +class ParentTypes(classSpecs: ClassSpecs) extends PrecompileStep { + override def run(): Iterable[CompilationProblem] = { classSpecs.foreach { case (_, curClass) => markup(curClass) } classSpecs.forEachTopLevel((_, spec) => { spec.parentClass = GenericStructClassSpec }) + None } def markup(curClass: ClassSpec): Unit = { diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/SpecsValueTypeDerive.scala b/shared/src/main/scala/io/kaitai/struct/precompile/SpecsValueTypeDerive.scala deleted file mode 100644 index 74686d1a1..000000000 --- a/shared/src/main/scala/io/kaitai/struct/precompile/SpecsValueTypeDerive.scala +++ /dev/null @@ -1,23 +0,0 @@ -package io.kaitai.struct.precompile - -import io.kaitai.struct.Log -import io.kaitai.struct.format.ClassSpecs - -class SpecsValueTypeDerive(specs: ClassSpecs) { - def run(): Unit = { - var iterNum = 1 - var hasChanged = false - do { - hasChanged = false - Log.typeProcValue.info(() => s"### SpecsValueTypeDerive: iteration #$iterNum") - specs.foreach { case (specName, spec) => - Log.typeProcValue.info(() => s"#### $specName") - val thisChanged = new ValueTypesDeriver(specs, spec).run() - Log.typeProcValue.info(() => ".... => " + (if (thisChanged) "changed" else "no changes")) - hasChanged |= thisChanged - } - iterNum += 1 - } while (hasChanged) - Log.typeProcValue.info(() => s"## value type deriving finished in ${iterNum - 1} iteration(s)") - } -} diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/StyleCheckIds.scala b/shared/src/main/scala/io/kaitai/struct/precompile/StyleCheckIds.scala index 4c4753847..597f3d9b4 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/StyleCheckIds.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/StyleCheckIds.scala @@ -40,20 +40,27 @@ class StyleCheckIds(specs: ClassSpecs) extends PrecompileStep { } def getSizeRefProblem(spec: ClassSpec, attr: MemberSpec): Option[CompilationProblem] = { - getSizeReference(spec, attr.dataType).flatMap(sizeRefAttr => { - val existingName = sizeRefAttr.id.humanReadable - val goodName = s"len_${attr.id.humanReadable}" - if (existingName != goodName) { - Some(StyleWarningSizeLen( - goodName, - existingName, - attr.id.humanReadable, - ProblemCoords(path = Some(sizeRefAttr.path ++ List("id"))) - )) - } else { - None - } - }) + try { + getSizeReference(spec, attr.dataType).flatMap(sizeRefAttr => { + val existingName = sizeRefAttr.id.humanReadable + val goodName = s"len_${attr.id.humanReadable}" + if (existingName != goodName) { + Some(StyleWarningSizeLen( + goodName, + existingName, + attr.id.humanReadable, + ProblemCoords(path = Some(sizeRefAttr.path ++ List("id"))) + )) + } else { + None + } + }) + } catch { + // This pass can be called on model with errors, in particular, types of + // value instances could not be calculated. In that case just ignore that + // instance + case _: ExpressionError => None + } } def getRepeatExprRefProblem(spec: ClassSpec, attr: AttrLikeSpec): Option[CompilationProblem] = { diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/TypeValidator.scala b/shared/src/main/scala/io/kaitai/struct/precompile/TypeValidator.scala index d80741484..b32b38609 100644 --- a/shared/src/main/scala/io/kaitai/struct/precompile/TypeValidator.scala +++ b/shared/src/main/scala/io/kaitai/struct/precompile/TypeValidator.scala @@ -105,6 +105,10 @@ class TypeValidator(specs: ClassSpecs) extends PrecompileStep { def validateValueInstance(vis: ValueInstanceSpec): Option[CompilationProblem] = { try { + // detectType performs some additional checks that validate does not (for example, + // applicability of operators to types, like `Not` to numbers). + // TODO: probably implement those checks in validate too? + detector.detectType(vis.value) detector.validate(vis.value) None } catch { diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala b/shared/src/main/scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala deleted file mode 100644 index e66c5c016..000000000 --- a/shared/src/main/scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala +++ /dev/null @@ -1,56 +0,0 @@ -package io.kaitai.struct.precompile - -import io.kaitai.struct.format.{ClassSpec, ClassSpecs, ValueInstanceSpec} -import io.kaitai.struct.problems.ErrorInInput -import io.kaitai.struct.translators.TypeDetector -import io.kaitai.struct.{ClassTypeProvider, Log} - -class ValueTypesDeriver(specs: ClassSpecs, topClass: ClassSpec) { - val provider = new ClassTypeProvider(specs, topClass) - val detector = new TypeDetector(provider) - - def run(): Boolean = - deriveValueType(topClass) - - def deriveValueType(curClass: ClassSpec): Boolean = { - Log.typeProcValue.info(() => s"deriveValueType(${curClass.nameAsStr})") - var hasChanged = false - var hasUndecided = false - - provider.nowClass = curClass - curClass.instances.foreach { - case (instName, inst) => - inst match { - case vi: ValueInstanceSpec => - vi.dataTypeOpt match { - case None => - try { - val viType = detector.detectType(vi.value) - vi.dataTypeOpt = Some(viType) - Log.typeProcValue.info(() => s"${instName.name} derived type: $viType") - hasChanged = true - } catch { - case tue: TypeUndecidedError => - Log.typeProcValue.info(() => s"${instName.name} type undecided: ${tue.getMessage}") - hasUndecided = true - // just ignore, we're not there yet, probably we'll get it on next iteration - case err: ExpressionError => - throw ErrorInInput(err, vi.path ++ List("value")).toException - } - case Some(_) => - // already derived, do nothing - } - case _ => - // do nothing - } - } - - // Continue with all nested types - curClass.types.foreach { - case (_, classSpec) => - hasChanged ||= deriveValueType(classSpec) - } - - hasChanged - } -}