Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[6.0] Fix SubstConstants: serialize ErgoTree size #995

Merged
merged 23 commits into from
Aug 5, 2024
Merged
Show file tree
Hide file tree
Changes from 18 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
c1d3ce1
v6.0-serialize: SigmaDslBuilder.serialize added SMethod declaration
aslesarenko May 16, 2024
eeb0d4a
v6.0-serialize: added SMethod.runtimeTypeArgs
aslesarenko May 17, 2024
5cbd065
v6.0-serialize: support of explicitTypeArgs in MethodCallSerializer
aslesarenko May 17, 2024
3f4508a
v6.0-serialize: SigmaParserTest for serialize()
aslesarenko May 17, 2024
06386d4
v6.0-serialize: SigmaBinderTest and SigmaTyperTest
aslesarenko May 17, 2024
f7beab4
v6.0-serialize: refactor SigmaDslSpecification towards LanguageSpecif…
aslesarenko May 17, 2024
cfd6e7b
v6.0-serialize: feature tests for Byte an Short
aslesarenko May 19, 2024
4fb3daa
v6.0-serialize: all new features move to LanguageSpecificationV6
aslesarenko May 19, 2024
d09d735
i994-fix-subst-constants: implementation + tests
aslesarenko May 22, 2024
d4bbccc
v6.0-serialize: fix JS tests (added reflection data)
aslesarenko May 23, 2024
9e312bb
Merge remote-tracking branch 'refs/remotes/origin/v6.0-serialize' int…
aslesarenko May 23, 2024
de18eeb
i994-fix-subst-constants: more tests
aslesarenko May 23, 2024
fd8e5d7
i994-fix-subst-constants: cleanup + ScalaDocs
aslesarenko Jun 3, 2024
07b8644
moving newFeature related code from serializePR
kushti Jun 6, 2024
93a100f
merging w. v6.0-newFeature
kushti Jun 7, 2024
91a9729
removing serialize from tests
kushti Jun 7, 2024
a5406ef
merging w. 6.0.0, serialize removed
kushti Jun 10, 2024
e7a1aaf
removing more serialize artefacts and HeaderTypeOps
kushti Jun 11, 2024
962a484
merging w. 6.0.0 , set tree size flag for trees v0 in SigmaDslTesting…
kushti Aug 3, 2024
04f6fee
test for substConstants with 5 elems
kushti Aug 4, 2024
f851b62
LSV5 fix
kushti Aug 5, 2024
3f7e784
Scala 2.11 compilation fix, ScalaDoc returned
kushti Aug 5, 2024
8b17643
fixed regression in CrossVersionProps
kushti Aug 5, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions data/shared/src/main/scala/sigma/ast/SigmaPredef.scala
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import org.ergoplatform.{ErgoAddressEncoder, P2PKAddress}
import scorex.util.encode.{Base16, Base58, Base64}
import sigma.ast.SCollection.{SByteArray, SIntArray}
import sigma.ast.SOption.SIntOption
import sigma.ast.SigmaPropConstant
import sigma.ast.syntax._
import sigma.data.Nullable
import sigma.exceptions.InvalidArguments
Expand Down Expand Up @@ -544,7 +543,7 @@ object SigmaPredef {

val funcs: Map[String, PredefinedFunc] = globalFuncs ++ infixFuncs ++ unaryFuncs

/** WARNING: This operations are not used in frontend, and should be be used.
/** WARNING: This operations are not used in frontend, and should not be used.
* They are used in SpecGen only the source of metadata for the corresponding ErgoTree nodes.
*/
val specialFuncs: Map[String, PredefinedFunc] = Seq(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,9 @@ class ErgoTreeSerializer {
* allow to use serialized scripts as pre-defined templates.
* See [[SubstConstants]] for details.
*
* Note, this operation doesn't require (de)serialization of ErgoTree expression,
* thus it is more efficient than serialization roundtrip.
*
* @param scriptBytes serialized ErgoTree with ConstantSegregationFlag set to 1.
* @param positions zero based indexes in ErgoTree.constants array which
* should be replaced with new values
Expand All @@ -304,39 +307,62 @@ class ErgoTreeSerializer {
s"expected positions and newVals to have the same length, got: positions: ${positions.toSeq},\n newVals: ${newVals.toSeq}")
val r = SigmaSerializer.startReader(scriptBytes)
val (header, _, constants, treeBytes) = deserializeHeaderWithTreeBytes(r)
val w = SigmaSerializer.startWriter()
w.put(header)
val nConstants = constants.length

val resBytes = if (VersionContext.current.isJitActivated) {
// need to measure the serialized size of the new constants
// by serializing them into a separate writer
val constW = SigmaSerializer.startWriter()

if (VersionContext.current.isJitActivated) {
// The following `constants.length` should not be serialized when segregation is off
// in the `header`, because in this case there is no `constants` section in the
// ErgoTree serialization format. Thus, applying this `substituteConstants` for
// non-segregated trees will return non-parsable ErgoTree bytes (when
// `constants.length` is put in `w`).
if (ErgoTree.isConstantSegregation(header)) {
w.putUInt(constants.length)
constW.putUInt(constants.length)
}

// The following is optimized O(nConstants + position.length) implementation
val nConstants = constants.length
if (nConstants > 0) {
val backrefs = getPositionsBackref(positions, nConstants)
cfor(0)(_ < nConstants, _ + 1) { i =>
val c = constants(i)
val iPos = backrefs(i) // index to `positions`
if (iPos == -1) {
// no position => no substitution, serialize original constant
constantSerializer.serialize(c, w)
constantSerializer.serialize(c, constW)
} else {
assert(positions(iPos) == i) // INV: backrefs and positions are mutually inverse
require(positions(iPos) == i) // INV: backrefs and positions are mutually inverse
val newConst = newVals(iPos)
require(c.tpe == newConst.tpe,
s"expected new constant to have the same ${c.tpe} tpe, got ${newConst.tpe}")
constantSerializer.serialize(newConst, w)
constantSerializer.serialize(newConst, constW)
}
}
}

val constBytes = constW.toBytes // nConstants + serialized new constants

// start composing the resulting tree bytes
val w = SigmaSerializer.startWriter()
w.put(header) // header byte

if (VersionContext.current.isV6SoftForkActivated) {
// fix in v6.0 to save tree size to respect size bit of the original tree
if (ErgoTree.hasSize(header)) {
val size = constBytes.length + treeBytes.length
w.putUInt(size) // tree size
}
}

w.putBytes(constBytes) // constants section
w.putBytes(treeBytes) // tree section
w.toBytes
} else {
val w = SigmaSerializer.startWriter()
w.put(header)

// for v4.x compatibility we save constants.length here (see the above comment to
// understand the consequences)
w.putUInt(constants.length)
Expand All @@ -357,10 +383,12 @@ class ErgoTreeSerializer {
case (c, _) =>
constantSerializer.serialize(c, w)
}

w.putBytes(treeBytes)
w.toBytes
}

w.putBytes(treeBytes)
(w.toBytes, constants.length)
(resBytes, nConstants)
}

}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,7 @@ trait CrossVersionProps extends AnyPropSpecLike with TestsBase {
System.gc()
}
forEachScriptAndErgoTreeVersion(activatedVersions, ergoTreeVersions) {
VersionContext.withVersions(activatedVersionInTests, ergoTreeVersionInTests) {
testFun_Run(testName, testFun)
}
}
}
}
Expand Down
24 changes: 22 additions & 2 deletions sc/shared/src/test/scala/sigma/LanguageSpecificationBase.scala
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
package sigma

import org.scalatest.BeforeAndAfterAll
import sigma.ast.JitCost
import sigma.eval.{EvalSettings, Profiler}
import sigma.ast.{Apply, FixedCostItem, FuncValue, GetVar, JitCost, OptionGet, ValUse}
import sigma.eval.{CostDetails, EvalSettings, Profiler}
import sigmastate.CompilerCrossVersionProps
import sigmastate.interpreter.CErgoTreeEvaluator

import scala.util.Success

/** Base class for language test suites (one suite for each language version: 5.0, 6.0, etc.)
Expand Down Expand Up @@ -123,4 +124,23 @@ abstract class LanguageSpecificationBase extends SigmaDslTesting
prepareSamples[(PreHeader, PreHeader)]
prepareSamples[(Header, Header)]
}

///=====================================================
/// CostDetails shared among test cases
///-----------------------------------------------------
val traceBase = Array(
FixedCostItem(Apply),
FixedCostItem(FuncValue),
FixedCostItem(GetVar),
FixedCostItem(OptionGet),
FixedCostItem(FuncValue.AddToEnvironmentDesc, FuncValue.AddToEnvironmentDesc_CostKind),
FixedCostItem(ValUse)
)

/** Helper method to create the given expected results for all tree versions. */
def expectedSuccessForAllTreeVersions[A](value: A, cost: Int, costDetails: CostDetails) = {
val res = ExpectedResult(Success(value), Some(cost)) -> Some(costDetails)
Seq(0, 1, 2, 3).map(version => version -> res)
}

}
Loading
Loading