Index: /reasoner/appendixTests.tex =================================================================== --- /reasoner/appendixTests.tex (revision 170) +++ /reasoner/appendixTests.tex (revision 170) @@ -0,0 +1,173 @@ +\section{Reasoner test cases}\label{appendixTestCases} + +In this section, we list a representative subset of the implemented reasoner test cases, which aim at validating translation and reasoning for specific IVML concepts and combinations. Please note that reasoner test cases defined as part of the test suite for the \class{ReasonerCore} bundle of EASy-Producer contains more than 310 test cases are applied to the reasoner, many constructed in a similar way for different types, i.e., exhaustively listing all of them is neither possible her nor intended. Among them, there are also more complex test cases taken from real scenarios, evaluation models as well as challenge test cases. + +For referencing the test cases, we use identifiers composed for the main concepts to be validated. We use the abbreviations shown in Table \ref{tab:testCaseIdSchema} to compose identifiers. For example, test case nuber one on Integer (I) variables used as compound (Co) slots is identified by 'ICo1'. Table \ref{tab:reasonerTestCasesBasics} and Table \ref{tab:reasonerTestCasesAdvanced} list the test cases referenced in this document. Test case names ending with 'Valid' indicate that there is an associate test case testing also the failing of the constraint under test. + +\begin{table*}[ht] +%\begin{adjustbox}{angle=90} +\centering +\begin{tabular}{|l|l|} +\hline +\textbf{Id part / abbreviation} & \textbf{IVML concept}\\ +\hline + +B & Boolean \\ +Cn & Compound \\ +Ct & Container\\ +Cs & Constraint\\ +A & Annotation, AssignmentBlock\\ +Ev & Eval\\ +I & Integer\\ +R & Real\\ +S & String\\ +E & Enum\protect\footnotemark\\ +T & Typedef\\ +F & Reference\\ +If & Interface\\ +Sc & Scope, re-assignment\\ +\hline +\end{tabular} +\caption{Test case naming schema.} +\label{tab:testCaseIdSchema} +%\end{adjustbox} +\end{table*} +\footnotetext{Similarly, test cases for ordered enums are in place, but not referenced explicitly in this document.} + + + +\begin{table*}[ht] +%\begin{adjustbox}{angle=90} +\centering +\begin{tabular}{|l|p{12.5cm}|} +\hline +\textbf{Id} & \textbf{Test model}\\ +\hline +B1 & \class{boolean/BooleanDefaultsTest.ivml}\\ +B2 & \class{boolean/BooleanAssignTest.ivml}\\ +B3 & \class{boolean/BooleanImpliesTest.ivml}\\ +BF1 & \class{boolean/BooleanRefAssignTest.ivml\protect\footnotemark}\\ +AB1 & \class{boolean/BooleanAnnotationTest.ivml}\\ +BCo1 & \class{boolean/BooleanInCompoundAssignTest.ivml}\\ +BCo3 & \class{boolean/BooleanInCompoundDefaultsTest.ivml}\\ +BCo3 & \class{boolean/BooleanInCompoundImpliesTest.ivml}\\ +BCt1 & \class{boolean/BooleanInContainerAssignTest.ivml}\\ +BCt2 & \class{boolean/BooleanInContainerDefaultsTest.ivml}\\ +I1 & \class{integer/IntegerDefaultsTest.ivml}\\ +I2 & \class{integer/IntegerAssignTest.ivml}\\ +AI1 & \class{integer/IntegerAnnotationTest.ivml}\\ +AI2 & \class{runtime/runtimeReasoning.ivml}\\ +IT1 & \class{operations/typedef.ivml}\\ +IT2 & \class{operations/typedefOfTypedefValid.ivml}\\ +ICo1 & \class{integer/IntegerInCompoundDefaultsTest.ivml}\\ +ICo2 & \class{integer/IntegerInCompoundAssignTest.ivml}\\ +ICt1 & \class{integer/IntegerInContainerAssignTest.ivml}\\ +ICt2 & \class{integer/IntegerInContainerDefaultsTest.ivml}\\ +R1 & \class{real/RealDefaultsTest.ivml}\\ +R2 & \class{real/RealAssignTest.ivml}\\ +AR1 & \class{real/RealAnnotationTest.ivml}\\ +RCo1 & \class{real/RealInCompoundDefaultsTest.ivml}\\ +RCo2 & \class{real/RealInCompoundAssignTest.ivml}\\ +RCt1 & \class{real/RealInContainerAssignTest.ivml}\\ +RCt2 & \class{real/RealInContainerDefaultsTest.ivml}\\ +S1 & \class{string/StringDefaultsTest.ivml}\\ +S2 & \class{string/StringAssignTest.ivml}\\ +AS1 & \class{string/StringAnnotationTest.ivml}\\ +SCo1 & \class{string/StringInCompoundDefaultsTest.ivml}\\ +SCo2 & \class{string/StringInCompoundAssignTest.ivml}\\ +SCt1 & \class{string/StringInContainerAssignTest.ivml}\\ +SCt2 & \class{string/StringInContainerDefaultsTest.ivml}\\ +\hline +\end{tabular} +\caption{Reasoner test cases for basic types combined with compounds or containers.} +\label{tab:reasonerTestCasesBasics} +%\end{adjustbox} +\end{table*} +\footnotetext{A similar test is defined for all basic types.} + +\begin{table*}[ht] +%\begin{adjustbox}{angle=90} +\centering +\begin{tabular}{|l|p{12.5cm}|} +\hline +\textbf{Id} & \textbf{Test model}\\ +\hline +E1 & \class{enums/EnumDefaultsTest.ivml}\\ +E2 & \class{enums/EnumAssignTest.ivml}\\ +AE1 & \class{enums/EnumAnnotationTest.ivml}\\ +EF1 & \class{enums/EnumRefAssignTest.ivml}\\ +ECo1 & \class{enums/EnumInCompoundDefaultsTest.ivml}\\ +ECo2 & \class{enums/EnumInCompoundAssignTest.ivml}\\ +ECt1 & \class{enums/EnumInContainerDefaultsTest.ivml}\\ +ECt2 & \class{enums/EnumInContainerAssignTest.ivml}\\ +CnCt1 & \class{collectionConstraints/constraintSetDefault.ivml}\\ +CnCt2 & \class{collectionConstraints/constraintSetSetDefault.ivml}\\ +CnCt3 & \class{compounds/CompoundContainerTest1.ivml}\\ +CnCt4 & \class{compounds/CompoundContainerTest2.ivml}\protect\footnotemark\\ +CnCs1 & \class{compounds/changingValues/changingValues.ivml}\\ +ACt1 & \class{collectionConstraints/ContainerAnnotationTest.ivml}\\ +ACn1 & \class{compounds/CompoundAnnotationTest.ivml}\\ +AFCn1 & \class{compounds/ReferenceCompoundAnnotationTest.ivml}\\ +ATCn1 & \class{compounds/DerivedCompoundAnnotationTest.ivml}\\ +CsCtCn1 & \class{constraintsVariables/changingConstraints/changingConstraints.ivml}\\ +CnTCt1 & \class{collectionConstraints/constraintSetDerivedCompound.ivml}\\ +CnTCt2 & \class{collectionConstraints/constraintSetSetDerivedCompound.ivml}\\ +CtTCt1 & \class{collectionConstraints/setDerivedSet.ivml}\\ +CtTF1 & \class{collectionConstraints/ReferenceDerivedCollectionTest.ivml}\\ +TFCnCtCsA1 & \class{collectionConstraints/QM.ivml}\\ +ITCn1 & \class{operations/typedefNestedInCompoundValid.ivml}\\ +ITCnCt1 & \class{operations/typedefCompoundNestedInSequenceValid1.ivml}\\ +ITCnCt2 & \class{operations/typedefCompoundNestedInSequenceValid2.ivml}\\ +ITCt1 & \class{operations/typedefNestedInSequenceValid.ivml}\\ +ITCt2 & \class{operations/typedefNestedInSetValid.ivml}\\ +CnAI1 & \class{attributes/BlockAssign.ivml}\\ +CnAI2 & \class{attributes/BlockAssignNested.ivml}\\ +CnAI3 & \class{attributes/BlockAssignNestedConstraint.ivml}\\ +FCn1 & \class{compounds/CompoundRefAssignTest1.ivml}\\ +FCn2 & \class{compounds/CompoundRefAssignTest2.ivml}\\ +FCn3 & \class{compounds/CompoundRefAssignTest3.ivml}\\ +Cn1 & \class{compounds/CompoundConstraintTest}\\ +Cn2 & \class{compounds/CompoundCompoundTest1.ivml}\\ +Cn3 & \class{compounds/CompoundCompoundTest2.ivml}\protect\footnotemark\\ +CnTF1 & \class{collectionConstraints/RefernceDerivedCompoundTest.ivml}\\ +\hline +\end{tabular} +\caption{Reasoner test cases for advanced concepts and types (part 1).} +\label{tab:reasonerTestCasesAdvanced} +%\end{adjustbox} +\end{table*} +\addtocounter{footnote}{-1} +\footnotetext{This test aims at recursive compound types.}\stepcounter{footnote} +\footnotetext{This test aims at recursive compound types.} + +\begin{table*}[ht] +%\begin{adjustbox}{angle=90} +\centering +\begin{tabular}{|l|p{12.5cm}|} +\hline +\textbf{Id} & \textbf{Test model}\\ +\hline +A1 & \class{attributes/DefaultAssign.ivml}\\ +A2 & \class{attributes/IndividualAssign.ivml}\\ +A3 & \class{attributes/IndividualAssignNested.ivml}\\ +ACs1 & \class{attributes/ConstraintAnnotationTest.ivml}\\ +CnA3 & \class{attributes/CompoundIndividualAssign.ivml}\\ +CnA4 & \class{attributes/BlockAssignNestedConstraintFail.ivml}\\ +Cs1 & \class{constraintVariables/constraintAssigned.ivml}\\ +Cs2 & \class{constraintVariables/constraintDefault.ivml}\\ +CsCn1 & \class{constraintVariables/constraintCompoundAssigned.ivml}\\ +CsCn2 & \class{constraintVariables/constraintCompoundDefault.ivml}\\ +CnFCt1 & \class{constraintVariables/QM\_constraintOrder.ivml}\\ +Ev1 & \class{evals/SimpleEval.ivml}\\ +CnEv1 & \class{evals/CompoundEval.ivml}\\ +If1 & \class{interfaces/Frozen.ivml}\\ +If2 & \class{interfaces/Unfrozen.ivml}\\ +Sc1 & \class{integer/VariableFailTest.ivml}\\ +Sc2 & \class{compounds/slotReassignment/Cfg.ivml}\\ +Sc3 & \class{constraintVariables/refinedConstraints/refinedConstraints.ivml}\\ +\hline +\end{tabular} +\caption{Reasoner test cases for advanced concepts and types (part 2).} +\label{tab:reasonerTestCasesAdvanced2} +%\end{adjustbox} +\end{table*} Index: /reasoner/evaluation.tex =================================================================== --- /reasoner/evaluation.tex (revision 169) +++ /reasoner/evaluation.tex (revision 170) @@ -1,3 +1,112 @@ \section{Evaluation}\label{sectEvaluation} + +The goal of this evaluation is to measure and discuss the performance of the actual implementation reasoner, in particular in comparison to the base version that acted as starting point for the revision. We also compare briefly potential impacts of different Java versions (Java 8 and Java 9) as well as different operating systems (Windows 7, Windows 10, Linux) as during the development of the new reasoner a hardware exchange forced switching the underlying versions of Windows, Java and even Eclipse. We focus here on response time and leave other potential interesting performance dimensions like memory usage for future evaluations. + +We discuss in Section \ref{sectEvaluationSetup} the setup of this evaluation, in Section \ref{sectModelComplexity} how to determine the complexity of IVML models and in Section \ref{sectEvaluationResults} the results. + +\subsection{Setup}\label{sectEvaluationSetup} + +\emph{Data collection:} The testing infrastructure of EASy-Producer implements a generic data collector, which can be feeded with key-value pairs representing measured (real) values. By default, the collector can account for coarse grained response time, which can help validating more detailed measures done during the actual measurement. For the measurements in this evaluation, we include the default statistics collected by the SSE reasoner such as translation time, evaluation time, number of failed constraints, number of re-evaluated constraints as well as model statistics and complexity measures (cf. Section \ref{sectModelComplexity}) delivered by EASy-Producer. + +\emph{Subjects:} The test cases of EASy-Producer involving reasoning, i.e., the test suites for the SSE reasoner (reused from the reasoner core), the runtime extension for VIL, the scenario test cases (including the models of QualiMaster) as well as the scenario test cases for the BMWi ScaleLog project. It is important to note that the base version of the reasoner\footnote{\label{reasonerBaseVersion}\TBD{Version 1.2.0-SNAPSHOT, git hash}} did not contain the generic data collector, so we had to backpack the original base version. Moreover, several test cases that have been created for testing the advanced features of the recent version of the SSE reasoner are also not included (and cannot be executed on the base version). However, while the subject sets differ in detail, the most imporant small and large models are the same in both subject sets. + +\emph{Procedure:} We run the four test suites mentioned above each 5 times to collect response time and model. Typically, each test suite runs individually in a JVM. To compensate delayed JIT optimization, we include a ramp-up run that warms up the JVM. For most test cases, a simple in-memory model with a compound type, a collection over that type and a quantor constraint over the container variable is sufficient. However, for the QualiMaster models, we added as ramp-up run a full run of one of the largest models without accounting for reasoning time or without performing instantiation. \TBD{We execute all tests in a script outside Eclipse to avoid disturbances caused by functionality of the IDE. We execute this procedure on the most recent version of EASy-Producer\footnote{Version 1.3.0-SNAPSHOT, TBD{git-hash}} on an actual development machine, a Dell laptop... with Windows 10 and JDK9. For comparison, we run the same version of EASy-Producer on a Dell laptop ... with Windows 7 and JDK8. For both windows machines, we disable first the virus scanner and terminate all programs that are not required for the execution of the tests. On the Windows 7 machine we also run the base version of the reasoner\footref{reasonerBaseVersion}. Finally, for curiosity, we run the the test execution script also on our continuous integration server, a Linux... VM at a point in time when no Jenkins tasks are running.} + +\emph{Analysis:} \TBD{XXX} + +\subsection{Model Complexity}\label{sectModelComplexity} + +$$ + cpx_v(e) = \begin{cases} + \sum_{v\in vars(e)}cpx_v(e) & \text{if } isConfiguration(e)\\ + w_{cpx}(type(e)) + cpx(default(e)) & \text{if } isVariable(e)\\ + 0 & \text{else} + \end{cases} +$$ + +$$ + cpx_c(e) = \begin{cases} + \sum_{v\in vars(e)}cpx_c(v) + cpx_c(project(e)) & \text{if } isConfiguration(e)\\ + \sum_{f\in elements(e)}cpx_c(f) & \text{if } isProject(e)\\ + \sum_{v\in vars(v)}cpx_c(e) + w_{cpx}(type(e)) & \\ + \text{ } + w_{cpx}(\IVML{Expr}) \cdot cpx_e(default(e)) & \text{if } isVariable(e)\\ + \sum_{c\in constraints(e)} cpx_c(c) & \text{if } isEval(e)\\ + \sum_{x\in constraints(e)~\cup~assignments(e)} cpx_c(x) & \text{if } isAssignment(e)\\ + cpx(function(e)) & \text{if } isOpDef(e)\\ + w_{cpx}(e) + w_{cpx}(\IVML{Expr}) \cdot cpx_e(expr(e)) & \text{if } isConstraint(e)\\ + 0 & \text{else} + \end{cases} +$$ + + +%$$ + % cpx(e) = \closedCases{ + % cpx(project(cfg)) & \text{if } isConfiguration(e)\\ + % %& \text{if } isProject(e)\\ only r_{cpx} + % w_{cpx}(type(e)) + cpx(default(e)) & \text{if } isVariable(e)\\ + % cpx(default(e)) & \text{if } isDeclaration(e)\\ % may count compound slots twice +% %\sum_{c\in constraints(e)} & \text{if } isEval(e)\\ only r_{cpx} + % %\sum_{x\in r_a(e)} cpx(x) & \text{if } isAssignment(e)\\ only r_{cpx} + % cpx(function(e)) & \text{if } isOpDef(e)\\ + % c_{cpx}(e) & \text{if } isConstraint(e)\\ + % c_{cpx}(e) & \text{if } isExpression(e)\\ + % 0 & \text{else} + % } + \sum_{x\in r_{cpx}(x)}cpx(e) +%$$ + +%$$ +% r_{cpx}(e) = \begin{cases} +% vars(e) & \text{if } isConfiguration(e)\\ +% constraints(p)~\cup~evals(p)~\cup~assignments(p) & \text{if } isProject(e)\\ +% vars(e)~\cup~allRefines^+(type(e)) & \text{if } isVariable(e)\\ +% constraints(e)~\cup~assignments(e) & \text{if } isCompound(e)\\ +% constraints(e)~\cup~evals(e) & \text{if } isEval(e)\\ +% vars(e)~\cup~constraints(e)~\cup~assignments(e) & \text{if } isAssignment(e)\\ +% \emptySet & \text{else} +% \end{cases} +%$$ + +$$ + cpx_{e}(e) = w_{cpx}(e) + \begin{cases} + \sum_{a\in args(e)}c_{cpx}(a) & \text{if } isCall(e)\\ + c_{cpx}(expr(e)) & \text{if } isParenthesis(e)~\vee \\ + & \text{ } isIter(e) \vee isLet(e)\\ + c_{cpx}(expr(e)) + c_{cpx}(then(e)) + c_{cpx}(else(e)) & \text{if } isIfThen(e)\\ + c_{cpx}(expr(e)) & \text{if } isAccessor(e)\\ + \sum_{e\in expressions(e)}c_{cpx}(e) & \text{if } isInitializer(e)~\vee \\ % compound + container + & \text{ } isBlock(e)\\ + 0 & \text{else} + \end{cases} +$$ + +Calibrated for the projects in this set + +$$ + w_{cpx}(e) = \begin{cases} + 2 & \text{if } isCompound(e)\\ + 2 & \text{if } isContainer(e)\\ +% 1 & \text{if } isExpression(e)\\ + 1 & \text{if } \IVML{Expr}\\ + 0.25 & \text{if } isVariable(e)\\ + 0.25 & \text{if } isConstant(e)\\ % 0 for isCall + 5 & \text{if } isIter(e)\\ + 1 & \text{if } isAccessor(e)\\ + 1 & \text{if } e = \IVMLself{}\\ + 0 & \text{else} + \end{cases} +$$ + +Tranditional +$$ + w_{cpx}(e) = \begin{cases} + 1 & \text{if } isConstraint(e)\\ % in particular 0 for Expr + 0 & \text{else} + \end{cases} +$$ + +\subsection{Results}\label{sectResults} + +% w(compound) = 2, w(container) = 2, w(variable) = 0.25, w(constant) = 0.25, w(call) = 0, w(iter) = 5 + \TBD{Keep or remove that section. However, some basic performance evaluations should not be so difficult. All executions need multiple runs, in particular to even out JIT compilation. It seems that this is done during/after the second reasoning run. Potential topics: Index: /reasoner/introduction.tex =================================================================== --- /reasoner/introduction.tex (revision 169) +++ /reasoner/introduction.tex (revision 170) @@ -16,3 +16,3 @@ \end{itemize} -This document is structured as follows: Section \ref{sectApproach} provides an overview to our approach. Section \ref{sectNotation} introduces the notation that we use throughout this document to specify the algorithms, i.e., we introduce notations for sets, sequences, substitutions as well as relevant parts of the IVML object model realizing the various IVML concepts, here in terms of notations for accessing (derived) properties of the IVML model. In Section \ref{sectReasoning}, we discuss the top-level algorithms for transforming constraints and reasoning. In Section \ref{sectTranslation}, we detail the translation of constraints for all IVML modeling concepts. The result is a set of instantiated constraints that can be used for reasoning. Section \ref{sectTranslation} includes also the mechanism for re-scheduling constraints when value types or constraint variables change, as this mechanism heavily relies on the constraint translations discussed before. In Section \ref{sectCompleteness}, we analyze the translations in order to assess their completeness with respect to IVML concepts and indicate the state of the validation, i.e., regression test cases checking that the constraint translations are applied correctly for specific modeling situations. In Section \ref{sectEvaluation}, we present an evaluation of the actual performance of the reasoning approach for different kinds of IVML models. In Section \ref{sectPerformance}, we summarize performance issues, findings and resolutions we made while revising the reasoner, in particular to avoid that future modifications violate already resolved performance anti-patterns. Finally, in Section \ref{sectConclusion} we conclude and provide an outlook on future work. +This document is structured as follows: Section \ref{sectApproach} provides an overview to our approach. Section \ref{sectNotation} introduces the notation that we use throughout this document to specify the algorithms, i.e., we introduce notations for sets, sequences, substitutions as well as relevant parts of the IVML object model realizing the various IVML concepts, here in terms of notations for accessing (derived) properties of the IVML model. In Section \ref{sectReasoning}, we discuss the top-level algorithms for transforming constraints and reasoning. In Section \ref{sectTranslation}, we detail the translation of constraints for all IVML modeling concepts. The result is a set of instantiated constraints that can be used for reasoning. Section \ref{sectTranslation} includes also the mechanism for re-scheduling constraints when value types or constraint variables change, as this mechanism heavily relies on the constraint translations discussed before. In Section \ref{sectCompleteness}, we analyze the translations in order to assess their completeness with respect to IVML concepts and indicate the state of the validation, i.e., regression test cases checking that the constraint translations are applied correctly for specific modeling situations. In Section \ref{sectPerformance}, we summarize performance issues, findings and resolutions we made while revising the reasoner, in particular to avoid that future modifications violate already resolved performance anti-patterns. In Section \ref{sectEvaluation}, we present an evaluation of the actual performance of the reasoning approach for different kinds of IVML models. Finally, in Section \ref{sectConclusion} we conclude and provide an outlook on future work. Index: /reasoner/notation.tex =================================================================== --- /reasoner/notation.tex (revision 169) +++ /reasoner/notation.tex (revision 170) @@ -45,5 +45,5 @@ While variable declarations as introduced in Section \ref{sectNotationVarDecls} as well as types as introduced below allow defining (a meta-model of) configuration options, \emph{configuration variables} are part of the configuration of a specific product and hold the actual configured value for a variable declaration. A configuration variable $v$ (representing a decision, inspired by decision modeling~\cite{CzarneckiGruenbacherRabiser+12, SchmidJohn04}) is defined through a variable declaration $d$, but for a single variable declaration $d$ in a model, multiple configuration variables can exist in different configurations. -Let $v$ be a configuration variable with declaration $d$ as part of a certain configuration $cfg$, i.e., $decision(cfg, d)=v$ and $parent(v)=cfg$. For such a combination of declaration $d$ and variable $v$, we define $declaration(v) = d$ as well as $type(v) = type(declaration(v))$. For a variable $v$, the function $isVariable(v)$ returns $true$, for any other IVML element/type $false$. +Let $v$ be a configuration variable with declaration $d$ as part of a certain \emph{configuration} $cfg$, i.e., $decision(cfg, d)=v$ and $parent(v)=cfg$. For such a combination of declaration $d$ and variable $v$, we define $declaration(v) = d$ as well as $type(v) = type(declaration(v))$. For a variable $v$, the function $isVariable(v)$ returns $true$, for any other IVML element/type $false$. A configuration $cfg$ returns all contained variables in terms of $vars(cfg)$. $value(v)$ denotes the actual value of $v$ and $type(value(v))$ compliant with $type(v)$. Please note, that $type(value(v))$ can be a subtype of $type(v)$, in particular if $type(v)$ is a refined type through inheritance (cf.~\cite{IVML-LS} for details on the IVML type compliance rules and the IVML type hierarchy). We will detail some relevant properties of values below in Section \ref{sectNotationValues}. @@ -82,5 +82,5 @@ \subsubsection{Project}\label{sectNotationProject} -A \emph{project} is a named scope, which can contain variable declarations, type declarations, assignment-blocks and eval-blocks. $decls(p)$ denotes the set of all variable declarations introduced in project $p$. $isProject(p)$ returns $true$ for a project $p$. +A \emph{project} is a named scope, which can contain variable declarations, type declarations, assignment-blocks and eval-blocks. $decls(p)$ denotes the set of all variable declarations introduced in project $p$. $isProject(p)$ returns $true$ for a project $p$. $evals(p)$ is the set of (potentially nested) \emph{eval-blocks} in project $p$. If $e\in evals(p)$ then $constraints(e)$ returns the constraints listed in $e$, whereby $constraints(e)$ may be empty, in particular if $e$ is an arbitrary model element. Similarly, $evals(e)$ returns the nested eval-blocks. Subsequently, the function $allEvalConstraints(e)$ returns all constraints in eval-block $e$ including all nested eval-blocks. @@ -93,5 +93,5 @@ In IVML, an \emph{annotation assignment} is a kind of block, which allows assigning annotation values to the variables defined within the block, i.e., the specified annotation values shall be applied to all contained variable declarations. The declared variables shall behave as if they are defined in the innermost containing element that is not an annotation assignment. For convenience, also constraints relating to the contained variable declarations can be stated within an annotation assignment. To indicate orthogonal combinations of annotation assignments in IVML models, annotation assignments can be nested referring to different annotations. For convenience, only the innermost annotation value assignment shall be applied in nested assignment blocks in order to avoid accidental multiple assignments to the same annotation within the same project scope. -$assignments(p)$ is the set of (potentially nested) annotation assignments of project $p$. If $a\in assignments(p)$ then $decls(a)$ returns the variables declared in $a$ and $constraints(a)$ the constraints declared within $a$. Further, $assignments(a)$ returns the annotation assignments nested within $a$. All three results may be empty. Further, $assignmentData(a)$ provides access to the affected annotations and the intended values values, i.e., if $f \in assignmentData(a)$ then $name(f)$ is the name of the affected attribute and $default(f)$ the intended default value expression that shall be assigned to all attributes with name $name(f)$ for all declarations within the containing assignment-block. $isAnnotationAssng(a)=true$ and for an arbitrary model element $e$, $isAnnotationAssng(e)$ returns whether $e$ is an assignment block. +$assignments(p)$ is the set of (potentially nested) annotation assignments of project $p$. If $a\in assignments(p)$ then $decls(a)$ returns the variables declared in $a$ and $constraints(a)$ the constraints declared within $a$. Further, $assignments(a)$ returns the annotation assignments nested within $a$. All three results may be empty. Further, $assignmentData(a)$ provides access to the intended value assignment, i.e., if $f \in assignmentData(a)$ then $name(f)$ is the name of the affected attribute and $default(f)$ the intended default value expression that shall be assigned to all attributes with name $name(f)$ for all declarations within the containing attribute assignment. $isAnnotationAssng(a)$ returns whether $e$ is an annotation assignment. Using these functions, $allAssignmentConstraints(a)$ determines all assignment constraints within an assignment $a$ considering potentially further nested annotation assignments: @@ -114,4 +114,6 @@ % returns the innermost annotation for variable $v$ with name $n$. + +A configuration $cfg$ as introduced in Section \ref{sectNotationConfigVars} always contains the decision variables for an underlying defining $project(cfg)$. % \subsubsection{Constraints}\label{sectNotationConstraints} Index: /reasoner/performance.tex =================================================================== --- /reasoner/performance.tex (revision 169) +++ /reasoner/performance.tex (revision 170) @@ -10,13 +10,17 @@ \item The \emph{constraint base} is used for storing the constraints to be processed. Initially, during reasoning a single project, a pointer-based iterator traversed the array-based constraint list and the algorithm kept all processed constraints in memory. Constraint re-scheduling just added more constraints. Basically, using a linked list avoids re-allocation of big memory blocks. Moreover, using the list as a queue enables dynamic memory usage according to the needs, i.e., the actual constraint can be obtained by removing the head element and the main reasoning loop in Algorithm \ref{algEvaluateConstraints} can just operate until the constraint base is empty (if no further re-scheduled constraints are added). To also speed up lookup and removal for re-scheduling, we extended the \class{ConstraintList} and integrated it with a hashtable (\class{ConstraintBase}). This avoids the initially quadratic complexity in constraint re-scheduling for forward reasoning. On large models, the difference in ad ho performance comparisons was roughly 600 ms, i.e., around 25\% of the reasoning time, even if we do not use JIT supported data structures. - \item In general, maintaining unneeded \emph{partitions of the constraint base} shall be avoided. Although we were able to speed up the creation of the constraint base by special transfer operations, it is better to directly add constraints to the constraint base where possible. Thus, we removed several unneeded temporary lists. However, adding constraints directly is not always possible due to the incremental construction while traversing the model and the prioritization of certain constraints, such as default values or constraints in eval-blocks. As an alterative, we could use a numeric prioritization stored along with the individual constraints (like the attached type information) and use a sorted data structure. So far, we tried to avoid additional information stored in the constraints or to investigate specific management records for each individual constraint that would be required when avoiding constraint translation by changing variable contexts as discussed in Section \ref{sectApproach}. + \item In general, maintaining unneeded \emph{partitions of the constraint base} shall be avoided. Although we were able to speed up the creation of the constraint base by special transfer operations, it is better to directly add constraints to the constraint base where possible. Thus, we removed several unneeded temporary lists. However, adding constraints directly is not always possible due to the incremental construction while traversing the model and the prioritization of certain constraints, such as default values or constraints in eval-blocks. As an alternative, we could use a numeric prioritization stored along with the individual constraints (like the attached type information) and use a sorted data structure. So far, we tried to avoid additional information stored in the constraints or to investigate specific management records for each individual constraint that would be required when avoiding constraint translation by changing variable contexts as discussed in Section \ref{sectApproach}. - \item Determine whether a certain \emph{constraint (type) is actually needed} for reasoning and avoid its translation/creation wherever feasible. While this is easy for some constraint types, e.g., annotations in incremental reasoning mode, checking for frozen constraints would require tests in various places of the constraint translation. So far, we opted for keeping these tests in a central place for better maintainance ($add$ function), but probably testing constraint properties more early would allow for better performance. + \item Determine whether a certain \emph{constraint (type) is actually needed} for reasoning and avoid its translation/creation wherever feasible. While this is easy for some constraint types, e.g., annotations in incremental reasoning mode, checking for frozen constraints would require tests in various places of the constraint translation. So far, we opted for keeping these tests in a central place for better maintenance ($add$ function), but probably testing constraint properties more early would allow for better performance. \item Create/translate a \emph{constraint in one step}, in particular, avoid multiple variable substitutions for the same constraint. Initially, some constraint types such as those for default values and derived types were substituted partially, then collected and finally substituted again. This implies iterating multiple times over the same constraint base as well as re-creating and throwing away constraint instances. We managed to avoid such multiple substitutions and, moreover, improved the variable substitution so that new constraint instances are only created if variable substitutions are actually applied. + \item \emph{Avoid output} during reasoning. The individual functions of the reasoner are called repeatedly for larger models and so emitting output, e.g., to the console, may sum up quickly to large runtime amounts. We experienced this problem for some models where the overall reasoning was considerably slow, while intuitively complexer models were processed significantly faster (approx. factor 7.5). The difference between these models was that the respective test case accepted for the slower models a huge number of reasoning errors (around 2300) while the faster ones did not cause reasoning errors. As the reasoner was emitting reasoning errors to the console, the difference in response time was spent on emitting the errors. In other words, while translation and constraint evaluation times for the slower models was significantly faster (approx. factor 13), the default output inverted the result. + +To prevent such behavior, the reasoner configuration contains a logger instance, that can be overridden to prevent output during reasoning. This logger instance shall define specific logging methods for complex types, such as reasoning messages, to prevent that independent of the concrete logging instance, complex types are turned by default into Strings (consuming superfluous reasoning time) and emitted through the basic logging method. Further, we moved the default output code from the reasoner to the reasoning result class allowing that such time consuming output is definitively performed outside the reasoner and that only executed if intended by the caller. + \item \textbf{Reuse visitor instances} if the same visitor applied more than once (as also for other types of instances~\cite{Shirazi02}) and the visitor maintains complex internal data structures such as maps or lists. If possible, even avoid internal storage structures that are consulted only once, e.g., for transferring the results of a traversal into another data structure. To avoid this, we created an abstract visitor which leaves the handing of the results open. A concrete visitor replaces the original visitor still storing the results as before in an internal datastructure. In contrast, the visitor using by the reasoner stores the results directly in the target datastructure. Informal performance observations showed that replacing a single visitor (out of 5 used by the implementation) improved the performance on large models by roughly 80 - 100 ms, i.e., 3-4\% of the reasoning time. - \item \textbf{Use instance pooling} if temporary use of datastructures is required, pooling of instances can be rather helpful~\cite{EichelbergerSassSchmid16, Shirazi02}, but requires explicit memory managment, i.e., obtaining and releasing. For example, we pool frequently used sets of compounds in the translation of compounds, which helps saving 20-30 ms on larger models. Moreover, we pool frequently used internal structures of the expression evaluation (but don't have measurements available as this was part of the original implementation). + \item \textbf{Use instance pooling} if temporary use of datastructures is required, pooling of instances can be rather helpful~\cite{EichelbergerSassSchmid16, Shirazi02}, but requires explicit memory management, i.e., obtaining and releasing. For example, we pool frequently used sets of compounds in the translation of compounds, which helps saving 20-30 ms on larger models. Moreover, we pool frequently used internal structures of the expression evaluation (but don't have measurements available as this was part of the original implementation). \item In general, \textbf{avoid temporary instances}, e.g., iterators, temporary constraint creation, or temporary data structures wherever possible. Possible strategies are maintaining a singleton instance in single-thread processing, using instance pools or just using different types of loops. Sometimes, also applying a (reusable) functor to a collection is more efficient than an explicit loop. If permitted by the compilation settings (EASy-Producer usually is compiled against JDK 1.6), also a lambda-function instead of a heavy-weight functor class may be appropriate. Index: /reasoner/reasoner.tex =================================================================== --- /reasoner/reasoner.tex (revision 169) +++ /reasoner/reasoner.tex (revision 170) @@ -82,6 +82,6 @@ \input{consRescheduling} \input{ivmlCompleteness} +\input{performance} \input{evaluation} -\input{performance} \input{conclusion} @@ -94,179 +94,5 @@ \begin{appendices} -\section{Reasoner test cases}\label{appendixTestCases} - -In this section, we list a representative subset of the implemented reasoner test cases, which aim at validating translation and reasoning for specific IVML concepts and combinations. Please note that reasoner test cases defined as part of the test suite for the \class{ReasonerCore} bundle of EASy-Producer contains more than 310 test cases are applied to the reasoner, many constructed in a similar way for different types, i.e., exhaustively listing all of them is neither possible her nor intended. Among them, there are also more complex test cases taken from real scenarios, evaluation models as well as challenge test cases. - -For referencing the test cases, we use identifiers composed for the main concepts to be validated. We use the abbreviations shown in Table \ref{tab:testCaseIdSchema} to compose identifiers. For example, test case nuber one on Integer (I) variables used as compound (Co) slots is identified by 'ICo1'. Table \ref{tab:reasonerTestCasesBasics} and Table \ref{tab:reasonerTestCasesAdvanced} list the test cases referenced in this document. Test case names ending with 'Valid' indicate that there is an associate test case testing also the failing of the constraint under test. - -\begin{table*}[ht] -%\begin{adjustbox}{angle=90} -\centering -\begin{tabular}{|l|l|} -\hline -\textbf{Id part / abbreviation} & \textbf{IVML concept}\\ -\hline - -B & Boolean \\ -Cn & Compound \\ -Ct & Container\\ -Cs & Constraint\\ -A & Annotation, AssignmentBlock\\ -Ev & Eval\\ -I & Integer\\ -R & Real\\ -S & String\\ -E & Enum\protect\footnotemark\\ -T & Typedef\\ -F & Reference\\ -If & Interface\\ -Sc & Scope, re-assignment\\ -\hline -\end{tabular} -\caption{Test case naming schema.} -\label{tab:testCaseIdSchema} -%\end{adjustbox} -\end{table*} -\footnotetext{Similarly, test cases for ordered enums are in place, but not referenced explicitly in this document.} - - - -\begin{table*}[ht] -%\begin{adjustbox}{angle=90} -\centering -\begin{tabular}{|l|p{12.5cm}|} -\hline -\textbf{Id} & \textbf{Test model}\\ -\hline -B1 & \class{boolean/BooleanDefaultsTest.ivml}\\ -B2 & \class{boolean/BooleanAssignTest.ivml}\\ -B3 & \class{boolean/BooleanImpliesTest.ivml}\\ -BF1 & \class{boolean/BooleanRefAssignTest.ivml\protect\footnotemark}\\ -AB1 & \class{boolean/BooleanAnnotationTest.ivml}\\ -BCo1 & \class{boolean/BooleanInCompoundAssignTest.ivml}\\ -BCo3 & \class{boolean/BooleanInCompoundDefaultsTest.ivml}\\ -BCo3 & \class{boolean/BooleanInCompoundImpliesTest.ivml}\\ -BCt1 & \class{boolean/BooleanInContainerAssignTest.ivml}\\ -BCt2 & \class{boolean/BooleanInContainerDefaultsTest.ivml}\\ -I1 & \class{integer/IntegerDefaultsTest.ivml}\\ -I2 & \class{integer/IntegerAssignTest.ivml}\\ -AI1 & \class{integer/IntegerAnnotationTest.ivml}\\ -AI2 & \class{runtime/runtimeReasoning.ivml}\\ -IT1 & \class{operations/typedef.ivml}\\ -IT2 & \class{operations/typedefOfTypedefValid.ivml}\\ -ICo1 & \class{integer/IntegerInCompoundDefaultsTest.ivml}\\ -ICo2 & \class{integer/IntegerInCompoundAssignTest.ivml}\\ -ICt1 & \class{integer/IntegerInContainerAssignTest.ivml}\\ -ICt2 & \class{integer/IntegerInContainerDefaultsTest.ivml}\\ -R1 & \class{real/RealDefaultsTest.ivml}\\ -R2 & \class{real/RealAssignTest.ivml}\\ -AR1 & \class{real/RealAnnotationTest.ivml}\\ -RCo1 & \class{real/RealInCompoundDefaultsTest.ivml}\\ -RCo2 & \class{real/RealInCompoundAssignTest.ivml}\\ -RCt1 & \class{real/RealInContainerAssignTest.ivml}\\ -RCt2 & \class{real/RealInContainerDefaultsTest.ivml}\\ -S1 & \class{string/StringDefaultsTest.ivml}\\ -S2 & \class{string/StringAssignTest.ivml}\\ -AS1 & \class{string/StringAnnotationTest.ivml}\\ -SCo1 & \class{string/StringInCompoundDefaultsTest.ivml}\\ -SCo2 & \class{string/StringInCompoundAssignTest.ivml}\\ -SCt1 & \class{string/StringInContainerAssignTest.ivml}\\ -SCt2 & \class{string/StringInContainerDefaultsTest.ivml}\\ -\hline -\end{tabular} -\caption{Reasoner test cases for basic types combined with compounds or containers.} -\label{tab:reasonerTestCasesBasics} -%\end{adjustbox} -\end{table*} -\footnotetext{A similar test is defined for all basic types.} - -\begin{table*}[ht] -%\begin{adjustbox}{angle=90} -\centering -\begin{tabular}{|l|p{12.5cm}|} -\hline -\textbf{Id} & \textbf{Test model}\\ -\hline -E1 & \class{enums/EnumDefaultsTest.ivml}\\ -E2 & \class{enums/EnumAssignTest.ivml}\\ -AE1 & \class{enums/EnumAnnotationTest.ivml}\\ -EF1 & \class{enums/EnumRefAssignTest.ivml}\\ -ECo1 & \class{enums/EnumInCompoundDefaultsTest.ivml}\\ -ECo2 & \class{enums/EnumInCompoundAssignTest.ivml}\\ -ECt1 & \class{enums/EnumInContainerDefaultsTest.ivml}\\ -ECt2 & \class{enums/EnumInContainerAssignTest.ivml}\\ -CnCt1 & \class{collectionConstraints/constraintSetDefault.ivml}\\ -CnCt2 & \class{collectionConstraints/constraintSetSetDefault.ivml}\\ -CnCt3 & \class{compounds/CompoundContainerTest1.ivml}\\ -CnCt4 & \class{compounds/CompoundContainerTest2.ivml}\protect\footnotemark\\ -CnCs1 & \class{compounds/changingValues/changingValues.ivml}\\ -ACt1 & \class{collectionConstraints/ContainerAnnotationTest.ivml}\\ -ACn1 & \class{compounds/CompoundAnnotationTest.ivml}\\ -AFCn1 & \class{compounds/ReferenceCompoundAnnotationTest.ivml}\\ -ATCn1 & \class{compounds/DerivedCompoundAnnotationTest.ivml}\\ -CsCtCn1 & \class{constraintsVariables/changingConstraints/changingConstraints.ivml}\\ -CnTCt1 & \class{collectionConstraints/constraintSetDerivedCompound.ivml}\\ -CnTCt2 & \class{collectionConstraints/constraintSetSetDerivedCompound.ivml}\\ -CtTCt1 & \class{collectionConstraints/setDerivedSet.ivml}\\ -CtTF1 & \class{collectionConstraints/ReferenceDerivedCollectionTest.ivml}\\ -TFCnCtCsA1 & \class{collectionConstraints/QM.ivml}\\ -ITCn1 & \class{operations/typedefNestedInCompoundValid.ivml}\\ -ITCnCt1 & \class{operations/typedefCompoundNestedInSequenceValid1.ivml}\\ -ITCnCt2 & \class{operations/typedefCompoundNestedInSequenceValid2.ivml}\\ -ITCt1 & \class{operations/typedefNestedInSequenceValid.ivml}\\ -ITCt2 & \class{operations/typedefNestedInSetValid.ivml}\\ -CnAI1 & \class{attributes/BlockAssign.ivml}\\ -CnAI2 & \class{attributes/BlockAssignNested.ivml}\\ -CnAI3 & \class{attributes/BlockAssignNestedConstraint.ivml}\\ -FCn1 & \class{compounds/CompoundRefAssignTest1.ivml}\\ -FCn2 & \class{compounds/CompoundRefAssignTest2.ivml}\\ -FCn3 & \class{compounds/CompoundRefAssignTest3.ivml}\\ -Cn1 & \class{compounds/CompoundConstraintTest}\\ -Cn2 & \class{compounds/CompoundCompoundTest1.ivml}\\ -Cn3 & \class{compounds/CompoundCompoundTest2.ivml}\protect\footnotemark\\ -CnTF1 & \class{collectionConstraints/RefernceDerivedCompoundTest.ivml}\\ -\hline -\end{tabular} -\caption{Reasoner test cases for advanced concepts and types (part 1).} -\label{tab:reasonerTestCasesAdvanced} -%\end{adjustbox} -\end{table*} -\addtocounter{footnote}{-1} -\footnotetext{This test aims at recursive compound types.}\stepcounter{footnote} -\footnotetext{This test aims at recursive compound types.} - -\begin{table*}[ht] -%\begin{adjustbox}{angle=90} -\centering -\begin{tabular}{|l|p{12.5cm}|} -\hline -\textbf{Id} & \textbf{Test model}\\ -\hline -A1 & \class{attributes/DefaultAssign.ivml}\\ -A2 & \class{attributes/IndividualAssign.ivml}\\ -A3 & \class{attributes/IndividualAssignNested.ivml}\\ -ACs1 & \class{attributes/ConstraintAnnotationTest.ivml}\\ -CnA3 & \class{attributes/CompoundIndividualAssign.ivml}\\ -CnA4 & \class{attributes/BlockAssignNestedConstraintFail.ivml}\\ -Cs1 & \class{constraintVariables/constraintAssigned.ivml}\\ -Cs2 & \class{constraintVariables/constraintDefault.ivml}\\ -CsCn1 & \class{constraintVariables/constraintCompoundAssigned.ivml}\\ -CsCn2 & \class{constraintVariables/constraintCompoundDefault.ivml}\\ -CnFCt1 & \class{constraintVariables/QM\_constraintOrder.ivml}\\ -Ev1 & \class{evals/SimpleEval.ivml}\\ -CnEv1 & \class{evals/CompoundEval.ivml}\\ -If1 & \class{interfaces/Frozen.ivml}\\ -If2 & \class{interfaces/Unfrozen.ivml}\\ -Sc1 & \class{integer/VariableFailTest.ivml}\\ -Sc2 & \class{compounds/slotReassignment/Cfg.ivml}\\ -Sc3 & \class{constraintVariables/refinedConstraints/refinedConstraints.ivml}\\ -\hline -\end{tabular} -\caption{Reasoner test cases for advanced concepts and types (part 2).} -\label{tab:reasonerTestCasesAdvanced2} -%\end{adjustbox} -\end{table*} - - +\input{appendixTests} \end{appendices}