Index: /reasoner/evaluation.tex =================================================================== --- /reasoner/evaluation.tex (revision 268) +++ /reasoner/evaluation.tex (revision 269) @@ -224,5 +224,5 @@ 4 & \ref{newLaptopId} & 1.3.0 & 10* & 433 & 15 & 493 & 53 & 6000 & 603 & 23696 & 55\\ 5 & \ref{jenkinsId} & 1.3.0 & 1.8.0 & 427 & 13 & 480 & 53 & 6000 & 494 & 19466 & 104\\ -6 & \ref{piId} & 1.1.0 & 1.8.0 & 400 & 958 & 28651 & 978 & 24110 & 1679 & 46487 &\\ +6 & \ref{piId} & 1.1.0 & 1.8.0 & 400 & 548 & 28613 & 723 & 24110 & 1092 & 46477 & 366\\ 7 & \ref{piId} & 1.3.0 & 1.8.0 & 433 & 255 & 6117 & 53 & 6000 & 815 & 23708 &\\ \hline @@ -234,5 +234,5 @@ \end{table*} -Table \ref{tab:experimentsDescSummary} summarizes for the initial run the number of tests, the measured reasoning time, the created number of constraints, the required constraint evaluations. For providing an indication of the overall time consumption of the experiment, we indicate the total execution time for the 5 repeated executions of the testsuites. We do not denote the minimum reasoning time, number of constraints or evaluations, respectively, the minimum number in all cases is 0 due to IVML tests without constraints. +Table \ref{tab:experimentsDescSummary} summarizes for the initial run the number of tests, the measured reasoning time, the created number of constraints, the required constraint evaluations. For providing an indication of the overall time consumption of the experiment, we indicate the total execution time for the 5 repeated executions of the testsuites. We list number of tests, constraints and evaluations although they shall be the same for the same treatments, because in the pre-experiments the numbers indicated that due to minor bugs the experiments were done over different models. We do not denote the minimum reasoning time, number of constraints or evaluations, respectively, the minimum number in all cases is 0 due to IVML tests without constraints. On the original device \ref{oldLaptopId}, we execute both versions of the reasoner on the same JDK (id 1 and 2). On the newer \ref{newLaptopId} we continue with the more recent reasoner version in a more modern (id 3, 4). In this setup, we use two different JDKs, Java 8 to enable a comparison with \ref{oldLaptopId}, and Java 10 to give an outlook on more recent JVMs. On the continuous integration machine (\ref{jenkinsId}, id 5) we just measure reasoner version 1.3.0 (corresponding to the most recent commit at that time). Finally, on the Pi device \ref{piId} we measure both reasoner versions on the same JDK (id 6 and 7). @@ -303,9 +303,9 @@ \begin{figure}[!htb] \centering - \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-old-1_1_0-jdk1_8-20190215_reasoningTime_all.pdf} - \hfill - \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-old-1_1_0-jdk1_8-20190215_translationTime_all.pdf} - \hfill - \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-old-1_1_0-jdk1_8-20190215_evaluationTime_all.pdf} + \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-old-1_1_0-jdk1_8-20190308_reasoningTime_all.pdf} + \hfill + \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-old-1_1_0-jdk1_8-20190308_translationTime_all.pdf} + \hfill + \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-old-1_1_0-jdk1_8-20190308_evaluationTime_all.pdf} \caption{Reasoner 1.10 on the Dell Latitude 6430u using Oracle JDK 1.8.}\label{fig:old-1_10-jdk8} \end{figure} @@ -315,9 +315,9 @@ \begin{figure}[!htb] \centering - \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-old-1_3_0-jdk1_8-20190215_reasoningTime_all.pdf} - \hfill - \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-old-1_3_0-jdk1_8-20190215_translationTime_all.pdf} - \hfill - \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-old-1_3_0-jdk1_8-20190215_evaluationTime_all.pdf} + \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-old-1_3_0-jdk1_8-20190308_reasoningTime_all.pdf} + \hfill + \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-old-1_3_0-jdk1_8-20190308_translationTime_all.pdf} + \hfill + \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-old-1_3_0-jdk1_8-20190308_evaluationTime_all.pdf} \caption{Reasoner 1.30 on the Dell Latitude 6430u using Oracle JDK 1.8.}\label{fig:old-1_30-jdk8} \end{figure} @@ -327,9 +327,9 @@ \begin{figure}[!htb] \centering - \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-new-1_3_0-ojdk1_8-20190215_reasoningTime_all.pdf} - \hfill - \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-new-1_3_0-ojdk1_8-20190215_translationTime_all.pdf} - \hfill - \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-new-1_3_0-ojdk1_8-20190215_evaluationTime_all.pdf} + \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-new-1_3_0-ojdk1_8-20190308_reasoningTime_all.pdf} + \hfill + \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-new-1_3_0-ojdk1_8-20190308_translationTime_all.pdf} + \hfill + \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-new-1_3_0-ojdk1_8-20190308_evaluationTime_all.pdf} \caption{Reasoner 1.30 on the Dell Latitude 7490 using Open JDK 1.8.}\label{fig:new-1_30-ojdk8} \end{figure} @@ -339,9 +339,9 @@ \begin{figure}[!htb] \centering - \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-new-1_3_0-ojdk10-20190215_reasoningTime_all.pdf} - \hfill - \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-new-1_3_0-ojdk10-20190215_translationTime_all.pdf} - \hfill - \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-new-1_3_0-ojdk10-20190215_evaluationTime_all.pdf} + \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-new-1_3_0-ojdk10-20190308_reasoningTime_all.pdf} + \hfill + \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-new-1_3_0-ojdk10-20190308_translationTime_all.pdf} + \hfill + \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-new-1_3_0-ojdk10-20190308_evaluationTime_all.pdf} \caption{Reasoner 1.30 on the Dell Latitude 7490 using Open JDK 10.}\label{fig:new-1_30-ojdk10} \end{figure} @@ -351,9 +351,9 @@ \begin{figure}[!htb] \centering - \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-jenkins-1_3_0-20190215_reasoningTime_all.pdf} - \hfill - \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-jenkins-1_3_0-20190215_translationTime_all.pdf} - \hfill - \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-jenkins-1_3_0-20190215_evaluationTime_all.pdf} + \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-jenkins-1_3_0-jdk1_8-20190308_reasoningTime_all.pdf} + \hfill + \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-jenkins-1_3_0-jdk1_8-20190308_translationTime_all.pdf} + \hfill + \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-jenkins-1_3_0-jdk1_8-20190308_evaluationTime_all.pdf} \caption{Reasoner 1.30 on Jenkins Ubuntu Open JDK 1.8.}\label{fig:jenkins-1_30-ojdk10} \end{figure} @@ -363,9 +363,9 @@ \begin{figure}[!htb] \centering - \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-pi-1_1_0-jdk1_8-20190219_reasoningTime_all.pdf} - \hfill - \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-pi-1_1_0-jdk1_8-20190219_translationTime_all.pdf} - \hfill - \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-pi-1_1_0-jdk1_8-20190219_evaluationTime_all.pdf} + \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-pi-1_1_0-jdk1_8-20190308_reasoningTime_all.pdf} + \hfill + \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-pi-1_1_0-jdk1_8-20190308_translationTime_all.pdf} + \hfill + \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-pi-1_1_0-jdk1_8-20190308_evaluationTime_all.pdf} \caption{Reasoner 1.10 on a Pi 3 using Oracle JDK 1.8.}\label{fig:pi-1_10-jdk8} \end{figure} @@ -380,5 +380,5 @@ \hfill \includegraphics[scale=\evalPDFscale]{figures/benchmark-results-pi-1_3_0-jdk1_8-20190219_evaluationTime_all.pdf} - \caption{Reasoner 1.30 on a Pi 3 using Oracle JDK 1.8.}\label{fig:pi-1_30-jdk8} + \caption{Reasoner 1.30 on a Pi 3 using Oracle JDK 1.8. \TBD{still old}}\label{fig:pi-1_30-jdk8} \end{figure} Index: /reasoner/measures/log.txt =================================================================== --- /reasoner/measures/log.txt (revision 268) +++ /reasoner/measures/log.txt (revision 269) @@ -1,3 +1,3 @@ -[1] "processing folder W:/offlineFiles/EASy-concepts/reasoner/measures/benchmark-results-jenkins-1_3_0-20190308" +[1] "processing folder W:/offlineFiles/EASy-concepts/reasoner/measures/benchmark-results-jenkins-1_3_0-jdk1_8-20190308" [1] "#tests;427" [1] "reasoning time;0;13.3681833389093;480.428571428571" @@ -12,5 +12,5 @@ [1] "constraints count;0;51.3064876957494;6000" [1] "reevaluation count;0;473.795877277085;19468.4" -[1] "processing folder W:/offlineFiles/EASy-concepts/reasoner/measures/benchmark-results-new-1.3-ojdk1.8-20190308" +[1] "processing folder W:/offlineFiles/EASy-concepts/reasoner/measures/benchmark-results-new-1_3_0-ojdk1_8-20190308" [1] "#tests;433" [1] "reasoning time;0;14.6421972946222;502.142857142857" @@ -25,5 +25,5 @@ [1] "constraints count;0;50.9177777777778;6000" [1] "reevaluation count;0;581.779746031746;23693.5714285714" -[1] "processing folder W:/offlineFiles/EASy-concepts/reasoner/measures/benchmark-results-new-1.3-ojdk10-20190308" +[1] "processing folder W:/offlineFiles/EASy-concepts/reasoner/measures/benchmark-results-new-1_3_0-ojdk10-20190308" [1] "#tests;433" [1] "reasoning time;0;14.8512042230287;492.857142857143" @@ -38,5 +38,5 @@ [1] "constraints count;0;50.7256637168142;6000" [1] "reevaluation count;0;579.44608091024;23693.5714285714" -[1] "processing folder W:/offlineFiles/EASy-concepts/reasoner/measures/benchmark-results-old-1.1.0-jdk1.8-20190308" +[1] "processing folder W:/offlineFiles/EASy-concepts/reasoner/measures/benchmark-results-old-1_1_0-jdk1_8-20190308" [1] "#tests;400" [1] "reasoning time;0;16.8614285714286;670.857142857143" @@ -51,5 +51,5 @@ [1] "constraints count;0;685.332151300236;24110" [1] "reevaluation count;0;1035.12715298886;46469.1428571429" -[1] "processing folder W:/offlineFiles/EASy-concepts/reasoner/measures/benchmark-results-old-1.3.0-jdk1.8-20190308" +[1] "processing folder W:/offlineFiles/EASy-concepts/reasoner/measures/benchmark-results-old-1_3_0-jdk1_8-20190308" [1] "#tests;433" [1] "reasoning time;0;11.6585285384362;390" @@ -64,28 +64,15 @@ [1] "constraints count;0;51.3139013452915;6000" [1] "reevaluation count;0;586.722293401666;23695.4285714286" -[1] "processing folder W:/offlineFiles/EASy-concepts/reasoner/measures/benchmark-results-pi-1.1-jdk1.8-20190308" +[1] "processing folder W:/offlineFiles/EASy-concepts/reasoner/measures/benchmark-results-pi-1_1_0-jdk1_8-20190308" [1] "#tests;400" -[1] "reasoning time;0;958.052678571429;28651.7142857143" -[1] "translation time;0;14.34125;252.428571428571" -[1] "evaluation time;0;831.131071428571;26645.4285714286" -[1] "constraints count;0;977.62375;24110" -[1] "reevaluation count;0;1679.00803571429;46487.8571428571" -[1] "#tests;416" -[1] "reasoning time;0.0285714285714286;920.879407051282;28460.2857142857" -[1] "translation time;0;13.8444482600733;242.2" -[1] "evaluation time;0;798.885966117216;26417.3428571429" -[1] "constraints count;0;941.034855769231;24110" -[1] "reevaluation count;0;1615.95147664835;46487.8571428571" -[1] "processing folder W:/offlineFiles/EASy-concepts/reasoner/measures/benchmark-results-pi-1.3-jdk1.8-20190308" -[1] "#tests;433" -[1] "reasoning time;0;254.985015946332;6117.28571428571" -[1] "translation time;0;104.677691630925;2750.14285714286" -[1] "evaluation time;0;78.3698724293413;2594.85714285714" -[1] "constraints count;0;52.6235565819861;6000" -[1] "reevaluation count;0;815.268228307489;23708.2857142857" -[1] "#tests;454" -[1] "reasoning time;0.0571428571428571;244.176431718062;6107.45714285714" -[1] "translation time;0;100.193953220055;2782.71428571429" -[1] "evaluation time;0;74.8047566603734;2528.48571428571" -[1] "constraints count;0;50.5374449339207;6000" -[1] "reevaluation count;0;779.189175582127;23702.2285714286" +[1] "reasoning time;0;548.705714285714;28613.7142857143" +[1] "translation time;0;10.8860714285714;217.571428571429" +[1] "evaluation time;0;458.746785714286;26559" +[1] "constraints count;0;723.18125;24110" +[1] "reevaluation count;0;1092.57839285714;46477.7142857143" +[1] "#tests;417" +[1] "reasoning time;0.0285714285714286;526.194826995546;28313.8857142857" +[1] "translation time;0;10.6558752997602;247.114285714286" +[1] "evaluation time;0;439.059883521754;26285.2" +[1] "constraints count;0;694.81654676259;24110" +[1] "reevaluation count;0;1049.49061322371;46483.8"