Index: /reasoner/evaluation.tex =================================================================== --- /reasoner/evaluation.tex (revision 252) +++ /reasoner/evaluation.tex (revision 253) @@ -58,5 +58,5 @@ \begin{itemize} \item an actual office computer: Dell Latitude 7490 laptop with Intel core i7-8650 U processor (4 physical cores at 1.9 GHz) , 32 GBytes RAM with Windows 10 Professional (10.0.17134) and Open JDK 10.0.2 64 bit. Besides Maven 3.2.3, ANT 1.10.3 with Maven-ANT-tasks 2.1.3, an Eclipse Oxygen 3a release 4.7.3a are installed. - \item a retired office computer used for the reason reveisions: Dell laptop \TBD{XXX} with Intel core i7-3367U (2 physical cores at 2.00GHz), Windows 7 version 6.1.761 SP 1 and Oracle JDK 1.8.0\_66 64 bit. Besides Maven 3.2.3, ANT 1.10.3 with Maven-ANT-tasks 2.1.3, an Eclipse Mars2 release 4.5.2 are installed. + \item a retired office computer used for the reason reveisions: Dell Latitude 6430u laptop with Intel core i7-3367U (2 physical cores at 2.00GHz), Windows 7 version 6.1.761 SP 1 and Oracle JDK 1.8.0\_66 64 bit. Besides Maven 3.2.3, ANT 1.10.3 with Maven-ANT-tasks 2.1.3, an Eclipse Mars2 release 4.5.2 are installed. \item our continuous integration server, a Ubuntu Linux 16.4.5 LTS VM with 4 GBytes RAM and OpenJDK 8 64bit. The measurement script is integrated as a manual build action that prevents any other continous build action running at the same time. \item a Raspberry Pi 3 by vendor element14 hosting an 8 GB SanDisk class-4 SD (one from \cite{KnocheEichelberger18}) with Raspbian Stretch Lite version November 2018, Linux Kernel 4.14 and Oracle JDK 1.8.\TBD{65} ARM and, as alternative for some experiments, Oracle JDK 1.8.0\_201 ARM. @@ -202,4 +202,89 @@ \subsection{Results}\label{sectEvaluationResults} +\begin{figure}[!htb] + \centering + \includegraphics[width=\linewidth]{figures/benchmark-results-old-1_1_0-jdk1_8-20190215_reasoningTime_all.pdf} + \hfill + \includegraphics[width=\linewidth]{figures/benchmark-results-old-1_1_0-jdk1_8-20190215_translationTime_all.pdf} + \hfill + \includegraphics[width=\linewidth]{figures/benchmark-results-old-1_1_0-jdk1_8-20190215_evaluationTime_all.pdf} + \caption{Reasoner 1.10 on the Dell Latitude 6430u using Oracle JDK 1.8.}\label{fig:old-1_10-jdk8} +\end{figure} + +\clearpage + +\begin{figure}[!htb] + \centering + \includegraphics[width=\linewidth]{figures/benchmark-results-old-1_3_0-jdk1_8-20190215_reasoningTime_all.pdf} + \hfill + \includegraphics[width=\linewidth]{figures/benchmark-results-old-1_3_0-jdk1_8-20190215_translationTime_all.pdf} + \hfill + \includegraphics[width=\linewidth]{figures/benchmark-results-old-1_3_0-jdk1_8-20190215_evaluationTime_all.pdf} + \caption{Reasoner 1.30 on the Dell Latitude 6430u using Oracle JDK 1.8.}\label{fig:old-1_30-jdk8} +\end{figure} + +\clearpage + +\begin{figure}[!htb] + \centering + \includegraphics[width=\linewidth]{figures/benchmark-results-new-1_3_0-ojdk1_8-20190215_reasoningTime_all.pdf} + \hfill + \includegraphics[width=\linewidth]{figures/benchmark-results-new-1_3_0-ojdk1_8-20190215_translationTime_all.pdf} + \hfill + \includegraphics[width=\linewidth]{figures/benchmark-results-new-1_3_0-ojdk1_8-20190215_evaluationTime_all.pdf} + \caption{Reasoner 1.30 on the Dell Latitude 7490 using Open JDK 1.8.}\label{fig:new-1_30-ojdk8} +\end{figure} + +\clearpage + +\begin{figure}[!htb] + \centering + \includegraphics[width=\linewidth]{figures/benchmark-results-new-1_3_0-ojdk10-20190215_reasoningTime_all.pdf} + \hfill + \includegraphics[width=\linewidth]{figures/benchmark-results-new-1_3_0-ojdk10-20190215_translationTime_all.pdf} + \hfill + \includegraphics[width=\linewidth]{figures/benchmark-results-new-1_3_0-ojdk10-20190215_evaluationTime_all.pdf} + \caption{Reasoner 1.30 on the Dell Latitude 7490 using Open JDK 10.}\label{fig:new-1_30-ojdk10} +\end{figure} + +\clearpage + +\begin{figure}[!htb] + \centering + \includegraphics[width=\linewidth]{figures/benchmark-results-jenkins-1_3_0-20190215_reasoningTime_all.pdf} + \hfill + \includegraphics[width=\linewidth]{figures/benchmark-results-jenkins-1_3_0-20190215_translationTime_all.pdf} + \hfill + \includegraphics[width=\linewidth]{figures/benchmark-results-jenkins-1_3_0-20190215_evaluationTime_all.pdf} + \caption{Reasoner 1.30 on Jenkins Ubuntu Open JDK 1.8.}\label{fig:new-1_30-ojdk10} +\end{figure} + +\clearpage + +\begin{figure}[!htb] + \centering + \includegraphics[width=\linewidth]{figures/benchmark-results-pi-1_1_0-jdk1_8-20190219_reasoningTime_all.pdf} + \hfill + \includegraphics[width=\linewidth]{figures/benchmark-results-pi-1_1_0-jdk1_8-20190219_translationTime_all.pdf} + \hfill + \includegraphics[width=\linewidth]{figures/benchmark-results-pi-1_1_0-jdk1_8-20190219_evaluationTime_all.pdf} + \caption{Reasoner 1.10 on a Pi 3 using Oracle JDK 1.8.}\label{fig:pi-1_10-jdk8} +\end{figure} + +\clearpage + +\begin{figure}[!htb] + \centering + \includegraphics[width=\linewidth]{figures/benchmark-results-pi-1_3_0-jdk1_8-20190219_reasoningTime_all.pdf} + \hfill + \includegraphics[width=\linewidth]{figures/benchmark-results-pi-1_3_0-jdk1_8-20190219_translationTime_all.pdf} + \hfill + \includegraphics[width=\linewidth]{figures/benchmark-results-pi-1_3_0-jdk1_8-20190219_evaluationTime_all.pdf} + \caption{Reasoner 1.30 on a Pi 3 using Oracle JDK 1.8.}\label{fig:pi-1_30-jdk8} +\end{figure} + +\clearpage + + \TBD{Potential topics: Index: /reasoner/measures/script.r =================================================================== --- /reasoner/measures/script.r (revision 252) +++ /reasoner/measures/script.r (revision 253) @@ -6,4 +6,6 @@ #setup +#data directory +dataBaseDir <- "W:/offlineFiles/EASy-concepts/reasoner/measures" #how many initial in-experiment runs (out of 10) shall be skipped skipRuns <- 3 @@ -125,5 +127,5 @@ #unpack result of summary so that names are taken over in colum names #see P. Tuma, do we teach the right statistics? that for some forms of statistics aggregation shall be ok - tmp.table <- tmp.table[,as.list(unlist(lapply(.SD, my.summary))), list(URI, tag, model.name, MODEL_COMPLEXITY), .SDcols=statCols] + tmp.table <- tmp.table[,as.list(unlist(lapply(.SD, my.summary))), list(URI, tag, model.name, MODEL_COMPLEXITY, REASONER_CONSTRAINT_COUNT), .SDcols=statCols] tmp.groupedTables[[f]] <- tmp.table } @@ -143,14 +145,31 @@ } +# emulate the ggplot2 colors for n given colors +#https://stackoverflow.com/questions/8197559/emulate-ggplot2-default-color-palette +gg_color_hue <- function(n) { + hues = seq(15, 375, length = n + 1) + hcl(h = hues, l = 65, c = 100)[1:n] +} + # draws a diagram for data using xCol (default MODEL_COMPLEXITY), y-axis and error-bar y values as given # URI as label (for interactive use ggplotly(tooltip=c("label")) after plot(gg)) and tags as values. # default settings can be changed through data$... or column names as strings -my.createErrorBarDiagram = function(data, yCol, yColName, yBarCol, diagramTitle="", xCol="MODEL_COMPLEXITY", xColName="model complexity", labelCol="URI", colorCol="tag", colorLegendName="", regression=TRUE, sizeFactor=5) { +my.createErrorBarDiagram = function(data, yCol, yColName, yBarCol, diagramTitle="", xCol="MODEL_COMPLEXITY", xColName="model complexity", labelCol="URI", colorCol="tag", colorLegendName="", regression=TRUE, sizeFactor=5, legendPos="right") { size <- length(data) + #ggplot seems to do levels(factor(.)) by its own + legendFactors <- my.get(data, colorCol) + #recalculate as otherwise legend is outdated + legendLabels <- levels(factor(legendFactors)) bar.width = size * sizeFactor # unclear, with seems to be correlated to the size, stackoverflow.com/questions/19420903/wideth-of-error-bars-in-ggplot2 - gg<-ggplot(data, aes(x=my.get(data, xCol),y=my.get(data, yCol),label=my.get(data, URI),color=my.get(data, tag))) + + gg<-ggplot(data, aes(x=my.get(data, xCol),y=my.get(data, yCol),label=my.get(data, URI),color=legendFactors,shape=legendFactors)) + geom_errorbar(aes(ymin=my.get(data, yCol)-my.get(data, yBarCol), ymax=my.get(data, yCol)+my.get(data, yBarCol)), width=bar.width) + geom_point() + - labs(x=xColName, y=yColName, title = diagramTitle, color=colorLegendName) + labs(x=xColName, y=yColName, title = diagramTitle, color=colorLegendName) + + #override color and shape in initial aes to have only a single legend + #http://www.sthda.com/english/wiki/ggplot2-point-shapes + #rep just to be sure if data changes + scale_shape_manual(name="", labels=legendLabels, values=rep(c(0, 1, 2, 5, 6, 15, 17, 19))) + + scale_color_manual(name="", labels=legendLabels, values=gg_color_hue(length(legendLabels))) + + theme(legend.position=legendPos) if (regression) { gg <- gg + geom_smooth(method=lm) @@ -186,4 +205,17 @@ name <- paste(prefix, name, "_", indicator, ".pdf", sep="") return (paste(dir, name, sep="/")) +} + +my.printStats = function(data, name) { + print(paste(name, min(data), mean(data), max(data), sep=" ")) +} + +my.printStatistics = function(data, dir, indicator) { + print(paste("#tests", nrow(data), sep=" ")) + my.printStats(data$REASONING_TIME.mean, "reasoning time") + my.printStats(data$REASONER_TRANSLATION_TIME.mean, "translation time") + my.printStats(data$REASONER_EVALUATION_TIME.mean, "evaluation time") + my.printStats(data$REASONER_INSTANCE_CREATION_TIME.mean, "instance creation time") + my.printStats(data$REASONER_CONSTRAINT_COUNT, "constraints count") } @@ -223,4 +255,6 @@ filename=my.composeFileName(dir, "evaluationTime", indicator), plot=gg) + + my.printStatistics(data, dir, indicator) } @@ -247,18 +281,9 @@ } -my.processAll("W:/offlineFiles/EASy-concepts/reasoner/measures") +my.processAll(dataBaseDir) + +# DEBUGGING STUFF #dirs <- list.dirs(path="W:/offlineFiles/EASy-concepts/reasoner/measures", recursive=FALSE) -#for (d in dirs) { -# if (!is.na(str_match(d, "benchmark-results.*"))) { -# print(paste("processing folder ", d)) -# print(" loading data (1)") -# data <- my.readData(d, 1) -# my.createDiagrams(data, d, 1) -# print(" loading data (all)") -# data <- my.readData(d) -# my.createDiagrams(data, d, "all") -# } -#} #data <- my.readFile('W:/offlineFiles/EASy-concepts/reasoner/measures/new', 'measurements-sse.tsv')