Merge pull request #5 from harschware/master

Added a suite runner that outputs a CSV with runtimes and log files for ...

.gitignore

@@ -0,0 +1,5 @@
+target/
+tpcds_kit.zip
+tpch_kit.zip
+*.sql.log
+derby.log

runSuite.pl

@@ -0,0 +1,75 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+use File::Basename;
+
+# PROTOTYPES
+sub dieWithUsage(;$);
+
+# GLOBALS
+my $SCRIPT_NAME = basename( __FILE__ );
+my $SCRIPT_PATH = dirname( __FILE__ );
+
+# MAIN
+dieWithUsage("one or more parameters not defined") unless @ARGV == 1;
+my $suite = shift;
+dieWithUsage("suite name required") unless $suite eq "tpcds" or $suite eq "tpch";
+
+chdir $SCRIPT_PATH;
+if( $suite eq 'tpcds' ) {
+	chdir "sample-queries-tpcds";
+} else {
+	chdir 'sample-queries-tpch';
+} # end if
+my @queries = glob '*.sql';
+
+my $db = { 
+	'tpcds' => 'tpcds_bin_partitioned_orc_2',
+	'tpch' => 'tpch_flat_orc_2'
+};
+
+print "filename,status,time,rows\n";
+for my $query ( @queries ) {
+	my $logname = "$query.log";
+	my $cmd="echo 'use $db->{${suite}}; source $query;' | hive -i testbench.settings 2>&1  | tee $query.log";
+#	my $cmd="cat $query.log";
+	#print $cmd ; exit;
+	
+	my $hiveStart = time();
+
+	my @hiveoutput=`$cmd`;
+	die "${SCRIPT_NAME}:: ERROR:  hive command unexpectedly exited \$? = '$?', \$! = '$!'" if $?;
+
+	my $hiveEnd = time();
+	my $hiveTime = $hiveEnd - $hiveStart;
+	foreach my $line ( @hiveoutput ) {
+		if( $line =~ /Time taken:\s+([\d\.]+)\s+seconds,\s+Fetched:\s+(\d+)\s+row/ ) {
+			print "$query,success,$hiveTime,$2\n"; 
+		} elsif( 
+			$line =~ /^FAILED: /
+			# || /Task failed!/ 
+			) {
+			print "$query,failed,$hiveTime\n"; 
+		} # end if
+	} # end while
+} # end for
+
+
+sub dieWithUsage(;$) {
+	my $err = shift || '';
+	if( $err ne '' ) {
+		chomp $err;
+		$err = "ERROR: $err\n\n";
+	} # end if
+
+	print STDERR <<USAGE;
+${err}Usage:
+	perl ${SCRIPT_NAME} [tpcds|tpch]
+
+Description:
+	This script runs the sample queries and outputs a CSV file of the time it took each query to run.  Also, all hive output is kept as a log file named 'queryXX.sql.log' for each query file of the form 'queryXX.sql'.
+USAGE
+	exit 1;
+}
+

tpcds-gen/Makefile

@@ -1,3 +1,4 @@
+MYOS=$(shell uname -s)
 
 all: target/lib/dsdgen.jar target/tpcds-gen-1.0-SNAPSHOT.jar
 
@@ -15,7 +16,8 @@ target/lib/dsdgen.jar: target/tools/dsdgen
 	cd target/; mkdir -p lib/; ( jar cvf lib/dsdgen.jar tools/ || gjar cvf lib/dsdgen.jar tools/ )
 
 target/tools/dsdgen: target/tpcds_kit.zip
-	test -d target/tools/ || (cd target; unzip tpcds_kit.zip; cd tools; cat ../../*.patch | patch -p0 )
+	test -d target/tools/ || (cd target; unzip tpcds_kit.zip; cd tools; cat ../../patches/all/*.patch | patch -p0 )
+	cd target/tools; cat ../../patches/${MYOS}/*.patch | patch -p1
 	cd target/tools; make clean; make dsdgen
 
 clean:

tpcds-gen/patches/Darwin/macosx.patch

@@ -0,0 +1,168 @@
+diff -rupN tools/Makefile.suite toolsnew/Makefile.suite
+--- tools/Makefile.suite	2012-04-25 11:03:50.000000000 -0700
++++ toolsnew/Makefile.suite	2014-06-25 13:15:00.000000000 -0700
+@@ -38,8 +38,8 @@
+ ################
+ ## TARGET OS HERE
+ ################
+-# OS Values: AIX, LINUX, SOLARIS, NCR, HPUX
+-OS	=	LINUX
++# OS Values: AIX, LINUX, SOLARIS, NCR, HPUX, OSX
++OS	=	OSX
+ ###########
+ # No changes should be necessary below this point
+ # Each compile variable is adjusted for the target platform using the OS setting above
+@@ -47,7 +47,8 @@ OS	=	LINUX
+ # CC
+ AIX_CC		= xlC
+ HPUX_CC		= gcc
+-LINUX_CC		= gcc
++LINUX_CC	= gcc
++OSX_CC		= gcc
+ NCR_CC		= cc
+ SOLARIS_CC	= gcc
+ SOL86_CC	= cc
+@@ -55,7 +56,8 @@ CC		= $($(OS)_CC)
+ # CFLAGS
+ AIX_CFLAGS		= -q64 -O3 -D_LARGE_FILES
+ HPUX_CFLAGS		= -O3 -Wall
+-LINUX_CFLAGS	= -g -Wall
++LINUX_CFLAGS		= -g -Wall
++OSX_CFLAGS		= -g -Wall
+ NCR_CFLAGS		= -g 
+ SOLARIS_CFLAGS	= -O3 -Wall
+ SOL86_CFLAGS	= -O3 
+@@ -65,6 +67,7 @@ CFLAGS			= $(BASE_CFLAGS) -D$(OS) $($(OS
+ AIX_EXE	= 
+ HPUX_EXE	= 
+ LINUX_EXE	= 
++OSX_EXE		= 
+ NCR_EXE		= 
+ SOLARIS_EXE	= 
+ SOL86_EXE	= 
+@@ -73,6 +76,7 @@ EXE		= $($(OS)_EXE)
+ AIX_LEX		= flex
+ HPUX_LEX	= flex
+ LINUX_LEX	= lex
++OSX_LEX		= lex
+ NCR_LEX		= lex
+ SOLARIS_LEX	= lex
+ SOL86_LEX	= lex
+@@ -81,6 +85,7 @@ LEX		= $($(OS)_LEX)
+ AIX_LIBS	= -lm
+ HPUX_LIBS	= -lm -ll
+ LINUX_LIBS	= -lm
++OSX_LIBS	= -lm
+ NCR_LIBS	= -lm -lc89
+ SOLARIS_LIBS	= -ly -ll -lm
+ SOL86_LIBS	= -ly -ll -lm
+@@ -89,6 +94,7 @@ LIBS		= $($(OS)_LIBS)
+ AIX_YACC	= yacc
+ HPUX_YACC	= bison -y
+ LINUX_YACC	= yacc
++OSX_YACC	= yacc
+ NCR_YACC	= yacc
+ SOLARIS_YACC	= yacc
+ SOL86_YACC	= yacc
+@@ -97,6 +103,7 @@ YACC		= $($(OS)_YACC)
+ AIX_YFLAGS	= -d -v
+ HPUX_YFLAGS	= -y -d -v
+ LINUX_YFLAGS	= -d -v
++OSX_YFLAGS	= -d -v
+ NCR_YFLAGS	= -d -v
+ SOLARIS_YFLAGS	= -d -v
+ SOL86_YFLAGS	= -d -v
+diff -rupN tools/config.h toolsnew/config.h
+--- tools/config.h	2012-04-25 11:03:52.000000000 -0700
++++ toolsnew/config.h	2014-06-25 13:15:00.000000000 -0700
+@@ -109,6 +109,18 @@
+ #define FLEX
+ #endif /* LINUX */
+ 
++#ifdef OSX
++#define SUPPORT_64BITS
++#define HUGE_TYPE       int64_t
++#define HUGE_FORMAT     "%lld"
++#define HUGE_COUNT      1
++#define USE_STRING_H
++#define USE_LIMITS_H
++#define MAXINT INT_MAX
++#define USE_STDLIB_H
++#define FLEX
++#endif /* OSX */
++
+ #ifdef SOLARIS
+ #define SUPPORT_64BITS
+ #define HUGE_TYPE	long long 
+diff -rupN tools/makefile toolsnew/makefile
+--- tools/makefile	2012-04-25 11:03:54.000000000 -0700
++++ toolsnew/makefile	2014-06-25 13:15:00.000000000 -0700
+@@ -38,8 +38,8 @@
+ ################
+ ## TARGET OS HERE
+ ################
+-# OS Values: AIX, LINUX, SOLARIS, NCR, HPUX
+-OS	=	LINUX
++# OS Values: AIX, LINUX, SOLARIS, NCR, HPUX, OSX
++OS	=	OSX
+ ###########
+ # No changes should be necessary below this point
+ # Each compile variable is adjusted for the target platform using the OS setting above
+@@ -47,7 +47,8 @@ OS	=	LINUX
+ # CC
+ AIX_CC		= xlC
+ HPUX_CC		= gcc
+-LINUX_CC		= gcc
++LINUX_CC	= gcc
++OSX_CC		= gcc
+ NCR_CC		= cc
+ SOLARIS_CC	= gcc
+ SOL86_CC	= cc
+@@ -56,6 +57,7 @@ CC		= $($(OS)_CC)
+ AIX_CFLAGS		= -q64 -O3 -D_LARGE_FILES
+ HPUX_CFLAGS		= -O3 -Wall
+ LINUX_CFLAGS	= -g -Wall
++OSX_CFLAGS	= -g -Wall -I/usr/include/malloc
+ NCR_CFLAGS		= -g 
+ SOLARIS_CFLAGS	= -O3 -Wall
+ SOL86_CFLAGS	= -O3 
+@@ -65,6 +67,7 @@ CFLAGS			= $(BASE_CFLAGS) -D$(OS) $($(OS
+ AIX_EXE	= 
+ HPUX_EXE	= 
+ LINUX_EXE	= 
++OSX_EXE	= 
+ NCR_EXE		= 
+ SOLARIS_EXE	= 
+ SOL86_EXE	= 
+@@ -73,6 +76,7 @@ EXE		= $($(OS)_EXE)
+ AIX_LEX		= flex
+ HPUX_LEX	= flex
+ LINUX_LEX	= lex
++OSX_LEX		= lex
+ NCR_LEX		= lex
+ SOLARIS_LEX	= lex
+ SOL86_LEX	= lex
+@@ -81,6 +85,7 @@ LEX		= $($(OS)_LEX)
+ AIX_LIBS	= -lm
+ HPUX_LIBS	= -lm -ll
+ LINUX_LIBS	= -lm
++OSX_LIBS	= -lm
+ NCR_LIBS	= -lm -lc89
+ SOLARIS_LIBS	= -ly -ll -lm
+ SOL86_LIBS	= -ly -ll -lm
+@@ -89,6 +94,7 @@ LIBS		= $($(OS)_LIBS)
+ AIX_YACC	= yacc
+ HPUX_YACC	= bison -y
+ LINUX_YACC	= yacc
++OSX_YACC	= yacc
+ NCR_YACC	= yacc
+ SOLARIS_YACC	= yacc
+ SOL86_YACC	= yacc
+@@ -97,6 +103,7 @@ YACC		= $($(OS)_YACC)
+ AIX_YFLAGS	= -d -v
+ HPUX_YFLAGS	= -y -d -v
+ LINUX_YFLAGS	= -d -v
++OSX_YFLAGS	= -d -v
+ NCR_YFLAGS	= -d -v
+ SOLARIS_YFLAGS	= -d -v
+ SOL86_YFLAGS	= -d -v

tpcds-gen/patches/all/tpcds_misspelled_header_guard.patch

@@ -0,0 +1,11 @@
+--- w_store_sales.h.orig	2014-06-25 10:58:19.000000000 -0700
++++ w_store_sales.h	2014-06-25 10:58:51.000000000 -0700
+@@ -34,7 +34,7 @@
+  * Gradient Systems
+  */ 
+ #ifndef W_STORE_SALES_H
+-#define W_STORE_SLAES_H
++#define W_STORE_SALES_H
+ 
+ #include "constants.h"
+ #include "pricing.h"

tpch-gen/Makefile

@@ -1,3 +1,4 @@
+MYOS=$(shell uname -s)
 
 all: target/lib/dbgen.jar target/tpch-gen-1.0-SNAPSHOT.jar
 
@@ -16,7 +17,8 @@ target/lib/dbgen.jar: target/tools/dbgen
 
 target/tools/dbgen: target/tpch_kit.zip
 	test -d target/tools/ || (cd target; unzip tpch_kit.zip -x __MACOSX/; ln -sf $$PWD/*/dbgen/ tools)
-	cd target/tools/; make -f makefile.suite clean; make -f makefile.suite CC=gcc DATABASE=ORACLE MACHINE=LINUX WORKLOAD=TPCH
+	cd target/tools; cat ../../../patches/${MYOS}/*.patch | patch -p0
+	cd target/tools; make -f makefile.suite clean; make -f makefile.suite CC=gcc DATABASE=ORACLE MACHINE=LINUX WORKLOAD=TPCH
 
 clean:
 	mvn clean

tpch-gen/patches/Darwin/macosx.patch

@@ -0,0 +1,11 @@
+--- makefile.suite.orig	2014-06-25 15:40:27.000000000 -0700
++++ makefile.suite	2014-06-25 15:42:03.000000000 -0700
+@@ -110,7 +110,7 @@
+ MACHINE = 
+ WORKLOAD = 
+ #
+-CFLAGS	= -g -DDBNAME=\"dss\" -D$(MACHINE) -D$(DATABASE) -D$(WORKLOAD) -DRNG_TEST -D_FILE_OFFSET_BITS=64 
++CFLAGS	= -g -DDBNAME=\"dss\" -D$(MACHINE) -D$(DATABASE) -D$(WORKLOAD) -DRNG_TEST -D_FILE_OFFSET_BITS=64  -I/usr/include/malloc
+ LDFLAGS = -O
+ # The OBJ,EXE and LIB macros will need to be changed for compilation under
+ #  Windows NT

tpch-setup.sh

@@ -67,12 +67,12 @@ runcommand "hive -i settings/load-flat.sql -f ddl-tpch/bin_flat/alltables.sql -d
 # Create the optimized tables.
 i=1
 total=8
-DATABASE=tpch_bin_partitioned_orc_${SCALE}
+DATABASE=tpch_flat_orc_${SCALE}
 for t in ${TABLES}
 do
 	echo "Optimizing table $t ($i/$total)."
 	COMMAND="hive -i settings/load-flat.sql -f ddl-tpch/bin_flat/${t}.sql \
-	    -d DB=tpch_bin_flat_orc_${SCALE} \
+	    -d DB=${DATABASE} \
 	    -d SOURCE=tpch_text_${SCALE} -d BUCKETS=${BUCKETS} \
 	    -d FILE=orc"
 	runcommand "$COMMAND"