Fix reducers max

settings/load-partitioned.sql

@@ -5,7 +5,7 @@ set hive.exec.max.dynamic.partitions.pernode=100000;
 set hive.exec.max.dynamic.partitions=100000;
 set hive.exec.max.created.files=1000000;
 set hive.exec.parallel=true;
-set hive.exec.reducers.max=${SCALE};
+set hive.exec.reducers.max=${REDUCERS};
 set hive.stats.autogather=true;
 set hive.optimize.sort.dynamic.partition=true;
 

tpcds-setup.sh

@@ -87,6 +87,8 @@ echo -e "all: ${DIMS} ${FACTS}" > $LOAD_FILE
 i=1
 total=24
 DATABASE=tpcds_bin_partitioned_${FORMAT}_${SCALE}
+MAX_REDUCERS=2500
+REDUCERS=$((test ${SCALE} -gt ${MAX_REDUCERS} && echo ${MAX_REDUCERS} || echo ${SCALE}))
 
 # Populate the smaller tables.
 for t in ${DIMS}
@@ -105,7 +107,7 @@ do
 	    -d DB=tpcds_bin_partitioned_${FORMAT}_${SCALE} \
             -d SCALE=${SCALE} \
 	    -d SOURCE=tpcds_text_${SCALE} -d BUCKETS=${BUCKETS} \
-	    -d RETURN_BUCKETS=${RETURN_BUCKETS} -d FILE=${FORMAT}"
+	    -d RETURN_BUCKETS=${RETURN_BUCKETS} -d REDUCERS=${REDUCERS} -d FILE=${FORMAT}"
 	echo -e "${t}:\n\t@$COMMAND $SILENCE && echo 'Optimizing table $t ($i/$total).'" >> $LOAD_FILE
 	i=`expr $i + 1`
 done

tpch-setup.sh

@@ -75,6 +75,8 @@ else
 fi
 
 DATABASE=tpch_${SCHEMA_TYPE}_orc_${SCALE}
+MAX_REDUCERS=2500
+REDUCERS=$((test ${SCALE} -gt ${MAX_REDUCERS} && echo ${MAX_REDUCERS} || echo ${SCALE}))
 
 for t in ${TABLES}
 do
@@ -82,7 +84,7 @@ do
 	COMMAND="hive -i settings/load-${SCHEMA_TYPE}.sql -f ddl-tpch/bin_${SCHEMA_TYPE}/${t}.sql \
 	    -d DB=${DATABASE} \
 	    -d SOURCE=tpch_text_${SCALE} -d BUCKETS=${BUCKETS} \
-            -d SCALE=${SCALE} \
+            -d SCALE=${SCALE} -d REDUCERS=${REDUCERS} \
 	    -d FILE=orc"
 	runcommand "$COMMAND"
 	if [ $? -ne 0 ]; then