object SparkConfigUtils { def sparkContext(args : Array[String]) : SparkContext = { val conf = new SparkConf() if (conf.getOption("app.name").isEmpty) { conf.setAppName("GitHub push counter") } else { println("APP NAME PROVIDED: " + conf.get("appName")) } if (conf.getOption("master").isEmpty) { conf.setMaster("local[*]") } else { println("MASTER PROVIDED: " + conf.get("master")) } //conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") conf.set("spark.io.compression.codec", "lz4") new SparkContext(conf) }
Spark Configuration
Every Spark job needs to configure either SparkContext or SQLContext. The intention of this helper class is to simplify interpretation of job arguments.
def sparkSQLContext(sparkContext : SparkContext) : SQLContext = new SQLContext(sparkContext)
def sparkHiveSQLContext(sparkContext: SparkContext) : HiveContext = new HiveContext(sparkContext) }