Java/Scala
Updated at:
This topic uses spark-3.x for code demonstrations.
Download the template project
If you have already downloaded the project, skip this step. For download instructions, see Use the Spark client in local mode.
The template project includes sample code for spark-1.x, spark-2.x, and spark-3.x. This topic focuses on spark-3.x for demonstration purposes.
Example 1: WordCount
package com.aliyun.odps.spark.examples import org.apache.spark.sql.SparkSession object WordCount { def main(args: Array[String]) { val spark = SparkSession .builder() .appName("WordCount") .getOrCreate() val sc = spark.sparkContext try { sc.parallelize(1 to 100, 10).map(word => (word, 1)).reduceByKey(_ + _, 10).take(100).foreach(println) } finally { sc.stop() } } }Submission method
cd PATH_TO_SPARK/MaxCompute-Spark/spark-3.x mvn clean package cd $SPARK_HOME bin/spark-submit --class \ com.aliyun.odps.spark.examples.WordCount \ PATH_TO_PROJECT/MaxCompute-Spark/spark-3.x/target/spark-examples_2.12-1.0.0-SNAPSHOT-shaded.jar
Example 2: GraphX PageRank
package com.aliyun.odps.spark.examples.graphx import org.apache.spark.graphx.{Edge, Graph, VertexId} import org.apache.spark.rdd.RDD import org.apache.spark.sql.SparkSession object PageRank { def main(args: Array[String]): Unit = { val spark = SparkSession .builder() .appName("PageRank") .getOrCreate() val sc = spark.sparkContext // build vertices val users: RDD[(VertexId, Array[String])] = sc.parallelize(List( "1,BarackObama,Barack Obama", "2,ladygaga,Goddess of Love", "3,jeresig,John Resig", "4,justinbieber,Justin Bieber", "6,matei_zaharia,Matei Zaharia", "7,odersky,Martin Odersky", "8,anonsys" ).map(line => line.split(",")).map(parts => (parts.head.toLong, parts.tail))) // build edges val followers: RDD[Edge[Double]] = sc.parallelize(Array( Edge(2L, 1L, 1.0), Edge(4L, 1L, 1.0), Edge(1L, 2L, 1.0), Edge(6L, 3L, 1.0), Edge(7L, 3L, 1.0), Edge(7L, 6L, 1.0), Edge(6L, 7L, 1.0), Edge(3L, 7L, 1.0) )) // build graph val followerGraph: Graph[Array[String], Double] = Graph(users, followers) // restrict the graph to users with usernames and names val subgraph = followerGraph.subgraph(vpred = (vid, attr) => attr.size == 2) // compute PageRank val pageRankGraph = subgraph.pageRank(0.001) // get attributes of the top pagerank users val userInfoWithPageRank = subgraph.outerJoinVertices(pageRankGraph.vertices) { case (uid, attrList, Some(pr)) => (pr, attrList.toList) case (uid, attrList, None) => (0.0, attrList.toList) } println(userInfoWithPageRank.vertices.top(5)(Ordering.by(_._2._1)).mkString("\n")) } }Submission method
cd PATH_TO_SPARK/MaxCompute-Spark/spark-3.x mvn clean package cd $SPARK_HOME bin/spark-submit --class \ com.aliyun.odps.spark.examples.graphx.PageRank \ PATH_TO_PROJECT/MaxCompute-Spark/spark-3.x/target/spark-examples_2.12-1.0.0-SNAPSHOT-shaded.jar
Example 3: Read and write MaxCompute tables
package com.aliyun.odps.spark.examples.sparksql import org.apache.spark.sql.{SaveMode, SparkSession} object SparkSQL { def main(args: Array[String]) { val spark = SparkSession .builder() .appName("SparkSQL-on-MaxCompute") .config("spark.sql.broadcastTimeout", 20 * 60) .config("spark.sql.crossJoin.enabled", true) .config("spark.sql.defaultCatalog","odps") .config("spark.sql.catalog.odps", "org.apache.spark.sql.execution.datasources.v2.odps.OdpsTableCatalog") .config("spark.sql.sources.partitionOverwriteMode", "dynamic") .config("spark.sql.extensions", "org.apache.spark.sql.execution.datasources.v2.odps.extension.OdpsExtensions") .config("spark.sql.catalogImplementation","hive") .getOrCreate() import spark._ import sqlContext.implicits._ val tableName = "mc_test_table" val ptTableName = "mc_test_pt_table" // Drop Create sql(s"DROP TABLE IF EXISTS ${tableName}") sql(s"DROP TABLE IF EXISTS ${ptTableName}") sql(s"CREATE TABLE ${tableName} (name STRING, num BIGINT)") sql(s"CREATE TABLE ${ptTableName} (name STRING, num BIGINT) PARTITIONED BY (pt1 STRING, pt2 STRING)") val df = spark.sparkContext.parallelize(0 to 99, 2).map(f => { (s"name-$f", f) }).toDF("name", "num") val ptDf = spark.sparkContext.parallelize(0 to 99, 2).map(f => { (s"name-$f", f, "2018", "0601") }).toDF("name", "num", "pt1", "pt2") // Write to standard table df.write.insertInto(tableName) // insertInto semantics df.writeTo(tableName).overwritePartitions() // insertOverwrite using DataSource V2 // Write to partitioned table // DataFrameWriter cannot specify partitions directly. Use a temporary table and SQL to write to specific partitions. df.createOrReplaceTempView(s"${ptTableName}_tmp_view") sql(s"insert into table ${ptTableName} partition (pt1='2018', pt2='0601') select * from ${ptTableName}_tmp_view") sql(s"insert overwrite table ${ptTableName} partition (pt1='2018', pt2='0601') select * from ${ptTableName}_tmp_view") ptDf.write.insertInto(ptTableName) // dynamic partition insertInto semantics ptDf.write.mode("overwrite").insertInto(ptTableName) // dynamic partition insertOverwrite semantics // Read standard table val rdf = sql(s"select name, num from $tableName") println(s"rdf show, ${rdf.count()}") rdf.show() rdf.printSchema() // Read partitioned table val rptdf = sql(s"select name, num, pt1, pt2 from $ptTableName where pt1 = '2018' and pt2 = '0601'") println(s"rptdf show, ${rptdf.count()}") rptdf.show() rptdf.printSchema() } }Submission method
cd PATH_TO_SPARK/MaxCompute-Spark/spark-3.x mvn clean package cd $SPARK_HOME bin/spark-submit --class \ com.aliyun.odps.spark.examples.sparksql.SparkSQL \ PATH_TO_PROJECT/MaxCompute-Spark/spark-3.x/target/spark-examples_2.12-1.0.0-SNAPSHOT-shaded.jar
Example 4: Access Alibaba Cloud OSS
See Access Alibaba Cloud OSS to add OSS dependencies.
package com.aliyun.odps.spark.examples.oss import org.apache.spark.{SparkConf, SparkContext} object JindoFsDemo { def main(args: Array[String]): Unit = { val bucket : String = args(0) val ossPath : String = args(1) // using access-key-id/access-key-secret val conf = new SparkConf() .setAppName("jindo-fs-demo") .set("spark.hadoop.fs.AbstractFileSystem.oss.impl", "com.aliyun.emr.fs.oss.OSS") .set("spark.hadoop.fs.oss.impl", "com.aliyun.emr.fs.oss.JindoOssFileSystem") .set("spark.hadoop.fs.oss.endpoint", "oss-cn-hangzhou-internal.aliyuncs.com") .set("spark.hadoop.fs.oss.accessKeyId", "xxx") .set("spark.hadoop.fs.oss.accessKeySecret", "xxx") val sc = new SparkContext(conf) try { read_oss_dir(sc, "demo", s"oss://${bucket}/${ossPath}") } finally { sc.stop() } } /** * compute cost time using jindo sdk */ def read_oss_dir(sc: SparkContext, job_des:String, ossPath: String): Unit = { val startTime: Long = System.currentTimeMillis() val inputData = sc.textFile(ossPath, 20) val cnt = inputData.count val endTime:Long = System.currentTimeMillis() val cost:Long = endTime - startTime println(s"job:$job_des, count:$cnt, consume:$cost") } }Submission method
cd {{PATH_TO_SPARK}}/MaxCompute-Spark/spark-3.x mvn clean package cd $SPARK_HOME bin/spark-submit --class \ com.aliyun.odps.spark.examples.oss.JindoFsDemo \ {{PATH_TO_PROJECT}}/MaxCompute-Spark/spark-3.x/target/spark-examples_2.12-1.0.0-SNAPSHOT-shaded.jar \ {{OSS_BUCKET}} \ {{OSS_PATH}}
Is this page helpful?