iteratively improving spark application performance
play

Iteratively Improving Spark Application Performance William C. - PowerPoint PPT Presentation

Feb 11, 2023 •210 likes •1.5k views

Iteratively Improving Spark Application Performance William C. Benton Red Hat, Inc. Forecast Background: Spark, RDDs, and Sparks execution model Case study overview Improving our prototype Background Apache Spark Introduced

  1. trait ActivitySliding { import org.apache.spark.rdd.RDD import com.freevariable.surlaplaque.data.{Trackpoint => TP} def windowsForActivities[U](data: RDD[TP], period: Int, data: RDD[TP] period: Int xform: (TP => U) = identity _) = { xform: (TP => U) = identity _ val pairs = data.groupBy((tp:TP) => tp.activity.getOrElse("UNKNOWN")) data.groupBy((tp:TP) => tp.activity.getOrElse("UNKNOWN")) pairs.flatMap { case (activity:String, stp:Seq[TP]) => (stp sliding period).zipWithIndex.map { case (s,i) => ((activity, i), s.map(xform)) } } } def identity(tp: Trackpoint) = tp }

  2. trait ActivitySliding { import org.apache.spark.rdd.RDD import com.freevariable.surlaplaque.data.{Trackpoint => TP} def windowsForActivities[U](data: RDD[TP], period: Int, data: RDD[TP] period: Int xform: (TP => U) = identity _) = { xform: (TP => U) = identity _ val pairs = data.groupBy((tp:TP) => tp.activity.getOrElse("UNKNOWN")) data.groupBy((tp:TP) => tp.activity.getOrElse("UNKNOWN")) pairs.flatMap { pairs.flatMap case (activity:String, stp:Seq[TP]) => (stp sliding period).zipWithIndex.map { case (s,i) => ((activity, i), s.map(xform)) } } } def identity(tp: Trackpoint) = tp }

  3. trait ActivitySliding { import org.apache.spark.rdd.RDD import com.freevariable.surlaplaque.data.{Trackpoint => TP} def windowsForActivities[U](data: RDD[TP], period: Int, data: RDD[TP] period: Int xform: (TP => U) = identity _) = { xform: (TP => U) = identity _ val pairs = data.groupBy((tp:TP) => tp.activity.getOrElse("UNKNOWN")) data.groupBy((tp:TP) => tp.activity.getOrElse("UNKNOWN")) pairs.flatMap { pairs.flatMap case (activity:String, stp:Seq[TP]) => (stp sliding period).zipWithIndex.map { (stp sliding period).zipWithIndex.map { case (s,i) => ((activity, i), s.map(xform)) case (s,i) => ((activity, i), s.map(xform)) } } } } def identity(tp: Trackpoint) = tp }

  4. Transform an RDD of TRACKPOINTS … trait ActivitySliding { import org.apache.spark.rdd.RDD import com.freevariable.surlaplaque.data.{Trackpoint => TP} def windowsForActivities[U](data: RDD[TP], period: Int, data: RDD[TP] period: Int xform: (TP => U) = identity _) = { xform: (TP => U) = identity _ val pairs = data.groupBy((tp:TP) => tp.activity.getOrElse("UNKNOWN")) data.groupBy((tp:TP) => tp.activity.getOrElse("UNKNOWN")) pairs.flatMap { pairs.flatMap case (activity:String, stp:Seq[TP]) => (stp sliding period).zipWithIndex.map { (stp sliding period).zipWithIndex.map { case (s,i) => ((activity, i), s.map(xform)) case (s,i) => ((activity, i), s.map(xform)) } } } } def identity(tp: Trackpoint) = tp } …TO an RDD of WINDOW ID S and SAMPLE WINDOWS

  5. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = windowsForActivities(data, period, minify _).cache val clusterPairs = windowedSamples.map { case ((act, idx), samples) => ((act, idx), clusterPairsForWindow(samples, model)) } val mmps = windowedSamples.map { case ((act, idx), samples) => ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) } // continued...

  6. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = windowsForActivities(data, period, minify _).cache val windowedSamples = windowsForActivities(data, period, minify _).cache val clusterPairs = windowedSamples.map { case ((act, idx), samples) => ((act, idx), clusterPairsForWindow(samples, model)) } val mmps = windowedSamples.map { case ((act, idx), samples) => ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) } // continued... Divide the input data into overlapping windows , keyed by ACTIVITY and OFFSET (we’ll call this key a WINDOW ID )

  7. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = windowsForActivities(data, period, minify _).cache val clusterPairs = windowedSamples.map { val clusterPairs = windowedSamples.map { case ((act, idx), samples) => case ((act, idx), samples) => ((act, idx), clusterPairsForWindow(samples, model)) ((act, idx), clusterPairsForWindow(samples, model)) } } val mmps = windowedSamples.map { case ((act, idx), samples) => ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) } // continued... Identify the spatial clusters that each window starts and ends in

  8. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = windowsForActivities(data, period, minify _).cache val clusterPairs = windowedSamples.map { case ((act, idx), samples) => ((act, idx), clusterPairsForWindow(samples, model)) } val mmps = windowedSamples.map { val mmps = windowedSamples.map { case ((act, idx), samples) => case ((act, idx), samples) => ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) } } // continued... Identify the MEAN WATTAGE for each window of samples

  9. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = /* window IDs and raw sample windows */ val clusterPairs = /* window IDs and spatial cluster pairs */ val mmps = /* window IDs and mean wattages for each window */ val top20 = mmps.join(clusterPairs) .map { case ((act, idx), (watts, (cls1, cls2))) => ((cls1, cls2), (watts, (act, idx))) } .reduceByKey ((a, b) => if (a._1 > b._1) a else b) .map { case ((cls1, cls2), (watts, (act, idx))) => (watts, (act, idx)) } .sortByKey(false) .take(20) app.context.parallelize(top20) .map { case (watts, (act, idx)) => ((act, idx), watts) } .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect }

  10. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = /* window IDs and raw sample windows */ val clusterPairs = /* window IDs and spatial cluster pairs */ val mmps = /* window IDs and mean wattages for each window */ val top20 = mmps.join(clusterPairs) mmps.join(clusterPairs) .map { case ((act, idx), (watts, (cls1, cls2))) => ((cls1, cls2), (watts, (act, idx))) } .reduceByKey ((a, b) => if (a._1 > b._1) a else b) .map { case ((cls1, cls2), (watts, (act, idx))) => (watts, (act, idx)) } .sortByKey(false) .take(20) app.context.parallelize(top20) .map { case (watts, (act, idx)) => ((act, idx), watts) } .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect } for each window ID , JOIN its mean wattages with its spatial clusters

  11. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = /* window IDs and raw sample windows */ val clusterPairs = /* window IDs and spatial cluster pairs */ val mmps = /* window IDs and mean wattages for each window */ val top20 = mmps.join(clusterPairs) .map { case ((act, idx), (watts, (cls1, cls2))) => .map { case ((act, idx), (watts, (cls1, cls2))) => ((cls1, cls2), (watts, (act, idx))) } ((cls1, cls2), (watts, (act, idx))) } .reduceByKey ((a, b) => if (a._1 > b._1) a else b) .map { case ((cls1, cls2), (watts, (act, idx))) => (watts, (act, idx)) } .sortByKey(false) .take(20) app.context.parallelize(top20) .map { case (watts, (act, idx)) => ((act, idx), watts) } .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect } transpose these tuples so they are keyed by spatial cluster pairs

  12. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = /* window IDs and raw sample windows */ val clusterPairs = /* window IDs and spatial cluster pairs */ val mmps = /* window IDs and mean wattages for each window */ val top20 = mmps.join(clusterPairs) .map { case ((act, idx), (watts, (cls1, cls2))) => ((cls1, cls2), (watts, (act, idx))) } .reduceByKey ((a, b) => if (a._1 > b._1) a else b) .reduceByKey ((a, b) => if (a._1 > b._1) a else b) .map { case ((cls1, cls2), (watts, (act, idx))) => (watts, (act, idx)) } .sortByKey(false) .take(20) app.context.parallelize(top20) .map { case (watts, (act, idx)) => ((act, idx), watts) } .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect } KEEP ONLY the BEST wattage for each spatial cluster pair

  13. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = /* window IDs and raw sample windows */ val clusterPairs = /* window IDs and spatial cluster pairs */ val mmps = /* window IDs and mean wattages for each window */ val top20 = mmps.join(clusterPairs) .map { case ((act, idx), (watts, (cls1, cls2))) => ((cls1, cls2), (watts, (act, idx))) } .reduceByKey ((a, b) => if (a._1 > b._1) a else b) .map { case ((cls1, cls2), (watts, (act, idx))) => (watts, (act, idx)) } .map { case ((cls1, cls2), (watts, (act, idx))) => (watts, (act, idx)) } .sortByKey(false) .take(20) app.context.parallelize(top20) .map { case (watts, (act, idx)) => ((act, idx), watts) } .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect } project away the cluster centers

  14. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = /* window IDs and raw sample windows */ val clusterPairs = /* window IDs and spatial cluster pairs */ val mmps = /* window IDs and mean wattages for each window */ val top20 = mmps.join(clusterPairs) .map { case ((act, idx), (watts, (cls1, cls2))) => ((cls1, cls2), (watts, (act, idx))) } .reduceByKey ((a, b) => if (a._1 > b._1) a else b) .map { case ((cls1, cls2), (watts, (act, idx))) => (watts, (act, idx)) } .sortByKey(false) .sortByKey(false) .take(20) .take(20) app.context.parallelize(top20) .map { case (watts, (act, idx)) => ((act, idx), watts) } .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect } SORT by wattage in descending order ; keep the best twenty

  15. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = /* window IDs and raw sample windows */ val clusterPairs = /* window IDs and spatial cluster pairs */ val mmps = /* window IDs and mean wattages for each window */ val top20 = mmps.join(clusterPairs) .map { case ((act, idx), (watts, (cls1, cls2))) => ((cls1, cls2), (watts, (act, idx))) } .reduceByKey ((a, b) => if (a._1 > b._1) a else b) .map { case ((cls1, cls2), (watts, (act, idx))) => (watts, (act, idx)) } .sortByKey(false) .take(20) app.context.parallelize(top20) .map { case (watts, (act, idx)) => ((act, idx), watts) } .map { case (watts, (act, idx)) => ((act, idx), watts) } .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect } Re-key the best efforts by window id

  16. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = /* window IDs and raw sample windows */ val clusterPairs = /* window IDs and spatial cluster pairs */ val mmps = /* window IDs and mean wattages for each window */ val top20 = mmps.join(clusterPairs) .map { case ((act, idx), (watts, (cls1, cls2))) => ((cls1, cls2), (watts, (act, idx))) } .reduceByKey ((a, b) => if (a._1 > b._1) a else b) .map { case ((cls1, cls2), (watts, (act, idx))) => (watts, (act, idx)) } .sortByKey(false) .take(20) app.context.parallelize(top20) .map { case (watts, (act, idx)) => ((act, idx), watts) } .join (windowedSamples) .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect } get the actual sample windows for each effort ; project away IDs

  17. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = /* window IDs and raw sample windows */ val clusterPairs = /* window IDs and spatial cluster pairs */ val mmps = /* window IDs and mean wattages for each window */ val top20 = mmps.join(clusterPairs) .map { case ((act, idx), (watts, (cls1, cls2))) => ((cls1, cls2), (watts, (act, idx))) } .reduceByKey ((a, b) => if (a._1 > b._1) a else b) .map { case ((cls1, cls2), (watts, (act, idx))) => (watts, (act, idx)) } .sortByKey(false) .take(20) app.context.parallelize(top20) .map { case (watts, (act, idx)) => ((act, idx), watts) } .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect .collect }

  18. Improving the prototype

  19. Broadcast large static data

  20. Broadcast variables // phoneBook maps (given name, surname) -> phone number digits val phoneBook: Map[(String, String), String] = initPhoneBook() val names: RDD[(String, String)] = /* ... */ val directory = names.map { case name @ (first, last) => (name, phoneBook.getOrElse("555-1212")) }

  21. Broadcast variables // phoneBook maps (given name, surname) -> phone number digits val phoneBook: Map[(String, String), String] = initPhoneBook() val names: RDD[(String, String)] = /* ... */ val directory = names.map { case name @ (first, last) => (name, phoneBook.getOrElse("555-1212")) } phoneBook will be copied and deserialized for each task !

  22. Broadcast variables // phoneBook maps (given name, surname) -> phone number digits val phoneBook: Map[(String, String), String] = initPhoneBook() val names: RDD[(String, String)] = /* ... */ val pbb = sparkContext.broadcast(phoneBook) val directory = names.map { case name @ (first, last) => (name, pbb.value.getOrElse(“555-1212")) } Broadcasting phoneBook means it can be deserialized once and cached on each Node !

  23. def bestsForPeriod(data: RDD[Trackpoint], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = windowsForActivities(data, period, minify _) val clusterPairs = windowedSamples.map { case ((act, idx), samples) => ((act, idx), clusterPairsForWindow(samples, model)) // ... }

  24. def bestsForPeriod(data: RDD[Trackpoint], period: Int, app: SLP, model: Broadcast[KMeansModel]) = { val windowedSamples = windowsForActivities(data, period, minify _) val clusterPairs = windowedSamples.map { case ((act, idx), samples) => ((act, idx), clusterPairsForWindow(samples, model.value)) // rest of function unchanged }

  25. Cache only when necessary

  26. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = windowsForActivities(data, period).cache // ... val top20 = mmps.join(clusterPairs) .map { case ((act, idx), (watts, (cls1, cls2))) => ((cls1, cls2), (watts, (act, idx))) } .reduceByKey ((a, b) => if (a._1 > b._1) a else b) .map { case ((cls1, cls2), (watts, (act, idx))) => (watts, (act, idx)) } .sortByKey(false) .take(20) // ... }

  27. Keeping EVERY WINDOW IN MEMORY… def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = windowsForActivities(data, period).cache // ... val top20 = mmps.join(clusterPairs) .map { case ((act, idx), (watts, (cls1, cls2))) => ((cls1, cls2), (watts, (act, idx))) } .reduceByKey ((a, b) => if (a._1 > b._1) a else b) .map { case ((cls1, cls2), (watts, (act, idx))) => (watts, (act, idx)) } .sortByKey(false) .take(20) // ... }

  28. Keeping EVERY WINDOW IN MEMORY… def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = windowsForActivities(data, period).cache // ... val top20 = mmps.join(clusterPairs) .map { case ((act, idx), (watts, (cls1, cls2))) => ((cls1, cls2), (watts, (act, idx))) } .reduceByKey ((a, b) => if (a._1 > b._1) a else b) .map { case ((cls1, cls2), (watts, (act, idx))) => (watts, (act, idx)) } .sortByKey(false) .take(20) // ... } …EVEN THOUGH RECOMPUTING windows IS incredibly CHEAP AND YOU’ll need only a tiny fraction of windows?

  29. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = windowsForActivities(data, period).cache // ... val top20 = mmps.join(clusterPairs) .map { case ((act, idx), (watts, (cls1, cls2))) => ((cls1, cls2), (watts, (act, idx))) } .reduceByKey ((a, b) => if (a._1 > b._1) a else b) .map { case ((cls1, cls2), (watts, (act, idx))) => (watts, (act, idx)) } .sortByKey(false) .take(20) // ... } Eliminating unnecessary memory pressure can lead to a substantial speedup!

  30. Avoid shu ffl es when possible

  31. tasks

  32. stages

  33. stages we want to Avoid all unnecessary shuffles

  34. def bestsForPeriod(data: RDD[Trackpoint], period: Int, app: SLP, model: Broadcast[KMeansModel]) = { val windowedSamples = windowsForActivities(data, period, minify _) val bests = windowedSamples.map { case ((act, idx), samples) => ( clusterPairsForWindow(samples, model.value), ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) ) }.cache val top20 = bests.reduceByKey ((a, b) => if (a._2 > b._2) a else b) .map { case ((_, _), keep) => keep } .takeOrdered(20)(Ordering.by[((String, Int), Double), Double] { case ((_, _), watts) => -watts }) app.context.parallelize(top20) .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect }

  35. def bestsForPeriod(data: RDD[Trackpoint], period: Int, app: SLP, model: Broadcast[KMeansModel]) = { val windowedSamples = windowsForActivities(data, period, minify _) val bests = windowedSamples.map { { case ((act, idx), samples) => ( case ((act, idx), samples) => ( clusterPairsForWindow(samples, model.value), clusterPairsForWindow(samples, model.value), ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) ) ) } }.cache val top20 = bests.reduceByKey ((a, b) => if (a._2 > b._2) a else b) .map { case ((_, _), keep) => keep } .takeOrdered(20)(Ordering.by[((String, Int), Double), Double] { case ((_, _), watts) => -watts }) app.context.parallelize(top20) .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect }

  36. def bestsForPeriod(data: RDD[Trackpoint], period: Int, app: SLP, model: Broadcast[KMeansModel]) = { val windowedSamples = windowsForActivities(data, period, minify _) start and end clusters { val bests = windowedSamples.map { case ((act, idx), samples) => ( case ((act, idx), samples) => ( clusterPairsForWindow(samples, model.value), clusterPairsForWindow(samples, model.value), ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) ) ) }.cache } val top20 = bests.reduceByKey ((a, b) => if (a._2 > b._2) a else b) .map { case ((_, _), keep) => keep } .takeOrdered(20)(Ordering.by[((String, Int), Double), Double] { case ((_, _), watts) => -watts }) app.context.parallelize(top20) .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect }

  37. def bestsForPeriod(data: RDD[Trackpoint], period: Int, app: SLP, model: Broadcast[KMeansModel]) = { val windowedSamples = windowsForActivities(data, period, minify _) start and end clusters val bests = windowedSamples.map { { case ((act, idx), samples) => ( case ((act, idx), samples) => ( clusterPairsForWindow(samples, model.value), clusterPairsForWindow(samples, model.value), ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) window id s and mean wattages ) ) }.cache } val top20 = bests.reduceByKey ((a, b) => if (a._2 > b._2) a else b) .map { case ((_, _), keep) => keep } .takeOrdered(20)(Ordering.by[((String, Int), Double), Double] { case ((_, _), watts) => -watts }) app.context.parallelize(top20) .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect }

  38. def bestsForPeriod(data: RDD[Trackpoint], period: Int, app: SLP, model: Broadcast[KMeansModel]) = { val windowedSamples = windowsForActivities(data, period, minify _) val bests = windowedSamples.map { case ((act, idx), samples) => ( clusterPairsForWindow(samples, model.value), ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) ) eliminate a join and a transpose }.cache bests.reduceByKey ((a, b) => if (a._2 > b._2) a else b) val top20 = bests.reduceByKey ((a, b) => if (a._2 > b._2) a else b) .map { case ((_, _), keep) => keep } .map { case ((_, _), keep) => keep } .takeOrdered(20)(Ordering.by[((String, Int), Double), Double] { case ((_, _), watts) => -watts }) app.context.parallelize(top20) .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect }

  39. def bestsForPeriod(data: RDD[Trackpoint], period: Int, app: SLP, model: Broadcast[KMeansModel]) = { val windowedSamples = windowsForActivities(data, period, minify _) val bests = windowedSamples.map { case ((act, idx), samples) => ( clusterPairsForWindow(samples, model.value), ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) ) }.cache val top20 = bests.reduceByKey ((a, b) => if (a._2 > b._2) a else b) .map { case ((_, _), keep) => keep } .takeOrdered(20)(Ordering.by[((String, Int), Double), Double] { .takeOrdered(20)(Ordering.by[((String, Int), Double), Double] { case ((_, _), watts) => -watts case ((_, _), watts) => -watts (use the right API calls!) }) } app.context.parallelize(top20) .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect }

  40. def bestsForPeriod(data: RDD[Trackpoint], period: Int, app: SLP, model: Broadcast[KMeansModel]) = { val windowedSamples = windowsForActivities(data, period, minify _) val bests = windowedSamples.map { case ((act, idx), samples) => ( clusterPairsForWindow(samples, model.value), ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) ) }.cache val top20 = bests.reduceByKey ((a, b) => if (a._2 > b._2) a else b) .map { case ((_, _), keep) => keep } .takeOrdered(20)(Ordering.by[((String, Int), Double), Double] { case ((_, _), watts) => -watts }) app.context.parallelize(top20) app.context.parallelize(top20) .join (windowedSamples) .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect .collect } eliminate a transpose

  41. def bestsForPeriod(data: RDD[Trackpoint], period: Int, app: SLP, model: Broadcast[KMeansModel]) = { val windowedSamples = windowsForActivities(data, period, minify _) val bests = windowedSamples.map { case ((act, idx), samples) => ( clusterPairsForWindow(samples, model.value), ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) ) }.cache val top20 = bests.reduceByKey ((a, b) => if (a._2 > b._2) a else b) .map { case ((_, _), keep) => keep } .takeOrdered(20)(Ordering.by[((String, Int), Double), Double] { case ((_, _), watts) => -watts }) app.context.parallelize(top20) app.context.parallelize(top20) app.context.parallelize(top20) .join (windowedSamples) .join (windowedSamples) .join (windowedSamples) .collect .map { case ((act, idx), (watts, samples)) => (watts, samples) } .map { case ((act, idx), (watts, samples)) => (watts, samples) } .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect .collect } eliminate a transpose …or two!

  42. Embrace laziness

  43. Embrace laziness (only pay for what you use)

  44. Windowed processing redux 20140909.tcx

Download Presentation
Download Policy: The content available on the website is offered to you 'AS IS' for your personal information and use only. It cannot be commercialized, licensed, or distributed on other websites without prior consent from the author. To download a presentation, simply click this link. If you encounter any difficulties during the download process, it's possible that the publisher has removed the file from their server.

Recommend

Improving Spark Performance with Zero-copy Buffer Management and RDMA Hu Li, Charley Chen and Wei

Improving Spark Performance with Zero-copy Buffer Management and RDMA Hu Li, Charley Chen and Wei

Improving Spark Performance with Zero-copy Buffer Management and RDMA Hu Li, Charley Chen and Wei Xu Institute for Interdisciplinary Information Sciences Tsinghua University, China Latency matters in big data Impala Query Dremel Query [2012]

541 views • 17 slides
Spark Technology 1. Spark main objectives 2. RDD concepts and operations 3. SPARK application

Spark Technology 1. Spark main objectives 2. RDD concepts and operations 3. SPARK application

10/05/2019 Big Data : Informatique pour les donnes et calculs massifs 7 SPARK technology Stphane Vialle Stephane.Vialle@centralesupelec.fr http://www.metz.supelec.fr/~vialle Spark Technology 1. Spark main objectives 2. RDD concepts

818 views • 39 slides
Intr Intro o to Spark to Spark and Spark and Spark SQL SQL AMP Camp 2014 Michael Armbrust -

Intr Intro o to Spark to Spark and Spark and Spark SQL SQL AMP Camp 2014 Michael Armbrust -

Intr Intro o to Spark to Spark and Spark and Spark SQL SQL AMP Camp 2014 Michael Armbrust - @michaelarmbrust What is Apache Spark? Fast and general cluster computing system, interoperable with Hadoop, included in all major distros

667 views • 43 slides
Spark Code Camp Discover Spark Streaming & Spark SQL Project Overview Focus on Spark

Spark Code Camp Discover Spark Streaming & Spark SQL Project Overview Focus on Spark

Spark Code Camp Discover Spark Streaming & Spark SQL Project Overview Focus on Spark Streaming and Spark SQL Explored Streaming API of Apache Spark on Ukko Cluster Window based Stream Content Direct Stream content

221 views • 9 slides
High Integrity Ada with SPARK Praxis Critical Systems 1 SPARK and the SPARK Examiner What is

High Integrity Ada with SPARK Praxis Critical Systems 1 SPARK and the SPARK Examiner What is

High Integrity Ada with SPARK Praxis Critical Systems 1 SPARK and the SPARK Examiner What is SPARK? A sub-language of Ada 83 and 95 with particular properties that make it ideally suited to the most critical of applications: completely

848 views • 10 slides
Exploring the Performance of Spark for a Scientific Use Case Saba Sehrish (ssehrish@fnal.gov), Jim

Exploring the Performance of Spark for a Scientific Use Case Saba Sehrish (ssehrish@fnal.gov), Jim

Exploring the Performance of Spark for a Scientific Use Case Saba Sehrish (ssehrish@fnal.gov), Jim Kowalkowski and Marc Paterno IEEE International Workshop on High-Performance Big Data Computing (HPBDC) In conjunction with the 30 th IEEE IPDPS

600 views • 22 slides
BUILDING APACHE SPARK APPLICATION PIPELINES FOR THE KUBERNETES ECOSYSTEM Michael McCune 14

BUILDING APACHE SPARK APPLICATION PIPELINES FOR THE KUBERNETES ECOSYSTEM Michael McCune 14

BUILDING APACHE SPARK APPLICATION PIPELINES FOR THE KUBERNETES ECOSYSTEM Michael McCune 14 November 2016 1 INTRODUCTION A little about me Embedded to Orchestration Red Hat emerging technologies OpenStack Sahara Oshinko project for

453 views • 31 slides
Improving Scalability and Fault Improving Scalability and Fault Tolerance in an Application

Improving Scalability and Fault Improving Scalability and Fault Tolerance in an Application

Improving Scalability and Fault Improving Scalability and Fault Tolerance in an Application Tolerance in an Application Management Infrastructure Management Infrastructure Nikolay Topilski , Jeannie Albrecht, and Amin Vahdat Williams College

560 views • 18 slides
Company Introduction Improving the security and performance of your data centres, networks and

Company Introduction Improving the security and performance of your data centres, networks and

SOLUTIONS & SUPPORT PROFESSIONAL & MANAGED SERVICES Company Introduction Improving the security and performance of your data centres, networks and applications Network Data Cyber Application Network Test www.phoenixdatacom.com

476 views • 10 slides
Spark Processing 101 September 10, 2015 Justin Sun Overview What is Spark? SparkContext

Spark Processing 101 September 10, 2015 Justin Sun Overview What is Spark? SparkContext

Spark Processing 101 September 10, 2015 Justin Sun Overview What is Spark? SparkContext Resilient Distributed Datasets (RDDs) Transformations Actions Code Examples Resources What is Spark? General cluster

356 views • 10 slides
SPARK G N I T E R R A M E V I T A E C u s t o m L o g o P r o j e c t Introduction

SPARK G N I T E R R A M E V I T A E C u s t o m L o g o P r o j e c t Introduction

C K PROPOSAL SPARK G N I T E R R A M E V I T A E C u s t o m L o g o P r o j e c t Introduction We are excited to quote on your logo design project and believe that Spark is a perfect fit for your needs. Concept Presentation

554 views • 30 slides
Using Adobe Spark Video in the Classroom Jessica McKenna jmckenna@glencoveschools.org &

Using Adobe Spark Video in the Classroom Jessica McKenna jmckenna@glencoveschools.org &

Using Adobe Spark Video in the Classroom Jessica McKenna jmckenna@glencoveschools.org & Giselle Taylor gjtaylor@glencoveschools.org Glen Cove City School District What is Adobe Spark Video? Story-telling video making application

331 views • 9 slides
SPARK NEW ZEALAND ANNUAL MEETING 2015 Spark New Zealand 2015 Spark New Zealand 2015 2 Order of

SPARK NEW ZEALAND ANNUAL MEETING 2015 Spark New Zealand 2015 Spark New Zealand 2015 2 Order of

SPARK NEW ZEALAND ANNUAL MEETING 2015 Spark New Zealand 2015 Spark New Zealand 2015 2 Order of Meeting: Introductions and formalities Chairmans address Managing Director update Resolutions Shareholder questions Conduct of polls Meeting

421 views • 38 slides
Improving Performance We want to improve the performance of our computation Unit 17

Improving Performance We want to improve the performance of our computation Unit 17

17.1 17.2 Improving Performance We want to improve the performance of our computation Unit 17 Question: What are we referring to when we say "performance"? __________________ Improving Performance __________________

337 views • 5 slides
Distributing Matrix Computations with Spark MLlib Reza Zadeh A General Platform Standard libraries

Distributing Matrix Computations with Spark MLlib Reza Zadeh A General Platform Standard libraries

Distributing Matrix Computations with Spark MLlib Reza Zadeh A General Platform Standard libraries included with Spark Spark MLlib Spark SQL GraphX Streaming machine structured graph learning real-time Spark Core Outline Introduction to

682 views • 40 slides
The Art of CPU-Pinning: Evaluating and Improving the Performance of Virtualization and

The Art of CPU-Pinning: Evaluating and Improving the Performance of Virtualization and

The Art of CPU-Pinning: Evaluating and Improving the Performance of Virtualization and Containerization Platforms Davood GhatrehSamani, Chavit Denninart Joseph Bacik Mohsen Amini Salehi High Performance Cloud Computing Lab (HPCC)

408 views • 30 slides
Web Application Stress Testing with Blaise Internet with Blaise Internet Jim OReilly Jim O

Web Application Stress Testing with Blaise Internet with Blaise Internet Jim OReilly Jim O

Web Application Stress Testing with Blaise Internet with Blaise Internet Jim OReilly Jim O Reilly Westat International Blaise Users Conference Annapolis Maryland, USA September 28, 2007 Septe be 8, 00 Introduction F Focus: web

460 views • 20 slides
Descriptive Methods 707.031: Evaluation Methodology Winter 2015/16 Eduardo Veas what we do with

Descriptive Methods 707.031: Evaluation Methodology Winter 2015/16 Eduardo Veas what we do with

Descriptive Methods 707.031: Evaluation Methodology Winter 2015/16 Eduardo Veas what we do with the data depends on the scales 2 Measurement Scales 3 The complexity of measurements Nominal Crude Ordinal Interval Ratio

847 views • 69 slides
If I Only Knew Then What I Know Now 5 Lessons Learned from my experiences leading agile

If I Only Knew Then What I Know Now 5 Lessons Learned from my experiences leading agile

If I Only Knew Then What I Know Now 5 Lessons Learned from my experiences leading agile transformations Larry Gorman , Chief Technology Evangelist, SkyTouch Technology Larry Gorman Chief Technology Evangelist SkyTouch Technology o: 602-337-2865

484 views • 21 slides
Organization of DSLE part Tooling Domain Specific Language Domain Specific Language

Organization of DSLE part Tooling Domain Specific Language Domain Specific Language

Organization of DSLE part Tooling Domain Specific Language Domain Specific Language Eclipse plus EMF Engineering Xtext, Xtend, Xpand, QVTo and ATL Topics: Prof.dr. Mark van den Brand Model driven software engineering Model

268 views • 13 slides
RRDtool Tips & Tricks Tobias Oetiker <tobi@oetiker.ch> OETIKER+PARTNER AG Tobi Oetiker

RRDtool Tips & Tricks Tobias Oetiker <tobi@oetiker.ch> OETIKER+PARTNER AG Tobi Oetiker

OETIKER+PARTNER AG RRDtool Tips & Tricks Tobias Oetiker <tobi@oetiker.ch> OETIKER+PARTNER AG Tobi Oetiker <tobi@oetiker.ch> OETIKER+PARTNER AG Recipe for Success Resolve the problems before anyone else finds them. Tobi

707 views • 48 slides
Picture of Tvlle in wet season Frogs love this Rick loves disease investigation, so when the Bohle

Picture of Tvlle in wet season Frogs love this Rick loves disease investigation, so when the Bohle

EHNV and BIV in Australia Ellen Ariel and Rick Speare James Cook University Townsville, Australia Ricks back yard in Bohle wet season Picture of Tvlle in wet season Frogs love this Rick loves disease investigation, so when the Bohle

385 views • 13 slides
EDAA40 EDAA40 Discrete Structures in Computer Science Discrete Structures in Computer Science

EDAA40 EDAA40 Discrete Structures in Computer Science Discrete Structures in Computer Science

EDAA40 EDAA40 Discrete Structures in Computer Science Discrete Structures in Computer Science 1: Sets 1: Sets Jrn W. Janneck, Dept. of Computer Science, Lund University axiomatic vs nave set theory Zermelo-Fraenkel Set Theory w/Choice

374 views • 8 slides
Time Agenda Item 8:00-8:30am Breakfast 8:30-9:00am Introductions and Recap from Last Night

Time Agenda Item 8:00-8:30am Breakfast 8:30-9:00am Introductions and Recap from Last Night

Collective Impact Funder Community of Practice: In-Person Meeting | September 29, 2016 PREPARED FOR: COLLECTIVE IMPACT FUNDER COMMUNITY OF PRACTICE PARTICIPANTS An Initiative of FSG and Aspen Institute Forum for Community Solutions Agenda for

962 views • 65 slides

More recommend