CODE EXAMPLE FOR R

How to create a new column with spark_apply

library(sparklyr)
sc <- spark_connect(master = "local")

df <- sdf_len(sc, 100)
df <- sdf_repartition(df, 5) %>%
    spark_apply(function(df){
        df['new']<-df$id * 10
        df
    })