PYTHON

column to list pyspark

# convert student Name to list using
# flatMap
print(dataframe.select('student Name').
      rdd.flatMap(lambda x: x).collect())
 
# convert student ID to list using
# flatMap
print(dataframe.select('student ID').
      rdd.flatMap(lambda x: x).collect())

convert python list to pyspark column

from pyspark.sql import Row
    li=[1,2,3,4]
    rdd1 = sc.parallelize(li)
    row_rdd = rdd1.map(lambda x: Row(x))
    df=sqlContext.createDataFrame(row_rdd,['numbers']).show()

column to list pyspark

>>> mvv_list = mvv_count_df.select('mvv').collect()
>>> mvv_list[0]
Out: Row(mvv=1)