CODE EXAMPLE FOR PYTHON

reading csv in spark

# Reading CSV file into a data frame
file = "your_csv_file.csv"
schema = "DEST_COUNTRY_NAME STRING, ORIGIN_COUNTRY_NAME STRING, COUNT INT" # If any
df = (spark.read.format("csv")
    .option("header", "true")
    .schema(schema)
    .option("mode", "FAILFAST") # Exit if any errors 
    .option("nullValue", "") # Replace any null data field with quotes / ""
    .load(file))

# Reading a CSV file into a Spark SQL table
spark.sql("""CREATE OR REPLACE TEMPORARY VIEW view_name
        USING csv 
        OPTION (
            path csv_path_location
            header "true"
            inferSchema "true"
            mode "FAILFAST)""")

# Writing DataFrames to CSV files 
df.wirte.format("csv").mode("overwrite").save("csv_path_you_want_to_save")