from pyspark.sql import SparkSession

spark = SparkSession.builder \
    .appName("Export-Full-Dataset-For-Sklearn") \
    .master("spark://10.0.0.15:7077") \
    .getOrCreate()

spark.sparkContext.setLogLevel("WARN")

df = spark.read.parquet("hdfs://10.0.0.15:9000/ids/dataset_full_clean_v2.parquet")
df = df.drop("Timestamp")

print("Full dataset rows: " + str(df.count()))

df.coalesce(1).write.mode("overwrite").option("header", "true").csv(
    "hdfs://10.0.0.15:9000/ids/full_100pct_for_sklearn_csv")

print("Export done.")
spark.stop()
