from pyspark.sql import SparkSession
spark = SparkSession.builder.appName("Verify").master("spark://10.0.0.15:7077").getOrCreate()
spark.sparkContext.setLogLevel("WARN")
df = spark.read.parquet("hdfs://10.0.0.15:9000/ids/dataset_full_clean_v2.parquet")
print("Row count: " + str(df.count()))
df.groupBy("Label").count().orderBy("count", ascending=False).show(20, truncate=False)
spark.stop()
