SimpleSparkAnalysis.py

# Databricks notebook source
# List contents of the sample datasets directory
# dbutils.fs.ls("/databricks-datasets/samples/online_retail/")
# display(dbutils.fs.ls("/databricks-datasets/samples"))
dbutils.fs.ls("/databricks-datasets/samples/population-vs-price/")

# COMMAND ----------

file_path = "/databricks-datasets/samples/population-vs-price/data_geo.csv"

df = spark.read.format("csv") \
  .option("header", "true") \
  .option("inferSchema", "true") \
  .load(file_path)

print("--- Data Preview ---")
df.show(5)

print("\n--- Schema ---")
df.printSchema()

# COMMAND ----------

# Import necessary functions
from pyspark.sql.functions import sum, col

# Perform grouping and aggregation
# 1. Group the DataFrame by the 'State' column
# 2. Calculate the sum of the '2014 Population estimate' column for each group
# 3. Alias the resulting sum column as 'TotalSales'
results_df = df.groupBy("State") \
               .agg(sum(col("2014 Population estimate").cast("integer")).alias("TotalSales"))

# Show the results - this is an Action!
print("\n--- Total Sales ---")
results_df.show(10) # Show top 10 results