Big Data Tools

python
Run Inline
import dask.dataframe as dd
# Dask: parallel computing
df = dd.read_csv('large_file_*.csv')
result = df.groupby('category').value.mean().compute()
print(result)
# PySpark basics
from pyspark.sql import SparkSession
spark = SparkSession.builder.appName("example").getOrCreate()
df = spark.read.csv("data.csv", header=True, inferSchema=True)
df.show()
df.groupBy("category").agg({"value": "mean"}).show()