# Create test files of different sizes

python3 << 'EOF'

import random

# Create 100 row test file

with open ( "test_100.csv" , "w" ) as f :

    f .write ( " id,name , age,salary ,department n " )

    for i in range ( 1 , 101 ) :

        name = random .choice ([ "Alice" , "Bob" , "Charlie" , "Diana" , "Eve" ])

        age = random .randint ( 20 , 65 )

        salary = 50000 + random .randint ( 0 , 50000 )

        dept = random .choice ([ "Engineering" , "Sales" , "Marketing" ])

        f .write ( f " { i } , { name } , { age } , { salary } , { dept } n " )

# Create 10,000 row test file

with open ( "test_10000.csv" , "w" ) as f :

    f .write ( " id,name , age,salary ,department n " )

    for i in range ( 1 , 10001 ) :

        name = random .choice ([ "Alice" , "Bob" , "Charlie" , "Diana" , "Eve" ])

        age = random .randint ( 20 , 65 )

        salary = 50000 + random .randint ( 0 , 50000 )

        dept = random .choice ([ "Engineering" , "Sales" , "Marketing" ])

        f .write ( f " { i } , { name } , { age } , { salary } , { dept } n " )

print ( "Test files created: test_100.csv, test_10000.csv" )

EOF

import time

from pyspark .sql import SparkSession

from pyspark .sql .types import *

spark = SparkSession .builder

    .appName ( " SchemaComparison " )

    .config ( " spark.sql.shuffle .partitions " , "4" )

    .getOrCreate ()

# Define explicit schema once

explicit_schema = StructType ( [

    StructField ( "id" , IntegerType ( ), True ),

    StructField ( "name" , StringType ( ), True ),

    StructField ( "age" , IntegerType ( ), True ),

    StructField ( "salary" , IntegerType ( ), True ),

    StructField ( "department" , StringType ( ), True )

])

# Test with 100 rows

print ( "=" * 60 )

print ( "TEST 1: Small file (100 rows)" )

print ( "=" * 60 )

# Inference

start = time .time ()

df_inf_100 = spark .read .csv ( "test_100.csv" , header = True , inferSchema = True )

count_inf = df_inf_100 .count ()  # Force evaluation

inf_time_100 = time .time () – start

# Explicit

start = time .time ()

df_exp_100 = spark .read .schema ( explicit_schema ).csv ( "test_100.csv" , header = True )

count_exp = df_exp_100 .count ()  # Force evaluation

exp_time_100 = time .time () – start

print ( f "Inference : { inf_time_100 :.3 f } s " )

print ( f "Explicit :  { exp_time_100 :.3 f } s " )

print ( f "Difference : { inf_time_100 – exp_time_100 :.3 f } s " )

# Test with 10,000 rows

print ( " n " + "=" * 60 )

print ( "TEST 2: Medium file (10,000 rows)" )

print ( "=" * 60 )

# Inference

start = time .time ()

df_inf_10k = spark .read .csv ( "test_10000.csv" , header = True , inferSchema = True )

count_inf = df_inf_10k .count ()

inf_time_10k = time .time () – start

# Explicit

start = time .time ()

df_exp_10k = spark .read .schema ( explicit_schema ).csv ( "test_10000.csv" , header = True )

count_exp = df_exp_10k .count ()

exp_time_10k = time .time () – start

print ( f "Inference : { inf_time_10k :.3 f } s " )

print ( f "Explicit :  { exp_time_10k :.3 f } s " )

print ( f "Speedup : { inf_time_10k / exp_time_10k :.1f } x faster" )

# Summary

print ( " n " + "=" * 60 )

print ( "SUMMARY" )

print ( "=" * 60 )

print ( f "100 rows:   Inference { inf_time_100 :.3 f } s vs Explicit { exp_time_100 :.3 f } s " )

print ( f "10K rows:   Inference { inf_time_10k :.3 f } s vs Explicit { exp_time_10k :.3 f } s " )

print ( f "Speedup on large file: { inf_time_10k / exp_time_10k :.1f } x" )

spark .stop ()

TEST 1 : Small file ( 100 rows )

Inference : 0.850s

Explicit :  0.620s

Difference : 0.230s

TEST 2 : Medium file ( 10 , 000 rows )

Inference : 2.150s

Explicit :  0.185s

Speedup : 11.6x faster

Start

  ↓

Exploring new data?  →  YES  → Use Inference

  ↓ NO

Production job?  →  YES  → Use Explicit

  ↓ NO

Large dataset?  →  YES  → Use Explicit

  ↓ NO

Performance matters?  →  YES  → Use Explicit

  ↓ NO

Use Inference (flexibility)

# EDA: Just exploring, performance not critical

df = spark .read .csv ( "unknown_data.csv" , header = True , inferSchema = True )

df .show ( 5 )

df .printSchema ()

df .describe ( ).show ()

# Production: Speed and reliability matter

schema = StructType ( [

    StructField ( " customer_id " , IntegerType ( ), False ),

    StructField ( "name" , StringType ( ), False ),

    StructField ( "email" , StringType ( ), False ),

    StructField ( " purchase_amount " , DoubleType ( ), True ),

    StructField ( "date" , DateType ( ), False ),

])

df = spark .read .schema ( schema ).csv ( "customers.csv" , header = True )

# Process with confidence

# Balanced approach: Define schema for consistency, but simple

# Usually explicit for consistency, but could use inference

df = spark .read .csv (

    "data.csv" ,

    header = True ,

    inferSchema = True ,

    samplingRatio = 0.5  # Sample 50% for faster inference

)

# This CSV might be misinterpreted

data_problematic = """ id,value , date,flag

1,100,2024-01- 15,true

2,200,2024-01- 16,false

3,300,2024-01- 17,yes

4,400,2024-01- 18,no

"""

# Spark infers:

# id: int (correct)

# value: int (correct)

# date: string (should be date!)

# flag: string (should be boolean !)

# With explicit schema, all types are correct

# Inference might give different results on different samples

# File 1: Infers age as IntegerType

# File 2: Infers age as StringType (had a non-numeric value)

# File 3: Different inference again

# Explicit schema ensures consistency across all files

💡 Recommendations

•   Development: Use inference for speed and flexibility

•   Production: Always use explicit schemas

•   Data Lakes: Use explicit schemas with versioning

•   Real-time: Use explicit schemas (streaming requires it)

•   Batch jobs: Prefer explicit schemas for consistency

📚 Study Notes

•   Inference: 5-10x slower, good for exploration

•   Explicit: Much faster, required for production

•   Accuracy: Inference can get types wrong

•   Consistency: Explicit ensures same types every time

•   Maintenance: Explicit requires schema updates

•   Trade-off: Development speed vs runtime speed