Interview Questions and Answers

Example:

from pyspark.sql import SparkSession

spark = SparkSession.builder.appName('example').getOrCreate()

Example:

grouped_data = df.groupBy('Category').agg({'Price': 'mean'})

Example:

from pyspark.sql import SparkSession

spark = SparkSession.builder.appName('example').getOrCreate()

Example:

data = df.collect()

Example:

result = df1.union(df2)

Example:

grouped_data = df.groupBy('Category').agg({'Price': 'mean'})

Example:

df.createOrReplaceTempView('temp_view')

Example:

result = df.orderBy('column')

Example:

data = [1, 2, 3, 4, 5]
rdd = spark.sparkContext.parallelize(data)

Example:

df = spark.createDataFrame([(1, 'John'), (2, 'Jane')], ['ID', 'Name'])

Example:

df.cache()

Example:

df.na.drop()

Example:

from pyspark.sql.functions import explode

exploded_df = df.select('ID', explode('items').alias('item'))

Example:

df.persist()

Example:

from pyspark.sql.functions import explode

exploded_df = df.select('ID', explode('items').alias('item'))

Example:

df.na.drop()

Example:

df.cache()

Example:

result = df.agg({'Sales': 'sum', 'Quantity': 'avg'})

Example:

df_coalesced = df.coalesce(5)

Example:

result = df1.join(df2, df1['key'] == df2['key'], 'inner')

Example:

from pyspark.sql.functions import broadcast

result = df1.join(broadcast(df2), 'key')

Example:

from pyspark.sql.window import Window
from pyspark.sql.functions import row_number

window_spec = Window.orderBy('column')
result = df.withColumn('row_num', row_number().over(window_spec))

Example:

spark.sparkContext.setCheckpointDir('hdfs://path/to/checkpoint')
df_checkpointed = df.checkpoint()

Example:

df.write.option('skew_hint', 'true').parquet('output_path')

Example:

from pyspark.sql.window import Window
from pyspark.sql.functions import sum

window_spec = Window.partitionBy('category').orderBy('value')
result = df.withColumn('sum_value', sum('value').over(window_spec))

Example:

from pyspark.sql.functions import broadcast

result = df1.join(broadcast(df2), 'key')

Example:

from pyspark.sql.functions import broadcast

result = df1.join(broadcast(df2), 'key')

Example:

accumulator = spark.sparkContext.accumulator(0)

# Inside a transformation or action
accumulator.add(1)

Example:

from pyspark.sql.functions import broadcast

result = df1.join(broadcast(df2), 'key')

Example:

df.write.option('skew_hint', 'true').parquet('output_path')