Work with Data

import os
from pyspark.sql import SparkSession

spark = SparkSession.builder.getOrCreate()

# the default configuration will pick up your credentials from environment variables
# No additional configuration is necessary

# test reading
df = spark.read.json("s3a://bucket/prefix1/prefix2/people.json")
df.show()

import os
from pyspark.sql import SparkSession

spark = SparkSession.builder.getOrCreate()

# for write operations you will need the ARN of the key to use
# Note that the credentials used need to have proper access to use the key
kms_key_arn = "<your key ARN here>"

# configure the connector
# This example assumes credentials from environment variables so no need to configure
# Note: The encryption config is not needed for read only operations
hadoop_conf = spark.sparkContext._jsc.hadoopConfiguration()
hadoop_conf.set("fs.s3a.server-side-encryption-algorithm", "SSE-KMS")
hadoop_conf.set("fs.s3a.server-side-encryption.key", kms_key_arn)

# test reading
df = spark.read.json("s3a://bucket/prefix1/prefix2/people.json")
df.show()

# test writing
df.write.mode("overwrite").parquet("3a://bucket/prefix1/prefix2/write-test/output")

import os
from pyspark.sql import SparkSession

try:
    spark.stop()
except:
    pass
spark = SparkSession.builder.getOrCreate()

#The name of one of the roles you are entitled to
profile_name="my-role-name-read-write"

# use boto3 for convenience to get credentials form credentials file populated by Domino
# can use any method desirable to extract the credentials
import boto3
role_creds = boto3.Session(profile_name=profile_name).get_credentials().get_frozen_credentials()

# configure the connector
# Use the TemporaryAWSCredentialsProvider
hadoop_conf = spark.sparkContext._jsc.hadoopConfiguration()
hadoop_conf.set("fs.s3a.aws.credentials.provider", "org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider")
hadoop_conf.set("fs.s3a.access.key", role_creds.access_key)
hadoop_conf.set("fs.s3a.secret.key", role_creds.secret_key)
hadoop_conf.set("fs.s3a.session.token", role_creds.token)

# test reading
df = spark.read.json("s3a://bucket/prefix1/prefix2/people.json")
df.show()

# test writing
df.write.mode("overwrite").parquet("s3a://bucket/prefix1/prefix2/write-test/output")