Class PslSparkUtils (1.0.0) 

public class PslSparkUtils

Inheritance

java.lang.Object > PslSparkUtils

Static Methods

convertAttributesToSparkMap(ListMultimap<String,ByteString> attributeMap)

public static ArrayBasedMapData convertAttributesToSparkMap(ListMultimap<String,ByteString> attributeMap)
Parameter
Name Description
attributeMap com.google.common.collect.ListMultimap<String,ByteString>
Returns
Type Description
org.apache.spark.sql.catalyst.util.ArrayBasedMapData

getSparkEndOffset(SparkSourceOffset headOffset, SparkSourceOffset startOffset, long maxMessagesPerBatch, long topicPartitionCount)

public static SparkSourceOffset getSparkEndOffset(SparkSourceOffset headOffset, SparkSourceOffset startOffset, long maxMessagesPerBatch, long topicPartitionCount)
Parameters
Name Description
headOffset SparkSourceOffset
startOffset SparkSourceOffset
maxMessagesPerBatch long
topicPartitionCount long
Returns
Type Description
SparkSourceOffset

toInternalRow(SequencedMessage msg, SubscriptionPath subscription, Partition partition)

public static InternalRow toInternalRow(SequencedMessage msg, SubscriptionPath subscription, Partition partition)
Parameters
Name Description
msg com.google.cloud.pubsublite.SequencedMessage
subscription com.google.cloud.pubsublite.SubscriptionPath
partition com.google.cloud.pubsublite.Partition
Returns
Type Description
org.apache.spark.sql.catalyst.InternalRow

toPslPartitionOffset(SparkPartitionOffset sparkPartitionOffset)

public static PslPartitionOffset toPslPartitionOffset(SparkPartitionOffset sparkPartitionOffset)
Parameter
Name Description
sparkPartitionOffset com.google.cloud.pubsublite.spark.SparkPartitionOffset
Returns
Type Description
PslPartitionOffset

toPslSourceOffset(SparkSourceOffset sparkSourceOffset)

public static PslSourceOffset toPslSourceOffset(SparkSourceOffset sparkSourceOffset)
Parameter
Name Description
sparkSourceOffset SparkSourceOffset
Returns
Type Description
PslSourceOffset

toPubSubMessage(StructType inputSchema, InternalRow row)

public static Message toPubSubMessage(StructType inputSchema, InternalRow row)
Parameters
Name Description
inputSchema org.apache.spark.sql.types.StructType
row org.apache.spark.sql.catalyst.InternalRow
Returns
Type Description
com.google.cloud.pubsublite.Message

toSparkSourceOffset(PslSourceOffset pslSourceOffset)

public static SparkSourceOffset toSparkSourceOffset(PslSourceOffset pslSourceOffset)
Parameter
Name Description
pslSourceOffset PslSourceOffset
Returns
Type Description
SparkSourceOffset

verifyWriteInputSchema(StructType inputSchema)

public static void verifyWriteInputSchema(StructType inputSchema)

Make sure data fields for publish have expected Spark DataType if they exist.

Parameter
Name Description
inputSchema org.apache.spark.sql.types.StructType

input table schema to write to Pub/Sub Lite.

Constructors

PslSparkUtils()

public PslSparkUtils()