org.apache.carbondata.streaming.segment

Class StreamSegment

java.lang.Object

org.apache.carbondata.streaming.segment.StreamSegment

public class StreamSegment
extends Object

streaming segment manager

Constructor Summary

Constructors

Constructor and Description
StreamSegment()

Method Summary

Modifier and Type	Method and Description
static StreamFileIndex	appendBatchData(org.apache.carbondata.common.CarbonIterator<Object[]> inputIterators, org.apache.hadoop.mapreduce.TaskAttemptContext job, org.apache.carbondata.processing.loading.model.CarbonLoadModel carbonLoadModel) invoke CarbonStreamOutputFormat to append batch data to existing carbondata file
static String	close(org.apache.carbondata.core.metadata.schema.table.CarbonTable table, String segmentId) marker old stream segment to finished status and create new stream segment
static org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex	collectMinMaxIndex(org.apache.carbondata.core.datastore.page.statistics.SimpleStatsResult[] dimStats, org.apache.carbondata.core.datastore.page.statistics.SimpleStatsResult[] mrsStats)
static void	finishStreaming(org.apache.carbondata.core.metadata.schema.table.CarbonTable carbonTable) change the status of the segment from "streaming" to "streaming finish"
static org.apache.carbondata.core.datastore.filesystem.CarbonFile[]	listDataFiles(String segmentDir, org.apache.carbondata.core.datastore.impl.FileFactory.FileType fileType) list all carbondata files of a segment
static org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex	mergeBlockletMinMax(org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex to, org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex from, org.apache.carbondata.core.metadata.datatype.DataType[] msrDataTypes) merge blocklet min/max to generate batch min/max
static String	open(org.apache.carbondata.core.metadata.schema.table.CarbonTable table) get stream segment or create new stream segment if not exists
static List<org.apache.carbondata.format.BlockIndex>	readIndexFile(String indexPath, org.apache.carbondata.core.datastore.impl.FileFactory.FileType fileType) read index file to list BlockIndex
static void	recoverFileIfRequired(String segmentDir, String fileName, String indexName) check the health of stream data file and try to recover data file from task fault this method will be invoked in following scenarios.
static void	recoverSegmentIfRequired(String segmentDir) check the health of stream segment and try to recover segment from job fault this method will be invoked in following scenarios.
static long	size(String segmentDir) calculate the size of the segment by the accumulation of data sizes in index file
static void	updateIndexFile(String segmentDir, StreamFileIndex[] blockIndexes, org.apache.carbondata.core.metadata.datatype.DataType[] msrDataTypes) update carbon index file after a stream batch.

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

StreamSegment

public StreamSegment()

Method Detail

open

public static String open(org.apache.carbondata.core.metadata.schema.table.CarbonTable table)
                   throws IOException

get stream segment or create new stream segment if not exists

Throws:

IOException

close

public static String close(org.apache.carbondata.core.metadata.schema.table.CarbonTable table,
                           String segmentId)
                    throws IOException

marker old stream segment to finished status and create new stream segment

Throws:

IOException

finishStreaming

public static void finishStreaming(org.apache.carbondata.core.metadata.schema.table.CarbonTable carbonTable)
                            throws IOException

change the status of the segment from "streaming" to "streaming finish"

Throws:

IOException

collectMinMaxIndex

public static org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex collectMinMaxIndex(org.apache.carbondata.core.datastore.page.statistics.SimpleStatsResult[] dimStats,
                                                                                                        org.apache.carbondata.core.datastore.page.statistics.SimpleStatsResult[] mrsStats)

appendBatchData

public static StreamFileIndex appendBatchData(org.apache.carbondata.common.CarbonIterator<Object[]> inputIterators,
                                              org.apache.hadoop.mapreduce.TaskAttemptContext job,
                                              org.apache.carbondata.processing.loading.model.CarbonLoadModel carbonLoadModel)
                                       throws Exception

invoke CarbonStreamOutputFormat to append batch data to existing carbondata file

Throws:

Exception

recoverSegmentIfRequired

public static void recoverSegmentIfRequired(String segmentDir)
                                     throws IOException

check the health of stream segment and try to recover segment from job fault this method will be invoked in following scenarios. 1. at the begin of the streaming (StreamSinkFactory.getStreamSegmentId) 2. after job failed (CarbonAppendableStreamSink.writeDataFileJob)

Throws:

IOException

recoverFileIfRequired

public static void recoverFileIfRequired(String segmentDir,
                                         String fileName,
                                         String indexName)
                                  throws IOException

check the health of stream data file and try to recover data file from task fault this method will be invoked in following scenarios. 1. at the begin of writing data file task

Throws:

IOException

listDataFiles

public static org.apache.carbondata.core.datastore.filesystem.CarbonFile[] listDataFiles(String segmentDir,
                                                                                         org.apache.carbondata.core.datastore.impl.FileFactory.FileType fileType)

list all carbondata files of a segment

readIndexFile

public static List<org.apache.carbondata.format.BlockIndex> readIndexFile(String indexPath,
                                                                          org.apache.carbondata.core.datastore.impl.FileFactory.FileType fileType)
                                                                   throws IOException

read index file to list BlockIndex

Parameters:

indexPath - path of the index file

fileType - file type of the index file

Returns:

the list of BlockIndex in the index file

Throws:

IOException

mergeBlockletMinMax

public static org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex mergeBlockletMinMax(org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex to,
                                                                                                         org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex from,
                                                                                                         org.apache.carbondata.core.metadata.datatype.DataType[] msrDataTypes)

merge blocklet min/max to generate batch min/max

updateIndexFile

public static void updateIndexFile(String segmentDir,
                                   StreamFileIndex[] blockIndexes,
                                   org.apache.carbondata.core.metadata.datatype.DataType[] msrDataTypes)
                            throws IOException

update carbon index file after a stream batch.

Throws:

IOException

size

public static long size(String segmentDir)
                 throws IOException

calculate the size of the segment by the accumulation of data sizes in index file

Throws:

IOException