io.druid.indexer

Class HadoopDruidIndexerConfig

java.lang.Object

io.druid.indexer.HadoopDruidIndexerConfig

public class HadoopDruidIndexerConfig
extends Object

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	HadoopDruidIndexerConfig.IndexJobCounters

Field Summary

Fields

Modifier and Type	Field and Description
static String	CONFIG_PROPERTY
static Charset	javaNativeCharset
static com.fasterxml.jackson.databind.ObjectMapper	jsonMapper
protected static Properties	properties
static com.google.common.base.Joiner	tabJoiner
static com.google.common.base.Splitter	tabSplitter

Constructor Summary

Constructors

Constructor and Description
HadoopDruidIndexerConfig(HadoopIngestionSpec spec)

Method Summary

Methods

Modifier and Type	Method and Description
org.apache.hadoop.mapreduce.Job	addInputPaths(org.apache.hadoop.mapreduce.Job job) Job instance should have Configuration set (by calling addJobProperties(Job) or via injected system properties) before this method is called.
void	addJobProperties(org.apache.hadoop.mapreduce.Job job)
static HadoopDruidIndexerConfig	fromConfiguration(org.apache.hadoop.conf.Configuration conf)
static HadoopDruidIndexerConfig	fromDistributedFileSystem(String path)
static HadoopDruidIndexerConfig	fromFile(File file)
static HadoopDruidIndexerConfig	fromMap(Map<String,Object> argSpec)
static HadoopDruidIndexerConfig	fromSpec(HadoopIngestionSpec spec)
static HadoopDruidIndexerConfig	fromString(String str)
com.google.common.base.Optional<Iterable<Bucket>>	getAllBuckets()
com.google.common.base.Optional<Bucket>	getBucket(io.druid.data.input.InputRow inputRow) Get the proper bucket for some input row.
String	getDataSource()
GranularitySpec	getGranularitySpec()
IndexSpec	getIndexSpec()
com.google.common.base.Optional<List<org.joda.time.Interval>>	getIntervals()
long	getMaxPartitionSize()
io.druid.data.input.impl.InputRowParser	getParser()
PartitionsSpec	getPartitionsSpec()
PathSpec	getPathSpec()
HadoopIngestionSpec	getSchema()
com.google.common.base.Optional<Set<org.joda.time.Interval>>	getSegmentGranularIntervals()
HadoopyShardSpec	getShardSpec(Bucket bucket)
Long	getTargetPartitionSize()
String	getWorkingPath()
void	intoConfiguration(org.apache.hadoop.mapreduce.Job job)
boolean	isCombineText()
boolean	isDeterminingPartitions()
boolean	isIgnoreInvalidRows()
boolean	isOverwriteFiles()
boolean	isPersistInHeap()
boolean	isUpdaterJobSpecSet()
org.apache.hadoop.fs.Path	makeDescriptorInfoDir()
org.apache.hadoop.fs.Path	makeDescriptorInfoPath(io.druid.timeline.DataSegment segment)
org.apache.hadoop.fs.Path	makeGroupedDataDir()
org.apache.hadoop.fs.Path	makeIntermediatePath() Make the intermediate path for this job run.
org.apache.hadoop.fs.Path	makeIntervalInfoPath()
org.apache.hadoop.fs.Path	makeSegmentPartitionInfoPath(org.joda.time.Interval bucketInterval)
void	setGranularitySpec(GranularitySpec granularitySpec)
void	setShardSpecs(Map<org.joda.time.DateTime,List<HadoopyShardSpec>> shardSpecs)
void	setVersion(String version)
void	verify()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

CONFIG_PROPERTY

public static final String CONFIG_PROPERTY

See Also:

Constant Field Values

javaNativeCharset

public static final Charset javaNativeCharset

tabSplitter

public static final com.google.common.base.Splitter tabSplitter

tabJoiner

public static final com.google.common.base.Joiner tabJoiner

jsonMapper

public static final com.fasterxml.jackson.databind.ObjectMapper jsonMapper

properties

protected static final Properties properties

Constructor Detail

HadoopDruidIndexerConfig

public HadoopDruidIndexerConfig(HadoopIngestionSpec spec)

Method Detail

fromSpec

public static HadoopDruidIndexerConfig fromSpec(HadoopIngestionSpec spec)

fromMap

public static HadoopDruidIndexerConfig fromMap(Map<String,Object> argSpec)

fromFile

public static HadoopDruidIndexerConfig fromFile(File file)

fromString

public static HadoopDruidIndexerConfig fromString(String str)

fromDistributedFileSystem

public static HadoopDruidIndexerConfig fromDistributedFileSystem(String path)

fromConfiguration

public static HadoopDruidIndexerConfig fromConfiguration(org.apache.hadoop.conf.Configuration conf)

getSchema

public HadoopIngestionSpec getSchema()

getPathSpec

public PathSpec getPathSpec()

getDataSource

public String getDataSource()

getGranularitySpec

public GranularitySpec getGranularitySpec()

setGranularitySpec

public void setGranularitySpec(GranularitySpec granularitySpec)

getPartitionsSpec

public PartitionsSpec getPartitionsSpec()

getIndexSpec

public IndexSpec getIndexSpec()

isOverwriteFiles

public boolean isOverwriteFiles()

isIgnoreInvalidRows

public boolean isIgnoreInvalidRows()

setVersion

public void setVersion(String version)

setShardSpecs

public void setShardSpecs(Map<org.joda.time.DateTime,List<HadoopyShardSpec>> shardSpecs)

getIntervals

public com.google.common.base.Optional<List<org.joda.time.Interval>> getIntervals()

isDeterminingPartitions

public boolean isDeterminingPartitions()

getTargetPartitionSize

public Long getTargetPartitionSize()

getMaxPartitionSize

public long getMaxPartitionSize()

isUpdaterJobSpecSet

public boolean isUpdaterJobSpecSet()

isCombineText

public boolean isCombineText()

getParser

public io.druid.data.input.impl.InputRowParser getParser()

getShardSpec

public HadoopyShardSpec getShardSpec(Bucket bucket)

addInputPaths

public org.apache.hadoop.mapreduce.Job addInputPaths(org.apache.hadoop.mapreduce.Job job)
                                              throws IOException

Job instance should have Configuration set (by calling addJobProperties(Job) or via injected system properties) before this method is called. The PathSpec may create objects which depend on the values of these configurations.

Parameters:

job -

Returns:

Throws:

IOException

getBucket

public com.google.common.base.Optional<Bucket> getBucket(io.druid.data.input.InputRow inputRow)

Get the proper bucket for some input row.

Parameters:

inputRow - an InputRow

Returns:

the Bucket that this row belongs to

getSegmentGranularIntervals

public com.google.common.base.Optional<Set<org.joda.time.Interval>> getSegmentGranularIntervals()

getAllBuckets

public com.google.common.base.Optional<Iterable<Bucket>> getAllBuckets()

isPersistInHeap

public boolean isPersistInHeap()

getWorkingPath

public String getWorkingPath()

makeIntermediatePath

public org.apache.hadoop.fs.Path makeIntermediatePath()

Make the intermediate path for this job run.

Returns:

the intermediate path for this job run.

makeSegmentPartitionInfoPath

public org.apache.hadoop.fs.Path makeSegmentPartitionInfoPath(org.joda.time.Interval bucketInterval)

makeIntervalInfoPath

public org.apache.hadoop.fs.Path makeIntervalInfoPath()

makeDescriptorInfoDir

public org.apache.hadoop.fs.Path makeDescriptorInfoDir()

makeGroupedDataDir

public org.apache.hadoop.fs.Path makeGroupedDataDir()

makeDescriptorInfoPath

public org.apache.hadoop.fs.Path makeDescriptorInfoPath(io.druid.timeline.DataSegment segment)

addJobProperties

public void addJobProperties(org.apache.hadoop.mapreduce.Job job)

intoConfiguration

public void intoConfiguration(org.apache.hadoop.mapreduce.Job job)

verify

public void verify()