public class HadoopDruidIndexerConfig extends Object
Modifier and Type | Class and Description |
---|---|
static class |
HadoopDruidIndexerConfig.IndexJobCounters |
Modifier and Type | Field and Description |
---|---|
static String |
CONFIG_PROPERTY |
static HadoopKerberosConfig |
HADOOP_KERBEROS_CONFIG |
static IndexIO |
INDEX_IO |
static IndexMerger |
INDEX_MERGER |
static IndexMergerV9 |
INDEX_MERGER_V9 |
static Charset |
JAVA_NATIVE_CHARSET |
static com.fasterxml.jackson.databind.ObjectMapper |
JSON_MAPPER |
static com.google.common.base.Joiner |
TAB_JOINER |
static com.google.common.base.Splitter |
TAB_SPLITTER |
Constructor and Description |
---|
HadoopDruidIndexerConfig(HadoopIngestionSpec spec) |
Modifier and Type | Method and Description |
---|---|
org.apache.hadoop.mapreduce.Job |
addInputPaths(org.apache.hadoop.mapreduce.Job job)
Job instance should have Configuration set (by calling
addJobProperties(Job)
or via injected system properties) before this method is called. |
void |
addJobProperties(org.apache.hadoop.mapreduce.Job job) |
static HadoopDruidIndexerConfig |
fromConfiguration(org.apache.hadoop.conf.Configuration conf) |
static HadoopDruidIndexerConfig |
fromDistributedFileSystem(String path) |
static HadoopDruidIndexerConfig |
fromFile(File file) |
static HadoopDruidIndexerConfig |
fromMap(Map<String,Object> argSpec) |
static HadoopDruidIndexerConfig |
fromSpec(HadoopIngestionSpec spec) |
static HadoopDruidIndexerConfig |
fromString(String str) |
com.google.common.base.Optional<Iterable<Bucket>> |
getAllBuckets() |
com.google.common.base.Optional<Bucket> |
getBucket(InputRow inputRow)
Get the proper bucket for some input row.
|
String |
getDataSource() |
GranularitySpec |
getGranularitySpec() |
IndexSpec |
getIndexSpec() |
List<org.joda.time.Interval> |
getInputIntervals() |
com.google.common.base.Optional<List<org.joda.time.Interval>> |
getIntervals() |
long |
getMaxPartitionSize() |
InputRowParser |
getParser() |
PartitionsSpec |
getPartitionsSpec() |
PathSpec |
getPathSpec() |
HadoopIngestionSpec |
getSchema() |
com.google.common.base.Optional<Set<org.joda.time.Interval>> |
getSegmentGranularIntervals() |
HadoopyShardSpec |
getShardSpec(Bucket bucket) |
int |
getShardSpecCount(Bucket bucket) |
Long |
getTargetPartitionSize() |
String |
getWorkingPath() |
void |
intoConfiguration(org.apache.hadoop.mapreduce.Job job) |
boolean |
isBuildV9Directly() |
boolean |
isCombineText() |
boolean |
isDeterminingPartitions() |
boolean |
isForceExtendableShardSpecs() |
boolean |
isIgnoreInvalidRows() |
boolean |
isOverwriteFiles() |
boolean |
isUpdaterJobSpecSet() |
org.apache.hadoop.fs.Path |
makeDescriptorInfoDir() |
org.apache.hadoop.fs.Path |
makeDescriptorInfoPath(DataSegment segment) |
org.apache.hadoop.fs.Path |
makeGroupedDataDir() |
org.apache.hadoop.fs.Path |
makeIntermediatePath()
Make the intermediate path for this job run.
|
org.apache.hadoop.fs.Path |
makeIntervalInfoPath() |
org.apache.hadoop.fs.Path |
makeSegmentPartitionInfoPath(org.joda.time.Interval bucketInterval) |
void |
setGranularitySpec(GranularitySpec granularitySpec) |
void |
setShardSpecs(Map<Long,List<HadoopyShardSpec>> shardSpecs) |
void |
setVersion(String version) |
void |
verify() |
public static final String CONFIG_PROPERTY
public static final Charset JAVA_NATIVE_CHARSET
public static final com.google.common.base.Splitter TAB_SPLITTER
public static final com.google.common.base.Joiner TAB_JOINER
public static final com.fasterxml.jackson.databind.ObjectMapper JSON_MAPPER
public static final IndexIO INDEX_IO
public static final IndexMerger INDEX_MERGER
public static final IndexMergerV9 INDEX_MERGER_V9
public static final HadoopKerberosConfig HADOOP_KERBEROS_CONFIG
public HadoopDruidIndexerConfig(HadoopIngestionSpec spec)
public static HadoopDruidIndexerConfig fromSpec(HadoopIngestionSpec spec)
public static HadoopDruidIndexerConfig fromMap(Map<String,Object> argSpec)
public static HadoopDruidIndexerConfig fromFile(File file)
public static HadoopDruidIndexerConfig fromString(String str)
public static HadoopDruidIndexerConfig fromDistributedFileSystem(String path)
public static HadoopDruidIndexerConfig fromConfiguration(org.apache.hadoop.conf.Configuration conf)
public HadoopIngestionSpec getSchema()
public PathSpec getPathSpec()
public String getDataSource()
public GranularitySpec getGranularitySpec()
public void setGranularitySpec(GranularitySpec granularitySpec)
public PartitionsSpec getPartitionsSpec()
public IndexSpec getIndexSpec()
public boolean isOverwriteFiles()
public boolean isIgnoreInvalidRows()
public void setVersion(String version)
public void setShardSpecs(Map<Long,List<HadoopyShardSpec>> shardSpecs)
public com.google.common.base.Optional<List<org.joda.time.Interval>> getIntervals()
public boolean isDeterminingPartitions()
public Long getTargetPartitionSize()
public boolean isForceExtendableShardSpecs()
public long getMaxPartitionSize()
public boolean isUpdaterJobSpecSet()
public boolean isCombineText()
public InputRowParser getParser()
public HadoopyShardSpec getShardSpec(Bucket bucket)
public int getShardSpecCount(Bucket bucket)
public boolean isBuildV9Directly()
public org.apache.hadoop.mapreduce.Job addInputPaths(org.apache.hadoop.mapreduce.Job job) throws IOException
addJobProperties(Job)
or via injected system properties) before this method is called. The PathSpec
may
create objects which depend on the values of these configurations.job
- IOException
public com.google.common.base.Optional<Bucket> getBucket(InputRow inputRow)
inputRow
- an InputRowpublic com.google.common.base.Optional<Set<org.joda.time.Interval>> getSegmentGranularIntervals()
public List<org.joda.time.Interval> getInputIntervals()
public String getWorkingPath()
public org.apache.hadoop.fs.Path makeIntermediatePath()
public org.apache.hadoop.fs.Path makeSegmentPartitionInfoPath(org.joda.time.Interval bucketInterval)
public org.apache.hadoop.fs.Path makeIntervalInfoPath()
public org.apache.hadoop.fs.Path makeDescriptorInfoDir()
public org.apache.hadoop.fs.Path makeGroupedDataDir()
public org.apache.hadoop.fs.Path makeDescriptorInfoPath(DataSegment segment)
public void addJobProperties(org.apache.hadoop.mapreduce.Job job)
public void intoConfiguration(org.apache.hadoop.mapreduce.Job job)
public void verify()
Copyright © 2011–2017. All rights reserved.