io.druid.indexer

Class JobHelper

java.lang.Object

io.druid.indexer.JobHelper

public class JobHelper
extends Object

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static interface	JobHelper.DataPusher Simple interface for retry operations

Constructor Summary

Constructors

Constructor and Description
JobHelper()

Method Summary

Methods

Modifier and Type	Method and Description
static long	copyFileToZipStream(File file, ZipOutputStream zipOutputStream, org.apache.hadoop.util.Progressable progressable)
static org.apache.hadoop.fs.Path	distributedClassPath(org.apache.hadoop.fs.Path base)
static org.apache.hadoop.fs.Path	distributedClassPath(String path)
static void	ensurePaths(HadoopDruidIndexerConfig config)
static URI	getURIFromSegment(io.druid.timeline.DataSegment dataSegment)
static org.apache.hadoop.conf.Configuration	injectSystemProperties(org.apache.hadoop.conf.Configuration conf)
static void	injectSystemProperties(org.apache.hadoop.mapreduce.Job job)
static org.apache.hadoop.fs.Path	makeSegmentOutputPath(org.apache.hadoop.fs.Path basePath, org.apache.hadoop.fs.FileSystem fileSystem, String dataSource, String version, org.joda.time.Interval interval, int partitionNum)
static org.apache.hadoop.fs.Path	prependFSIfNullScheme(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path)
static ProgressIndicator	progressIndicatorForContext(org.apache.hadoop.mapreduce.TaskAttemptContext context)
static boolean	runJobs(List<Jobby> jobs, HadoopDruidIndexerConfig config)
static io.druid.timeline.DataSegment	serializeOutIndex(io.druid.timeline.DataSegment segmentTemplate, org.apache.hadoop.conf.Configuration configuration, org.apache.hadoop.util.Progressable progressable, org.apache.hadoop.mapreduce.TaskAttemptID taskAttemptID, File mergedBase, org.apache.hadoop.fs.Path segmentBasePath)
static void	setupClasspath(org.apache.hadoop.fs.Path distributedClassPath, org.apache.hadoop.mapreduce.Job job)
static long	unzipNoGuava(org.apache.hadoop.fs.Path zip, org.apache.hadoop.conf.Configuration configuration, File outDir, org.apache.hadoop.util.Progressable progressable)
static void	writeSegmentDescriptor(org.apache.hadoop.fs.FileSystem outputFS, io.druid.timeline.DataSegment segment, org.apache.hadoop.fs.Path descriptorPath, org.apache.hadoop.util.Progressable progressable)
static long	zipAndCopyDir(File baseDir, OutputStream baseOutputStream, org.apache.hadoop.util.Progressable progressable)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

JobHelper

public JobHelper()

Method Detail

distributedClassPath

public static org.apache.hadoop.fs.Path distributedClassPath(String path)

distributedClassPath

public static org.apache.hadoop.fs.Path distributedClassPath(org.apache.hadoop.fs.Path base)

setupClasspath

public static void setupClasspath(org.apache.hadoop.fs.Path distributedClassPath,
                  org.apache.hadoop.mapreduce.Job job)
                           throws IOException

Throws:

IOException

injectSystemProperties

public static void injectSystemProperties(org.apache.hadoop.mapreduce.Job job)

injectSystemProperties

public static org.apache.hadoop.conf.Configuration injectSystemProperties(org.apache.hadoop.conf.Configuration conf)

ensurePaths

public static void ensurePaths(HadoopDruidIndexerConfig config)

runJobs

public static boolean runJobs(List<Jobby> jobs,
              HadoopDruidIndexerConfig config)

serializeOutIndex

public static io.druid.timeline.DataSegment serializeOutIndex(io.druid.timeline.DataSegment segmentTemplate,
                                              org.apache.hadoop.conf.Configuration configuration,
                                              org.apache.hadoop.util.Progressable progressable,
                                              org.apache.hadoop.mapreduce.TaskAttemptID taskAttemptID,
                                              File mergedBase,
                                              org.apache.hadoop.fs.Path segmentBasePath)
                                                       throws IOException

Throws:

IOException

writeSegmentDescriptor

public static void writeSegmentDescriptor(org.apache.hadoop.fs.FileSystem outputFS,
                          io.druid.timeline.DataSegment segment,
                          org.apache.hadoop.fs.Path descriptorPath,
                          org.apache.hadoop.util.Progressable progressable)
                                   throws IOException

Throws:

IOException

zipAndCopyDir

public static long zipAndCopyDir(File baseDir,
                 OutputStream baseOutputStream,
                 org.apache.hadoop.util.Progressable progressable)
                          throws IOException

Throws:

IOException

copyFileToZipStream

public static long copyFileToZipStream(File file,
                       ZipOutputStream zipOutputStream,
                       org.apache.hadoop.util.Progressable progressable)
                                throws IOException

Throws:

IOException

makeSegmentOutputPath

public static org.apache.hadoop.fs.Path makeSegmentOutputPath(org.apache.hadoop.fs.Path basePath,
                                              org.apache.hadoop.fs.FileSystem fileSystem,
                                              String dataSource,
                                              String version,
                                              org.joda.time.Interval interval,
                                              int partitionNum)

prependFSIfNullScheme

public static org.apache.hadoop.fs.Path prependFSIfNullScheme(org.apache.hadoop.fs.FileSystem fs,
                                              org.apache.hadoop.fs.Path path)

unzipNoGuava

public static long unzipNoGuava(org.apache.hadoop.fs.Path zip,
                org.apache.hadoop.conf.Configuration configuration,
                File outDir,
                org.apache.hadoop.util.Progressable progressable)
                         throws IOException

Throws:

IOException

getURIFromSegment

public static URI getURIFromSegment(io.druid.timeline.DataSegment dataSegment)

progressIndicatorForContext

public static ProgressIndicator progressIndicatorForContext(org.apache.hadoop.mapreduce.TaskAttemptContext context)