io.druid.indexer

Class JobHelper

java.lang.Object

io.druid.indexer.JobHelper

public class JobHelper
extends Object

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static interface	JobHelper.DataPusher Simple interface for retry operations

Field Summary

Fields

Modifier and Type	Field and Description
static String	DESCRIPTOR_JSON
static String	INDEX_ZIP

Constructor Summary

Constructors

Constructor and Description
JobHelper()

Method Summary

Modifier and Type	Method and Description
static void	authenticate(HadoopDruidIndexerConfig config) Dose authenticate against a secured hadoop cluster In case of any bug fix make sure to fix the code at HdfsStorageAuthentication#authenticate as well.
static long	copyFileToZipStream(File file, ZipOutputStream zipOutputStream, org.apache.hadoop.util.Progressable progressable)
static boolean	deleteWithRetry(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path, boolean recursive)
static org.apache.hadoop.fs.Path	distributedClassPath(org.apache.hadoop.fs.Path base)
static org.apache.hadoop.fs.Path	distributedClassPath(String path)
static void	ensurePaths(HadoopDruidIndexerConfig config)
static URI	getURIFromSegment(DataSegment dataSegment)
static org.apache.hadoop.conf.Configuration	injectSystemProperties(org.apache.hadoop.conf.Configuration conf)
static void	injectSystemProperties(org.apache.hadoop.mapreduce.Job job)
static boolean	isHdfs(org.apache.hadoop.fs.FileSystem fs)
static org.apache.hadoop.fs.Path	makeFileNamePath(org.apache.hadoop.fs.Path basePath, org.apache.hadoop.fs.FileSystem fs, DataSegment segmentTemplate, String baseFileName)
static org.apache.hadoop.fs.Path	makeTmpPath(org.apache.hadoop.fs.Path basePath, org.apache.hadoop.fs.FileSystem fs, DataSegment segmentTemplate, org.apache.hadoop.mapreduce.TaskAttemptID taskAttemptID)
static org.apache.hadoop.fs.Path	prependFSIfNullScheme(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path)
static ProgressIndicator	progressIndicatorForContext(org.apache.hadoop.mapreduce.TaskAttemptContext context)
static boolean	runJobs(List<Jobby> jobs, HadoopDruidIndexerConfig config)
static DataSegment	serializeOutIndex(DataSegment segmentTemplate, org.apache.hadoop.conf.Configuration configuration, org.apache.hadoop.util.Progressable progressable, File mergedBase, org.apache.hadoop.fs.Path finalIndexZipFilePath, org.apache.hadoop.fs.Path finalDescriptorPath, org.apache.hadoop.fs.Path tmpPath)
static void	setupClasspath(org.apache.hadoop.fs.Path distributedClassPath, org.apache.hadoop.fs.Path intermediateClassPath, org.apache.hadoop.mapreduce.Job job) Uploads jar files to hdfs and configures the classpath.
static com.google.common.base.Predicate<Throwable>	shouldRetryPredicate()
static long	unzipNoGuava(org.apache.hadoop.fs.Path zip, org.apache.hadoop.conf.Configuration configuration, File outDir, org.apache.hadoop.util.Progressable progressable)
static void	writeSegmentDescriptor(org.apache.hadoop.fs.FileSystem outputFS, DataSegment segment, org.apache.hadoop.fs.Path descriptorPath, org.apache.hadoop.util.Progressable progressable)
static long	zipAndCopyDir(File baseDir, OutputStream baseOutputStream, org.apache.hadoop.util.Progressable progressable)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

INDEX_ZIP

public static final String INDEX_ZIP

See Also:

Constant Field Values

DESCRIPTOR_JSON

public static final String DESCRIPTOR_JSON

See Also:

Constant Field Values

Constructor Detail

JobHelper

public JobHelper()

Method Detail

distributedClassPath

public static org.apache.hadoop.fs.Path distributedClassPath(String path)

distributedClassPath

public static org.apache.hadoop.fs.Path distributedClassPath(org.apache.hadoop.fs.Path base)

authenticate

public static void authenticate(HadoopDruidIndexerConfig config)

Dose authenticate against a secured hadoop cluster In case of any bug fix make sure to fix the code at HdfsStorageAuthentication#authenticate as well.

Parameters:

config - containing the principal name and keytab path.

setupClasspath

public static void setupClasspath(org.apache.hadoop.fs.Path distributedClassPath,
                                  org.apache.hadoop.fs.Path intermediateClassPath,
                                  org.apache.hadoop.mapreduce.Job job)
                           throws IOException

Uploads jar files to hdfs and configures the classpath. Snapshot jar files are uploaded to intermediateClasspath and not shared across multiple jobs. Non-Snapshot jar files are uploaded to a distributedClasspath and shared across multiple jobs.

Parameters:

distributedClassPath - classpath shared across multiple jobs

intermediateClassPath - classpath exclusive for this job. used to upload SNAPSHOT jar files.

job - job to run

Throws:

IOException

shouldRetryPredicate

public static final com.google.common.base.Predicate<Throwable> shouldRetryPredicate()

injectSystemProperties

public static void injectSystemProperties(org.apache.hadoop.mapreduce.Job job)

injectSystemProperties

public static org.apache.hadoop.conf.Configuration injectSystemProperties(org.apache.hadoop.conf.Configuration conf)

ensurePaths

public static void ensurePaths(HadoopDruidIndexerConfig config)

runJobs

public static boolean runJobs(List<Jobby> jobs,
                              HadoopDruidIndexerConfig config)

serializeOutIndex

public static DataSegment serializeOutIndex(DataSegment segmentTemplate,
                                            org.apache.hadoop.conf.Configuration configuration,
                                            org.apache.hadoop.util.Progressable progressable,
                                            File mergedBase,
                                            org.apache.hadoop.fs.Path finalIndexZipFilePath,
                                            org.apache.hadoop.fs.Path finalDescriptorPath,
                                            org.apache.hadoop.fs.Path tmpPath)
                                     throws IOException

Throws:

IOException

writeSegmentDescriptor

public static void writeSegmentDescriptor(org.apache.hadoop.fs.FileSystem outputFS,
                                          DataSegment segment,
                                          org.apache.hadoop.fs.Path descriptorPath,
                                          org.apache.hadoop.util.Progressable progressable)
                                   throws IOException

Throws:

IOException

zipAndCopyDir

public static long zipAndCopyDir(File baseDir,
                                 OutputStream baseOutputStream,
                                 org.apache.hadoop.util.Progressable progressable)
                          throws IOException

Throws:

IOException

copyFileToZipStream

public static long copyFileToZipStream(File file,
                                       ZipOutputStream zipOutputStream,
                                       org.apache.hadoop.util.Progressable progressable)
                                throws IOException

Throws:

IOException

isHdfs

public static boolean isHdfs(org.apache.hadoop.fs.FileSystem fs)

makeFileNamePath

public static org.apache.hadoop.fs.Path makeFileNamePath(org.apache.hadoop.fs.Path basePath,
                                                         org.apache.hadoop.fs.FileSystem fs,
                                                         DataSegment segmentTemplate,
                                                         String baseFileName)

makeTmpPath

public static org.apache.hadoop.fs.Path makeTmpPath(org.apache.hadoop.fs.Path basePath,
                                                    org.apache.hadoop.fs.FileSystem fs,
                                                    DataSegment segmentTemplate,
                                                    org.apache.hadoop.mapreduce.TaskAttemptID taskAttemptID)

prependFSIfNullScheme

public static org.apache.hadoop.fs.Path prependFSIfNullScheme(org.apache.hadoop.fs.FileSystem fs,
                                                              org.apache.hadoop.fs.Path path)

unzipNoGuava

public static long unzipNoGuava(org.apache.hadoop.fs.Path zip,
                                org.apache.hadoop.conf.Configuration configuration,
                                File outDir,
                                org.apache.hadoop.util.Progressable progressable)
                         throws IOException

Throws:

IOException

getURIFromSegment

public static URI getURIFromSegment(DataSegment dataSegment)

progressIndicatorForContext

public static ProgressIndicator progressIndicatorForContext(org.apache.hadoop.mapreduce.TaskAttemptContext context)

deleteWithRetry

public static boolean deleteWithRetry(org.apache.hadoop.fs.FileSystem fs,
                                      org.apache.hadoop.fs.Path path,
                                      boolean recursive)