java outputformat,Java TextOutputFormat.setCompressOutput方法代码示例

import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; //导入方法依赖的package包/类

/**

* @param inputs

* the input paths to be expected

* @param output

* the output location

* @param conf

* the job configuration

* @return the job to be launched in this stage

* @throws Exception

* @throws IOException

*/

public Job stage(Path[] inputs, Path output, Configuration conf) throws Exception {

final Job job = new Job(conf);

// A bit of a dirty hack, if any input file is an lzo file sneakily

// switch the input format class to LZOTextInput

if (inputFormatClass.equals(TextInputFormat.class) && containsLZO(inputs)) {

job.setInputFormatClass((Class extends InputFormat, ?>>) Class

.forName("com.hadoop.mapreduce.LzoTextInputFormat"));

}

else {

job.setInputFormatClass(inputFormatClass);

}

job.setMapOutputKeyClass(mapOutputKeyClass);

job.setMapOutputValueClass(mapOutputValueClass);

job.setOutputKeyClass(outputKeyClass);

job.setOutputValueClass(outputValueClass);

job.setOutputFormatClass(outputFormatClass);

if (outputFormatClass.equals(TextOutputFormat.class) && this.lzoCompress()) {

TextOutputFormat.setCompressOutput(job, true);

TextOutputFormat.setOutputCompressorClass(job,

(Class extends CompressionCodec>) Class.forName("com.hadoop.compression.lzo.LzopCodec"));

} else {

TextOutputFormat.setCompressOutput(job, false);

}

job.setJarByClass(this.getClass());

setInputPaths(job, inputs);

setOutputPath(job, output);

setMapperClass(job, mapper());

setReducerClass(job, reducer());

setCombinerClass(job, combiner());

setup(job);

return job;

}