[转帖]Hadoop二次排序_Hadoop，ERP及大数据讨论区_Weblogic技术|Tuxedo技术|中间件技术|Oracle论坛|JAVA论坛|Linux/Unix技术|hadoop论坛

总帖数

每页帖数

1/1页

返回列表

发起投票

查看: 3260 | 回复: 0

主题： [转帖]Hadoop二次排序

jie.liang

注册用户

等级：少校
经验：1003
发帖：77
精华：0
注册：2013-10-11
状态：离线
发送短消息息给jie.liang

加好友发送短消息息给jie.liang

发消息

发表于：

2013-10-17 17:08:39 | [全部帖] [楼主帖]

楼主

Hadoop二次排序：

import java.io.IOException;

import org.apache.Hadoop.conf.Configuration;
import org.apache.Hadoop.fs.Path;
import org.apache.Hadoop.io.IntWritable;
import org.apache.Hadoop.io.LongWritable;
import org.apache.Hadoop.io.Text;
import org.apache.Hadoop.io.WritableComparable;
import org.apache.Hadoop.io.WritableComparator;
import org.apache.Hadoop.mapreduce.Job;
import org.apache.Hadoop.mapreduce.Mapper;
import org.apache.Hadoop.mapreduce.Reducer;
import org.apache.Hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.Hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.Hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.Hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.Hadoop.mapreduce.lib.partition.HashPartitioner;
/**
* @author 吕桂强
* @email larry.lv.word@gmail.com
* @version 创建时间：2012-5-21 下午5:06:57
*/
public class SecondarySort {
// map阶段的最后会对整个map的List进行分区，每个分区映射到一个reducer
public static class FirstPartitioner extends HashPartitioner<Text, IntWritable> {
@Override
public int getPartition(Text key, IntWritable value, int numPartitions) {
return (key.toString().split(":")[0].hashCode() & Integer.MAX_VALUE) % numPartitions;
}
}
// 每个分区内又调用job.setSortComparatorClass或者key的比较函数进行排序
public static class SortComparator extends WritableComparator {
protected SortComparator() {
super(Text.class, true);
}
@SuppressWarnings("rawtypes")
@Override
public int compare(WritableComparable w1, WritableComparable w2) {
return -w1.toString().split(":")[0].compareTo(w2.toString().split(":")[0]);
}
}
// 只要这个比较器比较的两个key相同，他们就属于同一个组.
// 它们的value放在一个value迭代器，而这个迭代器的key使用属于同一个组的所有key的第一个key
public static class GroupingComparator extends WritableComparator {
protected GroupingComparator() {
super(Text.class, true);
}
@SuppressWarnings("rawtypes")
@Override
public int compare(WritableComparable w1, WritableComparable w2) {
return w1.toString().split(":")[0].compareTo(w2.toString().split(":")[0]);
}
}
// 自定义map
public static class Map extends Mapper<LongWritable, Text, Text, IntWritable> {
private final IntWritable intvalue = new IntWritable();
public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
context.write(value, intvalue);
}
}
// 自定义reduce
public static class Reduce extends Reducer<Text, IntWritable, Text, IntWritable> {
public void setup(Context context) {
context.getConfiguration();
System.out.println("reduce");
}
public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
context.write(new Text("-------------------------"), new IntWritable(1));
for (IntWritable val : values) {
// 虽然分在同一个组里，但是循环里每次输出的key都不相同(key看上去是个Text但实际也是一个list)
context.write(key, val);
}
}
}
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
Configuration conf = new Configuration();
Job job = new Job(conf, "secondarysort");
job.setJarByClass(SecondarySort.class);
job.setMapperClass(Map.class);
// job.setCombinerClass(Reduce.class);
job.setReducerClass(Reduce.class);
// 分区函数
job.setPartitionerClass(FirstPartitioner.class);
job.setSortComparatorClass(SortComparator.class);
// 分组函数
job.setGroupingComparatorClass(GroupingComparator.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
FileInputFormat.setInputPaths(job, new Path("/larry/wc/input"));
FileOutputFormat.setOutputPath(job, new Path("/larry/wc/output"));
job.setNumReduceTasks(1);
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}