Hadoop 自定义计数器

huijial

　　


public static class mapper extends Mapper<Text, BytesWritable, Text , Text>{
private Counter c ;
@Override
protected void setup(Context context) throws IOException,
InterruptedException {
c = context.getCounter("FILE", "COUNT");
}
@Override
protected void map(Text key, BytesWritable value, Context context)
throws IOException, InterruptedException {
c.increment(1);
context.write(key, new Text(value.getBytes()));
}
@Override
protected void cleanup(Context context)
throws IOException, InterruptedException {
}
}
public static class reducer extends Reducer<Text, Text, Text, Text>{
@Override
protected void reduce(Text arg0, Iterable<Text> arg1,
Context context)
throws IOException, InterruptedException {
Iterator<Text> itr = arg1.iterator();
while(itr.hasNext()){
itr.next();
}
context.write(arg0, new Text("heihei"));
}
}
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException{
Configuration conf = new Configuration();
Job job = new Job(conf);
job.setJarByClass(testCounter.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
job.setMapperClass(mapper.class);
job.setReducerClass(reducer.class);
job.setInputFormatClass(WholeFileInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.waitForCompletion(true);
String num = job.getCounters().findCounter("Map-Reduce Framework", "Map input records").getName();
System.out.println(num);
}
　　获取计数器只能在job完成之后，也就是job.waitForCompletion(true);之后，放在之前的话回报一个非法安全的错误，但是在reducer函数里面，则不错报错，也不能获取计数器值，可能设计人员没有扑捉reducer里面的异常吧...
　　计数器有自定义计数器和内置计数器
　　静态计数器和动态计数器，前者用枚举，更安全一些，后者直接用字符串，有时为了显示方便，会创建一个properties属性文件。