hadoop权威指南--气温最大值所遇到的--内部类为静态的问题

winson

　　老的API：

public class MaxTemperature {
public static  class  MaxTemperatureMapper extends MapReduceBase implements Mapper<LongWritable, Text, Text, IntWritable> {
private  static final  int MISSING = 9999;
@Override
public void map(LongWritable key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter)
throws IOException {
String line = value.toString();
String year = line.substring(15, 19);
int airTemperature;
if(line.charAt(87) == '+') {
airTemperature = Integer.parseInt(line.substring(88,92));
} else {
airTemperature = Integer.parseInt(line.substring(87, 92));
}
String  quality = line.substring(92, 93);
if(airTemperature != MISSING && quality.matches("[01459]")) {
output.collect(new Text(year), new IntWritable(airTemperature));
}
}
}
public static class MaxTemperatureReducer  extends  MapReduceBase implements  Reducer<Text,IntWritable, Text, IntWritable>{
@Override
public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output, Reporter reporter)
throws IOException {
int maxvalue = Integer.MIN_VALUE;
while (values.hasNext()) {
maxvalue = Math.max(maxvalue, values.next().get());
}
output.collect(key, new IntWritable(maxvalue));
}
}
public static void main(String[] args) throws IOException {
if (args.length != 2) {
System.err.println("Usage: MaxTemperature <input path> <output path>");
}
JobConf  conf  = new JobConf(MaxTemperature.class);
conf.setJobName("Max temperature");
FileInputFormat.addInputPath(conf, new Path(args[0]));
FileOutputFormat.setOutputPath(conf, new Path(args[1]));
conf.setMapperClass(MaxTemperatureMapper.class);
conf.setReducerClass(MaxTemperatureReducer.class);
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(IntWritable.class);
JobClient.runJob(conf);
}
}
　　新的api：

public class NewMaxTemperature {
public static  class NewMaxTemperatureMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
private static final int MISSING = 9999;
public void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, IntWritable>.Context context)
throws IOException, InterruptedException {
String line = value.toString();
String year = line.substring(15, 19);
int airTemperature;
if (line.charAt(87) == '+') {
airTemperature = Integer.parseInt(line.substring(88, 92));
} else {
airTemperature = Integer.parseInt(line.substring(87, 92));
}
String quality = line.substring(92, 93);
if (airTemperature != MISSING && quality.matches("[01459]")) {
context.write(new Text(year), new IntWritable(airTemperature));
}
}
}
public static class NewMaxTemperatureReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
@Override
public void reduce(Text key, Iterable<IntWritable> values,
Reducer<Text, IntWritable, Text, IntWritable>.Context context) throws IOException, InterruptedException {
int maxvalue = Integer.MIN_VALUE;
while (values.iterator().hasNext()) {
maxvalue = Math.max(maxvalue, values.iterator().next().get());
}
context.write(key, new IntWritable(maxvalue));
}
}
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
if (args.length != 2) {
System.err.println("Usage: MaxTemperature <input path> <output path>");
}
Configuration conf = new Configuration();
String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
HashMap<String, String> params = ParamsUtil.getJobParameters(otherArgs);
Job job = Job.getInstance(new Cluster(conf), conf);
job.setJarByClass(NewMaxTemperature.class);
job.setMapperClass(NewMaxTemperatureMapper.class);
job.setReducerClass(NewMaxTemperatureReducer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
String reduceTasksNum = params.get("r");
if (reduceTasksNum == null) {
System.out.println("No reduce tasks num");
System.exit(-1);
}
String input = params.get("input");
if (input == null) {
System.out.println("No input");
System.exit(-1);
}
String output = params.get("output");
if (output == null) {
System.out.println("No output");
System.exit(-1);
}
String name = params.get("name");
if (name == null || "".equals(name)) {
name = "MaxTemperature Test";
}
job.setNumReduceTasks(Integer.valueOf(reduceTasksNum));
FileInputFormat.setInputPaths(job, input);
FileOutputFormat.setOutputPath(job, new Path(output));
job.setJobName(name);
job.waitForCompletion(true);
}
}

　　在运行新的程序时，刚开始内部类没有用静态修饰，运行时报错：

WARN org.apache.hadoop.mapred.Child: Exception running child : java.lang.RuntimeException: java.lang.NoSuchMethodException: com.renren.dp.mapred.NewMaxTemperature$NewMaxTemperatureMapper.<init>()
at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:123)
at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:612)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:328)
at org.apache.hadoop.mapred.Child$4.run(Child.java:217)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:742)
at org.apache.hadoop.mapred.Child.main(Child.java:211)
Caused by: java.lang.NoSuchMethodException: com.renren.dp.mapred.NewMaxTemperature$NewMaxTemperatureMapper.<init>()
at java.lang.Class.getConstructor0(Class.java:2706)
at java.lang.Class.getDeclaredConstructor(Class.java:1985)
at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:117)
... 7 more
原因是：hadoop在调用map和reduce类时采用的反射调用，内部类不是静态的，没有获取到内部类的实例，因为必须要获取外部类的实例，new NewMaxTemperature（）.new NewMaxTemperatureMapper();如果是静态的内部类的话：直接new NewMaxTemperatureMapper() 得出实例。
在新api中封装了参数解析类：
public class ParamsUtil {
public static HashMap<String, String> getJobParameters(String[] args) {
Options options = new Options();
for (String arg : args) {
if (arg.startsWith("-")) {
Option option = OptionBuilder.hasArg(true).create(
arg.substring(1));
option.setArgName(arg.substring(1));
options.addOption(option);
}
}
PosixParser posixerParser = new PosixParser();
CommandLine cmd = null;
try {
cmd = posixerParser.parse(options, args);
HashMap<String, String> params = new HashMap<String, String>();
for (Option op : cmd.getOptions()) {
params.put(op.getArgName(), op.getValue());
System.out.println("Params:" + op.getArgName() + "="
+ op.getValue());
}
return params;
} catch (org.apache.commons.cli.ParseException e) {
e.printStackTrace();
System.exit(0);
}
return null;
}