hadoopバージョン互換性の問題

5253 ワード
hadoop
現在、hadoopを学ぶ書籍の資料はまだ多くありません.hadoop権威ガイドを学ぶとき、コードの例を書くのにエラーが発生しました.コードは最高気温を探すためです.
 public class MaxTemperature { 

  public static void main(String[] args) throws IOException {
    if (args.length != 2) {
      System.err.println("Usage: MaxTemperature <input path> <output path>");
      System.exit(-1);
    }
    
    JobConf conf = new JobConf(MaxTemperature.class);
    conf.setJobName("Max temperature");

    FileInputFormat.addInputPath(conf, new Path(args[0]));
    FileOutputFormat.setOutputPath(conf, new Path(args[1]));
    
    conf.setMapperClass(MaxTemperatureMapper.class);
    conf.setReducerClass(MaxTemperatureReducer.class);

    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(IntWritable.class);

    JobClient.runJob(conf);
  }
}

 public class MaxTemperatureMapper extends MapReduceBase 
  implements Mapper<LongWritable, Text, Text, IntWritable> {

  private static final int MISSING = 9999;
  
  public void map(LongWritable key, Text value,
      OutputCollector<Text, IntWritable> output, Reporter reporter)
      throws IOException {
    
    String line = value.toString();
    String year = line.substring(15, 19);
    int airTemperature;
    if (line.charAt(87) == '+') { // parseInt doesn't like leading plus signs
      airTemperature = Integer.parseInt(line.substring(88, 92));
    } else {
      airTemperature = Integer.parseInt(line.substring(87, 92));
    }
    String quality = line.substring(92, 93);
    if (airTemperature != MISSING && quality.matches("[01459]")) {
      output.collect(new Text(year), new IntWritable(airTemperature));
    }
  }
}

 public class MaxTemperatureReducer extends MapReduceBase 
  implements Reducer<Text, IntWritable, Text, IntWritable> {

  public void reduce(Text key, Iterator<IntWritable> values,
      OutputCollector<Text, IntWritable> output, Reporter reporter)
      throws IOException {
    
    int maxValue = Integer.MIN_VALUE;
    while (values.hasNext()) {
      maxValue = Math.max(maxValue, values.next().get());
    }
    output.collect(key, new IntWritable(maxValue));
  }
}

 
私がインストールしたhadoopは0.20.2バージョンで、このコードを実行すると「Usage:MaxTemperature」と表示され、最初から間違っていたことを示しています.バージョンが一致していないため、コードが互換性がないため、新しい方法でコードを作成します.
public class NewMaxTemperature {
  
  static class NewMaxTemperatureMapper
    /*[*/extends Mapper<LongWritable, Text, Text, IntWritable>/*]*/ {

    private static final int MISSING = 9999;
    
    public void map(LongWritable key, Text value, /*[*/Context context/*]*/)
        throws IOException, /*[*/InterruptedException/*]*/ {
      
      String line = value.toString();
      String year = line.substring(15, 19);
      int airTemperature;
      if (line.charAt(87) == '+') { // parseInt doesn't like leading plus signs
        airTemperature = Integer.parseInt(line.substring(88, 92));
      } else {
        airTemperature = Integer.parseInt(line.substring(87, 92));
      }
      String quality = line.substring(92, 93);
      if (airTemperature != MISSING && quality.matches("[01459]")) {
        /*[*/context.write/*]*/(new Text(year), new IntWritable(airTemperature));
      }
    }
  }
  
  static class NewMaxTemperatureReducer
    /*[*/extends Reducer<Text, IntWritable, Text, IntWritable>/*]*/ {
  
    public void reduce(Text key, /*[*/Iterable/*]*/<IntWritable> values,
        /*[*/Context context/*]*/)
        throws IOException, /*[*/InterruptedException/*]*/ {
      
      int maxValue = Integer.MIN_VALUE;
      for (IntWritable value : values) {
        maxValue = Math.max(maxValue, value.get());
      }
      /*[*/context.write/*]*/(key, new IntWritable(maxValue));
    }
  }

  public static void main(String[] args) throws Exception {
    if (args.length != 2) {
      System.err.println("Usage: NewMaxTemperature <input path> <output path>");
      System.exit(-1);
    }
    
    /*[*/Job job = new Job();
    job.setJarByClass(NewMaxTemperature.class);/*]*/

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    
    job.setMapperClass(NewMaxTemperatureMapper.class);
    job.setReducerClass(NewMaxTemperatureReducer.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    
    /*[*/System.exit(job.waitForCompletion(true) ? 0 : 1);/*]*/
  }
}
は正常に実行できます.
[HP]反応式Web復習
IntelliJ IDEAコメントテンプレートの設定(Mac)