2022-06-23 14:13:10
浏览数 (1)
代码语言：javascript
复制import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
public class Test {
public static class TokenizerMapper
extends Mapper<Object, Text, Text, Text>{
private Text record = new Text();
private Text time = new Text();
public void map(Object key, Text value, Context context
) throws IOException, InterruptedException {
StringTokenizer itr = new StringTokenizer(value.toString(), "n");
while (itr.hasMoreTokens()) {
String[] strs = itr.nextToken().split("\s ");
if(strs.length == 3) {
record.set(strs[0] " " strs[1]);
time.set(strs[2]);
context.write(record, time);
}
}
}
}
public static class AverageReducer
extends Reducer<Text, Text, Text, Text> {
private Text result = new Text();
private Text record = new Text();
public void reduce(Text key, Iterable<Text> values,
Context context
) throws IOException, InterruptedException {
int count = 0;
float aveScore = 0;
for (Text val : values) {
aveScore = Float.parseFloat(val.toString());
count ;
}
aveScore = aveScore / count;
record.set(key " " String.valueOf(count));
result.set(String .format("%.3f",aveScore));
context.write(record, result);
System.out.println(record.toString() " " String .format("%.3f",aveScore));
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
if(otherArgs.length < 2) {
System.err.println("Usage: wordcount <in> [<in>...] <out>");
System.exit(2);
}
Job job = new Job(conf, new Test().getClass().getName());
job.setJarByClass(Test.class);
job.setMapperClass(TokenizerMapper.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setReducerClass(AverageReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
for (int i = 0; i < otherArgs.length - 1; i) {
FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
}
FileOutputFormat.setOutputPath(job,
new Path(otherArgs[otherArgs.length - 1]));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
