mr案例

(1)创建maven项目

(2)在po,.xml添加下面代码

<dependencies>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
        </dependency>
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
            <version>1.7.30</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>3.1.3</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>3.1.3</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>3.1.3</version>
            <scope>test</scope>
        </dependency>
    </dependencies>

(3)在项目的src/main/resources目录下，新建一个文 件，命名为“log4j.properties”，在文件中填入。

log4j.rootLogger=INFO, stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n
log4j.appender.logfile=org.apache.log4j.FileAppender
log4j.appender.logfile.File=target/spring.log
log4j.appender.logfile.layout=org.apache.log4j.PatternLayout
log4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n

编写程序

（1）编写Mapper类

package com.scy;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class WordCountMapper extends Mapper<LongWritable,Text,Text,IntWritable> {
   //实例化数据类型对象
    Text k = new Text();
    IntWritable v = new IntWritable(1);

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        //1.获取一行数据
        String line =  value.toString();
        //2.切割字符串
       String[] words =  line.split(" ");
       //3.输出
        for(String word : words){
           k.set(word);
           context.write(k,v);
        }
    }
}

（2）编写Reducer类

package com.scy;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class WordCountReducer extends Reducer<Text, IntWritable,Text,IntWritable> {
    int sum;
    IntWritable v = new IntWritable();

    @Override
    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
        //1.累加求和
        sum = 0;
        for(IntWritable count: values){
            sum += count.get();
        }
        //2.输出
        v.set(sum);
        context.write(key,v);
    }
}

（3）编写Driver驱动类

package com.scy;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

public class WordCountDriver {
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        // 1 获取配置信息以及封装任务
        Configuration configuration = new Configuration();
        Job job = Job.getInstance();
        // 2 设置jar加载路径
        job.setJarByClass(WordCountDriver.class);
        // 3 设置map和reduce类
        job.setMapperClass(WordCountMapper.class);
        job.setReducerClass(WordCountReducer.class);
        // 4 设置map输出
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);
        // 5 设置最终输出kv类型
        job.setOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);
        // 6 设置输入和输出路径
         FileInputFormat.setInputPaths(job,new Path("D:\\input\\hello.txt"));
         FileOutputFormat.setOutputPath(job,new Path("D:\\output"));
        // 7 提交
        boolean result = job.waitForCompletion(true);
        System.out.println(result?0:1);
    }
}

（5）本地测试

（1）如果电脑系统是win7的就将win7的hadoop jar包
解压到非中文路径，并在Windows环境上配置
HADOOP_HOME环境变量。如果是电脑win10操作系
统，就解压win10的hadoop jar包，并配置
HADOOP_HOME环境变量。
注意：win8电脑和win10家庭版操作系统可能有问题，
需要重新编译源码或者更改操作系统。
（2）在Eclipse/Idea上运行程序

（6）集群上测试

将路径改成下方代码的路径

        // 6 设置输入和输出路径
        FileInputFormat.setInputPaths(job,new Path(args[0]));
        FileOutputFormat.setOutputPath(job,new Path(args[1]));

（0）用maven打jar包，需要添加的打包插件依赖
注意：标记红颜色的部分需要替换为自己工程主类

<build>
<plugins>
<plugin>
<artifactId>maven-compilerplugin</artifactId>
<version>2.3.2</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assemblyplugin </artifactId>
<configuration>
<descriptorRefs>
<descriptorRef>jarwith-dependencies</descriptorRef>
</descriptorRefs>
<archive>
<manifest>
<mainClass>com.xyd.WordcountDriver</mainCla
ss>
</manifest>
</archive>
</configuration>
<executions>
<execution>
<id>makeassembly</id>
<phase>package</phase>

<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>

注意：如果工程上显示红叉。在项目上右键->maven->update project即可。
（1）将程序打成jar包，然后拷贝到Hadoop集群中
(/opt/modle/hadoop-3.1.3)
步骤详情：右键->Run as->maven install。等待编译完
成就会在项目的target文件夹中生成jar包。如果看不
到。在项目上右键-》Refresh，即可看到。修改不带依
赖的jar包名称为mr.jar，并拷贝该jar包到Hadoop集
群。
（2）启动Hadoop集群(linux)
（3）执行WordCount程序 (linux)

[admin@hadoop1002 hadoop-3.1.3]$ hadoop jar
mr.jar com.xyd.WordCountDriver /input
/output

以上就是mr案例的所有的本地测试和集群测试，请大家参考。