[大数据]第四周mapreduce实战之空气质量

?AirBean.java

package org.jsoup;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.io.Writable;

public class AirBean implements Writable{
    String cityString;
    float pm25;
    float iaqi;

    public AirBean() {
    }

    public AirBean(String cityString, float pm25) {
        this.cityString = cityString;
        this.pm25 = pm25;
        float bpHi = 0,bpLo = 0,iaqiHi = 0,iaqiLo = 0;

        long[] pm25limits = {0,35,75,115,150,250,350,500};
        long[] airLimits = {0,50,100,150,200,300,400,500};

        for (int i = 0; i < 7; i++) {
            if (pm25>=pm25limits[i]&pm25<pm25limits[i+1]) {
                bpHi = pm25limits[i+1];
                bpLo = pm25limits[i];
                iaqiHi = airLimits[i+1];
                iaqiLo = airLimits[i];
                this.iaqi = ((iaqiHi-iaqiLo)/(bpHi-bpLo))*(pm25-bpLo)+iaqiLo;
            }

        }

    }

    public String getCityString() {
        return cityString;
    }

    public void setCityString(String cityString) {
        this.cityString = cityString;
    }

    public float getPm25() {
        return pm25;
    }

    public void setPm25(long pm25) {
        this.pm25 = pm25;
    }

    public float getIaqi() {
        return iaqi;
    }

    public void setIaqi(long iaqi) {
        this.iaqi = iaqi;
    }

    @Override
    public void write(DataOutput out) throws IOException {
        out.writeUTF(cityString);
        out.writeFloat(pm25);
        out.writeFloat(iaqi);
    }
    @Override
    public void readFields(DataInput in) throws IOException {
        cityString = in.readUTF();
        pm25 = in.readFloat();
        iaqi = in.readFloat();
    }
}

AirMapper.java?

package org.jsoup;

import java.io.IOException;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;


public class AirMapper extends Mapper<LongWritable, Text, Text, AirBean>{

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        if(key.toString().equals('0')){

        }else{
            String line = value.toString();
            String[] fields = line.split(",");
            String cityString = fields[16];
            float pm25 = Long.parseLong(fields[3]);
            context.write(new Text(cityString),new AirBean(cityString, pm25));
        }

    }
}

AirReducer.java

package org.jsoup;


import java.io.IOException;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;


public class AirReducer extends Reducer<Text, AirBean, Text, Text>{

    @Override
    protected void reduce(Text key, Iterable<AirBean> values, Context context) throws IOException, InterruptedException {
        float pm25Iaqi_av = 0f;
        float pm25Iaqi_sum = 0f;
        float count = 0f;
        for (AirBean value : values) {
            float iaqiLong = value.getIaqi();
            pm25Iaqi_sum = pm25Iaqi_sum + iaqiLong;
            count = count + 1;
        }
        pm25Iaqi_av = pm25Iaqi_sum/count;
        context.write(key,new Text(String.valueOf(pm25Iaqi_av)));
    }
}

AirRunner.java

package org.jsoup;

import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.log4j.BasicConfigurator;


public class AirRunner {
    public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
        BasicConfigurator.configure();
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);
        job.setJarByClass(AirRunner.class);

        job.setMapperClass(AirMapper.class);
        job.setReducerClass(AirReducer.class);

        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(AirBean.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);


        FileInputFormat.setInputPaths(job, new Path("hdfs://192.168.100.100:9000/input"));
        FileOutputFormat.setOutputPath(job,new Path("hdfs://192.168.100.100:9000/output") );

        job.waitForCompletion(true);
    }
}

GetFile.java

package org.jsoup;

import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

/**
 i am cloud
 1. @author Cloud
 */
public class GetFile {
    public static void main(String[] args) throws IOException{
        Configuration conf = new Configuration();
        //192.168.172.5这个是你的Linux中的master的ip地址，你自己换。
        conf.set("fs.defaultFS","hdfs://192.168.100.100:9000");
        FileSystem file = FileSystem.get(conf);
        file.copyFromLocalFile(new Path("/home/PM25city.txt"), new Path("/input"));
        //前面的Path是计算机的路径位置，后面是Hadoop路径位置
        file.close();
    }
}

OrderBean.java

package org.jsoup;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URI;
import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

/**
 *
 * @author Cloud
 */
public class OrderBean {
    public static void main(String[] args) throws IOException {
        String path = "hdfs://192.168.100.100:9000/output/part-r-00000" ;
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(URI.create(path), conf);
        FSDataInputStream hdfsInStream = fs.open(new Path(path));
        InputStreamReader isr = new InputStreamReader(hdfsInStream, "utf-8");
        BufferedReader br = new BufferedReader(isr);
        String line;
        //使用map保存，再排序
        HashMap<String,Float> iaqiMap = new HashMap<>();
        while ((line = br.readLine()) != null) {
            String[] val = line.split("\\s+");
            String city = val[0];
            float iaqi = Float.parseFloat(val[1]);
            iaqiMap.put(city,iaqi);
        }
        iaqiMap.entrySet()
                .stream()
                .sorted(Map.Entry.<String,Float>comparingByValue())
                .forEach(System.out::println);
    }
}