目录
1，项目目标分析
首先清洗数据，即去除缺省字段，说明字段和无用字段。对地图站点进行可视化，在地图上点击对应站点可以跳转到该站点的风速信息。
2，数据清洗
部分原始数据如下所示：
因此首先应该去除第一行的说明字段，去除空数据（“”），补充缺省字段（9999.9,999.9,99.99-->0）,去掉*、[A-I] 字符后拼接，然后去掉不需要的数据再重新进行拼接。
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.InputStreamReader;
import java.util.zip.GZIPInputStream;
public class SCombine {
public static void main(String[] args) {
//读取用的流
FileInputStream fis = null;
GZIPInputStream gis = null;
InputStreamReader isr = null;
BufferedReader bufr = null;
//源文件的路径
String inputPath="E:\\Hadoop-kaifa\\gsod50\\gsod_2016";
//写入相关的流
FileWriter fw = null;
BufferedWriter bufw = null;
//结果文件保存路径
String outPath = "yll.txt";
//处理文件的数量
int fileCount = 0;
//处理数据的条数
int dataCount = 0;
try {
File fileDir = new File(inputPath);
//初始化写相关的流
fw = new FileWriter(outPath);
bufw = new BufferedWriter(fw);
for (File file : fileDir.listFiles()) {
//读相关流的初始化
fis = new FileInputStream(file);
gis = new GZIPInputStream(fis);
isr = new InputStreamReader(gis);
bufr = new BufferedReader(isr);
//按行读取内容
String line = bufr.readLine();
String str ="";
while(line!=null) {
//1.去除第一行说明字段
if(line.startsWith("STN")){
line = bufr.readLine();
continue;
}
//切割字符串
String[] strs = line.split(" ");
//接收第一次清洗后的数据
String word = "";
//2.处理数据
for (int i = 0; i < strs.length; i++) {
//去掉空数据
if(strs[i].equals("")){
continue;
}
//补充缺省数据
if(strs[i].equals("9999.9")||strs[i].equals("999.9")||
strs[i].equals("99.99")){
strs[i] = "0.0";
}
//去除* [A-I]字符
if(strs[i].endsWith("*")){
strs[i] = strs[i].replace("*", "");
}
/**
* .代表任意字符
* *代表字符出现任意次
* [A-I]代表字符范围是A-I
*/
if(strs[i].matches(".*[A-I]")){
strs[i] = strs[i].substring(0, strs[i].length()-1);
}
//拼接
if(i==0) {
word += strs[i];
}else {
word += "/"+strs[i];
}
}
String[] newStrs = word.split("/");
for(int i=0;i<newStrs.length;i++) {
//去除不需要的数据，保留0,1,2,3,5,17,18
if(!(i==0 || i==1 || i==2 ||
i==3 || i==5 ||i==7||i==9||i==11||i==13||
i==15||i==16||i ==17 || i==18||
i==19||i==20||i==21)){
continue;
}
if(i!=0) {
str +="/"+newStrs[i];
}else {
str += newStrs[i];
}
}
//写操作
bufw.write(str);
bufw.newLine();
dataCount++;
str = "";
line = bufr.readLine();
}
if(bufr !=null) {
bufr.close();
}
if(isr !=null) {
isr.close();
}
if(gis !=null) {
gis.close();
}
if(fis !=null) {
fis.close();
}
//一个文件处理完毕
fileCount++;
if(fileCount % 50 ==0) {
System.out.println();
}else {
System.out.print(".");
}
}
bufw.flush();
System.out.println("一共处理了："+fileCount+"个文件，和"+dataCount+"条数据");
} catch (Exception e) {
e.printStackTrace();
}finally {
if(bufw !=null) {
try {
bufw.close();
} catch (Exception e) {
e.printStackTrace();
}
}
if(fw !=null) {
try {
fw.close();
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
}
2，存入数据库
在数据库中建表
CREATE TABLE sjbg (
id int(11) NOT NULL AUTO_INCREMENT,
STN varchar(8) DEFAULT NULL,
WBAN varchar(8) DEFAULT NULL,
YEARMODA date DEFAULT NULL,
TEMP double DEFAULT NULL,
DEWP double DEFAULT NULL,
SLP double DEFAULT NULL,
STP double DEFAULT NULL,
VISIB double DEFAULT NULL,
WDSP double DEFAULT NULL,
MXSPD double DEFAULT NULL,
GUST double DEFAULT NULL,
MAX double DEFAULT NULL,
MIN double DEFAULT NULL,
PRCP double DEFAULT NULL,
SNDP double DEFAULT NULL,
FRSHTT varchar(8) DEFAULT NULL,
PRIMARY KEY (id)
) ENGINE=InnoDB AUTO_INCREMENT=4292878 DEFAULT CHARSET=utf新建一个类用于将数据导入到表中。?
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.sql.Connection;
import java.sql.Date;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.text.SimpleDateFormat;
public class SCtoMysql {
static Connection conn=null;
static {
try {
Class.forName("com.mysql.jdbc.Driver");
String url="jdbc:mysql://localhost:53306/gsod1901"+
"?rewriteBatchStatements=true";
String user="root";
String password="root";
//创建连接对象
conn=DriverManager.getConnection(url,user,password);
} catch (Exception e) {
e.printStackTrace();
}
}
public static void main(String[] args) {
PreparedStatement ps=null;
Reader in=null;
BufferedReader bufr=null;
try {
in = new FileReader(new File("yll.txt")) ;
bufr = new BufferedReader(in);
String line=bufr.readLine();
//sql
String sql="insert into sjbg values(null,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
ps=conn.prepareStatement(sql);
SimpleDateFormat sdf=new SimpleDateFormat("yyyyMMdd");
//计数
int count=0;
//计时
long starttime=System.currentTimeMillis();
//设置事务提交为手动
conn.setAutoCommit(false);
while(line!=null) {
String[] strs=line.split("/");
ps.setString(1, strs[0]);
ps.setString(2, strs[1]);
ps.setDate(3, new Date(sdf.parse(strs[2]).getTime()));
ps.setDouble(4, Double.parseDouble(strs[3]));
ps.setDouble(5, Double.parseDouble(strs[4]));
ps.setDouble(6, Double.parseDouble(strs[5]));
ps.setDouble(7, Double.parseDouble(strs[6]));
ps.setDouble(8, Double.parseDouble(strs[7]));
ps.setDouble(9, Double.parseDouble(strs[8]));
ps.setDouble(10, Double.parseDouble(strs[9]));
ps.setDouble(11, Double.parseDouble(strs[10]));
ps.setDouble(12, Double.parseDouble(strs[11]));
ps.setDouble(13, Double.parseDouble(strs[12]));
ps.setDouble(14, Double.parseDouble(strs[13]));
ps.setDouble(15, Double.parseDouble(strs[14]));
ps.setString(16, strs[15]);
count++;
//批量操作
ps.addBatch();
//每隔50000条数据执行一次插入
if(count%50000==0) {
ps.executeBatch();
System.out.print("插入了："+count+"条数据");
}
line=bufr.readLine();
}
ps.executeBatch();
//手动提交
conn.commit();
long endTime =System.currentTimeMillis();
System.out.print("插入了："+count+"条数据，用时："+(endTime-starttime));
} catch (Exception e) {
e.printStackTrace();
}finally {
if(ps!=null) {
try {
ps.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
if(bufr!=null) {
try {
bufr.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if(in!=null) {
try {
in.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
}
3，地图站点显示开发
3.1，pojo类
public class Station {
//stn-wban
private String name;
//[经度lon，纬度lat]
private String[] value;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String[] getValue() {
return value;
}
public void setValue(String[] value) {
this.value = value;
}
}
3.2，dao层
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.ArrayList;
import java.util.List;
import pojo.Station;
import utils.MySqlUtils;
//Dao层，和数据库进行交互，用来从数据库中查询站点信息并进行封装
public class StationDao {
//从自定义工具类中获取连接对象
private Connection conn=MySqlUtils.getConn();
public List<Station> getStation() {
//保存站点信息的集合
List<Station> list =new ArrayList<Station>();
//sql
String sql = "select * from station_info";
PreparedStatement ps=null;
ResultSet rs=null;
try {
ps=conn.prepareStatement(sql);
rs = ps.executeQuery();
//遍历
while(rs.next()) {
Station s=new Station();
s.setName(rs.getString("stn")+"-"+rs.getString("wban"));
String[] value = {rs.getDouble("longitude")+"",
rs.getDouble("latitude")+""};
s.setValue(value);
list.add(s);
}
} catch (Exception e) {
e.printStackTrace();
}finally {
MySqlUtils.close(rs,ps);
}
return list;
}
}
?3.3，service层
import java.util.List;
import dao.StationDao;
import pojo.Station;
public class StationService {
private static StationDao dao = new StationDao();
//查询站点信息
public List<Station> getStation(){
return dao.getStation();
}
}
3.4，controller层
import java.io.IOException;
import java.util.List;
import javax.servlet.ServletException;
import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.alibaba.fastjson.JSON;
import pojo.Station;
import service.StationService;
@WebServlet("/StationController")
public class StationController extends HttpServlet {
private static final long SerialVersionUID =1L;
private static StationService service = new StationService();
@Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
List<Station> list = service.getStation();
//list转json字符串
String jsonStr=JSON.toJSONString(list);
resp.getWriter().write(jsonStr);
}
@Override
protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
doGet(req, resp);
}
}
?3.5，html页面
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<title>全球气象数据</title>
<script type="text/javascript" src="js/echarts.js"></script>
<script type="text/javascript" src="js/jquery-1.11.0.min.js"></script>
</head>
<body>
<div id="main" style="width:1200px;height:600px"></div>
<script type="text/javascript">
//初始化Echarts内置对象
var myEcharts = echarts.init(document.getElementById('main'));
//定义world.json的路径
var mapInforUrl = "data/world.json";
//定义访问后台的路径
var stationInforUrl = "StationController";
//通过JQuery的get方法获取world.json的数据
$.get(mapInforUrl,function(data){
echarts.registerMap('world',data);
//指定配置项
var option = {
//添加提示框组件
tooltip:{
trigger:'item',
formatter:'{b}'
},
//添加geo组件
geo:[{
name:'世界地图',
type:'map',
map:'world',
roam:true, //是否开启鼠标缩放和平移漫游。默认不开启
selectMode:false,//选中模式，表示是否支持多个选中，默认关闭
label:{
normal:{
show:false
},
emphasis:{
label:{
show:true
}
}
},
//样式
itemStyle:{
//普通
normal:{
areaColor:'#66CCCC',
//边框
borderColor:'white'
},
//高亮
emphasis:{
areaColor:"#CCFF33"
}
}
}],
};
//利用指定配置项显示图表
myEcharts.setOption(option);
//发送ajax请求，请求后台返回所有站点数据
$.get(stationInforUrl,function(data){
//将json字符串转换成json对象格式
data = JSON.parse(data);
myEcharts.setOption(option={
series:[{
name:'站点',
type:'scatter',
symbolSize:'3',
coordinateSystem:'geo',
//使用地理坐标系，通过 geoIndex 指定相应的地理坐标系组件。
itemStyle:{
color:'#66CC66'
},
data:data
}]
});
});
//添加点击事件
myEcharts.on('click',function(params){
//获取点击的站点的参数名称，按-分割
var str = params.name.split("-");
//如果数组长度为2，点击的是站点
if(str.length >=2){
window.location.href = "Spd.html?stn="
+str[0]+"&wban="+str[1];
}else{
alert("点击的不是站点，请重新点击")
}
});
});
</script>
</body>
</html>4，风速显示功能开发
4.1，pojo层
import java.sql.Date;
//对应数据库的
public class yll {
private String stn;
private String wban;
//日期
private Date yearmoda;
//温度
private double temp;
private double dewp;
private double slp;
private double stp;
private double visib;
private double wdsp;
private double mxspd;
private double gust;
private double max;
private double min;
private double prcp;
private double sndp;
private String frshtt;
public String getStn() {
return stn;
}
public void setStn(String stn) {
this.stn = stn;
}
public String getWban() {
return wban;
}
public void setWban(String wban) {
this.wban = wban;
}
public Date getYearmoda() {
return yearmoda;
}
public void setYearmoda(Date yearmoda) {
this.yearmoda = yearmoda;
}
public double getTemp() {
return temp;
}
public void setTemp(double temp) {
this.temp = temp;
}
public double getDewp() {
return dewp;
}
public void setDewp(double dewp) {
this.dewp = dewp;
}
public double getSlp() {
return slp;
}
public void setSlp(double slp) {
this.slp = slp;
}
public double getStp() {
return stp;
}
public void setStp(double stp) {
this.stp = stp;
}
public double getVisib() {
return visib;
}
public void setVisib(double visib) {
this.visib = visib;
}
public double getWdsp() {
return wdsp;
}
public void setWdsp(double wdsp) {
this.wdsp = wdsp;
}
public double getMxspd() {
return mxspd;
}
public void setMxspd(double mxspd) {
this.mxspd = mxspd;
}
public double getGust() {
return gust;
}
public void setGust(double gust) {
this.gust = gust;
}
public double getMax() {
return max;
}
public void setMax(double max) {
this.max = max;
}
public double getMin() {
return min;
}
public void setMin(double min) {
this.min = min;
}
public double getPrcp() {
return prcp;
}
public void setPrcp(double prcp) {
this.prcp = prcp;
}
public double getSndp() {
return sndp;
}
public void setSndp(double sndp) {
this.sndp = sndp;
}
public String getFrshtt() {
return frshtt;
}
public void setFrshtt(String frshtt) {
this.frshtt = frshtt;
}
@Override
public String toString() {
return "Weather [stn=" + stn + ", wban=" + wban + ", yearmoda=" + yearmoda + ", temp=" + temp + ", dewp=" + dewp
+ ", slp=" + slp + ", stp=" + stp + ", visib=" + visib + ", wdsp=" + wdsp + ", mxspd=" + mxspd
+ ", gust=" + gust + ", max=" + max + ", min=" + min + ", prcp=" + prcp + ", sndp=" + sndp + ", frshtt="
+ frshtt + "]";
}
}
public class yllSpd {
private String stn;
private String wban;
private String yearmoda;//年月日
private double visib;//平均可见度
private double wdsp;//平均风速
private double mxspd;//持续最大风速
private double gust;//瞬间最大风速
public String getStn() {
return stn;
}
public void setStn(String stn) {
this.stn = stn;
}
public String getWban() {
return wban;
}
public void setWban(String wban) {
this.wban = wban;
}
public String getYearmoda() {
return yearmoda;
}
public void setYearmoda(String yearmoda) {
this.yearmoda = yearmoda;
}
public double getVisib() {
return visib;
}
public void setVisib(double visib) {
this.visib = visib;
}
public double getWdsp() {
return wdsp;
}
public void setWdsp(double wdsp) {
this.wdsp = wdsp;
}
public double getMxspd() {
return mxspd;
}
public void setMxspd(double mxspd) {
this.mxspd = mxspd;
}
public double getGust() {
return gust;
}
public void setGust(double gust) {
this.gust = gust;
}
@Override
public String toString() {
return "yllSpd [stn=" + stn + ", wban=" + wban + ", yearmoda=" + yearmoda + ", visib=" + visib + ", wdsp="
+ wdsp + ", mxspd=" + mxspd + ", gust=" + gust + "]";
}
}
4.2，dao层
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.ArrayList;
import java.util.List;
import pojo.yll;
import utils.MySqlUtils;
public class SpdDao {
//获取连接对象
private Connection conn=MySqlUtils.getConn();
public List<yll> getSpd(String stn,String wban) {
List<yll> list = new ArrayList<yll>();
String sql = "select * from sjbg where stn=? and wban=?";
PreparedStatement ps=null;
ResultSet rs=null;
try {
ps=conn.prepareStatement(sql);
ps.setString(1, stn);
ps.setString(2, wban);
rs = ps.executeQuery();
while(rs.next()) {
yll y = new yll();
y.setStn(rs.getString(2));
y.setWban(rs.getString(3));
y.setYearmoda(rs.getDate(4));
y.setTemp(rs.getDouble(5));
y.setDewp(rs.getDouble(6));
y.setSlp(rs.getDouble(7));
y.setStp(rs.getDouble(8));
y.setVisib(rs.getDouble(9));
y.setWdsp(rs.getDouble(10));
y.setMxspd(rs.getDouble(11));
y.setGust(rs.getDouble(12));
y.setMax(rs.getDouble(13));
y.setMin(rs.getDouble(14));
y.setPrcp(rs.getDouble(15));
y.setSndp(rs.getDouble(16));
y.setFrshtt(rs.getString(17));
list.add(y);
}
} catch (Exception e) {
e.printStackTrace();
} finally {
MySqlUtils.close(rs,ps);
}
return list;
}
}
4.3，service层
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import dao.SpdDao;
import pojo.yllSpd;
import pojo.yll;
public class SpdService {
private static SpdDao dao =new SpdDao();
private static Map<String,List<yll>> ylls = new HashMap<>();
//通过stn，wban查询天气数据，并且把对应的数据添加到缓存中
private void cache(String stn,String wban) {
String stationFlag = stn+wban;
if(!ylls.containsKey(stationFlag)) {
List<yll> list = dao.getSpd(stn, wban);
ylls.put(stationFlag, list);
}
}
private List<yll> getSpd(String stn,String wban){
cache(stn,wban);
return ylls.get(stn+wban);
}
public List<yllSpd> getYll(String stn,String wban){
List<yll> list = getSpd(stn, wban);
List<yllSpd> spdList = new ArrayList<yllSpd>();
//封装
for (yll y : list) {
yllSpd ys = new yllSpd();
ys.setStn(y.getStn());
ys.setWban(y.getWban());
ys.setYearmoda(y.getYearmoda().toString());
ys.setVisib(y.getVisib());
ys.setWdsp(y.getWdsp());
ys.setMxspd(y.getMxspd());
ys.setGust(y.getGust());
spdList.add(ys);
}
return spdList;
}
}
4.4，controller层
import java.io.IOException;
import java.util.List;
import javax.servlet.ServletException;
import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.alibaba.fastjson.JSON;
import pojo.yllSpd;
import service.SpdService;
@WebServlet("/SpdServlet")
public class SpdServlet extends HttpServlet{
private static final long SerialVersionUID =1L;
private static SpdService service = new SpdService();
@Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
String stn = req.getParameter("stn");
String wban = req.getParameter("wban");
System.err.println(stn+"======"+wban);
List<yllSpd> list = service.getYll(stn, wban);
//将数据返回给前台
String jsonStr = JSON.toJSONString(list);
resp.getWriter().write(jsonStr);
}
@Override
protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
doGet(req, resp);
}
}
4.5，html页面
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<meta charset="UTF-8">
<title>气象站点风速信息</title>
<script type="text/javascript" src="js/echarts.js"></script>
<script type="text/javascript" src="js/jquery-1.11.0.min.js"></script>
</head>
<body>
<div id="spd" style="width:1200px;height:600px"></div>
<script type="text/javascript">
var myEcharts = echarts.init(document.getElementById('spd'));
var args = location.search;
var url = "SpdServlet"+args;
$.get(url,function(data){
data = JSON.parse(data);
alert(data);
var option = {
title:{
text:'气象站点风速信息折线图'
},
tooltip:{
trigger:'axis'
},
legend:{
data:['平均可见度','平均风速','持续最大风速','瞬间最大风速']
},
xAxis:{
type:'category',
boundaryGap:false,
data:data.map(function(item){
return item.yearmoda
})
},
yAxis:{
type:'value'
},
series:[{
name:'平均可见度',
type:'line',
data:data.map(function(item){
return item.visib
})
},
{
name:'平均风速',
type:'line',
data:data.map(function(item){
return item.wdsp
})
},
{
name:'持续最大风速',
type:'line',
data:data.map(function(item){
return item.mxspd
})
},{
name:'瞬间最大风速',
type:'line',
color:"lightblue",//设置颜色
data:data.map(function(item){
return item.gust
})
}]
};
myEcharts.setOption(option);
});
</script>
</body>
</html>5，结果展示
5.1，全球站点信息的显示
5.2，当点击的点为非站点时的显示?
?5.3，当点击的点为非空站点时的显示
