[Python知识库]全国城市数据及地图数据-python爬取

最近又在研究python，于是对全国的城市地图数据下手了，废话不多说，下面直接贴代码：
注释都有应该能看懂

import requests
import json
import os

# 所有地域的请求地址
ALL_AREA_URL = "http://geo.datav.aliyun.com/areas_v3/bound/all.json"
# 包含子域的json请求地址
ARREA_BOUND_URL = "http://geo.datav.aliyun.com/areas_v3/bound/geojson"
# 数据存储位置
FOLDER_PATH = "D:\\areaBound\\";
headers = {'content-type': 'charset=utf-8'}

allLen_g = 0
pullSize_g = 0;
fail_g = []


def getAllArea():
    """
    获取全国的所有地域
    :return: 返回元组
    """
    res = requests.get(url=ALL_AREA_URL, headers=headers)
    allArea = json.loads(res.text)
    global allLen_g
    allLen_g = len(allArea)
    root = None
    codemap = {}
    for area in allArea:
        code = area["adcode"]
        name = area['name']
        parent = area["parent"]
        current = {'code': code, 'name': name, "children": []}
        codemap[code] = current
        if parent is not None:
            codemap[parent]['children'].append(current)
        else:
            root = current
    return root


def getBoundData(areacode, full=False):
    if full:
        params = {"code": str(areacode) + "_full"}
    else:
        params = {"code": str(areacode)}
    res = requests.get(url=ARREA_BOUND_URL, params=params)
    return res.text


def load2file(area, folder, fail):
    code = area['code']
    name = area['name']
    if not os.path.exists(folder):
        os.makedirs(folder)

    has_fail = False
    global pullSize_g
    pullSize_g += 1
    # 无子区域
    try:
        filepath = folder + name + ".json"
        if not os.path.exists(filepath):
            res = getBoundData(code)
            with open(folder + name + ".json", "w") as f:
                f.write(res)
    except BaseException as e:
        print("pull [%s] fail!!!!" % name)
        print(e)
        has_fail = True
        # 包含子区域
    try:
        filepath = folder + name + "_full.json"
        if not os.path.exists(filepath):
            res = getBoundData(code, True)
            with open(folder + name + "_full.json", 'w') as f:
                f.write(res)
    except BaseException as e:
        print("pull [%s]_full fail!!!!" % name)
        print(e)
        has_fail = True

    if has_fail:
        fail.append(area)
    print("pull [%s] success~~~ >>> %.2f" % (name, pullSize_g / allLen_g))
    # 递归加载子区域
    folder = folder + area['name'] + "\\"
    for child in area['children']:
        load2file(child, folder, fail)


# 加载所有的地区
root = getAllArea()
folder = "D:\\qgdt\\"
if not os.path.exists(folder):
    os.makedirs(folder)
with open(folder+"qyhf.json",'w') as f:
    f.write(json.dumps(root,ensure_ascii=False))
load2file(root, "D:\\qgdt\\", fail_g)

附带爬取结果：
链接：https://pan.baidu.com/s/1taQPcS2x2rWF2H1yklVd-w
提取码：mlyu