# -*- coding: utf-8 -*-
from collections import defaultdict
def table_to_list(table):
dct = table_to_2d_dict(table)
return list(iter_2d_dict(dct))
def table_to_2d_dict(table):
result = defaultdict(lambda : defaultdict(str))
for row_i, row in enumerate(table.xpath('./tr')):
for col_i, col in enumerate(row.xpath('./td|./th')):
colspan = int(col.get('colspan', 1))
rowspan = int(col.get('rowspan', 1))
col_data = "".join(col.xpath('.//text()')).replace(" ", "").replace("\n", "")
while row_i in result and col_i in result[row_i]:
col_i += 1
for i in range(row_i, row_i + rowspan):
for j in range(col_i, col_i + colspan):
result[i][j] = col_data
return result
def iter_2d_dict(dct):
for i, row in sorted(dct.items()):
cols = []
for j, col in sorted(row.items()):
cols.append(col)
print(len(cols), cols)
yield cols
if __name__ == '__main__':
test_html = """
<div id="head"></div><!-- head finish --><!-- 中间部分 -->
"""
from lxml import etree
import pandas as pd
doc = etree.HTML(test_html.replace("<tbody>", '').replace("</tbody>", '').replace("\xa0", ''))
for table_el in doc.xpath('//table'):
table = table_to_list(table_el)
print(table)
df = pd.DataFrame(table)
for i in range(df.shape[1]):
values = [_[0] for _ in df.iloc[:, i:i+1].values.tolist() if _ != ['']]
if len(values) > 1 and (values[0] == '业主单位' or values[0] == '建设单位'):
print("miss list ", values[1:])
|
