python - 如何解析包含 HTML 的 XML

Question

我正在尝试用 Python 解析 Adium 的 XML 格式。我希望建立一个干净的聊天数据库，但清除所有格式和超链接。

我现在正在使用 xmltodict 创建列表/字典来迭代它。但每当我点击超链接或文本格式时，我都会遇到重大问题。我想是因为我试图通过 XML 进行暴力破解。它将附加标签放置在列表/字典的更深处。

基本上，我觉得我的做法是错误的。

以下是我正在使用的 XML 的两个片段。

XML 1

<?xml version="1.0" encoding="UTF-8" ?>
<chat xmlns="http://purl.org/net/ulf/ns/0.4-02" account="someusername" service="AIM">
<message sender="someusername" time="2008-07-27T18:02:34-0700"><div><span style="font-family: Arial; font-size: 10pt;">time is not of the essence</span></div></message>
<message sender="someusername" time="2008-07-27T18:02:43-0700"><div><span style="font-family: Arial; font-size: 10pt;">it <span style="font-style: italic;">is</span></span><span style="font-family: Helvetica; font-size: 12pt;"> </span><span style="font-family: Arial; font-size: 10pt;">the essence</span></div></message>
<message sender="anotherusername" time="2008-07-27T18:03:49-0700"><div><span style="color: #000000; font-family: Helvetica; font-size: 12pt;">yo</span></div></message>
<message sender="anotherusername" time="2008-07-27T18:03:51-0700"><div><span style="color: #000000; font-family: Helvetica; font-size: 12pt;">whats up?</span></div></message></chat>

XML 2

<?xml version="1.0" encoding="UTF-8" ?>
    <chat xmlns="http://purl.org/net/ulf/ns/0.4-02" account="someusername" service="AIM">
    <message sender="someusername" time="2009-09-26T05:54:23-0700"><div><a href="http://www.youtube.com/watch?v=LqbJx4TFFEE&amp;feature=related" style="color: #000000; font-family: Helvetica; font-size: 12pt;">http://www.youtube.com/watch?v=LqbJx4TFFEE&amp;feature=related</a></div></message>
    <message sender="someusername" time="2009-09-27T16:12:29-0700"><div><span style="color: #000000; font-family: Helvetica; font-size: 12pt;">2nd take, with the bonus stuff I think</span></div></message>
    <message sender="someusername" time="2009-09-27T17:18:52-0700"><div></div></message></chat>

这是我一直在使用的代码(抱歉，其中有一些废话):

import xmltodict
import os

def get_list_of_all_files_in_sub(dirName):
    # create a list of file and sub directories 
    # names in the given directory 
    listOfFile = os.listdir(dirName)
    allFiles = list()
    all_files_with_extension = list()

    # Iterate over all the entries
    for entry in listOfFile:
        # Create full path
        fullPath = os.path.join(dirName, entry)
        # If entry is a directory then get the list of files in this directory 
        if os.path.isdir(fullPath):
            allFiles = allFiles + get_list_of_all_files_in_sub(fullPath)
        else:
            allFiles.append(fullPath)
    return allFiles

def get_files_with_extension(path, file_extension=""):
    # gets a list of all files with a certain extension in a folder and all subfolders
    files = get_list_of_all_files_in_sub(path)

    all_files_with_extension = []
    for file in files:
        if file.split(".")[-1] == file_extension:
            all_files_with_extension.append(file)
    return all_files_with_extension

allmessages = []

files = get_files_with_extension("/Users/Desktop/chats", "chatlog")

for file in files:
    print (file)
    with open(file) as fd:
        doc = xmltodict.parse(fd.read())

    messages = doc['chat']['message']

    # this is gross, but apparently if "messages" only has one entry, it doesn't return a list.  So
    # to remedy this, im craming it into a list and back into itself to work with the rest of the code.
    if type(messages) is not list:
        print ("NOT A LIST")
        messages_container = messages
        messages = [messages_container]

    for message in messages:
        # Check to see if the SPAN exists inside DIV, which basically is checking to see if there's a real message in it.
        if 'span' in message["div"]:
            # checks if there's a sender, if there's no sender, it just doesn't include it in the output.
            if message["@sender"] != "":
                time =      (message["@time"])
                print (time)
                username =  (message["@sender"])
                print (username)

                # SET THE MESSAGE
                # If there are multiple messages within one message, then it comes in as a list.
                # But so far its just been things like warnings and offline notifications.
                # This seems to happen with AIM messages.
                if type(message["div"]['span']) is list:
                    print (message["div"]['span'])
                    for submessage in message["div"]['span']:
                        for subsubmessage in submessage:
                            print ("---------------1----------------")
                            print (subsubmessage)
                            print ("---------------2----------------")
                            if type(subsubmessage) is list:
                                print (subsubmessage["#text"])
                                if "Offline IM sent" not in subsubmessage["#text"]:
                                    text_message =  (subsubmessage["#text"])
                                    print (text_message)
                else:
                    text_message =  (message["div"]['span']["#text"])
                    print (text_message)

                if len(allmessages) > 0:
                    if (username == allmessages[-1]["sender"]):
                        if  (allmessages[-1]["message"].endswith('.')):
                            text_message = allmessages[-1]["message"] + "  " + text_message
                        else:
                            text_message = allmessages[-1]["message"] + ".  " + text_message

                        del allmessages[-1]

                newmessage = {  'time'      : time, 
                                'sender'    : username, 
                                'message'   : text_message
                            }

                allmessages.append (newmessage)
                #print ("{} {}: {}".format(time, username, message))

for message in x:
    print ("{} {}: {}".format(message['time'], message['sender'], message['message']))

我注意到xmltodict处理html标签的方式，它在输出时变成这样:

OrderedDict([('span', OrderedDict([('@style', 'font-family: Arial; font-size: 10pt;'), ('#text', 'time is not of the essence')]))])
OrderedDict([('span', [OrderedDict([('@style', 'font-family: Arial; font-size: 10pt;'), ('span', OrderedDict([('@style', 'font-style: italic;'), ('#text', 'is')])), ('#text', 'it')]), OrderedDict([('@style', 'font-family: Helvetica; font-size: 12pt;')]), OrderedDict([('@style', 'font-family: Arial; font-size: 10pt;'), ('#text', 'the essence')])])])

如您所见，带有格式的#text 被拉出并分离。关于如何做到这一点的任何其他方法或想法可能会效果更好？

Answer 1

您可以使用 BeautifulSoup，并使用 xml 作为解析器类型:

from bs4 import BeautifulSoup
import datetime
from pprint import pprint

xml_1 = '''<?xml version="1.0" encoding="UTF-8" ?>
<chat xmlns="http://purl.org/net/ulf/ns/0.4-02" account="someusername" service="AIM">
<message sender="someusername" time="2008-07-27T18:02:34-0700"><div><span style="font-family: Arial; font-size: 10pt;">time is not of the essence</span></div></message>
<message sender="someusername" time="2008-07-27T18:02:43-0700"><div><span style="font-family: Arial; font-size: 10pt;">it <span style="font-style: italic;">is</span></span><span style="font-family: Helvetica; font-size: 12pt;"> </span><span style="font-family: Arial; font-size: 10pt;">the essence</span></div></message>
<message sender="anotherusername" time="2008-07-27T18:03:49-0700"><div><span style="color: #000000; font-family: Helvetica; font-size: 12pt;">yo</span></div></message>
<message sender="anotherusername" time="2008-07-27T18:03:51-0700"><div><span style="color: #000000; font-family: Helvetica; font-size: 12pt;">whats up?</span></div></message></chat>'''

xml_2 = '''<?xml version="1.0" encoding="UTF-8" ?>
    <chat xmlns="http://purl.org/net/ulf/ns/0.4-02" account="someusername" service="AIM">
    <message sender="someusername" time="2009-09-26T05:54:23-0700"><div><a href="http://www.youtube.com/watch?v=LqbJx4TFFEE&amp;feature=related" style="color: #000000; font-family: Helvetica; font-size: 12pt;">http://www.youtube.com/watch?v=LqbJx4TFFEE&amp;feature=related</a></div></message>
    <message sender="someusername" time="2009-09-27T16:12:29-0700"><div><span style="color: #000000; font-family: Helvetica; font-size: 12pt;">2nd take, with the bonus stuff I think</span></div></message>
    <message sender="someusername" time="2009-09-27T17:18:52-0700"><div></div></message></chat>'''

def parse_xml(xml_string):
    soup = BeautifulSoup(xml_string, 'xml')
    data = []
    for message in soup.select('message[sender][time]'):
        account = message.find_parent('chat')['account']
        sender = message['sender']
        d = datetime.datetime.strptime(message['time'], "%Y-%m-%dT%H:%M:%S%z")  # ISO 8601 time format
        text = message.text.strip()
        data.append((account, sender, d, text))
    return data

pprint(parse_xml(xml_1))
pprint(parse_xml(xml_2))

打印:

[('someusername',
  'someusername',
  datetime.datetime(2008, 7, 27, 18, 2, 34, tzinfo=datetime.timezone(datetime.timedelta(-1, 61200))),
  'time is not of the essence'),
 ('someusername',
  'someusername',
  datetime.datetime(2008, 7, 27, 18, 2, 43, tzinfo=datetime.timezone(datetime.timedelta(-1, 61200))),
  'it is the essence'),
 ('someusername',
  'anotherusername',
  datetime.datetime(2008, 7, 27, 18, 3, 49, tzinfo=datetime.timezone(datetime.timedelta(-1, 61200))),
  'yo'),
 ('someusername',
  'anotherusername',
  datetime.datetime(2008, 7, 27, 18, 3, 51, tzinfo=datetime.timezone(datetime.timedelta(-1, 61200))),
  'whats up?')]
[('someusername',
  'someusername',
  datetime.datetime(2009, 9, 26, 5, 54, 23, tzinfo=datetime.timezone(datetime.timedelta(-1, 61200))),
  'http://www.youtube.com/watch?v=LqbJx4TFFEE&feature=related'),
 ('someusername',
  'someusername',
  datetime.datetime(2009, 9, 27, 16, 12, 29, tzinfo=datetime.timezone(datetime.timedelta(-1, 61200))),
  '2nd take, with the bonus stuff I think'),
 ('someusername',
  'someusername',
  datetime.datetime(2009, 9, 27, 17, 18, 52, tzinfo=datetime.timezone(datetime.timedelta(-1, 61200))),
  '')]