improve xml_dump processing.

examples/xml_dump.iteration.py

@@ -8,7 +8,9 @@
 
 # Iterate through pages
 for page in dump:
-
         # Iterate through a page's revisions
         for revision in page:
                 print(revision.id)
+
+# dump has a language attribute
+assert dump.lang == 'en'

mw/xml_dump/element_iterator.py

@@ -49,7 +49,7 @@ def __init__(self, element, pointer):
         self.pointer = pointer
         self.element = element
         self.depth = pointer.depth() - 1
-
+        self.element.attrib = {trim_ns(k): v for k, v in self.element.attrib.items()}
         self.done = False
 
     def __iter__(self):
@@ -93,7 +93,6 @@ def __getattr__(self, attr):
 
     @classmethod
     def from_file(cls, f):
-
         try:
             pointer = EventPointer.from_file(f)
             event, element = next(pointer)

mw/xml_dump/iteration/iterator.py

@@ -73,7 +73,7 @@ class Iterator(serializable.Type):
                  '__pages')
 
     def __init__(self, site_name=None, dbname=None, base=None, generator=None,
-                 case=None, namespaces=None, pages=None):
+                 case=None, namespaces=None, pages=None, lang=None):
 
         self.site_name = none_or(site_name, str)
         """
@@ -109,6 +109,11 @@ def __init__(self, site_name=None, dbname=None, base=None, generator=None,
         # Should be a lazy generator of page info
         self.__pages = pages
 
+        self.lang = none_or(lang, str)
+        """
+        A 2 character language code.
+        """
+
     def __iter__(self):
         return self.__pages
 
@@ -140,6 +145,9 @@ def load_site_info(cls, element):
         namespaces = {}
 
         for sub_element in element:
+
+            if sub_element.tag == 'siteinfo':
+                return(cls.load_site_info(sub_element))
             if sub_element.tag == 'sitename':
                 site_name = sub_element.text
             if sub_element.tag == 'dbname':
@@ -152,24 +160,21 @@ def load_site_info(cls, element):
                 case = sub_element.text
             elif sub_element.tag == 'namespaces':
                 namespaces = cls.load_namespaces(sub_element)
-
+        
         return site_name, dbname, base, generator, case, namespaces
 
     @classmethod
     def load_pages(cls, element):
-
         for sub_element in element:
             tag = sub_element.tag
-
             if tag == "page":
                 yield Page.from_element(sub_element)
             else:
                 assert MalformedXML("Expected to see 'page'.  " +
                                     "Instead saw '{0}'".format(tag))
 
     @classmethod
-    def from_element(cls, element):
-
+    def from_element(cls, element, lang=None):
         site_name = None
         base = None
         generator = None
@@ -187,20 +192,22 @@ def from_element(cls, element):
         # Consume all <page>
         pages = cls.load_pages(element)
 
-        return cls(site_name, dbname, base, generator, case, namespaces, pages)
+        return cls(site_name, dbname, base, generator, case, namespaces, pages, lang)
 
     @classmethod
     def from_file(cls, f):
         element = ElementIterator.from_file(f)
         assert element.tag == "mediawiki"
-        return cls.from_element(element)
+        lang = element.attr("lang")
+        return cls.from_element(element, lang=lang)
 
     @classmethod
     def from_string(cls, string):
         f = io.StringIO(string)
         element = ElementIterator.from_file(f)
         assert element.tag == "mediawiki"
-        return cls.from_element(element)
+        lang = element.attr("xml:lang")
+        return cls.from_element(element, lang=lang)
 
     @classmethod
     def from_page_xml(cls, page_xml):

mw/xml_dump/iteration/page.py

@@ -4,7 +4,6 @@
 from .redirect import Redirect
 from .revision import Revision
 
-
 class Page(serializable.Type):
     """
     Page meta data and a :class:`~mw.xml_dump.Revision` iterator.  Instances of
@@ -99,6 +98,8 @@ def from_element(cls, element):
                 restrictions.append(sub_element.text)
             elif tag == "DiscussionThreading":
                 continue
+            elif tag == "sha1":
+                continue
             elif tag == "revision":
                 first_revision = sub_element
                 break