3333# ----------------------------------------------------------------------------
3434import codecs
3535import gzip
36- import io
3736import logging
3837import os
3938import sys
@@ -75,49 +74,44 @@ def create_line_generator(self):
7574 Creates a generator function yielding lines in the file
7675 Should only yield non-empty lines
7776 """
77+
7878 if self .file_name .endswith (".gz" ):
7979 if sys .version_info .major == 3 :
8080 gz = gzip .open (self .file_name , mode = 'rt' , encoding = self .encoding )
81- f = io .BufferedReader (gz )
82- for line in gz :
83- if line == '' :
84- continue
85- yield line
86-
87- gz .close ()
8881 else :
8982 gz = gzip .open (self .file_name , mode = 'rt' )
90- f = io .BufferedReader (gz )
91- for line in f .readlines ():
92- if line == '' :
93- continue
94- yield line
9583
96- gz .close ()
84+ for line in gz .readlines ():
85+ if line == '' :
86+ continue
87+ yield line
88+
89+ gz .close ()
9790 else :
9891 if sys .version_info .major == 3 :
9992 # Python 3 native `open` is much faster
100- with open (self .file_name , mode = 'r' , encoding = self .encoding ) as file :
101- for line in file :
102- # Skip empty lines
103- if line == '' :
104- continue
105- yield line
93+ file = open (self .file_name , mode = 'r' , encoding = self .encoding )
10694 else :
107- with codecs .open (self .file_name , mode = 'r' , encoding = self .encoding ) as file :
108- for line in file :
109- # Skip empty lines
110- if line == '' :
111- continue
112- yield line
95+ # Python 2 needs the codecs package to deal with encoding
96+ file = codecs .open (self .file_name , mode = 'r' , encoding = self .encoding )
97+
98+ for line in file :
99+ if line == '' : # Skip empty lines
100+ continue
101+ yield line
102+
103+ file .close ()
113104
114105 def next_line (self ):
115106 """Read the next line from the line generator and split it"""
116- self .line = next (self .lines )
107+ self .line = next (self .lines ) # Will raise StopIteration when there are no more lines
117108 self .values = self .line .split ()
118109
119110 def consume_line (self ):
120- """Tell the parser we are done with this line"""
111+ """
112+ Tell the parser we are done with this line.
113+ This is simply by setting None values.
114+ """
121115 self .line = None
122116 self .values = None
123117
@@ -127,8 +121,12 @@ def parse(self):
127121 Determines what type of line we are and dispatch appropriately.
128122 """
129123 try :
124+ # Continues until `next_line()` raises StopIteration
125+ # This can trigger here or in parse functions in the subclass
130126 while True :
131- # Only advance the parser if the previous line was consumed
127+ # Only advance the parser if the previous line was consumed.
128+ # Parse functions reading multiple lines can end up reading one line too far,
129+ # so they return without consuming the line and we pick it up here
132130 if not self .line :
133131 self .next_line ()
134132
@@ -150,8 +148,6 @@ def parse_fallback(self):
150148 logging .warning ("Unimplemented OBJ format statement '%s' on line '%s'"
151149 % (self .values [0 ], self .line .rstrip ()))
152150
153- self .consume_line ()
154-
155151 def _build_dispatch_map (self ):
156152 """
157153 Build a dispatch map: {func name": func} dict
0 commit comments