File tree Expand file tree Collapse file tree 1 file changed +7
-2
lines changed
Expand file tree Collapse file tree 1 file changed +7
-2
lines changed Original file line number Diff line number Diff line change @@ -66,13 +66,18 @@ fn main() -> Result<(), Box<dyn Error>> {
6666}
6767
6868fn load_movielens ( path : & Path ) -> Result < Dataset < i32 , String > , Box < dyn Error > > {
69- // read movies, removing invalid UTF-8 bytes
69+ // read movies
7070 let mut movies = HashMap :: with_capacity ( 2000 ) ;
7171 let movies_file = File :: open ( path. join ( "u.item" ) ) ?;
7272 let rdr = BufReader :: new ( movies_file) ;
7373 for line in rdr. split ( b'\n' ) {
7474 let line = line?;
75- let line = String :: from_utf8_lossy ( & line) ;
75+ // convert encoding to UTF-8
76+ let line = String :: from_utf8 (
77+ line. into_iter ( )
78+ . flat_map ( |v| if v < 128 { vec ! [ v] } else { vec ! [ 195 , v - 64 ] } )
79+ . collect ( ) ,
80+ ) ?;
7681 let mut row = line. split ( '|' ) ;
7782 let id = row. next ( ) . unwrap ( ) . to_string ( ) ;
7883 let name = row. next ( ) . unwrap ( ) . to_string ( ) ;
You can’t perform that action at this time.
0 commit comments