- Fix missing data

Author: msorkhpar

README.md

@@ -42,4 +42,9 @@ with include drop, create tables, create indexes, reset sequences
 EOT
 
 pgloader ./sqlite-to-page-migration.load
+```
+
+After running the experiment we encountered some issues with the `wikimapper` library, so we developed the following script to correct the missing data:
+```shell
+python3 missing_data_correction.py
 ```

commons/storage.py

@@ -113,6 +113,7 @@ def fetch_wiki_mapping(identifier: str) -> tuple[str, str, str] | None:
                     record = (None, None, identifier)
                 else:
                     record = (None, None, None)
+                    print(query)
                     print("Identifier not found!!!", identifier)
     except (Exception, psycopg2.DatabaseError) as error:
         print("Error while fetching abstract", error)
@@ -246,6 +247,28 @@ def mark_wikipedia_page_process_failed(root_wikipedia_id):
             postgresql_pool.putconn(connection)
 
 
+def insert_missing_mappings(wikipedia_id, wikipedia_title, wikidata_id):
+    connection = None
+    try:
+        connection = postgresql_pool.getconn()
+        if connection:
+            with connection.cursor() as cursor:
+                if wikidata_id:
+                    cursor.execute(
+                        "INSERT INTO wiki_page_to_wiki_data_mappings(wikipedia_id, wikipedia_title, wikidata_id)"
+                        " VALUES (%s, %s, %s)",
+                        (wikipedia_id, wikipedia_title, wikidata_id))
+                if not wikidata_id:
+                    cursor.execute("UPDATE wikipedia_pages SET processed = TRUE WHERE id = %s",
+                                   (wikipedia_id,))
+                connection.commit()
+    except (Exception, psycopg2.DatabaseError) as error:
+        print("Error while updating processed field", error)
+    finally:
+        if connection:
+            postgresql_pool.putconn(connection)
+
+
 def create_neo4j_session() -> tuple[Driver, Session]:
     driver = GraphDatabase.driver(NEO4J_URI, auth=NEO4J_AUTH)
     return driver, driver.session(database=NEO4J_DB)

missing_data_correction.py

@@ -0,0 +1,32 @@
+import csv
+
+import pandas as pd
+import wptools
+
+from commons.storage import insert_missing_mappings
+from commons.wiki_entity import WikiEntity
+from commons.wiki_mapping import WikiMapping
+
+
+def fetch_wikidata_id(title):
+    try:
+        page = wptools.page(title, silent=True).get(show=False)
+        wikidata_id = page.data['wikibase']
+        return wikidata_id
+    except Exception as e:
+        return None
+
+
+if __name__ == '__main__':
+    csv_file = 'missing_data_pages.csv'
+    df = pd.read_csv(csv_file)
+    result = []
+    failed = []
+    insert_queries = []
+    for index, row in df.iterrows():
+        page_id = row['id']
+        title = row['title']
+        wikidata_id = fetch_wikidata_id(title)
+        insert_missing_mappings(page_id, title, wikidata_id)
+        if wikidata_id:
+            WikiEntity(WikiMapping(wikidata_id=wikidata_id)).get_summaries(False)