Triples validation

- Check if the extracted triples belong to English wikidata persist them
- Fresh fetch option for summary builder

Author: msorkhpar

commons/wiki_entity.py

@@ -17,18 +17,18 @@ def __init__(self, wiki_mapping: WikiMapping):
         self._wikidata_label = None
         self._wikidata_description = None
 
-    def get_summaries(self, keep_results=True):
+    def get_summaries(self, keep_results=True, fresh=False):
         if self._summaries:
             return self._summaries
         if not keep_results:
-            build_summaries(self.wikipedia_id, self.wikipedia_page_title, self.wikidata_id)
+            build_summaries(self.wikipedia_id, self.wikipedia_page_title, self.wikidata_id, fresh=fresh)
             return
-        self._summaries = build_summaries(self.wikipedia_id, self.wikipedia_page_title, self.wikidata_id)
+        self._summaries = build_summaries(self.wikipedia_id, self.wikipedia_page_title, self.wikidata_id, fresh)
         return self._summaries
 
-    def get_detailed_summaries(self):
+    def get_detailed_summaries(self, fresh=False):
         if not self._summaries:
-            self._summaries = self.get_summaries()
+            self._summaries = self.get_summaries(fresh=fresh)
         results = []
         for summary in self._summaries:
             from_wikipedia_id, from_wikipedia_title, from_wikidata_id = fetch_wiki_mapping(summary[0])

wiki-storage/src/main/java/com/github/msorkhpar/wikistorage/utils/WikidataEnglishInfoDTO.java

@@ -23,15 +23,5 @@ public WikidataEnglishInfoDTO(String title, AdditionalInfo data) {
         if (wikipediaInfo != null) {
             this.enWikiTitle = wikipediaInfo.getTitle();
         }
-        String key = "";
-        if (this.label == null && data.getLabels() != null && !data.getLabels().isEmpty()) {
-            key = data.getLabels().keySet().iterator().next();
-            this.label = data.getLabels().get(key).getValue();
-            this.description = data.getDescription() != null ? data.getDescription().getOrDefault(key, null).getValue() : null;
-        }
-
-        if (this.description == null && data.getDescription() != null && !data.getDescription().isEmpty()) {
-            this.description = data.getDescription().values().iterator().next().getValue();
-        }
     }
 }

wiki_summary/summary_builder.py

@@ -53,13 +53,28 @@ def _get_edge_candidates(root_wikidata_id, mentions: list[str]) -> dict[str, lis
     return edge_candidates
 
 
-def build_summaries(wikipedia_id, wikipedia_title, root_wikidata_id) -> list[tuple[str, str, str]]:
+def build_summaries(wikipedia_id, wikipedia_title, root_wikidata_id, fresh=False) -> list[tuple[str, str, str]]:
     """
     :param wikipedia_title:
     :param root_wikidata_id:
     :return: list of summaries: [ (from_entity, predicate, to_entity), ...]
     """
     # check if the summaries are already stored in Neo4j and return them
+    if fresh:
+        page_content = fetch_wikipedia_page_content(wikipedia_title)
+        raw_abstract = extract_raw_abstract(page_content)
+        mentions = extract_mention_titles(raw_abstract)
+        edge_candidates = _get_edge_candidates(root_wikidata_id, mentions)
+        summaries = []
+        for index, candidates in edge_candidates.items():
+            if len(candidates) > 1:
+                abstract_embedding = compute_embeddings(dewiki(raw_abstract))
+                candidate = _pick_most_relevant_predicate(abstract_embedding, candidates)
+            else:
+                candidate = candidates[0]
+            summaries.append(candidate)
+        return summaries
+
     summaries = fetch_summaries(root_wikidata_id)
     if summaries:
         mark_wikipedia_page_processed(wikipedia_id)

wikidata-graph-builder/src/main/java/com/github/msorkhpar/graphbuilder/service/WikidataDumpFileService.java

@@ -52,9 +52,16 @@ public CompletableFuture<Path> process(Path dumpFile, boolean multiStream) {
                     String pageString = xmlBuilder.toString();
                     try {
                         Element xmlPage = createPage(pageString);
-                        boolean triplePersistentStatus = processTriples(WikiDataEntityExtractor.extractTriples(xmlPage));
+                        Optional<Set<KGTriple>> tripleSet = WikiDataEntityExtractor.extractTriples(xmlPage);
+                        Optional<WikidataEnglishInfoDTO> metadata = extractMetadata(xmlPage);
+                        if (tripleSet.isEmpty() || metadata.isEmpty() ||
+                                (metadata.get().getLabel().isEmpty() && metadata.get().getDescription().isEmpty())
+                        ) {
+                            continue;
+                        }
+                        boolean triplePersistentStatus = processTriples(tripleSet);
                         if (triplePersistentStatus) {
-                            processMetadata(extractMetadata(xmlPage));
+                            processMetadata(metadata);
                         }
                     } catch (Exception e) {
                         logger.info("Extraction from the following text was not successful, {}", pageString);