Merged from master version 2.1.1

Author: reidmcy

.gitignore

@@ -20,9 +20,9 @@ profile_wosserver/
 metaknowledgeDocs.md
 j9Raws/
 /manualj9Abbreviations*
-metaknowledge/WOS/journalAbbreviations/j9Abbreviations.bak
-metaknowledge/WOS/journalAbbreviations/j9Abbreviations.dir
-metaknowledge/WOS/journalAbbreviations/j9Abbreviations.dat
+metaknowledge/journalAbbreviations/j9Abbreviations.bak
+metaknowledge/journalAbbreviations/j9Abbreviations.dir
+metaknowledge/journalAbbreviations/j9Abbreviations.dat
 !savedrecs.txt
 *.bib
 

metaknowledge/WOS/__init__.py

@@ -85,4 +85,6 @@
 
 from .tagProcessing.tagFunctions import *
 from .tagProcessing.funcDicts import tagToFullDict, fullToTagDict, tagNameConverterDict, tagsAndNameSet, knownTagsList
-from .journalAbbreviations.backend import updatej9DB, getj9dict, abrevDBname, excludeFromDB, addToDB, manaulDBname
+
+from .recordWOS import WOSRecord, recordParser
+from .wosHandlers import isWOSFile, wosParser

metaknowledge/WOS/tagProcessing/tagFunctions.py

@@ -1,6 +1,6 @@
 #Written by Reid McIlroy-Young for Dr. John McLevey, University of Waterloo 2015
 from .helpFuncs import getMonth
-from ..journalAbbreviations.wosCitations import WOSCitation
+from ...citation import Citation
 
 import collections
 
@@ -523,7 +523,7 @@ def citations(val):
     """
     retCites = []
     for c in val:
-        retCites.append(WOSCitation(c))
+        retCites.append(Citation(c))
     return retCites
 
 def publisherCity(val):

metaknowledge/WOS/wosHandlers.py

@@ -4,7 +4,23 @@
 from ..mkExceptions import cacheError, BadWOSFile, BadWOSRecord
 
 def isWOSFile(infile, checkedLines = 3):
-    """Checks if _infile_ has the right header in the first _checkedLines_ lines
+    """Determines if _infile_ is the path to a WOS file. A file is considerd to be a WOS file if it has the correct encoding (`utf-8` with a BOM) and within the first _checkedLines_ a line starts with `"VR 1.0"`.
+
+    # Parameters
+
+    _infile_ : `str`
+
+    > The path to the targets file
+
+    _checkedLines_ : `optional [int]`
+
+    > default 2, the number of lines to check for the header
+
+    # Returns
+
+    `bool`
+
+    > `True` if the file is a WOS file
     """
     try:
         with open(infile, 'r', encoding='utf-8-sig') as openfile:

metaknowledge/__init__.py

@@ -1,5 +1,5 @@
-#Written by Reid McIlroy-Young for Dr. John McLevey, University of Waterloo 2015
-"""metaknowledge is a Python3 package that simplifies bibliometric and computational analysis of Web of Science data.
+#Written by Reid McIlroy-Young for Dr. John McLevey, University of Waterloo 2016
+"""_metaknowledge_ is a Python3 package that simplifies bibliometric and computational analysis of Web of Science data.
 
 # Example
 
@@ -19,43 +19,33 @@
 
 # Overview
 
-This package can read the files downloaded from the [Thomson Reuters Web of Science](https://webofknowledge.com) (WOS) as plain text. These files contain metadata about scientific records, such as the authors, title, and citations. The records are exported in groups of up-to 500 individual records to a file.
+This package can read the files downloaded from the Thomson Reuters' [Web of Science](https://webofknowledge.com) (_WOS_), Elsevier's [Scopus](https://www.scopus.com/), [ProQuest](www.proquest.com/) and Medline files from [PubMed](www.ncbi.nlm.nih.gov/pubmed). These files contain entries on the metadata of scientific records, such as authors, title, and citations. _metaknowledge_ can also read grants from various organizations including _NSF_ and _NSERC_ which are handled similarly to records.
 
-The [metaknowledge.RecordCollection](#RecordCollection.RecordCollection) class can take a path to one or more of these files load and parse them. The object is the main way for work to be done on multiple records. For each individual record it creates an instance of the [metaknowledge.Record](#Record.Record) class that contains the results of the parsing of the record.
+The [metaknowledge.RecordCollection](#RecordCollection.RecordCollection) class can take a path to one or more of these files load and parse them. The object is the main way for work to be done on multiple records. For each individual record it creates an instance of the [metaknowledge.Record](#metaknowledge.Record) class that contains the results of the parsing of the record.
 
-The files given by WOS are a flat database containing a series of 2 character tags, e.g. 'TI' is the title. Each WOS tag has one or more values and metaknowledge can read them to extract useful information. The approximate meanings of the tags are listed in the [tagProcessing](#tagProcessing.tagProcessing) package, along with the parsing functions for each tag. If you simply want the mapping [`tagToFull()`](#metaknowledge.tagToFull) is a function that maps tags to their full names it, as well as a few other similar functions are provided by the base metaknowledge import. Note, the long names can be used in place of the short 2 character codes within metaknowledge. There are no full official public listings of tag the meanings available. metaknowledge is not attempting to provide the definitive or authoritative meanings.
+The files read by _metaknowledge_ are a databases containing a series of tags (implicitly or explicitly), e.g. `'TI'` is the title for WOS. Each tag has one or more values and metaknowledge can read them and extract useful information. As the tags differ between providers a small set of values can be accessed by special tags, the tags are listed in `specialRecordFields`. These special tags can act on the whole `Record` and as such may contain information provided by any number of other tags.
 
-Citations are handled by a special [Citation](#Citation.Citation) class. This class can parse the citations given by WOS as well as extra details about the full name of their journal and allow simple comparisons.
+Citations are handled by a special [Citation](#Citation.Citation) class. This class can parse the citations given by _WOS_ and journals cited by _Scopus_ and allows for better comparisons when they are used in graphs.
 
 Note for those reading the docstrings metaknowledge's docs are written in markdown and are processed to produce the documentation found at [networkslab.org/metaknowledge/documentation]({{ site.baseurl }}/documentation/), but you should have no problem reading them from the help function.
 """
 
-from .mkRecord import Record, ExtendedRecord
-from .citation import Citation, filterNonJournals
-from .grants.baseGrant import Grant, DefaultGrant
-from .grants.medlineGrant import MedlineGrant
-from .grants.cihrGrant import CIHRGrant
-from .grants.nsercGrant import NSERCGrant
-
+from .constants import VERBOSE_MODE, __version__, specialRecordFields, FAST_CITES
+from .mkExceptions import BadCitation, BadGrant, BadInputFile, BadProQuestFile, BadProQuestRecord, BadPubmedFile, BadPubmedRecord, BadRecord, BadWOSFile, BadWOSRecord, CollectionTypeError, GrantCollectionException, RCTypeError, RCValueError, RecordsNotCompatible, UnknownFile, cacheError, mkException, TagError, BadScopusRecord
 
+from .graphHelpers import writeEdgeList, writeNodeAttributeFile, writeGraph, readGraph, dropEdges, dropNodesByDegree, dropNodesByCount, mergeGraphs, graphStats, writeTnetFile
+from .diffusion import diffusionGraph, diffusionCount, diffusionAddCountsFromSource
 
-from .recordCollection import RecordCollection
-from .mkExceptions import BadCitation, BadGrant, BadInputFile, BadProQuestFile, BadProQuestRecord, BadPubmedFile, BadPubmedRecord, BadRecord, BadWOSFile, BadWOSRecord, CollectionTypeError, GrantCollectionException, RCTypeError, RCValueError, RecordsNotCompatible, UnknownFile, cacheError, mkException
-#from .progressBar import _ProgressBar
-from .grantCollection import GrantCollection
+from .citation import Citation, filterNonJournals
 from .mkCollection import Collection, CollectionWithIDs
+from .mkRecord import Record, ExtendedRecord
 
-from .graphHelpers import writeEdgeList, writeNodeAttributeFile, writeGraph, readGraph, dropEdges, dropNodesByDegree, dropNodesByCount, mergeGraphs, graphStats
-from .constants import VERBOSE_MODE, __version__, specialRecordFields, FAST_CITES
-from .diffusion import diffusionGraph, diffusionCount, diffusionAddCountsFromSource
 
-#from .WOS.tagProcessing.funcDicts import tagToFull, isTagOrName, normalizeToTag, normalizeToName
-#from .WOS.wosHandlers import wosParser
-from .WOS.recordWOS import WOSRecord#, recordParser
-from .WOS.journalAbbreviations.wosCitations import WOSCitation
+from .grantCollection import GrantCollection
+from .grants import NSERCGrant, CIHRGrant, MedlineGrant, NSFGrant, Grant, DefaultGrant
 
+from .recordCollection import RecordCollection
+from .WOS import WOSRecord
 from .medline import MedlineRecord
-
 from .proquest import ProQuestRecord
-
-#from .blondel import blondel, modularity #Better implementations can be found on Pypi so this has been discontinued
+from .scopus import ScopusRecord

metaknowledge/bin/metaknowledgeCLI.py

@@ -1,6 +1,6 @@
 #Written by Reid McIlroy-Young for Dr. John McLevey, University of Waterloo 2015
 import metaknowledge
-import metaknowledge.WOS
+import metaknowledge.journalAbbreviations
 import networkx as nx
 import argparse
 import os
@@ -180,10 +180,10 @@ def getWhatToDo(clargs, inRC):
             f.writelines(cites)
         return False
     else:
-        dbName = input("The default manual databse file is called {}, press Enter to use it or type the name of the database you wish to use:\n".format(metaknowledge.WOS.manaulDBname))
+        dbName = input("The default manual databse file is called {}, press Enter to use it or type the name of the database you wish to use:\n".format(metaknowledge.journalAbbreviations.manaulDBname))
         print("Starting to go over citations, to exit press ctr-C.")
         if dbName == '':
-            dbName = metaknowledge.WOS.manaulDBname
+            dbName = metaknowledge.journalAbbreviations.manaulDBname
         try:
             for R in inRC:
                 for c in R.get('citations', []):

metaknowledge/bin/metaknowledgeDocsGen.py

@@ -10,17 +10,20 @@
 import importlib
 import re
 
-documentedModules = ['contour', 'WOS', 'medline', 'proquest']#, 'journalAbbreviations', 'tagProcessing']
+documentedModules = ['contour', 'WOS', 'medline', 'proquest', 'scopus', 'journalAbbreviations']
 
 docsPrefix = time.strftime("%Y-%m-%d-")
 
 funcCounter = 0
+undocumented = 0
 
 blurbDict = {
     #modules
     'contour' : "A nicer matplotlib graph visualizer and contour plot",
     'WOS' : "The functions and classes associated with the Web of Science",
+    'journalAbbreviations' : "Handles the abbreviated journal names used by WOS",
     'medline' : "The functions and classes associated with Medline, the format used by Pubmed",
+    'scopus' : "The functions and classes associated with records from scopus",
     'proquest' : "The functions and classes associated with ProQuest",
 
     #Classes
@@ -33,12 +36,14 @@
     'WOSRecord' : "The object for containing and processing WOS entries",
     'ProQuestRecord' : "The object for containing and processing ProQuest entries",
     'MedlineRecord' : "The object for containing and processing Medline entries",
+    'ScopusRecord' : "The object for containing and processing Scopus entries",
 
     'Grant' : "The base for all the other Grants",
     'DefaultGrant' : "The Grant used if a file was not identifiable",
     'CIHRGrant' : "The container for CIHR grant entries",
     'NSERCGrant' : "The container for NSERC grant entries",
     'MedlineGrant' : "The container for grants derived from Medline Records entries",
+    'NSFGrant' : "The container for NSF grant entries",
 
     'Collection' : "The base of all other Collections, basically a set",
     'CollectionWithIDs' : "A Collection that only holds <i>metaknowledge</i> objects",
@@ -47,12 +52,11 @@
 
     #Deprecated
     'tagProcessing' : "All the tags and how they are handled",
-    'journalAbbreviations' : "Look here to get your J9 database",
 }
 
 singleFileYAML = """---
 layout: page
-title: Full Documentation
+title: Full Documentation {}
 author:
 - name: Reid McIlroy-Young
   department:
@@ -64,14 +68,14 @@
 shorttitle: metaknowledge
 search_omit: true
 ---
-"""
+""".format(metaknowledge.__version__)
 
 def makeBlurb(name):
     if name in blurbDict:
         return blurbDict[name]
     else:
         print("\033[94m{} had no blurb\033[0m".format(name))
-        return 'BLURB NEEDED FOR: {}'.format(name)
+        return 'BLURB NEEDED FOR {}'.format(name)
         #raise RuntimeError("{} needs a blurb".format(name))
 
 def makeHeader(title, excerpt, tags = (), weight = 10, layout = "doc", singleFile = False):
@@ -209,6 +213,8 @@ def writeFunc(fn, f, prefix = '', level = 5, singleFile = False):
         f.write(cleanedDoc(fn[1], lvl = level, singleFile = singleFile))
     except AttributeError:
         f.write("# Needs to be written\n\n")
+        global undocumented
+        undocumented += 1
         print("\033[93m{0}{1} had no docs\033[0m".format(prefix, fn[0]))
 
 def writeClass(cl, f, prefix = '', level = 4, singleFile = False, exceptMode = False):
@@ -219,6 +225,8 @@ def writeClass(cl, f, prefix = '', level = 4, singleFile = False, exceptMode = F
         f.write(cleanedDoc(cl[1], lvl = level, singleFile = singleFile))
     except AttributeError:
         f.write("# Needs to be written\n\n")
+        global undocumented
+        undocumented += 1
         print("\033[93m{0}{1} had no docs\033[0m".format(prefix, cl[0]))
 
 def proccessClass(cls, f, singleFile = False, exceptMode = False):
@@ -335,7 +343,7 @@ def main(args):
 
     if args.single:
         single = True
-        f = open("metaknowledge2Draft.md",'w')
+        f = open("metaknowledgeFull.md",'w')
         f.write(singleFileYAML)
 
         f.write(makeTable(documentedModules, header = '<a name="objlist"></a>The modules of <i>metaknowledge</i> are:', withBlurbs = True))
@@ -383,7 +391,8 @@ def main(args):
 def mkDocs():
     args = argumentParser()
     main(args)
-    print(funcCounter)
+    print("{} total functions".format(funcCounter))
+    print("{} undocumented".format(undocumented))
 
 if __name__ == '__main__':
     mkDocs()