Merged version 0.9.6

Former-commit-id: 818daf0

Author: reidmcy

README.rst

@@ -0,0 +1,26 @@
+metaknowledge
+=============
+
+metaknowledge is a Python3 library that simplifies bibliometric research
+using Web of Science data. It reads a directory of plain text files
+containing meta-data on publications and citations, and writes to a
+variety of data structures that are suitable for quantitative, network,
+and text analyses. It handles large datasets (e.g. several million
+records) efficiently.
+
+The website can be found at
+`networkslab.org/metaknowledge <http://networkslab.org/metaknowledge/>`__.
+
+Currently in Beta
+-----------------
+
+metaknowledge is in the final stages of testing before a 1.0 release
+different versions may behave differently.
+
+Installing
+----------
+
+To install run ``python3 setup.py install``
+
+For information on alternate installs read the documentation at the
+`website <http://networkslab.org/metaknowledge/installation/>`__.

metaknowledge/__init__.py

@@ -1,3 +1,4 @@
+#Written by Reid McIlroy-Young for Dr. John McLevey, University of Waterloo 2015
 """metaknowledge is a Python3 package that simplifies bibliometric and computational analysis of Web of Science data.
 
 # Example
@@ -12,25 +13,29 @@
     Done making a co-citation network of files-from-records                 1.1s
     >>> print(len(G.nodes()))
     223
-    >>> mk.write_graph(G, "Cocitation-Network-of-Journals")
+    >>> mk.writeGraph(G, "Cocitation-Network-of-Journals")
+
+There is also a simple command line program called `metaknowledge` that comes with the package. It allows for creating networks without any need to know Python. More information about it can be found at [networkslab.org/metaknowledge/cli]({{ site.baseurl }}/cli)
 
 # Overview
 
-This package can read the files downloaded from the [Thomson Reuters Web of Science](https://webofknowledge.com) (WOS) as plain text. These files contain metadata about scientific records, such as the authors, language, and citations. The records are saved in groups of up-to 500 individual records in a file.
+This package can read the files downloaded from the [Thomson Reuters Web of Science](https://webofknowledge.com) (WOS) as plain text. These files contain metadata about scientific records, such as the authors, title, and citations. The records are exported in groups of up-to 500 individual records to a file.
 
 The [metaknowledge.RecordCollection](#RecordCollection.RecordCollection) class can take a path to one or more of these files load and parse them. The object is the main way for work to be done on multiple records. For each individual record it creates an instance of the [metaknowledge.Record](#Record.Record) class that contains the results of the parsing of the record.
 
-The files given by WOS are a flat database containing a series of 2 character tags, e.g. 'TI' is the title. Each WOS tag has one or more values and metaknowledge makes use of them to extract useful information. The approximate meanings of the tags are listed in the [tagProcessing](#tagProcessing.tagProcessing) package, if you simply want the mapping [`tagToFull()`](#metaknowledge.tagToFull) is a function that maps tags to their full names it as well as a few other similar functions are provided by metaknowledge. There are no full official public listings of tag the meanings available. metaknowledge is not attempting to provide the definitive or authoritative meanings. Some
+The files given by WOS are a flat database containing a series of 2 character tags, e.g. 'TI' is the title. Each WOS tag has one or more values and metaknowledge can read them to extract useful information. The approximate meanings of the tags are listed in the [tagProcessing](#tagProcessing.tagProcessing) package, along with the parsing functions for each tag. If you simply want the mapping [`tagToFull()`](#metaknowledge.tagToFull) is a function that maps tags to their full names it, as well as a few other similar functions are provided by the base metaknowledge import. Note, the long names can be used in place of the short 2 character codes within metaknowledge. There are no full official public listings of tag the meanings available. metaknowledge is not attempting to provide the definitive or authoritative meanings.
 
-As citations are of great importance to sociology their handling is done with the [Citation](#Citation.Citation) class. This class can parse the citations given by WOS as well as extra details about the full name of their journal and allow simple comparisons.
+Citations are handled by a special [Citation](#Citation.Citation) class. This class can parse the citations given by WOS as well as extra details about the full name of their journal and allow simple comparisons.
 
-Note for those reading the docstring metaknowledge's docs are written in markdown and are processed to produce the documentation found at [networkslab.org/metaknowledge/documentation](http://networkslab.org/metaknowledge/documentation/).
+Note for those reading the docstrings metaknowledge's docs are written in markdown and are processed to produce the documentation found at [networkslab.org/metaknowledge/documentation]({{ site.baseurl }}/documentation/), but you should have no problem reading them from the help function.
 """
-from .record import Record, recordParser, BadISIRecord
+from .record import Record, recordParser, BadWOSRecord
 from .citation import Citation, BadCitation, filterNonJournals
-from .recordCollection import RecordCollection, isiParser
-from .graphHelpers import write_edgeList, write_nodeAttributeFile, write_graph, read_graph, _ProgressBar, drop_edges, drop_nodesByDegree, drop_nodesByCount, mergeGraphs, graphStats
+from .recordCollection import RecordCollection, wosParser
+
+from .graphHelpers import writeEdgeList, writeNodeAttributeFile, writeGraph, readGraph, _ProgressBar, dropEdges, dropNodesByDegree, dropNodesByCount, mergeGraphs, graphStats
 from .constants import VERBOSE_MODE
-#from .blondel import blondel, modularity
 from .diffusion import diffusionGraph, diffusionCount
 from .tagProcessing.funcDicts import tagToFull, isTagOrName, normalizeToTag, normalizeToName
+
+#from .blondel import blondel, modularity #Better implementations can be found on Pypi so this has been discontinued

metaknowledge/bin/__init__.py

@@ -1,3 +1,4 @@
+#Written by Reid McIlroy-Young for Dr. John McLevey, University of Waterloo 2015
 from .metaknowledgeCLI import mkCLI
 from .metaknowledgeMdToNb import mkMdToNb
 from .metaknowledgeDocsGen import mkDocs

metaknowledge/bin/metaknowledgeCLI.py

@@ -1,3 +1,4 @@
+#Written by Reid McIlroy-Young for Dr. John McLevey, University of Waterloo 2015
 import metaknowledge
 import metaknowledge.journalAbbreviations
 import networkx as nx
@@ -11,6 +12,7 @@
 
 #TODO: Figure output name stuff
 
+#These are used when the user goes to the interactive terminal, so they must be global
 RC = None
 G = None
 
@@ -21,7 +23,7 @@ def argumentParser():
     parser.add_argument("--name", "-n",  default = False, help = "The name used for the recordCollection and resulting files.")
     parser.add_argument("--debug", "-d", action = 'store_true', default = False, help = "Enables debug messages.")
     parser.add_argument("--progress", "-p", action = 'store_true' ,default = False, help = "Progress bar mode, shows progress bars where appropriate")
-    parser.add_argument("--suffix", "-s", default = '', help = "Progress bar mode, shows progress bars where appropriate")
+    parser.add_argument("--suffix", "-s", default = '', help = "The suffix of the WOS files you wish to extract Records from, by default all files are used and those that do not have Records are skipped")
     return parser.parse_args()
 
 def yesorNo(prompt):
@@ -81,7 +83,7 @@ def getOutputName(clargs, suffix, prompt = "What do you wish to call the output
         else:
             return s
 
-def getTag(prompt, nMode = False):
+def Tag(prompt, nMode = False):
     retTag = input(prompt).upper()
     if retTag in metaknowledge.tagsAndNames:
         return retTag
@@ -90,7 +92,7 @@ def getTag(prompt, nMode = False):
             return False
         else:
             print("{} is not a valid tag, please try again".format(retTag))
-            return getTag(prompt, nMode = nMode)
+            return Tag(prompt, nMode = nMode)
 
 def getNum(prompt):
     retNum = input(prompt)
@@ -218,21 +220,21 @@ def getNetwork(clargs, inRC):
     ])
     netID = int(inputMenu(netsDict, header = "What type of network do you wish to create?", promptMsg = "Input the number corresponding to the type of network you wish to generate? "))
     if netID == 1:
-        otg = getTag("What is the tag to use for the network? ")
+        otg = Tag("What is the tag to use for the network? ")
         print("Generating a network using the {0} tag.".format(otg))
         return inRC.oneModeNetwork(otg)
     elif netID == 2:
-        tg1 = getTag("What is the first tag to use for the network? ")
-        tg2 = getTag("And the second tag? ")
+        tg1 = Tag("What is the first tag to use for the network? ")
+        tg2 = Tag("And the second tag? ")
         print("Generating a network using the {0} and {1} tags.".format(tg1, tg2))
         return inRC.twoModeNetwork(tg1, tg2)
     elif netID == 3:
         tgs = []
-        tgs.append(getTag("What is the first tag to use for the network? "))
-        innertag = getTag("And the next tag (leave blank to continue)? ", nMode = True)
+        tgs.append(Tag("What is the first tag to use for the network? "))
+        innertag = Tag("And the next tag (leave blank to continue)? ", nMode = True)
         while innertag:
             tgs.append(innertag)
-            innertag = getTag("And the next tag (leave blank to continue)? ", nMode = True)
+            innertag = Tag("And the next tag (leave blank to continue)? ", nMode = True)
         print("Generating a network using the {0} and {1} tags".format(', '.join(tgs[:-1]), tgs[-1]))
         return inRC.nModeNetwork(tgs)
     elif netID == 4:
@@ -260,18 +262,23 @@ def getThresholds(clargs, grph):
     if thresID == 0:
         return grph
     elif thresID == 1:
-        return getThresholds(clargs, metaknowledge.drop_nodesByDegree(grph, minDegree = 1))
+        metaknowledge.dropNodesByDegree(grph, minDegree = 1)
+        return getThresholds(clargs, grph)
     elif thresID == 2:
-        return getThresholds(clargs, metaknowledge.drop_edges(grph, dropSelfLoops = True))
+        metaknowledge.dropEdges(grph, dropSelfLoops = True)
+        return getThresholds(clargs, grph)
     elif thresID == 3:
-        return getThresholds(clargs, metaknowledge.drop_edges(grph, minWeight = getNum("What is the minumum weight for an edge to be included? ")))
+        metaknowledge.dropEdges(grph, minWeight = getNum("What is the minumum weight for an edge to be included? "))
+        return getThresholds(clargs, grph)
     elif thresID == 4:
-        return getThresholds(clargs, metaknowledge.drop_edges(grph, minWeight = getNum("What is the maximum weight for an edge to be included? ")))
+        metaknowledge.dropEdges(grph, minWeight = getNum("What is the maximum weight for an edge to be included? "))
+        return getThresholds(clargs, grph)
     elif thresID == 5:
-        return getThresholds(clargs, metaknowledge.drop_nodesByDegree(grph, minDegree = getNum("What is the minumum degree for an edge to be included? ")))
+        metaknowledge.dropNodesByDegree(grph, minDegree = getNum("What is the minumum degree for an edge to be included? "))
+        return getThresholds(clargs, grph)
     else:
-        return getThresholds(clargs, metaknowledge.drop_nodesByDegree(grph, minDegree = getNum("What is the maximum degree for an edge to be included? ")))
-
+        metaknowledge.dropNodesByDegree(grph, minDegree = getNum("What is the maximum degree for an edge to be included? "))
+        return getThresholds(clargs, grph)
 
 def  outputNetwork(clargs, grph):
     outDict = collections.OrderedDict([
@@ -296,15 +303,15 @@ def  outputNetwork(clargs, grph):
         while True:
             try:
                 outName = getOutputName(clargs, '', checking = False)
-                metaknowledge.write_graph(grph, outName)
+                metaknowledge.writeGraph(grph, outName)
             except OSError:
                 if clargs.name:
-                    metaknowledge.write_graph(grph, outName, overwrite = True)
+                    metaknowledge.writeGraph(grph, outName, overwrite = True)
                     break
                 else:
                     overWrite = yesorNo("{}, overwrite (y/n)? ")
                     if overWrite:
-                        metaknowledge.write_graph(grph, outName, overwrite = True)
+                        metaknowledge.writeGraph(grph, outName, overwrite = True)
                         break
                     else:
                         pass
@@ -313,13 +320,13 @@ def  outputNetwork(clargs, grph):
 
     elif outID == 2:
         outName = getOutputName(clargs, '.csv')
-        metaknowledge.write_edgeList(grph, outName)
+        metaknowledge.writeEdgeList(grph, outName)
     elif outID == 3:
         outName = getOutputName(clargs, '.csv')
-        metaknowledge.write_nodeAttributeFile(grph, outName)
+        metaknowledge.writeNodeAttributeFile(grph, outName)
     else:
         outName = getOutputName(clargs, '.graphml')
-        nx.write_graphml(grph, outName)
+        nx.writeGraphml(grph, outName)
 
 def mkCLI():
     try:

metaknowledge/bin/metaknowledgeDocsGen.py

@@ -1,21 +1,21 @@
-#!/usr/bin/env python3
-
+#Written by Reid McIlroy-Young for Dr. John McLevey, University of Waterloo 2015
 """This is intended for metaknowledge only and may not work with anything else"""
 
 import inspect
 import argparse
 import os
 import time
 import metaknowledge
+import metaknowledge.tagProcessing
 import importlib
 import re
 
-documentedModules = ['visual', 'journalAbbreviations', 'tagProcessing']
+documentedModules = ['contour', 'journalAbbreviations', 'tagProcessing']
 
 docsPrefix = time.strftime("%Y-%m-%d-")
 
 blurbDict = {
-    'visual' : "A nicer matplotlib graph visualizer and contour plot",
+    'contour' : "A nicer matplotlib graph visualizer and contour plot",
     'tagProcessing' : "All the tags and how they are handled",
     'journalAbbreviations' : "Look here to get your J9 database",
     'Citation' : "Citation are special, here is how they are handled",
@@ -25,7 +25,7 @@
 
 singleFileYAML = """---
 layout: page
-title: ""
+title: Full Documentation
 author:
 - name: Reid McIlroy-Young
   department:
@@ -90,7 +90,7 @@ def cleanargs(obj, basic = False):
         return '()'
 
 def makeUrls(s):
-    return "[{0}]({{{{ site.baseurl }}}}{{% post_url /docs/{1}{2} %}}#{3})".format(s.group(1), docsPrefix, s.group(2), s.group(3))
+    return "[{0}]({{{{ site.baseurl }}}}{{{{ page.url }}}}#{3})".format(s.group(1), docsPrefix, s.group(2), s.group(3))
 
 def makeSingleFileUrls(s):
     return "[{0}](#{1})".format(s.group(1), s.group(3))
@@ -137,7 +137,7 @@ def makeLine():
     "border-top-style: solid;",
     "border-bottom-style: solid;",
     ]
-    return('<hr style="{}">'.format(''.join(style)))
+    return '<hr style="{}">'.format(''.join(style))
 
 def makeTable(entries, header = '', prefix = '', withBlurbs = False, bigTable = False):
     ents = []
@@ -165,26 +165,31 @@ def writeFunc(fn, f, prefix = '', level = 5, singleFile = False):
         f.write("# Needs to be written\n\n")
         print("\033[93m{0}{1} had no docs\033[0m".format(prefix, fn[0]))
 
-def writeClass(cl, f, prefix = '', level = 4, singleFile = False):
-    f.write(makeTitle(prefix, cl[0], cleanargs(cl[1].__init__), singleFile = singleFile))
+def writeClass(cl, f, prefix = '', level = 4, singleFile = False, exceptMode = False):
+    f.write(makeTitle(prefix, cl[0], "(_{}_)".format(cl[1].__bases__[0].__name__), singleFile = singleFile))
+    if not exceptMode:
+        f.write(makeTitle(prefix, "{}.__init__".format(cl[0]), cleanargs(cl[1].__init__), singleFile = singleFile))
     try:
         f.write(cleanedDoc(cl[1], lvl = level, singleFile = singleFile))
     except AttributeError:
         f.write("# Needs to be written\n\n")
         print("\033[93m{0}{1} had no docs\033[0m".format(prefix, cl[0]))
 
-def proccessClass(cl, f, singleFile = False):
-    writeClass(cl, f, singleFile = singleFile)
+def proccessClass(cl, f, singleFile = False, exceptMode = False):
+    writeClass(cl, f, singleFile = singleFile, exceptMode = exceptMode)
     baseMems = inspect.getmembers(cl[1].__bases__[0])
     funcs = []
+    if singleFile:
+        f.write(makeLine())
     for m in sorted(inspect.getmembers(cl[1]), key = getLineNumber):
-        if m[0][0] == '_' or m in baseMems:
+        if m[0][0] == '_' or m in baseMems or m[0] == 'with_traceback':
             pass
         elif inspect.isfunction(m[1]):
             funcs.append(m)
-    f.write(makeTable(funcs, prefix = cl[0], header = "The {} class has the following methods:".format(cl[0])))
-    for m in funcs:
-        writeFunc(m, f, prefix = '{}.'.format(cl[0], singleFile = singleFile))
+    if len(m) > 0 and not exceptMode:
+        f.write(makeTable(funcs, prefix = cl[0], header = "\nThe {} class has the following methods:".format(cl[0])))
+        for m in funcs:
+            writeFunc(m, f, prefix = '{}.'.format(cl[0], singleFile = singleFile))
 
 def writeClassFile(name, typ, targetFile = None, singleFile = False):
     fname = docsPrefix + "{}.md".format(name)
@@ -216,8 +221,11 @@ def writeModuleFile(mod, targetFile = None, singleFile = False):
             funcs.append(m)
     if mod != "tagProcessing":
         f.write(makeTable(funcs, prefix = mod, header = "The {} module provides the following functions:".format(mod)))
-    for fn in funcs:
-        writeFunc(fn, f, prefix = "{}.".format(mod))
+        for fn in funcs:
+            writeFunc(fn, f, prefix = "{}.".format(mod))
+    else:
+        for fn in metaknowledge.tagProcessing.tagToFunc.items():
+            writeFunc((metaknowledge.tagToFull(fn[0]), fn[1]), f, prefix = "{}.".format(mod))
     if targetFile is None:
         f.write("\n{% include docsFooter.md %}")
         f.close()
@@ -239,11 +247,8 @@ def writeMainBody(funcs, vrs, exceptions, targetFile = None, singleFile = False)
         writeFunc(fnc, f)
     first = True
     for excpt in exceptions:
-        if first:
-            first = False
-        else:
-            f.write(makeLine() + "\n\n")
-        proccessClass(excpt, f)
+        f.write(makeLine() + "\n\n")
+        proccessClass(excpt, f, exceptMode = True)
     if targetFile is None:
         f.write("\n{% include docsFooter.md %}")
         f.close()

metaknowledge/bin/metaknowledgeMdToNb.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python3
-
+#Written by Reid McIlroy-Young for Dr. John McLevey, University of Waterloo 2015
 import argparse
 import re
 import os.path

metaknowledge/blondel.py

@@ -1,3 +1,6 @@
+#Written by Reid McIlroy-Young for Dr. John McLevey, University of Waterloo 2015
+#Better implementations can be found on Pypi so this has been discontinued
+#None of these functions are tested
 import metaknowledge
 from .graphHelpers import _ProgressBar