From 3f07a57e74dd3a01209121319ea510845b7abee1 Mon Sep 17 00:00:00 2001 From: tracetidwell Date: Sun, 26 Apr 2020 12:05:51 -0400 Subject: [PATCH 01/18] added functionality for module docstring --- .vscode/launch.json | 2 +- module_docstring.md | 33 +++++++++ package.json | 10 ++- src/docstring/docstring_factory.ts | 2 + src/docstring/template_data.ts | 14 +++- src/docstring/templates/ncr.mustache | 60 +++++++++++++++ src/docstring_parts.ts | 10 +++ src/generate_docstring.ts | 9 +++ src/parse/get_body.ts | 4 + src/parse/parse.ts | 7 +- src/parse/parse_parameters.ts | 105 ++++++++++++++++++++++++--- src/test/run_integration_tests.ts | 1 + 12 files changed, 239 insertions(+), 18 deletions(-) create mode 100644 module_docstring.md create mode 100644 src/docstring/templates/ncr.mustache diff --git a/.vscode/launch.json b/.vscode/launch.json index 0ef00fd..e4f356c 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -28,4 +28,4 @@ "preLaunchTask": "npm: compile" } ] -} +} \ No newline at end of file diff --git a/module_docstring.md b/module_docstring.md new file mode 100644 index 0000000..2a41b23 --- /dev/null +++ b/module_docstring.md @@ -0,0 +1,33 @@ +# Changes made to autoDocstring to incorporate module level docstrings +- src/parse/get_body.ts + - getBody(): if docstring is being added at first line of document, return entire document as body +- src/docstring_parts.ts + - DocstringParts: added Class[] and Method[] attributes + - Class: Created Class interface + - Method: Created Method interface +- src/parse/parse_parameters.ts + - parseParameters(): added positionLine argument, if positionLine == 0 return only top-level classes and methods, otherwise return normal DocstringParts + - parseClasses(): returns list of top-level classes + - parseMethods(): returns list of top-level methods +- src/docstring/template_data.ts + - TemplateData: added classes and methods attributes, updated constructor to take list of classes and list of methods + - classesExist(): added method to check if classes exist, needed to parse template + - methodsExist(): added method to check if methods exist, needed to parse template +- src/docstring/templates/ncr.mustache + - Created template file that incorporates classes and methods +- package.json + - contributes.configuration.properties.autoDocstring.docstringFormat.enum + - Added "ncr" template (this should probably be removed) + - contributes.commands[0].title + - Changed to "Generate Docstring2" just to avoid any confusion, will change back when ready for PR + +# To run the code +- Open the autoDocstring folder in VS Code +- Open the command palette (Ctrl+Shift+P) and type `settings.json` to open the settings configuration file +- Add the line `"autoDocstring.customTemplatePath": "/home/shared/Projects/autoDocstring/src/docstring/templates/ncr.mustache"` and make sure the path to the ncr.mustache template is correct for your computer. +- Save and close settings.json. +- Press F5 +- Click "Debug Anyway" +- When the new screen opens, open a folder to any project you want to add a module level docstring to +- On line 1, click Ctrl+Shift+P or input """ and press enter. +- The docstring should populate and list all top-level classes and methods in the file. \ No newline at end of file diff --git a/package.json b/package.json index 44c13a5..e2c63d8 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ "commands": [ { "command": "autoDocstring.generateDocstring", - "title": "Generate Docstring" + "title": "Generate Docstring2" } ], "keybindings": [ @@ -70,12 +70,13 @@ "properties": { "autoDocstring.docstringFormat": { "type": "string", - "default": "default", + "default": "ncr", "enum": [ "default", "google", "sphinx", - "numpy" + "numpy", + "ncr" ], "description": "Which docstring format to use." }, @@ -141,6 +142,7 @@ "vscode-test": "^1.3.0" }, "dependencies": { - "mustache": "^3.2.1" + "mustache": "^3.2.1", + "vscode": "^1.1.37" } } diff --git a/src/docstring/docstring_factory.ts b/src/docstring/docstring_factory.ts index b7a293a..2214b32 100644 --- a/src/docstring/docstring_factory.ts +++ b/src/docstring/docstring_factory.ts @@ -24,6 +24,8 @@ export class DocstringFactory { this.includeDescription = includeDescription; this.template = template; + // console.log('template') + // console.log(this.template) } public generateDocstring(docstringParts: DocstringParts, indentation = ""): string { diff --git a/src/docstring/template_data.ts b/src/docstring/template_data.ts index dba5c1b..75ccac0 100644 --- a/src/docstring/template_data.ts +++ b/src/docstring/template_data.ts @@ -1,4 +1,4 @@ -import { Argument, Decorator, DocstringParts, Exception, KeywordArgument, Returns, Yields } from "../docstring_parts"; +import { Argument, Decorator, DocstringParts, Exception, KeywordArgument, Returns, Yields, Class, Method } from "../docstring_parts"; export class TemplateData { public name: string; @@ -8,6 +8,8 @@ export class TemplateData { public exceptions: Exception[]; public returns: Returns; public yields: Yields; + public classes: Class[]; + public methods: Method[]; private includeName: boolean; private includeExtendedSummary: boolean; @@ -22,6 +24,8 @@ export class TemplateData { this.exceptions = docstringParts.exceptions; this.returns = docstringParts.returns; this.yields = docstringParts.yields; + this.classes = docstringParts.classes; + this.methods = docstringParts.methods; this.includeName = includeName; this.includeExtendedSummary = includeExtendedSummary; @@ -88,6 +92,14 @@ export class TemplateData { return this.yields != undefined; } + public classesExist(): boolean { + return this.classes.length > 0; + } + + public methodsExist(): boolean { + return this.methods.length > 0; + } + private removeTypes(): void { for (const arg of this.args) { arg.type = undefined; diff --git a/src/docstring/templates/ncr.mustache b/src/docstring/templates/ncr.mustache new file mode 100644 index 0000000..1cc59cc --- /dev/null +++ b/src/docstring/templates/ncr.mustache @@ -0,0 +1,60 @@ +{{! NCR Docstring Template }} +{{summaryPlaceholder}} + +{{extendedSummaryPlaceholder}} + +{{#classesExist}} +Classes +------- +{{#classes}} +{{name}} +{{/classes}} +{{/classesExist}} + +{{#methodsExist}} +Methods +------- +{{#methods}} +{{name}} +{{/methods}} +{{/methodsExist}} + +{{#parametersExist}} +Parameters +---------- +{{#args}} +{{var}} : {{typePlaceholder}} + {{descriptionPlaceholder}} +{{/args}} +{{#kwargs}} +{{var}} : {{typePlaceholder}}, optional + {{descriptionPlaceholder}}, by default {{&default}} +{{/kwargs}} +{{/parametersExist}} + +{{#returnsExist}} +Returns +------- +{{#returns}} +{{typePlaceholder}} + {{descriptionPlaceholder}} +{{/returns}} +{{/returnsExist}} + +{{#yieldsExist}} +Yields +------- +{{#yields}} +{{typePlaceholder}} + {{descriptionPlaceholder}} +{{/yields}} +{{/yieldsExist}} + +{{#exceptionsExist}} +Raises +------ +{{#exceptions}} +{{type}} + {{descriptionPlaceholder}} +{{/exceptions}} +{{/exceptionsExist}} diff --git a/src/docstring_parts.ts b/src/docstring_parts.ts index a1b6be3..089f684 100644 --- a/src/docstring_parts.ts +++ b/src/docstring_parts.ts @@ -7,6 +7,8 @@ export interface DocstringParts { exceptions: Exception[]; returns: Returns; yields: Yields; + classes: Class[]; + methods: Method[]; } export interface Decorator { @@ -35,3 +37,11 @@ export interface Returns { export interface Yields { type: string; } + +export interface Class { + name: string; +} + +export interface Method { + name: string; +} \ No newline at end of file diff --git a/src/generate_docstring.ts b/src/generate_docstring.ts index c89e782..2d3892f 100644 --- a/src/generate_docstring.ts +++ b/src/generate_docstring.ts @@ -49,6 +49,15 @@ export class AutoDocstring { ); const docstringParts = parse(document, position.line); + // console.log("classes"); + // for (const cls of docstringParts.classes) { + // console.log(cls.name) + // } + // console.log("methods"); + // for (const method of docstringParts.methods) { + // console.log(method.name) + // } + // this.log(`Classes parsed:\n${docstringParts.classes}`) const indentation = getDocstringIndentation(document, position.line); const docstring = docstringFactory.generateDocstring(docstringParts, indentation); diff --git a/src/parse/get_body.ts b/src/parse/get_body.ts index a51a499..8409ed3 100644 --- a/src/parse/get_body.ts +++ b/src/parse/get_body.ts @@ -4,6 +4,10 @@ export function getBody(document: string, linePosition: number): string[] { const lines = document.split("\n"); const body = []; + if (linePosition === 0) { + return lines; + } + let currentLineNum = linePosition; const originalIndentation = getBodyBaseIndentation(lines, linePosition); diff --git a/src/parse/parse.ts b/src/parse/parse.ts index 2ccbd61..6c16383 100644 --- a/src/parse/parse.ts +++ b/src/parse/parse.ts @@ -4,9 +4,14 @@ import { DocstringParts } from "../docstring_parts"; export function parse(document: string, positionLine: number): DocstringParts { const definition = getDefinition(document, positionLine); const body = getBody(document, positionLine); + // console.log(`Body parsed:\n${body}`) + // console.log("Body parsed") + // for (const line of body) { + // console.log(line) + // } const parameterTokens = tokenizeDefinition(definition); const functionName = getFunctionName(definition); - return parseParameters(parameterTokens, body, functionName); + return parseParameters(positionLine, parameterTokens, body, functionName); } diff --git a/src/parse/parse_parameters.ts b/src/parse/parse_parameters.ts index 2b7e23b..a4b51d4 100644 --- a/src/parse/parse_parameters.ts +++ b/src/parse/parse_parameters.ts @@ -1,15 +1,35 @@ import { guessType } from "."; -import { Argument, Decorator, DocstringParts, Exception, KeywordArgument, Returns, Yields } from "../docstring_parts"; - -export function parseParameters(parameterTokens: string[], body: string[], functionName: string): DocstringParts { - return { - name: functionName, - decorators: parseDecorators(parameterTokens), - args: parseArguments(parameterTokens), - kwargs: parseKeywordArguments(parameterTokens), - returns: parseReturn(parameterTokens, body), - yields: parseYields(parameterTokens, body), - exceptions: parseExceptions(body), +import { indentationOf } from "./utilities"; +import {getFunctionName} from "./get_function_name"; +import { Argument, Decorator, DocstringParts, Exception, KeywordArgument, Returns, Yields, Class, Method } from "../docstring_parts"; + +export function parseParameters(positionLine: number, parameterTokens: string[], body: string[], functionName: string): DocstringParts { + + if (positionLine === 0) { + return { + name: functionName, + decorators: [], + args: [], + kwargs: [], + returns: undefined, + yields: undefined, + exceptions: [], + classes: parseClasses(body), + methods: parseMethods(body) + }; + } + else { + return { + name: functionName, + decorators: parseDecorators(parameterTokens), + args: parseArguments(parameterTokens), + kwargs: parseKeywordArguments(parameterTokens), + returns: parseReturn(parameterTokens, body), + yields: parseYields(parameterTokens, body), + exceptions: parseExceptions(body), + classes: [], + methods: [] + }; }; } @@ -136,6 +156,69 @@ function parseExceptions(body: string[]): Exception[] { return exceptions; } +function parseClasses(body: string[]): Class[] { + const classes: Class[] = [] + const pattern = /(?:class)\s+(\w+)\s*\(/; + //const pattern = /(?:class)\s+(\w+)\s*\(/; + + for (const line of body) { + + if (indentationOf(line) === 0) { + + console.log("class indentation match") + console.log(line) + + const match = line.match(pattern); + + if (match == null) { + continue + } + + console.log("class match") + console.log(match) + let className = getFunctionName(line); + console.log(className) + + classes.push({ + name: className, + }); + } + } + return classes; +} + +function parseMethods(body: string[]): Method[] { + const methods: Class[] = [] + const pattern = /(?:def)\s+(\w+)\s*\(/; + + for (const line of body) { + + if (indentationOf(line) === 0) { + + // console.log("indentation = 0") + // console.log(line) + + const match = line.match(pattern); + + if (match == null) { + continue + } + + // console.log("matches regex") + // console.log(match) + + let methodName = getFunctionName(line); + // console.log("method name") + // console.log(methodName) + + methods.push({ + name: methodName, + }); + } + } + return methods; +} + export function inArray(item: type, array: type[]) { return array.some((x) => item === x); } diff --git a/src/test/run_integration_tests.ts b/src/test/run_integration_tests.ts index 699c415..627ec06 100644 --- a/src/test/run_integration_tests.ts +++ b/src/test/run_integration_tests.ts @@ -1,3 +1,4 @@ + import * as path from "path"; import { runTests } from "vscode-test"; From a8f0ef316e0d3035a46166cd68d933fd9b437acc Mon Sep 17 00:00:00 2001 From: Bryan Nonni Date: Sun, 26 Apr 2020 12:44:36 -0400 Subject: [PATCH 02/18] adding regex expressions for classes & methods --- src/parse/parse_parameters.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/parse/parse_parameters.ts b/src/parse/parse_parameters.ts index a4b51d4..c163cd2 100644 --- a/src/parse/parse_parameters.ts +++ b/src/parse/parse_parameters.ts @@ -158,7 +158,7 @@ function parseExceptions(body: string[]): Exception[] { function parseClasses(body: string[]): Class[] { const classes: Class[] = [] - const pattern = /(?:class)\s+(\w+)\s*\(/; + const pattern = /(class) {1}(([A-Z]{1,2})*[a-z]*)*:{1}/g; //const pattern = /(?:class)\s+(\w+)\s*\(/; for (const line of body) { @@ -189,7 +189,7 @@ function parseClasses(body: string[]): Class[] { function parseMethods(body: string[]): Method[] { const methods: Class[] = [] - const pattern = /(?:def)\s+(\w+)\s*\(/; + const pattern = /(def) {1}([a-z\_])*\(([a-zA-Z, \=\_])+\):/g; for (const line of body) { From 6ad28ab867d549c4f41f12a17d6976e23e01b365 Mon Sep 17 00:00:00 2001 From: Bryan Nonni Date: Sun, 26 Apr 2020 13:08:23 -0400 Subject: [PATCH 03/18] Reversing regex --- src/parse/get_function_name.ts | 2 +- src/parse/parse_parameters.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/parse/get_function_name.ts b/src/parse/get_function_name.ts index 063e0a1..e1504a8 100644 --- a/src/parse/get_function_name.ts +++ b/src/parse/get_function_name.ts @@ -1,5 +1,5 @@ export function getFunctionName(functionDefinition: string): string { - const pattern = /(?:def|class)\s+(\w+)\s*\(/; + const pattern = /(?:def|class)\s+(\w+)\s*\(/g; const match = pattern.exec(functionDefinition); diff --git a/src/parse/parse_parameters.ts b/src/parse/parse_parameters.ts index c163cd2..a4b51d4 100644 --- a/src/parse/parse_parameters.ts +++ b/src/parse/parse_parameters.ts @@ -158,7 +158,7 @@ function parseExceptions(body: string[]): Exception[] { function parseClasses(body: string[]): Class[] { const classes: Class[] = [] - const pattern = /(class) {1}(([A-Z]{1,2})*[a-z]*)*:{1}/g; + const pattern = /(?:class)\s+(\w+)\s*\(/; //const pattern = /(?:class)\s+(\w+)\s*\(/; for (const line of body) { @@ -189,7 +189,7 @@ function parseClasses(body: string[]): Class[] { function parseMethods(body: string[]): Method[] { const methods: Class[] = [] - const pattern = /(def) {1}([a-z\_])*\(([a-zA-Z, \=\_])+\):/g; + const pattern = /(?:def)\s+(\w+)\s*\(/; for (const line of body) { From 07cb44eebaf2c26e79ac728e78aa8941a5aa6cc4 Mon Sep 17 00:00:00 2001 From: Bryan Nonni Date: Sun, 26 Apr 2020 13:14:02 -0400 Subject: [PATCH 04/18] removing a global flag --- src/parse/get_function_name.ts | 2 +- src/parse/parse_parameters.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/parse/get_function_name.ts b/src/parse/get_function_name.ts index e1504a8..063e0a1 100644 --- a/src/parse/get_function_name.ts +++ b/src/parse/get_function_name.ts @@ -1,5 +1,5 @@ export function getFunctionName(functionDefinition: string): string { - const pattern = /(?:def|class)\s+(\w+)\s*\(/g; + const pattern = /(?:def|class)\s+(\w+)\s*\(/; const match = pattern.exec(functionDefinition); diff --git a/src/parse/parse_parameters.ts b/src/parse/parse_parameters.ts index a4b51d4..3d1a808 100644 --- a/src/parse/parse_parameters.ts +++ b/src/parse/parse_parameters.ts @@ -189,7 +189,7 @@ function parseClasses(body: string[]): Class[] { function parseMethods(body: string[]): Method[] { const methods: Class[] = [] - const pattern = /(?:def)\s+(\w+)\s*\(/; + const pattern = /(def)\s+(\w+)\s*\(/; for (const line of body) { From 125110b2c784b839ac7e0d593ff358ef8d452cd8 Mon Sep 17 00:00:00 2001 From: tracetidwell Date: Sun, 26 Apr 2020 14:58:18 -0400 Subject: [PATCH 05/18] added src/parse/get_class_name to fix regex issue for classes --- src/parse/get_class_name.ts | 17 +++++++++++++++++ src/parse/parse_parameters.ts | 26 ++++++++++++-------------- 2 files changed, 29 insertions(+), 14 deletions(-) create mode 100644 src/parse/get_class_name.ts diff --git a/src/parse/get_class_name.ts b/src/parse/get_class_name.ts new file mode 100644 index 0000000..5b9b640 --- /dev/null +++ b/src/parse/get_class_name.ts @@ -0,0 +1,17 @@ +export function getClassName(functionDefinition: string): string { + const pattern1 = /(?:class)\s+(\w+)\s*\(/; + const pattern2 = /(?:class)\s+(\w+)/; + + const match1 = pattern1.exec(functionDefinition); + const match2 = pattern2.exec(functionDefinition); + + if (match1 != undefined && match1[1] != undefined) { + return match1[1]; + } + else if (match2 != undefined && match2[1] != undefined) { + return match2[1] + } + else { + return ""; + } +} diff --git a/src/parse/parse_parameters.ts b/src/parse/parse_parameters.ts index a4b51d4..354af4d 100644 --- a/src/parse/parse_parameters.ts +++ b/src/parse/parse_parameters.ts @@ -1,6 +1,7 @@ import { guessType } from "."; import { indentationOf } from "./utilities"; -import {getFunctionName} from "./get_function_name"; +import { getFunctionName } from "./get_function_name"; +import { getClassName } from "./get_class_name"; import { Argument, Decorator, DocstringParts, Exception, KeywordArgument, Returns, Yields, Class, Method } from "../docstring_parts"; export function parseParameters(positionLine: number, parameterTokens: string[], body: string[], functionName: string): DocstringParts { @@ -158,8 +159,7 @@ function parseExceptions(body: string[]): Exception[] { function parseClasses(body: string[]): Class[] { const classes: Class[] = [] - const pattern = /(?:class)\s+(\w+)\s*\(/; - //const pattern = /(?:class)\s+(\w+)\s*\(/; + const pattern = /(?:class)\s/; for (const line of body) { @@ -170,18 +170,16 @@ function parseClasses(body: string[]): Class[] { const match = line.match(pattern); - if (match == null) { - continue + if (match != null) { + console.log("class match2") + console.log(match) + let className = getClassName(line); + console.log(className) + + classes.push({ + name: className, + }); } - - console.log("class match") - console.log(match) - let className = getFunctionName(line); - console.log(className) - - classes.push({ - name: className, - }); } } return classes; From 180141e6ca134b6a5283e997ba9f32ca06ec8fe4 Mon Sep 17 00:00:00 2001 From: tracetidwell Date: Sun, 26 Apr 2020 14:59:51 -0400 Subject: [PATCH 06/18] added src/parse/get_class_name to fix regex issue with classes --- src/parse/parse_parameters.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parse/parse_parameters.ts b/src/parse/parse_parameters.ts index 65b9344..5cb4597 100644 --- a/src/parse/parse_parameters.ts +++ b/src/parse/parse_parameters.ts @@ -171,7 +171,7 @@ function parseClasses(body: string[]): Class[] { const match = line.match(pattern); if (match != null) { - console.log("class match2") + console.log("class match") console.log(match) let className = getClassName(line); console.log(className) From 739b166a91a75b766806f1d60076099702b93d6d Mon Sep 17 00:00:00 2001 From: tracetidwell Date: Mon, 27 Apr 2020 20:28:39 -0400 Subject: [PATCH 07/18] functionality for module and class docstrings. looks good, needs testing --- src/docstring/template_data.ts | 14 +- src/docstring/templates/ncr.mustache | 19 +- src/docstring_parts.ts | 6 + src/parse/get_body.ts | 21 +- src/parse/get_class_name.ts | 3 +- src/parse/get_definition.ts | 49 +++ src/parse/get_docstring_type.ts | 20 ++ src/parse/index.ts | 1 + src/parse/parse.ts | 13 +- src/parse/parse_parameters.ts | 78 ++++- .../python_test_files/depth_interpolator.py | 300 ++++++++++++++++++ .../python_test_files/dlc_project.py | 206 ++++++++++++ 12 files changed, 701 insertions(+), 29 deletions(-) create mode 100644 src/parse/get_docstring_type.ts create mode 100644 src/test/integration/python_test_files/depth_interpolator.py create mode 100644 src/test/integration/python_test_files/dlc_project.py diff --git a/src/docstring/template_data.ts b/src/docstring/template_data.ts index 75ccac0..a108f79 100644 --- a/src/docstring/template_data.ts +++ b/src/docstring/template_data.ts @@ -1,4 +1,4 @@ -import { Argument, Decorator, DocstringParts, Exception, KeywordArgument, Returns, Yields, Class, Method } from "../docstring_parts"; +import { Argument, Decorator, DocstringParts, Exception, KeywordArgument, Returns, Yields, Class, Method, Attribute } from "../docstring_parts"; export class TemplateData { public name: string; @@ -10,6 +10,7 @@ export class TemplateData { public yields: Yields; public classes: Class[]; public methods: Method[]; + public attributes: Attribute[]; private includeName: boolean; private includeExtendedSummary: boolean; @@ -26,6 +27,7 @@ export class TemplateData { this.yields = docstringParts.yields; this.classes = docstringParts.classes; this.methods = docstringParts.methods; + this.attributes = docstringParts.attributes; this.includeName = includeName; this.includeExtendedSummary = includeExtendedSummary; @@ -100,6 +102,10 @@ export class TemplateData { return this.methods.length > 0; } + public attributesExist(): boolean { + return this.attributes.length > 0; + } + private removeTypes(): void { for (const arg of this.args) { arg.type = undefined; @@ -140,5 +146,11 @@ export class TemplateData { if (yields != undefined && yields.type == undefined) { yields.type = placeholder; } + + for (const attribute of this.attributes) { + if (attribute.type === undefined) { + attribute.type = placeholder; + } + } } } diff --git a/src/docstring/templates/ncr.mustache b/src/docstring/templates/ncr.mustache index 1cc59cc..59aa18c 100644 --- a/src/docstring/templates/ncr.mustache +++ b/src/docstring/templates/ncr.mustache @@ -25,19 +25,32 @@ Parameters {{#args}} {{var}} : {{typePlaceholder}} {{descriptionPlaceholder}} + {{/args}} {{#kwargs}} {{var}} : {{typePlaceholder}}, optional {{descriptionPlaceholder}}, by default {{&default}} + {{/kwargs}} {{/parametersExist}} +{{#attributesExist}} +Attributes +---------- +{{#attributes}} +{{var}} : {{typePlaceholder}} + {{descriptionPlaceholder}} + +{{/attributes}} +{{/attributesExist}} + {{#returnsExist}} Returns ------- {{#returns}} -{{typePlaceholder}} +return_val : {{typePlaceholder}} {{descriptionPlaceholder}} + {{/returns}} {{/returnsExist}} @@ -45,8 +58,9 @@ Returns Yields ------- {{#yields}} -{{typePlaceholder}} +yield_val : {{typePlaceholder}} {{descriptionPlaceholder}} + {{/yields}} {{/yieldsExist}} @@ -56,5 +70,6 @@ Raises {{#exceptions}} {{type}} {{descriptionPlaceholder}} + {{/exceptions}} {{/exceptionsExist}} diff --git a/src/docstring_parts.ts b/src/docstring_parts.ts index 089f684..e7e78c6 100644 --- a/src/docstring_parts.ts +++ b/src/docstring_parts.ts @@ -9,6 +9,7 @@ export interface DocstringParts { yields: Yields; classes: Class[]; methods: Method[]; + attributes: Attribute[]; } export interface Decorator { @@ -44,4 +45,9 @@ export interface Class { export interface Method { name: string; +} + +export interface Attribute { + var: string; + type: string; } \ No newline at end of file diff --git a/src/parse/get_body.ts b/src/parse/get_body.ts index 8409ed3..d83f38e 100644 --- a/src/parse/get_body.ts +++ b/src/parse/get_body.ts @@ -1,18 +1,28 @@ import { blankLine, indentationOf } from "./utilities"; -export function getBody(document: string, linePosition: number): string[] { +export function getBody(docstringType: string, document: string, linePosition: number): string[] { const lines = document.split("\n"); const body = []; + let regex = '\s*def \w+' - if (linePosition === 0) { + if (docstringType === 'module') { return lines; } + else if (docstringType === 'class') { + let regex = '.'; + } let currentLineNum = linePosition; - const originalIndentation = getBodyBaseIndentation(lines, linePosition); + const originalIndentation = getBodyBaseIndentation(lines, linePosition, regex); + // console.log("original indentation") + // console.log(originalIndentation) + // console.log(currentLineNum) while (currentLineNum < lines.length) { const line = lines[currentLineNum]; + // console.log("current indentation") + // console.log(indentationOf(line)) + // console.log(currentLineNum) if (blankLine(line)) { currentLineNum++; @@ -30,9 +40,10 @@ export function getBody(document: string, linePosition: number): string[] { return body; } -function getBodyBaseIndentation(lines: string[], linePosition: number): number { +function getBodyBaseIndentation(lines: string[], linePosition: number, regex: string): number { let currentLineNum = linePosition; - const functionDefRegex = /\s*def \w+/; + //const functionDefRegex = /\s*def \w+/; + const functionDefRegex = RegExp(regex) while (currentLineNum < lines.length) { const line = lines[currentLineNum]; diff --git a/src/parse/get_class_name.ts b/src/parse/get_class_name.ts index 5b9b640..d09634c 100644 --- a/src/parse/get_class_name.ts +++ b/src/parse/get_class_name.ts @@ -1,5 +1,6 @@ export function getClassName(functionDefinition: string): string { - const pattern1 = /(?:class)\s+(\w+)\s*\(/; + //const pattern1 = /(?:class)\s+(\w+)\s*\(/; + const pattern1 = /(?:class)\s+(\w+)/; const pattern2 = /(?:class)\s+(\w+)/; const match1 = pattern1.exec(functionDefinition); diff --git a/src/parse/get_definition.ts b/src/parse/get_definition.ts index ddc0df3..f0d1436 100644 --- a/src/parse/get_definition.ts +++ b/src/parse/get_definition.ts @@ -10,6 +10,8 @@ export function getDefinition(document: string, linePosition: number): string { let currentLineNum = linePosition - 1; const originalIndentation = indentationOf(lines[currentLineNum]); + // console.log("original indentation") + // console.log(originalIndentation) while (currentLineNum >= 0) { const line = lines[currentLineNum]; @@ -22,5 +24,52 @@ export function getDefinition(document: string, linePosition: number): string { currentLineNum -= 1; } + //const classPattern = /(?:class)/; + const classPattern = /(?:class)\s+(\w+)/; + const classMatch = classPattern.exec(definition); + + if (classMatch != undefined && classMatch[0] != undefined) { + currentLineNum += 2; + let definition = classMatch[0]; + // console.log("new definition") + // console.log(definition) + // console.log(classMatch) + // console.log("current line num") + // console.log(currentLineNum) + // console.log("current indentation") + // console.log(indentationOf(lines[currentLineNum])) + // console.log(lines[currentLineNum]) + const initPattern = /(?:def __init__)/; + + while (currentLineNum < lines.length) { + const line = lines[currentLineNum]; + // console.log("current indentation") + // console.log(indentationOf(line)) + const initMatch = initPattern.exec(line) + + if (initMatch != undefined && initMatch[0] != undefined) { + const newIndentation = indentationOf(lines[currentLineNum]); + + while (currentLineNum < lines.length) { + const line = lines[currentLineNum]; + definition += line.trim(); + // console.log(definition) + + if (indentationOf(line) < newIndentation || blankLine(line)) { + return definition; + } + + currentLineNum += 1; + } + + } + else if (indentationOf(line) <= originalIndentation && !blankLine(line)) { + return definition; + } + currentLineNum += 1; + } + } + // console.log("last definition") + // console.log(definition) return definition; } diff --git a/src/parse/get_docstring_type.ts b/src/parse/get_docstring_type.ts new file mode 100644 index 0000000..15774dd --- /dev/null +++ b/src/parse/get_docstring_type.ts @@ -0,0 +1,20 @@ +export function getDocstringType(functionDefinition: string, linePosition: number): string { + + const class_pattern = /(?:class)/; + const class_match = class_pattern.exec(functionDefinition); + console.log(class_match) + console.log(functionDefinition) + + if (linePosition === 0) { + return "module" + } + + else if (class_match != undefined && class_match[0] != undefined) { + return "class"; + } + + else { + return "method"; + } + +} diff --git a/src/parse/index.ts b/src/parse/index.ts index 992fe29..86e3b9c 100644 --- a/src/parse/index.ts +++ b/src/parse/index.ts @@ -8,3 +8,4 @@ export { validDocstringPrefix } from "./valid_docstring_prefix"; export { parse } from "./parse"; export { parseParameters } from "./parse_parameters"; export { tokenizeDefinition } from "./tokenize_definition"; +export { getDocstringType } from "./get_docstring_type"; diff --git a/src/parse/parse.ts b/src/parse/parse.ts index 6c16383..a6c5a3c 100644 --- a/src/parse/parse.ts +++ b/src/parse/parse.ts @@ -1,17 +1,12 @@ -import { getBody, getDefinition, getFunctionName, parseParameters, tokenizeDefinition} from "."; +import { getBody, getDefinition, getFunctionName, parseParameters, tokenizeDefinition, getDocstringType } from "."; import { DocstringParts } from "../docstring_parts"; export function parse(document: string, positionLine: number): DocstringParts { const definition = getDefinition(document, positionLine); - const body = getBody(document, positionLine); - // console.log(`Body parsed:\n${body}`) - // console.log("Body parsed") - // for (const line of body) { - // console.log(line) - // } - + const docstringType = getDocstringType(definition, positionLine) + const body = getBody(docstringType, document, positionLine); const parameterTokens = tokenizeDefinition(definition); const functionName = getFunctionName(definition); - return parseParameters(positionLine, parameterTokens, body, functionName); + return parseParameters(docstringType, parameterTokens, body, functionName); } diff --git a/src/parse/parse_parameters.ts b/src/parse/parse_parameters.ts index 5cb4597..bf189fb 100644 --- a/src/parse/parse_parameters.ts +++ b/src/parse/parse_parameters.ts @@ -2,11 +2,11 @@ import { guessType } from "."; import { indentationOf } from "./utilities"; import { getFunctionName } from "./get_function_name"; import { getClassName } from "./get_class_name"; -import { Argument, Decorator, DocstringParts, Exception, KeywordArgument, Returns, Yields, Class, Method } from "../docstring_parts"; +import { Argument, Decorator, DocstringParts, Exception, KeywordArgument, Returns, Yields, Class, Method, Attribute } from "../docstring_parts"; -export function parseParameters(positionLine: number, parameterTokens: string[], body: string[], functionName: string): DocstringParts { +export function parseParameters(docstringType: string, parameterTokens: string[], body: string[], functionName: string): DocstringParts { - if (positionLine === 0) { + if (docstringType === "module") { return { name: functionName, decorators: [], @@ -16,10 +16,11 @@ export function parseParameters(positionLine: number, parameterTokens: string[], yields: undefined, exceptions: [], classes: parseClasses(body), - methods: parseMethods(body) + methods: parseMethods(body), + attributes: [] }; } - else { + else if (docstringType === "method") { return { name: functionName, decorators: parseDecorators(parameterTokens), @@ -29,7 +30,24 @@ export function parseParameters(positionLine: number, parameterTokens: string[], yields: parseYields(parameterTokens, body), exceptions: parseExceptions(body), classes: [], - methods: [] + methods: [], + attributes: [] + }; + } + else if (docstringType === "class") { + let args = parseArguments(parameterTokens); + let kwargs = parseKeywordArguments(parameterTokens); + return { + name: functionName, + decorators: parseDecorators(parameterTokens), + args: args, + kwargs: kwargs, + returns: undefined, + yields: undefined, + exceptions: parseExceptions(body), + classes: [], + methods: [], + attributes: parseAttributes(body, args, kwargs) }; }; } @@ -165,16 +183,16 @@ function parseClasses(body: string[]): Class[] { if (indentationOf(line) === 0) { - console.log("class indentation match") - console.log(line) + // console.log("class indentation match") + // console.log(line) const match = line.match(pattern); if (match != null) { - console.log("class match") - console.log(match) + // console.log("class match") + // console.log(match) let className = getClassName(line); - console.log(className) + // console.log(className) classes.push({ name: className, @@ -217,6 +235,35 @@ function parseMethods(body: string[]): Method[] { return methods; } +function parseAttributes(body: string[], args: Argument[], kwargs: KeywordArgument[]): Attribute[] { + const attributes: Attribute[] = []; + const pattern = /(?:self.)(\w+)(?:\s*:[^=]+)?\s*=\s*(.+)/; + //const pattern = /(?:self).(\w+)?\s*/ + + for (const line of body) { + // console.log(line) + const match = line.trim().match(pattern); + + // console.log(match) + + if (match == null) { + continue; + } + let var_ = match[1]; + let type_ = guessType(match[1]); + if (!containsAttribute(attributes, var_) && !containsAttribute(args, var_) && !containsAttribute(kwargs, var_)) { + attributes.push({ + var: var_, + type: type_ + }); + } + } + + // console.log(attributes) + + return attributes; +} + export function inArray(item: type, array: type[]) { return array.some((x) => item === x); } @@ -242,3 +289,12 @@ function parseFromBody(body: string[], pattern: RegExp): Returns | Yields { function isIterator(type: string): boolean { return type.startsWith("Generator") || type.startsWith("Iterator"); } + +function containsAttribute(attributes: Attribute[], name: string): boolean { + for (const attribute of attributes) { + if (attribute.var === name) { + return true; + } + } + return false; +} diff --git a/src/test/integration/python_test_files/depth_interpolator.py b/src/test/integration/python_test_files/depth_interpolator.py new file mode 100644 index 0000000..8d62b31 --- /dev/null +++ b/src/test/integration/python_test_files/depth_interpolator.py @@ -0,0 +1,300 @@ +from typing import Union, Tuple + +import cv2 +import numpy as np +import matplotlib.pyplot as plt +from scipy.interpolate import griddata + + +class DepthInterpolator: + + def run(self, fg_rgb_image: np.ndarray, fg_depth_image: np.ndarray, + bg_rgb_image: np.ndarray, bg_depth_image: np.ndarray, + debug: bool): + + pass + + def __repr__(self): + + return self.__class__.__name__ + + +class DefaultInterpolator(DepthInterpolator): + + def run(self, fg_rgb_image: np.ndarray, fg_depth_image: np.ndarray, + bg_rgb_image: np.ndarray, bg_depth_image: np.ndarray, + debug: bool = False) -> np.ndarray: + + return np.asarray(fg_depth_image).copy() + + +class SciPyInterpolator(DepthInterpolator): + + def __init__(self, depth_threshold: int, rgb_threshold: int): + + self.depth_threshold = depth_threshold + self.rgb_threshold = rgb_threshold + + def _subtract_depth_background( + self, fg_depth_image: np.ndarray) -> np.ndarray: + + fg_depth_image = np.asarray(fg_depth_image).copy() + mask = fg_depth_image >= self.depth_threshold + fg_depth_image[mask] = self.depth_threshold + + return fg_depth_image + + def _subtract_rgb_background(self, fg_image: np.ndarray, + bg_image: np.ndarray) -> np.ndarray: + + fg_image = np.asarray(fg_image).copy() + bg_image = np.asarray(bg_image).copy() + + diff = bg_image.astype(np.float64) - fg_image.astype(np.float64) + diff = np.abs(diff) + diff = np.sum(diff, axis=2) + + mask = (diff >= self.rgb_threshold) + + return mask + + +class GridDataInterpolator(SciPyInterpolator): + + def __init__(self, method: str = 'linear', depth_threshold: int = 1000, + rgb_threshold: int = 175, rgb_bg_sub: bool = False): + + super().__init__(depth_threshold, rgb_threshold) + + self.method = method + self.rgb_bg_sub = rgb_bg_sub + + def run(self, fg_rgb_image: np.ndarray, fg_depth_image: np.ndarray, + bg_rgb_image: np.ndarray, bg_depth_image: np.ndarray, + debug: bool = False) -> np.ndarray: + + fg_depth_image = self._subtract_depth_background(fg_depth_image) + points, values = _prepare_data(fg_depth_image) + if self.rgb_bg_sub: + mask = self._subtract_rgb_background(fg_rgb_image, bg_rgb_image) + fg_depth_image[~mask] = self.depth_threshold + hole_rows, hole_cols = np.where(fg_depth_image == 0) + + h, w = fg_depth_image.shape + grid_x, grid_y = np.mgrid[0:h:1, 0:w:1] + grid_z = griddata(points, values, (grid_x, grid_y), method=self.method) + + for row, col in zip(hole_rows, hole_cols): + if not np.isnan(grid_z[row, col]): + fg_depth_image[row, col] = grid_z[row, col] + + if debug: + + _plot(grid_z) + _plot(fg_depth_image) + + return fg_depth_image + + def __repr__(self): + + return '{}: {}'.format(self.__class__.__name__, self.method) + + +class MeanGridDataInterpolator(SciPyInterpolator): + + def __init__(self, methods, weights, depth_threshold: int = 1000, + rgb_threshold: int = 175, rgb_bg_sub: bool = True): + + super().__init__(depth_threshold, rgb_threshold) + + self.methods = methods + self.weights = weights + self.rgb_bg_sub = rgb_bg_sub + self.interpolators = [ + GridDataInterpolator( + method, depth_threshold, rgb_threshold, rgb_bg_sub) + for method in self.methods + ] + + def run(self, fg_rgb_image: np.ndarray, fg_depth_image: np.ndarray, + bg_rgb_image: np.ndarray, bg_depth_image: np.ndarray, + debug: bool = False) -> np.ndarray: + + depth_images = [ + interpolator.run(fg_rgb_image, fg_depth_image, + bg_rgb_image, bg_depth_image, debug) + for interpolator + in self.interpolators + ] + + mean_depth_image = np.average(depth_images, + axis=0, + weights=self.weights) + + if debug: + + _plot(mean_depth_image) + + + return mean_depth_image.astype(np.uint16) + + def __repr__(self): + + return '{}: {}'.format(self.__class__.__name__, self.methods) + + +class ForegroundInterpolator(DepthInterpolator): + + def __init__(self, depth_threshold: int, rgb_threshold: int, dilate_1: int, + dilate_2: int, dilate_3: int, erode: int, min_area: int): + + self.depth_threshold = depth_threshold + self.rgb_threshold = rgb_threshold + self.dilate_1 = dilate_1 + self.dilate_2 = dilate_2 + self.dilate_3 = dilate_3 + self.erode = erode + self.min_area = min_area + + def run(self, fg_rgb_image: np.ndarray, fg_depth_image: np.ndarray, + bg_rgb_image: np.ndarray, bg_depth_image: np.ndarray, + debug: bool = False) -> np.ndarray: + + fg_rgb_image = np.asarray(fg_rgb_image).copy() + fg_depth_image = np.asarray(fg_depth_image).copy() + bg_rgb_image = np.asarray(bg_rgb_image).copy() + bg_depth_image = np.asarray(bg_depth_image).copy() + + fg_threshold_mask = fg_depth_image >= self.depth_threshold + fg_depth_image[fg_threshold_mask] = self.depth_threshold + fg_rgb_image[fg_threshold_mask] = 0 + + bg_threshold_mask = bg_depth_image >= self.depth_threshold + bg_depth_image[bg_threshold_mask] = self.depth_threshold + bg_rgb_image[bg_threshold_mask] = 0 + + _, bg_sub_mask = self._subtract_background( + fg_rgb_image, bg_rgb_image) + + bg_sub_rgb_mask = bg_sub_mask.astype(np.uint8) + ded_rgb_mask = self._dilate_erode_dilate(bg_sub_rgb_mask) + contours_rgb_mask = self._select_contours(ded_rgb_mask) + contours_rgb_mask = contours_rgb_mask.astype(bool) + + contours_fg_depth_image = fg_depth_image.copy() + contours_fg_depth_image[~contours_rgb_mask] = 0 + + kernel = np.ones((self.dilate_3, self.dilate_3)) + dilate_depth_mask = cv2.dilate( + contours_fg_depth_image, kernel, iterations=1) + hole_mask = contours_fg_depth_image.copy().astype(bool) + hole_fill_mask = (contours_rgb_mask) & (~hole_mask) + + hole_fill_depth_image = contours_fg_depth_image.copy() + hole_fill_depth_image[hole_fill_mask] = \ + dilate_depth_mask[hole_fill_mask] + + fg_depth_image[contours_rgb_mask] = \ + hole_fill_depth_image[contours_rgb_mask] + + if debug: + + _plot(fg_depth_image) + + return fg_depth_image + + def _subtract_background( + self, fg_image: np.ndarray, + bg_image: np.ndarray) -> Tuple[np.ndarray]: + + diff = bg_image.astype(np.float64) - fg_image.astype(np.float64) + diff = np.abs(diff) + diff = np.sum(diff, axis=2) + + #diff = np.sqrt(np.sum((fg.astype(np.float64) - bg.astype(np.float64))**2, axis=2)) + + mask = (diff >= self.rgb_threshold) + + bg_sub_fg_image = fg_image.copy() + bg_sub_fg_image[~mask] = 255 + + return bg_sub_fg_image, mask + + def _dilate_erode_dilate(self, mask: np.ndarray) -> np.ndarray: + + kernel_dilate_1 = np.ones((self.dilate_1, self.dilate_1)) + kernel_erode = np.ones((self.erode, self.erode)) + kernel_dilate_2 = np.ones((self.dilate_2, self.dilate_2)) + + mask_dilate_1 = cv2.dilate(mask, kernel_dilate_1, iterations=1) + mask_erode = cv2.erode(mask_dilate_1, kernel_erode, iterations=1) + mask_dilate_2 = cv2.dilate(mask_erode, kernel_dilate_2, iterations=1) + + return mask_dilate_2 + + def _select_contours(self, mask: np.ndarray) -> np.ndarray: + + mask_contoured = mask.copy() + contours, _ = cv2.findContours(mask.copy(), + cv2.RETR_EXTERNAL, + cv2.CHAIN_APPROX_NONE) + + for contour in contours: + x, y, w, h = cv2.boundingRect(contour) + if cv2.contourArea(contour) < self.min_area: + mask_contoured[y:y+h+5, x:x+w+5] = 0 + + return mask_contoured + + +def initialize_depth_interpolator( + method: str, settings: dict) -> Union[GridDataInterpolator, + ForegroundInterpolator, + MeanGridDataInterpolator, + DefaultInterpolator]: + + if method == 'grid': + return GridDataInterpolator(settings['method'], + settings['depth_threshold'], + settings['rgb_threshold'], + settings['rgb_bg_sub']) + + + elif method == 'foreground': + return ForegroundInterpolator(settings['depth_threshold'], + settings['rgb_threshold'], + settings['dilate_1'], + settings['dilate_2'], + settings['dilate_3'], + settings['erode'], + settings['min_area']) + + elif method == 'mean_grid': + return MeanGridDataInterpolator(settings['methods'], + settings['weights'], + settings['depth_threshold'], + settings['rgb_threshold'], + settings['rgb_bg_sub']) + + else: + return DefaultInterpolator() + + +def _plot(image): + + plt.figure(figsize=(8, 8)) + plt.imshow(image) + plt.show() + + +def _prepare_data(fg_depth_image: np.ndarray) -> Tuple[np.ndarray]: + + points = [] + values = [] + nonzero_rows, nonzero_cols = np.where(fg_depth_image != 0) + + for i, j in zip(nonzero_rows, nonzero_cols): + points.append((i, j)) + values.append(fg_depth_image[i, j]) + + return np.array(points), np.array(values) diff --git a/src/test/integration/python_test_files/dlc_project.py b/src/test/integration/python_test_files/dlc_project.py new file mode 100644 index 0000000..23e587f --- /dev/null +++ b/src/test/integration/python_test_files/dlc_project.py @@ -0,0 +1,206 @@ +"""TODO +""" +import os +os.environ['DLCLight'] = 'True' +import glob +import datetime + +from deeplabcut import create_new_project as dlc_create_new_project + +from .deeplabcut_utils import load_yaml, dump_yaml + + +class DLCProject: + """Class to represent DeepLabCut project. Allows access to + config.yaml and all subdirectories in a DLC Project. + Parameters + ---------- + project_path : str + Full path to specific project directory. + config_init_path : str + Full path to yaml file that is used to specify any default + configuration settings that are to be overwritten along with + their values. + Attributes + ---------- + project_path : str + Full path to specific project directory. + config_path : str + Full path to specific project config.yaml file. + labeled_data_path : str + Full path to project labeled-data directory. + config_yaml : yaml + yaml object containing the project's configuration info. + """ + + def __init__(self, project_path: str, config_init_path: str = ''): + + self.project_path = project_path + self.config_path = os.path.join(project_path, 'config.yaml') + self.labeled_data_path = os.path.join(project_path, 'labeled-data') + if config_init_path: + self.config_yaml = self._initialize_config_yaml(config_init_path) + else: + self.config_yaml = load_yaml(self.config_path) + + @classmethod + def create_new_project( + cls, scorer: str, videos: list, task: str = 'TopDownPerson', + config_init_path: str = ( + '/home/shared/Projects/train-person-detector/configs/' + 'changes_to_config.yaml'), + working_directory: str = ( + '/home/shared/Projects/train-person-detector/dlc-projects')): + """Creates a new DeepLabCut project and DLCProject instance. + Parameters + ---------- + scorer : str + Name of scorer or experimenter. For Innovation Lab, this is + generally the name of the dataset e.g. data1-trace. + video_root : str + Path of the video files to be used just + before the camera name e.g. + /home/shared/mr_merchant_topdown/processed/20200302/data1-trace + task : str, default='TopDownPerson' + Name of task being performed in project. + config_init_path : str, default='changes_to_config.yaml' + YAML file containing all changes needing to be made to default + config.yaml created with new projects. + working_directory : str, default='/home/shared/Projects/train-person-detector/dlc-projects' + Root directory of 'dlc-projects' (differs between Linux and + Mac) + Returns + ------- + project : DLCProject + The DLCProject created from the given arguments. + """ + date = datetime.date.today().strftime('%Y-%m-%d') + project_name = f'{task}-{scorer}-{date}' + project_path = os.path.join(working_directory, project_name) + # videos = glob.glob(f'{video_root}*mp4') + + dlc_create_new_project(task, scorer, videos, copy_videos=False, + working_directory=working_directory) + project = DLCProject(project_path, config_init_path) + + return project + + @property + def scorer(self): + """Returns scorer from config_yaml""" + return self.config_yaml['scorer'] + + def _initialize_config_yaml(self, config_init_path): + """Updates the default config.yaml file with new values + specified in the yaml file located at config_init_path""" + config_yaml = load_yaml(self.config_path) + config_initialization_yaml = load_yaml(config_init_path) + + for key in config_initialization_yaml.keys(): + config_yaml[key] = config_initialization_yaml[key] + + dump_yaml(self.config_path, config_yaml) + + return config_yaml + + def _update_config_yaml(self, key, item): + """Updates the config.yaml at the given key with the given item. + """ + self.config_yaml[key] = item + dump_yaml(self.config_path, self.config_yaml) + + def _update_snapshotindex(self, snapshotindex): + """Sets the snapshotindex in the config.yaml file to a specified + value.""" + self._update_config_yaml('snapshotindex', snapshotindex) + + def _filepath_builder(self, subdir, filename): + """Builds filepaths to given filenames located in the project's + subdirectories.""" + iteration = self.config_yaml['iteration'] + task = self.config_yaml['Task'] + date = self.config_yaml['date'] + iteration_dir = f'iteration-{iteration}' + + if subdir in ['dlc-models', 'evaluation-results']: + training_fraction = self.config_yaml['TrainingFraction'][0] + training_fraction = str(training_fraction)[2:] + train_shuffle_dir = f'{task}{date}-trainset{training_fraction}shuffle1' + elif subdir == 'training-datasets': + train_shuffle_dir = f'UnaugmentedDataSet_{task}{date}' + + filepath = os.path.join(self.project_path, subdir, iteration_dir, + train_shuffle_dir, filename) + + return filepath + + def _update_pose_config_yaml(self, split, key, item): + """Updates the pose_config.yaml file used during training at + the given key with the given item.""" + filename = f'{split}/pose_config.yaml' + pose_config_path = self._filepath_builder('dlc-models', filename) + pose_config_yaml = load_yaml(pose_config_path) + pose_config_yaml[key] = item + dump_yaml(pose_config_path, pose_config_yaml) + + def update_pose_config_init_weights(self, init_weights): + """Updates the pose_config.yaml init_weights item in order to + carry out transfer learning or continue training from a + checkpoint.""" + self._update_pose_config_yaml('train', 'init_weights', init_weights) + self._update_pose_config_yaml('test', 'init_weights', init_weights) + + def __repr__(self): + + return self.config_path + + def get_labeled_data_by_camera(self, camera): + """Gets the labeled-data directory for a given camera/video.""" + return glob.glob(f'{self.labeled_data_path}/*{camera}')[0] + + def get_labeled_data_by_string(self, string): + """Gets the labeled-data directory for a given camera/video.""" + # print(string) + # print(glob.glob(f'{self.labeled_data_path}/{string}*')) + # print(os.path.join(self.labeled_data, string)) + #print(f'{self.labeled_data_path}/{string}*') + return glob.glob(f'{self.labeled_data_path}/{string}*')[0] + + def get_csv_by_string(self, string): + #print(f'{self.labeled_data_path}/{string}*/*csv') + return glob.glob(f'{self.labeled_data_path}/{string}*/*csv')[0] + + def get_labeled_data_dirs(self): + """Gets the labeled-data directory for all cameras.""" + return glob.glob(os.path.join(self.labeled_data_path, '*/')) + + def get_videos(self): + """Gets list of all videos used in project.""" + return os.listdir(self.labeled_data_path) + + def get_csv_filepath_by_video(self, video): + """Gets the CSV filepath for the given video.""" + csv_filename = f'*{self.scorer}*csv' + csv_glob = os.path.join(self.labeled_data_path, video, csv_filename) + return glob.glob(csv_glob)[0] + + +if __name__ == '__main__': + + # projects_root = '/home/shared/Projects' + # repo_name = 'train-person-detector' + SCORER = 'data1-trace-masked' + DATA_ROOT = '/home/shared/mr_merchant_topdown/processed' + DATASET_DATE = '20200302' + DATASET_DIR = os.path.join(DATA_ROOT, DATASET_DATE) + DATASET_NAME = 'data1-trace-masked' + VIDEO_ROOT = os.path.join(DATASET_DIR, DATASET_NAME) + CONFIG_INIT_PATH = '/home/shared/Projects/train-person-detector/configs/changes_to_config.yaml' + # project_root = '/home/shared/Projects/train-person-detector/projects' + # project_name = 'TopDownPerson-data1-10ids-exp1-2020-03-25' + # project = DLCProject(project_root, project_name) + + project = DLCProject.create_new_project(SCORER, VIDEO_ROOT, + config_init_path=CONFIG_INIT_PATH) + print(project) + print(project.config_yaml['TrainingFraction']) \ No newline at end of file From cff0855ba18e11f8f57c448c134b4dfb80b842d6 Mon Sep 17 00:00:00 2001 From: Bryan Nonni Date: Tue, 28 Apr 2020 10:11:09 -0400 Subject: [PATCH 08/18] pushing fixes to broken tests --- src/test/docstring/generate_docstring.spec.ts | 3 + src/test/parse/get_body.spec.ts | 16 ++--- src/test/parse/parse_parameters.spec.ts | 61 +++++++++++++------ 3 files changed, 53 insertions(+), 27 deletions(-) diff --git a/src/test/docstring/generate_docstring.spec.ts b/src/test/docstring/generate_docstring.spec.ts index d8b8121..eebfad8 100644 --- a/src/test/docstring/generate_docstring.spec.ts +++ b/src/test/docstring/generate_docstring.spec.ts @@ -449,6 +449,9 @@ const defaultDocstringComponents: DocstringParts = { returns: { type: "" }, yields: { type: "" }, exceptions: [], + classes: [], + methods: [], + attributes: [] }; const noTypesTemplate = `{{#args}}{{var}} {{type}}{{/args}} diff --git a/src/test/parse/get_body.spec.ts b/src/test/parse/get_body.spec.ts index 269146d..007715e 100644 --- a/src/test/parse/get_body.spec.ts +++ b/src/test/parse/get_body.spec.ts @@ -8,7 +8,7 @@ const expect = chai.expect; describe("getBody()", () => { it("should return the body of a function", () => { - const result = getBody(basicFunction, 4); + const result = getBody(docstringType,basicFunction, 4); expect(result).to.have.deep.members([ "print(\"HELLO WORLD\")", @@ -21,7 +21,7 @@ describe("getBody()", () => { }); it("should skip blank lines", () => { - const result = getBody(gapFunction, 5); + const result = getBody(docstringType,gapFunction, 5); expect(result).to.have.deep.members([ "print('HELLO WORLD')", @@ -30,7 +30,7 @@ describe("getBody()", () => { }); it("should handle multi line definitions", () => { - const result = getBody(multiLineDefFunction, 4); + const result = getBody(docstringType,multiLineDefFunction, 4); expect(result).to.have.deep.members([ "pass", @@ -38,13 +38,13 @@ describe("getBody()", () => { }); it("should handle indented functions", () => { - const result = getBody(indentedFunctions, 3); + const result = getBody(docstringType,indentedFunctions, 3); expect(result).to.have.deep.members([ "return 2", ]); - const result2 = getBody(indentedFunctions, 6); + const result2 = getBody(docstringType,indentedFunctions, 6); expect(result2).to.have.deep.members([ "pass", @@ -52,11 +52,11 @@ describe("getBody()", () => { }); it("should return an empty array if a function has no body", () => { - const result = getBody(noBody, 2); + const result = getBody(docstringType,noBody, 2); expect(result).to.have.deep.members([]); - const result2 = getBody(noBody, 4); + const result2 = getBody(docstringType,noBody, 4); expect(result2).to.have.deep.members([]); }); @@ -120,3 +120,5 @@ def no_body(): def next_no_body(): `; + +const docstringType = 'method'; diff --git a/src/test/parse/parse_parameters.spec.ts b/src/test/parse/parse_parameters.spec.ts index 9fbf708..a878918 100644 --- a/src/test/parse/parse_parameters.spec.ts +++ b/src/test/parse/parse_parameters.spec.ts @@ -26,7 +26,9 @@ describe("parseParameters()", () => { const functionName = "function"; - const result = parseParameters(parameterTokens, body, functionName); + const docstringType = "method"; + + const result = parseParameters(docstringType, parameterTokens, body, functionName); expect(result).to.eql({ name: "function", @@ -53,7 +55,8 @@ describe("parseParameters()", () => { it("should parse args with and without type hints", () => { const parameterTokens = ["param1: List[string]", "param2"]; - const result = parseParameters(parameterTokens, [], "name"); + const docstringType = "method"; + const result = parseParameters(docstringType, parameterTokens, [], "name"); expect(result.args).to.have.deep.members([ { var: "param1", type: "List[string]" }, @@ -63,7 +66,8 @@ describe("parseParameters()", () => { it("should parse kwargs with and without type hints", () => { const parameterTokens = ["param1: List[int] = [1,2]", "param2 = 'abc'"]; - const result = parseParameters(parameterTokens, [], "name"); + const docstringType = "method"; + const result = parseParameters(docstringType, parameterTokens, [], "name"); expect(result.kwargs).to.have.deep.members([ { var: "param1", default: "[1,2]", type: "List[int]" }, @@ -75,7 +79,8 @@ describe("parseParameters()", () => { it("should parse return types", () => { const parameterTokens = ["-> List[int]"]; - const result = parseParameters(parameterTokens, [], "name"); + const docstringType = "method"; + const result = parseParameters(docstringType, parameterTokens, [], "name"); expect(result.returns).to.deep.equal({ type: "List[int]", @@ -84,21 +89,24 @@ describe("parseParameters()", () => { it("should not parse '-> None' return types", () => { const parameterTokens = ["-> None"]; - const result = parseParameters(parameterTokens, [], "name"); + const docstringType = "method"; + const result = parseParameters(docstringType, parameterTokens, [], "name"); expect(result.returns).to.deep.equal(undefined); }); it("should not parse '-> Generator' return types", () => { const parameterTokens = ["-> Generator[int]"]; - const result = parseParameters(parameterTokens, [], "name"); + const docstringType = "method"; + const result = parseParameters(docstringType, parameterTokens, [], "name"); expect(result.returns).to.deep.equal(undefined); }); it("should not parse '-> Iterator' return types", () => { const parameterTokens = ["-> Iterator[int]"]; - const result = parseParameters(parameterTokens, [], "name"); + const docstringType = "method"; + const result = parseParameters(docstringType, parameterTokens, [], "name"); expect(result.returns).to.deep.equal(undefined); }); @@ -108,7 +116,8 @@ describe("parseParameters()", () => { it("should use the signature return type if it is an Iterator", () => { const parameterTokens = ["-> Iterator[int]"]; const body = []; - const result = parseParameters(parameterTokens, body, "name"); + const docstringType = "method"; + const result = parseParameters(docstringType, parameterTokens, body, "name"); expect(result.yields).to.deep.equal({ type: "Iterator[int]", @@ -118,7 +127,8 @@ describe("parseParameters()", () => { it("should use the signature return type if it is an Generator", () => { const parameterTokens = ["-> Generator[int]"]; const body = []; - const result = parseParameters(parameterTokens, body, "name"); + const docstringType = "method"; + const result = parseParameters(docstringType, parameterTokens, body, "name"); expect(result.yields).to.deep.equal({ type: "Generator[int]", @@ -128,7 +138,8 @@ describe("parseParameters()", () => { it("Should use the return type as the yield type if a yield exists in the body", () => { const parameterTokens = ["-> int"]; const body = ["yield 4"]; - const result = parseParameters(parameterTokens, body, "name"); + const docstringType = "method"; + const result = parseParameters(docstringType, parameterTokens, body, "name"); expect(result.yields).to.eql({ type: "Iterator[int]", @@ -138,7 +149,8 @@ describe("parseParameters()", () => { it("Should return a yield without type if a yield exists in the body but there is no return signature", () => { const parameterTokens = [""]; const body = ["yield 4"]; - const result = parseParameters(parameterTokens, body, "name"); + const docstringType = "method"; + const result = parseParameters(docstringType, parameterTokens, body, "name"); expect(result.yields).to.eql({ type: undefined, @@ -148,7 +160,8 @@ describe("parseParameters()", () => { it("Should return undefined if no yield exists in the signature or body", () => { const parameterTokens = ["-> List[int]"]; const body = []; - const result = parseParameters(parameterTokens, body, "name"); + const docstringType = "method"; + const result = parseParameters(docstringType, parameterTokens, body, "name"); expect(result.yields).to.eql(undefined); }); @@ -156,7 +169,8 @@ describe("parseParameters()", () => { }); it("should result in no yield if there is no yield type or yield in body", () => { - const result = parseParameters([], [], "name"); + const docstringType = "method"; + const result = parseParameters(docstringType, [], [], "name"); expect(result.returns).to.eql(undefined); }); @@ -164,7 +178,8 @@ describe("parseParameters()", () => { it("should parse the return from the body if there is no return type in the definition", () => { const parameterTokens = ["param1"]; const body = ["return 3"]; - const result = parseParameters(parameterTokens, body, ""); + const docstringType = "method"; + const result = parseParameters(docstringType, parameterTokens, body, ""); expect(result.returns).to.eql({ type: undefined, @@ -172,15 +187,17 @@ describe("parseParameters()", () => { }); it("should result in no return if there is no return type or return in body", () => { - const result = parseParameters([], [], "name"); + const docstringType = "method"; + const result = parseParameters(docstringType, [], [], "name"); expect(result.returns).to.eql(undefined); }); it("should parse simple exception", () => { + const docstringType = "method"; const functionContent = ["raise Exception"]; - const result = parseParameters([], functionContent, ""); + const result = parseParameters(docstringType, [], functionContent, ""); expect(result.exceptions).to.have.deep.members([ { type: "Exception" }, @@ -199,7 +216,8 @@ describe("parseParameters()", () => { "raise AlwaysCrapsOut", ]; - const result = parseParameters([], functionContent, ""); + const docstringType = "method"; + const result = parseParameters(docstringType, [], functionContent, ""); expect(result.exceptions).to.have.deep.members([ { type: "BadVar" }, @@ -210,8 +228,9 @@ describe("parseParameters()", () => { context("when the parameters have strange spacing", () => { it("should parse args with strange spacing", () => { + const docstringType = "method"; const parameterTokens = [" param1 : int ", " param2 ", "param3:List[int]"]; - const result = parseParameters(parameterTokens, [], "name"); + const result = parseParameters(docstringType, parameterTokens, [], "name"); expect(result.args).to.have.deep.members([ { var: "param1", type: "int" }, @@ -222,7 +241,8 @@ describe("parseParameters()", () => { it("should parse kwargs with strange spacing", () => { const parameterTokens = [" param1 : str\t=\t'abc'", " param2 = 1", "param3:int=2"]; - const result = parseParameters(parameterTokens, [], "name"); + const docstringType = "method"; + const result = parseParameters(docstringType, parameterTokens, [], "name"); expect(result.kwargs).to.have.deep.members([ { var: "param1", default: "'abc'", type: "str" }, @@ -233,7 +253,8 @@ describe("parseParameters()", () => { it("should parse return types with strange spacing", () => { const parameterTokens = ["\t -> \tint \t"]; - const result = parseParameters(parameterTokens, [], "name"); + const docstringType = "method"; + const result = parseParameters(docstringType, parameterTokens, [], "name"); expect(result.returns).to.deep.equal({ type: "int", From d099316092334b4ad063b484c5ac6b1b2c18b1da Mon Sep 17 00:00:00 2001 From: Bryan Nonni Date: Tue, 28 Apr 2020 10:18:39 -0400 Subject: [PATCH 09/18] parseAttributes: new regex pattern; parseClasses: different regex pattern allows use of getFunctionName() method; parseParameters: parseExceptions: [] inside docstringType === class --- src/parse/parse_parameters.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/parse/parse_parameters.ts b/src/parse/parse_parameters.ts index bf189fb..bac1bdb 100644 --- a/src/parse/parse_parameters.ts +++ b/src/parse/parse_parameters.ts @@ -44,7 +44,7 @@ export function parseParameters(docstringType: string, parameterTokens: string[] kwargs: kwargs, returns: undefined, yields: undefined, - exceptions: parseExceptions(body), + exceptions: [], classes: [], methods: [], attributes: parseAttributes(body, args, kwargs) @@ -177,7 +177,7 @@ function parseExceptions(body: string[]): Exception[] { function parseClasses(body: string[]): Class[] { const classes: Class[] = [] - const pattern = /(?:class)\s/; + const pattern = /(?:class)\s+(\w+)\s*\(*/; for (const line of body) { @@ -191,7 +191,8 @@ function parseClasses(body: string[]): Class[] { if (match != null) { // console.log("class match") // console.log(match) - let className = getClassName(line); + // let className = getClassName(line); + let className = getFunctionName(line); // console.log(className) classes.push({ @@ -237,7 +238,7 @@ function parseMethods(body: string[]): Method[] { function parseAttributes(body: string[], args: Argument[], kwargs: KeywordArgument[]): Attribute[] { const attributes: Attribute[] = []; - const pattern = /(?:self.)(\w+)(?:\s*:[^=]+)?\s*=\s*(.+)/; + const pattern = /(?:self\.|cls\.)(\w+)(?:\s*:[^=]+)?\s*=\s*(.+)/; //const pattern = /(?:self).(\w+)?\s*/ for (const line of body) { From 209f1edacf611092f35a43b9c09bf5201a0273f8 Mon Sep 17 00:00:00 2001 From: Bryan Nonni Date: Tue, 28 Apr 2020 10:21:32 -0400 Subject: [PATCH 10/18] getFunctionName: adding diff regex for both class and method --- src/parse/get_function_name.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parse/get_function_name.ts b/src/parse/get_function_name.ts index 063e0a1..f7d4ee0 100644 --- a/src/parse/get_function_name.ts +++ b/src/parse/get_function_name.ts @@ -1,5 +1,5 @@ export function getFunctionName(functionDefinition: string): string { - const pattern = /(?:def|class)\s+(\w+)\s*\(/; + const pattern = /(?:def|class)\s+(\w+)\s*\(*/; const match = pattern.exec(functionDefinition); From e620d1f2bd3c01d07565bad001592c1531d0abae Mon Sep 17 00:00:00 2001 From: tracetidwell Date: Fri, 29 Jan 2021 11:11:51 -0500 Subject: [PATCH 11/18] passing unit tests; failing integration tests --- .vscode/launch.json | 2 +- package-lock.json | 16 ++ package.json | 9 +- src/parse/get_body.ts | 10 +- src/parse/get_definition.ts | 190 ++++++++++++++++++------ src/test/parse/get_body.spec.ts | 2 +- src/test/parse/get_definition.spec.ts | 25 +++- src/test/parse/parse_parameters.spec.ts | 3 + src/test/run_integration_tests.js | 1 + src/test/run_integration_tests.ts | 1 + 10 files changed, 194 insertions(+), 65 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index e4f356c..3d30f88 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -20,7 +20,7 @@ "runtimeExecutable": "${execPath}", "args": [ "--extensionDevelopmentPath=${workspaceFolder}", - "--extensionTestsPath=${workspaceFolder}/out/test" + "--extensionTestsPath=${workspaceFolder}/src/test/integration/index.ts" ], "outFiles": [ "${workspaceFolder}/out/test/**/*.js" diff --git a/package-lock.json b/package-lock.json index d30bba1..0567cf7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,22 @@ "integrity": "sha512-t7uW6eFafjO+qJ3BIV2gGUyZs27egcNRkUdalkud+Qa3+kg//f129iuOFivHDXQ+vnU3fDXuwgv0cqMCbcE8sw==", "dev": true }, + "@types/glob": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.1.3.tgz", + "integrity": "sha512-SEYeGAIQIQX8NN6LDKprLjbrd5dARM5EXsd8GI/A5l0apYI1fGMWgPHSe4ZKL4eozlAyI+doUE9XbYS4xCkQ1w==", + "dev": true, + "requires": { + "@types/minimatch": "*", + "@types/node": "*" + } + }, + "@types/minimatch": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz", + "integrity": "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==", + "dev": true + }, "@types/mocha": { "version": "5.2.7", "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-5.2.7.tgz", diff --git a/package.json b/package.json index 6c73ca3..f6c5687 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "displayName": "Python Docstring Generator", "description": "Automatically generates detailed docstrings for python functions", "version": "0.5.4", - "publisher": "njpwerner", + "publisher": "tracetidwell", "license": "SEE LICENSE IN LICENSE", "icon": "images/icon.png", "extensionKind": [ @@ -45,7 +45,7 @@ "commands": [ { "command": "autoDocstring.generateDocstring", - "title": "Generate Docstring2" + "title": "Generate Docstring" } ], "keybindings": [ @@ -71,11 +71,7 @@ "properties": { "autoDocstring.docstringFormat": { "type": "string", -<<<<<<< HEAD "default": "ncr", -======= - "default": "google", ->>>>>>> master "enum": [ "docblockr", "google", @@ -150,6 +146,7 @@ }, "devDependencies": { "@types/chai": "^4.2.11", + "@types/glob": "^7.1.3", "@types/mocha": "^5.2.7", "@types/mustache": "^0.8.32", "@types/node": "^13.13.1", diff --git a/src/parse/get_body.ts b/src/parse/get_body.ts index 3469008..6c7936f 100644 --- a/src/parse/get_body.ts +++ b/src/parse/get_body.ts @@ -14,15 +14,13 @@ export function getBody(docstringType: string, document: string, linePosition: n let currentLineNum = linePosition; const originalIndentation = getBodyBaseIndentation(lines, linePosition, regex); - // console.log("original indentation") - // console.log(originalIndentation) - // console.log(currentLineNum) + + if (originalIndentation == 0) { + return body; + } while (currentLineNum < lines.length) { const line = lines[currentLineNum]; - // console.log("current indentation") - // console.log(indentationOf(line)) - // console.log(currentLineNum) if (blankLine(line)) { currentLineNum++; diff --git a/src/parse/get_definition.ts b/src/parse/get_definition.ts index 40fd04c..87f2949 100644 --- a/src/parse/get_definition.ts +++ b/src/parse/get_definition.ts @@ -1,75 +1,173 @@ -import { blankLine, indentationOf } from "./utilities"; +import { blankLine, indentationOf, preprocessLines } from "./utilities"; export function getDefinition(document: string, linePosition: number): string { - const lines = document.split("\n"); - let definition = ""; + // const lines = document.split("\n"); + // let definition = ""; - if (linePosition === 0) { - return definition; - } + // if (linePosition === 0) { + // return definition; + // } - let currentLineNum = linePosition - 1; - const originalIndentation = indentationOf(lines[currentLineNum]); - // console.log("original indentation") - // console.log(originalIndentation) + // let currentLineNum = linePosition - 1; + // const originalIndentation = indentationOf(lines[currentLineNum]); - while (currentLineNum >= 0) { - const line = lines[currentLineNum]; - definition = line.trim() + definition; + // while (currentLineNum >= 0) { + // const line = lines[currentLineNum]; + // definition = line.trim() + definition; - if (indentationOf(line) < originalIndentation || blankLine(line)) { - break; - } + // if (indentationOf(line) < originalIndentation || blankLine(line)) { + // break; + // } + + // currentLineNum -= 1; + // } + + // const classPattern = /(?:class)\s+(\w+)/; + // const classMatch = classPattern.exec(definition); + + // if (classMatch != undefined && classMatch[0] != undefined) { + // currentLineNum += 2; + // let definition = classMatch[0]; + // const initPattern = /(?:def __init__)/; + + // while (currentLineNum < lines.length) { + // const line = lines[currentLineNum]; + // const initMatch = initPattern.exec(line) + + // if (initMatch != undefined && initMatch[0] != undefined) { + // const newIndentation = indentationOf(lines[currentLineNum]); + + // while (currentLineNum < lines.length) { + // const line = lines[currentLineNum]; + // definition += line.trim(); + + // if (indentationOf(line) < newIndentation || blankLine(line)) { + // return definition; + // } - currentLineNum -= 1; + // currentLineNum += 1; + // } + + // } + // else if (indentationOf(line) <= originalIndentation && !blankLine(line)) { + // return definition; + // } + // currentLineNum += 1; + // } + + // return definition; + const precedingLines = getPrecedingLines(document, linePosition); + const precedingText = precedingLines.join(" "); + + // Don't parse if the preceding line is blank + const precedingLine = precedingLines[precedingLines.length - 1]; + if (precedingLine == undefined || blankLine(precedingLine)) { + return ""; } - //const classPattern = /(?:class)/; - const classPattern = /(?:class)\s+(\w+)/; - const classMatch = classPattern.exec(definition); + const pattern = /\b(((async\s+)?\s*def)|\s*class)\b/g; + + // Get starting index of last def match in the preceding text + let index: number; + while (pattern.test(precedingText)) { + index = pattern.lastIndex - RegExp.lastMatch.length; + } + + if (index == undefined) { + return ""; + } + + const lastFunctionDef = precedingText.slice(index).trim(); + + if (lastFunctionDef.startsWith('class')) { + const lines = document.split("\n"); - if (classMatch != undefined && classMatch[0] != undefined) { - currentLineNum += 2; + const originalIndentation = indentationOf(lines[linePosition]); + const classPattern = /(?:class)\s+(\w+)/; + const classMatch = classPattern.exec(lastFunctionDef); let definition = classMatch[0]; - // console.log("new definition") - // console.log(definition) - // console.log(classMatch) - // console.log("current line num") - // console.log(currentLineNum) - // console.log("current indentation") - // console.log(indentationOf(lines[currentLineNum])) - // console.log(lines[currentLineNum]) - const initPattern = /(?:def __init__)/; - - while (currentLineNum < lines.length) { - const line = lines[currentLineNum]; - // console.log("current indentation") - // console.log(indentationOf(line)) + // const initPattern = /(?:def __init__)/; + const initPattern = /(?<=def __init__).*/; + const defClosePattern = /(\)\:)/ + + while (linePosition < lines.length) { + const line = lines[linePosition]; const initMatch = initPattern.exec(line) if (initMatch != undefined && initMatch[0] != undefined) { - const newIndentation = indentationOf(lines[currentLineNum]); + definition += initMatch[0]; + const newIndentation = indentationOf(lines[linePosition]); + let defCloseMatch = defClosePattern.exec(line); + if (defCloseMatch != undefined && defCloseMatch[0] != undefined) { + return definition; + } + linePosition += 1; - while (currentLineNum < lines.length) { - const line = lines[currentLineNum]; + while (linePosition < lines.length) { + const line = lines[linePosition]; definition += line.trim(); - // console.log(definition) - + defCloseMatch = defClosePattern.exec(line); if (indentationOf(line) < newIndentation || blankLine(line)) { return definition; } + else if (defCloseMatch != undefined && defCloseMatch[0] != undefined) { + return definition; + } - currentLineNum += 1; + linePosition += 1; } } else if (indentationOf(line) <= originalIndentation && !blankLine(line)) { return definition; } - currentLineNum += 1; + linePosition += 1; } + + return definition } - // console.log("last definition") - // console.log(definition) - return definition; + + + // const classPattern = /(?:class)\s+(\w+)/; + // const classMatch = classPattern.exec(lastFunctionDef); + + // if (classMatch != undefined && classMatch[0] != undefined) { + // currentLineNum += 2; + // let definition = classMatch[0]; + // const initPattern = /(?:def __init__)/; + + // while (currentLineNum < lines.length) { + // const line = lines[currentLineNum]; + // const initMatch = initPattern.exec(line) + + // if (initMatch != undefined && initMatch[0] != undefined) { + // const newIndentation = indentationOf(lines[currentLineNum]); + + // while (currentLineNum < lines.length) { + // const line = lines[currentLineNum]; + // definition += line.trim(); + + // if (indentationOf(line) < newIndentation || blankLine(line)) { + // return definition; + // } + + // currentLineNum += 1; + // } + + // } + // else if (indentationOf(line) <= originalIndentation && !blankLine(line)) { + // return definition; + // } + // currentLineNum += 1; + // } + + return lastFunctionDef; +} + +function getPrecedingLines(document: string, linePosition: number): string[] { + const lines = document.split("\n"); + const rawPrecedingLines = lines.slice(0, linePosition); + + const precedingLines = preprocessLines(rawPrecedingLines); + return precedingLines; } \ No newline at end of file diff --git a/src/test/parse/get_body.spec.ts b/src/test/parse/get_body.spec.ts index 4db3027..4a588a3 100644 --- a/src/test/parse/get_body.spec.ts +++ b/src/test/parse/get_body.spec.ts @@ -123,7 +123,7 @@ def next_func(): const noBody = ` def no_body(): - + def next_no_body(): `; diff --git a/src/test/parse/get_definition.spec.ts b/src/test/parse/get_definition.spec.ts index 7c4f3cb..10d73e8 100644 --- a/src/test/parse/get_definition.spec.ts +++ b/src/test/parse/get_definition.spec.ts @@ -26,7 +26,7 @@ describe("getDefinition()", () => { expect(result).to.equal("def multi_line_function( param1, param2 = 1):"); }); - it("should get ignore commented lines in a multiline function definition", () => { + it("should ignore commented lines in a multiline function definition", () => { const result = getDefinition(multiLineCommentedLineFunction, 9); expect(result).to.equal( @@ -60,10 +60,15 @@ describe("getDefinition()", () => { }); context("when encountering a class", () => { - it("should get the class definition", () => { + it("should get the class definition from init", () => { const result = getDefinition(basicClass, 4); - expect(result).to.equal("class BasicClass(object):"); + expect(result).to.equal("class BasicClass(self, param1):"); + }); + it("should get the class definition from multiline init", () => { + const result = getDefinition(basicClass, 12); + + expect(result).to.equal("class AnotherBasicClass(self, param2):"); }); }); }); @@ -95,8 +100,8 @@ Something Else const multiLineFunction = ` Something Else -def multi_line_function( - param1, +def multi_line_function( + param1, param2 = 1): print("HELLO WORLD") @@ -158,4 +163,14 @@ class BasicClass(object): def hello(self): print("Hello world") + +class AnotherBasicClass(object): + + def __init__( + self, param2 + ): + self.param2 = param2 + + def hello(self): + print("Goodbye world") `; diff --git a/src/test/parse/parse_parameters.spec.ts b/src/test/parse/parse_parameters.spec.ts index fba039c..4eb0ae4 100644 --- a/src/test/parse/parse_parameters.spec.ts +++ b/src/test/parse/parse_parameters.spec.ts @@ -38,6 +38,9 @@ describe("parseParameters()", () => { returns: { type: "int" }, yields: undefined, exceptions: [{ type: "Exception" }, { type: "Exception2" }], + classes: [], + methods: [], + attributes: [], }); }); diff --git a/src/test/run_integration_tests.js b/src/test/run_integration_tests.js index f17ebab..7203b54 100644 --- a/src/test/run_integration_tests.js +++ b/src/test/run_integration_tests.js @@ -63,6 +63,7 @@ function main() { return [3 /*break*/, 4]; case 3: err_1 = _a.sent(); + console.error(err_1); console.error("Failed to run tests"); process.exit(1); return [3 /*break*/, 4]; diff --git a/src/test/run_integration_tests.ts b/src/test/run_integration_tests.ts index 627ec06..e6f09c0 100644 --- a/src/test/run_integration_tests.ts +++ b/src/test/run_integration_tests.ts @@ -18,6 +18,7 @@ async function main() { launchArgs: ["--disable-extensions"], }); } catch (err) { + console.error(err) console.error("Failed to run tests"); process.exit(1); } From 0b08830a34fc0be4ec719b5b6c0808287d642529 Mon Sep 17 00:00:00 2001 From: tracetidwell Date: Fri, 29 Jan 2021 13:38:39 -0500 Subject: [PATCH 12/18] all unit tests and integration tests passing! --- README.md | 11 +- package.json | 2 +- src/docstring/templates/sphinx.mustache | 16 + src/parse/get_body.ts | 4 +- src/parse/get_definition.ts | 92 +----- src/parse/get_docstring_indentation.ts | 4 + src/parse/get_docstring_type.ts | 2 - src/parse/parse.ts | 5 + src/parse/parse_parameters.ts | 25 -- src/test/integration/integration.test.ts | 22 ++ .../python_test_files/depth_interpolator.py | 300 ------------------ .../python_test_files/dlc_project.py | 206 ------------ .../integration/python_test_files/file_7.py | 10 + .../python_test_files/file_7_output.py | 18 ++ .../integration/python_test_files/file_8.py | 40 +++ .../python_test_files/file_8_output.py | 51 +++ src/test/parse/get_body.spec.ts | 144 +++++++-- .../parse/get_docstring_indentation.spec.ts | 11 +- src/test/parse/parse_parameters.spec.ts | 185 ++++++++--- 19 files changed, 438 insertions(+), 710 deletions(-) delete mode 100644 src/test/integration/python_test_files/depth_interpolator.py delete mode 100644 src/test/integration/python_test_files/dlc_project.py create mode 100644 src/test/integration/python_test_files/file_7.py create mode 100644 src/test/integration/python_test_files/file_7_output.py create mode 100644 src/test/integration/python_test_files/file_8.py create mode 100644 src/test/integration/python_test_files/file_8_output.py diff --git a/README.md b/README.md index 2627df5..4599839 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Visual Studio Code extension to quickly generate docstrings for python functions ## Usage -Cursor must be on the line directly below the definition to generate full auto-populated docstring +Cursor must be on the line directly below the class or method definition or on the first line of the file to generate full auto-populated docstring - Press enter after opening docstring with triple quotes (`"""` or `'''`) - Keyboard shortcut: `ctrl+shift+2` or `cmd+shift+2` for mac @@ -111,6 +111,15 @@ This extension now supports custom templates. The extension uses the [mustache.j {{#returnsExist}} - display contents if returns exist {{/returnsExist}} +{{#classesExist}} - display contents if classes exist +{{/classesExist}} + +{{#methodsExist}} - display contents if methods exist +{{/methodsExist}} + +{{#attributesExist}} - display contents if attributes exist +{{/attributesExist}} + {{#placeholder}} - makes contents a placeholder {{/placeholder}} ``` diff --git a/package.json b/package.json index f6c5687..631f7e9 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "displayName": "Python Docstring Generator", "description": "Automatically generates detailed docstrings for python functions", "version": "0.5.4", - "publisher": "tracetidwell", + "publisher": "njpwerner", "license": "SEE LICENSE IN LICENSE", "icon": "images/icon.png", "extensionKind": [ diff --git a/src/docstring/templates/sphinx.mustache b/src/docstring/templates/sphinx.mustache index b7ba7d7..502805d 100644 --- a/src/docstring/templates/sphinx.mustache +++ b/src/docstring/templates/sphinx.mustache @@ -3,6 +3,22 @@ {{extendedSummaryPlaceholder}} +{{#classesExist}} +Classes +------- +{{#classes}} +{{name}} +{{/classes}} +{{/classesExist}} + +{{#methodsExist}} +Methods +------- +{{#methods}} +{{name}} +{{/methods}} +{{/methodsExist}} + {{#args}} :param {{var}}: {{descriptionPlaceholder}} :type {{var}}: {{typePlaceholder}} diff --git a/src/parse/get_body.ts b/src/parse/get_body.ts index 6c7936f..87f8b34 100644 --- a/src/parse/get_body.ts +++ b/src/parse/get_body.ts @@ -9,13 +9,13 @@ export function getBody(docstringType: string, document: string, linePosition: n return lines; } else if (docstringType === 'class') { - let regex = '.'; + regex = '.'; } let currentLineNum = linePosition; const originalIndentation = getBodyBaseIndentation(lines, linePosition, regex); - if (originalIndentation == 0) { + if (originalIndentation === 0) { return body; } diff --git a/src/parse/get_definition.ts b/src/parse/get_definition.ts index 87f2949..1cc97cd 100644 --- a/src/parse/get_definition.ts +++ b/src/parse/get_definition.ts @@ -1,61 +1,11 @@ import { blankLine, indentationOf, preprocessLines } from "./utilities"; export function getDefinition(document: string, linePosition: number): string { - // const lines = document.split("\n"); - // let definition = ""; - // if (linePosition === 0) { - // return definition; - // } - - // let currentLineNum = linePosition - 1; - // const originalIndentation = indentationOf(lines[currentLineNum]); - - // while (currentLineNum >= 0) { - // const line = lines[currentLineNum]; - // definition = line.trim() + definition; - - // if (indentationOf(line) < originalIndentation || blankLine(line)) { - // break; - // } - - // currentLineNum -= 1; - // } - - // const classPattern = /(?:class)\s+(\w+)/; - // const classMatch = classPattern.exec(definition); - - // if (classMatch != undefined && classMatch[0] != undefined) { - // currentLineNum += 2; - // let definition = classMatch[0]; - // const initPattern = /(?:def __init__)/; - - // while (currentLineNum < lines.length) { - // const line = lines[currentLineNum]; - // const initMatch = initPattern.exec(line) - - // if (initMatch != undefined && initMatch[0] != undefined) { - // const newIndentation = indentationOf(lines[currentLineNum]); - - // while (currentLineNum < lines.length) { - // const line = lines[currentLineNum]; - // definition += line.trim(); - - // if (indentationOf(line) < newIndentation || blankLine(line)) { - // return definition; - // } + if (linePosition === 0) { + return ""; + } - // currentLineNum += 1; - // } - - // } - // else if (indentationOf(line) <= originalIndentation && !blankLine(line)) { - // return definition; - // } - // currentLineNum += 1; - // } - - // return definition; const precedingLines = getPrecedingLines(document, linePosition); const precedingText = precedingLines.join(" "); @@ -88,7 +38,7 @@ export function getDefinition(document: string, linePosition: number): string { let definition = classMatch[0]; // const initPattern = /(?:def __init__)/; const initPattern = /(?<=def __init__).*/; - const defClosePattern = /(\)\:)/ + const defClosePattern = /(\))/ while (linePosition < lines.length) { const line = lines[linePosition]; @@ -127,40 +77,6 @@ export function getDefinition(document: string, linePosition: number): string { return definition } - - // const classPattern = /(?:class)\s+(\w+)/; - // const classMatch = classPattern.exec(lastFunctionDef); - - // if (classMatch != undefined && classMatch[0] != undefined) { - // currentLineNum += 2; - // let definition = classMatch[0]; - // const initPattern = /(?:def __init__)/; - - // while (currentLineNum < lines.length) { - // const line = lines[currentLineNum]; - // const initMatch = initPattern.exec(line) - - // if (initMatch != undefined && initMatch[0] != undefined) { - // const newIndentation = indentationOf(lines[currentLineNum]); - - // while (currentLineNum < lines.length) { - // const line = lines[currentLineNum]; - // definition += line.trim(); - - // if (indentationOf(line) < newIndentation || blankLine(line)) { - // return definition; - // } - - // currentLineNum += 1; - // } - - // } - // else if (indentationOf(line) <= originalIndentation && !blankLine(line)) { - // return definition; - // } - // currentLineNum += 1; - // } - return lastFunctionDef; } diff --git a/src/parse/get_docstring_indentation.ts b/src/parse/get_docstring_indentation.ts index 3002d3c..f7882d0 100644 --- a/src/parse/get_docstring_indentation.ts +++ b/src/parse/get_docstring_indentation.ts @@ -5,6 +5,10 @@ export function getDocstringIndentation( linePosition: number, defaultIndentation: string, ): string { + + if (linePosition === 0) { + return "" + } const lines = document.split("\n"); const definitionPattern = /\b(((async\s+)?\s*def)|\s*class)\b/g; diff --git a/src/parse/get_docstring_type.ts b/src/parse/get_docstring_type.ts index 15774dd..5aad09a 100644 --- a/src/parse/get_docstring_type.ts +++ b/src/parse/get_docstring_type.ts @@ -2,8 +2,6 @@ export function getDocstringType(functionDefinition: string, linePosition: numbe const class_pattern = /(?:class)/; const class_match = class_pattern.exec(functionDefinition); - console.log(class_match) - console.log(functionDefinition) if (linePosition === 0) { return "module" diff --git a/src/parse/parse.ts b/src/parse/parse.ts index 4bb0ef0..3ce8643 100644 --- a/src/parse/parse.ts +++ b/src/parse/parse.ts @@ -10,5 +10,10 @@ export function parse(document: string, positionLine: number): DocstringParts { const parameterTokens = tokenizeDefinition(definition); const functionName = getFunctionName(definition); + console.log(docstringType) + console.log(body) + console.log(parameterTokens) + console.log(functionName) + return parseParameters(docstringType, parameterTokens, body, functionName); } diff --git a/src/parse/parse_parameters.ts b/src/parse/parse_parameters.ts index 22bc267..4006546 100644 --- a/src/parse/parse_parameters.ts +++ b/src/parse/parse_parameters.ts @@ -197,19 +197,10 @@ function parseClasses(body: string[]): Class[] { for (const line of body) { if (indentationOf(line) === 0) { - - // console.log("class indentation match") - // console.log(line) - const match = line.match(pattern); if (match != null) { - // console.log("class match") - // console.log(match) - // let className = getClassName(line); let className = getFunctionName(line); - // console.log(className) - classes.push({ name: className, }); @@ -226,23 +217,12 @@ function parseMethods(body: string[]): Method[] { for (const line of body) { if (indentationOf(line) === 0) { - - // console.log("indentation = 0") - // console.log(line) - const match = line.match(pattern); if (match == null) { continue } - - // console.log("matches regex") - // console.log(match) - let methodName = getFunctionName(line); - // console.log("method name") - // console.log(methodName) - methods.push({ name: methodName, }); @@ -257,11 +237,8 @@ function parseAttributes(body: string[], args: Argument[], kwargs: KeywordArgume //const pattern = /(?:self).(\w+)?\s*/ for (const line of body) { - // console.log(line) const match = line.trim().match(pattern); - // console.log(match) - if (match == null) { continue; } @@ -275,8 +252,6 @@ function parseAttributes(body: string[], args: Argument[], kwargs: KeywordArgume } } - // console.log(attributes) - return attributes; } diff --git a/src/test/integration/integration.test.ts b/src/test/integration/integration.test.ts index 3b77e9f..4805706 100644 --- a/src/test/integration/integration.test.ts +++ b/src/test/integration/integration.test.ts @@ -167,6 +167,28 @@ describe("Basic Integration Tests", function () { }); }); + it("generates class docstring by parsing __init__ in file 7", async function () { + await testDocstringGeneration({ + expectedOutputFilePath: path.resolve( + __dirname, + "./python_test_files/file_7_output.py", + ), + inputFilePath: path.resolve(__dirname, "./python_test_files/file_7.py"), + position: new vsc.Position(1, 0), + }); + }); + + it("generates module docstring by parsing document and listting classes and methods in file 8", async function () { + await testDocstringGeneration({ + expectedOutputFilePath: path.resolve( + __dirname, + "./python_test_files/file_8_output.py", + ), + inputFilePath: path.resolve(__dirname, "./python_test_files/file_8.py"), + position: new vsc.Position(0, 0), + }); + }); + it("generates a docstring for the starlark function", async function () { await testDocstringGeneration({ expectedOutputFilePath: path.resolve( diff --git a/src/test/integration/python_test_files/depth_interpolator.py b/src/test/integration/python_test_files/depth_interpolator.py deleted file mode 100644 index 8d62b31..0000000 --- a/src/test/integration/python_test_files/depth_interpolator.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Union, Tuple - -import cv2 -import numpy as np -import matplotlib.pyplot as plt -from scipy.interpolate import griddata - - -class DepthInterpolator: - - def run(self, fg_rgb_image: np.ndarray, fg_depth_image: np.ndarray, - bg_rgb_image: np.ndarray, bg_depth_image: np.ndarray, - debug: bool): - - pass - - def __repr__(self): - - return self.__class__.__name__ - - -class DefaultInterpolator(DepthInterpolator): - - def run(self, fg_rgb_image: np.ndarray, fg_depth_image: np.ndarray, - bg_rgb_image: np.ndarray, bg_depth_image: np.ndarray, - debug: bool = False) -> np.ndarray: - - return np.asarray(fg_depth_image).copy() - - -class SciPyInterpolator(DepthInterpolator): - - def __init__(self, depth_threshold: int, rgb_threshold: int): - - self.depth_threshold = depth_threshold - self.rgb_threshold = rgb_threshold - - def _subtract_depth_background( - self, fg_depth_image: np.ndarray) -> np.ndarray: - - fg_depth_image = np.asarray(fg_depth_image).copy() - mask = fg_depth_image >= self.depth_threshold - fg_depth_image[mask] = self.depth_threshold - - return fg_depth_image - - def _subtract_rgb_background(self, fg_image: np.ndarray, - bg_image: np.ndarray) -> np.ndarray: - - fg_image = np.asarray(fg_image).copy() - bg_image = np.asarray(bg_image).copy() - - diff = bg_image.astype(np.float64) - fg_image.astype(np.float64) - diff = np.abs(diff) - diff = np.sum(diff, axis=2) - - mask = (diff >= self.rgb_threshold) - - return mask - - -class GridDataInterpolator(SciPyInterpolator): - - def __init__(self, method: str = 'linear', depth_threshold: int = 1000, - rgb_threshold: int = 175, rgb_bg_sub: bool = False): - - super().__init__(depth_threshold, rgb_threshold) - - self.method = method - self.rgb_bg_sub = rgb_bg_sub - - def run(self, fg_rgb_image: np.ndarray, fg_depth_image: np.ndarray, - bg_rgb_image: np.ndarray, bg_depth_image: np.ndarray, - debug: bool = False) -> np.ndarray: - - fg_depth_image = self._subtract_depth_background(fg_depth_image) - points, values = _prepare_data(fg_depth_image) - if self.rgb_bg_sub: - mask = self._subtract_rgb_background(fg_rgb_image, bg_rgb_image) - fg_depth_image[~mask] = self.depth_threshold - hole_rows, hole_cols = np.where(fg_depth_image == 0) - - h, w = fg_depth_image.shape - grid_x, grid_y = np.mgrid[0:h:1, 0:w:1] - grid_z = griddata(points, values, (grid_x, grid_y), method=self.method) - - for row, col in zip(hole_rows, hole_cols): - if not np.isnan(grid_z[row, col]): - fg_depth_image[row, col] = grid_z[row, col] - - if debug: - - _plot(grid_z) - _plot(fg_depth_image) - - return fg_depth_image - - def __repr__(self): - - return '{}: {}'.format(self.__class__.__name__, self.method) - - -class MeanGridDataInterpolator(SciPyInterpolator): - - def __init__(self, methods, weights, depth_threshold: int = 1000, - rgb_threshold: int = 175, rgb_bg_sub: bool = True): - - super().__init__(depth_threshold, rgb_threshold) - - self.methods = methods - self.weights = weights - self.rgb_bg_sub = rgb_bg_sub - self.interpolators = [ - GridDataInterpolator( - method, depth_threshold, rgb_threshold, rgb_bg_sub) - for method in self.methods - ] - - def run(self, fg_rgb_image: np.ndarray, fg_depth_image: np.ndarray, - bg_rgb_image: np.ndarray, bg_depth_image: np.ndarray, - debug: bool = False) -> np.ndarray: - - depth_images = [ - interpolator.run(fg_rgb_image, fg_depth_image, - bg_rgb_image, bg_depth_image, debug) - for interpolator - in self.interpolators - ] - - mean_depth_image = np.average(depth_images, - axis=0, - weights=self.weights) - - if debug: - - _plot(mean_depth_image) - - - return mean_depth_image.astype(np.uint16) - - def __repr__(self): - - return '{}: {}'.format(self.__class__.__name__, self.methods) - - -class ForegroundInterpolator(DepthInterpolator): - - def __init__(self, depth_threshold: int, rgb_threshold: int, dilate_1: int, - dilate_2: int, dilate_3: int, erode: int, min_area: int): - - self.depth_threshold = depth_threshold - self.rgb_threshold = rgb_threshold - self.dilate_1 = dilate_1 - self.dilate_2 = dilate_2 - self.dilate_3 = dilate_3 - self.erode = erode - self.min_area = min_area - - def run(self, fg_rgb_image: np.ndarray, fg_depth_image: np.ndarray, - bg_rgb_image: np.ndarray, bg_depth_image: np.ndarray, - debug: bool = False) -> np.ndarray: - - fg_rgb_image = np.asarray(fg_rgb_image).copy() - fg_depth_image = np.asarray(fg_depth_image).copy() - bg_rgb_image = np.asarray(bg_rgb_image).copy() - bg_depth_image = np.asarray(bg_depth_image).copy() - - fg_threshold_mask = fg_depth_image >= self.depth_threshold - fg_depth_image[fg_threshold_mask] = self.depth_threshold - fg_rgb_image[fg_threshold_mask] = 0 - - bg_threshold_mask = bg_depth_image >= self.depth_threshold - bg_depth_image[bg_threshold_mask] = self.depth_threshold - bg_rgb_image[bg_threshold_mask] = 0 - - _, bg_sub_mask = self._subtract_background( - fg_rgb_image, bg_rgb_image) - - bg_sub_rgb_mask = bg_sub_mask.astype(np.uint8) - ded_rgb_mask = self._dilate_erode_dilate(bg_sub_rgb_mask) - contours_rgb_mask = self._select_contours(ded_rgb_mask) - contours_rgb_mask = contours_rgb_mask.astype(bool) - - contours_fg_depth_image = fg_depth_image.copy() - contours_fg_depth_image[~contours_rgb_mask] = 0 - - kernel = np.ones((self.dilate_3, self.dilate_3)) - dilate_depth_mask = cv2.dilate( - contours_fg_depth_image, kernel, iterations=1) - hole_mask = contours_fg_depth_image.copy().astype(bool) - hole_fill_mask = (contours_rgb_mask) & (~hole_mask) - - hole_fill_depth_image = contours_fg_depth_image.copy() - hole_fill_depth_image[hole_fill_mask] = \ - dilate_depth_mask[hole_fill_mask] - - fg_depth_image[contours_rgb_mask] = \ - hole_fill_depth_image[contours_rgb_mask] - - if debug: - - _plot(fg_depth_image) - - return fg_depth_image - - def _subtract_background( - self, fg_image: np.ndarray, - bg_image: np.ndarray) -> Tuple[np.ndarray]: - - diff = bg_image.astype(np.float64) - fg_image.astype(np.float64) - diff = np.abs(diff) - diff = np.sum(diff, axis=2) - - #diff = np.sqrt(np.sum((fg.astype(np.float64) - bg.astype(np.float64))**2, axis=2)) - - mask = (diff >= self.rgb_threshold) - - bg_sub_fg_image = fg_image.copy() - bg_sub_fg_image[~mask] = 255 - - return bg_sub_fg_image, mask - - def _dilate_erode_dilate(self, mask: np.ndarray) -> np.ndarray: - - kernel_dilate_1 = np.ones((self.dilate_1, self.dilate_1)) - kernel_erode = np.ones((self.erode, self.erode)) - kernel_dilate_2 = np.ones((self.dilate_2, self.dilate_2)) - - mask_dilate_1 = cv2.dilate(mask, kernel_dilate_1, iterations=1) - mask_erode = cv2.erode(mask_dilate_1, kernel_erode, iterations=1) - mask_dilate_2 = cv2.dilate(mask_erode, kernel_dilate_2, iterations=1) - - return mask_dilate_2 - - def _select_contours(self, mask: np.ndarray) -> np.ndarray: - - mask_contoured = mask.copy() - contours, _ = cv2.findContours(mask.copy(), - cv2.RETR_EXTERNAL, - cv2.CHAIN_APPROX_NONE) - - for contour in contours: - x, y, w, h = cv2.boundingRect(contour) - if cv2.contourArea(contour) < self.min_area: - mask_contoured[y:y+h+5, x:x+w+5] = 0 - - return mask_contoured - - -def initialize_depth_interpolator( - method: str, settings: dict) -> Union[GridDataInterpolator, - ForegroundInterpolator, - MeanGridDataInterpolator, - DefaultInterpolator]: - - if method == 'grid': - return GridDataInterpolator(settings['method'], - settings['depth_threshold'], - settings['rgb_threshold'], - settings['rgb_bg_sub']) - - - elif method == 'foreground': - return ForegroundInterpolator(settings['depth_threshold'], - settings['rgb_threshold'], - settings['dilate_1'], - settings['dilate_2'], - settings['dilate_3'], - settings['erode'], - settings['min_area']) - - elif method == 'mean_grid': - return MeanGridDataInterpolator(settings['methods'], - settings['weights'], - settings['depth_threshold'], - settings['rgb_threshold'], - settings['rgb_bg_sub']) - - else: - return DefaultInterpolator() - - -def _plot(image): - - plt.figure(figsize=(8, 8)) - plt.imshow(image) - plt.show() - - -def _prepare_data(fg_depth_image: np.ndarray) -> Tuple[np.ndarray]: - - points = [] - values = [] - nonzero_rows, nonzero_cols = np.where(fg_depth_image != 0) - - for i, j in zip(nonzero_rows, nonzero_cols): - points.append((i, j)) - values.append(fg_depth_image[i, j]) - - return np.array(points), np.array(values) diff --git a/src/test/integration/python_test_files/dlc_project.py b/src/test/integration/python_test_files/dlc_project.py deleted file mode 100644 index 23e587f..0000000 --- a/src/test/integration/python_test_files/dlc_project.py +++ /dev/null @@ -1,206 +0,0 @@ -"""TODO -""" -import os -os.environ['DLCLight'] = 'True' -import glob -import datetime - -from deeplabcut import create_new_project as dlc_create_new_project - -from .deeplabcut_utils import load_yaml, dump_yaml - - -class DLCProject: - """Class to represent DeepLabCut project. Allows access to - config.yaml and all subdirectories in a DLC Project. - Parameters - ---------- - project_path : str - Full path to specific project directory. - config_init_path : str - Full path to yaml file that is used to specify any default - configuration settings that are to be overwritten along with - their values. - Attributes - ---------- - project_path : str - Full path to specific project directory. - config_path : str - Full path to specific project config.yaml file. - labeled_data_path : str - Full path to project labeled-data directory. - config_yaml : yaml - yaml object containing the project's configuration info. - """ - - def __init__(self, project_path: str, config_init_path: str = ''): - - self.project_path = project_path - self.config_path = os.path.join(project_path, 'config.yaml') - self.labeled_data_path = os.path.join(project_path, 'labeled-data') - if config_init_path: - self.config_yaml = self._initialize_config_yaml(config_init_path) - else: - self.config_yaml = load_yaml(self.config_path) - - @classmethod - def create_new_project( - cls, scorer: str, videos: list, task: str = 'TopDownPerson', - config_init_path: str = ( - '/home/shared/Projects/train-person-detector/configs/' - 'changes_to_config.yaml'), - working_directory: str = ( - '/home/shared/Projects/train-person-detector/dlc-projects')): - """Creates a new DeepLabCut project and DLCProject instance. - Parameters - ---------- - scorer : str - Name of scorer or experimenter. For Innovation Lab, this is - generally the name of the dataset e.g. data1-trace. - video_root : str - Path of the video files to be used just - before the camera name e.g. - /home/shared/mr_merchant_topdown/processed/20200302/data1-trace - task : str, default='TopDownPerson' - Name of task being performed in project. - config_init_path : str, default='changes_to_config.yaml' - YAML file containing all changes needing to be made to default - config.yaml created with new projects. - working_directory : str, default='/home/shared/Projects/train-person-detector/dlc-projects' - Root directory of 'dlc-projects' (differs between Linux and - Mac) - Returns - ------- - project : DLCProject - The DLCProject created from the given arguments. - """ - date = datetime.date.today().strftime('%Y-%m-%d') - project_name = f'{task}-{scorer}-{date}' - project_path = os.path.join(working_directory, project_name) - # videos = glob.glob(f'{video_root}*mp4') - - dlc_create_new_project(task, scorer, videos, copy_videos=False, - working_directory=working_directory) - project = DLCProject(project_path, config_init_path) - - return project - - @property - def scorer(self): - """Returns scorer from config_yaml""" - return self.config_yaml['scorer'] - - def _initialize_config_yaml(self, config_init_path): - """Updates the default config.yaml file with new values - specified in the yaml file located at config_init_path""" - config_yaml = load_yaml(self.config_path) - config_initialization_yaml = load_yaml(config_init_path) - - for key in config_initialization_yaml.keys(): - config_yaml[key] = config_initialization_yaml[key] - - dump_yaml(self.config_path, config_yaml) - - return config_yaml - - def _update_config_yaml(self, key, item): - """Updates the config.yaml at the given key with the given item. - """ - self.config_yaml[key] = item - dump_yaml(self.config_path, self.config_yaml) - - def _update_snapshotindex(self, snapshotindex): - """Sets the snapshotindex in the config.yaml file to a specified - value.""" - self._update_config_yaml('snapshotindex', snapshotindex) - - def _filepath_builder(self, subdir, filename): - """Builds filepaths to given filenames located in the project's - subdirectories.""" - iteration = self.config_yaml['iteration'] - task = self.config_yaml['Task'] - date = self.config_yaml['date'] - iteration_dir = f'iteration-{iteration}' - - if subdir in ['dlc-models', 'evaluation-results']: - training_fraction = self.config_yaml['TrainingFraction'][0] - training_fraction = str(training_fraction)[2:] - train_shuffle_dir = f'{task}{date}-trainset{training_fraction}shuffle1' - elif subdir == 'training-datasets': - train_shuffle_dir = f'UnaugmentedDataSet_{task}{date}' - - filepath = os.path.join(self.project_path, subdir, iteration_dir, - train_shuffle_dir, filename) - - return filepath - - def _update_pose_config_yaml(self, split, key, item): - """Updates the pose_config.yaml file used during training at - the given key with the given item.""" - filename = f'{split}/pose_config.yaml' - pose_config_path = self._filepath_builder('dlc-models', filename) - pose_config_yaml = load_yaml(pose_config_path) - pose_config_yaml[key] = item - dump_yaml(pose_config_path, pose_config_yaml) - - def update_pose_config_init_weights(self, init_weights): - """Updates the pose_config.yaml init_weights item in order to - carry out transfer learning or continue training from a - checkpoint.""" - self._update_pose_config_yaml('train', 'init_weights', init_weights) - self._update_pose_config_yaml('test', 'init_weights', init_weights) - - def __repr__(self): - - return self.config_path - - def get_labeled_data_by_camera(self, camera): - """Gets the labeled-data directory for a given camera/video.""" - return glob.glob(f'{self.labeled_data_path}/*{camera}')[0] - - def get_labeled_data_by_string(self, string): - """Gets the labeled-data directory for a given camera/video.""" - # print(string) - # print(glob.glob(f'{self.labeled_data_path}/{string}*')) - # print(os.path.join(self.labeled_data, string)) - #print(f'{self.labeled_data_path}/{string}*') - return glob.glob(f'{self.labeled_data_path}/{string}*')[0] - - def get_csv_by_string(self, string): - #print(f'{self.labeled_data_path}/{string}*/*csv') - return glob.glob(f'{self.labeled_data_path}/{string}*/*csv')[0] - - def get_labeled_data_dirs(self): - """Gets the labeled-data directory for all cameras.""" - return glob.glob(os.path.join(self.labeled_data_path, '*/')) - - def get_videos(self): - """Gets list of all videos used in project.""" - return os.listdir(self.labeled_data_path) - - def get_csv_filepath_by_video(self, video): - """Gets the CSV filepath for the given video.""" - csv_filename = f'*{self.scorer}*csv' - csv_glob = os.path.join(self.labeled_data_path, video, csv_filename) - return glob.glob(csv_glob)[0] - - -if __name__ == '__main__': - - # projects_root = '/home/shared/Projects' - # repo_name = 'train-person-detector' - SCORER = 'data1-trace-masked' - DATA_ROOT = '/home/shared/mr_merchant_topdown/processed' - DATASET_DATE = '20200302' - DATASET_DIR = os.path.join(DATA_ROOT, DATASET_DATE) - DATASET_NAME = 'data1-trace-masked' - VIDEO_ROOT = os.path.join(DATASET_DIR, DATASET_NAME) - CONFIG_INIT_PATH = '/home/shared/Projects/train-person-detector/configs/changes_to_config.yaml' - # project_root = '/home/shared/Projects/train-person-detector/projects' - # project_name = 'TopDownPerson-data1-10ids-exp1-2020-03-25' - # project = DLCProject(project_root, project_name) - - project = DLCProject.create_new_project(SCORER, VIDEO_ROOT, - config_init_path=CONFIG_INIT_PATH) - print(project) - print(project.config_yaml['TrainingFraction']) \ No newline at end of file diff --git a/src/test/integration/python_test_files/file_7.py b/src/test/integration/python_test_files/file_7.py new file mode 100644 index 0000000..361f6d5 --- /dev/null +++ b/src/test/integration/python_test_files/file_7.py @@ -0,0 +1,10 @@ +class TestClass(object): + + def __init__(self, arg1: int, arg2: str, arg3: float = None): + self.arg1 = arg1 + self.arg2 = arg2 + self.arg3 = arg3 + self.arg4 = self._get_arg4() + + def _get_arg4(self): + return 4 diff --git a/src/test/integration/python_test_files/file_7_output.py b/src/test/integration/python_test_files/file_7_output.py new file mode 100644 index 0000000..e41776e --- /dev/null +++ b/src/test/integration/python_test_files/file_7_output.py @@ -0,0 +1,18 @@ +class TestClass(object): + """[summary] + + :param arg1: [description] + :type arg1: int + :param arg2: [description] + :type arg2: str + :param arg3: [description], defaults to None + :type arg3: float, optional + """ + def __init__(self, arg1: int, arg2: str, arg3: float = None): + self.arg1 = arg1 + self.arg2 = arg2 + self.arg3 = arg3 + self.arg4 = self._get_arg4() + + def _get_arg4(self): + return 4 diff --git a/src/test/integration/python_test_files/file_8.py b/src/test/integration/python_test_files/file_8.py new file mode 100644 index 0000000..5239d6b --- /dev/null +++ b/src/test/integration/python_test_files/file_8.py @@ -0,0 +1,40 @@ + +from typing import Union, List, Dict, Thing, Generator, Tuple + + +class TestClass(object): + + def __init__(self, arg1: int, arg2: str, arg3: float = None): + self.arg1 = arg1 + self.arg2 = arg2 + self.arg3 = arg3 + self.arg4 = self._get_arg4() + + def _get_arg4(self): + return 4 + + +def function_1(arg1: int) -> str: # comment + + if arg1 > 1: + raise FileExistsError() # comment + + return "abc" # comment + + +def function_2(arg1, arg2, kwarg1=1): + + if arg2 > 1: + raise FileExistsError() + + yield 1 + return arg1 + + +def function_3( + arg1: int, + arg2: Union[List[str], Dict[str, int], Thing], + kwarg1: int = 1 +) -> Generator[Tuple[str, str]]: + + yield ("abc", "def") diff --git a/src/test/integration/python_test_files/file_8_output.py b/src/test/integration/python_test_files/file_8_output.py new file mode 100644 index 0000000..8328cd1 --- /dev/null +++ b/src/test/integration/python_test_files/file_8_output.py @@ -0,0 +1,51 @@ +"""[summary] + +Classes +------- +TestClass + +Methods +------- +function_1 +function_2 +function_3 +""" +from typing import Union, List, Dict, Thing, Generator, Tuple + + +class TestClass(object): + + def __init__(self, arg1: int, arg2: str, arg3: float = None): + self.arg1 = arg1 + self.arg2 = arg2 + self.arg3 = arg3 + self.arg4 = self._get_arg4() + + def _get_arg4(self): + return 4 + + +def function_1(arg1: int) -> str: # comment + + if arg1 > 1: + raise FileExistsError() # comment + + return "abc" # comment + + +def function_2(arg1, arg2, kwarg1=1): + + if arg2 > 1: + raise FileExistsError() + + yield 1 + return arg1 + + +def function_3( + arg1: int, + arg2: Union[List[str], Dict[str, int], Thing], + kwarg1: int = 1 +) -> Generator[Tuple[str, str]]: + + yield ("abc", "def") diff --git a/src/test/parse/get_body.spec.ts b/src/test/parse/get_body.spec.ts index 4a588a3..c77a7e0 100644 --- a/src/test/parse/get_body.spec.ts +++ b/src/test/parse/get_body.spec.ts @@ -7,55 +7,109 @@ chai.config.truncateThreshold = 0; const expect = chai.expect; describe("getBody()", () => { - it("should return the body of a function", () => { - const result = getBody("method", basicFunction, 4); - - expect(result).to.have.deep.members([ - 'print("HELLO WORLD")', - "try:", - "something()", - "except Error:", - "raise SomethingWentWrong", - "return 3", - ]); - }); + context("when encoutering a function", () => { + it("should return the body of a function", () => { + const result = getBody("method", basicFunction, 4); - it("should skip blank lines", () => { - const result = getBody("method", gapFunction, 5); + expect(result).to.have.deep.members([ + 'print("HELLO WORLD")', + "try:", + "something()", + "except Error:", + "raise SomethingWentWrong", + "return 3", + ]); + }); - expect(result).to.have.deep.members(["print('HELLO WORLD')", "print('HELLO AGAIN')"]); - }); + it("should skip blank lines", () => { + const result = getBody("method", gapFunction, 5); - it("should skip comment lines", () => { - const result = getBody("method", commentFunction, 5); + expect(result).to.have.deep.members(["print('HELLO WORLD')", "print('HELLO AGAIN')"]); + }); - expect(result).to.have.deep.members(["print('HELLO AGAIN')"]); - }); + it("should skip comment lines", () => { + const result = getBody("method", commentFunction, 5); - it("should handle multi line definitions", () => { - const result = getBody("method", multiLineDefFunction, 4); + expect(result).to.have.deep.members(["print('HELLO AGAIN')"]); + }); - expect(result).to.have.deep.members(["pass"]); - }); + it("should handle multi line definitions", () => { + const result = getBody("method", multiLineDefFunction, 4); + + expect(result).to.have.deep.members(["pass"]); + }); + + it("should handle indented functions", () => { + const result = getBody("method", indentedFunctions, 3); + + expect(result).to.have.deep.members(["return 2"]); + + const result2 = getBody("method", indentedFunctions, 6); - it("should handle indented functions", () => { - const result = getBody("method", indentedFunctions, 3); + expect(result2).to.have.deep.members(["pass"]); + }); - expect(result).to.have.deep.members(["return 2"]); + it("should return an empty array if a function has no body", () => { + const result = getBody("method", noBody, 2); - const result2 = getBody("method", indentedFunctions, 6); + expect(result).to.have.deep.members([]); - expect(result2).to.have.deep.members(["pass"]); + const result2 = getBody("method", noBody, 4); + + expect(result2).to.have.deep.members([]); + }); }); + context("when encoutering a class", () => { + it("should return the body of a function", () => { + const result = getBody("method", basicFunction, 4); + + expect(result).to.have.deep.members([ + 'print("HELLO WORLD")', + "try:", + "something()", + "except Error:", + "raise SomethingWentWrong", + "return 3", + ]); + }); + + it("should skip blank lines", () => { + const result = getBody("method", gapFunction, 5); + + expect(result).to.have.deep.members(["print('HELLO WORLD')", "print('HELLO AGAIN')"]); + }); + + it("should skip comment lines", () => { + const result = getBody("method", commentFunction, 5); + + expect(result).to.have.deep.members(["print('HELLO AGAIN')"]); + }); + + it("should handle multi line definitions", () => { + const result = getBody("method", multiLineDefFunction, 4); - it("should return an empty array if a function has no body", () => { - const result = getBody("method", noBody, 2); + expect(result).to.have.deep.members(["pass"]); + }); - expect(result).to.have.deep.members([]); + it("should handle indented functions", () => { + const result = getBody("method", indentedFunctions, 3); - const result2 = getBody("method", noBody, 4); + expect(result).to.have.deep.members(["return 2"]); - expect(result2).to.have.deep.members([]); + const result2 = getBody("method", indentedFunctions, 6); + + expect(result2).to.have.deep.members(["pass"]); + }); + + it("should return an empty array if a function has no body", () => { + const result = getBody("method", noBody, 2); + + expect(result).to.have.deep.members([]); + + const result2 = getBody("method", noBody, 4); + + expect(result2).to.have.deep.members([]); + }); }); }); @@ -128,4 +182,26 @@ def next_no_body(): `; +const basicClass = ` +Something Else + +class BasicClass(object): + + def __init__(self, param1): + self.param1 = param1 + + def hello(self): + print("Hello world") + +class AnotherBasicClass(object): + + def __init__( + self, param2 + ): + self.param2 = param2 + + def hello(self): + print("Goodbye world") +`; + const docstringType = 'method'; diff --git a/src/test/parse/get_docstring_indentation.spec.ts b/src/test/parse/get_docstring_indentation.spec.ts index 8539d00..1bf0f22 100644 --- a/src/test/parse/get_docstring_indentation.spec.ts +++ b/src/test/parse/get_docstring_indentation.spec.ts @@ -43,11 +43,18 @@ describe("getDocstringIndentation()", () => { expect(result).to.equal("\t\t"); }); - it("should default to default indentation if no indentation is found", () => { - const result = getDocstringIndentation("", 0, " ".repeat(4)); + it("should default to default indentation if no indentation is found and line position is not 0", () => { + const result = getDocstringIndentation(" \n \n", 1, " ".repeat(4)); expect(result).to.equal(" ".repeat(4)); }); + + it("should default to no space if no indentation is found and line position is 0", () => { + const result = getDocstringIndentation("", 0, " ".repeat(4)); + + expect(result).to.equal(""); + }); + }); const fourSpaceIndentation = ` diff --git a/src/test/parse/parse_parameters.spec.ts b/src/test/parse/parse_parameters.spec.ts index 4eb0ae4..4f879cc 100644 --- a/src/test/parse/parse_parameters.spec.ts +++ b/src/test/parse/parse_parameters.spec.ts @@ -7,63 +7,150 @@ chai.config.truncateThreshold = 0; const expect = chai.expect; describe("parseParameters()", () => { - it("should parse an array of strings into a docstring struct", () => { - const parameterTokens = [ - "@decorator1", - "@decorator2", - "param1", - "param2: int", - "param3 = 1", - "param4: str = 'abc'", - "-> int", - ]; + context("when encoutering a method", () => { + it("should parse an array of strings into a docstring struct", () => { + const parameterTokens = [ + "@decorator1", + "@decorator2", + "param1", + "param2: int", + "param3 = 1", + "param4: str = 'abc'", + "-> int", + ]; + + const body = [" raise Exception", "raise Exception2"]; + + const functionName = "function"; + const docstringType = "method"; + const result = parseParameters(docstringType, parameterTokens, body, functionName); + + expect(result).to.eql({ + name: "function", + decorators: [{ name: "decorator1" }, { name: "decorator2" }], + args: [ + { var: "param1", type: undefined }, + { var: "param2", type: "int" }, + ], + kwargs: [ + { var: "param3", default: "1", type: "int" }, + { var: "param4", default: "'abc'", type: "str" }, + ], + returns: { type: "int" }, + yields: undefined, + exceptions: [{ type: "Exception" }, { type: "Exception2" }], + classes: [], + methods: [], + attributes: [], + }); + }); - const body = [" raise Exception", "raise Exception2"]; + it("should parse args with and without type hints", () => { + const parameterTokens = ["param1: List[string]", "param2"]; + const docstringType = "method"; + const result = parseParameters(docstringType, parameterTokens, [], "name"); - const functionName = "function"; - const docstringType = "method"; - const result = parseParameters(docstringType, parameterTokens, body, functionName); - - expect(result).to.eql({ - name: "function", - decorators: [{ name: "decorator1" }, { name: "decorator2" }], - args: [ - { var: "param1", type: undefined }, - { var: "param2", type: "int" }, - ], - kwargs: [ - { var: "param3", default: "1", type: "int" }, - { var: "param4", default: "'abc'", type: "str" }, - ], - returns: { type: "int" }, - yields: undefined, - exceptions: [{ type: "Exception" }, { type: "Exception2" }], - classes: [], - methods: [], - attributes: [], + expect(result.args).to.have.deep.members([ + { var: "param1", type: "List[string]" }, + { var: "param2", type: undefined }, + ]); }); - }); - it("should parse args with and without type hints", () => { - const parameterTokens = ["param1: List[string]", "param2"]; - const docstringType = "method"; - const result = parseParameters(docstringType, parameterTokens, [], "name"); + it("should parse kwargs with and without type hints", () => { + const parameterTokens = ["param1: List[int] = [1,2]", "param2 = 'abc'"]; + const docstringType = "method"; + const result = parseParameters(docstringType, parameterTokens, [], "name"); - expect(result.args).to.have.deep.members([ - { var: "param1", type: "List[string]" }, - { var: "param2", type: undefined }, - ]); + expect(result.kwargs).to.have.deep.members([ + { var: "param1", default: "[1,2]", type: "List[int]" }, + { var: "param2", default: "'abc'", type: "str" }, + ]); + }); }); - it("should parse kwargs with and without type hints", () => { - const parameterTokens = ["param1: List[int] = [1,2]", "param2 = 'abc'"]; - const docstringType = "method"; - const result = parseParameters(docstringType, parameterTokens, [], "name"); + context("when encoutering a class", () => { + it("should return only parameters when no attributes are defined in body", () => { + const parameterTokens = [ + "@decorator1", + "@decorator2", + "param1", + "param2: int", + "param3 = 1", + "param4: str = 'abc'", + "-> int", + ]; + + const body = [" raise Exception", "raise Exception2"]; + + const functionName = "testClass"; + const docstringType = "class"; + const result = parseParameters(docstringType, parameterTokens, body, functionName); + + expect(result).to.eql({ + name: "testClass", + decorators: [{ name: "decorator1" }, { name: "decorator2" }], + args: [ + { var: "param1", type: undefined }, + { var: "param2", type: "int" }, + ], + kwargs: [ + { var: "param3", default: "1", type: "int" }, + { var: "param4", default: "'abc'", type: "str" }, + ], + returns: undefined, + yields: undefined, + exceptions: [], + classes: [], + methods: [], + attributes: [], + }); + }); + + it("should return only parameters when no attributes are defined in body", () => { + const parameterTokens = [ + "@decorator1", + "@decorator2", + "param1", + "param2: int", + "param3 = 1", + "param4: str = 'abc'", + "-> int", + ]; + + const body = ["", + " def __init__(self, param1, param2: int, param3 = 1, param4: str = 'abc'):", + " self.param1 = param1", + " self.param2 = param2", + " self.param3 = param3", + " self.param4 = param4", + " self.param5 = 7"]; + + const functionName = "testClass"; + const docstringType = "class"; + const result = parseParameters(docstringType, parameterTokens, body, functionName); + + expect(result).to.eql({ + name: "testClass", + decorators: [{ name: "decorator1" }, { name: "decorator2" }], + args: [ + { var: "param1", type: undefined }, + { var: "param2", type: "int" }, + ], + kwargs: [ + { var: "param3", default: "1", type: "int" }, + { var: "param4", default: "'abc'", type: "str" }, + ], + returns: undefined, + yields: undefined, + exceptions: [], + classes: [], + methods: [], + attributes: [ + { var: "param5", type: undefined} + ], + }); + }); - expect(result.kwargs).to.have.deep.members([ - { var: "param1", default: "[1,2]", type: "List[int]" }, - { var: "param2", default: "'abc'", type: "str" }, - ]); }); describe("parseReturns", () => { From 29442b71d8fe9baeea55df924058fe3ef75b9f0c Mon Sep 17 00:00:00 2001 From: tracetidwell Date: Fri, 29 Jan 2021 14:09:05 -0500 Subject: [PATCH 13/18] updated changelog; cleaning up code --- CHANGELOG.md | 4 ++ src/parse/get_body.ts | 10 +++- src/parse/get_definition.ts | 88 ++++++++++++++++++----------------- src/parse/parse_parameters.ts | 68 +++++++++++++++++---------- 4 files changed, 101 insertions(+), 69 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bacdb1c..609b3c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Change Log +## [0.5.5] - 2021-01-17 + +- Added class and module docstrings (@tracetidwell) + ## [0.5.4] - 2020-11-17 - Added Starlark support #139 (@UebelAndre) diff --git a/src/parse/get_body.ts b/src/parse/get_body.ts index 87f8b34..4732ab9 100644 --- a/src/parse/get_body.ts +++ b/src/parse/get_body.ts @@ -2,7 +2,6 @@ import { blankLine, indentationOf, preprocessLines } from "./utilities"; export function getBody(docstringType: string, document: string, linePosition: number): string[] { const lines = document.split("\n"); - const body = []; let regex = '\s*def \w+' if (docstringType === 'module') { @@ -16,9 +15,16 @@ export function getBody(docstringType: string, document: string, linePosition: n const originalIndentation = getBodyBaseIndentation(lines, linePosition, regex); if (originalIndentation === 0) { - return body; + return []; } + return populateBody(currentLineNum, lines, originalIndentation); + +} + +function populateBody(currentLineNum: number, lines: string[], originalIndentation: number) { + + const body = []; while (currentLineNum < lines.length) { const line = lines[currentLineNum]; diff --git a/src/parse/get_definition.ts b/src/parse/get_definition.ts index 1cc97cd..6d9822c 100644 --- a/src/parse/get_definition.ts +++ b/src/parse/get_definition.ts @@ -27,57 +27,61 @@ export function getDefinition(document: string, linePosition: number): string { return ""; } - const lastFunctionDef = precedingText.slice(index).trim(); + let lastFunctionDef = precedingText.slice(index).trim(); if (lastFunctionDef.startsWith('class')) { - const lines = document.split("\n"); - - const originalIndentation = indentationOf(lines[linePosition]); - const classPattern = /(?:class)\s+(\w+)/; - const classMatch = classPattern.exec(lastFunctionDef); - let definition = classMatch[0]; - // const initPattern = /(?:def __init__)/; - const initPattern = /(?<=def __init__).*/; - const defClosePattern = /(\))/ - - while (linePosition < lines.length) { - const line = lines[linePosition]; - const initMatch = initPattern.exec(line) - - if (initMatch != undefined && initMatch[0] != undefined) { - definition += initMatch[0]; - const newIndentation = indentationOf(lines[linePosition]); - let defCloseMatch = defClosePattern.exec(line); - if (defCloseMatch != undefined && defCloseMatch[0] != undefined) { - return definition; - } - linePosition += 1; + lastFunctionDef = getClassDefinition(document, lastFunctionDef, linePosition) + } - while (linePosition < lines.length) { - const line = lines[linePosition]; - definition += line.trim(); - defCloseMatch = defClosePattern.exec(line); - if (indentationOf(line) < newIndentation || blankLine(line)) { - return definition; - } - else if (defCloseMatch != undefined && defCloseMatch[0] != undefined) { - return definition; - } - - linePosition += 1; - } - - } - else if (indentationOf(line) <= originalIndentation && !blankLine(line)) { + return lastFunctionDef; +} + +function getClassDefinition(document: string, lastFunctionDef: string, linePosition: number): string { + + const lines = document.split("\n"); + const originalIndentation = indentationOf(lines[linePosition]); + const classPattern = /(?:class)\s+(\w+)/; + const classMatch = classPattern.exec(lastFunctionDef); + let definition = classMatch[0]; + // const initPattern = /(?:def __init__)/; + const initPattern = /(?<=def __init__).*/; + const defClosePattern = /(\))/ + + while (linePosition < lines.length) { + const line = lines[linePosition]; + const initMatch = initPattern.exec(line) + + if (initMatch != undefined && initMatch[0] != undefined) { + definition += initMatch[0]; + const newIndentation = indentationOf(lines[linePosition]); + let defCloseMatch = defClosePattern.exec(line); + if (defCloseMatch != undefined && defCloseMatch[0] != undefined) { return definition; } linePosition += 1; - } - return definition + while (linePosition < lines.length) { + const line = lines[linePosition]; + definition += line.trim(); + defCloseMatch = defClosePattern.exec(line); + if (indentationOf(line) < newIndentation || blankLine(line)) { + return definition; + } + else if (defCloseMatch != undefined && defCloseMatch[0] != undefined) { + return definition; + } + + linePosition += 1; + } + + } + else if (indentationOf(line) <= originalIndentation && !blankLine(line)) { + return definition; + } + linePosition += 1; } - return lastFunctionDef; + return definition } function getPrecedingLines(document: string, linePosition: number): string[] { diff --git a/src/parse/parse_parameters.ts b/src/parse/parse_parameters.ts index 4006546..bb4f287 100644 --- a/src/parse/parse_parameters.ts +++ b/src/parse/parse_parameters.ts @@ -15,27 +15,10 @@ import { Attribute } from "../docstring_parts"; -export function parseParameters( - docstringType: string, +function parseMethod( parameterTokens: string[], body: string[], functionName: string): DocstringParts { - - if (docstringType === "module") { - return { - name: functionName, - decorators: [], - args: [], - kwargs: [], - returns: undefined, - yields: undefined, - exceptions: [], - classes: parseClasses(body), - methods: parseMethods(body), - attributes: [] - }; - } - else if (docstringType === "method") { return { name: functionName, decorators: parseDecorators(parameterTokens), @@ -48,8 +31,12 @@ export function parseParameters( methods: [], attributes: [] }; - } - else if (docstringType === "class") { +} + +function parseClass( + parameterTokens: string[], + body: string[], + functionName: string): DocstringParts { let args = parseArguments(parameterTokens); let kwargs = parseKeywordArguments(parameterTokens); return { @@ -64,6 +51,39 @@ export function parseParameters( methods: [], attributes: parseAttributes(body, args, kwargs) }; +} + +function parseModule( + body: string[], + functionName: string): DocstringParts { + return { + name: functionName, + decorators: [], + args: [], + kwargs: [], + returns: undefined, + yields: undefined, + exceptions: [], + classes: parseClasses(body), + methods: parseMethods(body), + attributes: [] + }; +} + +export function parseParameters( + docstringType: string, + parameterTokens: string[], + body: string[], + functionName: string): DocstringParts { + + if (docstringType === "module") { + return parseModule(body, functionName); + } + else if (docstringType === "method") { + return parseMethod(parameterTokens, body, functionName); + } + else if (docstringType === "class") { + return parseClass(parameterTokens, body, functionName); }; } @@ -200,9 +220,8 @@ function parseClasses(body: string[]): Class[] { const match = line.match(pattern); if (match != null) { - let className = getFunctionName(line); classes.push({ - name: className, + name: getFunctionName(line), }); } } @@ -211,7 +230,7 @@ function parseClasses(body: string[]): Class[] { } function parseMethods(body: string[]): Method[] { - const methods: Class[] = [] + const methods: Method[] = [] const pattern = /(def)\s+(\w+)\s*\(/; for (const line of body) { @@ -222,9 +241,8 @@ function parseMethods(body: string[]): Method[] { if (match == null) { continue } - let methodName = getFunctionName(line); methods.push({ - name: methodName, + name: getFunctionName(line), }); } } From c8ad6676166a858443935b989c0df095985ea933 Mon Sep 17 00:00:00 2001 From: tracetidwell Date: Fri, 29 Jan 2021 14:32:34 -0500 Subject: [PATCH 14/18] refactoring for code climate compliance --- src/parse/get_definition.ts | 58 +++++++++++++++-------------------- src/parse/parse_parameters.ts | 16 +++++----- 2 files changed, 32 insertions(+), 42 deletions(-) diff --git a/src/parse/get_definition.ts b/src/parse/get_definition.ts index 6d9822c..3b8ebfc 100644 --- a/src/parse/get_definition.ts +++ b/src/parse/get_definition.ts @@ -15,20 +15,12 @@ export function getDefinition(document: string, linePosition: number): string { return ""; } - const pattern = /\b(((async\s+)?\s*def)|\s*class)\b/g; - - // Get starting index of last def match in the preceding text - let index: number; - while (pattern.test(precedingText)) { - index = pattern.lastIndex - RegExp.lastMatch.length; - } - + const index = getIndex(precedingText) if (index == undefined) { return ""; } let lastFunctionDef = precedingText.slice(index).trim(); - if (lastFunctionDef.startsWith('class')) { lastFunctionDef = getClassDefinition(document, lastFunctionDef, linePosition) } @@ -49,34 +41,22 @@ function getClassDefinition(document: string, lastFunctionDef: string, linePosit while (linePosition < lines.length) { const line = lines[linePosition]; - const initMatch = initPattern.exec(line) + - if (initMatch != undefined && initMatch[0] != undefined) { - definition += initMatch[0]; - const newIndentation = indentationOf(lines[linePosition]); - let defCloseMatch = defClosePattern.exec(line); + if (indentationOf(line) <= originalIndentation && !blankLine(line)) { + return definition; + } + else { + + const initMatch = initPattern.exec(line) + if (initMatch != undefined && initMatch[0] != undefined) { + definition += initMatch[0]; + } + + const defCloseMatch = defClosePattern.exec(line); if (defCloseMatch != undefined && defCloseMatch[0] != undefined) { return definition; } - linePosition += 1; - - while (linePosition < lines.length) { - const line = lines[linePosition]; - definition += line.trim(); - defCloseMatch = defClosePattern.exec(line); - if (indentationOf(line) < newIndentation || blankLine(line)) { - return definition; - } - else if (defCloseMatch != undefined && defCloseMatch[0] != undefined) { - return definition; - } - - linePosition += 1; - } - - } - else if (indentationOf(line) <= originalIndentation && !blankLine(line)) { - return definition; } linePosition += 1; } @@ -84,6 +64,18 @@ function getClassDefinition(document: string, lastFunctionDef: string, linePosit return definition } +function getIndex(precedingText: string): number { + const pattern = /\b(((async\s+)?\s*def)|\s*class)\b/g; + + // Get starting index of last def match in the preceding text + let index: number; + while (pattern.test(precedingText)) { + index = pattern.lastIndex - RegExp.lastMatch.length; + } + + return index +} + function getPrecedingLines(document: string, linePosition: number): string[] { const lines = document.split("\n"); const rawPrecedingLines = lines.slice(0, linePosition); diff --git a/src/parse/parse_parameters.ts b/src/parse/parse_parameters.ts index bb4f287..fdc9eea 100644 --- a/src/parse/parse_parameters.ts +++ b/src/parse/parse_parameters.ts @@ -215,16 +215,14 @@ function parseClasses(body: string[]): Class[] { const pattern = /(?:class)\s+(\w+)\s*\(*/; for (const line of body) { - - if (indentationOf(line) === 0) { - const match = line.match(pattern); - - if (match != null) { - classes.push({ - name: getFunctionName(line), - }); - } + // if (indentationOf(line) === 0) { + const match = line.match(pattern); + if (indentationOf(line) === 0 && match != null) { + classes.push({ + name: getFunctionName(line), + }); } + // } } return classes; } From 187d850e02fd44a549651fc4ef8b8904a84a6262 Mon Sep 17 00:00:00 2001 From: tracetidwell Date: Fri, 29 Jan 2021 15:06:46 -0500 Subject: [PATCH 15/18] refactoring --- src/parse/get_definition.ts | 53 +++++++++++++++++++++++++---------- src/parse/parse_parameters.ts | 11 ++------ 2 files changed, 41 insertions(+), 23 deletions(-) diff --git a/src/parse/get_definition.ts b/src/parse/get_definition.ts index 3b8ebfc..6e965ae 100644 --- a/src/parse/get_definition.ts +++ b/src/parse/get_definition.ts @@ -32,30 +32,29 @@ function getClassDefinition(document: string, lastFunctionDef: string, linePosit const lines = document.split("\n"); const originalIndentation = indentationOf(lines[linePosition]); - const classPattern = /(?:class)\s+(\w+)/; - const classMatch = classPattern.exec(lastFunctionDef); - let definition = classMatch[0]; - // const initPattern = /(?:def __init__)/; - const initPattern = /(?<=def __init__).*/; - const defClosePattern = /(\))/ + let definition = getClassName(lastFunctionDef); while (linePosition < lines.length) { - const line = lines[linePosition]; - + const line = lines[linePosition]; if (indentationOf(line) <= originalIndentation && !blankLine(line)) { return definition; } else { - const initMatch = initPattern.exec(line) - if (initMatch != undefined && initMatch[0] != undefined) { - definition += initMatch[0]; + if (isInitMatch(line) && isCloseDefMatch(line)) { + return definition + getInitMatch(line); + + } + else if (isInitMatch(line)) { + definition += getInitMatch(line); + } - - const defCloseMatch = defClosePattern.exec(line); - if (defCloseMatch != undefined && defCloseMatch[0] != undefined) { - return definition; + else if (isCloseDefMatch(line)) { + return definition + line.trim(); + } + else { + definition += line.trim(); } } linePosition += 1; @@ -64,6 +63,30 @@ function getClassDefinition(document: string, lastFunctionDef: string, linePosit return definition } +function isInitMatch(line: string): boolean { + // const initPattern = /(?<=def __init__).*/; + const initPattern = /(?:def __init__)/; + const initMatch = initPattern.exec(line) + return initMatch != undefined && initMatch[0] != undefined; +} + +function getInitMatch(line: string): string { + const initPattern = /(?<=def __init__).*/; + return initPattern.exec(line)[0].trim() +} + +function isCloseDefMatch(line: string): boolean { + const defClosePattern = /(\))/ + const defCloseMatch = defClosePattern.exec(line); + return defCloseMatch != undefined && defCloseMatch[0] != undefined; +} + +function getClassName(lastFunctionDef: string): string { + const classPattern = /(?:class)\s+(\w+)/; + const classMatch = classPattern.exec(lastFunctionDef); + return classMatch[0]; +} + function getIndex(precedingText: string): number { const pattern = /\b(((async\s+)?\s*def)|\s*class)\b/g; diff --git a/src/parse/parse_parameters.ts b/src/parse/parse_parameters.ts index fdc9eea..fc8255f 100644 --- a/src/parse/parse_parameters.ts +++ b/src/parse/parse_parameters.ts @@ -215,14 +215,13 @@ function parseClasses(body: string[]): Class[] { const pattern = /(?:class)\s+(\w+)\s*\(*/; for (const line of body) { - // if (indentationOf(line) === 0) { + const match = line.match(pattern); if (indentationOf(line) === 0 && match != null) { classes.push({ name: getFunctionName(line), }); } - // } } return classes; } @@ -233,12 +232,8 @@ function parseMethods(body: string[]): Method[] { for (const line of body) { - if (indentationOf(line) === 0) { - const match = line.match(pattern); - - if (match == null) { - continue - } + const match = line.match(pattern); + if (indentationOf(line) === 0 && match != null) { methods.push({ name: getFunctionName(line), }); From b6eb8a41b532b8d8e8707ea09e05d722b9b987dd Mon Sep 17 00:00:00 2001 From: tracetidwell Date: Fri, 29 Jan 2021 15:29:45 -0500 Subject: [PATCH 16/18] still simplifying get_definition --- src/docstring/template_data.ts | 3 +-- src/docstring_parts.ts | 8 ++++---- src/parse/get_definition.ts | 37 +++++++++++++++++++--------------- src/parse/parse_parameters.ts | 27 +++++-------------------- 4 files changed, 31 insertions(+), 44 deletions(-) diff --git a/src/docstring/template_data.ts b/src/docstring/template_data.ts index 3aa3b69..34104cc 100644 --- a/src/docstring/template_data.ts +++ b/src/docstring/template_data.ts @@ -6,7 +6,6 @@ import { KeywordArgument, Returns, Yields, - Class, Method, Attribute } from "../docstring_parts"; @@ -20,7 +19,7 @@ export class TemplateData { public exceptions: Exception[]; public returns: Returns; public yields: Yields; - public classes: Class[]; + public classes: Method[]; public methods: Method[]; public attributes: Attribute[]; diff --git a/src/docstring_parts.ts b/src/docstring_parts.ts index 7409c32..995f6b1 100644 --- a/src/docstring_parts.ts +++ b/src/docstring_parts.ts @@ -6,7 +6,7 @@ export interface DocstringParts { exceptions: Exception[]; returns: Returns; yields: Yields; - classes: Class[]; + classes: Method[]; methods: Method[]; attributes: Attribute[]; } @@ -38,9 +38,9 @@ export interface Yields { type: string; } -export interface Class { - name: string; -} +// export interface Class { +// name: string; +// } export interface Method { name: string; diff --git a/src/parse/get_definition.ts b/src/parse/get_definition.ts index 6e965ae..093bd1f 100644 --- a/src/parse/get_definition.ts +++ b/src/parse/get_definition.ts @@ -40,22 +40,17 @@ function getClassDefinition(document: string, lastFunctionDef: string, linePosit if (indentationOf(line) <= originalIndentation && !blankLine(line)) { return definition; } - else { - - if (isInitMatch(line) && isCloseDefMatch(line)) { - return definition + getInitMatch(line); - - } - else if (isInitMatch(line)) { - definition += getInitMatch(line); - - } - else if (isCloseDefMatch(line)) { - return definition + line.trim(); - } - else { - definition += line.trim(); - } + + // if (isInitMatch(line)) { + // definition += getInitMatch(line); + // } + // else { + // definition += line.trim(); + // } + definition = updateDefinition(definition, line) + + if (isCloseDefMatch(line)) { + return definition } linePosition += 1; } @@ -63,6 +58,16 @@ function getClassDefinition(document: string, lastFunctionDef: string, linePosit return definition } +function updateDefinition(definition: string, line: string): string { + if (isInitMatch(line)) { + definition += getInitMatch(line); + } + else { + definition += line.trim(); + } + return definition +} + function isInitMatch(line: string): boolean { // const initPattern = /(?<=def __init__).*/; const initPattern = /(?:def __init__)/; diff --git a/src/parse/parse_parameters.ts b/src/parse/parse_parameters.ts index fc8255f..5e358ba 100644 --- a/src/parse/parse_parameters.ts +++ b/src/parse/parse_parameters.ts @@ -1,7 +1,6 @@ import { guessType } from "."; import { indentationOf } from "./utilities"; import { getFunctionName } from "./get_function_name"; -import { getClassName } from "./get_class_name"; import { Argument, Decorator, @@ -10,7 +9,6 @@ import { KeywordArgument, Returns, Yields, - Class, Method, Attribute } from "../docstring_parts"; @@ -64,8 +62,8 @@ function parseModule( returns: undefined, yields: undefined, exceptions: [], - classes: parseClasses(body), - methods: parseMethods(body), + classes: parseMethods(body, /(?:class)\s/), + methods: parseMethods(body, /(def)\s+(\w+)\s*\(/), attributes: [] }; } @@ -210,25 +208,10 @@ function parseExceptions(body: string[]): Exception[] { return exceptions; } -function parseClasses(body: string[]): Class[] { - const classes: Class[] = [] - const pattern = /(?:class)\s+(\w+)\s*\(*/; - - for (const line of body) { - - const match = line.match(pattern); - if (indentationOf(line) === 0 && match != null) { - classes.push({ - name: getFunctionName(line), - }); - } - } - return classes; -} - -function parseMethods(body: string[]): Method[] { +function parseMethods(body: string[], pattern: RegExp): Method[] { const methods: Method[] = [] - const pattern = /(def)\s+(\w+)\s*\(/; + // const pattern = /(def)\s+(\w+)\s*\(/; + // const pattern = /\b(((async\s+)?\s*def)|\s*class)\b/g; for (const line of body) { From 63e67d17a6abda500e59c7f9a7b1cfa95687141e Mon Sep 17 00:00:00 2001 From: tracetidwell Date: Fri, 29 Jan 2021 15:33:49 -0500 Subject: [PATCH 17/18] refatoring... --- src/parse/get_definition.ts | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/parse/get_definition.ts b/src/parse/get_definition.ts index 093bd1f..d8dd901 100644 --- a/src/parse/get_definition.ts +++ b/src/parse/get_definition.ts @@ -22,15 +22,14 @@ export function getDefinition(document: string, linePosition: number): string { let lastFunctionDef = precedingText.slice(index).trim(); if (lastFunctionDef.startsWith('class')) { - lastFunctionDef = getClassDefinition(document, lastFunctionDef, linePosition) + lastFunctionDef = getClassDefinition(document.split("\n"), lastFunctionDef, linePosition) } return lastFunctionDef; } -function getClassDefinition(document: string, lastFunctionDef: string, linePosition: number): string { +function getClassDefinition(lines: string[], lastFunctionDef: string, linePosition: number): string { - const lines = document.split("\n"); const originalIndentation = indentationOf(lines[linePosition]); let definition = getClassName(lastFunctionDef); @@ -41,12 +40,6 @@ function getClassDefinition(document: string, lastFunctionDef: string, linePosit return definition; } - // if (isInitMatch(line)) { - // definition += getInitMatch(line); - // } - // else { - // definition += line.trim(); - // } definition = updateDefinition(definition, line) if (isCloseDefMatch(line)) { From 5b4399210cb1b9bf55d1f8ad75e5492e973bedfe Mon Sep 17 00:00:00 2001 From: tracetidwell Date: Fri, 29 Jan 2021 15:38:21 -0500 Subject: [PATCH 18/18] one more try --- src/parse/get_definition.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/parse/get_definition.ts b/src/parse/get_definition.ts index d8dd901..8c6ce8b 100644 --- a/src/parse/get_definition.ts +++ b/src/parse/get_definition.ts @@ -29,17 +29,10 @@ export function getDefinition(document: string, linePosition: number): string { } function getClassDefinition(lines: string[], lastFunctionDef: string, linePosition: number): string { - - const originalIndentation = indentationOf(lines[linePosition]); let definition = getClassName(lastFunctionDef); while (linePosition < lines.length) { - const line = lines[linePosition]; - - if (indentationOf(line) <= originalIndentation && !blankLine(line)) { - return definition; - } - + const line = lines[linePosition]; definition = updateDefinition(definition, line) if (isCloseDefMatch(line)) {