From f5b012923ac81d51682e51dd54dd67b3666e4075 Mon Sep 17 00:00:00 2001 From: franck-mahieu Date: Tue, 24 Aug 2021 18:33:48 +0200 Subject: [PATCH 1/2] feat: add detectErrorInJsonsInsideFolder script --- README.md | 16 +++++++++++- package-lock.json | 4 +-- package.json | 1 + .../detectErrorInJsonsInsideFolder.js | 25 +++++++++++++++++++ 4 files changed, 43 insertions(+), 3 deletions(-) create mode 100755 scripts/validation/detectErrorInJsonsInsideFolder.js diff --git a/README.md b/README.md index c10c570..7fea68b 100644 --- a/README.md +++ b/README.md @@ -130,11 +130,25 @@ This line is not parsable : {"key4": "value2", "key5":} Launch this command, just update the sourcePath env var : ``` -cross-env sourcePath=pathToYourFileWithoutExtension node scripts/transform/detectErrorInJsonByLine.js +cross-env sourcePath=pathToYourFileWithoutExtension node scripts/validation/detectErrorInJsonByLine.js ``` Without sourcePath env var, the script choose the file in dataset folder +### detectErrorInJsonsInsideFolder + +Iterate over each json file in specified folder to log every not parsable files, like + +#### How to launch + +Launch this command, just update the sourcePath env var : + +``` +cross-env sourcePath=pathToYourFolder node scripts/validation/detectErrorInJsonsInsideFolder.js +``` + +Without sourcePath env var, the script choose the folder `folderOfJsons` in dataset folder + ## Ping Launch a ping with your nodeJS network configuration. It's helpfull if you want to check the connectivity between your diff --git a/package-lock.json b/package-lock.json index ef15623..3111ca3 100755 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { - "name": "toolbox", - "version": "0.1.0", + "name": "datasets-toolbox", + "version": "0.2.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 029f2bb..74cc843 100755 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "scripts": { "start-ping": "cross-env ipToPing= node ./scripts/ping.js", "start-detectErrorInJsonByLine": "cross-env sourcePath= node scripts/validation/detectErrorInJsonByLine.js", + "start-detectErrorInJsonsInsideFolder": "cross-env sourcePath= node scripts/validation/detectErrorInJsonsInsideFolder.js", "start-convertObjectByLineToObjectsArray": "cross-env sourcePath= node scripts/transform/convertObjectByLineToObjectsArray.js", "start-convertCsvToJson": "cross-env sourcePath= node scripts/transform/convertCsvToJson.js", "start-transformEveryObjectInObjectByLine": "cross-env sourcePath= node scripts/transform/transformEveryObjectsInObjectByLine.js", diff --git a/scripts/validation/detectErrorInJsonsInsideFolder.js b/scripts/validation/detectErrorInJsonsInsideFolder.js new file mode 100755 index 0000000..86f06a4 --- /dev/null +++ b/scripts/validation/detectErrorInJsonsInsideFolder.js @@ -0,0 +1,25 @@ +const fs = require('fs'); +const path = require('path'); + +const folderPath = process.env.sourcePath ? process.env.sourcePath : `${__dirname}/../../dataset/validation/folderOfJsons`; + +const jsonsToValidate = fs.readdirSync(path.resolve(folderPath)); + +(async () => { + console.log(`detectErrorInJsonsInsideFolder in process for this folder ${folderPath}`) + for (const jsonFile of jsonsToValidate) { + if(jsonFile.endsWith('.json')){ + let rawJson = fs.readFileSync(`${folderPath}/${jsonFile}`); + try { + if (rawJson) { + JSON.parse(rawJson.toString()); + } + } catch (e) { + console.error("ERROR : This file is not a valid json :", jsonFile, e) + } + } + + } + console.log(`detectErrorInJsonsInsideFolder is ended`) +})(); + From 82895af5f359dbc3a82f97f7803dc579c7ce9a54 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 24 Aug 2021 16:34:44 +0000 Subject: [PATCH 2/2] build(deps): bump lodash from 4.17.20 to 4.17.21 Bumps [lodash](https://github.com/lodash/lodash) from 4.17.20 to 4.17.21. - [Release notes](https://github.com/lodash/lodash/releases) - [Commits](https://github.com/lodash/lodash/compare/4.17.20...4.17.21) Signed-off-by: dependabot[bot] --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) mode change 100755 => 100644 package-lock.json diff --git a/package-lock.json b/package-lock.json old mode 100755 new mode 100644 index 3111ca3..bb4b5f8 --- a/package-lock.json +++ b/package-lock.json @@ -72,9 +72,9 @@ "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=" }, "lodash": { - "version": "4.17.20", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "map-stream": { "version": "0.0.7",