From 465548c273174f83e322f8927bd42f4e933a3d3e Mon Sep 17 00:00:00 2001 From: sauravraw Date: Mon, 15 Sep 2025 12:03:49 +0530 Subject: [PATCH 01/37] don't merge anything from here --- api/.eslintrc.json | 5 +- api/nodemon.json | 24 + api/package-lock.json | 244 ++- api/package.json | 16 +- api/src/constants/index.ts | 1 + api/src/helper/index.ts | 48 + api/src/models/project-lowdb.ts | 8 + api/src/services/drupal.service.ts | 63 + api/src/services/drupal/assets.service.ts | 428 ++++++ .../services/drupal/content-types.service.ts | 216 +++ api/src/services/drupal/entries.service.ts | 1359 +++++++++++++++++ .../services/drupal/field-analysis.service.ts | 427 ++++++ .../services/drupal/field-fetcher.service.ts | 228 +++ api/src/services/drupal/locales.service.ts | 271 ++++ api/src/services/drupal/query.service.ts | 369 +++++ api/src/services/drupal/references.service.ts | 421 +++++ api/src/services/drupal/taxonomy.service.ts | 443 ++++++ api/src/services/drupal/version.service.ts | 61 + api/src/services/migration.service.ts | 123 ++ api/src/services/projects.service.ts | 191 +++ api/src/utils/batch-processor.utils.ts | 108 ++ api/src/utils/content-type-creator.utils.ts | 50 +- .../utils/optimized-query-builder.utils.ts | 452 ++++++ api/test-drupal-locales.js | 150 ++ api/test-uid-format.js | 71 + test-drupal-locales-simple.js | 160 ++ test-drupal-locales.js | 75 + test-drupal-locales.mjs | 76 + ui/src/cmsData/legacyCms.json | 56 +- .../components/Common/AddStack/addStack.tsx | 49 +- .../Actions/LoadLanguageMapper.tsx | 305 +++- .../DestinationStack/Actions/LoadStacks.tsx | 114 +- .../LegacyCms/Actions/LoadFileFormat.tsx | 46 +- .../LegacyCms/Actions/LoadSelectCms.tsx | 7 + .../LegacyCms/Actions/LoadUploadFile.tsx | 38 +- ui/src/components/LegacyCms/index.tsx | 28 +- .../components/MigrationFlowHeader/index.tsx | 17 +- ui/src/context/app/app.interface.ts | 26 +- ui/src/pages/Migration/index.tsx | 38 +- ui/src/store/slice/migrationDataSlice.tsx | 14 +- upload-api/README.md | 0 .../drupalSchema/article.json | 263 ++++ .../drupalSchema/fact.json | 81 + .../drupalSchema/featured_content.json | 81 + .../drupalSchema/file_tree.json | 211 +++ .../drupalSchema/in_the_news.json | 123 ++ .../drupalSchema/institute.json | 141 ++ .../drupalSchema/link.json | 151 ++ .../drupalSchema/list.json | 151 ++ .../drupalSchema/page.json | 381 +++++ .../drupalSchema/partners.json | 141 ++ .../drupalSchema/profile.json | 251 +++ .../drupalSchema/school.json | 161 ++ .../drupalSchema/supporting_material.json | 163 ++ .../drupalSchema/tour.json | 111 ++ .../drupalSchema/video.json | 161 ++ .../taxonomySchema/taxonomySchema.json | 22 + upload-api/migration-drupal/config/index.json | 81 + upload-api/migration-drupal/index.js | 12 + .../libs/contentTypeMapper.js | 522 +++++++ .../libs/createInitialMapper.js | 147 ++ .../migration-drupal/libs/extractLocale.js | 100 ++ .../migration-drupal/libs/extractTaxonomy.js | 127 ++ upload-api/migration-drupal/utils/helper.js | 70 + .../utils/restrictedKeyWords/index.json | 47 + upload-api/nodemon.json | 15 + upload-api/package-lock.json | 623 ++++++++ upload-api/package.json | 9 +- upload-api/src/config/index.ts | 18 +- upload-api/src/helper/index.ts | 65 +- upload-api/src/models/types.ts | 14 +- upload-api/src/routes/index.ts | 49 +- upload-api/src/services/createMapper.ts | 7 + upload-api/src/services/drupal/index.ts | 88 ++ upload-api/src/services/fileProcessing.ts | 49 +- 75 files changed, 11298 insertions(+), 164 deletions(-) create mode 100644 api/nodemon.json create mode 100644 api/src/helper/index.ts create mode 100644 api/src/services/drupal.service.ts create mode 100644 api/src/services/drupal/assets.service.ts create mode 100644 api/src/services/drupal/content-types.service.ts create mode 100644 api/src/services/drupal/entries.service.ts create mode 100644 api/src/services/drupal/field-analysis.service.ts create mode 100644 api/src/services/drupal/field-fetcher.service.ts create mode 100644 api/src/services/drupal/locales.service.ts create mode 100644 api/src/services/drupal/query.service.ts create mode 100644 api/src/services/drupal/references.service.ts create mode 100644 api/src/services/drupal/taxonomy.service.ts create mode 100644 api/src/services/drupal/version.service.ts create mode 100644 api/src/utils/batch-processor.utils.ts create mode 100644 api/src/utils/optimized-query-builder.utils.ts create mode 100644 api/test-drupal-locales.js create mode 100644 api/test-uid-format.js create mode 100644 test-drupal-locales-simple.js create mode 100644 test-drupal-locales.js create mode 100644 test-drupal-locales.mjs delete mode 100644 upload-api/README.md create mode 100644 upload-api/drupalMigrationData/drupalSchema/article.json create mode 100644 upload-api/drupalMigrationData/drupalSchema/fact.json create mode 100644 upload-api/drupalMigrationData/drupalSchema/featured_content.json create mode 100644 upload-api/drupalMigrationData/drupalSchema/file_tree.json create mode 100644 upload-api/drupalMigrationData/drupalSchema/in_the_news.json create mode 100644 upload-api/drupalMigrationData/drupalSchema/institute.json create mode 100644 upload-api/drupalMigrationData/drupalSchema/link.json create mode 100644 upload-api/drupalMigrationData/drupalSchema/list.json create mode 100644 upload-api/drupalMigrationData/drupalSchema/page.json create mode 100644 upload-api/drupalMigrationData/drupalSchema/partners.json create mode 100644 upload-api/drupalMigrationData/drupalSchema/profile.json create mode 100644 upload-api/drupalMigrationData/drupalSchema/school.json create mode 100644 upload-api/drupalMigrationData/drupalSchema/supporting_material.json create mode 100644 upload-api/drupalMigrationData/drupalSchema/tour.json create mode 100644 upload-api/drupalMigrationData/drupalSchema/video.json create mode 100644 upload-api/drupalMigrationData/taxonomySchema/taxonomySchema.json create mode 100755 upload-api/migration-drupal/config/index.json create mode 100644 upload-api/migration-drupal/index.js create mode 100644 upload-api/migration-drupal/libs/contentTypeMapper.js create mode 100644 upload-api/migration-drupal/libs/createInitialMapper.js create mode 100644 upload-api/migration-drupal/libs/extractLocale.js create mode 100644 upload-api/migration-drupal/libs/extractTaxonomy.js create mode 100755 upload-api/migration-drupal/utils/helper.js create mode 100644 upload-api/migration-drupal/utils/restrictedKeyWords/index.json create mode 100644 upload-api/nodemon.json create mode 100644 upload-api/src/services/drupal/index.ts diff --git a/api/.eslintrc.json b/api/.eslintrc.json index 1c0a3d935..b7e913303 100644 --- a/api/.eslintrc.json +++ b/api/.eslintrc.json @@ -12,8 +12,9 @@ "parser": "@typescript-eslint/parser", "parserOptions": { "ecmaVersion": 2022, - "project": ["./tsconfig.json"], - "sourceType": "module" + "project": ["tsconfig.json"], + "sourceType": "module", + "tsconfigRootDir": __dirname }, "plugins": ["@typescript-eslint"], "rules": { diff --git a/api/nodemon.json b/api/nodemon.json new file mode 100644 index 000000000..8d9d481c1 --- /dev/null +++ b/api/nodemon.json @@ -0,0 +1,24 @@ +{ + "watch": ["src"], + "ext": "ts,js,json", + "ignore": [ + "database/**/*", + "logs/**/*", + "*.log", + "node_modules/**/*", + "dist/**/*", + "build/**/*", + "cmsMigrationData/**/*", + "test_output.log", + "combine.log", + "sample.log", + "upload-api/**/*", + "ui/**/*" + ], + "exec": "tsx ./src/server.ts", + "env": { + "NODE_ENV": "production" + }, + "delay": "1000", + "verbose": true +} diff --git a/api/package-lock.json b/api/package-lock.json index a6480f9e3..648b2c574 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -9,9 +9,9 @@ "version": "1.0.0", "license": "ISC", "dependencies": { - "@contentstack/cli": "1.41.0", + "@contentstack/cli": "^1.41.0", "@contentstack/cli-utilities": "^1.12.0", - "@contentstack/json-rte-serializer": "^2.0.7", + "@contentstack/json-rte-serializer": "^2.1.0", "@contentstack/marketplace-sdk": "^1.2.4", "axios": "^1.8.2", "chokidar": "^3.6.0", @@ -30,8 +30,11 @@ "jsonwebtoken": "^9.0.2", "lowdb": "^7.0.1", "mkdirp": "^3.0.1", + "mysql2": "^3.14.3", "p-limit": "^6.2.0", "path-to-regexp": "^8.2.0", + "php-serialize": "^5.1.3", + "php-unserialize": "^0.0.1", "router": "^2.0.0", "shelljs": "^0.9.0", "socket.io": "^4.7.5", @@ -57,6 +60,7 @@ "eslint-config-airbnb": "^19.0.0", "eslint-config-prettier": "^8.3.0", "lodash": "^4.17.21", + "nodemon": "^3.0.0", "prettier": "^2.4.1", "tsx": "^4.7.1", "typescript": "^5.4.3" @@ -4788,6 +4792,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/aws-ssl-profiles": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/aws-ssl-profiles/-/aws-ssl-profiles-1.1.2.tgz", + "integrity": "sha512-NZKeq9AfyQvEeNlN0zSYAaWrmBffJh3IELMZfRpJVWgrpEbtEpnjvzqBPf+mxoI287JohRDoa+/nsfqqiZmF6g==", + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/axe-core": { "version": "4.10.3", "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.10.3.tgz", @@ -5813,6 +5825,14 @@ "node": ">=0.4.0" } }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -7668,6 +7688,14 @@ "node": ">= 0.6.0" } }, + "node_modules/generate-function": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.3.1.tgz", + "integrity": "sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ==", + "dependencies": { + "is-property": "^1.0.2" + } + }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -8154,6 +8182,12 @@ "node": ">= 4" } }, + "node_modules/ignore-by-default": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz", + "integrity": "sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==", + "dev": true + }, "node_modules/immer": { "version": "10.1.1", "resolved": "https://registry.npmjs.org/immer/-/immer-10.1.1.tgz", @@ -9154,6 +9188,11 @@ "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.2.2.tgz", "integrity": "sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ==" }, + "node_modules/is-property": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", + "integrity": "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g==" + }, "node_modules/is-reference": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/is-reference/-/is-reference-1.2.1.tgz", @@ -10297,6 +10336,11 @@ "node": ">= 12.0.0" } }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==" + }, "node_modules/loose-envify": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", @@ -10327,6 +10371,20 @@ "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==" }, + "node_modules/lru.min": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/lru.min/-/lru.min-1.1.2.tgz", + "integrity": "sha512-Nv9KddBcQSlQopmBHXSsZVY5xsdlZkdH/Iey0BlcBYggMd4two7cZnKOK9vmy3nY0O5RGH99z1PCeTpPqszUYg==", + "engines": { + "bun": ">=1.0.0", + "deno": ">=1.30.0", + "node": ">=8.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wellwelwel" + } + }, "node_modules/magic-string": { "version": "0.30.17", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.17.tgz", @@ -10539,6 +10597,55 @@ "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==" }, + "node_modules/mysql2": { + "version": "3.14.3", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.14.3.tgz", + "integrity": "sha512-fD6MLV8XJ1KiNFIF0bS7Msl8eZyhlTDCDl75ajU5SJtpdx9ZPEACulJcqJWr1Y8OYyxsFc4j3+nflpmhxCU5aQ==", + "dependencies": { + "aws-ssl-profiles": "^1.1.1", + "denque": "^2.1.0", + "generate-function": "^2.3.1", + "iconv-lite": "^0.6.3", + "long": "^5.2.1", + "lru.min": "^1.0.0", + "named-placeholders": "^1.1.3", + "seq-queue": "^0.0.5", + "sqlstring": "^2.3.2" + }, + "engines": { + "node": ">= 8.0" + } + }, + "node_modules/mysql2/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/named-placeholders": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.3.tgz", + "integrity": "sha512-eLoBxg6wE/rZkJPhU/xRX1WTpkFEwDJEN96oxFrTsqBdbT5ec295Q+CoHrL9IT0DipqKhmGcaZmwOt8OON5x1w==", + "dependencies": { + "lru-cache": "^7.14.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/named-placeholders/node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "engines": { + "node": ">=12" + } + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -10626,6 +10733,77 @@ "resolved": "https://registry.npmjs.org/node-machine-id/-/node-machine-id-1.1.12.tgz", "integrity": "sha512-QNABxbrPa3qEIfrE6GOJ7BYIuignnJw7iQ2YPbc3Nla1HzRJjXzZOiikfF8m7eAMfichLt3M4VgLOetqgDmgGQ==" }, + "node_modules/nodemon": { + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.1.10.tgz", + "integrity": "sha512-WDjw3pJ0/0jMFmyNDp3gvY2YizjLmmOUQo6DEBY+JgdvW/yQ9mEeSw6H5ythl5Ny2ytb7f9C2nIbjSxMNzbJXw==", + "dev": true, + "dependencies": { + "chokidar": "^3.5.2", + "debug": "^4", + "ignore-by-default": "^1.0.1", + "minimatch": "^3.1.2", + "pstree.remy": "^1.1.8", + "semver": "^7.5.3", + "simple-update-notifier": "^2.0.0", + "supports-color": "^5.5.0", + "touch": "^3.1.0", + "undefsafe": "^2.0.5" + }, + "bin": { + "nodemon": "bin/nodemon.js" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/nodemon" + } + }, + "node_modules/nodemon/node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/nodemon/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/nodemon/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/nodemon/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -13658,6 +13836,22 @@ "node": ">=8" } }, + "node_modules/php-serialize": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/php-serialize/-/php-serialize-5.1.3.tgz", + "integrity": "sha512-p7zXX8xjGgddgP6byN+KmGKM0x6uoMZBRZteBa9LonqgrDV3LyMxUeGVX7RTFYwWaUAnTEsUWJfHI3N7eKvJgw==", + "engines": { + "node": ">= 8" + } + }, + "node_modules/php-unserialize": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/php-unserialize/-/php-unserialize-0.0.1.tgz", + "integrity": "sha512-aZWuX3gQ30Dui+Lff19q0jeu+3DHpSYXFEQPkeAx4WAyDtAp5VI30ZPC5wb4OrcHy6KUiZIRFRTWkvK8l8l6Rw==", + "engines": { + "node": "*" + } + }, "node_modules/picomatch": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", @@ -13934,6 +14128,12 @@ "url": "https://github.com/sponsors/lupomontero" } }, + "node_modules/pstree.remy": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz", + "integrity": "sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==", + "dev": true + }, "node_modules/pump": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", @@ -14613,6 +14813,11 @@ "node": ">= 0.8" } }, + "node_modules/seq-queue": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/seq-queue/-/seq-queue-0.0.5.tgz", + "integrity": "sha512-hr3Wtp/GZIc/6DAGPDcV4/9WoZhjrkXsi5B/07QgX8tsdc6ilr7BFM6PM6rbdAX1kFSDYeZGLipIZZKyQP0O5Q==" + }, "node_modules/serve-static": { "version": "1.16.2", "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", @@ -14946,6 +15151,18 @@ "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" }, + "node_modules/simple-update-notifier": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-2.0.0.tgz", + "integrity": "sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==", + "dev": true, + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/sinon": { "version": "19.0.5", "resolved": "https://registry.npmjs.org/sinon/-/sinon-19.0.5.tgz", @@ -15249,6 +15466,14 @@ "resolved": "https://registry.npmjs.org/speedometer/-/speedometer-1.0.0.tgz", "integrity": "sha512-lgxErLl/7A5+vgIIXsh9MbeukOaCb2axgQ+bKCdIE+ibNT4XNYGNCR1qFEGq6F+YDASXK3Fh/c5FgtZchFolxw==" }, + "node_modules/sqlstring": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.3.tgz", + "integrity": "sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg==", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/stack-trace": { "version": "0.0.10", "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", @@ -15645,6 +15870,15 @@ "node": ">=0.6" } }, + "node_modules/touch": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.1.tgz", + "integrity": "sha512-r0eojU4bI8MnHr8c5bNo7lJDdI2qXlWWJk6a9EAFG7vbhTjElYhBVS3/miuE0uOuoLdb8Mc/rVfsmm6eo5o9GA==", + "dev": true, + "bin": { + "nodetouch": "bin/nodetouch.js" + } + }, "node_modules/tough-cookie": { "version": "4.1.4", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.4.tgz", @@ -15958,6 +16192,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/undefsafe": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.5.tgz", + "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", + "dev": true + }, "node_modules/underscore": { "version": "1.12.1", "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.12.1.tgz", diff --git a/api/package.json b/api/package.json index dc864e161..9967eb006 100644 --- a/api/package.json +++ b/api/package.json @@ -5,8 +5,10 @@ "exports": "./src/server.ts", "scripts": { "build": "npx tsc", - "start": "NODE_ENV=production node dist/server.js", - "dev": "NODE_ENV=production tsx ./src/server.ts", + "start": "NODE_ENV=production tsx ./src/server.ts", + "start:prod": "NODE_ENV=production node dist/server.js", + "dev": "NODE_ENV=production nodemon --exec tsx ./src/server.ts", + "test:drupal": "npx tsx test-drupal-services.js", "prettify": "prettier --write .", "windev": "SET NODE_ENV=production&& tsx watch ./src/server.ts", "winstart": "SET NODE_ENV=production&& node dist/server.js", @@ -25,9 +27,9 @@ }, "homepage": "https://github.com/contentstack/migration-v2.git#readme", "dependencies": { - "@contentstack/cli": "1.41.0", + "@contentstack/cli": "^1.41.0", "@contentstack/cli-utilities": "^1.12.0", - "@contentstack/json-rte-serializer": "^2.0.7", + "@contentstack/json-rte-serializer": "^2.1.0", "@contentstack/marketplace-sdk": "^1.2.4", "axios": "^1.8.2", "chokidar": "^3.6.0", @@ -46,8 +48,11 @@ "jsonwebtoken": "^9.0.2", "lowdb": "^7.0.1", "mkdirp": "^3.0.1", + "mysql2": "^3.14.3", "p-limit": "^6.2.0", "path-to-regexp": "^8.2.0", + "php-serialize": "^5.1.3", + "php-unserialize": "^0.0.1", "router": "^2.0.0", "shelljs": "^0.9.0", "socket.io": "^4.7.5", @@ -73,9 +78,10 @@ "eslint-config-airbnb": "^19.0.0", "eslint-config-prettier": "^8.3.0", "lodash": "^4.17.21", + "nodemon": "^3.0.0", "prettier": "^2.4.1", "tsx": "^4.7.1", "typescript": "^5.4.3" }, "keywords": [] -} \ No newline at end of file +} diff --git a/api/src/constants/index.ts b/api/src/constants/index.ts index f782930ec..07f80836a 100644 --- a/api/src/constants/index.ts +++ b/api/src/constants/index.ts @@ -12,6 +12,7 @@ export const CMS = { SITECORE_V9: "sitecore v9", SITECORE_V10: "sitecore v10", WORDPRESS: "wordpress", + DRUPAL_V8: "drupal v8+", AEM: "aem", }; export const MODULES = [ diff --git a/api/src/helper/index.ts b/api/src/helper/index.ts new file mode 100644 index 000000000..d02bfb953 --- /dev/null +++ b/api/src/helper/index.ts @@ -0,0 +1,48 @@ +import mysql from 'mysql2'; +import customLogger from '../utils/custom-logger.utils.js'; + +const createDbConnection = async (config: any, projectId: string = '', stackId: string = ''): Promise => { + try { + // Create the connection with config values + const connection = mysql.createConnection({ + host: config?.host, + user: config?.user, + password: config?.password, + database: config?.database, + port: Number(config?.port) + }); + + // Test the connection by wrapping the connect method in a promise + return new Promise((resolve, reject) => { + connection.connect(async (err) => { + if (err) { + await customLogger(projectId, stackId, 'error', `Database connection failed: ${err.message}`); + reject(err); + return; + } + + await customLogger(projectId, stackId, 'info', 'Database connection established successfully'); + resolve(connection); + }); + }); + } catch (error: any) { + await customLogger(projectId, stackId, 'error', `Failed to create database connection: ${error.message}`); + return null; + } +}; + +// Usage example +const getDbConnection = async (config: any, projectId: string = '', stackId: string = '') => { + try { + const connection = await createDbConnection(config, projectId, stackId); + if (!connection) { + throw new Error('Could not establish database connection'); + } + return connection; + } catch (error: any) { + await customLogger(projectId, stackId, 'error', `Database connection error: ${error.message}`); + throw error; // Re-throw so caller can handle it + } +}; + +export { createDbConnection, getDbConnection }; \ No newline at end of file diff --git a/api/src/models/project-lowdb.ts b/api/src/models/project-lowdb.ts index e995da368..bb58f37e3 100644 --- a/api/src/models/project-lowdb.ts +++ b/api/src/models/project-lowdb.ts @@ -23,6 +23,14 @@ interface LegacyCMS { bucketName: string; buketKey: string; }; + mySQLDetails: { + host: string; + user: string; + password?: string; + database: string; + port?: number; + }; + is_sql: boolean; file_path: string; is_fileValid: boolean; is_localPath: boolean; diff --git a/api/src/services/drupal.service.ts b/api/src/services/drupal.service.ts new file mode 100644 index 000000000..3daf94cee --- /dev/null +++ b/api/src/services/drupal.service.ts @@ -0,0 +1,63 @@ +// Import modular Drupal services +import { createAssets } from './drupal/assets.service.js'; +import { createEntry } from './drupal/entries.service.js'; +import { createLocale } from './drupal/locales.service.js'; +import { createRefrence } from './drupal/references.service.js'; +import { createTaxonomy } from './drupal/taxonomy.service.js'; +import { createVersionFile } from './drupal/version.service.js'; +import { createQuery, createQueryConfig } from './drupal/query.service.js'; +import { generateContentTypeSchemas } from './drupal/content-types.service.js'; + +/** + * Drupal migration service with SQL-based data extraction. + * + * All functions use direct database connections to extract data from Drupal + * following the original migration patterns. + * + * IMPORTANT: Run in this order for proper dependency resolution: + * 1. createQuery - Generate dynamic queries from database analysis (MUST RUN FIRST) + * 2. generateContentTypeSchemas - Convert upload-api schema to API content types (MUST RUN AFTER upload-api) + * 3. createAssets - Extract assets first (needed by entries) + * 4. createRefrence - Create reference mappings (needed by entries) + * 5. createTaxonomy - Extract taxonomies (needed by entries for taxonomy references) + * 6. createEntry - Process entries (uses assets, references, and taxonomies) + * 7. createLocale - Create locale configurations + * 8. createVersionFile - Create version metadata file + */ +export const drupalService = { + createQuery, // Generate dynamic queries from database analysis (MUST RUN FIRST) + createQueryConfig, // Helper: Create query configuration file for dynamic SQL + generateContentTypeSchemas, // Convert upload-api schema to API content types (MUST RUN AFTER upload-api) + createAssets: (dbConfig: any, destination_stack_id: string, projectId: string, isTest = false, drupalAssetsConfig?: any) => { + console.info('๐Ÿ” === DRUPAL SERVICE - ASSETS WRAPPER ==='); + console.info('๐Ÿ“‹ Received DB Config from Migration Service:', JSON.stringify(dbConfig, null, 2)); + console.info('๐Ÿ“‹ Received Assets Config from Migration Service:', JSON.stringify(drupalAssetsConfig, null, 2)); + console.info('๐Ÿ“‹ Stack ID:', destination_stack_id); + console.info('๐Ÿ“‹ Project ID:', projectId); + console.info('๐Ÿ“‹ Is Test:', isTest); + console.info('=========================================='); + return createAssets( + dbConfig, + destination_stack_id, + projectId, + drupalAssetsConfig?.base_url || "", + drupalAssetsConfig?.public_path || "/sites/default/files/", + isTest + ); + }, + createRefrence, // Create reference mappings for relationships (run before entries) + createTaxonomy, // Extract and process Drupal taxonomies (vocabularies and terms) + createEntry: (dbConfig: any, destination_stack_id: string, projectId: string, isTest = false, masterLocale = 'en-us', contentTypeMapping: any[] = []) => { + console.info('๐Ÿ” === DRUPAL SERVICE - ENTRIES WRAPPER ==='); + console.info('๐Ÿ“‹ Received Config from Migration Service:', JSON.stringify(dbConfig, null, 2)); + console.info('๐Ÿ“‹ Stack ID:', destination_stack_id); + console.info('๐Ÿ“‹ Project ID:', projectId); + console.info('๐Ÿ“‹ Is Test:', isTest); + console.info('๐Ÿ“‹ Master Locale:', masterLocale); + console.info('๐Ÿ“‹ Content Type Mapping Count:', contentTypeMapping.length); + console.info('============================================'); + return createEntry(dbConfig, destination_stack_id, projectId, isTest, masterLocale, contentTypeMapping); + }, + createLocale, // Create locale configurations + createVersionFile, // Create version metadata file +}; diff --git a/api/src/services/drupal/assets.service.ts b/api/src/services/drupal/assets.service.ts new file mode 100644 index 000000000..faf3ccbed --- /dev/null +++ b/api/src/services/drupal/assets.service.ts @@ -0,0 +1,428 @@ +import fs from "fs"; +import path from "path"; +import axios from "axios"; +import pLimit from 'p-limit'; +import mysql from 'mysql2'; +import { MIGRATION_DATA_CONFIG } from "../../constants/index.js"; +import { getLogMessage } from "../../utils/index.js"; +import customLogger from "../../utils/custom-logger.utils.js"; +import { getDbConnection } from "../../helper/index.js"; +import { processBatches } from "../../utils/batch-processor.utils.js"; + +const { + DATA, + ASSETS_DIR_NAME, + ASSETS_FILE_NAME, + ASSETS_SCHEMA_FILE, + ASSETS_FAILED_FILE, +} = MIGRATION_DATA_CONFIG; + +interface AssetMetaData { + uid: string; + url: string; + filename: string; +} + +interface DrupalAsset { + fid: string | number; + uri: string; + filename: string; + filesize: string | number; + filemime?: string; + status?: string | number; + uid?: string | number; + timestamp?: string | number; + id?: string | number; // For file_usage table + count?: string | number; // For file_usage table +} + +/** + * Writes data to a specified file, ensuring the target directory exists. + */ +async function writeFile(dirPath: string, filename: string, data: any) { + let fileHandle; + try { + await fs.promises.mkdir(dirPath, { recursive: true }); + const filePath = path.join(dirPath, filename); + + // Use file handle for better control over file operations + fileHandle = await fs.promises.open(filePath, 'w'); + await fileHandle.writeFile(JSON.stringify(data), 'utf8'); + } catch (err) { + console.error(`Error writing ${dirPath}/${filename}::`, err); + throw err; // Re-throw to handle upstream + } finally { + // Ensure file handle is always closed + if (fileHandle) { + try { + await fileHandle.close(); + } catch (closeErr) { + console.error(`Error closing file handle for ${dirPath}/${filename}:`, closeErr); + } + } + } +} + +/** + * Constructs the full URL for Drupal assets, handling public:// and private:// schemes + */ +const constructAssetUrl = (uri: string, baseUrl: string, publicPath: string): string => { + let url = uri; + const replaceValue = baseUrl + publicPath; + + if (!url.startsWith("http")) { + url = url.replace("public://", replaceValue); + url = url.replace("private://", replaceValue); + } + + return encodeURI(url); +}; + +/** + * Saves an asset to the destination stack directory for Drupal migration. + * Based on the original Drupal v8 migration logic. + */ +const saveAsset = async ( + assets: DrupalAsset, + failedJSON: any, + assetData: any, + metadata: AssetMetaData[], + projectId: string, + destination_stack_id: string, + baseUrl: string = "", + publicPath: string = "/sites/default/files/", + retryCount = 0 +): Promise => { + try { + const srcFunc = 'saveAsset'; + const assetsSave = path.join(DATA, destination_stack_id, ASSETS_DIR_NAME); + + // Use Drupal-specific field names + const assetId = `assets_${assets.fid}`; + const fileName = assets.filename; + const fileUrl = constructAssetUrl(assets.uri, baseUrl, publicPath); + + // Check if asset already exists + if (fs.existsSync(path.resolve(assetsSave, assetId, fileName))) { + return assetId; // Asset already exists + } + + try { + const response = await axios.get(fileUrl, { + responseType: "arraybuffer", + timeout: 30000, + }); + + const assetPath = path.resolve(assetsSave, assetId); + + // Create asset data following Drupal migration pattern + assetData[assetId] = { + uid: assetId, + urlPath: `/assets/${assetId}`, + status: true, + content_type: assets.filemime || 'application/octet-stream', + file_size: assets.filesize.toString(), + tag: [], + filename: fileName, + url: fileUrl, + is_dir: false, + parent_uid: null, + _version: 1, + title: fileName, + publish_details: [], + }; + + const message = getLogMessage( + srcFunc, + `Asset "${fileName}" with id ${assets.fid} has been successfully transformed.`, + {} + ); + + await fs.promises.mkdir(assetPath, { recursive: true }); + await fs.promises.writeFile(path.join(assetPath, fileName), Buffer.from(response.data), "binary"); + await writeFile(assetPath, `_contentstack_${assetId}.json`, assetData[assetId]); + + metadata.push({ uid: assetId, url: fileUrl, filename: fileName }); + + // Remove from failed assets if it was previously failed + if (failedJSON[assetId]) { + delete failedJSON[assetId]; + } + + await customLogger(projectId, destination_stack_id, 'info', message); + return assetId; + + } catch (err: any) { + if (retryCount < 1) { + // Retry once + return await saveAsset(assets, failedJSON, assetData, metadata, projectId, destination_stack_id, baseUrl, publicPath, retryCount + 1); + } else { + // Mark as failed after retry + failedJSON[assetId] = { + failedUid: assets.fid, + name: fileName, + url: fileUrl, + file_size: assets.filesize, + reason_for_error: err?.message, + }; + + const message = getLogMessage( + srcFunc, + `Failed to download asset "${fileName}" with id ${assets.fid}: ${err.message}`, + {}, + err + ); + await customLogger(projectId, destination_stack_id, 'error', message); + return assetId; + } + } + } catch (error) { + console.error("Error in saveAsset:", error); + return `assets_${assets.fid}`; + } +}; + +/** + * Executes SQL query and returns results as Promise + */ +const executeQuery = (connection: mysql.Connection, query: string): Promise => { + return new Promise((resolve, reject) => { + connection.query(query, (error, results) => { + if (error) { + reject(error); + } else { + resolve(results as any[]); + } + }); + }); +}; + +/** + * Fetches assets from database using SQL query + */ +const fetchAssetsFromDB = async ( + connection: mysql.Connection, + projectId: string, + destination_stack_id: string +): Promise => { + const srcFunc = 'fetchAssetsFromDB'; + const assetsQuery = "SELECT a.fid, a.filename, a.uri, a.filesize, a.filemime FROM file_managed a"; + + try { + const results = await executeQuery(connection, assetsQuery); + + const message = getLogMessage( + srcFunc, + `Fetched ${results.length} assets from database.`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', message); + + return results as DrupalAsset[]; + } catch (error: any) { + const message = getLogMessage( + srcFunc, + `Failed to fetch assets from database: ${error.message}`, + {}, + error + ); + await customLogger(projectId, destination_stack_id, 'error', message); + throw error; + } +}; + +/** + * Retries failed assets using FID query + */ +const retryFailedAssets = async ( + connection: mysql.Connection, + failedAssetIds: string[], + failedJSON: any, + assetData: any, + metadata: AssetMetaData[], + projectId: string, + destination_stack_id: string, + baseUrl: string = "", + publicPath: string = "/sites/default/files/" +): Promise => { + const srcFunc = 'retryFailedAssets'; + + if (failedAssetIds.length === 0) { + return; + } + + try { + const assetsFIDQuery = `SELECT a.fid, a.filename, a.uri, a.filesize, a.filemime, b.id, b.count FROM file_managed a, file_usage b WHERE a.fid IN (${failedAssetIds.join(',')})`; + const results = await executeQuery(connection, assetsFIDQuery); + + if (results.length > 0) { + const limit = pLimit(1); // Reduce to 1 for large datasets to prevent EMFILE errors + const tasks = results.map((asset: DrupalAsset) => + limit(() => saveAsset(asset, failedJSON, assetData, metadata, projectId, destination_stack_id, baseUrl, publicPath)) + ); + + await Promise.all(tasks); + + const message = getLogMessage( + srcFunc, + `Retried ${results.length} failed assets.`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', message); + } + } catch (error: any) { + const message = getLogMessage( + srcFunc, + `Error retrying failed assets: ${error.message}`, + {}, + error + ); + await customLogger(projectId, destination_stack_id, 'error', message); + } +}; + +/** + * Creates and processes assets from Drupal database for migration to Contentstack. + * Based on the original Drupal v8 migration logic with direct SQL queries. + */ +export const createAssets = async ( + dbConfig: any, + destination_stack_id: string, + projectId: string, + baseUrl: string = "", + publicPath: string = "/sites/default/files/", + isTest = false +) => { + const srcFunc = 'createAssets'; + let connection: mysql.Connection | null = null; + + try { + console.info('๐Ÿ” === DRUPAL ASSETS SERVICE CONFIG ==='); + console.info('๐Ÿ“‹ Database Config:', JSON.stringify(dbConfig, null, 2)); + console.info('๐Ÿ“‹ Destination Stack ID:', destination_stack_id); + console.info('๐Ÿ“‹ Project ID:', projectId); + console.info('๐Ÿ“‹ Base URL:', baseUrl); + console.info('๐Ÿ“‹ Public Path:', publicPath); + console.info('๐Ÿ“‹ Is Test Migration:', isTest); + console.info('๐Ÿ“‹ Function:', srcFunc); + console.info('========================================'); + + const assetsSave = path.join(DATA, destination_stack_id, ASSETS_DIR_NAME); + const assetMasterFolderPath = path.join(DATA, destination_stack_id, "logs", ASSETS_DIR_NAME); + + // Initialize directories and files + await fs.promises.mkdir(assetsSave, { recursive: true }); + await fs.promises.mkdir(assetMasterFolderPath, { recursive: true }); + + // Initialize data structures + const failedJSON: any = {}; + const assetData: any = {}; + const metadata: AssetMetaData[] = []; + const fileMeta = { "1": ASSETS_SCHEMA_FILE }; + const failedAssetIds: string[] = []; + + const message = getLogMessage( + srcFunc, + `Exporting assets...`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', message); + + // Create database connection + connection = await getDbConnection(dbConfig, projectId, destination_stack_id); + + // Fetch assets from database + const assetsData = await fetchAssetsFromDB(connection, projectId, destination_stack_id); + + if (assetsData && assetsData.length > 0) { + let assets = assetsData; + if (isTest) { + assets = assets.slice(0, 10); + } + + console.log(`๐Ÿ“Š Processing ${assets.length} assets...`); + + // Use batch processing for large datasets to prevent EMFILE errors + const batchSize = assets.length > 10000 ? 100 : 1000; // Smaller batches for very large datasets + const results = await processBatches( + assets, + async (asset: DrupalAsset) => { + try { + return await saveAsset(asset, failedJSON, assetData, metadata, projectId, destination_stack_id, baseUrl, publicPath); + } catch (error) { + failedAssetIds.push(asset.fid.toString()); + return `assets_${asset.fid}`; + } + }, + { + batchSize, + concurrency: 1, // Process one at a time to prevent file handle exhaustion + delayBetweenBatches: 200 // 200ms delay between batches to allow file handles to close + }, + (batchIndex, totalBatches, batchResults) => { + console.log(`โœ… Completed batch ${batchIndex}/${totalBatches} - Processed ${batchResults.length} assets`); + + // Periodically save progress for very large datasets + if (batchIndex % 10 === 0) { + console.log(`๐Ÿ’พ Progress: ${batchIndex}/${totalBatches} batches completed (${(batchIndex/totalBatches*100).toFixed(1)}%)`); + } + } + ); + + // Retry failed assets if any + if (failedAssetIds.length > 0) { + await retryFailedAssets( + connection, + failedAssetIds, + failedJSON, + assetData, + metadata, + projectId, + destination_stack_id, + baseUrl, + publicPath + ); + } + + // Write files following the original pattern + await writeFile(assetsSave, ASSETS_SCHEMA_FILE, assetData); + await writeFile(assetsSave, ASSETS_FILE_NAME, fileMeta); + + if (Object.keys(failedJSON).length > 0) { + await writeFile(assetMasterFolderPath, ASSETS_FAILED_FILE, failedJSON); + } + + const successMessage = getLogMessage( + srcFunc, + `Successfully processed ${Object.keys(assetData).length} assets out of ${assets.length} total assets.`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', successMessage); + + return results; + + } else { + const message = getLogMessage( + srcFunc, + `No assets found.`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', message); + return []; + } + } catch (err) { + const message = getLogMessage( + srcFunc, + `Error encountered while creating assets.`, + {}, + err + ); + await customLogger(projectId, destination_stack_id, 'error', message); + throw err; + } finally { + // Close database connection + if (connection) { + connection.end(); + } + } +}; diff --git a/api/src/services/drupal/content-types.service.ts b/api/src/services/drupal/content-types.service.ts new file mode 100644 index 000000000..d04f7ea8a --- /dev/null +++ b/api/src/services/drupal/content-types.service.ts @@ -0,0 +1,216 @@ +import fs from 'fs'; +import path from 'path'; +import { MIGRATION_DATA_CONFIG } from '../../constants/index.js'; +import { convertToSchemaFormate } from '../../utils/content-type-creator.utils.js'; +import { getLogMessage } from '../../utils/index.js'; +import customLogger from '../../utils/custom-logger.utils.js'; + +const { DATA, CONTENT_TYPES_DIR_NAME } = MIGRATION_DATA_CONFIG; + +/** + * Generates API content types from upload-api drupal schema + * This service reads the upload-api generated schema and converts it to final API content types + * following the same pattern as other CMS services (Contentful, WordPress, Sitecore) + */ +export const generateContentTypeSchemas = async ( + destination_stack_id: string, + projectId: string +): Promise => { + const srcFunc = 'generateContentTypeSchemas'; + + try { + const message = getLogMessage( + srcFunc, + `Generating content type schemas from upload-api drupal schema...`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', message); + + // Path to upload-api generated schema + const uploadApiSchemaPath = path.join(process.cwd(), '..', 'upload-api', 'drupalMigrationData', 'drupalSchema'); + + // Path to API content types directory + const apiContentTypesPath = path.join(DATA, destination_stack_id, CONTENT_TYPES_DIR_NAME); + + // Ensure API content types directory exists + await fs.promises.mkdir(apiContentTypesPath, { recursive: true }); + + if (!fs.existsSync(uploadApiSchemaPath)) { + throw new Error(`Upload-API schema not found at: ${uploadApiSchemaPath}. Please run upload-api migration first.`); + } + + // Read all schema files from upload-api + const schemaFiles = fs.readdirSync(uploadApiSchemaPath).filter(file => file.endsWith('.json')); + + if (schemaFiles.length === 0) { + throw new Error(`No schema files found in upload-api directory: ${uploadApiSchemaPath}`); + } + + console.log(`๐Ÿ“‹ Found ${schemaFiles.length} content type schemas to convert`); + + for (const schemaFile of schemaFiles) { + try { + const uploadApiSchemaFilePath = path.join(uploadApiSchemaPath, schemaFile); + const uploadApiSchema = JSON.parse(fs.readFileSync(uploadApiSchemaFilePath, 'utf8')); + + // Convert upload-api schema to API format + const apiSchema = convertUploadApiSchemaToApiSchema(uploadApiSchema); + + // Write API schema file + const apiSchemaFilePath = path.join(apiContentTypesPath, schemaFile); + await fs.promises.writeFile(apiSchemaFilePath, JSON.stringify(apiSchema, null, 2), 'utf8'); + + const fieldMessage = getLogMessage( + srcFunc, + `Converted content type ${uploadApiSchema.uid} with ${uploadApiSchema.schema?.length || 0} fields`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', fieldMessage); + + console.log(`โœ… Converted ${schemaFile}: ${uploadApiSchema.schema?.length || 0} fields`); + + } catch (error: any) { + const errorMessage = getLogMessage( + srcFunc, + `Failed to convert schema file ${schemaFile}: ${error.message}`, + {}, + error + ); + await customLogger(projectId, destination_stack_id, 'error', errorMessage); + console.error(`โŒ Failed to convert ${schemaFile}:`, error.message); + } + } + + const successMessage = getLogMessage( + srcFunc, + `Successfully generated ${schemaFiles.length} content type schemas from upload-api`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', successMessage); + + console.log(`๐ŸŽ‰ Successfully converted ${schemaFiles.length} content type schemas`); + + } catch (error: any) { + const errorMessage = getLogMessage( + srcFunc, + `Failed to generate content type schemas: ${error.message}`, + {}, + error + ); + await customLogger(projectId, destination_stack_id, 'error', errorMessage); + throw error; + } +}; + +/** + * Converts upload-api drupal schema format to API content type format + * This preserves the original field types and user selections from the upload-api + */ +function convertUploadApiSchemaToApiSchema(uploadApiSchema: any): any { + const apiSchema = { + title: uploadApiSchema.title, + uid: uploadApiSchema.uid, + schema: [] as any[] + }; + + if (!uploadApiSchema.schema || !Array.isArray(uploadApiSchema.schema)) { + return apiSchema; + } + + // Convert each field from upload-api format to API format + for (const uploadField of uploadApiSchema.schema) { + try { + // Map upload-api field to API format using convertToSchemaFormate + const apiField = convertToSchemaFormate({ + field: { + title: uploadField.contentstackField || uploadField.otherCmsField, + uid: uploadField.contentstackFieldUid, + contentstackFieldType: uploadField.contentstackFieldType, + advanced: { + ...uploadField.advanced, + mandatory: uploadField.advanced?.mandatory || false, + multiple: uploadField.advanced?.multiple || false, + unique: uploadField.advanced?.unique || false, + nonLocalizable: uploadField.advanced?.non_localizable || false, + default_value: uploadField.advanced?.default_value || '', + validationRegex: uploadField.advanced?.format || '', + validationErrorMessage: uploadField.advanced?.error_message || '' + }, + // For reference fields, preserve reference_to from upload-api + refrenceTo: uploadField.advanced?.reference_to || uploadField.advanced?.embedObjects || [], + // For taxonomy fields, preserve taxonomies from upload-api + taxonomies: uploadField.advanced?.taxonomies || [] + }, + advanced: true + }); + + if (apiField) { + // Preserve additional metadata from upload-api + if (uploadField.contentstackFieldType === 'reference' && uploadField.advanced?.reference_to) { + apiField.reference_to = uploadField.advanced.reference_to; + } + + if (uploadField.contentstackFieldType === 'taxonomy' && uploadField.advanced?.taxonomies) { + apiField.taxonomies = uploadField.advanced.taxonomies; + } + + // Preserve field metadata for proper field type conversion + if (uploadField.advanced?.multiline !== undefined) { + apiField.field_metadata = apiField.field_metadata || {}; + apiField.field_metadata.multiline = uploadField.advanced.multiline; + } + + apiSchema.schema.push(apiField); + } + + } catch (error: any) { + console.warn(`Failed to convert field ${uploadField.uid}:`, error.message); + + // Fallback: create basic field structure + apiSchema.schema.push({ + display_name: uploadField.contentstackField || uploadField.otherCmsField || uploadField.uid, + uid: uploadField.contentstackFieldUid || uploadField.uid, + data_type: mapFieldTypeToDataType(uploadField.contentstackFieldType), + mandatory: uploadField.advanced?.mandatory || false, + unique: uploadField.advanced?.unique || false, + field_metadata: { _default: true }, + format: '', + error_messages: { format: '' }, + multiple: uploadField.advanced?.multiple || false, + non_localizable: uploadField.advanced?.non_localizable || false + }); + } + } + + return apiSchema; +} + +/** + * Maps upload-api field types to API data types + * This ensures proper field type preservation from upload-api to API + */ +function mapFieldTypeToDataType(fieldType: string): string { + const fieldTypeMap: { [key: string]: string } = { + 'single_line_text': 'text', + 'multi_line_text': 'text', + 'text': 'text', + 'html': 'html', + 'json': 'json', + 'markdown': 'text', + 'number': 'number', + 'boolean': 'boolean', + 'isodate': 'isodate', + 'file': 'file', + 'reference': 'reference', + 'taxonomy': 'taxonomy', + 'link': 'link', + 'dropdown': 'text', + 'radio': 'text', + 'checkbox': 'boolean', + 'global_field': 'global_field', + 'group': 'group', + 'url': 'text' + }; + + return fieldTypeMap[fieldType] || 'text'; +} diff --git a/api/src/services/drupal/entries.service.ts b/api/src/services/drupal/entries.service.ts new file mode 100644 index 000000000..449b192cc --- /dev/null +++ b/api/src/services/drupal/entries.service.ts @@ -0,0 +1,1359 @@ +import fs from "fs"; +import path from "path"; +import mysql from 'mysql2'; +import { v4 as uuidv4 } from "uuid"; +import { JSDOM } from "jsdom"; +import { htmlToJson, jsonToHtml, jsonToMarkdown } from '@contentstack/json-rte-serializer'; +import { CHUNK_SIZE, LOCALE_MAPPER, MIGRATION_DATA_CONFIG } from "../../constants/index.js"; +import { getLogMessage } from "../../utils/index.js"; +import customLogger from "../../utils/custom-logger.utils.js"; +import { getDbConnection } from "../../helper/index.js"; +import { analyzeFieldTypes, isTaxonomyField, isReferenceField, isAssetField, type TaxonomyFieldMapping, type ReferenceFieldMapping, type AssetFieldMapping } from "./field-analysis.service.js"; +import FieldFetcherService from './field-fetcher.service.js'; +// Dynamic import for phpUnserialize will be used in the function + +interface TaxonomyReference { + drupal_term_id: number; + taxonomy_uid: string; + term_uid: string; +} + +interface TaxonomyFieldOutput { + taxonomy_uid: string; + term_uid: string; +} + +const { + DATA, + ENTRIES_DIR_NAME, + ENTRIES_MASTER_FILE, + ASSETS_DIR_NAME, + ASSETS_SCHEMA_FILE, + REFERENCES_DIR_NAME, + REFERENCES_FILE_NAME, + TAXONOMIES_DIR_NAME, +} = MIGRATION_DATA_CONFIG; + +interface DrupalFieldConfig { + field_name: string; + field_type: string; + settings?: { + handler?: string; + [key: string]: any; + }; + [key: string]: any; +} + +interface DrupalEntry { + nid: number; + title: string; + langcode: string; + created: number; + type: string; + [key: string]: any; +} + +interface QueryConfig { + page: { + [contentType: string]: string; + }; + count: { + [contentTypeCount: string]: string; + }; +} + +const LIMIT = 5; // Pagination limit + +// NOTE: Hardcoded queries have been REMOVED. All queries are now generated dynamically +// by the query.service.ts based on actual database field analysis. + +/** + * Executes SQL query and returns results as Promise + */ +const executeQuery = (connection: mysql.Connection, query: string): Promise => { + return new Promise((resolve, reject) => { + connection.query(query, (error, results) => { + if (error) { + reject(error); + } else { + resolve(results as any[]); + } + }); + }); +}; + +/** + * Load taxonomy reference mappings from taxonomyReference.json + */ +const loadTaxonomyReferences = async (referencesPath: string): Promise> => { + try { + const taxonomyRefPath = path.join(referencesPath, 'taxonomyReference.json'); + + if (!fs.existsSync(taxonomyRefPath)) { + return {}; + } + + const taxonomyReferences: TaxonomyReference[] = JSON.parse( + fs.readFileSync(taxonomyRefPath, 'utf8') + ); + + // Create lookup map: drupal_term_id -> {taxonomy_uid, term_uid} + const lookup: Record = {}; + + taxonomyReferences.forEach(ref => { + lookup[ref.drupal_term_id] = { + taxonomy_uid: ref.taxonomy_uid, + term_uid: ref.term_uid + }; + }); + + return lookup; + } catch (error) { + console.warn('Could not load taxonomy references:', error); + return {}; + } +}; + +/** + * Writes data to a specified file, ensuring the target directory exists. + */ +async function writeFile(dirPath: string, filename: string, data: any) { + try { + await fs.promises.mkdir(dirPath, { recursive: true }); + const filePath = path.join(dirPath, filename); + await fs.promises.writeFile(filePath, JSON.stringify(data), 'utf8'); + } catch (err) { + console.error(`Error writing ${dirPath}/${filename}::`, err); + } +} + +/** + * Reads a file and returns its JSON content. + */ +async function readFile(filePath: string, fileName: string) { + try { + const data = await fs.promises.readFile(path.join(filePath, fileName), "utf8"); + return JSON.parse(data); + } catch (err) { + return {}; + } +} + +/** + * Splits the given entry data into chunks that are under the specified size in bytes. + */ +function makeChunks(entryData: any) { + let currentChunkSize = 0; + const chunkSize = CHUNK_SIZE; // 1 MB in bytes + let currentChunkId = uuidv4(); + const chunks: { [key: string]: any } = {}; + + for (const [key, value] of Object.entries(entryData)) { + const tempObj = { [(value as { uid: string }).uid]: value }; + chunks[currentChunkId] = { ...chunks[currentChunkId], ...tempObj }; + + currentChunkSize = Buffer.byteLength( + JSON.stringify(chunks[currentChunkId]), + "utf8" + ); + + if (currentChunkSize > chunkSize) { + currentChunkId = uuidv4(); + currentChunkSize = 0; + chunks[currentChunkId] = {}; + } + } + + return chunks; +} + +/** + * Fetches field configurations from Drupal database + */ +const fetchFieldConfigs = async ( + connection: mysql.Connection, + projectId: string, + destination_stack_id: string +): Promise => { + const srcFunc = 'fetchFieldConfigs'; + const contentTypeQuery = "SELECT *, CONVERT(data USING utf8) as data FROM config WHERE name LIKE '%field.field.node%'"; + + try { + const results = await executeQuery(connection, contentTypeQuery); + + const fieldConfigs: DrupalFieldConfig[] = []; + for (const row of results) { + try { + const { unserialize } = await import('php-serialize'); + const configData = unserialize(row.data); + if (configData && typeof configData === 'object') { + fieldConfigs.push(configData as DrupalFieldConfig); + } + } catch (parseError) { + console.warn(`Failed to parse field config for ${row.name}:`, parseError); + } + } + + const message = getLogMessage( + srcFunc, + `Fetched ${fieldConfigs.length} field configurations from database.`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', message); + + return fieldConfigs; + } catch (error: any) { + const message = getLogMessage( + srcFunc, + `Failed to fetch field configurations: ${error.message}`, + {}, + error + ); + await customLogger(projectId, destination_stack_id, 'error', message); + throw error; + } +}; + +/** + * Determines the source field type based on the value structure + */ +const determineSourceFieldType = (value: any): string => { + if (typeof value === 'object' && value !== null && value.type === 'doc') { + return 'json_rte'; + } + if (typeof value === 'string' && /<\/?[a-z][\s\S]*>/i.test(value)) { + return 'html_rte'; + } + if (typeof value === 'string') { + // Simple heuristic: if it has line breaks, consider it multi-line + return value.includes('\n') || value.includes('\r') ? 'multi_line' : 'single_line'; + } + return 'unknown'; +}; + +/** + * Checks if conversion is allowed based on the exact rules: + * 1. Single-line text โ†’ Single-line/Multi-line/HTML RTE/JSON RTE + * 2. Multi-line text โ†’ Multi-line/HTML RTE/JSON RTE (NOT Single-line) + * 3. HTML RTE โ†’ HTML RTE/JSON RTE (NOT Single-line or Multi-line) + * 4. JSON RTE โ†’ JSON RTE/HTML RTE (NOT Single-line or Multi-line) + */ +const isConversionAllowed = (sourceType: string, targetType: string): boolean => { + const conversionRules: { [key: string]: string[] } = { + 'single_line': ['single_line_text', 'text', 'multi_line_text', 'html', 'json'], + 'multi_line': ['multi_line_text', 'text', 'html', 'json'], // Cannot convert to single_line_text + 'html_rte': ['html', 'json'], // Cannot convert to single_line_text or multi_line_text + 'json_rte': ['json', 'html'] // Cannot convert to single_line_text or multi_line_text + }; + + return conversionRules[sourceType]?.includes(targetType) || false; +}; + +/** + * Processes field values based on content type mapping and field type switching + * Follows proper conversion rules for field type compatibility + */ +const processFieldByType = ( + value: any, + fieldMapping: any, + assetId: any, + referenceId: any +): any => { + if (!fieldMapping || !fieldMapping.contentstackFieldType) { + return value; + } + + // Determine source field type + const sourceType = determineSourceFieldType(value); + const targetType = fieldMapping.contentstackFieldType; + + // Check if conversion is allowed + if (!isConversionAllowed(sourceType, targetType)) { + console.warn(`Conversion not allowed: ${sourceType} โ†’ ${targetType}. Keeping original value.`); + return value; + } + + switch (targetType) { + case 'single_line_text': { + // Convert to single line text + if (typeof value === 'object' && value !== null && value.type === 'doc') { + // JSON RTE to plain text (extract text content) + try { + const htmlContent = jsonToHtml(value) || ''; + // Strip HTML tags and convert to single line + const textContent = htmlContent.replace(/<[^>]*>/g, '').replace(/\s+/g, ' ').trim(); + return textContent; + } catch (error) { + console.warn('Failed to convert JSON RTE to single line text:', error); + return String(value || ''); + } + } else if (typeof value === 'string') { + if (/<\/?[a-z][\s\S]*>/i.test(value)) { + // HTML to plain text + const textContent = value.replace(/<[^>]*>/g, '').replace(/\s+/g, ' ').trim(); + return textContent; + } + // Multi-line to single line + return value.replace(/\s+/g, ' ').trim(); + } + return String(value || ''); + } + + case 'text': + case 'multi_line_text': { + // Convert to multi-line text + if (typeof value === 'object' && value !== null && value.type === 'doc') { + // JSON RTE to HTML (preserving structure) + try { + return jsonToHtml(value, { + customElementTypes: { + "social-embed": (attrs, child, jsonBlock) => { + return `${child}`; + }, + }, + customTextWrapper: { + "color": (child, value) => { + return `${child}`; + }, + }, + }) || ''; + } catch (error) { + console.warn('Failed to convert JSON RTE to HTML:', error); + return String(value || ''); + } + } + // HTML and plain text can stay as-is for multi-line + return typeof value === 'string' ? value : String(value || ''); + } + + case 'json': { + // Convert to JSON RTE + if (typeof value === 'string' && /<\/?[a-z][\s\S]*>/i.test(value)) { + // HTML to JSON RTE + try { + const dom = new JSDOM(value); + const htmlDoc = dom.window.document.querySelector('body'); + if (htmlDoc) { + htmlDoc.innerHTML = value; + return htmlToJson(htmlDoc); + } + } catch (error) { + console.warn('Failed to convert HTML to JSON RTE:', error); + } + } else if (typeof value === 'string') { + // Plain text to JSON RTE + try { + const dom = new JSDOM(`

${value}

`); + const htmlDoc = dom.window.document.querySelector('body'); + if (htmlDoc) { + return htmlToJson(htmlDoc); + } + } catch (error) { + console.warn('Failed to convert text to JSON RTE:', error); + } + } + // If already JSON RTE or conversion failed, return as-is + return value; + } + + case 'html': { + // Convert to HTML RTE + if (typeof value === 'object' && value !== null && value.type === 'doc') { + // JSON RTE to HTML + try { + return jsonToHtml(value, { + customElementTypes: { + "social-embed": (attrs, child, jsonBlock) => { + return `${child}`; + }, + }, + customTextWrapper: { + "color": (child, value) => { + return `${child}`; + }, + }, + }) || '

'; + } catch (error) { + console.warn('Failed to convert JSON RTE to HTML:', error); + return value; + } + } + // If already HTML or plain text, return as-is + return typeof value === 'string' ? value : String(value || ''); + } + + case 'markdown': { + // Convert to Markdown + if (typeof value === 'object' && value !== null && value.type === 'doc') { + try { + return jsonToMarkdown(value); + } catch (error) { + console.warn('Failed to convert JSON RTE to Markdown:', error); + return value; + } + } + return typeof value === 'string' ? value : String(value || ''); + } + + case 'file': { + // File/Asset processing with proper validation and cleanup + if (fieldMapping.advanced?.multiple) { + // Multiple files + if (Array.isArray(value)) { + const validAssets = value + .map(assetRef => { + const assetKey = `assets_${assetRef}`; + const assetReference = assetId[assetKey]; + + if (assetReference && typeof assetReference === 'object') { + return assetReference; + } + + console.warn(`Asset ${assetKey} not found or invalid, excluding from array`); + return null; + }) + .filter(asset => asset !== null); // Remove null entries + + return validAssets.length > 0 ? validAssets : undefined; // Return undefined if no valid assets + } + } else { + // Single file + const assetKey = `assets_${value}`; + const assetReference = assetId[assetKey]; + + if (assetReference && typeof assetReference === 'object') { + return assetReference; + } + + console.warn(`Asset ${assetKey} not found or invalid, removing field`); + return undefined; // Return undefined to indicate field should be removed + } + return value; + } + + case 'reference': { + // Reference processing + if (fieldMapping.advanced?.multiple) { + // Multiple references + if (Array.isArray(value)) { + return value.map(refId => referenceId[`content_type_entries_title_${refId}`] || refId); + } + } else { + // Single reference + return [referenceId[`content_type_entries_title_${value}`] || value]; + } + return value; + } + + case 'number': { + // Number processing + if (typeof value === 'string') { + const parsed = parseInt(value); + return isNaN(parsed) ? 0 : parsed; + } + return typeof value === 'number' ? value : 0; + } + + case 'boolean': { + // Boolean processing + if (typeof value === 'string') { + return value === '1' || value.toLowerCase() === 'true'; + } + return Boolean(value); + } + + case 'isodate': { + // Date processing + if (typeof value === 'number') { + return new Date(value * 1000).toISOString(); + } + return value; + } + + default: { + // Default processing - handle HTML content + if (typeof value === 'string' && /<\/?[a-z][\s\S]*>/i.test(value)) { + try { + const dom = new JSDOM(value); + const htmlDoc = dom.window.document.querySelector('body'); + return htmlToJson(htmlDoc); + } catch (error) { + return value; + } + } + return value; + } + } +}; + +/** + * Consolidates all taxonomy fields into a single 'taxonomies' field with unique term_uid validation + * + * @param processedEntry - The processed entry data + * @param contentType - The content type being processed + * @param taxonomyFieldMapping - Mapping of taxonomy fields from field analysis + * @returns Entry with consolidated taxonomy field + */ +const consolidateTaxonomyFields = ( + processedEntry: any, + contentType: string, + taxonomyFieldMapping: TaxonomyFieldMapping +): any => { + const consolidatedTaxonomies: Array<{ taxonomy_uid: string; term_uid: string }> = []; + const fieldsToRemove: string[] = []; + const seenTermUids = new Set(); // Track unique term_uid values + + // Iterate through all fields in the processed entry + for (const [fieldKey, fieldValue] of Object.entries(processedEntry)) { + // Extract field name from key (remove _target_id suffix) + const fieldName = fieldKey.replace(/_target_id$/, ''); + + // Check if this is a taxonomy field using field analysis + if (isTaxonomyField(fieldName, contentType, taxonomyFieldMapping)) { + // Validate that field value is an array with taxonomy structure + if (Array.isArray(fieldValue)) { + for (const taxonomyItem of fieldValue) { + // Validate taxonomy structure + if (taxonomyItem && + typeof taxonomyItem === 'object' && + taxonomyItem.taxonomy_uid && + taxonomyItem.term_uid) { + + // Check for unique term_uid (avoid duplicates) + if (!seenTermUids.has(taxonomyItem.term_uid)) { + consolidatedTaxonomies.push({ + taxonomy_uid: taxonomyItem.taxonomy_uid, + term_uid: taxonomyItem.term_uid + }); + seenTermUids.add(taxonomyItem.term_uid); + } + } + } + } + + // Mark this field for removal + fieldsToRemove.push(fieldKey); + } + } + + // Create new entry object without the original taxonomy fields + const consolidatedEntry = { ...processedEntry }; + + // Remove original taxonomy fields + for (const fieldKey of fieldsToRemove) { + delete consolidatedEntry[fieldKey]; + } + + // Add consolidated taxonomy field if we have any taxonomies + if (consolidatedTaxonomies.length > 0) { + consolidatedEntry.taxonomies = consolidatedTaxonomies; + console.log(`๐Ÿท๏ธ Consolidated ${fieldsToRemove.length} taxonomy fields into 'taxonomies' with ${consolidatedTaxonomies.length} unique terms for ${contentType}`); + } + + // Replace existing 'taxonomies' field if it exists (as per requirement) + if ('taxonomies' in processedEntry && consolidatedTaxonomies.length > 0) { + console.log(`๐Ÿ”„ Replaced existing 'taxonomies' field with consolidated data for ${contentType}`); + } + + return consolidatedEntry; +}; + +/** + * Processes field values based on field configuration - following original Drupal logic + */ +const processFieldData = async ( + entryData: DrupalEntry, + fieldConfigs: DrupalFieldConfig[], + assetId: any, + referenceId: any, + taxonomyId: any, + taxonomyFieldMapping: TaxonomyFieldMapping, + referenceFieldMapping: ReferenceFieldMapping, + assetFieldMapping: any, + taxonomyReferenceLookup: Record, + contentType: string +): Promise => { + const fieldNames = Object.keys(entryData); + const isoDate = new Date(); + const processedData: any = {}; + const skippedFields = new Set(); // Track fields that should be skipped entirely + const processedFields = new Set(); // Track fields that have been processed to avoid duplicates + + // Process each field in the entry data + for (const [dataKey, value] of Object.entries(entryData)) { + // Extract field name from dataKey (remove _target_id suffix) + const fieldName = dataKey.replace(/_target_id$/, '').replace(/_value$/, '').replace(/_status$/, '').replace(/_uri$/, ''); + + // Handle asset fields using field analysis + if (dataKey.endsWith('_target_id') && isAssetField(fieldName, contentType, assetFieldMapping)) { + const assetKey = `assets_${value}`; + if (assetKey in assetId) { + // Transform to proper Contentstack asset reference format + const assetReference = assetId[assetKey]; + if (assetReference && typeof assetReference === 'object') { + processedData[dataKey] = assetReference; + } + // If asset reference is not properly structured, skip the field + } + // If asset not found in assets index, mark field as skipped + skippedFields.add(dataKey); + continue; // Skip further processing for this field + } + + // Handle entity references (taxonomy and node references) using field analysis + if (dataKey.endsWith('_target_id') && typeof value === 'number') { + // Check if this is a taxonomy field using our field analysis + if (isTaxonomyField(fieldName, contentType, taxonomyFieldMapping)) { + // Look up taxonomy reference using drupal_term_id + const taxonomyRef = taxonomyReferenceLookup[value]; + + if (taxonomyRef) { + // Transform to array format with taxonomy_uid and term_uid (no drupal_term_id) + processedData[dataKey] = [{ + taxonomy_uid: taxonomyRef.taxonomy_uid, + term_uid: taxonomyRef.term_uid + }]; + } else { + // Fallback to numeric tid if lookup failed + processedData[dataKey] = value; + } + continue; // Skip further processing for this field + } else if (isReferenceField(fieldName, contentType, referenceFieldMapping)) { + // Handle node reference fields using field analysis + const referenceKey = `content_type_entries_title_${value}`; + if (referenceKey in referenceId) { + // Transform to array format with proper reference structure + processedData[dataKey] = [referenceId[referenceKey]]; + } else { + // If reference not found, mark field as skipped + skippedFields.add(dataKey); + } + continue; // Skip further processing for this field + } + } + + // Handle other field types by checking field configs + const matchingFieldConfig = fieldConfigs.find(fc => + dataKey === `${fc.field_name}_value` || + dataKey === `${fc.field_name}_status` || + dataKey === fc.field_name + ); + + if (matchingFieldConfig) { + // Handle datetime and timestamps + if (matchingFieldConfig.field_type === 'datetime' || matchingFieldConfig.field_type === 'timestamp') { + if (dataKey === `${matchingFieldConfig.field_name}_value`) { + if (typeof value === 'number') { + processedData[dataKey] = new Date(value * 1000).toISOString(); + } else { + processedData[dataKey] = isoDate.toISOString(); + } + continue; + } + } + + // Handle boolean fields + if (matchingFieldConfig.field_type === 'boolean') { + if (dataKey === `${matchingFieldConfig.field_name}_value` && typeof value === 'number') { + processedData[dataKey] = value === 1; + continue; + } + } + + // Handle comment fields + if (matchingFieldConfig.field_type === 'comment') { + if (dataKey === `${matchingFieldConfig.field_name}_status` && typeof value === 'number') { + processedData[dataKey] = `${value}`; + continue; + } + } + } + + // Remove null, undefined, and empty values + if (value === null || value === undefined || value === '') { + // Skip null, undefined, and empty string values + continue; + } + + // Default case: copy field to processedData if it wasn't handled by special processing above + if (!(dataKey in processedData)) { + processedData[dataKey] = value; + } + } + + // Process standard field transformations + const ctValue: any = {}; + + for (const fieldName of fieldNames) { + // Skip fields that were intentionally excluded in the main processing loop + if (skippedFields.has(fieldName)) { + continue; + } + + const value = entryData[fieldName]; + + if (fieldName === 'created') { + ctValue[fieldName] = new Date(value * 1000).toISOString(); + } else if (fieldName === 'uid_name') { + ctValue[fieldName] = [value]; + } else if (fieldName.endsWith('_tid')) { + ctValue[fieldName] = [value]; + } else if (fieldName === 'nid') { + ctValue.uid = `content_type_entries_title_${value}`; + } else if (fieldName === 'langcode') { + // Use the actual langcode from the entry for proper multilingual support + ctValue.locale = value || 'en-us'; // fallback to en-us if langcode is empty + } else if (fieldName.endsWith('_uri')) { + // Skip if this field has already been processed + if (processedFields.has(fieldName)) { + continue; + } + + const baseFieldName = fieldName.replace('_uri', ''); + const titleFieldName = `${baseFieldName}_title`; + + // Check if we also have title data + const titleValue = entryData[titleFieldName]; + + if (value) { + ctValue[baseFieldName] = { + title: titleValue || value, // Use title if available, fallback to URI + href: value, + }; + } else { + ctValue[baseFieldName] = { + title: titleValue || '', + href: '', + }; + } + + // Mark title field as processed to avoid duplicate processing + if (titleValue) { + processedFields.add(titleFieldName); + } + } else if (fieldName.endsWith('_title')) { + // Skip _title fields as they're handled with _uri fields + if (processedFields.has(fieldName)) { + continue; + } + + // Check if there's a corresponding _uri field + const baseFieldName = fieldName.replace('_title', ''); + const uriFieldName = `${baseFieldName}_uri`; + + if (entryData[uriFieldName]) { + // URI field will handle this, skip processing here + continue; + } else { + // No URI field found, process title field standalone (rare case) + ctValue[baseFieldName] = { + title: value || '', + href: '', + }; + } + } else if (fieldName.endsWith('_value')) { + // Check if content contains HTML + if (/<\/?[a-z][\s\S]*>/i.test(value)) { + const dom = new JSDOM(value); + const htmlDoc = dom.window.document.querySelector('body'); + const jsonValue = htmlToJson(htmlDoc); + ctValue[fieldName.replace('_value', '')] = jsonValue; + } else { + ctValue[fieldName.replace('_value', '')] = value; + } + } else if (fieldName.endsWith('_status')) { + ctValue[fieldName.replace('_status', '')] = value; + } else { + // Check if content contains HTML + if (typeof value === 'string' && /<\/?[a-z][\s\S]*>/i.test(value)) { + const dom = new JSDOM(value); + const htmlDoc = dom.window.document.querySelector('body'); + const jsonValue = htmlToJson(htmlDoc); + ctValue[fieldName] = jsonValue; + } else { + ctValue[fieldName] = value; + } + } + } + + // Apply processed field data + Object.assign(ctValue, processedData); + + // Final cleanup: remove any null, undefined, or empty values from the final result + const cleanedEntry: any = {}; + for (const [key, val] of Object.entries(ctValue)) { + if (val !== null && val !== undefined && val !== '') { + cleanedEntry[key] = val; + } + } + + return cleanedEntry; +}; + +/** + * Processes entries for a specific content type and pagination offset + */ +const processEntries = async ( + connection: mysql.Connection, + contentType: string, + skip: number, + queryPageConfig: QueryConfig, + fieldConfigs: DrupalFieldConfig[], + assetId: any, + referenceId: any, + taxonomyId: any, + taxonomyFieldMapping: TaxonomyFieldMapping, + referenceFieldMapping: ReferenceFieldMapping, + assetFieldMapping: AssetFieldMapping, + taxonomyReferenceLookup: Record, + projectId: string, + destination_stack_id: string, + masterLocale: string, + contentTypeMapping: any[] = [], + isTest: boolean = false +): Promise<{ [key: string]: any } | null> => { + const srcFunc = 'processEntries'; + + try { + // Following original pattern: queryPageConfig['page']['' + pagename + ''] + const baseQuery = queryPageConfig['page'][contentType]; + if (!baseQuery) { + throw new Error(`No query found for content type: ${contentType}`); + } + + // Check if this is an optimized query (content type with many fields) + const isOptimizedQuery = baseQuery.includes('/* OPTIMIZED_NO_JOINS:'); + let entries: any[] = []; + + if (isOptimizedQuery) { + // Handle content types with many fields using optimized approach + const fieldCountMatch = baseQuery.match(/\/\* OPTIMIZED_NO_JOINS:(\d+) \*\//); + const fieldCount = fieldCountMatch ? parseInt(fieldCountMatch[1]) : 0; + + const optimizedMessage = getLogMessage( + srcFunc, + `Processing ${contentType} with optimized field fetching (${fieldCount} fields)`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', optimizedMessage); + + // Execute base query without field JOINs + const effectiveLimit = isTest ? 1 : LIMIT; + const cleanBaseQuery = baseQuery.replace(/\/\* OPTIMIZED_NO_JOINS:\d+ \*\//, '').trim(); + const query = cleanBaseQuery + ` LIMIT ${skip}, ${effectiveLimit}`; + const baseEntries = await executeQuery(connection, query); + + if (baseEntries.length === 0) { + return null; + } + + // Fetch field data separately using FieldFetcherService + const fieldFetcher = new FieldFetcherService(connection, projectId, destination_stack_id); + const nodeIds = baseEntries.map(entry => entry.nid); + const fieldsForType = await fieldFetcher.getFieldsForContentType(contentType); + + if (fieldsForType.length > 0) { + const fieldData = await fieldFetcher.fetchFieldDataForContentType( + contentType, + nodeIds, + fieldsForType + ); + + // Merge base entries with field data + entries = fieldFetcher.mergeNodeAndFieldData(baseEntries, fieldData); + + const mergeMessage = getLogMessage( + srcFunc, + `Merged ${baseEntries.length} base entries with field data for ${contentType}`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', mergeMessage); + } else { + entries = baseEntries; + } + + } else { + // Handle content types with few fields using traditional approach + const effectiveLimit = isTest ? 1 : LIMIT; + const query = baseQuery + ` LIMIT ${skip}, ${effectiveLimit}`; + entries = await executeQuery(connection, query); + + if (entries.length === 0) { + return null; + } + } + + // Group entries by their actual locale (langcode) for proper multilingual support + const entriesByLocale: { [locale: string]: any[] } = {}; + + // Group entries by their langcode + entries.forEach(entry => { + const entryLocale = entry.langcode || masterLocale; // fallback to masterLocale if no langcode + if (!entriesByLocale[entryLocale]) { + entriesByLocale[entryLocale] = []; + } + entriesByLocale[entryLocale].push(entry); + }); + + console.log(`๐Ÿ“ Found entries in ${Object.keys(entriesByLocale).length} locales for ${contentType}:`, Object.keys(entriesByLocale)); + + // ๐Ÿ”„ Apply locale folder transformation rules (same as locale service) + const transformedEntriesByLocale: { [locale: string]: any[] } = {}; + const allLocales = Object.keys(entriesByLocale); + const hasUnd = allLocales.includes('und'); + const hasEn = allLocales.includes('en'); + const hasEnUs = allLocales.includes('en-us'); + + console.log(`๐Ÿ” Locale Analysis: hasUnd=${hasUnd}, hasEn=${hasEn}, hasEnUs=${hasEnUs}`); + + // Transform locale folder names based on business rules + Object.entries(entriesByLocale).forEach(([originalLocale, entries]) => { + let targetFolder = originalLocale; + + if (originalLocale === 'und') { + if (hasEn && hasEnUs) { + // If all three present, "und" stays as "und" folder + targetFolder = 'und'; + console.log(`๐Ÿ”„ "und" entries โ†’ "und" folder (all three present)`); + } else if (hasEnUs) { + // If "und" + "en-us", "und" goes to "en" folder + targetFolder = 'en'; + console.log(`๐Ÿ”„ Transforming "und" entries โ†’ "en" folder (en-us exists)`); + } else { + // If only "und", use "en-us" folder + targetFolder = 'en-us'; + console.log(`๐Ÿ”„ Transforming "und" entries โ†’ "en-us" folder`); + } + } else if (originalLocale === 'en-us') { + if (hasUnd && !hasEn) { + // If "und" + "en-us" (no en), "und" becomes "en", so keep "en-us" + targetFolder = 'en-us'; + console.log(`๐Ÿ”„ "en-us" entries โ†’ "en-us" folder (und becomes en)`); + } else { + // Keep en-us as is in other cases + targetFolder = 'en-us'; + } + } else if (originalLocale === 'en') { + if (hasEnUs && !hasUnd) { + // If "en" + "en-us" (no und), "en" becomes "und" folder + targetFolder = 'und'; + console.log(`๐Ÿ”„ Transforming "en" entries โ†’ "und" folder (en-us exists, no und)`); + } else { + // Keep "en" as is in other cases + targetFolder = 'en'; + } + } + + // Merge entries if target folder already has entries + if (transformedEntriesByLocale[targetFolder]) { + transformedEntriesByLocale[targetFolder] = [ + ...transformedEntriesByLocale[targetFolder], + ...entries + ]; + console.log(`๐Ÿ“ Merging ${originalLocale} entries into existing ${targetFolder} folder`); + } else { + transformedEntriesByLocale[targetFolder] = entries; + console.log(`๐Ÿ“ Creating ${targetFolder} folder for ${originalLocale} entries`); + } + }); + + console.log(`๐Ÿ“‚ Final folder structure:`, Object.keys(transformedEntriesByLocale)); + + // Find content type mapping for field type switching + const currentContentTypeMapping = contentTypeMapping.find(ct => + ct.otherCmsUid === contentType || ct.contentstackUid === contentType + ); + + const allProcessedContent: { [key: string]: any } = {}; + + // Process entries for each transformed locale separately + for (const [currentLocale, localeEntries] of Object.entries(transformedEntriesByLocale)) { + console.log(`๐ŸŒ Processing ${localeEntries.length} entries for transformed locale: ${currentLocale}`); + + // Create folder structure: entries/contentType/locale/ + const contentTypeFolderPath = path.join(MIGRATION_DATA_CONFIG.DATA, destination_stack_id, MIGRATION_DATA_CONFIG.ENTRIES_DIR_NAME, contentType); + const localeFolderPath = path.join(contentTypeFolderPath, currentLocale); + await fs.promises.mkdir(localeFolderPath, { recursive: true }); + + // Read existing content for this locale or initialize + const localeFileName = `${currentLocale}.json`; + const existingLocaleContent = await readFile(localeFolderPath, localeFileName) || {}; + + // Process each entry in this locale + for (const entry of localeEntries) { + let processedEntry = await processFieldData(entry, fieldConfigs, assetId, referenceId, taxonomyId, taxonomyFieldMapping, referenceFieldMapping, assetFieldMapping, taxonomyReferenceLookup, contentType); + + // ๐Ÿท๏ธ TAXONOMY CONSOLIDATION: Merge all taxonomy fields into single 'taxonomies' field + processedEntry = consolidateTaxonomyFields(processedEntry, contentType, taxonomyFieldMapping); + + // Apply field type switching based on user's UI selections (from content type schema) + const enhancedEntry: any = {}; + + // Process each field with type switching support + for (const [fieldName, fieldValue] of Object.entries(processedEntry)) { + let fieldMapping = null; + + // First try to find mapping from UI content type mapping + if (currentContentTypeMapping && currentContentTypeMapping.fieldMapping) { + fieldMapping = currentContentTypeMapping.fieldMapping.find((fm: any) => + fm.uid === fieldName || + fm.otherCmsField === fieldName || + fieldName.startsWith(fm.uid) || + fieldName.includes(fm.uid) + ); + } + + // If no UI mapping found, try to infer from content type schema + if (!fieldMapping) { + // Load the content type schema to get user's field type selections + try { + const contentTypeSchemaPath = path.join(MIGRATION_DATA_CONFIG.DATA, destination_stack_id, 'content_types', `${contentType}.json`); + const contentTypeSchema = JSON.parse(await fs.promises.readFile(contentTypeSchemaPath, 'utf8')); + + // Find field in schema + const schemaField = contentTypeSchema.schema?.find((field: any) => + field.uid === fieldName || + field.uid === fieldName.replace(/_target_id$/, '') || + field.uid === fieldName.replace(/_value$/, '') || + fieldName.includes(field.uid) + ); + + if (schemaField) { + // Determine the proper field type based on schema configuration + let targetFieldType = schemaField.data_type; + + // Handle text fields with multiline metadata + if (schemaField.data_type === 'text' && schemaField.field_metadata?.multiline) { + targetFieldType = 'multi_line_text'; // This will be handled as HTML in processFieldByType + } + + // Create a mapping from schema field + fieldMapping = { + uid: fieldName, + contentstackFieldType: targetFieldType, + backupFieldType: schemaField.data_type, + advanced: schemaField + }; + + console.log(`๐Ÿ“‹ Field mapping created for ${fieldName}: ${targetFieldType} (from schema)`); + } + } catch (error: any) { + console.warn(`Failed to load content type schema for field ${fieldName}:`, error.message); + } + } + + if (fieldMapping) { + // Apply field type processing based on user's selection + const processedValue = processFieldByType(fieldValue, fieldMapping, assetId, referenceId); + + // Only add field if processed value is not undefined (undefined means remove field) + if (processedValue !== undefined) { + enhancedEntry[fieldName] = processedValue; + + // Log field type processing + if (fieldMapping.contentstackFieldType !== fieldMapping.backupFieldType) { + const message = getLogMessage( + srcFunc, + `Field ${fieldName} processed as ${fieldMapping.contentstackFieldType} (switched from ${fieldMapping.backupFieldType})`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', message); + } + } else { + // Log field removal + const message = getLogMessage( + srcFunc, + `Field ${fieldName} removed due to missing or invalid asset reference`, + {} + ); + await customLogger(projectId, destination_stack_id, 'warn', message); + } + } else { + // Keep original value if no mapping found + enhancedEntry[fieldName] = fieldValue; + } + } + + processedEntry = enhancedEntry; + + if (typeof entry.nid === 'number') { + existingLocaleContent[`content_type_entries_title_${entry.nid}`] = processedEntry; + allProcessedContent[`content_type_entries_title_${entry.nid}`] = processedEntry; + } + + // Log each entry transformation + const message = getLogMessage( + srcFunc, + `Entry with uid ${entry.nid} (locale: ${currentLocale}) for content type ${contentType} has been successfully transformed.`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', message); + } + + // Write processed content for this specific locale + await writeFile(localeFolderPath, localeFileName, existingLocaleContent); + + const localeMessage = getLogMessage( + srcFunc, + `Successfully processed ${localeEntries.length} entries for locale ${currentLocale} in content type ${contentType}`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', localeMessage); + } + + // ๐Ÿ“ Create mandatory index.json files for each transformed locale directory + for (const [currentLocale, localeEntries] of Object.entries(transformedEntriesByLocale)) { + if (localeEntries.length > 0) { + const contentTypeFolderPath = path.join(MIGRATION_DATA_CONFIG.DATA, destination_stack_id, MIGRATION_DATA_CONFIG.ENTRIES_DIR_NAME, contentType); + const localeFolderPath = path.join(contentTypeFolderPath, currentLocale); + const localeFileName = `${currentLocale}.json`; + + // Create mandatory index.json file that maps to the locale file + const indexData = { "1": localeFileName }; + await writeFile(localeFolderPath, 'index.json', indexData); + + console.log(`๐Ÿ“ Created mandatory index.json for ${contentType}/${currentLocale} โ†’ ${localeFileName}`); + } + } + + return allProcessedContent; + } catch (error: any) { + const message = getLogMessage( + srcFunc, + `Error processing entries for ${contentType}: ${error.message}`, + {}, + error + ); + await customLogger(projectId, destination_stack_id, 'error', message); + throw error; + } +}; + +/** + * Gets count and processes all entries for a specific content type + */ +const processContentType = async ( + connection: mysql.Connection, + contentType: string, + queryPageConfig: QueryConfig, + fieldConfigs: DrupalFieldConfig[], + assetId: any, + referenceId: any, + taxonomyId: any, + taxonomyFieldMapping: TaxonomyFieldMapping, + referenceFieldMapping: ReferenceFieldMapping, + assetFieldMapping: AssetFieldMapping, + taxonomyReferenceLookup: Record, + projectId: string, + destination_stack_id: string, + masterLocale: string, + contentTypeMapping: any[] = [], + isTest: boolean = false +): Promise => { + const srcFunc = 'processContentType'; + + try { + // Get total count for pagination (if count query exists) + const countKey = `${contentType}Count`; + let totalCount = 1; // Default to process at least one batch + + if (queryPageConfig.count && queryPageConfig.count[countKey]) { + const countQuery = queryPageConfig.count[countKey]; + const countResults = await executeQuery(connection, countQuery); + totalCount = countResults[0]?.countentry || 0; + } + + if (totalCount === 0) { + const message = getLogMessage( + srcFunc, + `No entries found for content type ${contentType}.`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', message); + return; + } + + // ๐Ÿงช Process entries in batches (test migration: single entry, main migration: all entries) + const effectiveLimit = isTest ? 1 : LIMIT; + const maxIterations = isTest ? 1 : Math.ceil(totalCount / LIMIT); // Test: single iteration, Main: full pagination + + for (let i = 0; i < (isTest ? effectiveLimit : totalCount + LIMIT); i += effectiveLimit) { + const result = await processEntries( + connection, + contentType, + i, + queryPageConfig, + fieldConfigs, + assetId, + referenceId, + taxonomyId, + taxonomyFieldMapping, + referenceFieldMapping, + assetFieldMapping, + taxonomyReferenceLookup, + projectId, + destination_stack_id, + masterLocale, + contentTypeMapping, + isTest + ); + + // If no entries returned, break the loop + if (!result) { + break; + } + } + + } catch (error: any) { + const message = getLogMessage( + srcFunc, + `Error processing content type ${contentType}: ${error.message}`, + {}, + error + ); + await customLogger(projectId, destination_stack_id, 'error', message); + throw error; + } +}; + +/** + * Reads dynamic query configuration file generated by query.service.ts + * Following original pattern: helper.readFile(path.join(process.cwd(), config.data, 'query', 'index.json')) + * + * NOTE: No fallback to hardcoded queries - dynamic queries MUST be generated first + */ +async function readQueryConfig(destination_stack_id: string): Promise { + try { + const queryPath = path.join(DATA, destination_stack_id, 'query', 'index.json'); + const data = await fs.promises.readFile(queryPath, "utf8"); + return JSON.parse(data); + } catch (err) { + // No fallback - dynamic queries must be generated first by createQuery() service + throw new Error(`โŒ No dynamic query configuration found at query/index.json. Dynamic queries must be generated first using createQuery() service. Original error: ${err}`); + } +} + +/** + * Creates and processes entries from Drupal database for migration to Contentstack. + * Based on the original Drupal v8 migration logic with direct SQL queries. + * + * Supports dynamic SQL queries from query/index.json file following original pattern: + * var queryPageConfig = helper.readFile(path.join(process.cwd(), config.data, 'query', 'index.json')); + * var query = queryPageConfig['page']['' + pagename + '']; + */ +export const createEntry = async ( + dbConfig: any, + destination_stack_id: string, + projectId: string, + isTest = false, + masterLocale = 'en-us', + contentTypeMapping: any[] = [] +): Promise => { + const srcFunc = 'createEntry'; + let connection: mysql.Connection | null = null; + + try { + console.info('๐Ÿ” === DRUPAL ENTRIES SERVICE CONFIG ==='); + console.info('๐Ÿ“‹ Database Config:', JSON.stringify(dbConfig, null, 2)); + console.info('๐Ÿ“‹ Destination Stack ID:', destination_stack_id); + console.info('๐Ÿ“‹ Project ID:', projectId); + console.info('๐Ÿ“‹ Is Test Migration:', isTest); + console.info('๐Ÿ“‹ Function:', srcFunc); + console.info('========================================='); + + const entriesSave = path.join(DATA, destination_stack_id, ENTRIES_DIR_NAME); + const assetsSave = path.join(DATA, destination_stack_id, ASSETS_DIR_NAME); + const referencesSave = path.join(DATA, destination_stack_id, REFERENCES_DIR_NAME); + const taxonomiesSave = path.join(DATA, destination_stack_id, TAXONOMIES_DIR_NAME); + + // Initialize directories + await fs.promises.mkdir(entriesSave, { recursive: true }); + + const message = getLogMessage( + srcFunc, + `Exporting entries...`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', message); + + // Read query configuration (following original pattern) + const queryPageConfig = await readQueryConfig(destination_stack_id); + + // Create database connection + connection = await getDbConnection(dbConfig, projectId, destination_stack_id); + + // Analyze field types to identify taxonomy, reference, and asset fields + const { taxonomyFields: taxonomyFieldMapping, referenceFields: referenceFieldMapping, assetFields: assetFieldMapping } = await analyzeFieldTypes(dbConfig, destination_stack_id, projectId); + + // Fetch field configurations + const fieldConfigs = await fetchFieldConfigs(connection, projectId, destination_stack_id); + + // Read supporting data - following original page.js pattern + // Load assets from index.json (your new format) + const assetId = await readFile(assetsSave, 'index.json') || {}; + console.log(`๐Ÿ“ Loaded ${Object.keys(assetId).length} assets from index.json`); + + const referenceId = await readFile(referencesSave, REFERENCES_FILE_NAME) || {}; + const taxonomyId = await readFile(path.join(entriesSave, 'taxonomy'), `${masterLocale}.json`) || {}; + + // Load taxonomy reference mappings for field transformation + const taxonomyReferenceLookup = await loadTaxonomyReferences(referencesSave); + + // Process each content type from query config (like original) + const pageQuery = queryPageConfig.page; + const contentTypes = Object.keys(pageQuery); + // ๐Ÿงช Test migration: Process ALL content types but with limited data per content type + const typesToProcess = contentTypes; // Always process all content types + + for (const contentType of typesToProcess) { + await processContentType( + connection, + contentType, + queryPageConfig, + fieldConfigs, + assetId, + referenceId, + taxonomyId, + taxonomyFieldMapping, + referenceFieldMapping, + assetFieldMapping, + taxonomyReferenceLookup, + projectId, + destination_stack_id, + masterLocale, + contentTypeMapping, + isTest + ); + } + + const successMessage = getLogMessage( + srcFunc, + `Successfully processed entries for ${typesToProcess.length} content types with multilingual support.`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', successMessage); + + // Log multilingual structure summary + const structureSummary = getLogMessage( + srcFunc, + `Multilingual entries structure created at: ${DATA}/${destination_stack_id}/${ENTRIES_DIR_NAME}/[contentType]/[locale]/[locale].json`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', structureSummary); + + } catch (err) { + const message = getLogMessage( + srcFunc, + `Error encountered while creating entries.`, + {}, + err + ); + await customLogger(projectId, destination_stack_id, 'error', message); + throw err; + } finally { + // Close database connection + if (connection) { + connection.end(); + } + } +}; \ No newline at end of file diff --git a/api/src/services/drupal/field-analysis.service.ts b/api/src/services/drupal/field-analysis.service.ts new file mode 100644 index 000000000..fdfac7e90 --- /dev/null +++ b/api/src/services/drupal/field-analysis.service.ts @@ -0,0 +1,427 @@ +import mysql from "mysql2"; +import { getDbConnection } from "../../helper/index.js"; +import customLogger from "../../utils/custom-logger.utils.js"; +import { getLogMessage } from "../../utils/index.js"; +// Dynamic import for phpUnserialize will be used in the function + +interface FieldInfo { + field_name: string; + content_types: string; + field_type: string; + content_handler?: string; + target_type?: string; + handler_settings?: any; +} + +export interface TaxonomyFieldMapping { + [contentType: string]: { + [fieldName: string]: { + vocabulary?: string; + handler: string; + field_type: string; + }; + }; +} + +export interface ReferenceFieldMapping { + [contentType: string]: { + [fieldName: string]: { + target_type: string; + handler: string; + field_type: string; + }; + }; +} + +export interface AssetFieldMapping { + [contentType: string]: { + [fieldName: string]: { + field_type: string; + file_extensions?: string[]; + upload_location?: string; + max_filesize?: string; + }; + }; +} + +/** + * Execute SQL query with promise support + */ +const executeQuery = async (connection: mysql.Connection, query: string): Promise => { + return new Promise((resolve, reject) => { + connection.query(query, (error, results) => { + if (error) { + reject(error); + return; + } + resolve(results as any[]); + }); + }); +}; + +/** + * Analyze field configuration to identify taxonomy and reference fields + * Based on the original query.js logic that checks content_handler + */ +export const analyzeFieldTypes = async ( + dbConfig: any, + destination_stack_id: string, + projectId: string +): Promise<{ taxonomyFields: TaxonomyFieldMapping; referenceFields: ReferenceFieldMapping; assetFields: AssetFieldMapping }> => { + const srcFunc = 'analyzeFieldTypes'; + let connection: mysql.Connection | null = null; + + try { + const message = getLogMessage( + srcFunc, + `Analyzing field types to identify taxonomy fields...`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', message); + + // Create database connection + connection = await getDbConnection(dbConfig, projectId, destination_stack_id); + + // Query to get field configurations (same as original ct_mapped query) + const fieldConfigQuery = ` + SELECT *, CONVERT(data USING utf8) as data + FROM config + WHERE name LIKE '%field.field.node%' + `; + + const fieldConfigs = await executeQuery(connection, fieldConfigQuery); + + const taxonomyFieldMapping: TaxonomyFieldMapping = {}; + const referenceFieldMapping: ReferenceFieldMapping = {}; + const assetFieldMapping: AssetFieldMapping = {}; + let taxonomyFieldCount = 0; + let referenceFieldCount = 0; + let assetFieldCount = 0; + let totalFieldCount = 0; + + for (const fieldConfig of fieldConfigs) { + try { + // Unserialize the PHP data to get field details + const { unserialize } = await import('php-serialize'); + const fieldData = unserialize(fieldConfig.data); + + if (fieldData && fieldData.field_name && fieldData.bundle) { + totalFieldCount++; + + const fieldInfo: FieldInfo = { + field_name: fieldData.field_name, + content_types: fieldData.bundle, + field_type: fieldData.field_type || 'unknown', + content_handler: fieldData?.settings?.handler, + target_type: fieldData?.settings?.target_type, + handler_settings: fieldData?.settings?.handler_settings + }; + + // Initialize content type mappings if not exists + if (!taxonomyFieldMapping[fieldInfo.content_types]) { + taxonomyFieldMapping[fieldInfo.content_types] = {}; + } + if (!referenceFieldMapping[fieldInfo.content_types]) { + referenceFieldMapping[fieldInfo.content_types] = {}; + } + if (!assetFieldMapping[fieldInfo.content_types]) { + assetFieldMapping[fieldInfo.content_types] = {}; + } + + // Check if this is a taxonomy reference field + const isTaxonomyField = + // Check handler for taxonomy references + (fieldInfo.content_handler && fieldInfo.content_handler.includes('taxonomy_term')) || + // Check target_type for entity references to taxonomy terms + (fieldInfo.target_type === 'taxonomy_term') || + // Check field type for direct taxonomy reference fields + (fieldInfo.field_type === 'entity_reference' && fieldInfo.target_type === 'taxonomy_term') || + (fieldInfo.field_type === 'taxonomy_term_reference') || + // Check handler settings for vocabulary restrictions (taxonomy specific) + (fieldInfo.handler_settings?.target_bundles && + Object.keys(fieldInfo.handler_settings.target_bundles).some(bundle => + fieldInfo.target_type === 'taxonomy_term')); + + // Check if this is a node reference field (non-taxonomy entity reference) + const isReferenceField = + // Check for entity_reference field type + (fieldInfo.field_type === 'entity_reference') && + // Check handler for node references + (fieldInfo.content_handler && fieldInfo.content_handler.includes('node')) || + // Check target_type for entity references to nodes + (fieldInfo.target_type === 'node') && + // Make sure it's NOT a taxonomy field + !isTaxonomyField; + + if (isTaxonomyField) { + taxonomyFieldCount++; + + // Try to determine the vocabulary from handler settings + let vocabulary = 'unknown'; + if (fieldInfo.handler_settings?.target_bundles) { + const vocabularies = Object.keys(fieldInfo.handler_settings.target_bundles); + vocabulary = vocabularies.length === 1 ? vocabularies[0] : vocabularies.join(','); + } + + taxonomyFieldMapping[fieldInfo.content_types][fieldInfo.field_name] = { + vocabulary, + handler: fieldInfo.content_handler || 'default:taxonomy_term', + field_type: fieldInfo.field_type + }; + + const taxonomyMessage = getLogMessage( + srcFunc, + `Found taxonomy field: ${fieldInfo.content_types}.${fieldInfo.field_name} โ†’ vocabulary: ${vocabulary}`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', taxonomyMessage); + } else if (isReferenceField) { + referenceFieldCount++; + + referenceFieldMapping[fieldInfo.content_types][fieldInfo.field_name] = { + target_type: fieldInfo.target_type || 'node', + handler: fieldInfo.content_handler || 'default:node', + field_type: fieldInfo.field_type + }; + + const referenceMessage = getLogMessage( + srcFunc, + `Found reference field: ${fieldInfo.content_types}.${fieldInfo.field_name} โ†’ target_type: ${fieldInfo.target_type || 'node'}`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', referenceMessage); + } + + // Check if this is an asset/file field + const isAssetField = + // Check for file field type + (fieldInfo.field_type === 'file') || + // Check for image field type + (fieldInfo.field_type === 'image') || + // Check for managed_file field type + (fieldInfo.field_type === 'managed_file') || + // Check for entity_reference to file entities + (fieldInfo.field_type === 'entity_reference' && fieldInfo.target_type === 'file'); + + if (isAssetField) { + assetFieldCount++; + + // Extract file-related settings + const fileExtensions = fieldData?.settings?.file_extensions ? + fieldData.settings.file_extensions.split(' ') : []; + const uploadLocation = fieldData?.settings?.file_directory || + fieldData?.settings?.uri_scheme || 'public://'; + const maxFilesize = fieldData?.settings?.max_filesize || + fieldData?.settings?.file_size || ''; + + assetFieldMapping[fieldInfo.content_types][fieldInfo.field_name] = { + field_type: fieldInfo.field_type, + file_extensions: fileExtensions, + upload_location: uploadLocation, + max_filesize: maxFilesize + }; + + const assetMessage = getLogMessage( + srcFunc, + `Found asset field: ${fieldInfo.content_types}.${fieldInfo.field_name} โ†’ type: ${fieldInfo.field_type}, extensions: [${fileExtensions.join(', ')}]`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', assetMessage); + } + } + + } catch (parseError: any) { + // Log parsing error but continue with other fields + const parseMessage = getLogMessage( + srcFunc, + `Could not parse field config: ${parseError.message}`, + {}, + parseError + ); + await customLogger(projectId, destination_stack_id, 'warn', parseMessage); + } + } + + const summaryMessage = getLogMessage( + srcFunc, + `Field analysis complete: ${taxonomyFieldCount} taxonomy fields, ${referenceFieldCount} reference fields, and ${assetFieldCount} asset fields found out of ${totalFieldCount} total fields.`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', summaryMessage); + + return { + taxonomyFields: taxonomyFieldMapping, + referenceFields: referenceFieldMapping, + assetFields: assetFieldMapping + }; + + } catch (error: any) { + const message = getLogMessage( + srcFunc, + `Error analyzing field types: ${error.message}`, + {}, + error + ); + await customLogger(projectId, destination_stack_id, 'error', message); + throw error; + } finally { + if (connection) { + connection.end(); + } + } +}; + +/** + * Check if a specific field is a taxonomy field + */ +export const isTaxonomyField = ( + fieldName: string, + contentType: string, + taxonomyMapping: TaxonomyFieldMapping +): boolean => { + return !!(taxonomyMapping[contentType] && taxonomyMapping[contentType][fieldName]); +}; + +/** + * Check if a specific field is a reference field + */ +export const isReferenceField = ( + fieldName: string, + contentType: string, + referenceMapping: ReferenceFieldMapping +): boolean => { + return !!(referenceMapping[contentType] && referenceMapping[contentType][fieldName]); +}; + +/** + * Check if a specific field is an asset field + */ +export const isAssetField = ( + fieldName: string, + contentType: string, + assetMapping: AssetFieldMapping +): boolean => { + return !!(assetMapping[contentType] && assetMapping[contentType][fieldName]); +}; + +/** + * Get taxonomy field information + */ +export const getTaxonomyFieldInfo = ( + fieldName: string, + contentType: string, + taxonomyMapping: TaxonomyFieldMapping +) => { + return taxonomyMapping[contentType]?.[fieldName] || null; +}; + +/** + * Get reference field information + */ +export const getReferenceFieldInfo = ( + fieldName: string, + contentType: string, + referenceMapping: ReferenceFieldMapping +) => { + return referenceMapping[contentType]?.[fieldName] || null; +}; + +/** + * Get asset field information + */ +export const getAssetFieldInfo = ( + fieldName: string, + contentType: string, + assetMapping: AssetFieldMapping +) => { + return assetMapping[contentType]?.[fieldName] || null; +}; + +/** + * Transform taxonomy field value to Contentstack format + * Converts tid to taxonomy term uid based on our taxonomy data + * + * The taxonomyData should contain individual vocabulary files: + * - taxonomies/list.json + * - taxonomies/news.json + * etc. + */ +export const transformTaxonomyValue = async ( + value: any, + fieldName: string, + contentType: string, + taxonomyMapping: TaxonomyFieldMapping, + taxonomyBasePath: string +): Promise => { + const fieldInfo = getTaxonomyFieldInfo(fieldName, contentType, taxonomyMapping); + + if (!fieldInfo || !value) { + return value; + } + + // If it's a taxonomy field with tid value, try to find the corresponding term + if (typeof value === 'number' || (typeof value === 'string' && /^\d+$/.test(value))) { + const tid = parseInt(value.toString()); + + try { + // Try to determine which vocabulary to look in based on field info + const vocabularies = fieldInfo.vocabulary ? fieldInfo.vocabulary.split(',') : ['unknown']; + + for (const vocabulary of vocabularies) { + try { + const fs = await import('fs'); + const path = await import('path'); + + const taxonomyFilePath = path.join(taxonomyBasePath, `${vocabulary}.json`); + + if (fs.existsSync(taxonomyFilePath)) { + const taxonomyContent = JSON.parse(fs.readFileSync(taxonomyFilePath, 'utf8')); + + if (taxonomyContent.terms && Array.isArray(taxonomyContent.terms)) { + for (const term of taxonomyContent.terms) { + if (term.drupal_term_id === tid) { + return term.uid; + } + } + } + } + } catch (vocabError) { + // Continue to next vocabulary if this one fails + continue; + } + } + + // If we couldn't find it in specific vocabularies, try all taxonomy files + const fs = await import('fs'); + const path = await import('path'); + + if (fs.existsSync(taxonomyBasePath)) { + const taxonomyFiles = fs.readdirSync(taxonomyBasePath) + .filter(file => file.endsWith('.json') && file !== 'taxonomies.json'); + + for (const file of taxonomyFiles) { + try { + const taxonomyContent = JSON.parse(fs.readFileSync(path.join(taxonomyBasePath, file), 'utf8')); + + if (taxonomyContent.terms && Array.isArray(taxonomyContent.terms)) { + for (const term of taxonomyContent.terms) { + if (term.drupal_term_id === tid) { + return term.uid; + } + } + } + } catch (fileError) { + // Continue to next file if this one fails + continue; + } + } + } + + } catch (error) { + // Return original value if transformation fails + return value; + } + } + + return value; +}; diff --git a/api/src/services/drupal/field-fetcher.service.ts b/api/src/services/drupal/field-fetcher.service.ts new file mode 100644 index 000000000..bb2c6ab6c --- /dev/null +++ b/api/src/services/drupal/field-fetcher.service.ts @@ -0,0 +1,228 @@ +import mysql from 'mysql2'; +import { getLogMessage } from '../../utils/index.js'; +import customLogger from '../../utils/custom-logger.utils.js'; + +/** + * Field Fetcher Service for Content Types with Many Fields + * Handles field data fetching for content types that exceed MySQL JOIN limits + */ + +interface DrupalFieldData { + field_name: string; + content_types: string; + type: string; + content_handler?: string; +} + +interface FieldDataResult { + [nid: number]: { + [fieldName: string]: any; + }; +} + +export class FieldFetcherService { + private connection: mysql.Connection; + private projectId: string; + private destinationStackId: string; + + constructor(connection: mysql.Connection, projectId: string, destinationStackId: string) { + this.connection = connection; + this.projectId = projectId; + this.destinationStackId = destinationStackId; + } + + /** + * Fetch field data for content types with many fields using individual queries + * This avoids the MySQL 61-table JOIN limit + */ + async fetchFieldDataForContentType( + contentType: string, + nodeIds: number[], + fieldsForType: DrupalFieldData[] + ): Promise { + const srcFunc = 'fetchFieldDataForContentType'; + const fieldData: FieldDataResult = {}; + + if (nodeIds.length === 0) { + return fieldData; + } + + // Initialize field data structure + nodeIds.forEach(nid => { + fieldData[nid] = {}; + }); + + const message = getLogMessage( + srcFunc, + `Fetching field data for ${contentType}: ${fieldsForType.length} fields, ${nodeIds.length} nodes`, + {} + ); + await customLogger(this.projectId, this.destinationStackId, 'info', message); + + // Process fields in batches to avoid overwhelming the database + const batchSize = 10; + for (let i = 0; i < fieldsForType.length; i += batchSize) { + const fieldBatch = fieldsForType.slice(i, i + batchSize); + + await Promise.all( + fieldBatch.map(field => this.fetchSingleFieldData(field, nodeIds, fieldData)) + ); + } + + const successMessage = getLogMessage( + srcFunc, + `Successfully fetched field data for ${contentType}: ${Object.keys(fieldData).length} nodes processed`, + {} + ); + await customLogger(this.projectId, this.destinationStackId, 'info', successMessage); + + return fieldData; + } + + /** + * Fetch data for a single field across multiple nodes + */ + private async fetchSingleFieldData( + field: DrupalFieldData, + nodeIds: number[], + fieldData: FieldDataResult + ): Promise { + const fieldTableName = `node__${field.field_name}`; + + try { + // Check if field table exists + const tableExistsQuery = ` + SELECT 1 FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = ? + LIMIT 1 + `; + + const [tableExists] = await this.connection.promise().query(tableExistsQuery, [fieldTableName]) as any[]; + + if (tableExists.length === 0) { + console.warn(`Field table ${fieldTableName} does not exist`); + return; + } + + // Get field columns dynamically + const columnQuery = ` + SELECT COLUMN_NAME + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = ? + AND COLUMN_NAME LIKE ? + `; + + const [columns] = await this.connection.promise().query(columnQuery, [fieldTableName, `${field.field_name}_%`]) as any[]; + + if (columns.length === 0) { + console.warn(`No columns found for field ${field.field_name}`); + return; + } + + // Build field query with all relevant columns + const fieldColumns = columns.map((col: any) => col.COLUMN_NAME); + const selectColumns = fieldColumns.join(', '); + + const fieldQuery = ` + SELECT + entity_id, + ${selectColumns} + FROM ${fieldTableName} + WHERE entity_id IN (${nodeIds.map(() => '?').join(',')}) + `; + + const [fieldResults] = await this.connection.promise().query(fieldQuery, nodeIds) as any[]; + + // Merge field results into main data structure + fieldResults.forEach((row: any) => { + const nid = row.entity_id; + if (fieldData[nid]) { + // Add all field columns to the node data + fieldColumns.forEach((columnName: string) => { + if (row[columnName] !== null && row[columnName] !== undefined) { + fieldData[nid][columnName] = row[columnName]; + } + }); + } + }); + + } catch (error: any) { + console.warn(`Error fetching data for field ${field.field_name}:`, error.message); + + const errorMessage = getLogMessage( + 'fetchSingleFieldData', + `Failed to fetch data for field ${field.field_name}: ${error.message}`, + {}, + error + ); + await customLogger(this.projectId, this.destinationStackId, 'warn', errorMessage); + } + } + + /** + * Merge base node data with field data + */ + mergeNodeAndFieldData( + baseNodes: any[], + fieldData: FieldDataResult + ): any[] { + return baseNodes.map(node => { + const nid = node.nid; + const nodeFieldData = fieldData[nid] || {}; + + return { + ...node, + ...nodeFieldData + }; + }); + } + + /** + * Get field configuration for a content type + */ + async getFieldsForContentType(contentType: string): Promise { + const configQuery = ` + SELECT *, CONVERT(data USING utf8) as data + FROM config + WHERE name LIKE '%field.field.node%' + `; + + try { + const [rows] = await this.connection.promise().query(configQuery) as any[]; + const fields: DrupalFieldData[] = []; + + for (const row of rows) { + try { + const { unserialize } = await import('php-serialize'); + const configData = unserialize(row.data); + + if (configData && configData.bundle === contentType) { + fields.push({ + field_name: configData.field_name, + content_types: configData.bundle, + type: configData.field_type, + content_handler: configData?.settings?.handler + }); + } + } catch (parseError) { + console.warn(`Failed to parse field config for ${row.name}:`, parseError); + } + } + + return fields; + } catch (error: any) { + const errorMessage = getLogMessage( + 'getFieldsForContentType', + `Failed to get fields for content type ${contentType}: ${error.message}`, + {}, + error + ); + await customLogger(this.projectId, this.destinationStackId, 'error', errorMessage); + throw error; + } + } +} + +export default FieldFetcherService; diff --git a/api/src/services/drupal/locales.service.ts b/api/src/services/drupal/locales.service.ts new file mode 100644 index 000000000..0bf30c667 --- /dev/null +++ b/api/src/services/drupal/locales.service.ts @@ -0,0 +1,271 @@ +import fs from "fs"; +import path from "path"; +import axios from "axios"; +import { LOCALE_MAPPER, MIGRATION_DATA_CONFIG } from "../../constants/index.js"; +import { Locale } from "../../models/types.js"; +import { getAllLocales, getLogMessage } from "../../utils/index.js"; +import customLogger from "../../utils/custom-logger.utils.js"; +import { createDbConnection } from "../../helper/index.js"; + +const { + DATA: MIGRATION_DATA_PATH, + LOCALE_DIR_NAME, + LOCALE_FILE_NAME, + LOCALE_MASTER_LOCALE, + LOCALE_CF_LANGUAGE, +} = MIGRATION_DATA_CONFIG; + +/** + * Writes data to a specified file, ensuring the target directory exists. + */ +async function writeFile(dirPath: string, filename: string, data: any) { + try { + await fs.promises.mkdir(dirPath, { recursive: true }); + const filePath = path.join(dirPath, filename); + await fs.promises.writeFile(filePath, JSON.stringify(data), 'utf8'); + } catch (err) { + console.error(`Error writing ${dirPath}/${filename}::`, err); + } +} + +/** + * Helper function to get key by value from an object + */ +function getKeyByValue(obj: Record, targetValue: string): string | undefined { + return Object.entries(obj).find(([_, value]) => value === targetValue)?.[0]; +} + +/** + * Fetches locale names from Contentstack API + */ +async function fetchContentstackLocales(): Promise> { + try { + const response = await axios.get('https://app.contentstack.com/api/v3/locales?include_all=true'); + return response.data?.locales || {}; + } catch (error) { + console.error('Error fetching Contentstack locales:', error); + return {}; + } +} + +/** + * Applies special locale code transformations based on business rules + */ +function applyLocaleTransformations(locales: string[], masterLocale: string): { code: string; name: string; isMaster: boolean }[] { + const hasUnd = locales.includes('und'); + const hasEn = locales.includes('en'); + const hasEnUs = locales.includes('en-us'); + + return locales.map(locale => { + let code = locale.toLowerCase(); + let name = ''; + let isMaster = locale === masterLocale; + + // Apply transformation rules + if (locale === 'und') { + if (hasEn && hasEnUs) { + // If all three present, "und" stays as "und" + code = 'und'; + name = 'Language Neutral'; + } else if (hasEnUs) { + // If "und" + "en-us", "und" becomes "en" + code = 'en'; + name = 'English'; + } else { + // If only "und", becomes "en-us" + code = 'en-us'; + name = 'English - United States'; + } + } else if (locale === 'en-us') { + if (hasUnd && !hasEn) { + // If "und" + "en-us" (no en), "und" becomes "en", so keep "en-us" + code = 'en-us'; + name = 'English - United States'; + } else { + // Keep en-us as is in other cases + code = 'en-us'; + name = 'English - United States'; + } + } else if (locale === 'en') { + if (hasEnUs && !hasUnd) { + // If "en" + "en-us" (no und), "en" becomes "und" + code = 'und'; + name = 'Language Neutral'; + } else { + // Keep "en" as is in other cases + code = 'en'; + name = 'English'; + } + } + + return { code, name, isMaster }; + }); +} + +/** + * Processes and creates locale configurations from Drupal database for migration to Contentstack. + * + * This function: + * 1. Fetches master locale from Drupal system.site config + * 2. Fetches all locales from node_field_data + * 3. Fetches non-master locales separately + * 4. Gets locale names from Contentstack API + * 5. Applies special transformation rules for "und", "en", "en-us" + * 6. Creates 3 JSON files: master-locale.json, locales.json, language.json + */ +export const createLocale = async ( + dbConfig: any, + destination_stack_id: string, + projectId: string, + project: any +) => { + const srcFunc = 'createLocale'; + const localeSave = path.join(MIGRATION_DATA_PATH, destination_stack_id, LOCALE_DIR_NAME); + + try { + const msLocale: Record = {}; + const allLocales: Record = {}; + const localeList: Record = {}; + + // Create database connection using dbConfig + console.log('๐Ÿ” Database config for locales:', { + host: dbConfig?.host, + user: dbConfig?.user, + database: dbConfig?.database, + port: dbConfig?.port, + hasPassword: !!dbConfig?.password + }); + + if (!dbConfig || !dbConfig.host || !dbConfig.user || !dbConfig.database) { + throw new Error('Invalid database configuration provided to createLocale'); + } + + const connection = await createDbConnection(dbConfig); + + if (!connection) { + throw new Error('Failed to create database connection'); + } + + // Helper function to execute queries (same pattern as entries.service.ts) + const executeQuery = (query: string): Promise => { + return new Promise((resolve, reject) => { + connection.query(query, (error, results) => { + if (error) { + reject(error); + } else { + resolve(results as any[]); + } + }); + }); + }; + + // 1. Get master locale from Drupal system.site config + const masterLocaleQuery = ` + SELECT SUBSTRING_INDEX( + SUBSTRING_INDEX(CONVERT(data USING utf8), 'default_langcode";s:2:"', -1), + '"', 1 + ) as master_locale + FROM config + WHERE name = 'system.site' + `; + + const masterRows: any = await executeQuery(masterLocaleQuery); + const masterLocaleCode = masterRows[0]?.master_locale || 'en'; + + // 2. Get all locales from node_field_data + const allLocalesQuery = ` + SELECT DISTINCT langcode + FROM node_field_data + WHERE langcode IS NOT NULL AND langcode != '' + ORDER BY langcode + `; + + const allLocaleRows: any = await executeQuery(allLocalesQuery); + const allLocaleCodes = allLocaleRows.map((row: any) => row.langcode); + + // 3. Get non-master locales + const nonMasterLocalesQuery = ` + SELECT DISTINCT n.langcode + FROM node_field_data n + WHERE n.langcode IS NOT NULL + AND n.langcode != '' + AND n.langcode != ( + SELECT + SUBSTRING_INDEX( + SUBSTRING_INDEX(CONVERT(data USING utf8), 'default_langcode";s:2:"', -1), + '"', + 1 + ) + FROM config + WHERE name = 'system.site' + LIMIT 1 + ) + ORDER BY n.langcode + `; + + const nonMasterRows: any = await executeQuery(nonMasterLocalesQuery); + const nonMasterLocaleCodes = nonMasterRows.map((row: any) => row.langcode); + + // Close database connection + connection.end(); + + // 4. Fetch locale names from Contentstack API + const contentstackLocales = await fetchContentstackLocales(); + + // 5. Apply special transformation rules + const transformedLocales = applyLocaleTransformations(allLocaleCodes, masterLocaleCode); + + // 6. Process each locale + transformedLocales.forEach((localeInfo, index) => { + const { code, name, isMaster } = localeInfo; + + // Create UID using original langcode from database + const originalLangcode = allLocaleCodes[index]; // Get original langcode from database + const uid = `drupallocale_${originalLangcode.toLowerCase().replace(/-/g, '_')}`; + + // Get name from Contentstack API or use transformed name + const localeName = name || contentstackLocales[code] || contentstackLocales[code.toLowerCase()] || 'Unknown Language'; + + const newLocale: Locale = { + code: code.toLowerCase(), + name: localeName, + fallback_locale: isMaster ? null : masterLocaleCode.toLowerCase(), + uid: uid, + }; + + // Add to appropriate collections using UID as key + if (isMaster) { + msLocale[uid] = newLocale; + } else { + allLocales[uid] = newLocale; + } + + localeList[uid] = newLocale; + }); + + // Handle case where no non-master locales exist + const finalAllLocales = Object.keys(allLocales).length > 0 ? allLocales : {}; + + // 7. Write locale files (same structure as Contentful) + await writeFile(localeSave, LOCALE_FILE_NAME, finalAllLocales); // locales.json (non-master only) + await writeFile(localeSave, LOCALE_MASTER_LOCALE, msLocale); // master-locale.json (master only) + await writeFile(localeSave, LOCALE_CF_LANGUAGE, localeList); // language.json (all locales) + + const message = getLogMessage( + srcFunc, + `Drupal locales have been successfully transformed. Master: ${masterLocaleCode}, Total: ${allLocaleCodes.length}, Non-master: ${nonMasterLocaleCodes.length}`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', message); + + } catch (err) { + const message = getLogMessage( + srcFunc, + `Error while creating Drupal locales.`, + {}, + err + ); + await customLogger(projectId, destination_stack_id, 'error', message); + throw err; + } +}; diff --git a/api/src/services/drupal/query.service.ts b/api/src/services/drupal/query.service.ts new file mode 100644 index 000000000..2a8db407b --- /dev/null +++ b/api/src/services/drupal/query.service.ts @@ -0,0 +1,369 @@ +import fs from 'fs'; +import path from 'path'; +import mysql from 'mysql2'; +import { getDbConnection } from '../../helper/index.js'; +import customLogger from '../../utils/custom-logger.utils.js'; +import { getLogMessage } from '../../utils/index.js'; +import { MIGRATION_DATA_CONFIG } from '../../constants/index.js'; +import OptimizedQueryBuilder from '../../utils/optimized-query-builder.utils.js'; + +const { DATA } = MIGRATION_DATA_CONFIG; + +// PHP unserialize functionality (simplified for Node.js) +// Dynamic import for phpUnserialize will be used in the function + +/** + * Interface for field data extracted from Drupal config + */ +interface DrupalFieldData { + field_name: string; + content_types: string; + type: string; + content_handler?: string; +} + +/** + * Interface for query configuration + */ +interface QueryConfig { + page: { [contentType: string]: string }; + count: { [contentType: string]: string }; +} + +/** + * Get field information by querying the database for a specific field + * Enhanced to handle link fields with both URI and TITLE columns + */ +const getQuery = (connection: mysql.Connection, data: DrupalFieldData): Promise => { + return new Promise((resolve, reject) => { + try { + const tableName = `node__${data.field_name}`; + + // Check if this is a link field first + if (data.type !== 'link') { + // For non-link fields, use existing logic + const value = data.field_name; + const handlerType = data.content_handler === undefined ? 'invalid' : data.content_handler; + const query = `SELECT *, '${handlerType}' as handler, '${data.type}' as fieldType FROM ${tableName}`; + + connection.query(query, (error: any, rows: any, fields: any) => { + if (!error && fields) { + // Look for field patterns in the database columns + for (const field of fields) { + const fieldName = field.name; + + // Check for various Drupal field suffixes + if (fieldName === `${value}_value` || + fieldName === `${value}_fid` || + fieldName === `${value}_tid` || + fieldName === `${value}_status` || + fieldName === `${value}_target_id` || + fieldName === `${value}_uri`) { + const fieldTable = `node__${data.field_name}.${fieldName}`; + resolve(fieldTable); + return; + } + } + // If no matching field was found + resolve(''); + } else { + console.error(`Error executing query for field ${value}:`, error); + resolve(''); // Resolve with empty string on error to continue process + } + }); + return; + } + + // For LINK fields only - get both URI and TITLE columns + connection.query(`SHOW COLUMNS FROM ${tableName}`, (error: any, columns: any) => { + if (error) { + console.error(`Error querying columns for link field ${data.field_name}:`, error); + resolve(''); + return; + } + + // Filter for link-specific columns only + const linkColumns = columns + .map((col: any) => col.Field) + .filter((field: string) => + (field === `${data.field_name}_uri` || field === `${data.field_name}_title`) && + field.startsWith(data.field_name) + ); + + if (linkColumns.length > 0) { + // Return both columns as MAX aggregations for link fields + const maxColumns = linkColumns.map((col: string) => `MAX(${tableName}.${col}) as ${col}`); + resolve(maxColumns.join(',')); + } else { + // Fallback to just URI if title doesn't exist + const uriColumn = `${data.field_name}_uri`; + resolve(`MAX(${tableName}.${uriColumn}) as ${uriColumn}`); + } + }); + } catch (error) { + console.error('Error in getQuery', error); + resolve(''); // Resolve with empty string on error to continue process + } + }); +}; + +/** + * Process field data and generate SQL queries for each content type + */ +const generateQueriesForFields = async ( + connection: mysql.Connection, + fieldData: DrupalFieldData[], + projectId: string, + destination_stack_id: string +): Promise => { + const srcFunc = 'generateQueriesForFields'; + + try { + const select: { [contentType: string]: string } = {}; + const countQuery: { [contentType: string]: string } = {}; + + // Group fields by content type + const contentTypes = [...new Set(fieldData.map(field => field.content_types))]; + + const message = `Processing ${contentTypes.length} content types for query generation...`; + await customLogger(projectId, destination_stack_id, 'info', message); + + // Process each content type + for (const contentType of contentTypes) { + const fieldsForType = fieldData.filter(field => field.content_types === contentType); + const fieldCount = fieldsForType.length; + const maxJoinLimit = 50; // Conservative limit to avoid MySQL's 61-table limit + + // Check if content type has too many fields for single query + if (fieldCount > maxJoinLimit) { + const warningMessage = `Content type '${contentType}' has ${fieldCount} fields (>${maxJoinLimit} limit). Using optimized base query only.`; + await customLogger(projectId, destination_stack_id, 'warn', warningMessage); + + // Generate simple base query without field JOINs to avoid MySQL limit + const baseQuery = ` + SELECT + node.nid, + node.title, + node.langcode, + node.type, + users.name as author_name + FROM node_field_data node + LEFT JOIN users ON users.uid = node.uid + WHERE node.type = '${contentType}' + GROUP BY node.nid + `.replace(/\s+/g, ' ').trim(); + + const baseCountQuery = ` + SELECT COUNT(DISTINCT node.nid) as countentry + FROM node_field_data node + WHERE node.type = '${contentType}' + `.replace(/\s+/g, ' ').trim(); + + select[contentType] = baseQuery + ` /* OPTIMIZED_NO_JOINS:${fieldCount} */`; + countQuery[`${contentType}Count`] = baseCountQuery; + + const optimizedMessage = `Generated optimized base query for ${contentType} (avoiding ${fieldCount} JOINs)`; + await customLogger(projectId, destination_stack_id, 'info', optimizedMessage); + + continue; // Skip to next content type + } + + const tableJoins: string[] = []; + const queries: Promise[] = []; + + // Collect all field queries (only for content types with manageable field count) + fieldsForType.forEach(fieldData => { + tableJoins.push(`node__${fieldData.field_name}`); + queries.push(getQuery(connection, fieldData)); + }); + + try { + // Wait for all field queries to complete + const results = await Promise.all(queries); + + // Filter out empty results + const validResults = results.filter(item => item); + + if (validResults.length === 0) { + console.log(`No valid fields found for content type ${contentType}`); + continue; + } + + // Build the SELECT clause with proper handling for link fields + const modifiedResults = validResults.map(item => { + // Check if this is already a MAX aggregation (link fields) + if (item.includes('MAX(') && item.includes(' as ')) { + return item; // Link fields are already properly formatted + } + // For other fields, apply MAX aggregation + return `MAX(${item}) as ${item.split('.').pop()}`; + }); + + // Build LEFT JOIN clauses + const leftJoins = tableJoins.map( + table => `LEFT JOIN ${table} ON ${table}.entity_id = node.nid` + ); + leftJoins.push('LEFT JOIN users ON users.uid = node.uid'); + + // Construct the complete query + const selectClause = [ + 'SELECT node.nid, MAX(node.title) AS title, MAX(node.langcode) AS langcode, MAX(node.created) as created, MAX(node.type) as type', + ...modifiedResults + ].join(','); + + const fromClause = 'FROM node_field_data node'; + const joinClause = leftJoins.join(' '); + const whereClause = `WHERE node.type = '${contentType}'`; + const groupClause = 'GROUP BY node.nid'; + + // Final query construction + const finalQuery = `${selectClause} ${fromClause} ${joinClause} ${whereClause} ${groupClause}`; + + // Clean up any double commas + select[contentType] = finalQuery.replace(/,,/g, ',').replace(/, ,/g, ','); + + // Build count query + const countQueryStr = `SELECT count(distinct(node.nid)) as countentry ${fromClause} ${joinClause} ${whereClause}`; + countQuery[`${contentType}Count`] = countQueryStr; + + const fieldMessage = `Generated queries for content type: ${contentType} with ${validResults.length} fields`; + await customLogger(projectId, destination_stack_id, 'info', fieldMessage); + + } catch (error) { + const errorMessage = `Error processing queries for content type: ${contentType}`; + await customLogger(projectId, destination_stack_id, 'error', errorMessage); + console.error('Error processing queries for content type:', contentType, error); + } + } + + return { + page: select, + count: countQuery + }; + + } catch (error: any) { + const errorMessage = `Error in generateQueriesForFields: ${error.message}`; + await customLogger(projectId, destination_stack_id, 'error', errorMessage); + throw error; + } +}; + +/** + * Extract field configuration from Drupal database and generate dynamic queries + * Based on upload-api/migration-drupal/libs/extractQueries.js + */ +/** + * Validates that query configuration file exists (legacy compatibility) + * + * NOTE: This function is for backward compatibility. + * The new dynamic query system uses createQuery() which generates queries + * based on actual database field analysis. + */ +export const createQueryConfig = async ( + destination_stack_id: string, + customQueries?: any +): Promise => { + const queryDir = path.join(DATA, destination_stack_id, 'query'); + const queryPath = path.join(queryDir, 'index.json'); + + try { + // Check if dynamic query file exists (should be created by createQuery service) + await fs.promises.access(queryPath); + console.log(`โœ… Dynamic query configuration found at: ${queryPath}`); + } catch (error) { + // If no dynamic queries exist, this is an error since we removed hardcoded fallbacks + throw new Error(`โŒ No query configuration found at ${queryPath}. Dynamic queries must be generated first using createQuery() service.`); + } +}; + +export const createQuery = async ( + dbConfig: any, + destination_stack_id: string, + projectId: string +): Promise => { + const srcFunc = 'createQuery'; + let connection: mysql.Connection | null = null; + + try { + console.info('๐Ÿ” === DRUPAL QUERY SERVICE CONFIG ==='); + console.info('๐Ÿ“‹ Database Config:', JSON.stringify(dbConfig, null, 2)); + console.info('๐Ÿ“‹ Destination Stack ID:', destination_stack_id); + console.info('๐Ÿ“‹ Project ID:', projectId); + console.info('๐Ÿ“‹ Function:', srcFunc); + console.info('======================================'); + + const queryDir = path.join(DATA, destination_stack_id, 'query'); + const queryPath = path.join(queryDir, 'index.json'); + + // Create query directory + await fs.promises.mkdir(queryDir, { recursive: true }); + + const message = `Generating dynamic queries from Drupal database...`; + await customLogger(projectId, destination_stack_id, 'info', message); + + // Create database connection + connection = await getDbConnection(dbConfig, projectId, destination_stack_id); + + // SQL query to extract field configuration from Drupal + const configQuery = "SELECT *, CONVERT(data USING utf8) as data FROM config WHERE name LIKE '%field.field.node%'"; + + // Execute query using promise-based approach + const [rows] = await connection.promise().query(configQuery) as any[]; + + let fieldData: DrupalFieldData[] = []; + + // Process results and extract field information + for (let i = 0; i < rows.length; i++) { + try { + const { unserialize } = await import('php-serialize'); + const convDetails = unserialize(rows[i].data); + if (convDetails && typeof convDetails === 'object' && 'field_name' in convDetails) { + fieldData.push({ + field_name: convDetails.field_name, + content_types: convDetails.bundle, + type: convDetails.field_type, + content_handler: convDetails?.settings?.handler + }); + } + } catch (err: any) { + console.warn(`Couldn't parse row ${i}:`, err.message); + } + } + + if (fieldData.length === 0) { + throw new Error('No field configuration found in Drupal database'); + } + + const fieldMessage = `Found ${fieldData.length} field configurations in database`; + await customLogger(projectId, destination_stack_id, 'info', fieldMessage); + + // Generate queries based on field data + const queryConfig = await generateQueriesForFields(connection, fieldData, projectId, destination_stack_id); + + // Write query configuration to file + await fs.promises.writeFile(queryPath, JSON.stringify(queryConfig, null, 4), 'utf8'); + + const successMessage = `Successfully generated and saved dynamic queries to: ${queryPath}`; + await customLogger(projectId, destination_stack_id, 'info', successMessage); + + console.info(`โœ… Dynamic query configuration created at: ${queryPath}`); + console.info(`๐Ÿ“Š Generated queries for ${Object.keys(queryConfig.page).length} content types`); + + } catch (error: any) { + const errorMessage = `Failed to generate dynamic queries: ${error.message}`; + await customLogger(projectId, destination_stack_id, 'error', errorMessage); + + console.error('โŒ Error generating dynamic queries:', error); + throw new Error(`Failed to connect to database or generate queries: ${error.message}`); + } finally { + // Always close the connection when done + if (connection) { + try { + connection.end(); + console.info('Database connection closed'); + } catch (err: any) { + console.warn('Connection was already closed:', err.message); + } + } + } +}; diff --git a/api/src/services/drupal/references.service.ts b/api/src/services/drupal/references.service.ts new file mode 100644 index 000000000..9f70e16ff --- /dev/null +++ b/api/src/services/drupal/references.service.ts @@ -0,0 +1,421 @@ +import fs from "fs"; +import path from "path"; +import mysql from 'mysql2'; +import { MIGRATION_DATA_CONFIG } from "../../constants/index.js"; +import { getLogMessage } from "../../utils/index.js"; +import customLogger from "../../utils/custom-logger.utils.js"; +import { getDbConnection } from "../../helper/index.js"; + +const { + DATA, + REFERENCES_DIR_NAME, + REFERENCES_FILE_NAME, +} = MIGRATION_DATA_CONFIG; + +interface QueryConfig { + page: { + [contentType: string]: string; + }; + count: { + [contentTypeCount: string]: string; + }; +} + +interface DrupalEntry { + nid: number; + title: string; + langcode: string; + created: number; + type: string; + [key: string]: any; +} + +interface TaxonomyReference { + drupal_term_id: number; + taxonomy_uid: string; + term_uid: string; +} + +interface DrupalTaxonomyTerm { + taxonomy_uid: string; // vid (vocabulary id) + drupal_term_id: number; // term id + term_name: string; // term name + term_description: string | null; // term description +} + +const LIMIT = 100; // Pagination limit for references + +// NOTE: Hardcoded queries have been REMOVED. All queries are now generated dynamically +// by the query.service.ts based on actual database field analysis. + +/** + * Executes SQL query and returns results as Promise + */ +const executeQuery = (connection: mysql.Connection, query: string): Promise => { + return new Promise((resolve, reject) => { + connection.query(query, (error, results) => { + if (error) { + reject(error); + } else { + resolve(results as any[]); + } + }); + }); +}; + +/** + * Writes data to a specified file, ensuring the target directory exists. + */ +async function writeFile(dirPath: string, filename: string, data: any) { + try { + await fs.promises.mkdir(dirPath, { recursive: true }); + const filePath = path.join(dirPath, filename); + await fs.promises.writeFile(filePath, JSON.stringify(data, null, 4), 'utf8'); + } catch (err) { + console.error(`Error writing ${dirPath}/${filename}::`, err); + } +} + +/** + * Reads existing references file or returns empty object + */ +async function readReferencesFile(referencesPath: string): Promise { + try { + const data = await fs.promises.readFile(referencesPath, "utf8"); + return JSON.parse(data); + } catch (err) { + return {}; + } +} + +/** + * Processes entries for a specific content type and creates reference mappings + * Following the original putPosts logic from references.js + */ +const putPosts = async ( + entries: DrupalEntry[], + contentType: string, + referencesPath: string, + projectId: string, + destination_stack_id: string +): Promise => { + const srcFunc = 'putPosts'; + + try { + // Read existing references data + const referenceData = await readReferencesFile(referencesPath); + + // Process each entry and create reference mapping + entries.forEach((entry) => { + const referenceKey = `content_type_entries_title_${entry.nid}`; + referenceData[referenceKey] = { + uid: referenceKey, + _content_type_uid: contentType, + }; + }); + + // Write updated references back to file + await fs.promises.writeFile(referencesPath, JSON.stringify(referenceData, null, 4), 'utf8'); + + const message = getLogMessage( + srcFunc, + `Created ${entries.length} reference mappings for content type ${contentType}.`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', message); + + } catch (error: any) { + const message = getLogMessage( + srcFunc, + `Error creating references for ${contentType}: ${error.message}`, + {}, + error + ); + await customLogger(projectId, destination_stack_id, 'error', message); + throw error; + } +}; + +/** + * Processes entries for a specific content type with pagination + * Following the original getQuery logic from references.js + */ +const getQuery = async ( + connection: mysql.Connection, + contentType: string, + skip: number, + queryPageConfig: QueryConfig, + referencesPath: string, + projectId: string, + destination_stack_id: string +): Promise => { + const srcFunc = 'getQuery'; + + try { + // Following original pattern: queryPageConfig['page']['' + pagename + ''] + const baseQuery = queryPageConfig['page'][contentType]; + if (!baseQuery) { + throw new Error(`No query found for content type: ${contentType}`); + } + + const query = baseQuery + ` LIMIT ${skip}, ${LIMIT}`; + const entries = await executeQuery(connection, query); + + if (entries.length === 0) { + return false; // No more entries + } + + await putPosts(entries, contentType, referencesPath, projectId, destination_stack_id); + return true; // More entries might exist + + } catch (error: any) { + const message = getLogMessage( + srcFunc, + `Error querying references for ${contentType}: ${error.message}`, + {}, + error + ); + await customLogger(projectId, destination_stack_id, 'error', message); + throw error; + } +}; + +/** + * Processes all entries for a specific content type + * Following the original getPageCount logic from references.js + */ +const getPageCount = async ( + connection: mysql.Connection, + contentType: string, + queryPageConfig: QueryConfig, + referencesPath: string, + projectId: string, + destination_stack_id: string +): Promise => { + const srcFunc = 'getPageCount'; + + try { + // Process entries in batches + let skip = 0; + let hasMoreEntries = true; + + while (hasMoreEntries) { + hasMoreEntries = await getQuery( + connection, + contentType, + skip, + queryPageConfig, + referencesPath, + projectId, + destination_stack_id + ); + skip += LIMIT; + } + + const message = getLogMessage( + srcFunc, + `Completed reference extraction for content type ${contentType}.`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', message); + + } catch (error: any) { + const message = getLogMessage( + srcFunc, + `Error processing content type ${contentType}: ${error.message}`, + {}, + error + ); + await customLogger(projectId, destination_stack_id, 'error', message); + throw error; + } +}; + +/** + * Reads dynamic query configuration file generated by query.service.ts + * Following original pattern: helper.readFile(path.join(process.cwd(), config.data, 'query', 'index.json')) + * + * NOTE: No fallback to hardcoded queries - dynamic queries MUST be generated first + */ +async function readQueryConfig(destination_stack_id: string): Promise { + try { + const queryPath = path.join(DATA, destination_stack_id, 'query', 'index.json'); + const data = await fs.promises.readFile(queryPath, "utf8"); + return JSON.parse(data); + } catch (err) { + // No fallback - dynamic queries must be generated first by createQuery() service + throw new Error(`โŒ No dynamic query configuration found at query/index.json. Dynamic queries must be generated first using createQuery() service. Original error: ${err}`); + } +} + +/** + * Creates taxonomy reference mappings from Drupal database + * Using the taxonomy query to create a flat mapping file: taxonomyReference.json + */ +const createTaxonomyReferences = async ( + connection: mysql.Connection, + referencesSave: string, + projectId: string, + destination_stack_id: string +): Promise => { + const srcFunc = 'createTaxonomyReferences'; + + try { + const message = getLogMessage( + srcFunc, + `Creating taxonomy reference mappings...`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', message); + + // Use the same SQL query as taxonomy.service.ts + const taxonomyQuery = ` + SELECT + f.vid AS taxonomy_uid, + f.tid AS drupal_term_id, + f.name AS term_name, + f.description__value AS term_description + FROM taxonomy_term_field_data f + ORDER BY f.vid, f.tid + `; + + const taxonomyTerms = await executeQuery(connection, taxonomyQuery); + + if (taxonomyTerms.length === 0) { + const noDataMessage = getLogMessage( + srcFunc, + `No taxonomy terms found in database.`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', noDataMessage); + return; + } + + // Transform to taxonomy reference format + const taxonomyReferences: TaxonomyReference[] = []; + + for (const term of taxonomyTerms as DrupalTaxonomyTerm[]) { + const termUid = `${term.taxonomy_uid}_${term.drupal_term_id}`; + + taxonomyReferences.push({ + drupal_term_id: term.drupal_term_id, + taxonomy_uid: term.taxonomy_uid, + term_uid: termUid + }); + } + + // Save taxonomy references to taxonomyReference.json + await writeFile(referencesSave, 'taxonomyReference.json', taxonomyReferences); + + const successMessage = getLogMessage( + srcFunc, + `Created ${taxonomyReferences.length} taxonomy reference mappings.`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', successMessage); + + } catch (error: any) { + const message = getLogMessage( + srcFunc, + `Error creating taxonomy references: ${error.message}`, + {}, + error + ); + await customLogger(projectId, destination_stack_id, 'error', message); + throw error; + } +}; + +/** + * Creates reference mappings from Drupal database for migration to Contentstack. + * Based on the original Drupal v8 references.js logic with direct SQL queries. + * + * This creates a references.json file that maps node IDs to content types, + * which is then used by the entries service to resolve entity references. + * + * Supports dynamic SQL queries from query/index.json file following original pattern: + * var queryPageConfig = helper.readFile(path.join(process.cwd(), config.data, 'query', 'index.json')); + * var query = queryPageConfig['page']['' + pagename + '']; + */ +export const createRefrence = async ( + dbConfig: any, + destination_stack_id: string, + projectId: string, + isTest = false +): Promise => { + const srcFunc = 'createRefrence'; + let connection: mysql.Connection | null = null; + + try { + const referencesSave = path.join(DATA, destination_stack_id, REFERENCES_DIR_NAME); + const referencesPath = path.join(referencesSave, REFERENCES_FILE_NAME); + + // Initialize directories and files + await fs.promises.mkdir(referencesSave, { recursive: true }); + + // Initialize empty references file if it doesn't exist + if (!await fs.promises.access(referencesPath).then(() => true).catch(() => false)) { + await fs.promises.writeFile(referencesPath, JSON.stringify({}, null, 4), 'utf8'); + } + + const message = getLogMessage( + srcFunc, + `Exporting references...`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', message); + + // Read query configuration (following original pattern) + const queryPageConfig = await readQueryConfig(destination_stack_id); + + // Create database connection + connection = await getDbConnection(dbConfig, projectId, destination_stack_id); + + // Process each content type from query config (like original) + const pageQuery = queryPageConfig.page; + const contentTypes = Object.keys(pageQuery); + const typesToProcess = isTest ? contentTypes.slice(0, 2) : contentTypes; + + // Process content types sequentially (like original sequence logic) + for (const contentType of typesToProcess) { + await getPageCount( + connection, + contentType, + queryPageConfig, + referencesPath, + projectId, + destination_stack_id + ); + } + + // Create taxonomy reference mappings + await createTaxonomyReferences( + connection, + referencesSave, + projectId, + destination_stack_id + ); + + const successMessage = getLogMessage( + srcFunc, + `Successfully created reference mappings for ${typesToProcess.length} content types and taxonomy references.`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', successMessage); + + } catch (err) { + const message = getLogMessage( + srcFunc, + `Error encountered while creating references.`, + {}, + err + ); + await customLogger(projectId, destination_stack_id, 'error', message); + throw err; + } finally { + // Close database connection + if (connection) { + connection.end(); + } + } +}; \ No newline at end of file diff --git a/api/src/services/drupal/taxonomy.service.ts b/api/src/services/drupal/taxonomy.service.ts new file mode 100644 index 000000000..8c1e78106 --- /dev/null +++ b/api/src/services/drupal/taxonomy.service.ts @@ -0,0 +1,443 @@ +import fs from "fs"; +import path from "path"; +import mysql from "mysql2"; +import { getDbConnection } from "../../helper/index.js"; +import customLogger from "../../utils/custom-logger.utils.js"; +import { getLogMessage } from "../../utils/index.js"; +import { MIGRATION_DATA_CONFIG } from "../../constants/index.js"; + +const { DATA, TAXONOMIES_DIR_NAME } = MIGRATION_DATA_CONFIG; + +interface DrupalTaxonomyTerm { + taxonomy_uid: string; // vid (vocabulary id) + term_tid: number; // term id + term_name: string; // term name + term_description: string | null; // term description +} + +interface TaxonomyTerm { + uid: string; + name: string; + parent_uid: string | null; + description?: string; +} + +interface TaxonomyStructure { + taxonomy: { + uid: string; + name: string; + description: string; + }; + terms: TaxonomyTerm[]; +} + +/** + * Execute SQL query with promise support + */ +const executeQuery = async (connection: mysql.Connection, query: string): Promise => { + return new Promise((resolve, reject) => { + connection.query(query, (error, results) => { + if (error) { + reject(error); + return; + } + resolve(results as any[]); + }); + }); +}; + +/** + * Generate slug from name (similar to Contentstack uid format) + */ +const generateSlug = (name: string): string => { + return name + .toLowerCase() + .replace(/[^a-z0-9]+/g, '_') + .replace(/^_+|_+$/g, '') + .substring(0, 50); // Limit length +}; + +/** + * Get vocabulary names from Drupal database + * Note: In Drupal 8+, vocabulary names are in the config table + */ +const getVocabularyNames = async ( + connection: mysql.Connection, + projectId: string, + destination_stack_id: string +): Promise> => { + const srcFunc = 'getVocabularyNames'; + + try { + // Try to get vocabulary names from config table (Drupal 8+) + const configQuery = ` + SELECT + SUBSTRING_INDEX(SUBSTRING_INDEX(name, '.', 3), '.', -1) as vid, + JSON_UNQUOTE(JSON_EXTRACT(data, '$.name')) as name + FROM config + WHERE name LIKE 'taxonomy.vocabulary.%' + AND data IS NOT NULL + `; + + const vocabularies = await executeQuery(connection, configQuery); + + const vocabNames: Record = {}; + + for (const vocab of vocabularies) { + if (vocab.vid && vocab.name) { + vocabNames[vocab.vid] = vocab.name; + } + } + + const message = getLogMessage( + srcFunc, + `Found ${Object.keys(vocabNames).length} vocabularies in config.`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', message); + + return vocabNames; + + } catch (error: any) { + // Fallback: use vid as name if config method fails + const message = getLogMessage( + srcFunc, + `Could not fetch vocabulary names from config, will use vid as name: ${error.message}`, + {}, + error + ); + await customLogger(projectId, destination_stack_id, 'warn', message); + + return {}; + } +}; + +/** + * Fetch taxonomy hierarchy information + * Note: Drupal uses taxonomy_term__parent or taxonomy_term_hierarchy table for hierarchy + */ +const getTermHierarchy = async ( + connection: mysql.Connection, + projectId: string, + destination_stack_id: string +): Promise> => { + const srcFunc = 'getTermHierarchy'; + + try { + // Try different possible hierarchy table structures + const hierarchyQueries = [ + // Drupal 8+ field-based hierarchy + `SELECT entity_id as tid, parent_target_id as parent_tid + FROM taxonomy_term__parent + WHERE parent_target_id IS NOT NULL AND parent_target_id != 0`, + + // Drupal 7 style hierarchy + `SELECT tid, parent + FROM taxonomy_term_hierarchy + WHERE parent IS NOT NULL AND parent != 0` + ]; + + let hierarchyData: any[] = []; + + for (const query of hierarchyQueries) { + try { + hierarchyData = await executeQuery(connection, query); + if (hierarchyData.length > 0) { + break; // Use the first successful query + } + } catch (queryError) { + // Continue to next query if this one fails + continue; + } + } + + const hierarchy: Record = {}; + + for (const item of hierarchyData) { + const childTid = item.tid || item.entity_id; + const parentTid = item.parent || item.parent_tid || item.parent_target_id; + + if (childTid && parentTid) { + if (!hierarchy[parentTid]) { + hierarchy[parentTid] = []; + } + hierarchy[parentTid].push(childTid); + } + } + + const message = getLogMessage( + srcFunc, + `Found ${Object.keys(hierarchy).length} parent-child relationships.`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', message); + + return hierarchy; + + } catch (error: any) { + const message = getLogMessage( + srcFunc, + `Could not fetch term hierarchy: ${error.message}`, + {}, + error + ); + await customLogger(projectId, destination_stack_id, 'warn', message); + + return {}; + } +}; + +/** + * Process taxonomy terms and organize by vocabulary + */ +const processTaxonomyData = async ( + terms: DrupalTaxonomyTerm[], + vocabularyNames: Record, + hierarchy: Record, + projectId: string, + destination_stack_id: string +): Promise> => { + const srcFunc = 'processTaxonomyData'; + + try { + const taxonomies: Record = {}; + + // Group terms by vocabulary + const termsByVocabulary: Record = {}; + + for (const term of terms) { + if (!termsByVocabulary[term.taxonomy_uid]) { + termsByVocabulary[term.taxonomy_uid] = []; + } + termsByVocabulary[term.taxonomy_uid].push(term); + } + + // Create taxonomy structure for each vocabulary + for (const [vid, vocabTerms] of Object.entries(termsByVocabulary)) { + const vocabularyName = vocabularyNames[vid] || vid; + + const taxonomyStructure: TaxonomyStructure = { + taxonomy: { + uid: vid, + name: vocabularyName, + description: "" + }, + terms: [] + }; + + // Convert terms to Contentstack format + for (const term of vocabTerms) { + // ๐Ÿท๏ธ Generate term UID using vocabulary prefix + term ID format + const vocabularyPrefix = vid.toLowerCase(); + const termUid = `${vocabularyPrefix}_${term.term_tid}`; + + // Find parent if exists + let parentUid: string | null = null; + for (const [parentTid, childTids] of Object.entries(hierarchy)) { + if (childTids.includes(term.term_tid)) { + // Find parent term in the same vocabulary + const parentTerm = vocabTerms.find(t => t.term_tid === parseInt(parentTid)); + if (parentTerm) { + // ๐Ÿท๏ธ Generate parent UID using same vocabulary prefix + term ID format + parentUid = `${vocabularyPrefix}_${parentTerm.term_tid}`; + } + break; + } + } + + taxonomyStructure.terms.push({ + uid: termUid, + name: term.term_name, + parent_uid: parentUid, + description: term.term_description || "" + }); + } + + taxonomies[vid] = taxonomyStructure; + } + + const message = getLogMessage( + srcFunc, + `Processed ${Object.keys(taxonomies).length} vocabularies with ${terms.length} total terms.`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', message); + + return taxonomies; + + } catch (error: any) { + const message = getLogMessage( + srcFunc, + `Error processing taxonomy data: ${error.message}`, + {}, + error + ); + await customLogger(projectId, destination_stack_id, 'error', message); + throw error; + } +}; + +/** + * Save taxonomy files to disk + */ +const saveTaxonomyFiles = async ( + taxonomies: Record, + taxonomiesPath: string, + projectId: string, + destination_stack_id: string +): Promise => { + const srcFunc = 'saveTaxonomyFiles'; + + try { + let filesSaved = 0; + + // Save individual taxonomy files (existing functionality) + for (const [vid, taxonomy] of Object.entries(taxonomies)) { + const filePath = path.join(taxonomiesPath, `${vid}.json`); + await fs.promises.writeFile(filePath, JSON.stringify(taxonomy, null, 2), 'utf8'); + filesSaved++; + + const message = getLogMessage( + srcFunc, + `Saved taxonomy file: ${vid}.json with ${taxonomy.terms.length} terms.`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', message); + } + + // Create consolidated taxonomies.json file with just vocabulary metadata + const taxonomiesDataObject: Record = {}; + + for (const [vid, taxonomy] of Object.entries(taxonomies)) { + taxonomiesDataObject[vid] = { + uid: taxonomy.taxonomy.uid, + name: taxonomy.taxonomy.name, + description: taxonomy.taxonomy.description + }; + } + + const taxonomiesFilePath = path.join(taxonomiesPath, 'taxonomies.json'); + await fs.promises.writeFile(taxonomiesFilePath, JSON.stringify(taxonomiesDataObject, null, 2), 'utf8'); + + const consolidatedMessage = getLogMessage( + srcFunc, + `Saved consolidated taxonomies.json with ${Object.keys(taxonomiesDataObject).length} vocabularies.`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', consolidatedMessage); + + const summaryMessage = getLogMessage( + srcFunc, + `Successfully saved ${filesSaved} individual taxonomy files + 1 consolidated taxonomies.json file.`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', summaryMessage); + + } catch (error: any) { + const message = getLogMessage( + srcFunc, + `Error saving taxonomy files: ${error.message}`, + {}, + error + ); + await customLogger(projectId, destination_stack_id, 'error', message); + throw error; + } +}; + +/** + * Creates taxonomy files from Drupal database for migration to Contentstack. + * + * Extracts taxonomy vocabularies and terms from Drupal database, + * organizes them by vocabulary, and saves individual JSON files + * for each vocabulary in the format expected by Contentstack. + */ +export const createTaxonomy = async ( + dbConfig: any, + destination_stack_id: string, + projectId: string +): Promise => { + const srcFunc = 'createTaxonomy'; + let connection: mysql.Connection | null = null; + + try { + const taxonomiesPath = path.join(DATA, destination_stack_id, TAXONOMIES_DIR_NAME); + + // Create taxonomies directory + await fs.promises.mkdir(taxonomiesPath, { recursive: true }); + + const message = getLogMessage( + srcFunc, + `Exporting taxonomies...`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', message); + + // Create database connection + connection = await getDbConnection(dbConfig, projectId, destination_stack_id); + + // Main SQL query to fetch taxonomy terms + const taxonomyQuery = ` + SELECT + f.vid AS taxonomy_uid, + f.tid AS term_tid, + f.name AS term_name, + f.description__value AS term_description + FROM taxonomy_term_field_data f + ORDER BY f.vid, f.tid + `; + + // Fetch taxonomy data + const taxonomyTerms = await executeQuery(connection, taxonomyQuery); + + if (taxonomyTerms.length === 0) { + const noDataMessage = getLogMessage( + srcFunc, + `No taxonomy terms found in database.`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', noDataMessage); + return; + } + + // Get vocabulary names and hierarchy + const [vocabularyNames, hierarchy] = await Promise.all([ + getVocabularyNames(connection, projectId, destination_stack_id), + getTermHierarchy(connection, projectId, destination_stack_id) + ]); + + // Process taxonomy data + const taxonomies = await processTaxonomyData( + taxonomyTerms, + vocabularyNames, + hierarchy, + projectId, + destination_stack_id + ); + + // Save taxonomy files + await saveTaxonomyFiles(taxonomies, taxonomiesPath, projectId, destination_stack_id); + + const successMessage = getLogMessage( + srcFunc, + `Successfully exported ${Object.keys(taxonomies).length} taxonomies with ${taxonomyTerms.length} total terms.`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', successMessage); + + } catch (err) { + const message = getLogMessage( + srcFunc, + `Error encountered while creating taxonomies.`, + {}, + err + ); + await customLogger(projectId, destination_stack_id, 'error', message); + throw err; + } finally { + // Close database connection + if (connection) { + connection.end(); + } + } +}; diff --git a/api/src/services/drupal/version.service.ts b/api/src/services/drupal/version.service.ts new file mode 100644 index 000000000..20e6ff6be --- /dev/null +++ b/api/src/services/drupal/version.service.ts @@ -0,0 +1,61 @@ +import fs from "fs"; +import path from "path"; +import { MIGRATION_DATA_CONFIG } from "../../constants/index.js"; +import { getLogMessage } from "../../utils/index.js"; +import customLogger from "../../utils/custom-logger.utils.js"; + +const { DATA, EXPORT_INFO_FILE } = MIGRATION_DATA_CONFIG; + +/** + * Writes data to a specified file, ensuring the target directory exists. + */ +async function writeFile(dirPath: string, filename: string, data: any) { + try { + await fs.promises.mkdir(dirPath, { recursive: true }); + const filePath = path.join(dirPath, filename); + await fs.promises.writeFile(filePath, JSON.stringify(data), 'utf8'); + } catch (err) { + console.error(`Error writing ${dirPath}/${filename}::`, err); + } +} + +/** + * Creates a version file for the given destination stack for Drupal migration. + */ +export const createVersionFile = async ( + destination_stack_id: string, + projectId: string +): Promise => { + const srcFunc = 'createVersionFile'; + + try { + const versionData = { + contentVersion: 2, + logsPath: "", + migrationSource: "drupal", + migrationTimestamp: new Date().toISOString(), + }; + + await writeFile( + path.join(DATA, destination_stack_id), + EXPORT_INFO_FILE, + versionData + ); + + const message = getLogMessage( + srcFunc, + `Version file has been successfully created.`, + {} + ); + await customLogger(projectId, destination_stack_id, 'info', message); + + } catch (err) { + const message = getLogMessage( + srcFunc, + `Error writing version file: ${err}`, + {}, + err + ); + await customLogger(projectId, destination_stack_id, 'error', message); + } +}; diff --git a/api/src/services/migration.service.ts b/api/src/services/migration.service.ts index bb07b5775..7b319b74c 100644 --- a/api/src/services/migration.service.ts +++ b/api/src/services/migration.service.ts @@ -22,6 +22,7 @@ import { import { fieldAttacher } from '../utils/field-attacher.utils.js'; import { siteCoreService } from './sitecore.service.js'; import { wordpressService } from './wordpress.service.js'; +import { drupalService } from './drupal.service.js'; import { testFolderCreator } from '../utils/test-folder-creator.utils.js'; import { utilsCli } from './runCli.service.js'; import customLogger from '../utils/custom-logger.utils.js'; @@ -109,6 +110,47 @@ const createTestStack = async (req: Request): Promise => { .findIndex({ id: projectId }) .value(); if (index > -1) { + // โœ… NEW: Generate queries for new test stack (Drupal only) + const project = ProjectModelLowdb.data.projects[index]; + if (project?.legacy_cms?.cms === CMS.DRUPAL_V8) { + try { + const startMessage = getLogMessage( + srcFun, + `Generating dynamic queries for new test stack (${res?.data?.stack?.api_key})...`, + token_payload + ); + await customLogger(projectId, res?.data?.stack?.api_key, 'info', startMessage); + + // Get database configuration from project + const dbConfig = { + host: project?.legacy_cms?.mySQLDetails?.host, + user: project?.legacy_cms?.mySQLDetails?.user, + password: project?.legacy_cms?.mySQLDetails?.password || '', + database: project?.legacy_cms?.mySQLDetails?.database, + port: project?.legacy_cms?.mySQLDetails?.port || 3306 + }; + + // Generate dynamic queries for the new test stack + await drupalService.createQuery(dbConfig, res?.data?.stack?.api_key, projectId); + + const successMessage = getLogMessage( + srcFun, + `Successfully generated queries for test stack (${res?.data?.stack?.api_key})`, + token_payload + ); + await customLogger(projectId, res?.data?.stack?.api_key, 'info', successMessage); + } catch (error: any) { + const errorMessage = getLogMessage( + srcFun, + `Failed to generate queries for test stack: ${error.message}. Test migration may fail.`, + token_payload, + error + ); + await customLogger(projectId, res?.data?.stack?.api_key, 'error', errorMessage); + // Don't throw error - let test stack creation succeed even if query generation fails + } + } + ProjectModelLowdb.update((data: any) => { data.projects[index].current_step = STEPPER_STEPS['TESTING']; data.projects[index].current_test_stack_id = res?.data?.stack?.api_key; @@ -433,6 +475,43 @@ const startTestMigration = async (req: Request): Promise => { ); break; } + case CMS.DRUPAL_V8: { + // Get database configuration from project + const dbConfig = { + host: project?.legacy_cms?.mySQLDetails?.host, + user: project?.legacy_cms?.mySQLDetails?.user, + password: project?.legacy_cms?.mySQLDetails?.password || '', + database: project?.legacy_cms?.mySQLDetails?.database, + port: project?.legacy_cms?.mySQLDetails?.port || 3306 + }; + + // Get Drupal assets URL configuration from project + const drupalAssetsConfig = { + base_url: project?.legacy_cms?.drupalAssetsUrl?.base_url || "", + public_path: project?.legacy_cms?.drupalAssetsUrl?.public_path || "/sites/default/files/" + }; + + console.info('๐Ÿ” === DRUPAL TEST MIGRATION CONFIG ==='); + console.info('๐Ÿ“‹ Database Config:', JSON.stringify(dbConfig, null, 2)); + console.info('๐Ÿ“‹ Drupal Assets Config:', JSON.stringify(drupalAssetsConfig, null, 2)); + console.info('๐Ÿ“‹ Project ID:', projectId); + console.info('๐Ÿ“‹ Test Stack ID:', project?.current_test_stack_id); + console.info('====================================='); + + // Run Drupal migration services in proper order (following test-drupal-services sequence) + // NOTE: Dynamic queries are generated during Step 2โ†’3 transition, no need for createQueryConfig + + // Generate content type schemas from upload-api (CRITICAL: Must run after upload-api generates schema) + await drupalService?.generateContentTypeSchemas(project?.current_test_stack_id, projectId); + + await drupalService?.createAssets(dbConfig, project?.current_test_stack_id, projectId, true, drupalAssetsConfig); + await drupalService?.createRefrence(dbConfig, project?.current_test_stack_id, projectId, true); + await drupalService?.createTaxonomy(dbConfig, project?.current_test_stack_id, projectId); + await drupalService?.createEntry(dbConfig, project?.current_test_stack_id, projectId, true, project?.stackDetails?.master_locale, project?.content_mapper || []); + await drupalService?.createLocale(dbConfig, project?.current_test_stack_id, projectId, project); + await drupalService?.createVersionFile(project?.current_test_stack_id, projectId); + break; + } default: break; } @@ -667,6 +746,43 @@ const startMigration = async (req: Request): Promise => { ); break; } + case CMS.DRUPAL_V8: { + // Get database configuration from project + const dbConfig = { + host: project?.legacy_cms?.mySQLDetails?.host, + user: project?.legacy_cms?.mySQLDetails?.user, + password: project?.legacy_cms?.mySQLDetails?.password || '', + database: project?.legacy_cms?.mySQLDetails?.database, + port: project?.legacy_cms?.mySQLDetails?.port || 3306 + }; + + // Get Drupal assets URL configuration from project + const drupalAssetsConfig = { + base_url: project?.legacy_cms?.drupalAssetsUrl?.base_url || "", + public_path: project?.legacy_cms?.drupalAssetsUrl?.public_path || "/sites/default/files/" + }; + + console.info('๐Ÿ” === DRUPAL FINAL MIGRATION CONFIG ==='); + console.info('๐Ÿ“‹ Database Config:', JSON.stringify(dbConfig, null, 2)); + console.info('๐Ÿ“‹ Drupal Assets Config:', JSON.stringify(drupalAssetsConfig, null, 2)); + console.info('๐Ÿ“‹ Project ID:', projectId); + console.info('๐Ÿ“‹ Destination Stack ID:', project?.destination_stack_id); + console.info('====================================='); + + // Run Drupal migration services in proper order (following test-drupal-services sequence) + // NOTE: Dynamic queries are generated during Step 2โ†’3 transition, no need for createQueryConfig + + // Generate content type schemas from upload-api (CRITICAL: Must run after upload-api generates schema) + await drupalService?.generateContentTypeSchemas(project?.destination_stack_id, projectId); + + await drupalService?.createAssets(dbConfig, project?.destination_stack_id, projectId, false, drupalAssetsConfig); + await drupalService?.createRefrence(dbConfig, project?.destination_stack_id, projectId, false); + await drupalService?.createTaxonomy(dbConfig, project?.destination_stack_id, projectId); + await drupalService?.createLocale(dbConfig, project?.destination_stack_id, projectId, project); + await drupalService?.createEntry(dbConfig, project?.destination_stack_id, projectId, false, project?.stackDetails?.master_locale, project?.content_mapper || []); + await drupalService?.createVersionFile(project?.destination_stack_id, projectId); + break; + } default: break; } @@ -1004,6 +1120,13 @@ const getLogs = async (req: Request): Promise => { export const createSourceLocales = async (req: Request) => { const projectId = req?.params?.projectId; const locales = req?.body?.locale; + + console.log('๐Ÿ” DEBUG: createSourceLocales received:', { + projectId, + locales, + localesType: typeof locales, + localesArray: Array.isArray(locales) + }); try { // Find the project with the specified projectId diff --git a/api/src/services/projects.service.ts b/api/src/services/projects.service.ts index 05d625e8e..9121bb4fd 100644 --- a/api/src/services/projects.service.ts +++ b/api/src/services/projects.service.ts @@ -2,6 +2,7 @@ import { Request } from "express"; import ProjectModelLowdb from "../models/project-lowdb.js"; import ContentTypesMapperModelLowdb from "../models/contentTypesMapper-lowdb.js"; import FieldMapperModel from "../models/FieldMapper.js"; +import { drupalService } from "./drupal.service.js"; import { BadRequestError, @@ -13,6 +14,7 @@ import { HTTP_CODES, STEPPER_STEPS, NEW_PROJECT_STATUS, + CMS, } from "../constants/index.js"; import { config } from "../config/index.js"; import { getLogMessage, isEmpty, safePromise } from "../utils/index.js"; @@ -20,6 +22,7 @@ import getAuthtoken from "../utils/auth.utils.js"; import https from "../utils/https.utils.js"; import getProjectUtil from "../utils/get-project.utils.js"; import logger from "../utils/logger.js"; +import customLogger from "../utils/custom-logger.utils.js"; // import { contentMapperService } from "./contentMapper.service.js"; import { v4 as uuidv4 } from "uuid"; @@ -109,6 +112,16 @@ const createProject = async (req: Request) => { bucketName: "", buketKey: "", }, + is_sql: false, + mySQLDetails: { + host: "", + user: "", + database:"" + }, + drupalAssetsUrl: { + base_url: "", + public_path: "" + } }, content_mapper: [], execution_log: [], @@ -435,6 +448,9 @@ const updateFileFormat = async (req: Request) => { is_localPath, is_fileValid, awsDetails, + is_sql, + mySQLDetails, + drupalAssetsUrl, } = req.body; const srcFunc = "updateFileFormat"; const projectIndex = (await getProjectUtil( @@ -491,6 +507,17 @@ const updateFileFormat = async (req: Request) => { awsDetails.bucketName; data.projects[projectIndex].legacy_cms.awsDetails.buketKey = awsDetails.buketKey; + data.projects[ projectIndex ].legacy_cms.is_sql = is_sql; + data.projects[projectIndex].legacy_cms.mySQLDetails.host = + mySQLDetails.host; + data.projects[projectIndex].legacy_cms.mySQLDetails.user = + mySQLDetails.user; + data.projects[projectIndex].legacy_cms.mySQLDetails.database = + mySQLDetails.database; + data.projects[projectIndex].legacy_cms.drupalAssetsUrl.base_url = + drupalAssetsUrl?.base_url || ""; + data.projects[projectIndex].legacy_cms.drupalAssetsUrl.public_path = + drupalAssetsUrl?.public_path || ""; }); logger.info( @@ -680,6 +707,113 @@ const updateDestinationStack = async (req: Request) => { } }; +/** + * Generates dynamic queries for Drupal projects with retry logic + * @param project - The project object + * @param stackId - The stack ID to generate queries for + * @param projectId - The project ID + * @param stackType - Type of stack ('destination' or 'test') + * @param token_payload - Token payload for logging + * @returns Promise - Success/failure of query generation + */ +const generateQueriesWithRetry = async ( + project: any, + stackId: string, + projectId: string, + stackType: string, + token_payload: any +): Promise => { + const srcFunc = 'generateQueriesWithRetry'; + const maxRetries = 3; + + // Only generate queries for Drupal projects + if (project?.legacy_cms?.cms !== CMS.DRUPAL_V8) { + return true; // Skip for non-Drupal projects + } + + if (!stackId) { + const message = getLogMessage( + srcFunc, + `No ${stackType} stack ID found, skipping query generation`, + token_payload + ); + await customLogger(projectId, stackId || 'unknown', 'warn', message); + return true; // Skip if no stack ID + } + + // Get database configuration from project + const dbConfig = { + host: project?.legacy_cms?.mySQLDetails?.host, + user: project?.legacy_cms?.mySQLDetails?.user, + password: project?.legacy_cms?.mySQLDetails?.password || '', + database: project?.legacy_cms?.mySQLDetails?.database, + port: project?.legacy_cms?.mySQLDetails?.port || 3306 + }; + + const logMessage = getLogMessage( + srcFunc, + `Starting query generation for ${stackType} stack (${stackId})...`, + token_payload + ); + await customLogger(projectId, stackId, 'info', logMessage); + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + const attemptMessage = getLogMessage( + srcFunc, + `Query generation attempt ${attempt}/${maxRetries} for ${stackType} stack`, + token_payload + ); + await customLogger(projectId, stackId, 'info', attemptMessage); + + // Generate dynamic queries using the query service + await drupalService.createQuery(dbConfig, stackId, projectId); + + const successMessage = getLogMessage( + srcFunc, + `Successfully generated queries for ${stackType} stack (${stackId}) on attempt ${attempt}`, + token_payload + ); + await customLogger(projectId, stackId, 'info', successMessage); + + return true; // Success + } catch (error: any) { + const errorMessage = getLogMessage( + srcFunc, + `Query generation attempt ${attempt}/${maxRetries} failed for ${stackType} stack: ${error.message}`, + token_payload, + error + ); + await customLogger(projectId, stackId, 'error', errorMessage); + + if (attempt === maxRetries) { + // Final attempt failed + const finalErrorMessage = getLogMessage( + srcFunc, + `Query generation failed after ${maxRetries} attempts for ${stackType} stack. Please try again.`, + token_payload, + error + ); + await customLogger(projectId, stackId, 'error', finalErrorMessage); + return false; // All attempts failed + } + + // Wait before retry (exponential backoff) + const retryDelay = Math.pow(2, attempt) * 1000; // 2s, 4s, 8s + const retryMessage = getLogMessage( + srcFunc, + `Retrying query generation in ${retryDelay / 1000} seconds...`, + token_payload + ); + await customLogger(projectId, stackId, 'info', retryMessage); + + await new Promise(resolve => setTimeout(resolve, retryDelay)); + } + } + + return false; // Should never reach here +}; + /** * Updates the current step of a project based on the provided request. * @param req - The request object containing the parameters and body. @@ -755,6 +889,63 @@ const updateCurrentStep = async (req: Request) => { ); } + // โœ… NEW: Generate dynamic queries for both destination and test stacks (Drupal only) + if (project?.legacy_cms?.cms === CMS.DRUPAL_V8) { + const startMessage = getLogMessage( + srcFunc, + `Generating dynamic queries for Drupal project before proceeding to Content Mapping...`, + token_payload + ); + await customLogger(projectId, project?.destination_stack_id, 'info', startMessage); + + // Generate queries for destination stack + const destinationSuccess = await generateQueriesWithRetry( + project, + project?.destination_stack_id, + projectId, + 'destination', + token_payload + ); + + // Generate queries for test stack (if exists) + let testSuccess = true; + if (project?.current_test_stack_id) { + testSuccess = await generateQueriesWithRetry( + project, + project?.current_test_stack_id, + projectId, + 'test', + token_payload + ); + } + + // Check if query generation failed + if (!destinationSuccess || !testSuccess) { + const failedStacks = []; + if (!destinationSuccess) failedStacks.push('destination'); + if (!testSuccess) failedStacks.push('test'); + + const errorMessage = `Query generation failed for ${failedStacks.join(' and ')} stack(s). Something went wrong. Please try again.`; + + logger.error( + getLogMessage( + srcFunc, + errorMessage, + token_payload + ) + ); + + throw new BadRequestError(errorMessage); + } + + const completeMessage = getLogMessage( + srcFunc, + `Dynamic queries successfully generated for all stacks. Proceeding to Content Mapping step.`, + token_payload + ); + await customLogger(projectId, project?.destination_stack_id, 'info', completeMessage); + } + await ProjectModelLowdb.update((data: any) => { data.projects[projectIndex].current_step = STEPPER_STEPS.CONTENT_MAPPING; diff --git a/api/src/utils/batch-processor.utils.ts b/api/src/utils/batch-processor.utils.ts new file mode 100644 index 000000000..2f2e25488 --- /dev/null +++ b/api/src/utils/batch-processor.utils.ts @@ -0,0 +1,108 @@ +/** + * Batch Processor Utility + * Handles large datasets by processing them in smaller batches to prevent resource exhaustion + */ + +export interface BatchProcessorOptions { + batchSize: number; + concurrency: number; + delayBetweenBatches?: number; +} + +export class BatchProcessor { + private options: BatchProcessorOptions; + + constructor(options: BatchProcessorOptions) { + this.options = { + delayBetweenBatches: 100, // Default 100ms delay + ...options + }; + } + + /** + * Process large array in batches to prevent resource exhaustion + */ + async processBatches( + items: T[], + processor: (item: T) => Promise, + onBatchComplete?: (batchIndex: number, totalBatches: number, results: R[]) => void + ): Promise { + const { batchSize, concurrency, delayBetweenBatches } = this.options; + const totalBatches = Math.ceil(items.length / batchSize); + const allResults: R[] = []; + + console.log(`๐Ÿ“ฆ Processing ${items.length} items in ${totalBatches} batches (${batchSize} items per batch, concurrency: ${concurrency})`); + + for (let batchIndex = 0; batchIndex < totalBatches; batchIndex++) { + const startIndex = batchIndex * batchSize; + const endIndex = Math.min(startIndex + batchSize, items.length); + const batch = items.slice(startIndex, endIndex); + + console.log(`๐Ÿ“‹ Processing batch ${batchIndex + 1}/${totalBatches} (${batch.length} items)`); + + // Process batch with controlled concurrency + const batchResults = await this.processBatchWithConcurrency(batch, processor, concurrency); + allResults.push(...batchResults); + + // Callback for batch completion + if (onBatchComplete) { + onBatchComplete(batchIndex + 1, totalBatches, batchResults); + } + + // Delay between batches to allow file handles to close + if (batchIndex < totalBatches - 1 && delayBetweenBatches && delayBetweenBatches > 0) { + await this.delay(delayBetweenBatches); + } + + // Force garbage collection if available + if (global.gc) { + global.gc(); + } + } + + console.log(`โœ… Completed processing ${items.length} items in ${totalBatches} batches`); + return allResults; + } + + /** + * Process a single batch with controlled concurrency + */ + private async processBatchWithConcurrency( + batch: T[], + processor: (item: T) => Promise, + concurrency: number + ): Promise { + const results: R[] = []; + + for (let i = 0; i < batch.length; i += concurrency) { + const chunk = batch.slice(i, i + concurrency); + const chunkPromises = chunk.map(processor); + const chunkResults = await Promise.all(chunkPromises); + results.push(...chunkResults); + } + + return results; + } + + /** + * Delay utility + */ + private delay(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); + } +} + +/** + * Quick utility function for batch processing + */ +export async function processBatches( + items: T[], + processor: (item: T) => Promise, + options: BatchProcessorOptions, + onBatchComplete?: (batchIndex: number, totalBatches: number, results: R[]) => void +): Promise { + const batchProcessor = new BatchProcessor(options); + return batchProcessor.processBatches(items, processor, onBatchComplete); +} + +export default BatchProcessor; diff --git a/api/src/utils/content-type-creator.utils.ts b/api/src/utils/content-type-creator.utils.ts index 215d2829d..ec4f7466b 100644 --- a/api/src/utils/content-type-creator.utils.ts +++ b/api/src/utils/content-type-creator.utils.ts @@ -5,7 +5,7 @@ import customLogger from './custom-logger.utils.js'; import { getLogMessage } from './index.js'; import { LIST_EXTENSION_UID, MIGRATION_DATA_CONFIG } from '../constants/index.js'; import { contentMapperService } from "../services/contentMapper.service.js"; -import appMeta from '../constants/app/index.json'; +import appMeta from '../constants/app/index.json' with { type: 'json' }; const { GLOBAL_FIELDS_FILE_NAME, @@ -122,7 +122,7 @@ const saveAppMapper = async ({ marketPlacePath, data, fileName }: any) => { } } -const convertToSchemaFormate = ({ field, advanced = true, marketPlacePath, keyMapper }: any) => { +export const convertToSchemaFormate = ({ field, advanced = true, marketPlacePath, keyMapper }: any) => { switch (field?.contentstackFieldType) { case 'single_line_text': { return { @@ -364,6 +364,29 @@ const convertToSchemaFormate = ({ field, advanced = true, marketPlacePath, keyMa "non_localizable": field.advanced?.nonLocalizable ?? false } } + + case "html": { + return { + "data_type": "html", + "display_name": field?.title, + uid: field?.uid, + "field_metadata": { + description: "", + default_value: field?.advanced?.default_value ?? '', + "allow_json_rte": true, + "embed_entry": false, + "rich_text_type": "advanced" + }, + "format": field?.advanced?.validationRegex ?? '', + "error_messages": { + "format": field?.advanced?.validationErrorMessage ?? '', + }, + "multiple": field?.advanced?.multiple ?? false, + "mandatory": field?.advanced?.mandatory ?? false, + "unique": field?.advanced?.unique ?? false, + "non_localizable": field.advanced?.nonLocalizable ?? false + } + } case 'markdown': { return { "data_type": "text", @@ -462,6 +485,27 @@ const convertToSchemaFormate = ({ field, advanced = true, marketPlacePath, keyMa }; } + case "taxonomy": { + return { + data_type: "taxonomy", + display_name: field?.title, + uid: field?.uid, + taxonomies: field?.advanced?.taxonomies || [], + field_metadata: { + description: field?.advanced?.field_metadata?.description || "", + default_value: field?.advanced?.field_metadata?.default_value || "" + }, + format: field?.advanced?.validationRegex ?? '', + error_messages: { + format: field?.advanced?.validationErrorMessage ?? '', + }, + mandatory: field?.advanced?.mandatory ?? false, + multiple: field?.advanced?.multiple ?? true, + non_localizable: field?.advanced?.non_localizable ?? false, + unique: field?.advanced?.unique ?? false + }; + } + case 'html': { const htmlField: any = { "data_type": "text", @@ -770,7 +814,7 @@ export const contenTypeMaker = async ({ contentType, destinationStackId, project marketPlacePath, keyMapper }); - if (dt && item?.isDeleted === false) { + if (dt && item?.isDeleted !== true) { ct?.schema?.push(dt); } } diff --git a/api/src/utils/optimized-query-builder.utils.ts b/api/src/utils/optimized-query-builder.utils.ts new file mode 100644 index 000000000..0dfd70c9d --- /dev/null +++ b/api/src/utils/optimized-query-builder.utils.ts @@ -0,0 +1,452 @@ +import mysql from 'mysql2'; +import { getLogMessage } from './index.js'; +import customLogger from './custom-logger.utils.js'; + +/** + * Optimized Query Builder for Drupal Field Data + * Eliminates the 61-table JOIN limit by using sequential queries + */ + +interface DrupalFieldData { + field_name: string; + content_types: string; + type: string; + content_handler?: string; +} + +interface FieldResult { + entity_id: number; + [key: string]: any; +} + +interface OptimizedQueryResult { + baseQuery: string; + countQuery: string; + fieldQueries: string[]; +} + +export class OptimizedQueryBuilder { + private connection: mysql.Connection; + private projectId: string; + private destinationStackId: string; + + constructor(connection: mysql.Connection, projectId: string, destinationStackId: string) { + this.connection = connection; + this.projectId = projectId; + this.destinationStackId = destinationStackId; + } + + /** + * Strategy 1: Sequential Field Queries (No JOINs) + * Fetch base node data first, then field data separately + */ + async generateSequentialQueries( + contentType: string, + fieldsForType: DrupalFieldData[] + ): Promise { + const srcFunc = 'generateSequentialQueries'; + + // 1. Base query for node data (no JOINs) + const baseQuery = ` + SELECT + node.nid, + node.title, + node.langcode, + node.created, + node.type, + users.name as author_name + FROM node_field_data node + LEFT JOIN users ON users.uid = node.uid + WHERE node.type = '${contentType}' + ORDER BY node.nid + `; + + // 2. Count query (simple, no JOINs) + const countQuery = ` + SELECT COUNT(DISTINCT node.nid) as countentry + FROM node_field_data node + WHERE node.type = '${contentType}' + `; + + // 3. Individual field queries (one per field table) + const fieldQueries: string[] = []; + + for (const field of fieldsForType) { + // Check if field table exists and get column structure + const fieldTableName = `node__${field.field_name}`; + + try { + // Get field columns dynamically + const columnQuery = ` + SELECT COLUMN_NAME + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = '${fieldTableName}' + AND COLUMN_NAME LIKE '${field.field_name}_%' + `; + + const [columns] = await this.connection.promise().query(columnQuery) as any[]; + + if (columns.length > 0) { + // Build field-specific query + const fieldColumns = columns.map((col: any) => col.COLUMN_NAME).join(', '); + + const fieldQuery = ` + SELECT + entity_id, + ${fieldColumns} + FROM ${fieldTableName} + WHERE entity_id IN ( + SELECT nid FROM node_field_data WHERE type = '${contentType}' + ) + `; + + fieldQueries.push(fieldQuery); + } + } catch (error) { + console.warn(`Field table ${fieldTableName} not found or inaccessible:`, error); + } + } + + const message = getLogMessage( + srcFunc, + `Generated optimized queries for ${contentType}: 1 base + ${fieldQueries.length} field queries (0 JOINs)`, + {} + ); + await customLogger(this.projectId, this.destinationStackId, 'info', message); + + return { + baseQuery, + countQuery, + fieldQueries + }; + } + + /** + * Strategy 2: Batch Field Queries (Limited JOINs) + * Group fields into batches with max 15 JOINs each + */ + async generateBatchedQueries( + contentType: string, + fieldsForType: DrupalFieldData[], + batchSize: number = 15 + ): Promise<{ baseQuery: string; batchQueries: string[]; countQuery: string }> { + const srcFunc = 'generateBatchedQueries'; + + // Base query (always the same) + const baseQuery = ` + SELECT + node.nid, + node.title, + node.langcode, + node.created, + node.type + FROM node_field_data node + WHERE node.type = '${contentType}' + ORDER BY node.nid + `; + + // Count query + const countQuery = ` + SELECT COUNT(DISTINCT node.nid) as countentry + FROM node_field_data node + WHERE node.type = '${contentType}' + `; + + // Create batches of fields + const fieldBatches = this.createFieldBatches(fieldsForType, batchSize); + const batchQueries: string[] = []; + + for (let i = 0; i < fieldBatches.length; i++) { + const batch = fieldBatches[i]; + const validFields: string[] = []; + const joinClauses: string[] = []; + + // Validate each field in the batch + for (const field of batch) { + try { + const fieldTableName = `node__${field.field_name}`; + + // Check if table exists + const tableExistsQuery = ` + SELECT 1 FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = '${fieldTableName}' + `; + + const [tableExists] = await this.connection.promise().query(tableExistsQuery) as any[]; + + if (tableExists.length > 0) { + // Get field columns + const columnQuery = ` + SELECT COLUMN_NAME + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = '${fieldTableName}' + AND COLUMN_NAME LIKE '${field.field_name}_%' + LIMIT 1 + `; + + const [columns] = await this.connection.promise().query(columnQuery) as any[]; + + if (columns.length > 0) { + const columnName = columns[0].COLUMN_NAME; + validFields.push(`MAX(${fieldTableName}.${columnName}) as ${columnName}`); + joinClauses.push(`LEFT JOIN ${fieldTableName} ON ${fieldTableName}.entity_id = node.nid`); + } + } + } catch (error) { + console.warn(`Skipping field ${field.field_name}:`, error); + } + } + + if (validFields.length > 0) { + const batchQuery = ` + SELECT + node.nid, + ${validFields.join(',\n ')} + FROM node_field_data node + ${joinClauses.join('\n ')} + WHERE node.type = '${contentType}' + GROUP BY node.nid + ORDER BY node.nid + `; + + batchQueries.push(batchQuery); + } + } + + const message = getLogMessage( + srcFunc, + `Generated ${batchQueries.length} batched queries for ${contentType} (max ${batchSize} JOINs each)`, + {} + ); + await customLogger(this.projectId, this.destinationStackId, 'info', message); + + return { + baseQuery, + batchQueries, + countQuery + }; + } + + /** + * Strategy 3: Union-Based Field Queries + * Use UNION to combine field data without JOINs + */ + async generateUnionQueries( + contentType: string, + fieldsForType: DrupalFieldData[] + ): Promise<{ baseQuery: string; unionQuery: string; countQuery: string }> { + const srcFunc = 'generateUnionQueries'; + + // Base query + const baseQuery = ` + SELECT + node.nid, + node.title, + node.langcode, + node.created, + node.type + FROM node_field_data node + WHERE node.type = '${contentType}' + ORDER BY node.nid + `; + + // Count query + const countQuery = ` + SELECT COUNT(DISTINCT node.nid) as countentry + FROM node_field_data node + WHERE node.type = '${contentType}' + `; + + // Union query for all field data + const unionParts: string[] = []; + + for (const field of fieldsForType) { + const fieldTableName = `node__${field.field_name}`; + + try { + // Get field columns + const columnQuery = ` + SELECT COLUMN_NAME + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = '${fieldTableName}' + AND COLUMN_NAME LIKE '${field.field_name}_%' + LIMIT 1 + `; + + const [columns] = await this.connection.promise().query(columnQuery) as any[]; + + if (columns.length > 0) { + const columnName = columns[0].COLUMN_NAME; + + unionParts.push(` + SELECT + entity_id as nid, + '${field.field_name}' as field_name, + ${columnName} as field_value + FROM ${fieldTableName} + WHERE entity_id IN ( + SELECT nid FROM node_field_data WHERE type = '${contentType}' + ) + `); + } + } catch (error) { + console.warn(`Skipping field ${field.field_name} in union:`, error); + } + } + + const unionQuery = unionParts.length > 0 ? unionParts.join('\nUNION ALL\n') : ''; + + const message = getLogMessage( + srcFunc, + `Generated union query for ${contentType} with ${unionParts.length} field parts`, + {} + ); + await customLogger(this.projectId, this.destinationStackId, 'info', message); + + return { + baseQuery, + unionQuery, + countQuery + }; + } + + /** + * Execute optimized queries and merge results + */ + async executeOptimizedQueries( + strategy: 'sequential' | 'batched' | 'union', + contentType: string, + fieldsForType: DrupalFieldData[], + batchSize: number = 15 + ): Promise { + const srcFunc = 'executeOptimizedQueries'; + + try { + switch (strategy) { + case 'sequential': + return await this.executeSequentialQueries(contentType, fieldsForType); + + case 'batched': + return await this.executeBatchedQueries(contentType, fieldsForType, batchSize); + + case 'union': + return await this.executeUnionQueries(contentType, fieldsForType); + + default: + throw new Error(`Unknown strategy: ${strategy}`); + } + } catch (error: any) { + const message = getLogMessage( + srcFunc, + `Failed to execute optimized queries for ${contentType}: ${error.message}`, + {}, + error + ); + await customLogger(this.projectId, this.destinationStackId, 'error', message); + throw error; + } + } + + private async executeSequentialQueries(contentType: string, fieldsForType: DrupalFieldData[]): Promise { + const { baseQuery, fieldQueries } = await this.generateSequentialQueries(contentType, fieldsForType); + + // Execute base query + const [baseResults] = await this.connection.promise().query(baseQuery) as any[]; + + // Create result map + const resultMap = new Map(); + baseResults.forEach((row: any) => { + resultMap.set(row.nid, { ...row }); + }); + + // Execute field queries and merge results + for (const fieldQuery of fieldQueries) { + const [fieldResults] = await this.connection.promise().query(fieldQuery) as any[]; + + fieldResults.forEach((fieldRow: any) => { + const nid = fieldRow.entity_id; + if (resultMap.has(nid)) { + const existingRow = resultMap.get(nid); + // Merge field data (exclude entity_id) + const { entity_id, ...fieldData } = fieldRow; + Object.assign(existingRow, fieldData); + } + }); + } + + return Array.from(resultMap.values()); + } + + private async executeBatchedQueries(contentType: string, fieldsForType: DrupalFieldData[], batchSize: number): Promise { + const { baseQuery, batchQueries } = await this.generateBatchedQueries(contentType, fieldsForType, batchSize); + + // Execute base query + const [baseResults] = await this.connection.promise().query(baseQuery) as any[]; + + // Create result map + const resultMap = new Map(); + baseResults.forEach((row: any) => { + resultMap.set(row.nid, { ...row }); + }); + + // Execute batch queries and merge results + for (const batchQuery of batchQueries) { + const [batchResults] = await this.connection.promise().query(batchQuery) as any[]; + + batchResults.forEach((batchRow: any) => { + const nid = batchRow.nid; + if (resultMap.has(nid)) { + const existingRow = resultMap.get(nid); + // Merge batch data (exclude nid) + const { nid: _, ...batchData } = batchRow; + Object.assign(existingRow, batchData); + } + }); + } + + return Array.from(resultMap.values()); + } + + private async executeUnionQueries(contentType: string, fieldsForType: DrupalFieldData[]): Promise { + const { baseQuery, unionQuery } = await this.generateUnionQueries(contentType, fieldsForType); + + // Execute base query + const [baseResults] = await this.connection.promise().query(baseQuery) as any[]; + + // Create result map + const resultMap = new Map(); + baseResults.forEach((row: any) => { + resultMap.set(row.nid, { ...row }); + }); + + // Execute union query if it exists + if (unionQuery) { + const [unionResults] = await this.connection.promise().query(unionQuery) as any[]; + + // Group union results by nid + unionResults.forEach((unionRow: any) => { + const nid = unionRow.nid; + if (resultMap.has(nid)) { + const existingRow = resultMap.get(nid); + existingRow[unionRow.field_name] = unionRow.field_value; + } + }); + } + + return Array.from(resultMap.values()); + } + + private createFieldBatches(fields: DrupalFieldData[], batchSize: number): DrupalFieldData[][] { + const batches: DrupalFieldData[][] = []; + for (let i = 0; i < fields.length; i += batchSize) { + batches.push(fields.slice(i, i + batchSize)); + } + return batches; + } +} + +export default OptimizedQueryBuilder; diff --git a/api/test-drupal-locales.js b/api/test-drupal-locales.js new file mode 100644 index 000000000..00ae177ec --- /dev/null +++ b/api/test-drupal-locales.js @@ -0,0 +1,150 @@ +import { createLocale } from './src/services/drupal/locales.service.js'; +import mysql from 'mysql2/promise'; +import axios from 'axios'; +import fs from 'fs'; +import path from 'path'; + +// Test configuration using upload-api database (riceuniversity2) +const testProject = { + mysql: { + host: 'localhost', + user: 'root', + password: '', + database: 'riceuniversity2', + port: 3306 + } +}; + +const testDestinationStackId = 'test-drupal-locale-stack'; +const testProjectId = 'test-project-123'; + +async function testDrupalLocaleQueries() { + console.log('๐Ÿงช Testing Drupal Locale SQL Queries...'); + console.log('๐Ÿ“Š Database:', testProject.mysql.database); + + let connection; + + try { + // Create database connection + connection = await mysql.createConnection(testProject.mysql); + console.log('โœ… Database connection established'); + + // 1. Test master locale query + console.log('\n๐Ÿ” Testing Master Locale Query...'); + const masterLocaleQuery = ` + SELECT SUBSTRING_INDEX( + SUBSTRING_INDEX(CONVERT(data USING utf8), 'default_langcode";s:2:"', -1), + '"', 1 + ) as master_locale + FROM config + WHERE name = 'system.site' + `; + + const [masterRows] = await connection.execute(masterLocaleQuery); + const masterLocaleCode = masterRows[0]?.master_locale || 'en'; + console.log('โœ… Master Locale:', masterLocaleCode); + + // 2. Test all locales query + console.log('\n๐Ÿ” Testing All Locales Query...'); + const allLocalesQuery = ` + SELECT DISTINCT langcode + FROM node_field_data + WHERE langcode IS NOT NULL AND langcode != '' + ORDER BY langcode + `; + + const [allLocaleRows] = await connection.execute(allLocalesQuery); + const allLocaleCodes = allLocaleRows.map(row => row.langcode); + console.log('โœ… All Locales:', allLocaleCodes); + + // 3. Test non-master locales query + console.log('\n๐Ÿ” Testing Non-Master Locales Query...'); + const nonMasterLocalesQuery = ` + SELECT DISTINCT n.langcode + FROM node_field_data n + WHERE n.langcode IS NOT NULL + AND n.langcode != '' + AND n.langcode != ( + SELECT + SUBSTRING_INDEX( + SUBSTRING_INDEX(CONVERT(data USING utf8), 'default_langcode";s:2:"', -1), + '"', + 1 + ) + FROM config + WHERE name = 'system.site' + LIMIT 1 + ) + ORDER BY n.langcode + `; + + const [nonMasterRows] = await connection.execute(nonMasterLocalesQuery); + const nonMasterLocaleCodes = nonMasterRows.map(row => row.langcode); + console.log('โœ… Non-Master Locales:', nonMasterLocaleCodes); + + await connection.end(); + + // 4. Test Contentstack API + console.log('\n๐Ÿ” Testing Contentstack Locales API...'); + try { + const response = await axios.get('https://app.contentstack.com/api/v3/locales?include_all=true'); + const contentstackLocales = response.data?.locales || {}; + const localeCount = Object.keys(contentstackLocales).length; + console.log('โœ… Contentstack API Response:', `${localeCount} locales fetched`); + + // Test locale name lookup for found codes + console.log('\n๐Ÿ” Testing Locale Name Mapping...'); + allLocaleCodes.forEach(code => { + const name = contentstackLocales[code] || contentstackLocales[code.toLowerCase()] || 'Unknown'; + console.log(` ${code} โ†’ ${name}`); + }); + + } catch (apiError) { + console.error('โŒ Contentstack API Error:', apiError.message); + } + + // 5. Test the actual createLocale function + console.log('\n๐Ÿ” Testing createLocale Function...'); + try { + await createLocale(testDestinationStackId, testProjectId, testProject); + console.log('โœ… createLocale function executed successfully'); + + // Check if files were created + const localesDir = path.join('./cmsMigrationData', testDestinationStackId, 'locales'); + console.log('๐Ÿ“‚ Checking directory:', localesDir); + + if (fs.existsSync(localesDir)) { + const files = fs.readdirSync(localesDir); + console.log('๐Ÿ“ Created files:', files); + + // Read and display each file + files.forEach(file => { + const filePath = path.join(localesDir, file); + const content = JSON.parse(fs.readFileSync(filePath, 'utf8')); + console.log(`\n๐Ÿ“„ ${file}:`, JSON.stringify(content, null, 2)); + }); + } else { + console.log('โŒ Locales directory not found'); + } + + } catch (createError) { + console.error('โŒ createLocale function failed:', createError); + console.error('Stack trace:', createError.stack); + } + + console.log('\nโœ… All tests completed!'); + + } catch (error) { + console.error('โŒ Test failed:', error); + console.error('Stack trace:', error.stack); + } +} + +// Run the test +testDrupalLocaleQueries().then(() => { + console.log('\n๐Ÿ Test completed'); + process.exit(0); +}).catch((error) => { + console.error('๐Ÿ’ฅ Test crashed:', error); + process.exit(1); +}); diff --git a/api/test-uid-format.js b/api/test-uid-format.js new file mode 100644 index 000000000..c57ac56d5 --- /dev/null +++ b/api/test-uid-format.js @@ -0,0 +1,71 @@ +// Test the new UID format and JSON structure + +const testLocales = ['en', 'und', 'fr-fr', 'es-mx']; +const masterLocale = 'en'; + +console.log('๐Ÿงช Testing New UID Format and JSON Structure...\n'); + +// Test UID generation +console.log('๐Ÿ” UID Generation:'); +testLocales.forEach(langcode => { + const uid = `drupallocale_${langcode.toLowerCase().replace(/-/g, '_')}`; + console.log(` ${langcode} โ†’ ${uid}`); +}); + +console.log('\n๐Ÿ“„ Expected JSON Output:\n'); + +// Simulate the expected output +const msLocale = {}; +const allLocales = {}; +const localeList = {}; + +testLocales.forEach(langcode => { + const uid = `drupallocale_${langcode.toLowerCase().replace(/-/g, '_')}`; + const isMaster = langcode === masterLocale; + + // Apply transformation (simplified) + let code = langcode.toLowerCase(); + let name = ''; + + if (langcode === 'und') { + code = 'en-us'; + name = 'English - United States'; + } else if (langcode === 'en') { + name = 'English'; + } else if (langcode === 'fr-fr') { + name = 'French - France'; + } else if (langcode === 'es-mx') { + name = 'Spanish - Mexico'; + } + + const locale = { + code: code, + name: name, + fallback_locale: isMaster ? null : masterLocale.toLowerCase(), + uid: uid + }; + + if (isMaster) { + msLocale[uid] = locale; + } else { + allLocales[uid] = locale; + } + + localeList[uid] = locale; +}); + +console.log('โœ… master-locale.json:'); +console.log(JSON.stringify(msLocale, null, 2)); + +console.log('\nโœ… locales.json:'); +console.log(JSON.stringify(allLocales, null, 2)); + +console.log('\nโœ… language.json:'); +console.log(JSON.stringify(localeList, null, 2)); + +console.log('\n๐ŸŽฏ Key Features:'); +console.log(' โœ… UID format: drupallocale_{langcode}'); +console.log(' โœ… Hyphens replaced with underscores'); +console.log(' โœ… UID used as JSON key (not random UUID)'); +console.log(' โœ… Master locale has null fallback'); +console.log(' โœ… Non-master locales use master as fallback'); diff --git a/test-drupal-locales-simple.js b/test-drupal-locales-simple.js new file mode 100644 index 000000000..9b9fb35fe --- /dev/null +++ b/test-drupal-locales-simple.js @@ -0,0 +1,160 @@ +const mysql = require('mysql2/promise'); +const axios = require('axios'); + +// Test configuration using upload-api database +const dbConfig = { + host: 'localhost', + user: 'root', + password: '', + database: 'riceuniversity2', + port: 3306 +}; + +async function testDrupalLocaleQueries() { + console.log('๐Ÿงช Testing Drupal Locale SQL Queries...'); + console.log('๐Ÿ“Š Database:', dbConfig.database); + + let connection; + + try { + // Create database connection + connection = await mysql.createConnection(dbConfig); + console.log('โœ… Database connection established'); + + // 1. Test master locale query + console.log('\n๐Ÿ” Testing Master Locale Query...'); + const masterLocaleQuery = ` + SELECT SUBSTRING_INDEX( + SUBSTRING_INDEX(CONVERT(data USING utf8), 'default_langcode";s:2:"', -1), + '"', 1 + ) as master_locale + FROM config + WHERE name = 'system.site' + `; + + const [masterRows] = await connection.execute(masterLocaleQuery); + const masterLocaleCode = masterRows[0]?.master_locale || 'en'; + console.log('โœ… Master Locale:', masterLocaleCode); + + // 2. Test all locales query + console.log('\n๐Ÿ” Testing All Locales Query...'); + const allLocalesQuery = ` + SELECT DISTINCT langcode + FROM node_field_data + WHERE langcode IS NOT NULL AND langcode != '' + ORDER BY langcode + `; + + const [allLocaleRows] = await connection.execute(allLocalesQuery); + const allLocaleCodes = allLocaleRows.map(row => row.langcode); + console.log('โœ… All Locales:', allLocaleCodes); + + // 3. Test non-master locales query + console.log('\n๐Ÿ” Testing Non-Master Locales Query...'); + const nonMasterLocalesQuery = ` + SELECT DISTINCT n.langcode + FROM node_field_data n + WHERE n.langcode IS NOT NULL + AND n.langcode != '' + AND n.langcode != ( + SELECT + SUBSTRING_INDEX( + SUBSTRING_INDEX(CONVERT(data USING utf8), 'default_langcode";s:2:"', -1), + '"', + 1 + ) + FROM config + WHERE name = 'system.site' + LIMIT 1 + ) + ORDER BY n.langcode + `; + + const [nonMasterRows] = await connection.execute(nonMasterLocalesQuery); + const nonMasterLocaleCodes = nonMasterRows.map(row => row.langcode); + console.log('โœ… Non-Master Locales:', nonMasterLocaleCodes); + + // 4. Test Contentstack API + console.log('\n๐Ÿ” Testing Contentstack Locales API...'); + try { + const response = await axios.get('https://app.contentstack.com/api/v3/locales?include_all=true'); + const contentstackLocales = response.data?.locales || {}; + const localeCount = Object.keys(contentstackLocales).length; + console.log('โœ… Contentstack API Response:', `${localeCount} locales fetched`); + + // Show sample locales + const sampleLocales = Object.entries(contentstackLocales).slice(0, 5); + console.log('๐Ÿ“‹ Sample Locales:', sampleLocales); + + // Test locale name lookup for found codes + console.log('\n๐Ÿ” Testing Locale Name Mapping...'); + allLocaleCodes.forEach(code => { + const name = contentstackLocales[code] || contentstackLocales[code.toLowerCase()] || 'Unknown'; + console.log(` ${code} โ†’ ${name}`); + }); + + } catch (apiError) { + console.error('โŒ Contentstack API Error:', apiError.message); + } + + // 5. Test transformation logic + console.log('\n๐Ÿ” Testing Transformation Logic...'); + const hasUnd = allLocaleCodes.includes('und'); + const hasEn = allLocaleCodes.includes('en'); + const hasEnUs = allLocaleCodes.includes('en-us'); + + console.log('๐Ÿ“Š Locale Analysis:'); + console.log(` Has "und": ${hasUnd}`); + console.log(` Has "en": ${hasEn}`); + console.log(` Has "en-us": ${hasEnUs}`); + console.log(` Master Locale: ${masterLocaleCode}`); + + // Apply transformation rules + console.log('\n๐Ÿ”„ Applying Transformation Rules...'); + allLocaleCodes.forEach(locale => { + let transformedCode = locale.toLowerCase(); + let transformedName = ''; + let isMaster = locale === masterLocaleCode; + + if (locale === 'und') { + if (hasEnUs) { + transformedCode = 'en'; + transformedName = 'English'; + } else { + transformedCode = 'en-us'; + transformedName = 'English - United States'; + } + } else if (locale === 'en-us') { + if (hasUnd) { + transformedCode = 'en'; + transformedName = 'English'; + } + } else if (locale === 'en' && hasEnUs) { + transformedCode = 'und'; + transformedName = 'Language Neutral'; + } + + console.log(` ${locale} โ†’ ${transformedCode} (${transformedName || 'API lookup'}) [${isMaster ? 'MASTER' : 'regular'}]`); + }); + + console.log('\nโœ… All tests completed successfully!'); + + } catch (error) { + console.error('โŒ Test failed:', error); + console.error('Stack trace:', error.stack); + } finally { + if (connection) { + await connection.end(); + console.log('๐Ÿ”Œ Database connection closed'); + } + } +} + +// Run the test +testDrupalLocaleQueries().then(() => { + console.log('\n๐Ÿ Test completed'); + process.exit(0); +}).catch((error) => { + console.error('๐Ÿ’ฅ Test crashed:', error); + process.exit(1); +}); diff --git a/test-drupal-locales.js b/test-drupal-locales.js new file mode 100644 index 000000000..0d389fa93 --- /dev/null +++ b/test-drupal-locales.js @@ -0,0 +1,75 @@ +const { createLocale } = await import('./api/src/services/drupal/locales.service.js'); +const path = await import('path'); +const fs = await import('fs'); + +// Test configuration using upload-api database +const testProject = { + mysql: { + host: 'localhost', + user: 'root', + password: '', + database: 'riceuniversity2', + port: '3306' + } +}; + +const testDestinationStackId = 'test-drupal-locale-stack'; +const testProjectId = 'test-project-123'; + +async function testDrupalLocales() { + console.log('๐Ÿงช Testing Drupal Locale System...'); + console.log('๐Ÿ“Š Database:', testProject.mysql.database); + console.log('๐ŸŽฏ Stack ID:', testDestinationStackId); + + try { + // Test the createLocale function + await createLocale(testDestinationStackId, testProjectId, testProject); + + console.log('โœ… Locale creation completed successfully!'); + + // Check if files were created + const localesDir = path.join('./api/cmsMigrationData', testDestinationStackId, 'locales'); + + console.log('\n๐Ÿ“ Checking created files:'); + + // Check master-locale.json + const masterLocalePath = path.join(localesDir, 'master-locale.json'); + if (fs.existsSync(masterLocalePath)) { + const masterLocale = JSON.parse(fs.readFileSync(masterLocalePath, 'utf8')); + console.log('โœ… master-locale.json:', JSON.stringify(masterLocale, null, 2)); + } else { + console.log('โŒ master-locale.json not found'); + } + + // Check locales.json + const localesPath = path.join(localesDir, 'locales.json'); + if (fs.existsSync(localesPath)) { + const locales = JSON.parse(fs.readFileSync(localesPath, 'utf8')); + console.log('โœ… locales.json:', JSON.stringify(locales, null, 2)); + } else { + console.log('โŒ locales.json not found'); + } + + // Check language.json + const languagePath = path.join(localesDir, 'language.json'); + if (fs.existsSync(languagePath)) { + const language = JSON.parse(fs.readFileSync(languagePath, 'utf8')); + console.log('โœ… language.json:', JSON.stringify(language, null, 2)); + } else { + console.log('โŒ language.json not found'); + } + + } catch (error) { + console.error('โŒ Test failed:', error); + console.error('Stack trace:', error.stack); + } +} + +// Run the test +testDrupalLocales().then(() => { + console.log('\n๐Ÿ Test completed'); + process.exit(0); +}).catch((error) => { + console.error('๐Ÿ’ฅ Test crashed:', error); + process.exit(1); +}); diff --git a/test-drupal-locales.mjs b/test-drupal-locales.mjs new file mode 100644 index 000000000..1ab0e3bd3 --- /dev/null +++ b/test-drupal-locales.mjs @@ -0,0 +1,76 @@ +import { createLocale } from './api/src/services/drupal/locales.service.js'; +import path from 'path'; +import fs from 'fs'; + +// Test configuration using upload-api database +const testProject = { + mysql: { + host: 'localhost', + user: 'root', + password: '', + database: 'riceuniversity2', + port: '3306' + } +}; + +const testDestinationStackId = 'test-drupal-locale-stack'; +const testProjectId = 'test-project-123'; + +async function testDrupalLocales() { + console.log('๐Ÿงช Testing Drupal Locale System...'); + console.log('๐Ÿ“Š Database:', testProject.mysql.database); + console.log('๐ŸŽฏ Stack ID:', testDestinationStackId); + + try { + // Test the createLocale function + await createLocale(testDestinationStackId, testProjectId, testProject); + + console.log('โœ… Locale creation completed successfully!'); + + // Check if files were created + const localesDir = path.join('./api/cmsMigrationData', testDestinationStackId, 'locales'); + + console.log('\n๐Ÿ“ Checking created files:'); + console.log('๐Ÿ“‚ Directory:', localesDir); + + // Check master-locale.json + const masterLocalePath = path.join(localesDir, 'master-locale.json'); + if (fs.existsSync(masterLocalePath)) { + const masterLocale = JSON.parse(fs.readFileSync(masterLocalePath, 'utf8')); + console.log('โœ… master-locale.json:', JSON.stringify(masterLocale, null, 2)); + } else { + console.log('โŒ master-locale.json not found at:', masterLocalePath); + } + + // Check locales.json + const localesPath = path.join(localesDir, 'locales.json'); + if (fs.existsSync(localesPath)) { + const locales = JSON.parse(fs.readFileSync(localesPath, 'utf8')); + console.log('โœ… locales.json:', JSON.stringify(locales, null, 2)); + } else { + console.log('โŒ locales.json not found at:', localesPath); + } + + // Check language.json + const languagePath = path.join(localesDir, 'language.json'); + if (fs.existsSync(languagePath)) { + const language = JSON.parse(fs.readFileSync(languagePath, 'utf8')); + console.log('โœ… language.json:', JSON.stringify(language, null, 2)); + } else { + console.log('โŒ language.json not found at:', languagePath); + } + + } catch (error) { + console.error('โŒ Test failed:', error); + console.error('Stack trace:', error.stack); + } +} + +// Run the test +testDrupalLocales().then(() => { + console.log('\n๐Ÿ Test completed'); + process.exit(0); +}).catch((error) => { + console.error('๐Ÿ’ฅ Test crashed:', error); + process.exit(1); +}); diff --git a/ui/src/cmsData/legacyCms.json b/ui/src/cmsData/legacyCms.json index 9e866e611..d4f169925 100644 --- a/ui/src/cmsData/legacyCms.json +++ b/ui/src/cmsData/legacyCms.json @@ -92,60 +92,6 @@ } ] }, - { - "cms_id": "drupal", - "_metadata": { - "uid": "csb96887a2cfee9e8c" - }, - "title": "Drupal", - "description": "", - "group_name": "lightning", - "doc_url": { - "title": "https://www.drupal.org/", - "href": "https://www.drupal.org/" - }, - "parent": "Drupal", - "isactive": true, - "allowed_file_formats": [ - { - "fileformat_id": "zip", - "_metadata": { - "uid": "cs5d9c8914dc21ea80" - }, - "title": "Zip", - "description": "", - "group_name": "zip", - "isactive": true - } - ] - }, - { - "cms_id": "drupal v7", - "title": "Drupal v7", - "description": "", - "group_name": "lightning", - "doc_url": { - "title": "https://www.drupal.org/", - "href": "https://www.drupal.org/" - }, - "parent": "Drupal", - "isactive": true, - "allowed_file_formats": [ - { - "fileformat_id": "sql", - "title": "Sql", - "description": "", - "group_name": "sql", - "isactive": true, - "_metadata": { - "uid": "csceba83e388748bf1" - } - } - ], - "_metadata": { - "uid": "cs88cc83ef30625782" - } - }, { "cms_id": "drupal v8+", "title": "Drupal v8+", @@ -160,7 +106,7 @@ "allowed_file_formats": [ { "fileformat_id": "sql", - "title": "Sql", + "title": "SQL", "description": "", "group_name": "sql", "isactive": true, diff --git a/ui/src/components/Common/AddStack/addStack.tsx b/ui/src/components/Common/AddStack/addStack.tsx index 2d0c78417..0c1136281 100644 --- a/ui/src/components/Common/AddStack/addStack.tsx +++ b/ui/src/components/Common/AddStack/addStack.tsx @@ -1,5 +1,5 @@ // Libraries -import { useEffect, useState } from 'react'; +import { useEffect, useState, useRef } from 'react'; import { Form as FinalForm, Field as ReactFinalField } from 'react-final-form'; import { ModalBody, @@ -43,7 +43,8 @@ const AddStack = (props: any): JSX.Element => { const [isLoading, setIsLoading] = useState(true); const [allLocales, setAllLocales] = useState([]); const [addStackCMSData, setAddStackCMSData] = useState(defaultAddStackCMSData); - + const formRef = useRef(null); + /** * Handles the form submission. * @param formData - The form data. @@ -104,10 +105,27 @@ const AddStack = (props: any): JSX.Element => { created_at: key })) : []; + + // Detect master locale from source CMS + const sourceLocales = props?.newMigrationData?.destination_stack?.sourceLocale || props?.sourceLocales || []; + const masterLocale = sourceLocales.length > 0 ? sourceLocales[0] : 'en-us'; + + // Find matching Contentstack locale + const matchingLocale = rawMappedLocalesMapped.find(locale => + locale.value === masterLocale || + locale.value === `${masterLocale}-us` || + locale.value === `${masterLocale}-${masterLocale}` + ); + setAllLocales(rawMappedLocalesMapped); + + // Update form with correct master locale after locales are loaded + if (formRef.current && matchingLocale) { + formRef.current.change('locale', matchingLocale); + } }) .catch((err: any) => { - console.error(err); + console.error('โŒ Error fetching locales:', err); }); window.addEventListener('popstate', props?.closeModal); @@ -116,6 +134,25 @@ const AddStack = (props: any): JSX.Element => { window.removeEventListener('popstate', props?.closeModal); }; }, []); + + // Effect to update form with master locale when allLocales are loaded + useEffect(() => { + if (allLocales.length > 0 && formRef.current) { + const sourceLocales = props?.newMigrationData?.destination_stack?.sourceLocale || props?.sourceLocales || []; + const masterLocale = sourceLocales.length > 0 ? sourceLocales[0] : 'en-us'; + + // Find matching Contentstack locale + const matchingLocale = allLocales.find(locale => + locale.value === masterLocale || + locale.value === `${masterLocale}-us` || + locale.value === `${masterLocale}-${masterLocale}` + ); + + if (matchingLocale) { + formRef.current.change('locale', matchingLocale); + } + } + }, [allLocales, props?.newMigrationData?.destination_stack?.sourceLocale, props?.sourceLocales]); return ( <> @@ -126,7 +163,7 @@ const AddStack = (props: any): JSX.Element => { ) : ( - { @@ -148,7 +185,9 @@ const AddStack = (props: any): JSX.Element => { initialValues={{ locale: { label: 'English - United States', value: 'en-us' } }} - render={({ handleSubmit }): JSX.Element => { + render={({ handleSubmit, form }): JSX.Element => { + // Store form reference for updating values + formRef.current = form; return (
diff --git a/ui/src/components/DestinationStack/Actions/LoadLanguageMapper.tsx b/ui/src/components/DestinationStack/Actions/LoadLanguageMapper.tsx index 04ad10a81..f2a017b27 100644 --- a/ui/src/components/DestinationStack/Actions/LoadLanguageMapper.tsx +++ b/ui/src/components/DestinationStack/Actions/LoadLanguageMapper.tsx @@ -39,6 +39,9 @@ const Mapper = ({ isDisabled, isStackChanged, stack, + autoSelectedSourceLocale, + onLocaleStateUpdate, + parentLocaleState, }: { key: string; uid:string; @@ -48,11 +51,25 @@ const Mapper = ({ sourceOptions: Array<{ label: string; value: string }>; isDisabled: boolean; isStackChanged: boolean; - stack: IDropDown + stack: IDropDown; + autoSelectedSourceLocale?: { label: string; value: string } | null; + onLocaleStateUpdate?: (updater: (prev: ExistingFieldType) => ExistingFieldType) => void; + parentLocaleState?: ExistingFieldType; }) => { const [selectedMappings, setSelectedMappings] = useState<{ [key: string]: string }>({}); const [existingField, setExistingField] = useState({}); const [existingLocale, setexistingLocale] = useState({}); + + // ๐Ÿ”ฅ Sync with parent state when auto-mapping occurs + useEffect(() => { + if (parentLocaleState && Object.keys(parentLocaleState).length > 0) { + console.info('๐Ÿ”„ Syncing child existingLocale with parent state:', parentLocaleState); + setexistingLocale(prev => ({ + ...prev, + ...parentLocaleState + })); + } + }, [parentLocaleState]); const [selectedCsOptions, setselectedCsOption] = useState([]); const [selectedSourceOption, setselectedSourceOption] = useState([]); const [csOptions, setcsOptions] = useState(options); @@ -91,6 +108,7 @@ const Mapper = ({ } }, [sourceOptions]); + useEffect(() => { const formattedoptions = options?.filter( (item: { label: string; value: string }) => @@ -170,7 +188,25 @@ const Mapper = ({ }, [cmsLocaleOptions]); - + + // ๐Ÿš€ Auto-select single source locale in the master locale row + // This runs after the clearing logic to ensure auto-selection persists + useEffect(() => { + if (autoSelectedSourceLocale && cmsLocaleOptions?.length > 0) { + const masterLocaleRow = cmsLocaleOptions.find(locale => locale.value === 'master_locale'); + if (masterLocaleRow) { + // Use setTimeout to ensure this runs after other state updates + setTimeout(() => { + const updater = (prev: ExistingFieldType) => ({ + ...prev, + [masterLocaleRow.label]: autoSelectedSourceLocale + }); + setexistingLocale(updater); + onLocaleStateUpdate?.(updater); + }, 0); + } + } + }, [autoSelectedSourceLocale, cmsLocaleOptions, isStackChanged]); // function for change select value const handleSelectedCsLocale = ( @@ -510,6 +546,7 @@ const Mapper = ({ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { const newMigrationData = useSelector((state: RootState) => state?.migration?.newMigrationData); + const dispatch = useDispatch(); const [options, setoptions] = useState<{ label: string; value: string }[]>([]); const [cmsLocaleOptions, setcmsLocaleOptions] = useState<{ label: string; value: string }[]>([]); const [sourceLocales, setsourceLocales] = useState<{ label: string; value: string }[]>([]); @@ -518,6 +555,8 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { const [previousStack, setPreviousStack] = useState(); const [isStackChanged, setisStackChanged] = useState(false); const [stackValue, setStackValue] = useState(stack?.value) + const [autoSelectedSourceLocale, setAutoSelectedSourceLocale] = useState<{ label: string; value: string } | null>(null); + const [mapperLocaleState, setMapperLocaleState] = useState({}); const prevStackRef:any = useRef(null); @@ -531,6 +570,263 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { prevStackRef.current = stack; }, [stack]); + // Smart locale mapping function - works for all CMS platforms + const getSmartLocaleMapping = (sourceLocaleCode: string, availableContentstackLocales: { label: string; value: string }[]): string => { + // First, try direct match + const directMatch = availableContentstackLocales.find(locale => locale.value === sourceLocaleCode); + if (directMatch) { + return sourceLocaleCode; + } + + // Smart mapping for common locale patterns + const commonMappings: { [key: string]: string[] } = { + 'en': ['en-us', 'en-gb', 'en-au', 'en-ca'], + 'es': ['es-es', 'es-mx', 'es-ar', 'es-co'], + 'fr': ['fr-fr', 'fr-ca', 'fr-be', 'fr-ch'], + 'de': ['de-de', 'de-at', 'de-ch'], + 'it': ['it-it'], + 'pt': ['pt-pt', 'pt-br'], + 'ja': ['ja-jp'], + 'zh': ['zh-cn', 'zh-tw', 'zh-hk'], + 'ar': ['ar-ae', 'ar-sa', 'ar-eg', 'ar-ma'], + 'hi': ['hi-in'], + 'ru': ['ru-ru'], + 'ko': ['ko-kr'], + 'nl': ['nl-nl', 'nl-be'], + 'sv': ['sv-se'], + 'no': ['nb-no', 'nn-no'], + 'da': ['da-dk'], + 'fi': ['fi-fi'], + 'pl': ['pl-pl'], + 'tr': ['tr-tr'], + 'th': ['th-th'], + 'vi': ['vi-vn'], + 'uk': ['uk-ua'], + 'cs': ['cs-cz'], + 'hu': ['hu-hu'], + 'ro': ['ro-ro'], + 'bg': ['bg-bg'], + 'hr': ['hr-hr'], + 'sk': ['sk-sk'], + 'sl': ['sl-si'], + 'et': ['et-ee'], + 'lv': ['lv-lv'], + 'lt': ['lt-lt'], + 'eg': ['ar-eg', 'en-eg', 'ar-ae'] // Egyptian - prefer Arabic Egypt + }; + + const possibleMappings = commonMappings[sourceLocaleCode] || []; + + // Find the first available mapping + for (const candidate of possibleMappings) { + const match = availableContentstackLocales.find(locale => locale.value === candidate); + if (match) { + return candidate; + } + } + + // Fallback to en-us if available, otherwise first available locale + const fallback = availableContentstackLocales.find(locale => locale.value === 'en-us'); + return fallback ? 'en-us' : availableContentstackLocales[0]?.value || sourceLocaleCode; + }; + + // ๐Ÿš€ UNIVERSAL LOCALE AUTO-MAPPING (All CMS platforms) + // Handles both single and multiple locale scenarios + useEffect(() => { + const sourceLocale = newMigrationData?.destination_stack?.sourceLocale?.map((item) => ({ + label: item, + value: item + })); + + const allLocales: { label: string; value: string }[] = Object?.entries( + newMigrationData?.destination_stack?.csLocale ?? {} + ).map(([key]) => ({ + label: key, + value: key + })); + + // ๐Ÿ”„ Improved stack change detection + const stackHasChanged = currentStack?.uid !== previousStack?.uid || + isStackChanged || + previousStack === undefined; // Also trigger when no previous stack + + // โœ… Declare keys before using it + const keys = Object?.keys(newMigrationData?.destination_stack?.localeMapping || {})?.find( key => key === `${newMigrationData?.destination_stack?.selectedStack?.master_locale}-master_locale`); + + // ๐Ÿ” Debug logging to understand what's happening + if (sourceLocale && allLocales) { + console.info('๐Ÿ” Auto-mapping Debug Info:'); + console.info(' Source Locales:', sourceLocale); + console.info(' Destination Locales:', allLocales); + console.info(' Current Stack:', currentStack?.uid); + console.info(' Previous Stack:', previousStack?.uid); + console.info(' Is Stack Changed:', isStackChanged); + console.info(' Stack Has Changed (Improved):', stackHasChanged); + console.info(' Locale Mapping:', newMigrationData?.destination_stack?.localeMapping); + console.info(' Project Step:', newMigrationData?.project_current_step); + console.info(' Master Locale:', stack?.master_locale); + console.info(' Keys Found:', keys); + + // Check which condition will be met + const isSingleLocale = sourceLocale?.length === 1; + const isMultiLocale = sourceLocale?.length > 1; + const hasAllLocales = allLocales?.length > 0; + const hasCmsLocaleOptions = cmsLocaleOptions?.length > 0; + const shouldAutoMap = (Object?.entries(newMigrationData?.destination_stack?.localeMapping || {})?.length === 0 || + !keys || + stackHasChanged); + const isCorrectStep = newMigrationData?.project_current_step <= 2; + + console.info('๐Ÿ” Condition Check:'); + console.info(' Is Single Locale:', isSingleLocale); + console.info(' Is Multi Locale:', isMultiLocale); + console.info(' Has All Locales:', hasAllLocales); + console.info(' Has CMS Locale Options:', hasCmsLocaleOptions); + console.info(' Should Auto Map:', shouldAutoMap); + console.info(' Is Correct Step:', isCorrectStep); + } + + // โœ… EXISTING: Single locale auto-mapping (PRESERVED) + // Enhanced condition: Also trigger on stack changes for existing templates + const shouldAutoMapSingle = (Object?.entries(newMigrationData?.destination_stack?.localeMapping || {})?.length === 0 || + !keys || + stackHasChanged); + + // Clear existing mappings when stack changes to allow fresh auto-mapping + if (stackHasChanged && Object?.entries(newMigrationData?.destination_stack?.localeMapping || {})?.length > 0) { + const newMigrationDataObj: INewMigration = { + ...newMigrationData, + destination_stack: { + ...newMigrationData?.destination_stack, + localeMapping: {} // Clear existing mappings for fresh auto-mapping + } + }; + dispatch(updateNewMigrationData(newMigrationDataObj)); + return; // Exit early to let the cleared state trigger auto-mapping in next render + } + + if (sourceLocale?.length === 1 && + allLocales?.length > 0 && + shouldAutoMapSingle && + newMigrationData?.project_current_step <= 2) { + + const singleSourceLocale = sourceLocale[0]; + const smartDestinationLocale = getSmartLocaleMapping(singleSourceLocale.value, allLocales); + + // Set the auto-selected source locale for the Mapper component + setAutoSelectedSourceLocale({ + label: singleSourceLocale.value, // Source locale (e.g., "en") + value: singleSourceLocale.value // Source locale for dropdown selection + }); + + // Set the mapping in Redux state + const autoMapping = { + [`${stack?.master_locale}-master_locale`]: stack?.master_locale, + [singleSourceLocale.value]: smartDestinationLocale + }; + + const newMigrationDataObj: INewMigration = { + ...newMigrationData, + destination_stack: { + ...newMigrationData?.destination_stack, + localeMapping: autoMapping + } + }; + dispatch(updateNewMigrationData(newMigrationDataObj)); + + // Reset stack changed flag after auto-mapping + if (isStackChanged) { + setisStackChanged(false); + } + } + // ๐Ÿ†• NEW: Enhanced multi-locale auto-mapping + // Enhanced condition: Also trigger on stack changes for existing templates + else if (sourceLocale?.length > 1 && + allLocales?.length > 0 && + cmsLocaleOptions?.length > 0 && // โœ… CRITICAL: Wait for cmsLocaleOptions to be ready + (Object?.entries(newMigrationData?.destination_stack?.localeMapping || {})?.length === 0 || + !keys || + stackHasChanged) && + newMigrationData?.project_current_step <= 2) { + + // console.info('๐Ÿš€ EXECUTING Multi-locale auto-mapping...'); + // console.info(' Source Locales for matching:', sourceLocale); + // console.info(' Available Destination Locales:', allLocales.slice(0, 10)); + + // Build auto-mapping for exact matches (case-insensitive) + const autoMapping: Record = { + [`${stack?.master_locale}-master_locale`]: stack?.master_locale + }; + + sourceLocale.forEach(source => { + // Case-insensitive exact matching only + const exactMatch = allLocales.find(dest => + source.value.toLowerCase() === dest.value.toLowerCase() + ); + + // console.info(` Checking ${source.value} -> Found match:`, exactMatch?.value || 'No match'); + + if (exactMatch) { + autoMapping[source.value] = exactMatch.value; + } + }); + + // console.info('๐ŸŽฏ Final Auto-mapping Result:', autoMapping); + + // Update Redux state with auto-mappings + const newMigrationDataObj: INewMigration = { + ...newMigrationData, + destination_stack: { + ...newMigrationData?.destination_stack, + localeMapping: autoMapping + } + }; + dispatch(updateNewMigrationData(newMigrationDataObj)); + + // console.info('โœ… Redux state updated with auto-mapping'); + + // ๐Ÿ”ฅ CRITICAL FIX: Update existingLocale state for dropdown display + // The dropdown reads from existingLocale, not from Redux localeMapping + const updatedExistingLocale: ExistingFieldType = {}; + + // Map each auto-mapped source locale to the dropdown state + sourceLocale.forEach(source => { + if (autoMapping[source.value]) { + // Find the corresponding cmsLocaleOptions index for this source locale + const localeRow = cmsLocaleOptions?.find(locale => { + const isDirectMatch = locale.value === source.value; + const isMasterMatch = locale.value === 'master_locale' && source.value === 'en'; + return isDirectMatch || isMasterMatch; + }); + + if (localeRow) { + updatedExistingLocale[localeRow.label] = { + label: source.value, + value: source.value + }; + } + } + }); + + // Update the existingLocale state + setMapperLocaleState(prev => ({ + ...prev, + ...updatedExistingLocale + })); + + // Clear auto-selected source locale for multi-locale scenario + setAutoSelectedSourceLocale(null); + + // Reset stack changed flag after auto-mapping + if (isStackChanged) { + setisStackChanged(false); + } + } + else { + setAutoSelectedSourceLocale(null); + } + }, [newMigrationData?.destination_stack?.sourceLocale, newMigrationData?.destination_stack?.csLocale, newMigrationData?.project_current_step, stack?.master_locale, isStackChanged, currentStack?.uid, previousStack?.uid, newMigrationData?.destination_stack?.selectedStack, cmsLocaleOptions]); + useEffect(() => { const fetchData = async () => { try { @@ -547,6 +843,8 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { })); setsourceLocales(sourceLocale); setoptions(allLocales); + + // Original logic for multiple locales or existing mappings const keys = Object?.keys(newMigrationData?.destination_stack?.localeMapping || {})?.find( key => key === `${newMigrationData?.destination_stack?.selectedStack?.master_locale}-master_locale`); if((Object?.entries(newMigrationData?.destination_stack?.localeMapping)?.length === 0 || !keys || @@ -659,6 +957,9 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { isDisabled={newMigrationData?.project_current_step > 2} isStackChanged={isStackChanged} stack={stack ?? DEFAULT_DROPDOWN} + autoSelectedSourceLocale={autoSelectedSourceLocale} + onLocaleStateUpdate={setMapperLocaleState} + parentLocaleState={mapperLocaleState} /> } type="Secondary" diff --git a/ui/src/components/DestinationStack/Actions/LoadStacks.tsx b/ui/src/components/DestinationStack/Actions/LoadStacks.tsx index c5dfa427b..841c3ea50 100644 --- a/ui/src/components/DestinationStack/Actions/LoadStacks.tsx +++ b/ui/src/components/DestinationStack/Actions/LoadStacks.tsx @@ -199,48 +199,61 @@ const LoadStacks = (props: LoadFileFormatProps) => { }; const fetchData = async () => { - try { - if (allStack?.length <= 0) { - setAllStack(loadingOption); - const stackData = await getAllStacksInOrg(selectedOrganisation?.value, ''); // org id will always be there - const csLocales = await getStackLocales(selectedOrganisation?.value); - const stackArray = validateArray(stackData?.data?.stacks) - ? stackData?.data?.stacks?.map((stack: StackResponse) => ({ - label: stack?.name, - value: stack?.api_key, - uid: stack?.api_key, - master_locale: stack?.master_locale, - locales: stack?.locales, - created_at: stack?.created_at, - isNewStack: newStackCreated, - isDisabled: newMigrationDataRef?.current?.destination_stack?.migratedStacks?.includes( - stack?.api_key - ) - })) - : []; - - stackArray.sort( - (a: IDropDown, b: IDropDown) => - new Date(b?.created_at)?.getTime() - new Date(a?.created_at)?.getTime() - ); + try { + if (allStack?.length <= 0) { + setAllStack(loadingOption); + const stackData = await getAllStacksInOrg(selectedOrganisation?.value, ''); + const csLocales = await getStackLocales(selectedOrganisation?.value); + + const stackArray = validateArray(stackData?.data?.stacks) + ? stackData?.data?.stacks?.map((stack: StackResponse) => ({ + label: stack?.name, + value: stack?.api_key, + uid: stack?.api_key, + master_locale: stack?.master_locale, + locales: stack?.locales, + created_at: stack?.created_at, + isNewStack: newStackCreated, + isDisabled: newMigrationDataRef?.current?.destination_stack?.migratedStacks?.includes( + stack?.api_key + ) + })) + : []; + + stackArray.sort( + (a: IDropDown, b: IDropDown) => + new Date(b?.created_at)?.getTime() - new Date(a?.created_at)?.getTime() + ); + + setAllStack(stackArray); + + // โœ… Auto-select if only one option + if (stackArray.length === 1) { + const onlyStack = stackArray[0]; + setSelectedStack(onlyStack); - setAllStack(stackArray); - //Set selected Stack + const newMigrationDataObj: INewMigration = { + ...newMigrationData, + destination_stack: { + ...newMigrationData?.destination_stack, + selectedStack: onlyStack, + stackArray: stackArray + } + }; + dispatch(updateNewMigrationData(newMigrationDataObj)); + } else { + // If multiple, restore previously selected const selectedStackData = validateArray(stackArray) - ? stackArray.find((stack: IDropDown) => { - return stack?.value === newMigrationData?.destination_stack?.selectedStack?.value; - }) + ? stackArray.find( + (stack: IDropDown) => + stack?.value === newMigrationData?.destination_stack?.selectedStack?.value + ) : null; - // if (stackData?.data?.stacks?.length === 0 && (!stackData?.data?.stack)) { - // setIsError(true); - // setErrorMessage("Please create new stack there is no stack available"); - // } if (selectedStackData) { setSelectedStack(selectedStackData); setNewStackCreated(false); const newMigrationDataObj: INewMigration = { - // ...newMigrationDataRef?.current, ...newMigrationData, destination_stack: { ...newMigrationData?.destination_stack, @@ -248,24 +261,25 @@ const LoadStacks = (props: LoadFileFormatProps) => { stackArray: stackArray } }; - // Dispatch the updated migration data to Redux dispatch(updateNewMigrationData(newMigrationDataObj)); } - const newMigrationDataObj: INewMigration = { - ...newMigrationDataRef?.current, - //...newMigrationData, - destination_stack: { - ...newMigrationDataRef?.current?.destination_stack, - csLocale: csLocales?.data?.locales - } - }; - // Dispatch the updated migration data to Redux - dispatch(updateNewMigrationData(newMigrationDataObj)); } - } catch (error) { - return error; + + // update locales in Redux + const newMigrationDataObj: INewMigration = { + ...newMigrationDataRef?.current, + destination_stack: { + ...newMigrationDataRef?.current?.destination_stack, + csLocale: csLocales?.data?.locales + } + }; + dispatch(updateNewMigrationData(newMigrationDataObj)); } - }; + } catch (error) { + return error; + } +}; + const handleCreateNewStack = () => { cbModal({ @@ -278,6 +292,8 @@ const LoadStacks = (props: LoadFileFormatProps) => { onSubmit={handleOnSave} defaultValues={defaultStack} selectedOrganisation={selectedOrganisation?.value} + sourceLocales={newMigrationData?.destination_stack?.sourceLocale} + newMigrationData={newMigrationData} {...props} /> ), @@ -364,7 +380,7 @@ const LoadStacks = (props: LoadFileFormatProps) => { && (
-
Language Mapping
+
Language Mappings
{ const cmsType = !isEmptyString(newMigrationData?.legacy_cms?.selectedCms?.parent) ? newMigrationData?.legacy_cms?.selectedCms?.parent : newMigrationData?.legacy_cms?.uploadedFile?.cmsType; const filePath = newMigrationData?.legacy_cms?.uploadedFile?.file_details?.localPath?.toLowerCase(); - const fileFormat: string = newMigrationData?.legacy_cms?.selectedFileFormat?.title?.toLowerCase(); + + // Get file format from selectedFileFormat or detect from upload response + let fileFormat: string = newMigrationData?.legacy_cms?.selectedFileFormat?.title?.toLowerCase(); + + // If fileFormat is not set, try to detect from upload response + if (!fileFormat && newMigrationData?.legacy_cms?.uploadedFile?.file_details?.isSQL) { + fileFormat = 'sql'; + } if(! isEmptyString(selectedCard?.fileformat_id) && selectedCard?.fileformat_id !== fileFormat && newMigrationData?.project_current_step > 1){ setFileIcon(selectedCard?.title); } else { @@ -72,16 +79,33 @@ const LoadFileFormat = (props: LoadFileFormatProps) => { ); } - const isFormatValid = filteredCmsData[0]?.allowed_file_formats?.find( - (format: ICardType) => { - const isValid = format?.fileformat_id?.toLowerCase() === fileFormat?.toLowerCase(); - return isValid; - } - ); + // Special handling for Drupal SQL format + const isDrupal = cmsType?.toLowerCase() === 'drupal'; + const isSQLFormat = fileFormat?.toLowerCase() === 'sql'; + + let isFormatValid = false; + + if (isDrupal && isSQLFormat) { + // For Drupal, automatically accept SQL format + isFormatValid = true; + } else { + // For other CMS types, use the original validation logic + const foundFormat = filteredCmsData[0]?.allowed_file_formats?.find( + (format: ICardType) => { + const isValid = format?.fileformat_id?.toLowerCase() === fileFormat?.toLowerCase(); + return isValid; + } + ); + isFormatValid = !!foundFormat; + } if (!isFormatValid) { setIsError(true); setError('File format does not support, please add the correct file format.'); + } else { + // Clear any previous errors + setIsError(false); + setError(''); } const selectedFileFormatObj = { @@ -92,8 +116,12 @@ const LoadFileFormat = (props: LoadFileFormatProps) => { title: fileFormat === 'zip' ? fileFormat?.charAt?.(0)?.toUpperCase() + fileFormat?.slice?.(1) : fileFormat?.toUpperCase() } - - setFileIcon(fileFormat === 'zip' ? fileFormat?.charAt?.(0).toUpperCase() + fileFormat?.slice?.(1) : fileFormat?.toUpperCase()); + // Set file icon based on format + if (isDrupal && isSQLFormat) { + setFileIcon('SQL'); + } else { + setFileIcon(fileFormat === 'zip' ? fileFormat?.charAt?.(0).toUpperCase() + fileFormat?.slice?.(1) : fileFormat?.toUpperCase()); + } } } catch (error) { diff --git a/ui/src/components/LegacyCms/Actions/LoadSelectCms.tsx b/ui/src/components/LegacyCms/Actions/LoadSelectCms.tsx index 7a01428ae..0b6d15c86 100644 --- a/ui/src/components/LegacyCms/Actions/LoadSelectCms.tsx +++ b/ui/src/components/LegacyCms/Actions/LoadSelectCms.tsx @@ -104,6 +104,13 @@ const LoadSelectCms = (props: LoadSelectCmsProps) => { } }; + // Debug logging for selectedFileFormat setting + console.info('๐Ÿ”ง === LOAD SELECT CMS DEBUG ==='); + console.info('๐Ÿ“‹ filteredCmsData[0]:', filteredCmsData[0]); + console.info('๐Ÿ“‹ allowed_file_formats[0]:', filteredCmsData[0]?.allowed_file_formats[0]); + console.info('๐Ÿ“‹ selectedFileFormat being set:', newMigrationDataObj.legacy_cms.selectedFileFormat); + console.info('================================'); + //dispatch(updateNewMigrationData(newMigrationDataObj)); setCmsData(filteredCmsData); diff --git a/ui/src/components/LegacyCms/Actions/LoadUploadFile.tsx b/ui/src/components/LegacyCms/Actions/LoadUploadFile.tsx index 817219684..1528815a6 100644 --- a/ui/src/components/LegacyCms/Actions/LoadUploadFile.tsx +++ b/ui/src/components/LegacyCms/Actions/LoadUploadFile.tsx @@ -137,14 +137,45 @@ const LoadUploadFile = (props: LoadUploadFileProps) => { awsRegion: data?.file_details?.awsData?.awsRegion, bucketName: data?.file_details?.awsData?.bucketName, buketKey: data?.file_details?.awsData?.buketKey + }, + isSQL: data?.file_details?.isSQL, + mySQLDetails: { + host: data?.file_details?.mySQLDetails?.host, + user: data?.file_details?.mySQLDetails?.user, + password: data?.file_details?.mySQLDetails?.password, + database: data?.file_details?.mySQLDetails?.database, + port: data?.file_details?.mySQLDetails?.port + }, + drupalAssetsUrl: { + base_url: data?.file_details?.drupalAssetsUrl?.base_url, + public_path: data?.file_details?.drupalAssetsUrl?.public_path } }, cmsType: data?.cmsType - } } }; + // For Drupal SQL files, ensure selectedFileFormat is set in the same update + if (status === 200 && data?.file_details?.isSQL && data?.file_details?.cmsType === 'drupal') { + console.info('๐Ÿ”ง === LOAD UPLOAD FILE DEBUG ==='); + console.info('๐Ÿ“‹ Setting selectedFileFormat for Drupal SQL in combined update'); + + // Add selectedFileFormat to the existing newMigrationDataObj + newMigrationDataObj.legacy_cms.selectedFileFormat = { + fileformat_id: 'sql', + title: 'SQL', + description: '', + group_name: 'sql', + isactive: true + }; + + console.info('๐Ÿ“‹ Combined newMigrationDataObj:', newMigrationDataObj); + console.info('================================'); + } + + console.info('๐Ÿ“‹ Dispatching combined newMigrationDataObj:', newMigrationDataObj); + console.info('๐Ÿ” DEBUG: uploadedFile.isValidated in dispatch:', newMigrationDataObj.legacy_cms.uploadedFile.isValidated); dispatch(updateNewMigrationData(newMigrationDataObj)); if (status === 200) { @@ -153,10 +184,13 @@ const LoadUploadFile = (props: LoadUploadFileProps) => { setIsDisabled(true); + + if ( !isEmptyString(newMigrationData?.legacy_cms?.affix) && !isEmptyString(newMigrationData?.legacy_cms?.selectedCms?.cms_id) && - !isEmptyString(newMigrationData?.legacy_cms?.selectedFileFormat?.fileformat_id) + (data?.file_details?.isSQL || + !isEmptyString(newMigrationData?.legacy_cms?.selectedFileFormat?.fileformat_id)) ) { props.handleStepChange(props?.currentStep, true); } diff --git a/ui/src/components/LegacyCms/index.tsx b/ui/src/components/LegacyCms/index.tsx index 347880869..b61cf1557 100644 --- a/ui/src/components/LegacyCms/index.tsx +++ b/ui/src/components/LegacyCms/index.tsx @@ -211,17 +211,39 @@ const LegacyCMSComponent = forwardRef(({ legacyCMSData, isCompleted, handleOnAll },[newMigrationData]); useEffect(()=>{ - if(! isEmptyString(newMigrationData?.legacy_cms?.affix) + // Debug logging for completion check + console.info('๐Ÿ” === LEGACY CMS COMPLETION CHECK ==='); + console.info('๐Ÿ“Š Current state:', { + affix: newMigrationData?.legacy_cms?.affix, + selectedFileFormat: newMigrationData?.legacy_cms?.selectedFileFormat, + selectedCms: newMigrationData?.legacy_cms?.selectedCms, + uploadedFile: newMigrationData?.legacy_cms?.uploadedFile, + isValidated: newMigrationData?.legacy_cms?.uploadedFile?.isValidated + }); + + console.info('โœ… Individual conditions:'); + console.info(' - affix exists:', !isEmptyString(newMigrationData?.legacy_cms?.affix)); + console.info(' - selectedFileFormat.title exists:', !isEmptyString(newMigrationData?.legacy_cms?.selectedFileFormat?.title)); + console.info(' - selectedCms.title exists:', !isEmptyString(newMigrationData?.legacy_cms?.selectedCms?.title)); + console.info(' - uploadedFile.isValidated:', newMigrationData?.legacy_cms?.uploadedFile?.isValidated); + + const allConditionsMet = !isEmptyString(newMigrationData?.legacy_cms?.affix) && !isEmptyString(newMigrationData?.legacy_cms?.selectedFileFormat?.title) && ! isEmptyString(newMigrationData?.legacy_cms?.selectedCms?.title) && - newMigrationData?.legacy_cms?.uploadedFile?.isValidated){ + newMigrationData?.legacy_cms?.uploadedFile?.isValidated; + + console.info('๐ŸŽฏ All conditions met:', allConditionsMet); + console.info('====================================='); + + if(allConditionsMet){ + console.info('โœ… Setting all steps completed to TRUE'); setIsAllStepsCompleted(true); handleAllStepsComplete(true); } else{ + console.info('โŒ Setting all steps completed to FALSE'); setIsAllStepsCompleted(false); handleAllStepsComplete(false); - } },[newMigrationData,isAllStepsCompleted]) diff --git a/ui/src/components/MigrationFlowHeader/index.tsx b/ui/src/components/MigrationFlowHeader/index.tsx index 196c35ad9..f245212d0 100644 --- a/ui/src/components/MigrationFlowHeader/index.tsx +++ b/ui/src/components/MigrationFlowHeader/index.tsx @@ -47,19 +47,28 @@ const MigrationFlowHeader = ({ useEffect(() => { fetchProject(); - }, [selectedOrganisation?.value, params?.projectId]); + }, [selectedOrganisation?.value, params?.projectId, projectData?.current_step]); /******** Function to get project ********/ /** * Fetch the project details project name and current step. */ const fetchProject = async () => { + if (!projectData?.name || !projectData?.current_step) return; + setProjectName(projectData?.name); setCurrentStep(projectData?.current_step); - //Navigate to lastest or active Step - const url = `/projects/${params?.projectId}/migration/steps/${projectData?.current_step}`; - navigate(url, { replace: true }); + // Check if URL step matches database step + const urlStep = parseInt(params?.stepId || '1'); + const dbStep = projectData?.current_step; + + // Only navigate if there's a mismatch and we have valid data + if (urlStep !== dbStep && dbStep && params?.projectId) { + console.info(`๐Ÿ”„ Step mismatch detected: URL shows step ${urlStep}, DB shows step ${dbStep}. Redirecting...`); + const url = `/projects/${params?.projectId}/migration/steps/${dbStep}`; + navigate(url, { replace: true }); + } }; let stepValue; diff --git a/ui/src/context/app/app.interface.ts b/ui/src/context/app/app.interface.ts index 98d326872..a589a1591 100644 --- a/ui/src/context/app/app.interface.ts +++ b/ui/src/context/app/app.interface.ts @@ -57,6 +57,18 @@ export interface FileDetails { buketKey?: string; }; filePath?: string | undefined; + mySQLDetails?: { + host?: string; + user?: string; + password?: string; + database?: string; + port?: number; + }; + drupalAssetsUrl?: { + base_url?: string; + public_path?: string; + }; + isSQL?: boolean; } export interface IFile { id?: string; @@ -314,7 +326,19 @@ export const DEFAULT_FILE: IFile = { awsRegion: '', bucketName: '', buketKey: '' - } + }, + mySQLDetails: { + host: '', + user: '', + password: '', + database: '', + port: 0 + }, + drupalAssetsUrl: { + base_url: '', + public_path: '' + }, + isSQL: false }, isValidated: false, reValidate: false, diff --git a/ui/src/pages/Migration/index.tsx b/ui/src/pages/Migration/index.tsx index 07f6d8ea0..472c8e012 100644 --- a/ui/src/pages/Migration/index.tsx +++ b/ui/src/pages/Migration/index.tsx @@ -503,8 +503,9 @@ const Migration = () => { await affixConfirmation(selectedOrganisation?.value, projectId, { affix_confirmation: true }); - await updateFileFormatData(selectedOrganisation?.value, projectId, { + const fileFormatData = { file_format: + newMigrationData?.legacy_cms?.selectedFileFormat?.fileformat_id?.toString() || newMigrationData?.legacy_cms?.selectedCms?.allowed_file_formats[0]?.fileformat_id?.toString(), file_path: newMigrationData?.legacy_cms?.uploadedFile?.file_details?.localPath, is_fileValid: newMigrationData?.legacy_cms?.uploadedFile?.isValidated, @@ -513,31 +514,48 @@ const Migration = () => { awsRegion: newMigrationData?.legacy_cms?.uploadedFile?.file_details?.awsData?.awsRegion, bucketName: newMigrationData?.legacy_cms?.uploadedFile?.file_details?.awsData?.bucketName, buketKey: newMigrationData?.legacy_cms?.uploadedFile?.file_details?.awsData?.buketKey - } - }); - const res = await updateCurrentStepData(selectedOrganisation.value, projectId); + }, + mySQLDetails: newMigrationData?.legacy_cms?.uploadedFile?.file_details?.mySQLDetails + }; + + console.info('๐Ÿ” === FILE FORMAT API CALL DEBUG ==='); + console.info('๐Ÿ“‹ Sending fileFormatData:', fileFormatData); + console.info('๐Ÿ“‹ mySQLDetails:', newMigrationData?.legacy_cms?.uploadedFile?.file_details?.mySQLDetails); + console.info('====================================='); + + await updateFileFormatData(selectedOrganisation?.value, projectId, fileFormatData); + + // Check current project state before attempting step update + const currentProjectData = await getMigrationData(selectedOrganisation?.value, projectId); + const currentStep = currentProjectData?.data?.current_step; + + console.info(`๐Ÿ” Current project step: ${currentStep}, attempting to proceed...`); + + // Only call updateCurrentStepData if we're at step 1 (to avoid 400 error) + let res; + if (currentStep === 1) { + res = await updateCurrentStepData(selectedOrganisation.value, projectId); + } else { + console.info(`โš ๏ธ Project already at step ${currentStep}, skipping step update`); + res = { status: 200 }; // Simulate success to continue flow + } if (res?.status === 200) { setIsLoading(false); // Check if stack is already selected if (newMigrationData?.destination_stack?.selectedStack?.value) { const url = `/projects/${projectId}/migration/steps/3`; - - await updateCurrentStepData(selectedOrganisation?.value, projectId); - handleStepChange(2); navigate(url, { replace: true }); } else { const url = `/projects/${projectId}/migration/steps/2`; - await updateCurrentStepData(selectedOrganisation?.value, projectId); - handleStepChange(1); navigate(url, { replace: true }); } } else { setIsLoading(false); Notification({ - notificationContent: { text: res?.data?.error?.message }, + notificationContent: { text: res?.data?.error?.message || 'Failed to update project step' }, type: 'error' }); } diff --git a/ui/src/store/slice/migrationDataSlice.tsx b/ui/src/store/slice/migrationDataSlice.tsx index 55de090b4..693706961 100644 --- a/ui/src/store/slice/migrationDataSlice.tsx +++ b/ui/src/store/slice/migrationDataSlice.tsx @@ -27,7 +27,19 @@ const migrationSlice = createSlice({ state.newMigrationData = action?.payload; }, updateNewMigrationData: (state, action) => { - state.newMigrationData = { ...state?.newMigrationData, ...action?.payload }; + // Deep merge for nested objects + state.newMigrationData = { + ...state?.newMigrationData, + ...action?.payload, + legacy_cms: { + ...state?.newMigrationData?.legacy_cms, + ...action?.payload?.legacy_cms, + uploadedFile: { + ...state?.newMigrationData?.legacy_cms?.uploadedFile, + ...action?.payload?.legacy_cms?.uploadedFile + } + } + }; }, setMigrationData: (state, action) => { state.migrationData = action?.payload; diff --git a/upload-api/README.md b/upload-api/README.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/upload-api/drupalMigrationData/drupalSchema/article.json b/upload-api/drupalMigrationData/drupalSchema/article.json new file mode 100644 index 000000000..a4d3d674e --- /dev/null +++ b/upload-api/drupalMigrationData/drupalSchema/article.json @@ -0,0 +1,263 @@ +{ + "title": "article", + "uid": "article", + "schema": [ + { + "uid": "title", + "otherCmsField": "title", + "otherCmsType": "text", + "contentstackField": "title", + "contentstackFieldUid": "title", + "contentstackFieldType": "text", + "backupFieldType": "text", + "backupFieldUid": "title", + "advanced": { + "mandatory": true + } + }, + { + "uid": "url", + "otherCmsField": "url", + "otherCmsType": "text", + "contentstackField": "Url", + "contentstackFieldUid": "url", + "contentstackFieldType": "url", + "backupFieldType": "url", + "backupFieldUid": "url", + "advanced": { + "mandatory": true + } + }, + { + "uid": "body", + "otherCmsField": "Body", + "otherCmsType": "text_with_summary", + "contentstackField": "Body", + "contentstackFieldUid": "body", + "contentstackFieldType": "json", + "backupFieldType": "html", + "backupFieldUid": "body", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_author", + "otherCmsField": "Author", + "otherCmsType": "entity_reference", + "contentstackField": "Author", + "contentstackFieldUid": "field_author_target_id", + "contentstackFieldType": "reference", + "backupFieldType": "reference", + "backupFieldUid": "field_author_target_id", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [ + "profile" + ], + "description": "" + } + }, + { + "uid": "field_external_link", + "otherCmsField": "External Link", + "otherCmsType": "string", + "contentstackField": "External Link", + "contentstackFieldUid": "field_external_link", + "contentstackFieldType": "single_line_text", + "backupFieldType": "multi_line_text", + "backupFieldUid": "field_external_link", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [ + "fact", + "institute", + "link", + "list", + "page", + "profile", + "school", + "tour", + "video" + ], + "description": "" + } + }, + { + "uid": "field_formatted_subtitle", + "otherCmsField": "Formatted Subtitle", + "otherCmsType": "text", + "contentstackField": "Formatted Subtitle", + "contentstackFieldUid": "field_formatted_subtitle", + "contentstackFieldType": "single_line_text", + "backupFieldType": "multi_line_text", + "backupFieldUid": "field_formatted_subtitle", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [ + "fact", + "institute", + "link", + "list", + "page", + "profile", + "school", + "tour", + "video" + ], + "description": "" + } + }, + { + "uid": "field_formatted_title", + "otherCmsField": "Formatted Title", + "otherCmsType": "text_long", + "contentstackField": "Formatted Title", + "contentstackFieldUid": "field_formatted_title", + "contentstackFieldType": "json", + "backupFieldType": "html", + "backupFieldUid": "field_formatted_title", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_image", + "otherCmsField": "Image", + "otherCmsType": "image", + "contentstackField": "Image", + "contentstackFieldUid": "field_image_target_id", + "contentstackFieldType": "file", + "backupFieldType": "file", + "backupFieldUid": "field_image_target_id", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_note", + "otherCmsField": "Note", + "otherCmsType": "string_long", + "contentstackField": "Note", + "contentstackFieldUid": "field_note", + "contentstackFieldType": "json", + "backupFieldType": "html", + "backupFieldUid": "field_note", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_subtitle", + "otherCmsField": "Subtitle", + "otherCmsType": "string", + "contentstackField": "Subtitle", + "contentstackFieldUid": "field_subtitle", + "contentstackFieldType": "single_line_text", + "backupFieldType": "multi_line_text", + "backupFieldUid": "field_subtitle", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [ + "fact", + "institute", + "link", + "list", + "page", + "profile", + "school", + "tour", + "video" + ], + "description": "" + } + }, + { + "uid": "taxonomies", + "otherCmsField": "Taxonomy", + "otherCmsType": "taxonomy", + "contentstackField": "Taxonomy", + "contentstackFieldUid": "taxonomies", + "contentstackFieldType": "taxonomy", + "backupFieldType": "taxonomy", + "backupFieldUid": "taxonomies", + "advanced": { + "taxonomies": [ + { + "taxonomy_uid": "news", + "mandatory": false, + "multiple": true, + "non_localizable": false + }, + { + "taxonomy_uid": "tags", + "mandatory": false, + "multiple": true, + "non_localizable": false + } + ], + "mandatory": false, + "multiple": true, + "non_localizable": false, + "unique": false + } + } + ], + "description": "Schema for article", + "options": { + "is_page": true, + "singleton": false, + "sub_title": [], + "title": "title", + "url_pattern": "/:title", + "url_prefix": "/article/" + } +} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/drupalSchema/fact.json b/upload-api/drupalMigrationData/drupalSchema/fact.json new file mode 100644 index 000000000..3352891b6 --- /dev/null +++ b/upload-api/drupalMigrationData/drupalSchema/fact.json @@ -0,0 +1,81 @@ +{ + "title": "fact", + "uid": "fact", + "schema": [ + { + "uid": "title", + "otherCmsField": "title", + "otherCmsType": "text", + "contentstackField": "title", + "contentstackFieldUid": "title", + "contentstackFieldType": "text", + "backupFieldType": "text", + "backupFieldUid": "title", + "advanced": { + "mandatory": true + } + }, + { + "uid": "url", + "otherCmsField": "url", + "otherCmsType": "text", + "contentstackField": "Url", + "contentstackFieldUid": "url", + "contentstackFieldType": "url", + "backupFieldType": "url", + "backupFieldUid": "url", + "advanced": { + "mandatory": true + } + }, + { + "uid": "field_body", + "otherCmsField": "Body", + "otherCmsType": "text_long", + "contentstackField": "Body", + "contentstackFieldUid": "field_body", + "contentstackFieldType": "json", + "backupFieldType": "html", + "backupFieldUid": "field_body", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_image", + "otherCmsField": "Image", + "otherCmsType": "image", + "contentstackField": "Image", + "contentstackFieldUid": "field_image_target_id", + "contentstackFieldType": "file", + "backupFieldType": "file", + "backupFieldUid": "field_image_target_id", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + } + ], + "description": "Schema for fact", + "options": { + "is_page": true, + "singleton": false, + "sub_title": [], + "title": "title", + "url_pattern": "/:title", + "url_prefix": "/fact/" + } +} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/drupalSchema/featured_content.json b/upload-api/drupalMigrationData/drupalSchema/featured_content.json new file mode 100644 index 000000000..b2a795dee --- /dev/null +++ b/upload-api/drupalMigrationData/drupalSchema/featured_content.json @@ -0,0 +1,81 @@ +{ + "title": "featured content", + "uid": "featured_content", + "schema": [ + { + "uid": "title", + "otherCmsField": "title", + "otherCmsType": "text", + "contentstackField": "title", + "contentstackFieldUid": "title", + "contentstackFieldType": "text", + "backupFieldType": "text", + "backupFieldUid": "title", + "advanced": { + "mandatory": true + } + }, + { + "uid": "url", + "otherCmsField": "url", + "otherCmsType": "text", + "contentstackField": "Url", + "contentstackFieldUid": "url", + "contentstackFieldType": "url", + "backupFieldType": "url", + "backupFieldUid": "url", + "advanced": { + "mandatory": true + } + }, + { + "uid": "body", + "otherCmsField": "Body", + "otherCmsType": "text_with_summary", + "contentstackField": "Body", + "contentstackFieldUid": "body", + "contentstackFieldType": "json", + "backupFieldType": "html", + "backupFieldUid": "body", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_featured_content", + "otherCmsField": "Featured Content", + "otherCmsType": "entity_reference_revisions", + "contentstackField": "Featured Content", + "contentstackFieldUid": "field_featured_content", + "contentstackFieldType": "single_line_text", + "backupFieldType": "single_line_text", + "backupFieldUid": "field_featured_content", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + } + ], + "description": "Schema for featured content", + "options": { + "is_page": true, + "singleton": false, + "sub_title": [], + "title": "title", + "url_pattern": "/:title", + "url_prefix": "/featuredcontent/" + } +} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/drupalSchema/file_tree.json b/upload-api/drupalMigrationData/drupalSchema/file_tree.json new file mode 100644 index 000000000..bffd06702 --- /dev/null +++ b/upload-api/drupalMigrationData/drupalSchema/file_tree.json @@ -0,0 +1,211 @@ +{ + "title": "file tree", + "uid": "file_tree", + "schema": [ + { + "uid": "title", + "otherCmsField": "title", + "otherCmsType": "text", + "contentstackField": "title", + "contentstackFieldUid": "title", + "contentstackFieldType": "text", + "backupFieldType": "text", + "backupFieldUid": "title", + "advanced": { + "mandatory": true + } + }, + { + "uid": "url", + "otherCmsField": "url", + "otherCmsType": "text", + "contentstackField": "Url", + "contentstackFieldUid": "url", + "contentstackFieldType": "url", + "backupFieldType": "url", + "backupFieldUid": "url", + "advanced": { + "mandatory": true + } + }, + { + "uid": "field_content", + "otherCmsField": "Content", + "otherCmsType": "entity_reference_revisions", + "contentstackField": "Content", + "contentstackFieldUid": "field_content", + "contentstackFieldType": "single_line_text", + "backupFieldType": "single_line_text", + "backupFieldUid": "field_content", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_file", + "otherCmsField": "File", + "otherCmsType": "file", + "contentstackField": "File", + "contentstackFieldUid": "field_file_target_id", + "contentstackFieldType": "file", + "backupFieldType": "file", + "backupFieldUid": "field_file_target_id", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_formatted_title", + "otherCmsField": "Formatted Title", + "otherCmsType": "text_long", + "contentstackField": "Formatted Title", + "contentstackFieldUid": "field_formatted_title", + "contentstackFieldType": "json", + "backupFieldType": "html", + "backupFieldUid": "field_formatted_title", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_link", + "otherCmsField": "Link", + "otherCmsType": "link", + "contentstackField": "Link", + "contentstackFieldUid": "field_link", + "contentstackFieldType": "link", + "backupFieldType": "link", + "backupFieldUid": "field_link", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "taxonomies", + "otherCmsField": "Taxonomy", + "otherCmsType": "taxonomy", + "contentstackField": "Taxonomy", + "contentstackFieldUid": "taxonomies", + "contentstackFieldType": "taxonomy", + "backupFieldType": "taxonomy", + "backupFieldUid": "taxonomies", + "advanced": { + "taxonomies": [ + { + "taxonomy_uid": "centers", + "mandatory": false, + "multiple": true, + "non_localizable": false + }, + { + "taxonomy_uid": "eventtypes", + "mandatory": false, + "multiple": true, + "non_localizable": false + }, + { + "taxonomy_uid": "initiativetype", + "mandatory": false, + "multiple": true, + "non_localizable": false + }, + { + "taxonomy_uid": "issues", + "mandatory": false, + "multiple": true, + "non_localizable": false + }, + { + "taxonomy_uid": "metroarea", + "mandatory": false, + "multiple": true, + "non_localizable": false + }, + { + "taxonomy_uid": "people", + "mandatory": false, + "multiple": true, + "non_localizable": false + }, + { + "taxonomy_uid": "researchformat", + "mandatory": false, + "multiple": true, + "non_localizable": false + }, + { + "taxonomy_uid": "ricenewsgroup", + "mandatory": false, + "multiple": true, + "non_localizable": false + }, + { + "taxonomy_uid": "sponsors", + "mandatory": false, + "multiple": true, + "non_localizable": false + }, + { + "taxonomy_uid": "states", + "mandatory": false, + "multiple": true, + "non_localizable": false + }, + { + "taxonomy_uid": "tags", + "mandatory": false, + "multiple": true, + "non_localizable": false + }, + { + "taxonomy_uid": "urbanedgeposttypes", + "mandatory": false, + "multiple": true, + "non_localizable": false + } + ], + "mandatory": false, + "multiple": true, + "non_localizable": false, + "unique": false + } + } + ], + "description": "Schema for file tree", + "options": { + "is_page": true, + "singleton": false, + "sub_title": [], + "title": "title", + "url_pattern": "/:title", + "url_prefix": "/filetree/" + } +} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/drupalSchema/in_the_news.json b/upload-api/drupalMigrationData/drupalSchema/in_the_news.json new file mode 100644 index 000000000..b30c52daf --- /dev/null +++ b/upload-api/drupalMigrationData/drupalSchema/in_the_news.json @@ -0,0 +1,123 @@ +{ + "title": "in the news", + "uid": "in_the_news", + "schema": [ + { + "uid": "title", + "otherCmsField": "title", + "otherCmsType": "text", + "contentstackField": "title", + "contentstackFieldUid": "title", + "contentstackFieldType": "text", + "backupFieldType": "text", + "backupFieldUid": "title", + "advanced": { + "mandatory": true + } + }, + { + "uid": "url", + "otherCmsField": "url", + "otherCmsType": "text", + "contentstackField": "Url", + "contentstackFieldUid": "url", + "contentstackFieldType": "url", + "backupFieldType": "url", + "backupFieldUid": "url", + "advanced": { + "mandatory": true + } + }, + { + "uid": "field_external_link", + "otherCmsField": "External Link", + "otherCmsType": "string", + "contentstackField": "External Link", + "contentstackFieldUid": "field_external_link", + "contentstackFieldType": "single_line_text", + "backupFieldType": "multi_line_text", + "backupFieldUid": "field_external_link", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [ + "article", + "fact", + "featured_content", + "file_tree", + "institute", + "link", + "list", + "page", + "partners", + "profile" + ], + "description": "" + } + }, + { + "uid": "field_publication_date", + "otherCmsField": "Publication Date", + "otherCmsType": "datetime", + "contentstackField": "Publication Date", + "contentstackFieldUid": "field_publication_date", + "contentstackFieldType": "isodate", + "backupFieldType": "isodate", + "backupFieldUid": "field_publication_date", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_source", + "otherCmsField": "Source", + "otherCmsType": "string", + "contentstackField": "Source", + "contentstackFieldUid": "field_source", + "contentstackFieldType": "single_line_text", + "backupFieldType": "multi_line_text", + "backupFieldUid": "field_source", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [ + "article", + "fact", + "featured_content", + "file_tree", + "institute", + "link", + "list", + "page", + "partners", + "profile" + ], + "description": "" + } + } + ], + "description": "Schema for in the news", + "options": { + "is_page": true, + "singleton": false, + "sub_title": [], + "title": "title", + "url_pattern": "/:title", + "url_prefix": "/inthenews/" + } +} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/drupalSchema/institute.json b/upload-api/drupalMigrationData/drupalSchema/institute.json new file mode 100644 index 000000000..e096500d8 --- /dev/null +++ b/upload-api/drupalMigrationData/drupalSchema/institute.json @@ -0,0 +1,141 @@ +{ + "title": "institute", + "uid": "institute", + "schema": [ + { + "uid": "title", + "otherCmsField": "title", + "otherCmsType": "text", + "contentstackField": "title", + "contentstackFieldUid": "title", + "contentstackFieldType": "text", + "backupFieldType": "text", + "backupFieldUid": "title", + "advanced": { + "mandatory": true + } + }, + { + "uid": "url", + "otherCmsField": "url", + "otherCmsType": "text", + "contentstackField": "Url", + "contentstackFieldUid": "url", + "contentstackFieldType": "url", + "backupFieldType": "url", + "backupFieldUid": "url", + "advanced": { + "mandatory": true + } + }, + { + "uid": "field_formatted_title", + "otherCmsField": "Formatted Title", + "otherCmsType": "text_long", + "contentstackField": "Formatted Title", + "contentstackFieldUid": "field_formatted_title", + "contentstackFieldType": "json", + "backupFieldType": "html", + "backupFieldUid": "field_formatted_title", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_image", + "otherCmsField": "Image", + "otherCmsType": "image", + "contentstackField": "Image", + "contentstackFieldUid": "field_image_target_id", + "contentstackFieldType": "file", + "backupFieldType": "file", + "backupFieldUid": "field_image_target_id", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_link", + "otherCmsField": "Link", + "otherCmsType": "link", + "contentstackField": "Link", + "contentstackFieldUid": "field_link", + "contentstackFieldType": "link", + "backupFieldType": "link", + "backupFieldUid": "field_link", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_links", + "otherCmsField": "Links", + "otherCmsType": "link", + "contentstackField": "Links", + "contentstackFieldUid": "field_links", + "contentstackFieldType": "link", + "backupFieldType": "link", + "backupFieldUid": "field_links", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_summary", + "otherCmsField": "Summary", + "otherCmsType": "string_long", + "contentstackField": "Summary", + "contentstackFieldUid": "field_summary", + "contentstackFieldType": "json", + "backupFieldType": "html", + "backupFieldUid": "field_summary", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + } + ], + "description": "Schema for institute", + "options": { + "is_page": true, + "singleton": false, + "sub_title": [], + "title": "title", + "url_pattern": "/:title", + "url_prefix": "/institute/" + } +} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/drupalSchema/link.json b/upload-api/drupalMigrationData/drupalSchema/link.json new file mode 100644 index 000000000..0baa3108c --- /dev/null +++ b/upload-api/drupalMigrationData/drupalSchema/link.json @@ -0,0 +1,151 @@ +{ + "title": "link", + "uid": "link", + "schema": [ + { + "uid": "title", + "otherCmsField": "title", + "otherCmsType": "text", + "contentstackField": "title", + "contentstackFieldUid": "title", + "contentstackFieldType": "text", + "backupFieldType": "text", + "backupFieldUid": "title", + "advanced": { + "mandatory": true + } + }, + { + "uid": "url", + "otherCmsField": "url", + "otherCmsType": "text", + "contentstackField": "Url", + "contentstackFieldUid": "url", + "contentstackFieldType": "url", + "backupFieldType": "url", + "backupFieldUid": "url", + "advanced": { + "mandatory": true + } + }, + { + "uid": "field_formatted_title", + "otherCmsField": "Formatted Title", + "otherCmsType": "text_long", + "contentstackField": "Formatted Title", + "contentstackFieldUid": "field_formatted_title", + "contentstackFieldType": "json", + "backupFieldType": "html", + "backupFieldUid": "field_formatted_title", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_image", + "otherCmsField": "Image", + "otherCmsType": "image", + "contentstackField": "Image", + "contentstackFieldUid": "field_image_target_id", + "contentstackFieldType": "file", + "backupFieldType": "file", + "backupFieldUid": "field_image_target_id", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_link", + "otherCmsField": "Link", + "otherCmsType": "link", + "contentstackField": "Link", + "contentstackFieldUid": "field_link", + "contentstackFieldType": "link", + "backupFieldType": "link", + "backupFieldUid": "field_link", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_link_order", + "otherCmsField": "Link Order", + "otherCmsType": "integer", + "contentstackField": "Link Order", + "contentstackFieldUid": "field_link_order", + "contentstackFieldType": "number", + "backupFieldType": "number", + "backupFieldUid": "field_link_order", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "taxonomies", + "otherCmsField": "Taxonomy", + "otherCmsType": "taxonomy", + "contentstackField": "Taxonomy", + "contentstackFieldUid": "taxonomies", + "contentstackFieldType": "taxonomy", + "backupFieldType": "taxonomy", + "backupFieldUid": "taxonomies", + "advanced": { + "taxonomies": [ + { + "taxonomy_uid": "pages", + "mandatory": false, + "multiple": true, + "non_localizable": false + }, + { + "taxonomy_uid": "tags", + "mandatory": false, + "multiple": true, + "non_localizable": false + } + ], + "mandatory": false, + "multiple": true, + "non_localizable": false, + "unique": false + } + } + ], + "description": "Schema for link", + "options": { + "is_page": true, + "singleton": false, + "sub_title": [], + "title": "title", + "url_pattern": "/:title", + "url_prefix": "/link/" + } +} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/drupalSchema/list.json b/upload-api/drupalMigrationData/drupalSchema/list.json new file mode 100644 index 000000000..010b3c35b --- /dev/null +++ b/upload-api/drupalMigrationData/drupalSchema/list.json @@ -0,0 +1,151 @@ +{ + "title": "list", + "uid": "list", + "schema": [ + { + "uid": "title", + "otherCmsField": "title", + "otherCmsType": "text", + "contentstackField": "title", + "contentstackFieldUid": "title", + "contentstackFieldType": "text", + "backupFieldType": "text", + "backupFieldUid": "title", + "advanced": { + "mandatory": true + } + }, + { + "uid": "url", + "otherCmsField": "url", + "otherCmsType": "text", + "contentstackField": "Url", + "contentstackFieldUid": "url", + "contentstackFieldType": "url", + "backupFieldType": "url", + "backupFieldUid": "url", + "advanced": { + "mandatory": true + } + }, + { + "uid": "field_formatted_title", + "otherCmsField": "Formatted Title", + "otherCmsType": "text_long", + "contentstackField": "Formatted Title", + "contentstackFieldUid": "field_formatted_title", + "contentstackFieldType": "json", + "backupFieldType": "html", + "backupFieldUid": "field_formatted_title", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_link", + "otherCmsField": "Link", + "otherCmsType": "link", + "contentstackField": "Link", + "contentstackFieldUid": "field_link", + "contentstackFieldType": "link", + "backupFieldType": "link", + "backupFieldUid": "field_link", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_links", + "otherCmsField": "Links", + "otherCmsType": "link", + "contentstackField": "Links", + "contentstackFieldUid": "field_links", + "contentstackFieldType": "link", + "backupFieldType": "link", + "backupFieldUid": "field_links", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_list_order", + "otherCmsField": "List Order", + "otherCmsType": "integer", + "contentstackField": "List Order", + "contentstackFieldUid": "field_list_order", + "contentstackFieldType": "number", + "backupFieldType": "number", + "backupFieldUid": "field_list_order", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "taxonomies", + "otherCmsField": "Taxonomy", + "otherCmsType": "taxonomy", + "contentstackField": "Taxonomy", + "contentstackFieldUid": "taxonomies", + "contentstackFieldType": "taxonomy", + "backupFieldType": "taxonomy", + "backupFieldUid": "taxonomies", + "advanced": { + "taxonomies": [ + { + "taxonomy_uid": "list", + "mandatory": false, + "multiple": true, + "non_localizable": false + }, + { + "taxonomy_uid": "tags", + "mandatory": false, + "multiple": true, + "non_localizable": false + } + ], + "mandatory": false, + "multiple": true, + "non_localizable": false, + "unique": false + } + } + ], + "description": "Schema for list", + "options": { + "is_page": true, + "singleton": false, + "sub_title": [], + "title": "title", + "url_pattern": "/:title", + "url_prefix": "/list/" + } +} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/drupalSchema/page.json b/upload-api/drupalMigrationData/drupalSchema/page.json new file mode 100644 index 000000000..f826794aa --- /dev/null +++ b/upload-api/drupalMigrationData/drupalSchema/page.json @@ -0,0 +1,381 @@ +{ + "title": "page", + "uid": "page", + "schema": [ + { + "uid": "title", + "otherCmsField": "title", + "otherCmsType": "text", + "contentstackField": "title", + "contentstackFieldUid": "title", + "contentstackFieldType": "text", + "backupFieldType": "text", + "backupFieldUid": "title", + "advanced": { + "mandatory": true + } + }, + { + "uid": "url", + "otherCmsField": "url", + "otherCmsType": "text", + "contentstackField": "Url", + "contentstackFieldUid": "url", + "contentstackFieldType": "url", + "backupFieldType": "url", + "backupFieldUid": "url", + "advanced": { + "mandatory": true + } + }, + { + "uid": "field_body", + "otherCmsField": "Body", + "otherCmsType": "text_long", + "contentstackField": "Body", + "contentstackFieldUid": "field_body", + "contentstackFieldType": "json", + "backupFieldType": "html", + "backupFieldUid": "field_body", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_external_link", + "otherCmsField": "External Link", + "otherCmsType": "string", + "contentstackField": "External Link", + "contentstackFieldUid": "field_external_link", + "contentstackFieldType": "single_line_text", + "backupFieldType": "multi_line_text", + "backupFieldUid": "field_external_link", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [ + "article", + "fact", + "institute", + "link", + "list", + "profile", + "school", + "tour", + "video" + ], + "description": "" + } + }, + { + "uid": "field_formatted_title", + "otherCmsField": "Formatted Title", + "otherCmsType": "text_long", + "contentstackField": "Formatted Title", + "contentstackFieldUid": "field_formatted_title", + "contentstackFieldType": "json", + "backupFieldType": "html", + "backupFieldUid": "field_formatted_title", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_hero_body", + "otherCmsField": "Hero Body", + "otherCmsType": "text_long", + "contentstackField": "Hero Body", + "contentstackFieldUid": "field_hero_body", + "contentstackFieldType": "json", + "backupFieldType": "html", + "backupFieldUid": "field_hero_body", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_hero_image", + "otherCmsField": "Hero Image", + "otherCmsType": "image", + "contentstackField": "Hero Image", + "contentstackFieldUid": "field_hero_image_target_id", + "contentstackFieldType": "file", + "backupFieldType": "file", + "backupFieldUid": "field_hero_image_target_id", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_hero_intro", + "otherCmsField": "Hero Intro", + "otherCmsType": "string", + "contentstackField": "Hero Intro", + "contentstackFieldUid": "field_hero_intro", + "contentstackFieldType": "single_line_text", + "backupFieldType": "multi_line_text", + "backupFieldUid": "field_hero_intro", + "advanced": { + "default_value": "ABBREVIATED INTRODUCTORY", + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [ + "article", + "fact", + "institute", + "link", + "list", + "profile", + "school", + "tour", + "video" + ], + "description": "" + } + }, + { + "uid": "field_hero_title", + "otherCmsField": "Hero Title", + "otherCmsType": "text_long", + "contentstackField": "Hero Title", + "contentstackFieldUid": "field_hero_title", + "contentstackFieldType": "json", + "backupFieldType": "html", + "backupFieldUid": "field_hero_title", + "advanced": { + "default_value": "

Page Title Here

\r\n", + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_image", + "otherCmsField": "Image", + "otherCmsType": "image", + "contentstackField": "Image", + "contentstackFieldUid": "field_image_target_id", + "contentstackFieldType": "file", + "backupFieldType": "file", + "backupFieldUid": "field_image_target_id", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_link", + "otherCmsField": "Link", + "otherCmsType": "link", + "contentstackField": "Link", + "contentstackFieldUid": "field_link", + "contentstackFieldType": "link", + "backupFieldType": "link", + "backupFieldUid": "field_link", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_meta_tags", + "otherCmsField": "Meta Tags", + "otherCmsType": "metatag", + "contentstackField": "Meta Tags", + "contentstackFieldUid": "field_meta_tags", + "contentstackFieldType": "single_line_text", + "backupFieldType": "single_line_text", + "backupFieldUid": "field_meta_tags", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_paragraphs", + "otherCmsField": "Paragraphs", + "otherCmsType": "entity_reference_revisions", + "contentstackField": "Paragraphs", + "contentstackFieldUid": "field_paragraphs", + "contentstackFieldType": "single_line_text", + "backupFieldType": "single_line_text", + "backupFieldUid": "field_paragraphs", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_quick_links", + "otherCmsField": "Quick Links", + "otherCmsType": "link", + "contentstackField": "Quick Links", + "contentstackFieldUid": "field_quick_links", + "contentstackFieldType": "link", + "backupFieldType": "link", + "backupFieldUid": "field_quick_links", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_subtitle", + "otherCmsField": "Subtitle", + "otherCmsType": "string", + "contentstackField": "Subtitle", + "contentstackFieldUid": "field_subtitle", + "contentstackFieldType": "single_line_text", + "backupFieldType": "multi_line_text", + "backupFieldUid": "field_subtitle", + "advanced": { + "default_value": "LEARN MORE", + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [ + "article", + "fact", + "institute", + "link", + "list", + "profile", + "school", + "tour", + "video" + ], + "description": "" + } + }, + { + "uid": "field_summary", + "otherCmsField": "Summary", + "otherCmsType": "string_long", + "contentstackField": "Summary", + "contentstackFieldUid": "field_summary", + "contentstackFieldType": "json", + "backupFieldType": "html", + "backupFieldUid": "field_summary", + "advanced": { + "default_value": "Fostering diversity and an intellectual environment, Rice University is a comprehensive research university located on a 300-acre tree-lined campus in Houston, Texas. Rice produces the next generation of leaders and advances tomorrowโ€™s thinking.", + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "taxonomies", + "otherCmsField": "Taxonomy", + "otherCmsType": "taxonomy", + "contentstackField": "Taxonomy", + "contentstackFieldUid": "taxonomies", + "contentstackFieldType": "taxonomy", + "backupFieldType": "taxonomy", + "backupFieldUid": "taxonomies", + "advanced": { + "taxonomies": [ + { + "taxonomy_uid": "pages", + "mandatory": false, + "multiple": true, + "non_localizable": false + }, + { + "taxonomy_uid": "tags", + "mandatory": false, + "multiple": true, + "non_localizable": false + } + ], + "mandatory": false, + "multiple": true, + "non_localizable": false, + "unique": false + } + } + ], + "description": "Schema for page", + "options": { + "is_page": true, + "singleton": false, + "sub_title": [], + "title": "title", + "url_pattern": "/:title", + "url_prefix": "/page/" + } +} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/drupalSchema/partners.json b/upload-api/drupalMigrationData/drupalSchema/partners.json new file mode 100644 index 000000000..6a069a625 --- /dev/null +++ b/upload-api/drupalMigrationData/drupalSchema/partners.json @@ -0,0 +1,141 @@ +{ + "title": "partners", + "uid": "partners", + "schema": [ + { + "uid": "title", + "otherCmsField": "title", + "otherCmsType": "text", + "contentstackField": "title", + "contentstackFieldUid": "title", + "contentstackFieldType": "text", + "backupFieldType": "text", + "backupFieldUid": "title", + "advanced": { + "mandatory": true + } + }, + { + "uid": "url", + "otherCmsField": "url", + "otherCmsType": "text", + "contentstackField": "Url", + "contentstackFieldUid": "url", + "contentstackFieldType": "url", + "backupFieldType": "url", + "backupFieldUid": "url", + "advanced": { + "mandatory": true + } + }, + { + "uid": "field_formatted_title", + "otherCmsField": "Formatted Title", + "otherCmsType": "text_long", + "contentstackField": "Formatted Title", + "contentstackFieldUid": "field_formatted_title", + "contentstackFieldType": "json", + "backupFieldType": "html", + "backupFieldUid": "field_formatted_title", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_image", + "otherCmsField": "Image", + "otherCmsType": "image", + "contentstackField": "Image", + "contentstackFieldUid": "field_image_target_id", + "contentstackFieldType": "file", + "backupFieldType": "file", + "backupFieldUid": "field_image_target_id", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_link", + "otherCmsField": "Link", + "otherCmsType": "link", + "contentstackField": "Link", + "contentstackFieldUid": "field_link", + "contentstackFieldType": "link", + "backupFieldType": "link", + "backupFieldUid": "field_link", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_links", + "otherCmsField": "Links", + "otherCmsType": "link", + "contentstackField": "Links", + "contentstackFieldUid": "field_links", + "contentstackFieldType": "link", + "backupFieldType": "link", + "backupFieldUid": "field_links", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_summary", + "otherCmsField": "Summary", + "otherCmsType": "string_long", + "contentstackField": "Summary", + "contentstackFieldUid": "field_summary", + "contentstackFieldType": "json", + "backupFieldType": "html", + "backupFieldUid": "field_summary", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + } + ], + "description": "Schema for partners", + "options": { + "is_page": true, + "singleton": false, + "sub_title": [], + "title": "title", + "url_pattern": "/:title", + "url_prefix": "/partners/" + } +} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/drupalSchema/profile.json b/upload-api/drupalMigrationData/drupalSchema/profile.json new file mode 100644 index 000000000..4c8160fad --- /dev/null +++ b/upload-api/drupalMigrationData/drupalSchema/profile.json @@ -0,0 +1,251 @@ +{ + "title": "profile", + "uid": "profile", + "schema": [ + { + "uid": "title", + "otherCmsField": "title", + "otherCmsType": "text", + "contentstackField": "title", + "contentstackFieldUid": "title", + "contentstackFieldType": "text", + "backupFieldType": "text", + "backupFieldUid": "title", + "advanced": { + "mandatory": true + } + }, + { + "uid": "url", + "otherCmsField": "url", + "otherCmsType": "text", + "contentstackField": "Url", + "contentstackFieldUid": "url", + "contentstackFieldType": "url", + "backupFieldType": "url", + "backupFieldUid": "url", + "advanced": { + "mandatory": true + } + }, + { + "uid": "body", + "otherCmsField": "Body", + "otherCmsType": "text_with_summary", + "contentstackField": "Body", + "contentstackFieldUid": "body", + "contentstackFieldType": "json", + "backupFieldType": "html", + "backupFieldUid": "body", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_address", + "otherCmsField": "Address", + "otherCmsType": "string", + "contentstackField": "Address", + "contentstackFieldUid": "field_address", + "contentstackFieldType": "single_line_text", + "backupFieldType": "multi_line_text", + "backupFieldUid": "field_address", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [ + "article", + "fact", + "institute", + "link", + "list", + "page", + "school", + "tour", + "video" + ], + "description": "" + } + }, + { + "uid": "field_email", + "otherCmsField": "Email", + "otherCmsType": "email", + "contentstackField": "Email", + "contentstackFieldUid": "field_email", + "contentstackFieldType": "text", + "backupFieldType": "text", + "backupFieldUid": "field_email", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_hours", + "otherCmsField": "Hours", + "otherCmsType": "string", + "contentstackField": "Hours", + "contentstackFieldUid": "field_hours", + "contentstackFieldType": "single_line_text", + "backupFieldType": "multi_line_text", + "backupFieldUid": "field_hours", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [ + "article", + "fact", + "institute", + "link", + "list", + "page", + "school", + "tour", + "video" + ], + "description": "" + } + }, + { + "uid": "field_image", + "otherCmsField": "Image", + "otherCmsType": "image", + "contentstackField": "Image", + "contentstackFieldUid": "field_image_target_id", + "contentstackFieldType": "file", + "backupFieldType": "file", + "backupFieldUid": "field_image_target_id", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_links", + "otherCmsField": "Links", + "otherCmsType": "link", + "contentstackField": "Links", + "contentstackFieldUid": "field_links", + "contentstackFieldType": "link", + "backupFieldType": "link", + "backupFieldUid": "field_links", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_phone", + "otherCmsField": "Phone", + "otherCmsType": "string", + "contentstackField": "Phone", + "contentstackFieldUid": "field_phone", + "contentstackFieldType": "single_line_text", + "backupFieldType": "multi_line_text", + "backupFieldUid": "field_phone", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [ + "article", + "fact", + "institute", + "link", + "list", + "page", + "school", + "tour", + "video" + ], + "description": "" + } + }, + { + "uid": "field_summary", + "otherCmsField": "Summary", + "otherCmsType": "string_long", + "contentstackField": "Summary", + "contentstackFieldUid": "field_summary", + "contentstackFieldType": "json", + "backupFieldType": "html", + "backupFieldUid": "field_summary", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_title", + "otherCmsField": "Title", + "otherCmsType": "string_long", + "contentstackField": "Title", + "contentstackFieldUid": "field_title", + "contentstackFieldType": "json", + "backupFieldType": "html", + "backupFieldUid": "field_title", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + } + ], + "description": "Schema for profile", + "options": { + "is_page": true, + "singleton": false, + "sub_title": [], + "title": "title", + "url_pattern": "/:title", + "url_prefix": "/profile/" + } +} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/drupalSchema/school.json b/upload-api/drupalMigrationData/drupalSchema/school.json new file mode 100644 index 000000000..6cd733e64 --- /dev/null +++ b/upload-api/drupalMigrationData/drupalSchema/school.json @@ -0,0 +1,161 @@ +{ + "title": "school", + "uid": "school", + "schema": [ + { + "uid": "title", + "otherCmsField": "title", + "otherCmsType": "text", + "contentstackField": "title", + "contentstackFieldUid": "title", + "contentstackFieldType": "text", + "backupFieldType": "text", + "backupFieldUid": "title", + "advanced": { + "mandatory": true + } + }, + { + "uid": "url", + "otherCmsField": "url", + "otherCmsType": "text", + "contentstackField": "Url", + "contentstackFieldUid": "url", + "contentstackFieldType": "url", + "backupFieldType": "url", + "backupFieldUid": "url", + "advanced": { + "mandatory": true + } + }, + { + "uid": "field_formatted_title", + "otherCmsField": "Formatted Title", + "otherCmsType": "text_long", + "contentstackField": "Formatted Title", + "contentstackFieldUid": "field_formatted_title", + "contentstackFieldType": "json", + "backupFieldType": "html", + "backupFieldUid": "field_formatted_title", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_image", + "otherCmsField": "Image", + "otherCmsType": "image", + "contentstackField": "Image", + "contentstackFieldUid": "field_image_target_id", + "contentstackFieldType": "file", + "backupFieldType": "file", + "backupFieldUid": "field_image_target_id", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_link", + "otherCmsField": "Link", + "otherCmsType": "link", + "contentstackField": "Link", + "contentstackFieldUid": "field_link", + "contentstackFieldType": "link", + "backupFieldType": "link", + "backupFieldUid": "field_link", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_links", + "otherCmsField": "Links", + "otherCmsType": "link", + "contentstackField": "Links", + "contentstackFieldUid": "field_links", + "contentstackFieldType": "link", + "backupFieldType": "link", + "backupFieldUid": "field_links", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_school_order", + "otherCmsField": "School Order", + "otherCmsType": "integer", + "contentstackField": "School Order", + "contentstackFieldUid": "field_school_order", + "contentstackFieldType": "number", + "backupFieldType": "number", + "backupFieldUid": "field_school_order", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_summary", + "otherCmsField": "Summary", + "otherCmsType": "string_long", + "contentstackField": "Summary", + "contentstackFieldUid": "field_summary", + "contentstackFieldType": "json", + "backupFieldType": "html", + "backupFieldUid": "field_summary", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + } + ], + "description": "Schema for school", + "options": { + "is_page": true, + "singleton": false, + "sub_title": [], + "title": "title", + "url_pattern": "/:title", + "url_prefix": "/school/" + } +} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/drupalSchema/supporting_material.json b/upload-api/drupalMigrationData/drupalSchema/supporting_material.json new file mode 100644 index 000000000..586642a9b --- /dev/null +++ b/upload-api/drupalMigrationData/drupalSchema/supporting_material.json @@ -0,0 +1,163 @@ +{ + "title": "supporting material", + "uid": "supporting_material", + "schema": [ + { + "uid": "title", + "otherCmsField": "title", + "otherCmsType": "text", + "contentstackField": "title", + "contentstackFieldUid": "title", + "contentstackFieldType": "text", + "backupFieldType": "text", + "backupFieldUid": "title", + "advanced": { + "mandatory": true + } + }, + { + "uid": "url", + "otherCmsField": "url", + "otherCmsType": "text", + "contentstackField": "Url", + "contentstackFieldUid": "url", + "contentstackFieldType": "url", + "backupFieldType": "url", + "backupFieldUid": "url", + "advanced": { + "mandatory": true + } + }, + { + "uid": "field_document_source", + "otherCmsField": "Document Source", + "otherCmsType": "link", + "contentstackField": "Document Source", + "contentstackFieldUid": "field_document_source", + "contentstackFieldType": "link", + "backupFieldType": "link", + "backupFieldUid": "field_document_source", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_document_upload", + "otherCmsField": "Document Upload", + "otherCmsType": "entity_reference", + "contentstackField": "Document Upload", + "contentstackFieldUid": "field_document_upload_target_id", + "contentstackFieldType": "reference", + "backupFieldType": "reference", + "backupFieldUid": "field_document_upload_target_id", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [ + "article", + "fact", + "featured_content", + "file_tree", + "institute", + "in_the_news", + "link", + "list", + "page", + "partners" + ], + "description": "" + } + }, + { + "uid": "field_media_type", + "otherCmsField": "Media Type", + "otherCmsType": "list_string", + "contentstackField": "Media Type", + "contentstackFieldUid": "field_media_type", + "contentstackFieldType": "single_line_text", + "backupFieldType": "multi_line_text", + "backupFieldUid": "field_media_type", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [ + "article", + "fact", + "featured_content", + "file_tree", + "institute", + "in_the_news", + "link", + "list", + "page", + "partners" + ], + "description": "" + } + }, + { + "uid": "field_press_kit", + "otherCmsField": "Press Kit", + "otherCmsType": "boolean", + "contentstackField": "Press Kit", + "contentstackFieldUid": "field_press_kit", + "contentstackFieldType": "boolean", + "backupFieldType": "boolean", + "backupFieldUid": "field_press_kit", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_publication_date", + "otherCmsField": "Publication Date", + "otherCmsType": "datetime", + "contentstackField": "Publication Date", + "contentstackFieldUid": "field_publication_date", + "contentstackFieldType": "isodate", + "backupFieldType": "isodate", + "backupFieldUid": "field_publication_date", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + } + ], + "description": "Schema for supporting material", + "options": { + "is_page": true, + "singleton": false, + "sub_title": [], + "title": "title", + "url_pattern": "/:title", + "url_prefix": "/supportingmaterial/" + } +} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/drupalSchema/tour.json b/upload-api/drupalMigrationData/drupalSchema/tour.json new file mode 100644 index 000000000..4d6be850c --- /dev/null +++ b/upload-api/drupalMigrationData/drupalSchema/tour.json @@ -0,0 +1,111 @@ +{ + "title": "tour", + "uid": "tour", + "schema": [ + { + "uid": "title", + "otherCmsField": "title", + "otherCmsType": "text", + "contentstackField": "title", + "contentstackFieldUid": "title", + "contentstackFieldType": "text", + "backupFieldType": "text", + "backupFieldUid": "title", + "advanced": { + "mandatory": true + } + }, + { + "uid": "url", + "otherCmsField": "url", + "otherCmsType": "text", + "contentstackField": "Url", + "contentstackFieldUid": "url", + "contentstackFieldType": "url", + "backupFieldType": "url", + "backupFieldUid": "url", + "advanced": { + "mandatory": true + } + }, + { + "uid": "field_external_link", + "otherCmsField": "External Link", + "otherCmsType": "string", + "contentstackField": "External Link", + "contentstackFieldUid": "field_external_link", + "contentstackFieldType": "single_line_text", + "backupFieldType": "multi_line_text", + "backupFieldUid": "field_external_link", + "advanced": { + "default_value": "https://www.rice.edu/virtualtours/graduatestudies/tourfiles/flash/index.html?utm_source=Tour&utm_medium=Virtual&utm_campaign=GraduateWindow", + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [ + "article", + "fact", + "institute", + "link", + "list", + "page", + "profile", + "school", + "video" + ], + "description": "" + } + }, + { + "uid": "field_formatted_title", + "otherCmsField": "Formatted Title", + "otherCmsType": "text_long", + "contentstackField": "Formatted Title", + "contentstackFieldUid": "field_formatted_title", + "contentstackFieldType": "json", + "backupFieldType": "html", + "backupFieldUid": "field_formatted_title", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_image", + "otherCmsField": "Image", + "otherCmsType": "image", + "contentstackField": "Image", + "contentstackFieldUid": "field_image_target_id", + "contentstackFieldType": "file", + "backupFieldType": "file", + "backupFieldUid": "field_image_target_id", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + } + ], + "description": "Schema for tour", + "options": { + "is_page": true, + "singleton": false, + "sub_title": [], + "title": "title", + "url_pattern": "/:title", + "url_prefix": "/tour/" + } +} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/drupalSchema/video.json b/upload-api/drupalMigrationData/drupalSchema/video.json new file mode 100644 index 000000000..d59d47866 --- /dev/null +++ b/upload-api/drupalMigrationData/drupalSchema/video.json @@ -0,0 +1,161 @@ +{ + "title": "video", + "uid": "video", + "schema": [ + { + "uid": "title", + "otherCmsField": "title", + "otherCmsType": "text", + "contentstackField": "title", + "contentstackFieldUid": "title", + "contentstackFieldType": "text", + "backupFieldType": "text", + "backupFieldUid": "title", + "advanced": { + "mandatory": true + } + }, + { + "uid": "url", + "otherCmsField": "url", + "otherCmsType": "text", + "contentstackField": "Url", + "contentstackFieldUid": "url", + "contentstackFieldType": "url", + "backupFieldType": "url", + "backupFieldUid": "url", + "advanced": { + "mandatory": true + } + }, + { + "uid": "field_formatted_title", + "otherCmsField": "Formatted Title", + "otherCmsType": "text_long", + "contentstackField": "Formatted Title", + "contentstackFieldUid": "field_formatted_title", + "contentstackFieldType": "json", + "backupFieldType": "html", + "backupFieldUid": "field_formatted_title", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_image", + "otherCmsField": "Image", + "otherCmsType": "image", + "contentstackField": "Image", + "contentstackFieldUid": "field_image_target_id", + "contentstackFieldType": "file", + "backupFieldType": "file", + "backupFieldUid": "field_image_target_id", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "field_video", + "otherCmsField": "Video", + "otherCmsType": "string", + "contentstackField": "Video", + "contentstackFieldUid": "field_video", + "contentstackFieldType": "single_line_text", + "backupFieldType": "multi_line_text", + "backupFieldUid": "field_video", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [ + "article", + "fact", + "institute", + "link", + "list", + "page", + "profile", + "school", + "tour" + ], + "description": "" + } + }, + { + "uid": "field_video_order", + "otherCmsField": "Video Order", + "otherCmsType": "integer", + "contentstackField": "Video Order", + "contentstackFieldUid": "field_video_order", + "contentstackFieldType": "number", + "backupFieldType": "number", + "backupFieldUid": "field_video_order", + "advanced": { + "default_value": null, + "mandatory": false, + "multiple": false, + "unique": false, + "nonLocalizable": false, + "validationErrorMessage": "", + "embedObjects": [], + "description": "" + } + }, + { + "uid": "taxonomies", + "otherCmsField": "Taxonomy", + "otherCmsType": "taxonomy", + "contentstackField": "Taxonomy", + "contentstackFieldUid": "taxonomies", + "contentstackFieldType": "taxonomy", + "backupFieldType": "taxonomy", + "backupFieldUid": "taxonomies", + "advanced": { + "taxonomies": [ + { + "taxonomy_uid": "videos", + "mandatory": false, + "multiple": true, + "non_localizable": false + }, + { + "taxonomy_uid": "tags", + "mandatory": false, + "multiple": true, + "non_localizable": false + } + ], + "mandatory": false, + "multiple": true, + "non_localizable": false, + "unique": false + } + } + ], + "description": "Schema for video", + "options": { + "is_page": true, + "singleton": false, + "sub_title": [], + "title": "title", + "url_pattern": "/:title", + "url_prefix": "/video/" + } +} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/taxonomySchema/taxonomySchema.json b/upload-api/drupalMigrationData/taxonomySchema/taxonomySchema.json new file mode 100644 index 000000000..ebc3fe8aa --- /dev/null +++ b/upload-api/drupalMigrationData/taxonomySchema/taxonomySchema.json @@ -0,0 +1,22 @@ +[ + { + "uid": "list", + "name": "List" + }, + { + "uid": "news", + "name": "News" + }, + { + "uid": "pages", + "name": "Pages" + }, + { + "uid": "tags", + "name": "Tags" + }, + { + "uid": "videos", + "name": "Videos" + } +] \ No newline at end of file diff --git a/upload-api/migration-drupal/config/index.json b/upload-api/migration-drupal/config/index.json new file mode 100755 index 000000000..b96777d4d --- /dev/null +++ b/upload-api/migration-drupal/config/index.json @@ -0,0 +1,81 @@ +{ + "data": "./drupalMigrationData", + "entryfolder": "entries", + "drupal": { + "drupal": "drupalSchema" + }, + "modules": { + "locales": { + "dirName": "locales", + "fileName": "locales.json" + }, + "contentTypes": { + "dirName": "content_types", + "fileName": "contenttype.json", + "masterfile": "contenttypes.json", + "validKeys": ["title", "uid", "schema", "options", "singleton", "description"] + }, + "references": { + "dirName": "references", + "fileName": "references.json", + "masterfile": "references.json" + }, + "authors": { + "dirName": "authors", + "fileName": "en-us.json", + "masterfile": "authors.json" + }, + "entries": { + "dirName": "entries", + "fileName": "en-us.json", + "masterfile": "entries.json" + }, + "vocabulary": { + "dirName": "vocabulary", + "fileName": "en-us.json", + "masterfile": "vocabulary.json" + }, + "taxonomy": { + "dirName": "taxonomy", + "fileName": "en-us.json", + "masterfile": "taxonomy.json" + }, + + "asset": { + "dirName": "assets", + "fileName": "assets.json", + "featuredfileName": "_featured.json", + "masterfile": "url_master.json" + }, + + "query": { + "dirName": "query", + "fileName": "index.json" + } + }, + "base_locale": { "name": "English US", "code": "en-us" }, + "mysql": { + "host": "localhost", + "user": "root", + "password": "", + "database": "riceunivercity1" + }, + "base_url": "", + "public_path": "sites/g/files/bxs4201/files/", + "private_path": "", + "drupal_base_url": "", + "mysql-query": { + "locale": "SELECT languages.language,languages.name FROM `languages`", + "taxonomy_term_data": "SELECT a.name, max(a.description__value) as description__value ,max(b.tid) as tid,max(b.vid) as vid FROM taxonomy_term_field_data a, taxonomy_term_data b WHERE a.vid=b.vid group by a.name", + "taxonomyCount": "SELECT count(b.tid) as taxonomycount FROM taxonomy_term_field_data a, taxonomy_term_data b WHERE a.tid = b.tid AND a.vid=b.vid", + "ct_mapped": "SELECT *, CONVERT(data USING utf8) as data FROM config WHERE name LIKE '%field.field.node%'", + "fileID": "SELECT * FROM `file_usage`", + "assetCount": "SELECT count(a.fid) as assetcount FROM file_managed a", + "assets": "SELECT a.fid, a.filename, a.uri, a.filesize FROM file_managed a", + "assetsFID": "SELECT a.fid, a.filename, a.uri, b.id,b.count FROM file_managed a, file_usage b WHERE a.fid IN", + "authorCount": "SELECT count(users_field_data.uid) as usercount FROM users_field_data LEFT JOIN file_managed ON file_managed.fid = users_field_data.uid", + "authors": "SELECT a.uid, a.name, a.mail, a.timezone, a.langcode, b.user_picture_target_id as picture FROM users_field_data a LEFT JOIN user__user_picture b ON a.uid = b.entity_id", + "vocabulary": "SELECT taxonomy_term_field_data.vid, taxonomy_term_field_data.name AS title, taxonomy_term_field_data.description__value FROM taxonomy_term_field_data", + "vocabularyCount": "SELECT count(taxonomy_term_field_data.vid) as vocabularycount FROM taxonomy_term_field_data" + } +} diff --git a/upload-api/migration-drupal/index.js b/upload-api/migration-drupal/index.js new file mode 100644 index 000000000..395c84b50 --- /dev/null +++ b/upload-api/migration-drupal/index.js @@ -0,0 +1,12 @@ +/* eslint-disable @typescript-eslint/no-var-requires */ + +const extractTaxonomy = require('./libs/extractTaxonomy'); +const createInitialMapper = require( './libs/createInitialMapper' ); +const extractLocale= require('./libs/extractLocale'); + +module.exports = { + // extractContentTypes, + extractTaxonomy, + createInitialMapper, + extractLocale +}; diff --git a/upload-api/migration-drupal/libs/contentTypeMapper.js b/upload-api/migration-drupal/libs/contentTypeMapper.js new file mode 100644 index 000000000..72b09cd74 --- /dev/null +++ b/upload-api/migration-drupal/libs/contentTypeMapper.js @@ -0,0 +1,522 @@ +/* eslint-disable @typescript-eslint/no-var-requires */ + +const restrictedUid = require('../utils/restrictedKeyWords'); +const fs = require('fs'); +const path = require('path'); + +/** + * Loads taxonomy schema from drupalMigrationData/taxonomySchema/taxonomySchema.json + * If not found, attempts to generate it using extractTaxonomy + * + * @param {Object} dbConfig - Database configuration for fallback taxonomy extraction + * @returns {Array} Array of taxonomy vocabularies with uid and name + */ +const loadTaxonomySchema = async (dbConfig = null) => { + try { + const taxonomySchemaPath = path.join(__dirname, '..', '..', 'drupalMigrationData', 'taxonomySchema', 'taxonomySchema.json'); + + // Check if taxonomy schema exists + if (fs.existsSync(taxonomySchemaPath)) { + const taxonomyData = fs.readFileSync(taxonomySchemaPath, 'utf8'); + const taxonomies = JSON.parse(taxonomyData); + console.log(`โœ… Loaded ${taxonomies.length} taxonomies from schema file`); + return taxonomies; + } + + // If not found and dbConfig available, try to generate it + if (dbConfig) { + console.log('โš ๏ธ Taxonomy schema not found, attempting to generate...'); + const extractTaxonomy = require('./extractTaxonomy'); + const taxonomies = await extractTaxonomy(dbConfig); + console.log(`โœ… Generated ${taxonomies.length} taxonomies`); + return taxonomies; + } + + console.warn('โš ๏ธ Taxonomy schema not found and no dbConfig provided for generation'); + return []; + + } catch (error) { + console.error('โŒ Error loading taxonomy schema:', error.message); + return []; + } +}; + +/** + * Corrects the UID by applying a custom affix and sanitizing the string. + * + * @param {string} uid - The original UID that needs to be corrected. + * @param {string} affix - The affix to be prepended to the UID if it's restricted. + * @returns {string} The corrected UID with the affix (if applicable) and sanitized characters. + * + * @description + * This function checks if the provided `uid` is included in the `restrictedUid` list. If it is, the function will: + * 1. Prepend the provided `affix` to the `uid`. + * 2. Replace any non-alphanumeric characters in the `uid` with underscores. + * + * It then converts any uppercase letters to lowercase and prefixes them with an underscore (to match a typical snake_case format). + * + * If the `uid` is not restricted, the function simply returns it after converting uppercase letters to lowercase and adding an underscore before each uppercase letter. + * // Outputs: 'prefix_my_restricted_uid' + */ +const uidCorrector = (uid, affix) => { + let newId = uid; + if (restrictedUid?.includes?.(uid) || uid?.startsWith?.('_ids') || uid?.endsWith?.('_ids')) { + newId = uid?.replace?.(uid, `${affix}_${uid}`); + newId = newId?.replace?.(/[^a-zA-Z0-9]+/g, '_'); + } + return newId.replace(/([A-Z])/g, (match) => `${match?.toLowerCase?.()}`); +}; + +/** + * Extracts advanced field configurations from a Drupal field item. + * + * @param {Object} item - The Drupal field item containing properties like `default_value`, `description`, etc. + * @param {Array} [referenceFields=[]] - Optional array of reference field names to associate with the field. + * @returns {Object} An object containing advanced field configurations, such as default value, validation rules, mandatory status, and more. + * + * @description + * This function extracts advanced configuration details for a Drupal field from the provided `item`. It gathers + * various settings like default values, mandatory status, localization settings, and description (with a maximum length of 255 characters). + * + * The result is an object that includes all these advanced properties, which can be used to configure fields in Contentstack. + * + * // Outputs an object with the advanced field configurations, including default value, mandatory, and more. + */ +const extractAdvancedFields = (item, referenceFields = []) => { + let description = item?.description || ''; + if (description.length > 255) { + description = description.slice(0, 255); + } + + return { + default_value: item?.default_value || null, + mandatory: item?.required || false, + multiple: item?.max > 1 || false, + unique: false, + nonLocalizable: false, + validationErrorMessage: '', + embedObjects: referenceFields.length ? referenceFields : (referenceFields.length === 0 && Array.isArray(referenceFields) ? [] : undefined), + description: description, + }; +}; + +/** + * Creates a field object for a content type, including both the main and backup field configurations. + * + * @param {Object} item - The Drupal field item that contains properties like `field_name`, `field_label`, and `type`. + * @param {string} contentstackFieldType - The type of field for Contentstack (e.g., 'text', 'json'). + * @param {string} backupFieldType - The type of backup field (e.g., 'text', 'json'). + * @param {Array} [referenceFields=[]] - Optional array of reference field names to associate with the field. + * @returns {Object} A field object that includes the UID, CMS field names, field types, and advanced configurations. + * + * @description + * This function generates a field object to be used in the context of a content management system (CMS), + * specifically for fields that have both a primary and backup configuration. It extracts the necessary field + * details from the provided `item` and augments it with additional information such as UID, field names, and field types. + * + * The advanced field properties are extracted using the `extractAdvancedFields` function, including any reference fields, + * field types, and other metadata related to the field configuration. + * + * // Outputs an object containing the field configuration for Contentstack and backup fields + */ +const createFieldObject = (item, contentstackFieldType, backupFieldType, referenceFields = []) => { + // Add suffix for specific field types (following old migration pattern) + const needsSuffix = ['reference', 'file'].includes(contentstackFieldType); + const fieldNameWithSuffix = needsSuffix ? `${item?.field_name}_target_id` : item?.field_name; + + // ๐Ÿšซ For json and html fields, always use empty embedObjects array + const shouldUseEmptyEmbedObjects = ['json', 'html'].includes(contentstackFieldType); + const finalReferenceFields = shouldUseEmptyEmbedObjects ? [] : referenceFields; + + return { + uid: item?.field_name, + otherCmsField: item?.field_label, + otherCmsType: item?.type, + contentstackField: item?.field_label, + contentstackFieldUid: uidCorrector(fieldNameWithSuffix, item?.prefix), + contentstackFieldType: contentstackFieldType, + backupFieldType: backupFieldType, + backupFieldUid: uidCorrector(fieldNameWithSuffix, item?.prefix), + advanced: extractAdvancedFields(item, finalReferenceFields) + }; +}; + +/** + * Creates a field object for dropdown or radio field types with appropriate options and validations. + * + * @param {Object} item - The Drupal field item that includes field details like `type`, etc. + * @param {string} fieldType - The type of field being created (e.g., 'dropdown', 'radio'). + * @returns {Object} A field object that includes the field configuration and validation options. + * + * @description + * This function generates a field object for dropdown or radio field types based on the provided item. + * It ensures that the field's advanced properties are extracted from the item, including validation options. + * + */ +const createDropdownOrRadioFieldObject = (item, fieldType) => { + return { + ...createFieldObject(item, fieldType, fieldType), + advanced: { + ...extractAdvancedFields(item), + options: [{ value: 'value', key: 'key' }] + } + }; +}; + +/** + * Creates a taxonomy field object with specialized structure for Contentstack + * + * @param {Object} item - The Drupal field item containing field details + * @param {Array} taxonomySchema - Array of taxonomy vocabularies from taxonomySchema.json + * @param {Array} targetVocabularies - Optional array of specific vocabularies this field references + * @returns {Object} A taxonomy field object with the required structure + */ +const createTaxonomyFieldObject = (item, taxonomySchema, targetVocabularies = []) => { + // Determine which taxonomies to include + let taxonomiesToInclude = []; + + if (targetVocabularies && targetVocabularies.length > 0) { + // If specific vocabularies are provided, use only those + taxonomiesToInclude = taxonomySchema.filter(taxonomy => + targetVocabularies.includes(taxonomy.uid) || targetVocabularies.includes(taxonomy.name) + ); + } else { + // If no specific vocabularies, include all available taxonomies + taxonomiesToInclude = taxonomySchema; + } + + // Build taxonomies array with default properties + const taxonomiesArray = taxonomiesToInclude.map(taxonomy => ({ + taxonomy_uid: taxonomy.uid, + mandatory: false, + multiple: true, + non_localizable: false + })); + + // Get advanced field properties from the original field + const advancedFields = extractAdvancedFields(item); + + // Add _target_id suffix for taxonomy fields (following old migration pattern) + const fieldNameWithSuffix = `${item?.field_name}_target_id`; + + return { + uid: item?.field_name, + otherCmsField: item?.field_label, + otherCmsType: item?.type, + contentstackField: item?.field_label, + contentstackFieldUid: uidCorrector(fieldNameWithSuffix, item?.prefix), + contentstackFieldType: 'taxonomy', + backupFieldType: 'reference', + backupFieldUid: uidCorrector(fieldNameWithSuffix, item?.prefix), + advanced: { + data_type: "taxonomy", + display_name: item?.field_label || item?.field_name, + uid: uidCorrector(fieldNameWithSuffix, item?.prefix), + taxonomies: taxonomiesArray, + field_metadata: { + description: advancedFields?.field_metadata?.description || "", + default_value: advancedFields?.field_metadata?.default_value || "" + }, + format: "", + error_messages: { format: "" }, + mandatory: advancedFields?.mandatory || false, + multiple: advancedFields?.multiple !== undefined ? advancedFields?.multiple : true, + non_localizable: advancedFields?.non_localizable || false, + unique: advancedFields?.unique || false + } + }; +}; + +/** + * Maps a collection of Drupal content type items to a schema array with specific field types and properties. + * + * @param {Array} data - An array of Drupal field items, each containing metadata like type, field_name, field_label, etc. + * @param {Array} contentTypes - Array of available content types for reference resolution. + * @param {string} prefix - The prefix to be used for UID correction. + * @param {Object} dbConfig - Database configuration for taxonomy extraction fallback. + * @returns {Promise} A schema array with field objects and corresponding properties based on the Drupal field item. + * + * @description + * This function processes each Drupal field item from the input data and maps them to a specific schema structure. + * It handles various Drupal field types and maps them to Contentstack field types with the following adaptations: + * + * Field Type Mappings: + * - Single line text โ†’ Multiline/HTML RTE/JSON RTE + * - Multiline text โ†’ HTML RTE/JSON RTE + * - HTML RTE โ†’ JSON RTE + * - JSON RTE โ†’ HTML RTE + * - Taxonomy term references โ†’ Taxonomy fields with vocabulary mappings + * + * The function supports processing of: + * - Text fields with various widget types + * - Rich text fields with associated reference fields + * - Integer/Number fields with widget-specific mappings + * - Date, Link, Array, Boolean, Object, and Location fields + * - Special handling for complex types like Entity references, File fields, and Taxonomy terms. + * - Taxonomy fields with automatic vocabulary detection and mapping. + */ +const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => { + // Load taxonomy schema for taxonomy field processing + const taxonomySchema = await loadTaxonomySchema(dbConfig); + console.log(`๐Ÿท๏ธ Loaded ${taxonomySchema.length} taxonomy vocabularies for field mapping`); + + // ๐Ÿท๏ธ Collect taxonomy fields for consolidation + const collectedTaxonomies = []; + + const schemaArray = data.reduce((acc, item) => { + // Add prefix to item for UID correction + item.prefix = prefix; + + switch (item.type) { + case 'text_with_summary': + case 'text_long': { + // Rich text with switching options: JSON RTE โ†’ HTML RTE โ†’ multiline โ†’ text + const availableContentTypes = contentTypes?.filter(ct => ct !== item.content_types) || []; + const referenceFields = availableContentTypes.slice(0, 10); + console.log(`๐Ÿ”„ Rich text field ${item.field_name} using backup content types:`, referenceFields); + acc.push(createFieldObject(item, 'json', 'html', referenceFields)); + break; + } + case 'text': { + // Single line with switching options: single_line โ†’ multiline โ†’ HTML RTE โ†’ JSON RTE + const availableContentTypes = contentTypes?.filter(ct => ct !== item.content_types) || []; + const referenceFields = availableContentTypes.slice(0, 10); + console.log(`๐Ÿ”„ Text field ${item.field_name} using backup content types:`, referenceFields); + acc.push(createFieldObject(item, 'single_line_text', 'multi_line_text', referenceFields)); + break; + } + case 'string_long': + case 'comment': { + // Rich text with switching options: JSON RTE โ†’ HTML RTE โ†’ multiline โ†’ text + const availableContentTypes = contentTypes?.filter(ct => ct !== item.content_types) || []; + const referenceFields = availableContentTypes.slice(0, 10); + console.log(`๐Ÿ”„ Rich text field ${item.field_name} using backup content types:`, referenceFields); + acc.push(createFieldObject(item, 'json', 'html', referenceFields)); + break; + } + case 'string': + case 'list_string': { + // Single line with switching options: single_line โ†’ multiline โ†’ HTML RTE โ†’ JSON RTE + const availableContentTypes = contentTypes?.filter(ct => ct !== item.content_types) || []; + const referenceFields = availableContentTypes.slice(0, 10); + console.log(`๐Ÿ”„ Text field ${item.field_name} using backup content types:`, referenceFields); + acc.push(createFieldObject(item, 'single_line_text', 'multi_line_text', referenceFields)); + break; + } + case 'email': { + acc.push(createFieldObject(item, 'text', 'text')); + break; + } + case 'taxonomy_term_reference': { + // ๐Ÿท๏ธ Collect taxonomy field for consolidation instead of creating individual fields + if (taxonomySchema && taxonomySchema.length > 0) { + console.log(`๐Ÿท๏ธ Collecting taxonomy field for consolidation: ${item.field_name}`); + + // Try to determine specific vocabularies this field references + let targetVocabularies = []; + + // Check if field has handler settings that specify target vocabularies + if (item.handler_settings && item.handler_settings.target_bundles) { + targetVocabularies = Object.keys(item.handler_settings.target_bundles); + console.log(`๐ŸŽฏ Found target vocabularies for ${item.field_name}:`, targetVocabularies); + } + + // Add vocabularies to collection (avoid duplicates) + if (targetVocabularies && targetVocabularies.length > 0) { + targetVocabularies.forEach(vocab => { + if (!collectedTaxonomies.includes(vocab)) { + collectedTaxonomies.push(vocab); + } + }); + } else { + // Backup: Use all available taxonomies from taxonomySchema.json + taxonomySchema.forEach(taxonomy => { + if (!collectedTaxonomies.includes(taxonomy.uid)) { + collectedTaxonomies.push(taxonomy.uid); + } + }); + } + } else { + // Fallback to regular reference field if no taxonomy schema available + console.warn(`โš ๏ธ No taxonomy schema available for field: ${item.field_name}, using reference field`); + acc.push(createFieldObject(item, 'reference', 'reference', ['taxonomy'])); + } + break; + } + case 'entity_reference': { + // Check if this is a taxonomy field by handler + if (item.handler === 'default:taxonomy_term') { + // ๐Ÿท๏ธ Collect taxonomy field for consolidation instead of creating individual fields + console.log(`๐Ÿท๏ธ Collecting taxonomy field for consolidation: ${item.field_name}`); + + // Try to determine specific vocabularies this field references + let targetVocabularies = []; + + // Check if field has handler settings that specify target vocabularies + if (item.reference) { + targetVocabularies = Object.keys(item.reference); + console.log(`๐ŸŽฏ Found target vocabularies for ${item.field_name}:`, targetVocabularies); + } + + if (taxonomySchema && taxonomySchema.length > 0) { + // Add vocabularies to collection (avoid duplicates) + if (targetVocabularies && targetVocabularies.length > 0) { + targetVocabularies.forEach(vocab => { + if (!collectedTaxonomies.includes(vocab)) { + collectedTaxonomies.push(vocab); + } + }); + } else { + // Backup: Use all available taxonomies from taxonomySchema.json + taxonomySchema.forEach(taxonomy => { + if (!collectedTaxonomies.includes(taxonomy.uid)) { + collectedTaxonomies.push(taxonomy.uid); + } + }); + } + } else { + // Fallback to regular reference field if no taxonomy schema available + console.warn(`โš ๏ธ No taxonomy schema available for field: ${item.field_name}, using reference field`); + // Use available content types instead of generic 'taxonomy' + const availableContentTypes = contentTypes?.filter(ct => ct !== item.content_types) || []; + const referenceFields = availableContentTypes.slice(0, 10); + acc.push(createFieldObject(item, 'reference', 'reference', referenceFields)); + } + } else if (item.handler === 'default:node') { + // Handle node reference fields - use specific content types from reference settings + let referenceFields = []; + + if (item.reference && Object.keys(item.reference).length > 0) { + // Use specific content types from field configuration + referenceFields = Object.keys(item.reference); + console.log(`๐ŸŽฏ Found specific reference content types for ${item.field_name}:`, referenceFields); + } else { + // Backup: Use up to 10 content types from available content types + const availableContentTypes = contentTypes?.filter(ct => ct !== item.content_types) || []; + referenceFields = availableContentTypes.slice(0, 10); + console.log(`๐Ÿ”„ No specific content types found for ${item.field_name}, using backup content types:`, referenceFields); + } + + acc.push(createFieldObject(item, 'reference', 'reference', referenceFields)); + } else { + // Handle other entity references - exclude taxonomy and limit to 10 content types + const availableContentTypes = contentTypes?.filter(ct => ct !== item.content_types) || []; + const referenceFields = availableContentTypes.slice(0, 10); + console.log(`๐Ÿ“‹ Using available content types for ${item.field_name}:`, referenceFields); + acc.push(createFieldObject(item, 'reference', 'reference', referenceFields)); + } + break; + } + case 'image': + case 'file': { + acc.push(createFieldObject(item, 'file', 'file')); + break; + } + case 'list_boolean': + case 'boolean': { + acc.push(createFieldObject(item, 'boolean', 'boolean')); + break; + } + case 'datetime': + case 'timestamp': { + acc.push(createFieldObject(item, 'isodate', 'isodate')); + break; + } + case 'integer': + case 'decimal': + case 'float': + case 'list_integer': + case 'list_float': { + acc.push(createFieldObject(item, 'number', 'number')); + break; + } + case 'link': { + acc.push(createFieldObject(item, 'link', 'link')); + break; + } + case 'list_text': { + acc.push(createDropdownOrRadioFieldObject(item, 'dropdown')); + break; + } + case 'list_number': { + acc.push(createDropdownOrRadioFieldObject(item, 'dropdown')); + break; + } + default: { + // Default to single line text for unknown types + acc.push(createFieldObject(item, 'single_line_text', 'single_line_text')); + break; + } + } + return acc; + }, []); + + // Add default title and url fields if not present + const hasTitle = schemaArray.some(field => field.uid === 'title'); + const hasUrl = schemaArray.some(field => field.uid === 'url'); + + if (!hasTitle) { + schemaArray.unshift({ + uid: 'title', + otherCmsField: 'title', + otherCmsType: 'text', + contentstackField: 'title', + contentstackFieldUid: 'title', + contentstackFieldType: 'text', + backupFieldType: 'text', + backupFieldUid: 'title', + advanced: { mandatory: true } + }); + } + + if (!hasUrl) { + schemaArray.splice(1, 0, { + uid: 'url', + otherCmsField: 'url', + otherCmsType: 'text', + contentstackField: 'Url', + contentstackFieldUid: 'url', + contentstackFieldType: 'url', + backupFieldType: 'url', + backupFieldUid: 'url', + advanced: { mandatory: true } + }); + } + + // ๐Ÿท๏ธ TAXONOMY CONSOLIDATION: Create single consolidated taxonomy field if any taxonomies were collected + if (collectedTaxonomies.length > 0) { + console.log(`๐Ÿท๏ธ Creating consolidated taxonomy field with ${collectedTaxonomies.length} taxonomies:`, collectedTaxonomies); + + // Create consolidated taxonomy field with fixed properties + const consolidatedTaxonomyField = { + uid: 'taxonomies', + otherCmsField: 'Taxonomy', + otherCmsType: 'taxonomy', + contentstackField: 'Taxonomy', + contentstackFieldUid: 'taxonomies', + contentstackFieldType: 'taxonomy', + backupFieldType: 'taxonomy', + backupFieldUid: 'taxonomies', + advanced: { + taxonomies: collectedTaxonomies.map(taxonomyUid => ({ + taxonomy_uid: taxonomyUid, + mandatory: false, + multiple: true, + non_localizable: false + })), + mandatory: false, + multiple: true, + non_localizable: false, + unique: false + } + }; + + // Add consolidated taxonomy field at the end of schema + schemaArray.push(consolidatedTaxonomyField); + console.log(`โœ… Added consolidated taxonomy field 'taxonomies' with ${collectedTaxonomies.length} vocabularies`); + } + + return schemaArray; +}; + +module.exports = contentTypeMapper; diff --git a/upload-api/migration-drupal/libs/createInitialMapper.js b/upload-api/migration-drupal/libs/createInitialMapper.js new file mode 100644 index 000000000..9582b9781 --- /dev/null +++ b/upload-api/migration-drupal/libs/createInitialMapper.js @@ -0,0 +1,147 @@ +'use strict'; +/* eslint-disable @typescript-eslint/no-var-requires */ + +/** + * External module dependencies. + */ +const fs = require('fs'); // for existsSync +const fsp = require('fs/promises'); // for async file operations +const path = require('path'); +const contentTypeMapper = require('./contentTypeMapper'); + +/** + * Internal module dependencies. + */ +const { dbConnection, writeFile } = require('../utils/helper'); +const config = require('../config'); +const idArray = require('../utils/restrictedKeyWords'); + +/** + * Corrects the UID by adding a prefix and sanitizing the string if it is found in a specified list. + */ +const uidCorrector = (uid, prefix) => { + let newId = uid; + if (idArray.includes(uid) || uid.startsWith('_ids') || uid.endsWith('_ids')) { + newId = uid.replace(uid, `${prefix}_${uid}`); + newId = newId.replace(/[^a-zA-Z0-9]+/g, '_'); + } + return newId.replace(/([A-Z])/g, (match) => `_${match.toLowerCase()}`); +}; + +/** + * Creates an initial mapping for content types by processing Drupal database data. + */ +const createInitialMapper = async (systemConfig, prefix) => { + try { + const drupalFolderPath = path.resolve(config.data, config.drupal.drupal); + + if (!fs.existsSync(drupalFolderPath)) { + await fsp.mkdir(drupalFolderPath, { recursive: true }); + } + + console.log('Extracting content types from Drupal database...'); + + // Get database connection + const connection = dbConnection(systemConfig); + + // SQL query to get field configurations + const query = "SELECT *, CONVERT(data USING utf8) as data FROM config WHERE name LIKE '%field.field.node%'"; + + // Execute query + const [rows] = await connection.promise().query(query); + + const details_data = []; + + // Process each row to extract field data + for (let i = 0; i < rows.length; i++) { + try { + const { unserialize } = require('php-serialize'); + const conv_details = unserialize(rows[i].data); + + details_data.push({ + field_label: conv_details?.label, + description: conv_details?.description, + field_name: conv_details?.field_name, + content_types: conv_details?.bundle, + type: conv_details?.field_type, + handler: conv_details?.settings?.handler, + reference: conv_details?.settings?.handler_settings?.target_bundles, + min: conv_details?.settings?.min, + max: conv_details?.settings?.max, + default_value: conv_details?.default_value?.[0]?.value + }); + } catch (error) { + console.warn(`Couldn't parse row ${i}:`, error.message); + } + } + + if (details_data.length === 0) { + console.log('No content types found to process'); + return { contentTypes: [] }; + } + + console.log(`Processing ${details_data.length} content type entries...`); + + const initialMapper = []; + const contentTypes = Object.keys(require('lodash').keyBy(details_data, 'content_types')); + + // Process each content type + for (const contentType of contentTypes) { + const contentTypeFields = require('lodash').filter(details_data, { content_types: contentType }); + const contenttypeTitle = contentType.split('_').join(' '); + + const contentTypeObject = { + status: 1, + isUpdated: false, + updateAt: '', + otherCmsTitle: contenttypeTitle, + otherCmsUid: contenttypeTitle, + contentstackTitle: contenttypeTitle.charAt(0).toUpperCase() + contenttypeTitle.slice(1), + contentstackUid: uidCorrector(contenttypeTitle, prefix), + type: 'content_type', + fieldMapping: [] + }; + + // Map fields using contentTypeMapper + const contentstackFields = await contentTypeMapper(contentTypeFields, contentTypes, prefix, systemConfig.mysql); + contentTypeObject.fieldMapping = contentstackFields; + + initialMapper.push(contentTypeObject); + + // Save individual content type file + const main = { + title: contenttypeTitle, + uid: contentType, + schema: contentstackFields, + description: `Schema for ${contenttypeTitle}`, + options: { + is_page: true, + singleton: false, + sub_title: [], + title: `title`, + url_pattern: '/:title', + url_prefix: `/${contenttypeTitle.replace(/[^a-zA-Z0-9]+/g, '').toLowerCase()}/` + } + }; + + await fsp.writeFile( + path.join(drupalFolderPath, `${contentType}.json`), + JSON.stringify(main, null, 4) + ); + } + + console.log(`Successfully processed ${initialMapper.length} content types`); + console.log('Content type extraction completed successfully'); + + // Close database connection + connection.end(); + console.log('Database connection closed'); + + return { contentTypes: initialMapper }; + } catch (error) { + console.error('Error in content type extraction:', error); + throw error; + } +}; + +module.exports = createInitialMapper; diff --git a/upload-api/migration-drupal/libs/extractLocale.js b/upload-api/migration-drupal/libs/extractLocale.js new file mode 100644 index 000000000..fea1e77f4 --- /dev/null +++ b/upload-api/migration-drupal/libs/extractLocale.js @@ -0,0 +1,100 @@ +'use strict'; +/* eslint-disable @typescript-eslint/no-var-requires */ +/** + * External module dependencies. + */ +const { dbConnection} = require('../utils/helper'); + +/** + * Apply locale transformation rules (same logic as API side) + * - "und" alone โ†’ "en-us" + * - "und" + "en-us" โ†’ both become "en" + * - "en" + "en-us" โ†’ "en" becomes "und", "en-us" stays + * - All three โ†’ "en"+"und" become "und", "en-us" stays + */ +function applyLocaleTransformations(originalLocales) { + const locales = [...originalLocales]; // Copy to avoid mutation + const hasUnd = locales.includes('und'); + const hasEn = locales.includes('en'); + const hasEnUs = locales.includes('en-us'); + + console.log(`๐Ÿ” Locale Analysis: hasUnd=${hasUnd}, hasEn=${hasEn}, hasEnUs=${hasEnUs}`); + + const transformedSet = new Set(); + + // Apply transformation rules + locales.forEach(locale => { + if (locale === 'und') { + if (hasEn && hasEnUs) { + // If all three present, "und" stays as "und" + transformedSet.add('und'); + console.log(`๐Ÿ”„ "und" stays as "und" (all three present)`); + } else if (hasEnUs) { + // If "und" + "en-us", "und" becomes "en" + transformedSet.add('en'); + console.log(`๐Ÿ”„ Transforming "und" โ†’ "en" (en-us exists)`); + } else { + // If only "und", becomes "en-us" + transformedSet.add('en-us'); + console.log(`๐Ÿ”„ Transforming "und" โ†’ "en-us"`); + } + } else if (locale === 'en-us') { + if (hasUnd && !hasEn) { + // If "und" + "en-us" (no en), "und" becomes "en", so keep "en-us" + transformedSet.add('en-us'); + console.log(`๐Ÿ”„ "en-us" stays as "en-us" (und becomes en)`); + } else { + // Keep en-us as is in other cases + transformedSet.add('en-us'); + } + } else if (locale === 'en') { + if (hasEnUs && !hasUnd) { + // If "en" + "en-us" (no und), "en" becomes "und" + transformedSet.add('und'); + console.log(`๐Ÿ”„ Transforming "en" โ†’ "und" (en-us exists, no und)`); + } else { + // Keep "en" as is in other cases + transformedSet.add('en'); + } + } else { + // Keep other locales as is + transformedSet.add(locale); + } + }); + + return Array.from(transformedSet).sort(); +} + +const extractLocale = async ( systemConfig ) => +{ + let connection; + try + { + // Get database connection - pass your MySQL config + connection = await dbConnection( systemConfig ); + + // DYNAMIC locale extraction - query directly for unique language codes + console.log('๐ŸŒ Extracting locales dynamically from Drupal database...'); + + // Simple query to get all unique language codes from content + const localeQuery = "SELECT DISTINCT langcode FROM node_field_data WHERE langcode IS NOT NULL AND langcode != '' ORDER BY langcode"; + + const [localeRows] = await connection.promise().query(localeQuery); + const originalLocales = localeRows.map(row => row.langcode).filter(locale => locale && locale.trim()); + + console.log(`๐Ÿ“ Found ${originalLocales.length} original locales:`, originalLocales); + + // ๐Ÿ”„ Apply locale transformation rules for UI consistency + const transformedLocales = applyLocaleTransformations(originalLocales); + + console.log(`โœ… Transformed to ${transformedLocales.length} locales for UI:`, transformedLocales); + + return transformedLocales; + + } catch (error) { + console.error(`Error reading JSON file:`, error); + return []; + } +}; + +module.exports = extractLocale; diff --git a/upload-api/migration-drupal/libs/extractTaxonomy.js b/upload-api/migration-drupal/libs/extractTaxonomy.js new file mode 100644 index 000000000..750174728 --- /dev/null +++ b/upload-api/migration-drupal/libs/extractTaxonomy.js @@ -0,0 +1,127 @@ +const mysql = require('mysql2'); +const fs = require('fs'); +const path = require('path'); +const { unserialize } = require('php-unserialize'); + +/** + * Execute SQL query with promise support + */ +const executeQuery = async (connection, query) => { + return new Promise((resolve, reject) => { + connection.query(query, (error, results) => { + if (error) { + reject(error); + return; + } + resolve(results); + }); + }); +}; + +/** + * Generate slug from name (similar to Contentstack uid format) + */ +const generateSlug = (name) => { + return name + .toLowerCase() + .replace(/[^a-z0-9\s-]/g, '') // Remove special characters + .replace(/\s+/g, '_') // Replace spaces with underscores + .replace(/-+/g, '_') // Replace hyphens with underscores + .replace(/_+/g, '_') // Replace multiple underscores with single + .replace(/^_|_$/g, ''); // Remove leading/trailing underscores +}; + +/** + * Extract taxonomy vocabularies (parent-level only) from Drupal database + * and save them to drupalMigrationData/taxonomySchema/taxonomySchema.json + * + * @param {Object} dbConfig - Database configuration + * @returns {Promise} Array of vocabulary objects with uid and name + */ +const extractTaxonomy = async (dbConfig) => { + console.log('๐Ÿ” === EXTRACTING TAXONOMY VOCABULARIES ==='); + console.log('๐Ÿ“‹ Database Config:', JSON.stringify(dbConfig, null, 2)); + + let connection; + + try { + // Create database connection + connection = mysql.createConnection(dbConfig); + + // Test connection + await new Promise((resolve, reject) => { + connection.connect((err) => { + if (err) reject(err); + else resolve(); + }); + }); + + console.log('โœ… Database connection successful'); + + // Extract vocabularies from Drupal config table (Drupal 8+) + // This gets only the parent vocabularies, not individual terms + const vocabularyQuery = ` + SELECT + SUBSTRING_INDEX(SUBSTRING_INDEX(name, '.', 3), '.', -1) as vid, + CONVERT(data USING utf8) as data + FROM config + WHERE name LIKE 'taxonomy.vocabulary.%' + AND data IS NOT NULL + `; + + console.log('๐Ÿ” Executing vocabulary query...'); + const vocabularies = await executeQuery(connection, vocabularyQuery); + + console.log(`๐Ÿ“‹ Found ${vocabularies.length} vocabularies`); + + // Transform vocabularies to required format + const taxonomySchema = []; + + for (const vocab of vocabularies) { + try { + if (vocab.vid && vocab.data) { + // Unserialize the PHP data to get vocabulary details + const vocabularyData = unserialize(vocab.data); + + if (vocabularyData && vocabularyData.name) { + const uid = generateSlug(vocab.vid); // Use vid as base for uid + const name = vocabularyData.name; + + taxonomySchema.push({ + uid: uid, + name: name + }); + + console.log(`โœ… Added vocabulary: ${uid} (${name})`); + } + } + } catch (parseError) { + console.warn(`โš ๏ธ Failed to parse vocabulary data for ${vocab.vid}:`, parseError.message); + } + } + + // Create output directory + const outputDir = path.join(__dirname, '..', '..', 'drupalMigrationData', 'taxonomySchema'); + await fs.promises.mkdir(outputDir, { recursive: true }); + + // Save taxonomy schema to file + const outputPath = path.join(outputDir, 'taxonomySchema.json'); + await fs.promises.writeFile(outputPath, JSON.stringify(taxonomySchema, null, 2)); + + console.log(`โœ… Taxonomy schema saved to: ${outputPath}`); + console.log(`๐Ÿ“Š Total vocabularies extracted: ${taxonomySchema.length}`); + console.log('=========================================='); + + return taxonomySchema; + + } catch (error) { + console.error('โŒ Error extracting taxonomy:', error.message); + throw error; + } finally { + if (connection) { + connection.end(); + } + } +}; + +module.exports = extractTaxonomy; diff --git a/upload-api/migration-drupal/utils/helper.js b/upload-api/migration-drupal/utils/helper.js new file mode 100755 index 000000000..5de737297 --- /dev/null +++ b/upload-api/migration-drupal/utils/helper.js @@ -0,0 +1,70 @@ +/* eslint-disable @typescript-eslint/no-var-requires */ +/** + * External module Dependencies. + */ +const mkdirp = require('mkdirp'); +const path = require('path'); +const fs = require('fs'); +const mysql = require('mysql2'); + +const readFile = function (filePath, parse) { + parse = typeof parse == 'undefined' ? true : parse; + filePath = path.resolve(filePath); + let data; + if (fs.existsSync(filePath)) data = parse ? JSON.parse(fs.readFileSync(filePath, 'utf-8')) : data; + return data; +}; + +const writeFile = function (filePath, data) { + filePath = path.resolve(filePath); + data = typeof data == 'object' ? JSON.stringify(data) : data || '{}'; + fs.writeFileSync(filePath, data, 'utf-8'); +}; + +const appendFile = function (filePath, data) { + filePath = path.resolve(filePath); + fs.appendFileSync(filePath, data); +}; + +const makeDirectory = function () { + for (let key in arguments) { + let dirname = path.resolve(arguments[key]); + if (!fs.existsSync(dirname)) mkdirp.sync(dirname); + } +}; + +function deleteFolderSync(folderPath) { + if (fs.existsSync(folderPath)) { + fs.readdirSync(folderPath).forEach((file) => { + const currentPath = path.join(folderPath, file); + if (fs.lstatSync(currentPath).isDirectory()) { + // Recurse + deleteFolderSync(currentPath); + } else { + // Delete file + fs.unlinkSync(currentPath); + } + }); + // Delete now-empty folder + fs.rmdirSync(folderPath); + } +} + +function dbConnection(config) { + var connection = mysql.createConnection({ + host: config['mysql']['host'], + user: config['mysql']['user'], + password: config['mysql']['password'], + database: config['mysql']['database'] + }); + return connection; +} + +module.exports = { + readFile, + writeFile, + appendFile, + makeDirectory, + deleteFolderSync, + dbConnection +}; diff --git a/upload-api/migration-drupal/utils/restrictedKeyWords/index.json b/upload-api/migration-drupal/utils/restrictedKeyWords/index.json new file mode 100644 index 000000000..ff93e6f3b --- /dev/null +++ b/upload-api/migration-drupal/utils/restrictedKeyWords/index.json @@ -0,0 +1,47 @@ +[ + "uid", + "api_key", + "created_at", + "deleted_at", + "updated_at", + "tags_array", + "klass_id", + "applikation_id", + "id", + "_id", + "ACL", + "SYS_ACL", + "DEFAULT_ACL", + "app_user_object_uid", + "built_io_upload", + "__loc", + "tags", + "_owner", + "_version", + "toJSON", + "save", + "update", + "domain", + "share_account", + "shard_app", + "shard_random", + "hook", + "__indexes", + "__meta", + "created_by", + "updated_by", + "inbuilt_class", + "tenant_id", + "isSystemUser", + "isApplicationUser", + "isNew", + "_shouldLean", + "_shouldFilter", + "options", + "_version", + "__v", + "locale", + "publish_details", + "title", + "url" +] \ No newline at end of file diff --git a/upload-api/nodemon.json b/upload-api/nodemon.json new file mode 100644 index 000000000..04640944f --- /dev/null +++ b/upload-api/nodemon.json @@ -0,0 +1,15 @@ +{ + "watch": ["src"], + "ext": "ts,js,json", + "ignore": [ + "drupalMigrationData/**/*", + "migration-*/**/*", + "build/**/*", + "node_modules/**/*", + "*.log", + "extracted_files/**/*" + ], + "env": { + "NODE_ENV": "development" + } +} diff --git a/upload-api/package-lock.json b/upload-api/package-lock.json index 7ae610fed..171df3e30 100644 --- a/upload-api/package-lock.json +++ b/upload-api/package-lock.json @@ -22,12 +22,17 @@ "jszip": "^3.10.1", "lodash.isempty": "^4.4.0", "migration-contentful": "file:migration-contentful", + "migration-drupal": "file:migration-drupal", "migration-sitecore": "file:migration-sitecore", "migration-wordpress": "file:migration-wordpress", "multer": "^2.0.0", + "mysql2": "^3.14.3", "node-fetch": "^2.7.0", "nodemon": "^3.1.9", + "php-serialize": "^5.1.3", + "php-unserialize": "^0.0.1", "prettier": "^3.3.3", + "when": "^3.7.8", "winston": "^3.17.0", "xml2js": "^0.6.2" }, @@ -44,6 +49,7 @@ "eslint-config-prettier": "^9.1.0", "eslint-plugin-prettier": "^5.1.3", "helmet": "^7.1.0", + "tsx": "^4.7.1", "typescript": "^5.3.3" } }, @@ -74,6 +80,7 @@ "uuid": "dist/bin/uuid" } }, + "migration-drupal": {}, "migration-sitecore": { "name": "migration-v2-sitecore", "version": "1.0.0", @@ -1181,6 +1188,422 @@ "kuler": "^2.0.0" } }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.9.tgz", + "integrity": "sha512-OaGtL73Jck6pBKjNIe24BnFE6agGl+6KxDtTfHhy1HmhthfKouEcOhqpSL64K4/0WCtbKFLOdzD/44cJ4k9opA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.9.tgz", + "integrity": "sha512-5WNI1DaMtxQ7t7B6xa572XMXpHAaI/9Hnhk8lcxF4zVN4xstUgTlvuGDorBguKEnZO70qwEcLpfifMLoxiPqHQ==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.9.tgz", + "integrity": "sha512-IDrddSmpSv51ftWslJMvl3Q2ZT98fUSL2/rlUXuVqRXHCs5EUF1/f+jbjF5+NG9UffUDMCiTyh8iec7u8RlTLg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.9.tgz", + "integrity": "sha512-I853iMZ1hWZdNllhVZKm34f4wErd4lMyeV7BLzEExGEIZYsOzqDWDf+y082izYUE8gtJnYHdeDpN/6tUdwvfiw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.9.tgz", + "integrity": "sha512-XIpIDMAjOELi/9PB30vEbVMs3GV1v2zkkPnuyRRURbhqjyzIINwj+nbQATh4H9GxUgH1kFsEyQMxwiLFKUS6Rg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.9.tgz", + "integrity": "sha512-jhHfBzjYTA1IQu8VyrjCX4ApJDnH+ez+IYVEoJHeqJm9VhG9Dh2BYaJritkYK3vMaXrf7Ogr/0MQ8/MeIefsPQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.9.tgz", + "integrity": "sha512-z93DmbnY6fX9+KdD4Ue/H6sYs+bhFQJNCPZsi4XWJoYblUqT06MQUdBCpcSfuiN72AbqeBFu5LVQTjfXDE2A6Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.9.tgz", + "integrity": "sha512-mrKX6H/vOyo5v71YfXWJxLVxgy1kyt1MQaD8wZJgJfG4gq4DpQGpgTB74e5yBeQdyMTbgxp0YtNj7NuHN0PoZg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.9.tgz", + "integrity": "sha512-HBU2Xv78SMgaydBmdor38lg8YDnFKSARg1Q6AT0/y2ezUAKiZvc211RDFHlEZRFNRVhcMamiToo7bDx3VEOYQw==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.9.tgz", + "integrity": "sha512-BlB7bIcLT3G26urh5Dmse7fiLmLXnRlopw4s8DalgZ8ef79Jj4aUcYbk90g8iCa2467HX8SAIidbL7gsqXHdRw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.9.tgz", + "integrity": "sha512-e7S3MOJPZGp2QW6AK6+Ly81rC7oOSerQ+P8L0ta4FhVi+/j/v2yZzx5CqqDaWjtPFfYz21Vi1S0auHrap3Ma3A==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.9.tgz", + "integrity": "sha512-Sbe10Bnn0oUAB2AalYztvGcK+o6YFFA/9829PhOCUS9vkJElXGdphz0A3DbMdP8gmKkqPmPcMJmJOrI3VYB1JQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.9.tgz", + "integrity": "sha512-YcM5br0mVyZw2jcQeLIkhWtKPeVfAerES5PvOzaDxVtIyZ2NUBZKNLjC5z3/fUlDgT6w89VsxP2qzNipOaaDyA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.9.tgz", + "integrity": "sha512-++0HQvasdo20JytyDpFvQtNrEsAgNG2CY1CLMwGXfFTKGBGQT3bOeLSYE2l1fYdvML5KUuwn9Z8L1EWe2tzs1w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.9.tgz", + "integrity": "sha512-uNIBa279Y3fkjV+2cUjx36xkx7eSjb8IvnL01eXUKXez/CBHNRw5ekCGMPM0BcmqBxBcdgUWuUXmVWwm4CH9kg==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.9.tgz", + "integrity": "sha512-Mfiphvp3MjC/lctb+7D287Xw1DGzqJPb/J2aHHcHxflUo+8tmN/6d4k6I2yFR7BVo5/g7x2Monq4+Yew0EHRIA==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.9.tgz", + "integrity": "sha512-iSwByxzRe48YVkmpbgoxVzn76BXjlYFXC7NvLYq+b+kDjyyk30J0JY47DIn8z1MO3K0oSl9fZoRmZPQI4Hklzg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.9.tgz", + "integrity": "sha512-9jNJl6FqaUG+COdQMjSCGW4QiMHH88xWbvZ+kRVblZsWrkXlABuGdFJ1E9L7HK+T0Yqd4akKNa/lO0+jDxQD4Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.9.tgz", + "integrity": "sha512-RLLdkflmqRG8KanPGOU7Rpg829ZHu8nFy5Pqdi9U01VYtG9Y0zOG6Vr2z4/S+/3zIyOxiK6cCeYNWOFR9QP87g==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.9.tgz", + "integrity": "sha512-YaFBlPGeDasft5IIM+CQAhJAqS3St3nJzDEgsgFixcfZeyGPCd6eJBWzke5piZuZ7CtL656eOSYKk4Ls2C0FRQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.9.tgz", + "integrity": "sha512-1MkgTCuvMGWuqVtAvkpkXFmtL8XhWy+j4jaSO2wxfJtilVCi0ZE37b8uOdMItIHz4I6z1bWWtEX4CJwcKYLcuA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.9.tgz", + "integrity": "sha512-4Xd0xNiMVXKh6Fa7HEJQbrpP3m3DDn43jKxMjxLLRjWnRsfxjORYJlXPO4JNcXtOyfajXorRKY9NkOpTHptErg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.9.tgz", + "integrity": "sha512-WjH4s6hzo00nNezhp3wFIAfmGZ8U7KtrJNlFMRKxiI9mxEK1scOMAaa9i4crUtu+tBr+0IN6JCuAcSBJZfnphw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.9.tgz", + "integrity": "sha512-mGFrVJHmZiRqmP8xFOc6b84/7xa5y5YvR1x8djzXpJBSv/UsNK6aqec+6JDjConTgvvQefdGhFDAs2DLAds6gQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.9.tgz", + "integrity": "sha512-b33gLVU2k11nVx1OhX3C8QQP6UHQK4ZtN56oFWvVXvz2VkDoe6fbG8TOgHFxEvqeqohmRnIHe5A1+HADk4OQww==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.9.tgz", + "integrity": "sha512-PPOl1mi6lpLNQxnGoyAfschAodRFYXJ+9fs6WHXz7CSWKbOqiMZsubC+BQsVKuul+3vKLuwTHsS2c2y9EoKwxQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.7.0", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.7.0.tgz", @@ -3413,6 +3836,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/aws-ssl-profiles": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/aws-ssl-profiles/-/aws-ssl-profiles-1.1.2.tgz", + "integrity": "sha512-NZKeq9AfyQvEeNlN0zSYAaWrmBffJh3IELMZfRpJVWgrpEbtEpnjvzqBPf+mxoI287JohRDoa+/nsfqqiZmF6g==", + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/axios": { "version": "1.11.0", "resolved": "https://registry.npmjs.org/axios/-/axios-1.11.0.tgz", @@ -4351,6 +4782,14 @@ "node": ">=0.4.0" } }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -4663,6 +5102,47 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/esbuild": { + "version": "0.25.9", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.9.tgz", + "integrity": "sha512-CRbODhYyQx3qp7ZEwzxOk4JBqmD/seJrzPa/cGjY1VtIn5E09Oi9/dB4JwctnfZ8Q8iT7rioVv5k/FNT/uf54g==", + "dev": true, + "hasInstallScript": true, + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.9", + "@esbuild/android-arm": "0.25.9", + "@esbuild/android-arm64": "0.25.9", + "@esbuild/android-x64": "0.25.9", + "@esbuild/darwin-arm64": "0.25.9", + "@esbuild/darwin-x64": "0.25.9", + "@esbuild/freebsd-arm64": "0.25.9", + "@esbuild/freebsd-x64": "0.25.9", + "@esbuild/linux-arm": "0.25.9", + "@esbuild/linux-arm64": "0.25.9", + "@esbuild/linux-ia32": "0.25.9", + "@esbuild/linux-loong64": "0.25.9", + "@esbuild/linux-mips64el": "0.25.9", + "@esbuild/linux-ppc64": "0.25.9", + "@esbuild/linux-riscv64": "0.25.9", + "@esbuild/linux-s390x": "0.25.9", + "@esbuild/linux-x64": "0.25.9", + "@esbuild/netbsd-arm64": "0.25.9", + "@esbuild/netbsd-x64": "0.25.9", + "@esbuild/openbsd-arm64": "0.25.9", + "@esbuild/openbsd-x64": "0.25.9", + "@esbuild/openharmony-arm64": "0.25.9", + "@esbuild/sunos-x64": "0.25.9", + "@esbuild/win32-arm64": "0.25.9", + "@esbuild/win32-ia32": "0.25.9", + "@esbuild/win32-x64": "0.25.9" + } + }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", @@ -5532,6 +6012,14 @@ "node": ">= 0.6.0" } }, + "node_modules/generate-function": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.3.1.tgz", + "integrity": "sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ==", + "dependencies": { + "is-property": "^1.0.2" + } + }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -5599,6 +6087,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-tsconfig": { + "version": "4.10.1", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.10.1.tgz", + "integrity": "sha512-auHyJ4AgMz7vgS8Hp3N6HXSmlMdUyhSUrfBF16w153rxtLIEOE+HGqaBppczZvnHLqQJfiHotCYpNhl0lUROFQ==", + "dev": true, + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, "node_modules/glob": { "version": "9.3.5", "resolved": "https://registry.npmjs.org/glob/-/glob-9.3.5.tgz", @@ -6756,6 +7256,11 @@ "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==" }, + "node_modules/is-property": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", + "integrity": "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g==" + }, "node_modules/is-regex": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.1.tgz", @@ -7251,11 +7756,30 @@ "node": ">= 12.0.0" } }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==" + }, "node_modules/lru-cache": { "version": "10.4.3", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==" }, + "node_modules/lru.min": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/lru.min/-/lru.min-1.1.2.tgz", + "integrity": "sha512-Nv9KddBcQSlQopmBHXSsZVY5xsdlZkdH/Iey0BlcBYggMd4two7cZnKOK9vmy3nY0O5RGH99z1PCeTpPqszUYg==", + "engines": { + "bun": ">=1.0.0", + "deno": ">=1.30.0", + "node": ">=8.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wellwelwel" + } + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -7321,6 +7845,10 @@ "resolved": "migration-contentful", "link": true }, + "node_modules/migration-drupal": { + "resolved": "migration-drupal", + "link": true + }, "node_modules/migration-sitecore": { "resolved": "migration-sitecore", "link": true @@ -7490,6 +8018,44 @@ "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==" }, + "node_modules/mysql2": { + "version": "3.14.3", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.14.3.tgz", + "integrity": "sha512-fD6MLV8XJ1KiNFIF0bS7Msl8eZyhlTDCDl75ajU5SJtpdx9ZPEACulJcqJWr1Y8OYyxsFc4j3+nflpmhxCU5aQ==", + "dependencies": { + "aws-ssl-profiles": "^1.1.1", + "denque": "^2.1.0", + "generate-function": "^2.3.1", + "iconv-lite": "^0.6.3", + "long": "^5.2.1", + "lru.min": "^1.0.0", + "named-placeholders": "^1.1.3", + "seq-queue": "^0.0.5", + "sqlstring": "^2.3.2" + }, + "engines": { + "node": ">= 8.0" + } + }, + "node_modules/named-placeholders": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.3.tgz", + "integrity": "sha512-eLoBxg6wE/rZkJPhU/xRX1WTpkFEwDJEN96oxFrTsqBdbT5ec295Q+CoHrL9IT0DipqKhmGcaZmwOt8OON5x1w==", + "dependencies": { + "lru-cache": "^7.14.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/named-placeholders/node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "engines": { + "node": ">=12" + } + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -7980,6 +8546,22 @@ "node": ">=16" } }, + "node_modules/php-serialize": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/php-serialize/-/php-serialize-5.1.3.tgz", + "integrity": "sha512-p7zXX8xjGgddgP6byN+KmGKM0x6uoMZBRZteBa9LonqgrDV3LyMxUeGVX7RTFYwWaUAnTEsUWJfHI3N7eKvJgw==", + "engines": { + "node": ">= 8" + } + }, + "node_modules/php-unserialize": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/php-unserialize/-/php-unserialize-0.0.1.tgz", + "integrity": "sha512-aZWuX3gQ30Dui+Lff19q0jeu+3DHpSYXFEQPkeAx4WAyDtAp5VI30ZPC5wb4OrcHy6KUiZIRFRTWkvK8l8l6Rw==", + "engines": { + "node": "*" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -8373,6 +8955,15 @@ "node": ">=4" } }, + "node_modules/resolve-pkg-maps": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", + "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", + "dev": true, + "funding": { + "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" + } + }, "node_modules/restore-cursor": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", @@ -8618,6 +9209,11 @@ "node": ">= 18" } }, + "node_modules/seq-queue": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/seq-queue/-/seq-queue-0.0.5.tgz", + "integrity": "sha512-hr3Wtp/GZIc/6DAGPDcV4/9WoZhjrkXsi5B/07QgX8tsdc6ilr7BFM6PM6rbdAX1kFSDYeZGLipIZZKyQP0O5Q==" + }, "node_modules/serve-static": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.0.tgz", @@ -8891,6 +9487,14 @@ "node": ">=0.10.0" } }, + "node_modules/sqlstring": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.3.tgz", + "integrity": "sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg==", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/stack-trace": { "version": "0.0.10", "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", @@ -9331,6 +9935,25 @@ "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==" }, + "node_modules/tsx": { + "version": "4.20.5", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.20.5.tgz", + "integrity": "sha512-+wKjMNU9w/EaQayHXb7WA7ZaHY6hN8WgfvHNQ3t1PnU91/7O8TcTnIhCDYTZwnt8JsO9IBqZ30Ln1r7pPF52Aw==", + "dev": true, + "dependencies": { + "esbuild": "~0.25.0", + "get-tsconfig": "^4.7.5" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, "node_modules/tty-table": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/tty-table/-/tty-table-4.2.3.tgz", diff --git a/upload-api/package.json b/upload-api/package.json index 99cbd9091..81eb2fba3 100644 --- a/upload-api/package.json +++ b/upload-api/package.json @@ -5,7 +5,8 @@ "main": "index.js", "scripts": { "test": "echo \"Error: no test specified\" && exit 1", - "start": "tsc && node build/index.js", + "start": "PORT=4002 nodemon --exec tsx src/index.ts --ignore drupalMigrationData --ignore migration-* --ignore build --ignore *.log --ignore extracted_files", + "start:prod": "tsc && node build/index.js", "validation": "tsc && node build/main.js", "prettify": "prettier --write .", "lint": "eslint .", @@ -29,6 +30,7 @@ "eslint-config-prettier": "^9.1.0", "eslint-plugin-prettier": "^5.1.3", "helmet": "^7.1.0", + "tsx": "^4.7.1", "typescript": "^5.3.3" }, "dependencies": { @@ -45,12 +47,17 @@ "jszip": "^3.10.1", "lodash.isempty": "^4.4.0", "migration-contentful": "file:migration-contentful", + "migration-drupal": "file:migration-drupal", "migration-sitecore": "file:migration-sitecore", "migration-wordpress": "file:migration-wordpress", "multer": "^2.0.0", + "mysql2": "^3.14.3", "node-fetch": "^2.7.0", "nodemon": "^3.1.9", + "php-serialize": "^5.1.3", + "php-unserialize": "^0.0.1", "prettier": "^3.3.3", + "when": "^3.7.8", "winston": "^3.17.0", "xml2js": "^0.6.2" } diff --git a/upload-api/src/config/index.ts b/upload-api/src/config/index.ts index ca683e8b0..dc2d5cc35 100644 --- a/upload-api/src/config/index.ts +++ b/upload-api/src/config/index.ts @@ -2,8 +2,8 @@ export default { plan: { dropdown: { optionLimit: 100 } }, - cmsType: process.env.CMS_TYPE || 'cmsType', - isLocalPath: true, + cmsType: process.env.CMS_TYPE || 'drupal', + isLocalPath: false, awsData: { awsRegion: 'us-east-2', awsAccessKeyId: '', @@ -12,5 +12,17 @@ export default { bucketName: '', bucketKey: '' }, - localPath: process.env.CONTAINER_PATH || 'your-local-legacy-cms-path', + isSQL: true, + mysql: { + host: 'localhost', + user: 'root', + password: '', + database: 'riceuniversity1', + port: '3306' + }, + drupalAssetsUrl: { + base_url: "", + public_path: "", + }, + localPath: process.env.CONTAINER_PATH || 'sql', }; \ No newline at end of file diff --git a/upload-api/src/helper/index.ts b/upload-api/src/helper/index.ts index ee4783347..75c68b282 100644 --- a/upload-api/src/helper/index.ts +++ b/upload-api/src/helper/index.ts @@ -4,6 +4,7 @@ import path from "path"; import xml2js from 'xml2js' import { HTTP_TEXTS, HTTP_CODES, MACOSX_FOLDER } from '../constants'; import logger from "../utils/logger"; +import mysql from 'mysql2'; const getFileName = (params: { Key: string }) => { const obj: { fileName?: string; fileExt?: string } = {}; @@ -149,5 +150,67 @@ function deleteFolderSync(folderPath: string): void { } } +/** + * Establishes a MySQL database connection + * @returns Promise that resolves to the connection object or null if connection fails + */ +const createDbConnection = async (config: any): Promise => { + try { + // Create the connection with config values + const connection = mysql.createConnection({ + host: config?.host, + user: config?.user, + password: config?.password, + database: config?.database, + port: Number(config?.port) + }); + + // Test the connection by wrapping the connect method in a promise + return new Promise((resolve, reject) => { + connection.connect((err) => { + if (err) { + logger.error('Database connection failed:', { + error: err.message, + code: err.code, + stack: err.stack + }); + reject(err); + return; + } + + logger.info('Database connection established successfully', { + host: config?.mysql?.host, + database: config?.mysql?.database + }); + + resolve(connection); + }); + }); + } catch (error: any) { + logger.error('Failed to create database connection:', { + error: error.message, + stack: error.stack + }); + return null; + } +}; + +// Usage example +const getDbConnection = async (config: any) => { + try { + const connection = await createDbConnection(config); + if (!connection) { + throw new Error('Could not establish database connection'); + } + return connection; + } catch (error: any) { + logger.error('Database connection error:', { + message: error.message, + stack: error.stack + }); + throw error; // Re-throw so caller can handle it + } +}; + -export { getFileName, saveZip, saveJson, fileOperationLimiter, deleteFolderSync, parseXmlToJson }; +export { getFileName, saveZip, saveJson, fileOperationLimiter, deleteFolderSync, parseXmlToJson, getDbConnection}; diff --git a/upload-api/src/models/types.ts b/upload-api/src/models/types.ts index 2492a9b94..228b03fca 100644 --- a/upload-api/src/models/types.ts +++ b/upload-api/src/models/types.ts @@ -15,4 +15,16 @@ export interface Config { bucketKey: string; }; localPath: string; -} + isSQL: boolean; + mysql: { + host: string; + user: string; + password: string; + database: string; + port: string; + }; + drupalAssetsUrl: { + base_url: string; + public_path: string; + }; +} \ No newline at end of file diff --git a/upload-api/src/routes/index.ts b/upload-api/src/routes/index.ts index 0b87309e6..8d53af57f 100644 --- a/upload-api/src/routes/index.ts +++ b/upload-api/src/routes/index.ts @@ -10,7 +10,7 @@ import { UploadPartCommand } from '@aws-sdk/client-s3'; import { client } from '../services/aws/client'; -import { fileOperationLimiter } from '../helper'; +import { fileOperationLimiter, getDbConnection } from '../helper'; import handleFileProcessing from '../services/fileProcessing'; import config from '../config/index'; import createMapper from '../services/createMapper'; @@ -175,7 +175,42 @@ router.get('/validator', express.json(), fileOperationLimiter, async function (r } } } else { - const params = { + if ( config?.isSQL ) + { + const fileExt = 'sql'; + const name = 'sql'; + + console.log('Processing SQL database connection'); + // For SQL files, we don't need to read from S3, just validate the database connection + const result = await handleFileProcessing(fileExt, null, cmsType, name); + if (!result) { + console.error('File processing returned no result'); + return res.status(500).json({ + status: 500, + message: 'File processing failed to return a result', + file_details: config + }); + } + + const filePath = ''; + createMapper(filePath, projectId, app_token, affix, config); + + // Ensure we're sending back the complete file_details + const response = { + ...result, + file_details: { + ...result.file_details, + isSQL: config.isSQL, + mySQLDetails: config.mysql, // Changed from mysql to mySQLDetails + drupalAssetsUrl: config.drupalAssetsUrl + } + }; + + console.log('Sending SQL validation response:', JSON.stringify(response, null, 2)); + return res.status(result.status).json(response); + } else + { + const params = { Bucket: config?.awsData?.bucketName, Key: config?.awsData?.bucketKey }; @@ -228,8 +263,16 @@ router.get('/validator', express.json(), fileOperationLimiter, async function (r }); } } - catch (err: any) { + } catch (err: any) { console.error('๐Ÿš€ ~ router.get ~ err:', err); + // Only send error response if no response has been sent yet + if (!res.headersSent) { + res.status(500).json({ + status: 500, + message: 'Internal server error', + error: err.message + }); + } } }); diff --git a/upload-api/src/services/createMapper.ts b/upload-api/src/services/createMapper.ts index a8efebfea..ee4f5ab79 100644 --- a/upload-api/src/services/createMapper.ts +++ b/upload-api/src/services/createMapper.ts @@ -2,6 +2,7 @@ import createSitecoreMapper from '../controllers/sitecore'; import createWordpressMapper from '../controllers/wordpress'; import { Config } from '../models/types'; import createContentfulMapper from './contentful'; +import createDrupalMapper from './drupal'; const createMapper = async ( filePath: string = '', @@ -11,6 +12,8 @@ const createMapper = async ( config: Config ) => { const CMSIdentifier = config?.cmsType?.toLowerCase(); + console.log('CMSIdentifier', CMSIdentifier); + switch (CMSIdentifier) { case 'sitecore': { return await createSitecoreMapper(filePath, projectId, app_token, affix, config); @@ -24,6 +27,10 @@ const createMapper = async ( return createWordpressMapper(filePath, projectId, app_token, affix); } + case 'drupal': { + return createDrupalMapper(config, projectId, app_token, affix); + } + // case 'aem': { // return createAemMapper({ data }); // } diff --git a/upload-api/src/services/drupal/index.ts b/upload-api/src/services/drupal/index.ts new file mode 100644 index 000000000..627e43fcb --- /dev/null +++ b/upload-api/src/services/drupal/index.ts @@ -0,0 +1,88 @@ +/* eslint-disable @typescript-eslint/no-var-requires */ +import axios from 'axios'; + +import logger from '../../utils/logger'; +import { HTTP_CODES, HTTP_TEXTS } from '../../constants'; +import { Config } from '../../models/types'; + +const { createInitialMapper, extractLocale, extractTaxonomy } = require('migration-drupal'); + +const createDrupalMapper = async ( + config: Config, + projectId: string | string[], + app_token: string | string[], + affix: string | string[] +) => { + try { + console.log('hey we are in createDrupalMapper'); + + // this is to fetch the locales from the drupal database + // const fetchedLocales:[]= await extractLocale(config) + + const localeData = await extractLocale(config); + console.log('๐Ÿ” DEBUG: Locale data from extractLocale:', localeData); + + // Extract taxonomy vocabularies and save to drupalMigrationData + console.log('๐Ÿท๏ธ Extracting taxonomy vocabularies...'); + const taxonomyData = await extractTaxonomy(config.mysql); + console.log(`โœ… Extracted ${taxonomyData.length} taxonomy vocabularies`); + + const initialMapper = await createInitialMapper(config, affix); + + const req = { + method: 'post', + maxBodyLength: Infinity, + url: `${process.env.NODE_BACKEND_API}/v2/mapper/createDummyData/${projectId}`, + headers: { + app_token, + 'Content-Type': 'application/json' + }, + data: JSON.stringify(initialMapper) + }; + + const { data } = await axios.request(req); + console.log('๐Ÿš€ ~ createDrupalMapper ~ data:', data?.data); + if (data?.data?.content_mapper?.length) { + console.log('Inside the if block of createDrupalMapper'); + + logger.info('Validation success:', { + status: HTTP_CODES?.OK, + message: HTTP_TEXTS?.MAPPER_SAVED + }); + } else { + console.log('Inside the else block of createDrupalMapper'); + } + + const localeArray = Array.from(localeData); + console.log('๐Ÿ” DEBUG: Sending locales to API:', localeArray); + + const mapperConfig = { + method: 'post', + maxBodyLength: Infinity, + url: `${process.env.NODE_BACKEND_API}/v2/migration/localeMapper/${projectId}`, + headers: { + app_token, + 'Content-Type': 'application/json' + }, + data: { + locale: localeArray + } + }; + + const mapRes = await axios.request(mapperConfig); + if (mapRes?.status == 200) { + logger.info('Legacy CMS', { + status: HTTP_CODES?.OK, + message: HTTP_TEXTS?.LOCALE_SAVED + }); + } + } catch (err: any) { + console.error('๐Ÿš€ ~ createDrupalMapper ~ err:', err?.response?.data ?? err); + logger.warn('Validation error:', { + status: HTTP_CODES?.UNAUTHORIZED, + message: HTTP_TEXTS?.VALIDATION_ERROR + }); + } +}; + +export default createDrupalMapper; diff --git a/upload-api/src/services/fileProcessing.ts b/upload-api/src/services/fileProcessing.ts index 025ef3b66..3c2d86273 100644 --- a/upload-api/src/services/fileProcessing.ts +++ b/upload-api/src/services/fileProcessing.ts @@ -1,5 +1,5 @@ import { HTTP_TEXTS, HTTP_CODES } from '../constants'; -import { parseXmlToJson, saveJson, saveZip } from '../helper'; +import { parseXmlToJson, saveJson, saveZip, getDbConnection } from '../helper'; import JSZip from 'jszip'; import validator from '../validators'; import config from '../config/index'; @@ -68,7 +68,52 @@ const handleFileProcessing = async ( }; } } - } else { + }else if (fileExt === 'sql') { + console.log('SQL file processing'); + try { + // Get database connection + const dbConnection = await getDbConnection(config.mysql); + + if (dbConnection) { + await validator({ data: config, type: cmsType, extension: fileExt }); + logger.info('Database connection success:', { + status: HTTP_CODES?.OK, + message: 'Successfully connected to database' + }); + const successResponse = { + status: HTTP_CODES?.OK, + message: 'Successfully connected to database', + file_details: config + }; + console.log('=== Sending response (sql success) ==='); + console.log('Response object:', JSON.stringify(successResponse, null, 2)); + return successResponse; + } else { + logger.warn('Database connection error:', { + status: HTTP_CODES?.UNAUTHORIZED, + message: 'Failed to connect to database' + }); + const authErrorResponse = { + status: HTTP_CODES?.UNAUTHORIZED, + message: 'Failed to connect to database', + file_details: config + }; + console.log('=== Sending response (sql auth error) ==='); + console.log('Response object:', JSON.stringify(authErrorResponse, null, 2)); + return authErrorResponse; + } + } catch (error) { + logger.error('Database connection error:', error); + console.log('=== Sending response (sql server error) ==='); + const errorResponse = { + status: HTTP_CODES?.SERVER_ERROR, + message: 'Failed to connect to database', + file_details: config + }; + console.log('Response object:', JSON.stringify(errorResponse, null, 2)); + return errorResponse; + } + }else { // if file is not zip // Convert the buffer to a string assuming it's UTF-8 encoded const jsonString = Buffer?.from?.(zipBuffer)?.toString?.('utf8'); From 0e4ac356d58e294dfd0e29fcf29b730f362487e2 Mon Sep 17 00:00:00 2001 From: sauravraw Date: Fri, 19 Sep 2025 12:19:13 +0530 Subject: [PATCH 02/37] added locale support test cases --- .gitignore | 3 +- api/src/services/drupal/assets.service.ts | 216 +++-- .../services/drupal/content-types.service.ts | 188 ++-- api/src/services/drupal/entries.service.ts | 836 ++++++++++++------ api/src/services/drupal/locales.service.ts | 193 ++-- api/src/services/drupal/query.service.ts | 241 +++-- ui/package.json | 2 + .../Actions/LoadLanguageMapper.tsx | 378 +++++++- .../drupalSchema/featured_content.json | 81 -- .../drupalSchema/file_tree.json | 211 ----- .../drupalSchema/in_the_news.json | 123 --- .../drupalSchema/partners.json | 141 --- .../drupalSchema/supporting_material.json | 163 ---- .../libs/contentTypeMapper.js | 20 - .../libs/createInitialMapper.js | 9 - .../migration-drupal/libs/extractLocale.js | 129 ++- .../migration-drupal/libs/extractTaxonomy.js | 47 +- upload-api/src/config/index.ts | 8 +- upload-api/src/routes/index.ts | 323 +++---- upload-api/src/services/drupal/index.ts | 16 +- upload-api/src/services/fileProcessing.ts | 52 +- upload-api/src/validators/drupal/index.ts | 142 +++ upload-api/src/validators/index.ts | 5 + 23 files changed, 1826 insertions(+), 1701 deletions(-) delete mode 100644 upload-api/drupalMigrationData/drupalSchema/featured_content.json delete mode 100644 upload-api/drupalMigrationData/drupalSchema/file_tree.json delete mode 100644 upload-api/drupalMigrationData/drupalSchema/in_the_news.json delete mode 100644 upload-api/drupalMigrationData/drupalSchema/partners.json delete mode 100644 upload-api/drupalMigrationData/drupalSchema/supporting_material.json create mode 100644 upload-api/src/validators/drupal/index.ts diff --git a/.gitignore b/.gitignore index ffe1acc3d..25b20b358 100644 --- a/.gitignore +++ b/.gitignore @@ -359,4 +359,5 @@ upload-api/cmsMigrationData upload-api/extracted_files *copy* .qodo -.vscode \ No newline at end of file +.vscode +*MigrationData* \ No newline at end of file diff --git a/api/src/services/drupal/assets.service.ts b/api/src/services/drupal/assets.service.ts index faf3ccbed..70adc8c81 100644 --- a/api/src/services/drupal/assets.service.ts +++ b/api/src/services/drupal/assets.service.ts @@ -1,13 +1,13 @@ -import fs from "fs"; -import path from "path"; -import axios from "axios"; +import fs from 'fs'; +import path from 'path'; +import axios from 'axios'; import pLimit from 'p-limit'; import mysql from 'mysql2'; -import { MIGRATION_DATA_CONFIG } from "../../constants/index.js"; -import { getLogMessage } from "../../utils/index.js"; -import customLogger from "../../utils/custom-logger.utils.js"; -import { getDbConnection } from "../../helper/index.js"; -import { processBatches } from "../../utils/batch-processor.utils.js"; +import { MIGRATION_DATA_CONFIG } from '../../constants/index.js'; +import { getLogMessage } from '../../utils/index.js'; +import customLogger from '../../utils/custom-logger.utils.js'; +import { getDbConnection } from '../../helper/index.js'; +import { processBatches } from '../../utils/batch-processor.utils.js'; const { DATA, @@ -32,7 +32,7 @@ interface DrupalAsset { status?: string | number; uid?: string | number; timestamp?: string | number; - id?: string | number; // For file_usage table + id?: string | number; // For file_usage table count?: string | number; // For file_usage table } @@ -44,7 +44,7 @@ async function writeFile(dirPath: string, filename: string, data: any) { try { await fs.promises.mkdir(dirPath, { recursive: true }); const filePath = path.join(dirPath, filename); - + // Use file handle for better control over file operations fileHandle = await fs.promises.open(filePath, 'w'); await fileHandle.writeFile(JSON.stringify(data), 'utf8'); @@ -57,7 +57,10 @@ async function writeFile(dirPath: string, filename: string, data: any) { try { await fileHandle.close(); } catch (closeErr) { - console.error(`Error closing file handle for ${dirPath}/${filename}:`, closeErr); + console.error( + `Error closing file handle for ${dirPath}/${filename}:`, + closeErr + ); } } } @@ -66,15 +69,19 @@ async function writeFile(dirPath: string, filename: string, data: any) { /** * Constructs the full URL for Drupal assets, handling public:// and private:// schemes */ -const constructAssetUrl = (uri: string, baseUrl: string, publicPath: string): string => { +const constructAssetUrl = ( + uri: string, + baseUrl: string, + publicPath: string +): string => { let url = uri; const replaceValue = baseUrl + publicPath; - - if (!url.startsWith("http")) { - url = url.replace("public://", replaceValue); - url = url.replace("private://", replaceValue); + + if (!url.startsWith('http')) { + url = url.replace('public://', replaceValue); + url = url.replace('private://', replaceValue); } - + return encodeURI(url); }; @@ -89,14 +96,14 @@ const saveAsset = async ( metadata: AssetMetaData[], projectId: string, destination_stack_id: string, - baseUrl: string = "", - publicPath: string = "/sites/default/files/", + baseUrl: string = '', + publicPath: string = '/sites/default/files/', retryCount = 0 ): Promise => { try { const srcFunc = 'saveAsset'; const assetsSave = path.join(DATA, destination_stack_id, ASSETS_DIR_NAME); - + // Use Drupal-specific field names const assetId = `assets_${assets.fid}`; const fileName = assets.filename; @@ -109,10 +116,10 @@ const saveAsset = async ( try { const response = await axios.get(fileUrl, { - responseType: "arraybuffer", + responseType: 'arraybuffer', timeout: 30000, }); - + const assetPath = path.resolve(assetsSave, assetId); // Create asset data following Drupal migration pattern @@ -139,23 +146,40 @@ const saveAsset = async ( ); await fs.promises.mkdir(assetPath, { recursive: true }); - await fs.promises.writeFile(path.join(assetPath, fileName), Buffer.from(response.data), "binary"); - await writeFile(assetPath, `_contentstack_${assetId}.json`, assetData[assetId]); - + await fs.promises.writeFile( + path.join(assetPath, fileName), + Buffer.from(response.data), + 'binary' + ); + await writeFile( + assetPath, + `_contentstack_${assetId}.json`, + assetData[assetId] + ); + metadata.push({ uid: assetId, url: fileUrl, filename: fileName }); - + // Remove from failed assets if it was previously failed if (failedJSON[assetId]) { delete failedJSON[assetId]; } - + await customLogger(projectId, destination_stack_id, 'info', message); return assetId; - } catch (err: any) { if (retryCount < 1) { // Retry once - return await saveAsset(assets, failedJSON, assetData, metadata, projectId, destination_stack_id, baseUrl, publicPath, retryCount + 1); + return await saveAsset( + assets, + failedJSON, + assetData, + metadata, + projectId, + destination_stack_id, + baseUrl, + publicPath, + retryCount + 1 + ); } else { // Mark as failed after retry failedJSON[assetId] = { @@ -165,7 +189,7 @@ const saveAsset = async ( file_size: assets.filesize, reason_for_error: err?.message, }; - + const message = getLogMessage( srcFunc, `Failed to download asset "${fileName}" with id ${assets.fid}: ${err.message}`, @@ -177,7 +201,7 @@ const saveAsset = async ( } } } catch (error) { - console.error("Error in saveAsset:", error); + console.error('Error in saveAsset:', error); return `assets_${assets.fid}`; } }; @@ -185,7 +209,10 @@ const saveAsset = async ( /** * Executes SQL query and returns results as Promise */ -const executeQuery = (connection: mysql.Connection, query: string): Promise => { +const executeQuery = ( + connection: mysql.Connection, + query: string +): Promise => { return new Promise((resolve, reject) => { connection.query(query, (error, results) => { if (error) { @@ -206,18 +233,19 @@ const fetchAssetsFromDB = async ( destination_stack_id: string ): Promise => { const srcFunc = 'fetchAssetsFromDB'; - const assetsQuery = "SELECT a.fid, a.filename, a.uri, a.filesize, a.filemime FROM file_managed a"; - + const assetsQuery = + 'SELECT a.fid, a.filename, a.uri, a.filesize, a.filemime FROM file_managed a'; + try { const results = await executeQuery(connection, assetsQuery); - + const message = getLogMessage( srcFunc, `Fetched ${results.length} assets from database.`, {} ); await customLogger(projectId, destination_stack_id, 'info', message); - + return results as DrupalAsset[]; } catch (error: any) { const message = getLogMessage( @@ -242,27 +270,40 @@ const retryFailedAssets = async ( metadata: AssetMetaData[], projectId: string, destination_stack_id: string, - baseUrl: string = "", - publicPath: string = "/sites/default/files/" + baseUrl: string = '', + publicPath: string = '/sites/default/files/' ): Promise => { const srcFunc = 'retryFailedAssets'; - + if (failedAssetIds.length === 0) { return; } - + try { - const assetsFIDQuery = `SELECT a.fid, a.filename, a.uri, a.filesize, a.filemime, b.id, b.count FROM file_managed a, file_usage b WHERE a.fid IN (${failedAssetIds.join(',')})`; + const assetsFIDQuery = `SELECT a.fid, a.filename, a.uri, a.filesize, a.filemime, b.id, b.count FROM file_managed a, file_usage b WHERE a.fid IN (${failedAssetIds.join( + ',' + )})`; const results = await executeQuery(connection, assetsFIDQuery); - + if (results.length > 0) { const limit = pLimit(1); // Reduce to 1 for large datasets to prevent EMFILE errors const tasks = results.map((asset: DrupalAsset) => - limit(() => saveAsset(asset, failedJSON, assetData, metadata, projectId, destination_stack_id, baseUrl, publicPath)) + limit(() => + saveAsset( + asset, + failedJSON, + assetData, + metadata, + projectId, + destination_stack_id, + baseUrl, + publicPath + ) + ) ); - + await Promise.all(tasks); - + const message = getLogMessage( srcFunc, `Retried ${results.length} failed assets.`, @@ -289,13 +330,13 @@ export const createAssets = async ( dbConfig: any, destination_stack_id: string, projectId: string, - baseUrl: string = "", - publicPath: string = "/sites/default/files/", + baseUrl: string = '', + publicPath: string = '/sites/default/files/', isTest = false ) => { const srcFunc = 'createAssets'; let connection: mysql.Connection | null = null; - + try { console.info('๐Ÿ” === DRUPAL ASSETS SERVICE CONFIG ==='); console.info('๐Ÿ“‹ Database Config:', JSON.stringify(dbConfig, null, 2)); @@ -308,31 +349,40 @@ export const createAssets = async ( console.info('========================================'); const assetsSave = path.join(DATA, destination_stack_id, ASSETS_DIR_NAME); - const assetMasterFolderPath = path.join(DATA, destination_stack_id, "logs", ASSETS_DIR_NAME); - + const assetMasterFolderPath = path.join( + DATA, + destination_stack_id, + 'logs', + ASSETS_DIR_NAME + ); + // Initialize directories and files await fs.promises.mkdir(assetsSave, { recursive: true }); await fs.promises.mkdir(assetMasterFolderPath, { recursive: true }); - + // Initialize data structures const failedJSON: any = {}; const assetData: any = {}; const metadata: AssetMetaData[] = []; - const fileMeta = { "1": ASSETS_SCHEMA_FILE }; + const fileMeta = { '1': ASSETS_SCHEMA_FILE }; const failedAssetIds: string[] = []; - const message = getLogMessage( - srcFunc, - `Exporting assets...`, - {} - ); + const message = getLogMessage(srcFunc, `Exporting assets...`, {}); await customLogger(projectId, destination_stack_id, 'info', message); // Create database connection - connection = await getDbConnection(dbConfig, projectId, destination_stack_id); - + connection = await getDbConnection( + dbConfig, + projectId, + destination_stack_id + ); + // Fetch assets from database - const assetsData = await fetchAssetsFromDB(connection, projectId, destination_stack_id); + const assetsData = await fetchAssetsFromDB( + connection, + projectId, + destination_stack_id + ); if (assetsData && assetsData.length > 0) { let assets = assetsData; @@ -340,15 +390,22 @@ export const createAssets = async ( assets = assets.slice(0, 10); } - console.log(`๐Ÿ“Š Processing ${assets.length} assets...`); - // Use batch processing for large datasets to prevent EMFILE errors const batchSize = assets.length > 10000 ? 100 : 1000; // Smaller batches for very large datasets const results = await processBatches( assets, async (asset: DrupalAsset) => { try { - return await saveAsset(asset, failedJSON, assetData, metadata, projectId, destination_stack_id, baseUrl, publicPath); + return await saveAsset( + asset, + failedJSON, + assetData, + metadata, + projectId, + destination_stack_id, + baseUrl, + publicPath + ); } catch (error) { failedAssetIds.push(asset.fid.toString()); return `assets_${asset.fid}`; @@ -357,14 +414,17 @@ export const createAssets = async ( { batchSize, concurrency: 1, // Process one at a time to prevent file handle exhaustion - delayBetweenBatches: 200 // 200ms delay between batches to allow file handles to close + delayBetweenBatches: 200, // 200ms delay between batches to allow file handles to close }, (batchIndex, totalBatches, batchResults) => { - console.log(`โœ… Completed batch ${batchIndex}/${totalBatches} - Processed ${batchResults.length} assets`); - // Periodically save progress for very large datasets if (batchIndex % 10 === 0) { - console.log(`๐Ÿ’พ Progress: ${batchIndex}/${totalBatches} batches completed (${(batchIndex/totalBatches*100).toFixed(1)}%)`); + console.log( + `๐Ÿ’พ Progress: ${batchIndex}/${totalBatches} batches completed (${( + (batchIndex / totalBatches) * + 100 + ).toFixed(1)}%)` + ); } } ); @@ -387,26 +447,28 @@ export const createAssets = async ( // Write files following the original pattern await writeFile(assetsSave, ASSETS_SCHEMA_FILE, assetData); await writeFile(assetsSave, ASSETS_FILE_NAME, fileMeta); - + if (Object.keys(failedJSON).length > 0) { await writeFile(assetMasterFolderPath, ASSETS_FAILED_FILE, failedJSON); } const successMessage = getLogMessage( srcFunc, - `Successfully processed ${Object.keys(assetData).length} assets out of ${assets.length} total assets.`, + `Successfully processed ${ + Object.keys(assetData).length + } assets out of ${assets.length} total assets.`, {} ); - await customLogger(projectId, destination_stack_id, 'info', successMessage); - + await customLogger( + projectId, + destination_stack_id, + 'info', + successMessage + ); + return results; - } else { - const message = getLogMessage( - srcFunc, - `No assets found.`, - {} - ); + const message = getLogMessage(srcFunc, `No assets found.`, {}); await customLogger(projectId, destination_stack_id, 'info', message); return []; } diff --git a/api/src/services/drupal/content-types.service.ts b/api/src/services/drupal/content-types.service.ts index d04f7ea8a..3941e1b01 100644 --- a/api/src/services/drupal/content-types.service.ts +++ b/api/src/services/drupal/content-types.service.ts @@ -17,7 +17,7 @@ export const generateContentTypeSchemas = async ( projectId: string ): Promise => { const srcFunc = 'generateContentTypeSchemas'; - + try { const message = getLogMessage( srcFunc, @@ -27,48 +27,75 @@ export const generateContentTypeSchemas = async ( await customLogger(projectId, destination_stack_id, 'info', message); // Path to upload-api generated schema - const uploadApiSchemaPath = path.join(process.cwd(), '..', 'upload-api', 'drupalMigrationData', 'drupalSchema'); - + const uploadApiSchemaPath = path.join( + process.cwd(), + '..', + 'upload-api', + 'drupalMigrationData', + 'drupalSchema' + ); + // Path to API content types directory - const apiContentTypesPath = path.join(DATA, destination_stack_id, CONTENT_TYPES_DIR_NAME); - + const apiContentTypesPath = path.join( + DATA, + destination_stack_id, + CONTENT_TYPES_DIR_NAME + ); + // Ensure API content types directory exists await fs.promises.mkdir(apiContentTypesPath, { recursive: true }); - + if (!fs.existsSync(uploadApiSchemaPath)) { - throw new Error(`Upload-API schema not found at: ${uploadApiSchemaPath}. Please run upload-api migration first.`); + throw new Error( + `Upload-API schema not found at: ${uploadApiSchemaPath}. Please run upload-api migration first.` + ); } - + // Read all schema files from upload-api - const schemaFiles = fs.readdirSync(uploadApiSchemaPath).filter(file => file.endsWith('.json')); - + const schemaFiles = fs + .readdirSync(uploadApiSchemaPath) + .filter((file) => file.endsWith('.json')); + if (schemaFiles.length === 0) { - throw new Error(`No schema files found in upload-api directory: ${uploadApiSchemaPath}`); + throw new Error( + `No schema files found in upload-api directory: ${uploadApiSchemaPath}` + ); } - - console.log(`๐Ÿ“‹ Found ${schemaFiles.length} content type schemas to convert`); - + for (const schemaFile of schemaFiles) { try { - const uploadApiSchemaFilePath = path.join(uploadApiSchemaPath, schemaFile); - const uploadApiSchema = JSON.parse(fs.readFileSync(uploadApiSchemaFilePath, 'utf8')); - + const uploadApiSchemaFilePath = path.join( + uploadApiSchemaPath, + schemaFile + ); + const uploadApiSchema = JSON.parse( + fs.readFileSync(uploadApiSchemaFilePath, 'utf8') + ); + // Convert upload-api schema to API format const apiSchema = convertUploadApiSchemaToApiSchema(uploadApiSchema); - + // Write API schema file const apiSchemaFilePath = path.join(apiContentTypesPath, schemaFile); - await fs.promises.writeFile(apiSchemaFilePath, JSON.stringify(apiSchema, null, 2), 'utf8'); - + await fs.promises.writeFile( + apiSchemaFilePath, + JSON.stringify(apiSchema, null, 2), + 'utf8' + ); + const fieldMessage = getLogMessage( srcFunc, - `Converted content type ${uploadApiSchema.uid} with ${uploadApiSchema.schema?.length || 0} fields`, + `Converted content type ${uploadApiSchema.uid} with ${ + uploadApiSchema.schema?.length || 0 + } fields`, {} ); - await customLogger(projectId, destination_stack_id, 'info', fieldMessage); - - console.log(`โœ… Converted ${schemaFile}: ${uploadApiSchema.schema?.length || 0} fields`); - + await customLogger( + projectId, + destination_stack_id, + 'info', + fieldMessage + ); } catch (error: any) { const errorMessage = getLogMessage( srcFunc, @@ -76,20 +103,25 @@ export const generateContentTypeSchemas = async ( {}, error ); - await customLogger(projectId, destination_stack_id, 'error', errorMessage); - console.error(`โŒ Failed to convert ${schemaFile}:`, error.message); + await customLogger( + projectId, + destination_stack_id, + 'error', + errorMessage + ); } } - + const successMessage = getLogMessage( srcFunc, `Successfully generated ${schemaFiles.length} content type schemas from upload-api`, {} ); await customLogger(projectId, destination_stack_id, 'info', successMessage); - - console.log(`๐ŸŽ‰ Successfully converted ${schemaFiles.length} content type schemas`); - + + console.log( + `๐ŸŽ‰ Successfully converted ${schemaFiles.length} content type schemas` + ); } catch (error: any) { const errorMessage = getLogMessage( srcFunc, @@ -110,13 +142,13 @@ function convertUploadApiSchemaToApiSchema(uploadApiSchema: any): any { const apiSchema = { title: uploadApiSchema.title, uid: uploadApiSchema.uid, - schema: [] as any[] + schema: [] as any[], }; - + if (!uploadApiSchema.schema || !Array.isArray(uploadApiSchema.schema)) { return apiSchema; } - + // Convert each field from upload-api format to API format for (const uploadField of uploadApiSchema.schema) { try { @@ -134,41 +166,55 @@ function convertUploadApiSchemaToApiSchema(uploadApiSchema: any): any { nonLocalizable: uploadField.advanced?.non_localizable || false, default_value: uploadField.advanced?.default_value || '', validationRegex: uploadField.advanced?.format || '', - validationErrorMessage: uploadField.advanced?.error_message || '' + validationErrorMessage: uploadField.advanced?.error_message || '', }, // For reference fields, preserve reference_to from upload-api - refrenceTo: uploadField.advanced?.reference_to || uploadField.advanced?.embedObjects || [], - // For taxonomy fields, preserve taxonomies from upload-api - taxonomies: uploadField.advanced?.taxonomies || [] + refrenceTo: + uploadField.advanced?.reference_to || + uploadField.advanced?.embedObjects || + [], + // For taxonomy fields, preserve taxonomies from upload-api + taxonomies: uploadField.advanced?.taxonomies || [], }, - advanced: true + advanced: true, }); - + if (apiField) { // Preserve additional metadata from upload-api - if (uploadField.contentstackFieldType === 'reference' && uploadField.advanced?.reference_to) { + if ( + uploadField.contentstackFieldType === 'reference' && + uploadField.advanced?.reference_to + ) { apiField.reference_to = uploadField.advanced.reference_to; } - - if (uploadField.contentstackFieldType === 'taxonomy' && uploadField.advanced?.taxonomies) { + + if ( + uploadField.contentstackFieldType === 'taxonomy' && + uploadField.advanced?.taxonomies + ) { apiField.taxonomies = uploadField.advanced.taxonomies; } - + // Preserve field metadata for proper field type conversion if (uploadField.advanced?.multiline !== undefined) { apiField.field_metadata = apiField.field_metadata || {}; apiField.field_metadata.multiline = uploadField.advanced.multiline; } - + apiSchema.schema.push(apiField); } - } catch (error: any) { - console.warn(`Failed to convert field ${uploadField.uid}:`, error.message); - + console.warn( + `Failed to convert field ${uploadField.uid}:`, + error.message + ); + // Fallback: create basic field structure apiSchema.schema.push({ - display_name: uploadField.contentstackField || uploadField.otherCmsField || uploadField.uid, + display_name: + uploadField.contentstackField || + uploadField.otherCmsField || + uploadField.uid, uid: uploadField.contentstackFieldUid || uploadField.uid, data_type: mapFieldTypeToDataType(uploadField.contentstackFieldType), mandatory: uploadField.advanced?.mandatory || false, @@ -177,11 +223,11 @@ function convertUploadApiSchemaToApiSchema(uploadApiSchema: any): any { format: '', error_messages: { format: '' }, multiple: uploadField.advanced?.multiple || false, - non_localizable: uploadField.advanced?.non_localizable || false + non_localizable: uploadField.advanced?.non_localizable || false, }); } } - + return apiSchema; } @@ -191,26 +237,26 @@ function convertUploadApiSchemaToApiSchema(uploadApiSchema: any): any { */ function mapFieldTypeToDataType(fieldType: string): string { const fieldTypeMap: { [key: string]: string } = { - 'single_line_text': 'text', - 'multi_line_text': 'text', - 'text': 'text', - 'html': 'html', - 'json': 'json', - 'markdown': 'text', - 'number': 'number', - 'boolean': 'boolean', - 'isodate': 'isodate', - 'file': 'file', - 'reference': 'reference', - 'taxonomy': 'taxonomy', - 'link': 'link', - 'dropdown': 'text', - 'radio': 'text', - 'checkbox': 'boolean', - 'global_field': 'global_field', - 'group': 'group', - 'url': 'text' + single_line_text: 'text', + multi_line_text: 'text', + text: 'text', + html: 'html', + json: 'json', + markdown: 'text', + number: 'number', + boolean: 'boolean', + isodate: 'isodate', + file: 'file', + reference: 'reference', + taxonomy: 'taxonomy', + link: 'link', + dropdown: 'text', + radio: 'text', + checkbox: 'boolean', + global_field: 'global_field', + group: 'group', + url: 'text', }; - + return fieldTypeMap[fieldType] || 'text'; } diff --git a/api/src/services/drupal/entries.service.ts b/api/src/services/drupal/entries.service.ts index 449b192cc..37216a944 100644 --- a/api/src/services/drupal/entries.service.ts +++ b/api/src/services/drupal/entries.service.ts @@ -1,14 +1,30 @@ -import fs from "fs"; -import path from "path"; +import fs from 'fs'; +import path from 'path'; import mysql from 'mysql2'; -import { v4 as uuidv4 } from "uuid"; -import { JSDOM } from "jsdom"; -import { htmlToJson, jsonToHtml, jsonToMarkdown } from '@contentstack/json-rte-serializer'; -import { CHUNK_SIZE, LOCALE_MAPPER, MIGRATION_DATA_CONFIG } from "../../constants/index.js"; -import { getLogMessage } from "../../utils/index.js"; -import customLogger from "../../utils/custom-logger.utils.js"; -import { getDbConnection } from "../../helper/index.js"; -import { analyzeFieldTypes, isTaxonomyField, isReferenceField, isAssetField, type TaxonomyFieldMapping, type ReferenceFieldMapping, type AssetFieldMapping } from "./field-analysis.service.js"; +import { v4 as uuidv4 } from 'uuid'; +import { JSDOM } from 'jsdom'; +import { + htmlToJson, + jsonToHtml, + jsonToMarkdown, +} from '@contentstack/json-rte-serializer'; +import { + CHUNK_SIZE, + LOCALE_MAPPER, + MIGRATION_DATA_CONFIG, +} from '../../constants/index.js'; +import { getLogMessage } from '../../utils/index.js'; +import customLogger from '../../utils/custom-logger.utils.js'; +import { getDbConnection } from '../../helper/index.js'; +import { + analyzeFieldTypes, + isTaxonomyField, + isReferenceField, + isAssetField, + type TaxonomyFieldMapping, + type ReferenceFieldMapping, + type AssetFieldMapping, +} from './field-analysis.service.js'; import FieldFetcherService from './field-fetcher.service.js'; // Dynamic import for phpUnserialize will be used in the function @@ -64,13 +80,16 @@ interface QueryConfig { const LIMIT = 5; // Pagination limit -// NOTE: Hardcoded queries have been REMOVED. All queries are now generated dynamically +// NOTE: Hardcoded queries have been REMOVED. All queries are now generated dynamically // by the query.service.ts based on actual database field analysis. /** * Executes SQL query and returns results as Promise */ -const executeQuery = (connection: mysql.Connection, query: string): Promise => { +const executeQuery = ( + connection: mysql.Connection, + query: string +): Promise => { return new Promise((resolve, reject) => { connection.query(query, (error, results) => { if (error) { @@ -85,28 +104,30 @@ const executeQuery = (connection: mysql.Connection, query: string): Promise> => { +const loadTaxonomyReferences = async ( + referencesPath: string +): Promise> => { try { const taxonomyRefPath = path.join(referencesPath, 'taxonomyReference.json'); - + if (!fs.existsSync(taxonomyRefPath)) { return {}; } - + const taxonomyReferences: TaxonomyReference[] = JSON.parse( fs.readFileSync(taxonomyRefPath, 'utf8') ); - + // Create lookup map: drupal_term_id -> {taxonomy_uid, term_uid} const lookup: Record = {}; - - taxonomyReferences.forEach(ref => { + + taxonomyReferences.forEach((ref) => { lookup[ref.drupal_term_id] = { taxonomy_uid: ref.taxonomy_uid, - term_uid: ref.term_uid + term_uid: ref.term_uid, }; }); - + return lookup; } catch (error) { console.warn('Could not load taxonomy references:', error); @@ -132,7 +153,10 @@ async function writeFile(dirPath: string, filename: string, data: any) { */ async function readFile(filePath: string, fileName: string) { try { - const data = await fs.promises.readFile(path.join(filePath, fileName), "utf8"); + const data = await fs.promises.readFile( + path.join(filePath, fileName), + 'utf8' + ); return JSON.parse(data); } catch (err) { return {}; @@ -154,7 +178,7 @@ function makeChunks(entryData: any) { currentChunkSize = Buffer.byteLength( JSON.stringify(chunks[currentChunkId]), - "utf8" + 'utf8' ); if (currentChunkSize > chunkSize) { @@ -176,11 +200,12 @@ const fetchFieldConfigs = async ( destination_stack_id: string ): Promise => { const srcFunc = 'fetchFieldConfigs'; - const contentTypeQuery = "SELECT *, CONVERT(data USING utf8) as data FROM config WHERE name LIKE '%field.field.node%'"; - + const contentTypeQuery = + "SELECT *, CONVERT(data USING utf8) as data FROM config WHERE name LIKE '%field.field.node%'"; + try { const results = await executeQuery(connection, contentTypeQuery); - + const fieldConfigs: DrupalFieldConfig[] = []; for (const row of results) { try { @@ -190,17 +215,20 @@ const fetchFieldConfigs = async ( fieldConfigs.push(configData as DrupalFieldConfig); } } catch (parseError) { - console.warn(`Failed to parse field config for ${row.name}:`, parseError); + console.warn( + `Failed to parse field config for ${row.name}:`, + parseError + ); } } - + const message = getLogMessage( srcFunc, `Fetched ${fieldConfigs.length} field configurations from database.`, {} ); await customLogger(projectId, destination_stack_id, 'info', message); - + return fieldConfigs; } catch (error: any) { const message = getLogMessage( @@ -226,7 +254,9 @@ const determineSourceFieldType = (value: any): string => { } if (typeof value === 'string') { // Simple heuristic: if it has line breaks, consider it multi-line - return value.includes('\n') || value.includes('\r') ? 'multi_line' : 'single_line'; + return value.includes('\n') || value.includes('\r') + ? 'multi_line' + : 'single_line'; } return 'unknown'; }; @@ -238,12 +268,21 @@ const determineSourceFieldType = (value: any): string => { * 3. HTML RTE โ†’ HTML RTE/JSON RTE (NOT Single-line or Multi-line) * 4. JSON RTE โ†’ JSON RTE/HTML RTE (NOT Single-line or Multi-line) */ -const isConversionAllowed = (sourceType: string, targetType: string): boolean => { +const isConversionAllowed = ( + sourceType: string, + targetType: string +): boolean => { const conversionRules: { [key: string]: string[] } = { - 'single_line': ['single_line_text', 'text', 'multi_line_text', 'html', 'json'], - 'multi_line': ['multi_line_text', 'text', 'html', 'json'], // Cannot convert to single_line_text - 'html_rte': ['html', 'json'], // Cannot convert to single_line_text or multi_line_text - 'json_rte': ['json', 'html'] // Cannot convert to single_line_text or multi_line_text + single_line: [ + 'single_line_text', + 'text', + 'multi_line_text', + 'html', + 'json', + ], + multi_line: ['multi_line_text', 'text', 'html', 'json'], // Cannot convert to single_line_text + html_rte: ['html', 'json'], // Cannot convert to single_line_text or multi_line_text + json_rte: ['json', 'html'], // Cannot convert to single_line_text or multi_line_text }; return conversionRules[sourceType]?.includes(targetType) || false; @@ -269,7 +308,9 @@ const processFieldByType = ( // Check if conversion is allowed if (!isConversionAllowed(sourceType, targetType)) { - console.warn(`Conversion not allowed: ${sourceType} โ†’ ${targetType}. Keeping original value.`); + console.warn( + `Conversion not allowed: ${sourceType} โ†’ ${targetType}. Keeping original value.` + ); return value; } @@ -281,16 +322,25 @@ const processFieldByType = ( try { const htmlContent = jsonToHtml(value) || ''; // Strip HTML tags and convert to single line - const textContent = htmlContent.replace(/<[^>]*>/g, '').replace(/\s+/g, ' ').trim(); + const textContent = htmlContent + .replace(/<[^>]*>/g, '') + .replace(/\s+/g, ' ') + .trim(); return textContent; } catch (error) { - console.warn('Failed to convert JSON RTE to single line text:', error); + console.warn( + 'Failed to convert JSON RTE to single line text:', + error + ); return String(value || ''); } } else if (typeof value === 'string') { if (/<\/?[a-z][\s\S]*>/i.test(value)) { // HTML to plain text - const textContent = value.replace(/<[^>]*>/g, '').replace(/\s+/g, ' ').trim(); + const textContent = value + .replace(/<[^>]*>/g, '') + .replace(/\s+/g, ' ') + .trim(); return textContent; } // Multi-line to single line @@ -305,18 +355,20 @@ const processFieldByType = ( if (typeof value === 'object' && value !== null && value.type === 'doc') { // JSON RTE to HTML (preserving structure) try { - return jsonToHtml(value, { - customElementTypes: { - "social-embed": (attrs, child, jsonBlock) => { - return `${child}`; + return ( + jsonToHtml(value, { + customElementTypes: { + 'social-embed': (attrs, child, jsonBlock) => { + return `${child}`; + }, }, - }, - customTextWrapper: { - "color": (child, value) => { - return `${child}`; + customTextWrapper: { + color: (child, value) => { + return `${child}`; + }, }, - }, - }) || ''; + }) || '' + ); } catch (error) { console.warn('Failed to convert JSON RTE to HTML:', error); return String(value || ''); @@ -361,18 +413,20 @@ const processFieldByType = ( if (typeof value === 'object' && value !== null && value.type === 'doc') { // JSON RTE to HTML try { - return jsonToHtml(value, { - customElementTypes: { - "social-embed": (attrs, child, jsonBlock) => { - return `${child}`; + return ( + jsonToHtml(value, { + customElementTypes: { + 'social-embed': (attrs, child, jsonBlock) => { + return `${child}`; + }, }, - }, - customTextWrapper: { - "color": (child, value) => { - return `${child}`; + customTextWrapper: { + color: (child, value) => { + return `${child}`; + }, }, - }, - }) || '

'; + }) || '

' + ); } catch (error) { console.warn('Failed to convert JSON RTE to HTML:', error); return value; @@ -401,30 +455,32 @@ const processFieldByType = ( // Multiple files if (Array.isArray(value)) { const validAssets = value - .map(assetRef => { + .map((assetRef) => { const assetKey = `assets_${assetRef}`; const assetReference = assetId[assetKey]; - + if (assetReference && typeof assetReference === 'object') { return assetReference; } - - console.warn(`Asset ${assetKey} not found or invalid, excluding from array`); + + console.warn( + `Asset ${assetKey} not found or invalid, excluding from array` + ); return null; }) - .filter(asset => asset !== null); // Remove null entries - + .filter((asset) => asset !== null); // Remove null entries + return validAssets.length > 0 ? validAssets : undefined; // Return undefined if no valid assets } } else { // Single file const assetKey = `assets_${value}`; const assetReference = assetId[assetKey]; - + if (assetReference && typeof assetReference === 'object') { return assetReference; } - + console.warn(`Asset ${assetKey} not found or invalid, removing field`); return undefined; // Return undefined to indicate field should be removed } @@ -436,7 +492,10 @@ const processFieldByType = ( if (fieldMapping.advanced?.multiple) { // Multiple references if (Array.isArray(value)) { - return value.map(refId => referenceId[`content_type_entries_title_${refId}`] || refId); + return value.map( + (refId) => + referenceId[`content_type_entries_title_${refId}`] || refId + ); } } else { // Single reference @@ -488,7 +547,7 @@ const processFieldByType = ( /** * Consolidates all taxonomy fields into a single 'taxonomies' field with unique term_uid validation - * + * * @param processedEntry - The processed entry data * @param contentType - The content type being processed * @param taxonomyFieldMapping - Mapping of taxonomy fields from field analysis @@ -499,62 +558,70 @@ const consolidateTaxonomyFields = ( contentType: string, taxonomyFieldMapping: TaxonomyFieldMapping ): any => { - const consolidatedTaxonomies: Array<{ taxonomy_uid: string; term_uid: string }> = []; + const consolidatedTaxonomies: Array<{ + taxonomy_uid: string; + term_uid: string; + }> = []; const fieldsToRemove: string[] = []; const seenTermUids = new Set(); // Track unique term_uid values - + // Iterate through all fields in the processed entry for (const [fieldKey, fieldValue] of Object.entries(processedEntry)) { // Extract field name from key (remove _target_id suffix) const fieldName = fieldKey.replace(/_target_id$/, ''); - + // Check if this is a taxonomy field using field analysis if (isTaxonomyField(fieldName, contentType, taxonomyFieldMapping)) { // Validate that field value is an array with taxonomy structure if (Array.isArray(fieldValue)) { for (const taxonomyItem of fieldValue) { // Validate taxonomy structure - if (taxonomyItem && - typeof taxonomyItem === 'object' && - taxonomyItem.taxonomy_uid && - taxonomyItem.term_uid) { - + if ( + taxonomyItem && + typeof taxonomyItem === 'object' && + taxonomyItem.taxonomy_uid && + taxonomyItem.term_uid + ) { // Check for unique term_uid (avoid duplicates) if (!seenTermUids.has(taxonomyItem.term_uid)) { consolidatedTaxonomies.push({ taxonomy_uid: taxonomyItem.taxonomy_uid, - term_uid: taxonomyItem.term_uid + term_uid: taxonomyItem.term_uid, }); seenTermUids.add(taxonomyItem.term_uid); } } } } - + // Mark this field for removal fieldsToRemove.push(fieldKey); } } - + // Create new entry object without the original taxonomy fields const consolidatedEntry = { ...processedEntry }; - + // Remove original taxonomy fields for (const fieldKey of fieldsToRemove) { delete consolidatedEntry[fieldKey]; } - + // Add consolidated taxonomy field if we have any taxonomies if (consolidatedTaxonomies.length > 0) { consolidatedEntry.taxonomies = consolidatedTaxonomies; - console.log(`๐Ÿท๏ธ Consolidated ${fieldsToRemove.length} taxonomy fields into 'taxonomies' with ${consolidatedTaxonomies.length} unique terms for ${contentType}`); + console.log( + `๐Ÿท๏ธ Consolidated ${fieldsToRemove.length} taxonomy fields into 'taxonomies' with ${consolidatedTaxonomies.length} unique terms for ${contentType}` + ); } - + // Replace existing 'taxonomies' field if it exists (as per requirement) if ('taxonomies' in processedEntry && consolidatedTaxonomies.length > 0) { - console.log(`๐Ÿ”„ Replaced existing 'taxonomies' field with consolidated data for ${contentType}`); + console.log( + `๐Ÿ”„ Replaced existing 'taxonomies' field with consolidated data for ${contentType}` + ); } - + return consolidatedEntry; }; @@ -580,14 +647,21 @@ const processFieldData = async ( const processedFields = new Set(); // Track fields that have been processed to avoid duplicates // Process each field in the entry data - for (const [dataKey, value] of Object.entries(entryData)) { + for (const [dataKey, value] of Object.entries(entryData)) { // Extract field name from dataKey (remove _target_id suffix) - const fieldName = dataKey.replace(/_target_id$/, '').replace(/_value$/, '').replace(/_status$/, '').replace(/_uri$/, ''); - + const fieldName = dataKey + .replace(/_target_id$/, '') + .replace(/_value$/, '') + .replace(/_status$/, '') + .replace(/_uri$/, ''); + // Handle asset fields using field analysis - if (dataKey.endsWith('_target_id') && isAssetField(fieldName, contentType, assetFieldMapping)) { - const assetKey = `assets_${value}`; - if (assetKey in assetId) { + if ( + dataKey.endsWith('_target_id') && + isAssetField(fieldName, contentType, assetFieldMapping) + ) { + const assetKey = `assets_${value}`; + if (assetKey in assetId) { // Transform to proper Contentstack asset reference format const assetReference = assetId[assetKey]; if (assetReference && typeof assetReference === 'object') { @@ -602,29 +676,33 @@ const processFieldData = async ( // Handle entity references (taxonomy and node references) using field analysis if (dataKey.endsWith('_target_id') && typeof value === 'number') { - // Check if this is a taxonomy field using our field analysis + // Check if this is a taxonomy field using our field analysis if (isTaxonomyField(fieldName, contentType, taxonomyFieldMapping)) { - // Look up taxonomy reference using drupal_term_id - const taxonomyRef = taxonomyReferenceLookup[value]; - - if (taxonomyRef) { - // Transform to array format with taxonomy_uid and term_uid (no drupal_term_id) - processedData[dataKey] = [{ - taxonomy_uid: taxonomyRef.taxonomy_uid, - term_uid: taxonomyRef.term_uid - }]; - } else { - // Fallback to numeric tid if lookup failed - processedData[dataKey] = value; - } + // Look up taxonomy reference using drupal_term_id + const taxonomyRef = taxonomyReferenceLookup[value]; + + if (taxonomyRef) { + // Transform to array format with taxonomy_uid and term_uid (no drupal_term_id) + processedData[dataKey] = [ + { + taxonomy_uid: taxonomyRef.taxonomy_uid, + term_uid: taxonomyRef.term_uid, + }, + ]; + } else { + // Fallback to numeric tid if lookup failed + processedData[dataKey] = value; + } continue; // Skip further processing for this field - } else if (isReferenceField(fieldName, contentType, referenceFieldMapping)) { - // Handle node reference fields using field analysis - const referenceKey = `content_type_entries_title_${value}`; - if (referenceKey in referenceId) { - // Transform to array format with proper reference structure - processedData[dataKey] = [referenceId[referenceKey]]; - } else { + } else if ( + isReferenceField(fieldName, contentType, referenceFieldMapping) + ) { + // Handle node reference fields using field analysis + const referenceKey = `content_type_entries_title_${value}`; + if (referenceKey in referenceId) { + // Transform to array format with proper reference structure + processedData[dataKey] = [referenceId[referenceKey]]; + } else { // If reference not found, mark field as skipped skippedFields.add(dataKey); } @@ -633,15 +711,19 @@ const processFieldData = async ( } // Handle other field types by checking field configs - const matchingFieldConfig = fieldConfigs.find(fc => - dataKey === `${fc.field_name}_value` || - dataKey === `${fc.field_name}_status` || - dataKey === fc.field_name + const matchingFieldConfig = fieldConfigs.find( + (fc) => + dataKey === `${fc.field_name}_value` || + dataKey === `${fc.field_name}_status` || + dataKey === fc.field_name ); if (matchingFieldConfig) { // Handle datetime and timestamps - if (matchingFieldConfig.field_type === 'datetime' || matchingFieldConfig.field_type === 'timestamp') { + if ( + matchingFieldConfig.field_type === 'datetime' || + matchingFieldConfig.field_type === 'timestamp' + ) { if (dataKey === `${matchingFieldConfig.field_name}_value`) { if (typeof value === 'number') { processedData[dataKey] = new Date(value * 1000).toISOString(); @@ -654,7 +736,10 @@ const processFieldData = async ( // Handle boolean fields if (matchingFieldConfig.field_type === 'boolean') { - if (dataKey === `${matchingFieldConfig.field_name}_value` && typeof value === 'number') { + if ( + dataKey === `${matchingFieldConfig.field_name}_value` && + typeof value === 'number' + ) { processedData[dataKey] = value === 1; continue; } @@ -662,7 +747,10 @@ const processFieldData = async ( // Handle comment fields if (matchingFieldConfig.field_type === 'comment') { - if (dataKey === `${matchingFieldConfig.field_name}_status` && typeof value === 'number') { + if ( + dataKey === `${matchingFieldConfig.field_name}_status` && + typeof value === 'number' + ) { processedData[dataKey] = `${value}`; continue; } @@ -683,15 +771,15 @@ const processFieldData = async ( // Process standard field transformations const ctValue: any = {}; - + for (const fieldName of fieldNames) { // Skip fields that were intentionally excluded in the main processing loop if (skippedFields.has(fieldName)) { continue; } - + const value = entryData[fieldName]; - + if (fieldName === 'created') { ctValue[fieldName] = new Date(value * 1000).toISOString(); } else if (fieldName === 'uid_name') { @@ -708,13 +796,13 @@ const processFieldData = async ( if (processedFields.has(fieldName)) { continue; } - + const baseFieldName = fieldName.replace('_uri', ''); const titleFieldName = `${baseFieldName}_title`; - + // Check if we also have title data const titleValue = entryData[titleFieldName]; - + if (value) { ctValue[baseFieldName] = { title: titleValue || value, // Use title if available, fallback to URI @@ -726,7 +814,7 @@ const processFieldData = async ( href: '', }; } - + // Mark title field as processed to avoid duplicate processing if (titleValue) { processedFields.add(titleFieldName); @@ -736,11 +824,11 @@ const processFieldData = async ( if (processedFields.has(fieldName)) { continue; } - + // Check if there's a corresponding _uri field const baseFieldName = fieldName.replace('_title', ''); const uriFieldName = `${baseFieldName}_uri`; - + if (entryData[uriFieldName]) { // URI field will handle this, skip processing here continue; @@ -813,71 +901,90 @@ const processEntries = async ( isTest: boolean = false ): Promise<{ [key: string]: any } | null> => { const srcFunc = 'processEntries'; - + try { // Following original pattern: queryPageConfig['page']['' + pagename + ''] const baseQuery = queryPageConfig['page'][contentType]; if (!baseQuery) { throw new Error(`No query found for content type: ${contentType}`); } - + // Check if this is an optimized query (content type with many fields) const isOptimizedQuery = baseQuery.includes('/* OPTIMIZED_NO_JOINS:'); let entries: any[] = []; - + if (isOptimizedQuery) { // Handle content types with many fields using optimized approach - const fieldCountMatch = baseQuery.match(/\/\* OPTIMIZED_NO_JOINS:(\d+) \*\//); + const fieldCountMatch = baseQuery.match( + /\/\* OPTIMIZED_NO_JOINS:(\d+) \*\// + ); const fieldCount = fieldCountMatch ? parseInt(fieldCountMatch[1]) : 0; - + const optimizedMessage = getLogMessage( srcFunc, `Processing ${contentType} with optimized field fetching (${fieldCount} fields)`, {} ); - await customLogger(projectId, destination_stack_id, 'info', optimizedMessage); - + await customLogger( + projectId, + destination_stack_id, + 'info', + optimizedMessage + ); + // Execute base query without field JOINs const effectiveLimit = isTest ? 1 : LIMIT; - const cleanBaseQuery = baseQuery.replace(/\/\* OPTIMIZED_NO_JOINS:\d+ \*\//, '').trim(); + const cleanBaseQuery = baseQuery + .replace(/\/\* OPTIMIZED_NO_JOINS:\d+ \*\//, '') + .trim(); const query = cleanBaseQuery + ` LIMIT ${skip}, ${effectiveLimit}`; const baseEntries = await executeQuery(connection, query); - + if (baseEntries.length === 0) { return null; } - + // Fetch field data separately using FieldFetcherService - const fieldFetcher = new FieldFetcherService(connection, projectId, destination_stack_id); - const nodeIds = baseEntries.map(entry => entry.nid); - const fieldsForType = await fieldFetcher.getFieldsForContentType(contentType); - + const fieldFetcher = new FieldFetcherService( + connection, + projectId, + destination_stack_id + ); + const nodeIds = baseEntries.map((entry) => entry.nid); + const fieldsForType = await fieldFetcher.getFieldsForContentType( + contentType + ); + if (fieldsForType.length > 0) { const fieldData = await fieldFetcher.fetchFieldDataForContentType( contentType, nodeIds, fieldsForType ); - + // Merge base entries with field data entries = fieldFetcher.mergeNodeAndFieldData(baseEntries, fieldData); - + const mergeMessage = getLogMessage( srcFunc, `Merged ${baseEntries.length} base entries with field data for ${contentType}`, {} ); - await customLogger(projectId, destination_stack_id, 'info', mergeMessage); + await customLogger( + projectId, + destination_stack_id, + 'info', + mergeMessage + ); } else { entries = baseEntries; } - } else { // Handle content types with few fields using traditional approach const effectiveLimit = isTest ? 1 : LIMIT; const query = baseQuery + ` LIMIT ${skip}, ${effectiveLimit}`; entries = await executeQuery(connection, query); - + if (entries.length === 0) { return null; } @@ -885,9 +992,9 @@ const processEntries = async ( // Group entries by their actual locale (langcode) for proper multilingual support const entriesByLocale: { [locale: string]: any[] } = {}; - + // Group entries by their langcode - entries.forEach(entry => { + entries.forEach((entry) => { const entryLocale = entry.langcode || masterLocale; // fallback to masterLocale if no langcode if (!entriesByLocale[entryLocale]) { entriesByLocale[entryLocale] = []; @@ -895,16 +1002,29 @@ const processEntries = async ( entriesByLocale[entryLocale].push(entry); }); - console.log(`๐Ÿ“ Found entries in ${Object.keys(entriesByLocale).length} locales for ${contentType}:`, Object.keys(entriesByLocale)); + console.log( + `๐Ÿ“ Found entries in ${ + Object.keys(entriesByLocale).length + } locales for ${contentType}:`, + Object.keys(entriesByLocale) + ); // ๐Ÿ”„ Apply locale folder transformation rules (same as locale service) + // Rules: + // - "und" alone โ†’ "en-us" + // - "und" + "en-us" โ†’ "und" become "en", "en-us" stays + // - "en" + "und" โ†’ "und" becomes "en-us", "en" stays + // - All three "en" + "und" + "en-us" โ†’ all three stays + // - Apart from these, all other locales stay as is const transformedEntriesByLocale: { [locale: string]: any[] } = {}; const allLocales = Object.keys(entriesByLocale); const hasUnd = allLocales.includes('und'); const hasEn = allLocales.includes('en'); const hasEnUs = allLocales.includes('en-us'); - console.log(`๐Ÿ” Locale Analysis: hasUnd=${hasUnd}, hasEn=${hasEn}, hasEnUs=${hasEnUs}`); + console.log( + `๐Ÿ” Locale Analysis: hasUnd=${hasUnd}, hasEn=${hasEn}, hasEnUs=${hasEnUs}` + ); // Transform locale folder names based on business rules Object.entries(entriesByLocale).forEach(([originalLocale, entries]) => { @@ -912,207 +1032,300 @@ const processEntries = async ( if (originalLocale === 'und') { if (hasEn && hasEnUs) { - // If all three present, "und" stays as "und" folder + // Rule 4: All three "en" + "und" + "en-us" โ†’ all three stays targetFolder = 'und'; console.log(`๐Ÿ”„ "und" entries โ†’ "und" folder (all three present)`); - } else if (hasEnUs) { - // If "und" + "en-us", "und" goes to "en" folder + } else if (hasEnUs && !hasEn) { + // Rule 2: "und" + "en-us" โ†’ "und" become "en", "en-us" stays targetFolder = 'en'; - console.log(`๐Ÿ”„ Transforming "und" entries โ†’ "en" folder (en-us exists)`); - } else { - // If only "und", use "en-us" folder + console.log( + `๐Ÿ”„ Transforming "und" entries โ†’ "en" folder (Rule 2: und+en-us)` + ); + } else if (hasEn && !hasEnUs) { + // Rule 3: "en" + "und" โ†’ "und" becomes "en-us", "en" stays targetFolder = 'en-us'; - console.log(`๐Ÿ”„ Transforming "und" entries โ†’ "en-us" folder`); - } - } else if (originalLocale === 'en-us') { - if (hasUnd && !hasEn) { - // If "und" + "en-us" (no en), "und" becomes "en", so keep "en-us" + console.log( + `๐Ÿ”„ Transforming "und" entries โ†’ "en-us" folder (Rule 3: en+und)` + ); + } else if (!hasEn && !hasEnUs) { + // Rule 1: "und" alone โ†’ "en-us" targetFolder = 'en-us'; - console.log(`๐Ÿ”„ "en-us" entries โ†’ "en-us" folder (und becomes en)`); + console.log( + `๐Ÿ”„ Transforming "und" entries โ†’ "en-us" folder (Rule 1: und alone)` + ); } else { - // Keep en-us as is in other cases - targetFolder = 'en-us'; - } - } else if (originalLocale === 'en') { - if (hasEnUs && !hasUnd) { - // If "en" + "en-us" (no und), "en" becomes "und" folder + // Keep as is for any other combinations targetFolder = 'und'; - console.log(`๐Ÿ”„ Transforming "en" entries โ†’ "und" folder (en-us exists, no und)`); - } else { - // Keep "en" as is in other cases - targetFolder = 'en'; } + } else if (originalLocale === 'en-us') { + // "en-us" always stays as "en-us" in all rules + targetFolder = 'en-us'; + console.log(`๐Ÿ”„ "en-us" entries โ†’ "en-us" folder (stays as is)`); + } else if (originalLocale === 'en') { + // "en" always stays as "en" in all rules (never transforms to "und") + targetFolder = 'en'; + console.log(`๐Ÿ”„ "en" entries โ†’ "en" folder (stays as is)`); } // Merge entries if target folder already has entries if (transformedEntriesByLocale[targetFolder]) { transformedEntriesByLocale[targetFolder] = [ ...transformedEntriesByLocale[targetFolder], - ...entries + ...entries, ]; - console.log(`๐Ÿ“ Merging ${originalLocale} entries into existing ${targetFolder} folder`); + console.log( + `๐Ÿ“ Merging ${originalLocale} entries into existing ${targetFolder} folder` + ); } else { transformedEntriesByLocale[targetFolder] = entries; - console.log(`๐Ÿ“ Creating ${targetFolder} folder for ${originalLocale} entries`); + console.log( + `๐Ÿ“ Creating ${targetFolder} folder for ${originalLocale} entries` + ); } }); - console.log(`๐Ÿ“‚ Final folder structure:`, Object.keys(transformedEntriesByLocale)); + console.log( + `๐Ÿ“‚ Final folder structure:`, + Object.keys(transformedEntriesByLocale) + ); // Find content type mapping for field type switching - const currentContentTypeMapping = contentTypeMapping.find(ct => - ct.otherCmsUid === contentType || ct.contentstackUid === contentType + const currentContentTypeMapping = contentTypeMapping.find( + (ct) => + ct.otherCmsUid === contentType || ct.contentstackUid === contentType ); const allProcessedContent: { [key: string]: any } = {}; // Process entries for each transformed locale separately - for (const [currentLocale, localeEntries] of Object.entries(transformedEntriesByLocale)) { - console.log(`๐ŸŒ Processing ${localeEntries.length} entries for transformed locale: ${currentLocale}`); - + for (const [currentLocale, localeEntries] of Object.entries( + transformedEntriesByLocale + )) { + console.log( + `๐ŸŒ Processing ${localeEntries.length} entries for transformed locale: ${currentLocale}` + ); + // Create folder structure: entries/contentType/locale/ - const contentTypeFolderPath = path.join(MIGRATION_DATA_CONFIG.DATA, destination_stack_id, MIGRATION_DATA_CONFIG.ENTRIES_DIR_NAME, contentType); + const contentTypeFolderPath = path.join( + MIGRATION_DATA_CONFIG.DATA, + destination_stack_id, + MIGRATION_DATA_CONFIG.ENTRIES_DIR_NAME, + contentType + ); const localeFolderPath = path.join(contentTypeFolderPath, currentLocale); await fs.promises.mkdir(localeFolderPath, { recursive: true }); // Read existing content for this locale or initialize const localeFileName = `${currentLocale}.json`; - const existingLocaleContent = await readFile(localeFolderPath, localeFileName) || {}; + const existingLocaleContent = + (await readFile(localeFolderPath, localeFileName)) || {}; // Process each entry in this locale for (const entry of localeEntries) { - let processedEntry = await processFieldData(entry, fieldConfigs, assetId, referenceId, taxonomyId, taxonomyFieldMapping, referenceFieldMapping, assetFieldMapping, taxonomyReferenceLookup, contentType); - + let processedEntry = await processFieldData( + entry, + fieldConfigs, + assetId, + referenceId, + taxonomyId, + taxonomyFieldMapping, + referenceFieldMapping, + assetFieldMapping, + taxonomyReferenceLookup, + contentType + ); + // ๐Ÿท๏ธ TAXONOMY CONSOLIDATION: Merge all taxonomy fields into single 'taxonomies' field - processedEntry = consolidateTaxonomyFields(processedEntry, contentType, taxonomyFieldMapping); - + processedEntry = consolidateTaxonomyFields( + processedEntry, + contentType, + taxonomyFieldMapping + ); + // Apply field type switching based on user's UI selections (from content type schema) const enhancedEntry: any = {}; - + // Process each field with type switching support for (const [fieldName, fieldValue] of Object.entries(processedEntry)) { let fieldMapping = null; - + // First try to find mapping from UI content type mapping - if (currentContentTypeMapping && currentContentTypeMapping.fieldMapping) { - fieldMapping = currentContentTypeMapping.fieldMapping.find((fm: any) => - fm.uid === fieldName || - fm.otherCmsField === fieldName || - fieldName.startsWith(fm.uid) || - fieldName.includes(fm.uid) + if ( + currentContentTypeMapping && + currentContentTypeMapping.fieldMapping + ) { + fieldMapping = currentContentTypeMapping.fieldMapping.find( + (fm: any) => + fm.uid === fieldName || + fm.otherCmsField === fieldName || + fieldName.startsWith(fm.uid) || + fieldName.includes(fm.uid) ); } - + // If no UI mapping found, try to infer from content type schema if (!fieldMapping) { // Load the content type schema to get user's field type selections try { - const contentTypeSchemaPath = path.join(MIGRATION_DATA_CONFIG.DATA, destination_stack_id, 'content_types', `${contentType}.json`); - const contentTypeSchema = JSON.parse(await fs.promises.readFile(contentTypeSchemaPath, 'utf8')); - + const contentTypeSchemaPath = path.join( + MIGRATION_DATA_CONFIG.DATA, + destination_stack_id, + 'content_types', + `${contentType}.json` + ); + const contentTypeSchema = JSON.parse( + await fs.promises.readFile(contentTypeSchemaPath, 'utf8') + ); + // Find field in schema - const schemaField = contentTypeSchema.schema?.find((field: any) => - field.uid === fieldName || - field.uid === fieldName.replace(/_target_id$/, '') || - field.uid === fieldName.replace(/_value$/, '') || - fieldName.includes(field.uid) + const schemaField = contentTypeSchema.schema?.find( + (field: any) => + field.uid === fieldName || + field.uid === fieldName.replace(/_target_id$/, '') || + field.uid === fieldName.replace(/_value$/, '') || + fieldName.includes(field.uid) ); - + if (schemaField) { // Determine the proper field type based on schema configuration let targetFieldType = schemaField.data_type; - + // Handle text fields with multiline metadata - if (schemaField.data_type === 'text' && schemaField.field_metadata?.multiline) { + if ( + schemaField.data_type === 'text' && + schemaField.field_metadata?.multiline + ) { targetFieldType = 'multi_line_text'; // This will be handled as HTML in processFieldByType } - + // Create a mapping from schema field fieldMapping = { uid: fieldName, contentstackFieldType: targetFieldType, backupFieldType: schemaField.data_type, - advanced: schemaField + advanced: schemaField, }; - - console.log(`๐Ÿ“‹ Field mapping created for ${fieldName}: ${targetFieldType} (from schema)`); + + console.log( + `๐Ÿ“‹ Field mapping created for ${fieldName}: ${targetFieldType} (from schema)` + ); } } catch (error: any) { - console.warn(`Failed to load content type schema for field ${fieldName}:`, error.message); + console.warn( + `Failed to load content type schema for field ${fieldName}:`, + error.message + ); } } - + if (fieldMapping) { // Apply field type processing based on user's selection - const processedValue = processFieldByType(fieldValue, fieldMapping, assetId, referenceId); - - // Only add field if processed value is not undefined (undefined means remove field) - if (processedValue !== undefined) { - enhancedEntry[fieldName] = processedValue; - - // Log field type processing - if (fieldMapping.contentstackFieldType !== fieldMapping.backupFieldType) { - const message = getLogMessage( - srcFunc, - `Field ${fieldName} processed as ${fieldMapping.contentstackFieldType} (switched from ${fieldMapping.backupFieldType})`, - {} - ); - await customLogger(projectId, destination_stack_id, 'info', message); - } - } else { - // Log field removal + const processedValue = processFieldByType( + fieldValue, + fieldMapping, + assetId, + referenceId + ); + + // Only add field if processed value is not undefined (undefined means remove field) + if (processedValue !== undefined) { + enhancedEntry[fieldName] = processedValue; + + // Log field type processing + if ( + fieldMapping.contentstackFieldType !== + fieldMapping.backupFieldType + ) { const message = getLogMessage( srcFunc, - `Field ${fieldName} removed due to missing or invalid asset reference`, + `Field ${fieldName} processed as ${fieldMapping.contentstackFieldType} (switched from ${fieldMapping.backupFieldType})`, {} ); - await customLogger(projectId, destination_stack_id, 'warn', message); + await customLogger( + projectId, + destination_stack_id, + 'info', + message + ); } + } else { + // Log field removal + const message = getLogMessage( + srcFunc, + `Field ${fieldName} removed due to missing or invalid asset reference`, + {} + ); + await customLogger( + projectId, + destination_stack_id, + 'warn', + message + ); + } } else { // Keep original value if no mapping found enhancedEntry[fieldName] = fieldValue; } } - + processedEntry = enhancedEntry; - - if (typeof entry.nid === 'number') { - existingLocaleContent[`content_type_entries_title_${entry.nid}`] = processedEntry; - allProcessedContent[`content_type_entries_title_${entry.nid}`] = processedEntry; - } - // Log each entry transformation - const message = getLogMessage( - srcFunc, + if (typeof entry.nid === 'number') { + existingLocaleContent[`content_type_entries_title_${entry.nid}`] = + processedEntry; + allProcessedContent[`content_type_entries_title_${entry.nid}`] = + processedEntry; + } + + // Log each entry transformation + const message = getLogMessage( + srcFunc, `Entry with uid ${entry.nid} (locale: ${currentLocale}) for content type ${contentType} has been successfully transformed.`, - {} - ); - await customLogger(projectId, destination_stack_id, 'info', message); - } + {} + ); + await customLogger(projectId, destination_stack_id, 'info', message); + } // Write processed content for this specific locale await writeFile(localeFolderPath, localeFileName, existingLocaleContent); - + const localeMessage = getLogMessage( srcFunc, `Successfully processed ${localeEntries.length} entries for locale ${currentLocale} in content type ${contentType}`, {} ); - await customLogger(projectId, destination_stack_id, 'info', localeMessage); + await customLogger( + projectId, + destination_stack_id, + 'info', + localeMessage + ); } // ๐Ÿ“ Create mandatory index.json files for each transformed locale directory - for (const [currentLocale, localeEntries] of Object.entries(transformedEntriesByLocale)) { + for (const [currentLocale, localeEntries] of Object.entries( + transformedEntriesByLocale + )) { if (localeEntries.length > 0) { - const contentTypeFolderPath = path.join(MIGRATION_DATA_CONFIG.DATA, destination_stack_id, MIGRATION_DATA_CONFIG.ENTRIES_DIR_NAME, contentType); - const localeFolderPath = path.join(contentTypeFolderPath, currentLocale); + const contentTypeFolderPath = path.join( + MIGRATION_DATA_CONFIG.DATA, + destination_stack_id, + MIGRATION_DATA_CONFIG.ENTRIES_DIR_NAME, + contentType + ); + const localeFolderPath = path.join( + contentTypeFolderPath, + currentLocale + ); const localeFileName = `${currentLocale}.json`; - + // Create mandatory index.json file that maps to the locale file - const indexData = { "1": localeFileName }; + const indexData = { '1': localeFileName }; await writeFile(localeFolderPath, 'index.json', indexData); - - console.log(`๐Ÿ“ Created mandatory index.json for ${contentType}/${currentLocale} โ†’ ${localeFileName}`); + + console.log( + `๐Ÿ“ Created mandatory index.json for ${contentType}/${currentLocale} โ†’ ${localeFileName}` + ); } } @@ -1151,12 +1364,12 @@ const processContentType = async ( isTest: boolean = false ): Promise => { const srcFunc = 'processContentType'; - + try { // Get total count for pagination (if count query exists) const countKey = `${contentType}Count`; let totalCount = 1; // Default to process at least one batch - + if (queryPageConfig.count && queryPageConfig.count[countKey]) { const countQuery = queryPageConfig.count[countKey]; const countResults = await executeQuery(connection, countQuery); @@ -1176,8 +1389,12 @@ const processContentType = async ( // ๐Ÿงช Process entries in batches (test migration: single entry, main migration: all entries) const effectiveLimit = isTest ? 1 : LIMIT; const maxIterations = isTest ? 1 : Math.ceil(totalCount / LIMIT); // Test: single iteration, Main: full pagination - - for (let i = 0; i < (isTest ? effectiveLimit : totalCount + LIMIT); i += effectiveLimit) { + + for ( + let i = 0; + i < (isTest ? effectiveLimit : totalCount + LIMIT); + i += effectiveLimit + ) { const result = await processEntries( connection, contentType, @@ -1197,13 +1414,12 @@ const processContentType = async ( contentTypeMapping, isTest ); - + // If no entries returned, break the loop if (!result) { break; } } - } catch (error: any) { const message = getLogMessage( srcFunc, @@ -1219,24 +1435,33 @@ const processContentType = async ( /** * Reads dynamic query configuration file generated by query.service.ts * Following original pattern: helper.readFile(path.join(process.cwd(), config.data, 'query', 'index.json')) - * + * * NOTE: No fallback to hardcoded queries - dynamic queries MUST be generated first */ -async function readQueryConfig(destination_stack_id: string): Promise { +async function readQueryConfig( + destination_stack_id: string +): Promise { try { - const queryPath = path.join(DATA, destination_stack_id, 'query', 'index.json'); - const data = await fs.promises.readFile(queryPath, "utf8"); + const queryPath = path.join( + DATA, + destination_stack_id, + 'query', + 'index.json' + ); + const data = await fs.promises.readFile(queryPath, 'utf8'); return JSON.parse(data); } catch (err) { // No fallback - dynamic queries must be generated first by createQuery() service - throw new Error(`โŒ No dynamic query configuration found at query/index.json. Dynamic queries must be generated first using createQuery() service. Original error: ${err}`); + throw new Error( + `โŒ No dynamic query configuration found at query/index.json. Dynamic queries must be generated first using createQuery() service. Original error: ${err}` + ); } } /** * Creates and processes entries from Drupal database for migration to Contentstack. * Based on the original Drupal v8 migration logic with direct SQL queries. - * + * * Supports dynamic SQL queries from query/index.json file following original pattern: * var queryPageConfig = helper.readFile(path.join(process.cwd(), config.data, 'query', 'index.json')); * var query = queryPageConfig['page']['' + pagename + '']; @@ -1251,7 +1476,7 @@ export const createEntry = async ( ): Promise => { const srcFunc = 'createEntry'; let connection: mysql.Connection | null = null; - + try { console.info('๐Ÿ” === DRUPAL ENTRIES SERVICE CONFIG ==='); console.info('๐Ÿ“‹ Database Config:', JSON.stringify(dbConfig, null, 2)); @@ -1263,41 +1488,66 @@ export const createEntry = async ( const entriesSave = path.join(DATA, destination_stack_id, ENTRIES_DIR_NAME); const assetsSave = path.join(DATA, destination_stack_id, ASSETS_DIR_NAME); - const referencesSave = path.join(DATA, destination_stack_id, REFERENCES_DIR_NAME); - const taxonomiesSave = path.join(DATA, destination_stack_id, TAXONOMIES_DIR_NAME); - + const referencesSave = path.join( + DATA, + destination_stack_id, + REFERENCES_DIR_NAME + ); + const taxonomiesSave = path.join( + DATA, + destination_stack_id, + TAXONOMIES_DIR_NAME + ); + // Initialize directories await fs.promises.mkdir(entriesSave, { recursive: true }); - const message = getLogMessage( - srcFunc, - `Exporting entries...`, - {} - ); + const message = getLogMessage(srcFunc, `Exporting entries...`, {}); await customLogger(projectId, destination_stack_id, 'info', message); // Read query configuration (following original pattern) const queryPageConfig = await readQueryConfig(destination_stack_id); // Create database connection - connection = await getDbConnection(dbConfig, projectId, destination_stack_id); - + connection = await getDbConnection( + dbConfig, + projectId, + destination_stack_id + ); + // Analyze field types to identify taxonomy, reference, and asset fields - const { taxonomyFields: taxonomyFieldMapping, referenceFields: referenceFieldMapping, assetFields: assetFieldMapping } = await analyzeFieldTypes(dbConfig, destination_stack_id, projectId); - + const { + taxonomyFields: taxonomyFieldMapping, + referenceFields: referenceFieldMapping, + assetFields: assetFieldMapping, + } = await analyzeFieldTypes(dbConfig, destination_stack_id, projectId); + // Fetch field configurations - const fieldConfigs = await fetchFieldConfigs(connection, projectId, destination_stack_id); + const fieldConfigs = await fetchFieldConfigs( + connection, + projectId, + destination_stack_id + ); // Read supporting data - following original page.js pattern // Load assets from index.json (your new format) - const assetId = await readFile(assetsSave, 'index.json') || {}; - console.log(`๐Ÿ“ Loaded ${Object.keys(assetId).length} assets from index.json`); - - const referenceId = await readFile(referencesSave, REFERENCES_FILE_NAME) || {}; - const taxonomyId = await readFile(path.join(entriesSave, 'taxonomy'), `${masterLocale}.json`) || {}; - + const assetId = (await readFile(assetsSave, 'index.json')) || {}; + console.log( + `๐Ÿ“ Loaded ${Object.keys(assetId).length} assets from index.json` + ); + + const referenceId = + (await readFile(referencesSave, REFERENCES_FILE_NAME)) || {}; + const taxonomyId = + (await readFile( + path.join(entriesSave, 'taxonomy'), + `${masterLocale}.json` + )) || {}; + // Load taxonomy reference mappings for field transformation - const taxonomyReferenceLookup = await loadTaxonomyReferences(referencesSave); + const taxonomyReferenceLookup = await loadTaxonomyReferences( + referencesSave + ); // Process each content type from query config (like original) const pageQuery = queryPageConfig.page; @@ -1339,8 +1589,12 @@ export const createEntry = async ( `Multilingual entries structure created at: ${DATA}/${destination_stack_id}/${ENTRIES_DIR_NAME}/[contentType]/[locale]/[locale].json`, {} ); - await customLogger(projectId, destination_stack_id, 'info', structureSummary); - + await customLogger( + projectId, + destination_stack_id, + 'info', + structureSummary + ); } catch (err) { const message = getLogMessage( srcFunc, @@ -1356,4 +1610,4 @@ export const createEntry = async ( connection.end(); } } -}; \ No newline at end of file +}; diff --git a/api/src/services/drupal/locales.service.ts b/api/src/services/drupal/locales.service.ts index 0bf30c667..0cf2edb88 100644 --- a/api/src/services/drupal/locales.service.ts +++ b/api/src/services/drupal/locales.service.ts @@ -1,11 +1,11 @@ -import fs from "fs"; -import path from "path"; -import axios from "axios"; -import { LOCALE_MAPPER, MIGRATION_DATA_CONFIG } from "../../constants/index.js"; -import { Locale } from "../../models/types.js"; -import { getAllLocales, getLogMessage } from "../../utils/index.js"; -import customLogger from "../../utils/custom-logger.utils.js"; -import { createDbConnection } from "../../helper/index.js"; +import fs from 'fs'; +import path from 'path'; +import axios from 'axios'; +import { LOCALE_MAPPER, MIGRATION_DATA_CONFIG } from '../../constants/index.js'; +import { Locale } from '../../models/types.js'; +import { getAllLocales, getLogMessage } from '../../utils/index.js'; +import customLogger from '../../utils/custom-logger.utils.js'; +import { createDbConnection } from '../../helper/index.js'; const { DATA: MIGRATION_DATA_PATH, @@ -31,7 +31,10 @@ async function writeFile(dirPath: string, filename: string, data: any) { /** * Helper function to get key by value from an object */ -function getKeyByValue(obj: Record, targetValue: string): string | undefined { +function getKeyByValue( + obj: Record, + targetValue: string +): string | undefined { return Object.entries(obj).find(([_, value]) => value === targetValue)?.[0]; } @@ -40,7 +43,9 @@ function getKeyByValue(obj: Record, targetValue: string): string */ async function fetchContentstackLocales(): Promise> { try { - const response = await axios.get('https://app.contentstack.com/api/v3/locales?include_all=true'); + const response = await axios.get( + 'https://app.contentstack.com/api/v3/locales?include_all=true' + ); return response.data?.locales || {}; } catch (error) { console.error('Error fetching Contentstack locales:', error); @@ -50,61 +55,87 @@ async function fetchContentstackLocales(): Promise> { /** * Applies special locale code transformations based on business rules + * - "und" alone โ†’ "en-us" + * - "und" + "en-us" โ†’ "und" become "en", "en-us" stays + * - "en" + "und" โ†’ "und" becomes "en-us", "en" stays + * - All three "en" + "und" + "en-us" โ†’ all three stays + * - Apart from these, all other locales stay as is */ -function applyLocaleTransformations(locales: string[], masterLocale: string): { code: string; name: string; isMaster: boolean }[] { +function applyLocaleTransformations( + locales: string[], + masterLocale: string +): { code: string; name: string; isMaster: boolean }[] { const hasUnd = locales.includes('und'); const hasEn = locales.includes('en'); const hasEnUs = locales.includes('en-us'); - - return locales.map(locale => { - let code = locale.toLowerCase(); + + // First, apply the transformation rules to get the correct locale codes + const transformedCodes: string[] = []; + + // Start with all non-special locales (not und, en, en-us) + const nonSpecialLocales = locales.filter( + (locale) => !['und', 'en', 'en-us'].includes(locale) + ); + transformedCodes.push(...nonSpecialLocales); + + // Apply transformation rules based on combinations + if (hasEn && hasUnd && hasEnUs) { + // Rule 4: All three "en" + "und" + "en-us" โ†’ all three stays + transformedCodes.push('en', 'und', 'en-us'); + } else if (hasUnd && hasEnUs && !hasEn) { + // Rule 2: "und" + "en-us" โ†’ "und" become "en", "en-us" stays + transformedCodes.push('en', 'en-us'); + } else if (hasEn && hasUnd && !hasEnUs) { + // Rule 3: "en" + "und" โ†’ "und" becomes "en-us", "en" stays + transformedCodes.push('en', 'en-us'); + } else if (hasUnd && !hasEn && !hasEnUs) { + // Rule 1: "und" alone โ†’ "en-us" + transformedCodes.push('en-us'); + } else { + // For any other combinations, keep locales as they are + if (hasEn) transformedCodes.push('en'); + if (hasUnd) transformedCodes.push('und'); + if (hasEnUs) transformedCodes.push('en-us'); + } + + // Remove duplicates and sort + const uniqueTransformedCodes = Array.from(new Set(transformedCodes)).sort(); + + // Now map each transformed code to the proper format with names + return uniqueTransformedCodes.map((code) => { let name = ''; - let isMaster = locale === masterLocale; - - // Apply transformation rules - if (locale === 'und') { - if (hasEn && hasEnUs) { - // If all three present, "und" stays as "und" - code = 'und'; - name = 'Language Neutral'; - } else if (hasEnUs) { - // If "und" + "en-us", "und" becomes "en" - code = 'en'; + let isMaster = false; + + // Determine if this is the master locale (check against original and transformed) + isMaster = + code === masterLocale || + (masterLocale === 'und' && code === 'en-us') || // Rule 1 transformation + (masterLocale === 'und' && hasEnUs && code === 'en') || // Rule 2 transformation + (masterLocale === 'und' && hasEn && code === 'en-us'); // Rule 3 transformation + + // Set appropriate names + switch (code) { + case 'en': name = 'English'; - } else { - // If only "und", becomes "en-us" - code = 'en-us'; - name = 'English - United States'; - } - } else if (locale === 'en-us') { - if (hasUnd && !hasEn) { - // If "und" + "en-us" (no en), "und" becomes "en", so keep "en-us" - code = 'en-us'; - name = 'English - United States'; - } else { - // Keep en-us as is in other cases - code = 'en-us'; + break; + case 'en-us': name = 'English - United States'; - } - } else if (locale === 'en') { - if (hasEnUs && !hasUnd) { - // If "en" + "en-us" (no und), "en" becomes "und" - code = 'und'; + break; + case 'und': name = 'Language Neutral'; - } else { - // Keep "en" as is in other cases - code = 'en'; - name = 'English'; - } + break; + default: + name = ''; // Will be filled from Contentstack API later + break; } - - return { code, name, isMaster }; + + return { code: code.toLowerCase(), name, isMaster }; }); } /** * Processes and creates locale configurations from Drupal database for migration to Contentstack. - * + * * This function: * 1. Fetches master locale from Drupal system.site config * 2. Fetches all locales from node_field_data @@ -120,28 +151,25 @@ export const createLocale = async ( project: any ) => { const srcFunc = 'createLocale'; - const localeSave = path.join(MIGRATION_DATA_PATH, destination_stack_id, LOCALE_DIR_NAME); + const localeSave = path.join( + MIGRATION_DATA_PATH, + destination_stack_id, + LOCALE_DIR_NAME + ); try { const msLocale: Record = {}; const allLocales: Record = {}; const localeList: Record = {}; - // Create database connection using dbConfig - console.log('๐Ÿ” Database config for locales:', { - host: dbConfig?.host, - user: dbConfig?.user, - database: dbConfig?.database, - port: dbConfig?.port, - hasPassword: !!dbConfig?.password - }); - if (!dbConfig || !dbConfig.host || !dbConfig.user || !dbConfig.database) { - throw new Error('Invalid database configuration provided to createLocale'); + throw new Error( + 'Invalid database configuration provided to createLocale' + ); } - + const connection = await createDbConnection(dbConfig); - + if (!connection) { throw new Error('Failed to create database connection'); } @@ -179,7 +207,7 @@ export const createLocale = async ( WHERE langcode IS NOT NULL AND langcode != '' ORDER BY langcode `; - + const allLocaleRows: any = await executeQuery(allLocalesQuery); const allLocaleCodes = allLocaleRows.map((row: any) => row.langcode); @@ -202,7 +230,7 @@ export const createLocale = async ( ) ORDER BY n.langcode `; - + const nonMasterRows: any = await executeQuery(nonMasterLocalesQuery); const nonMasterLocaleCodes = nonMasterRows.map((row: any) => row.langcode); @@ -213,19 +241,28 @@ export const createLocale = async ( const contentstackLocales = await fetchContentstackLocales(); // 5. Apply special transformation rules - const transformedLocales = applyLocaleTransformations(allLocaleCodes, masterLocaleCode); + const transformedLocales = applyLocaleTransformations( + allLocaleCodes, + masterLocaleCode + ); // 6. Process each locale transformedLocales.forEach((localeInfo, index) => { const { code, name, isMaster } = localeInfo; - + // Create UID using original langcode from database const originalLangcode = allLocaleCodes[index]; // Get original langcode from database - const uid = `drupallocale_${originalLangcode.toLowerCase().replace(/-/g, '_')}`; - + const uid = `drupallocale_${originalLangcode + .toLowerCase() + .replace(/-/g, '_')}`; + // Get name from Contentstack API or use transformed name - const localeName = name || contentstackLocales[code] || contentstackLocales[code.toLowerCase()] || 'Unknown Language'; - + const localeName = + name || + contentstackLocales[code] || + contentstackLocales[code.toLowerCase()] || + 'Unknown Language'; + const newLocale: Locale = { code: code.toLowerCase(), name: localeName, @@ -239,17 +276,18 @@ export const createLocale = async ( } else { allLocales[uid] = newLocale; } - + localeList[uid] = newLocale; }); // Handle case where no non-master locales exist - const finalAllLocales = Object.keys(allLocales).length > 0 ? allLocales : {}; + const finalAllLocales = + Object.keys(allLocales).length > 0 ? allLocales : {}; // 7. Write locale files (same structure as Contentful) - await writeFile(localeSave, LOCALE_FILE_NAME, finalAllLocales); // locales.json (non-master only) - await writeFile(localeSave, LOCALE_MASTER_LOCALE, msLocale); // master-locale.json (master only) - await writeFile(localeSave, LOCALE_CF_LANGUAGE, localeList); // language.json (all locales) + await writeFile(localeSave, LOCALE_FILE_NAME, finalAllLocales); // locales.json (non-master only) + await writeFile(localeSave, LOCALE_MASTER_LOCALE, msLocale); // master-locale.json (master only) + await writeFile(localeSave, LOCALE_CF_LANGUAGE, localeList); // language.json (all locales) const message = getLogMessage( srcFunc, @@ -257,7 +295,6 @@ export const createLocale = async ( {} ); await customLogger(projectId, destination_stack_id, 'info', message); - } catch (err) { const message = getLogMessage( srcFunc, diff --git a/api/src/services/drupal/query.service.ts b/api/src/services/drupal/query.service.ts index 2a8db407b..e36b55926 100644 --- a/api/src/services/drupal/query.service.ts +++ b/api/src/services/drupal/query.service.ts @@ -34,16 +34,20 @@ interface QueryConfig { * Get field information by querying the database for a specific field * Enhanced to handle link fields with both URI and TITLE columns */ -const getQuery = (connection: mysql.Connection, data: DrupalFieldData): Promise => { +const getQuery = ( + connection: mysql.Connection, + data: DrupalFieldData +): Promise => { return new Promise((resolve, reject) => { try { const tableName = `node__${data.field_name}`; - + // Check if this is a link field first if (data.type !== 'link') { // For non-link fields, use existing logic const value = data.field_name; - const handlerType = data.content_handler === undefined ? 'invalid' : data.content_handler; + const handlerType = + data.content_handler === undefined ? 'invalid' : data.content_handler; const query = `SELECT *, '${handlerType}' as handler, '${data.type}' as fieldType FROM ${tableName}`; connection.query(query, (error: any, rows: any, fields: any) => { @@ -51,14 +55,16 @@ const getQuery = (connection: mysql.Connection, data: DrupalFieldData): Promise< // Look for field patterns in the database columns for (const field of fields) { const fieldName = field.name; - + // Check for various Drupal field suffixes - if (fieldName === `${value}_value` || - fieldName === `${value}_fid` || - fieldName === `${value}_tid` || - fieldName === `${value}_status` || - fieldName === `${value}_target_id` || - fieldName === `${value}_uri`) { + if ( + fieldName === `${value}_value` || + fieldName === `${value}_fid` || + fieldName === `${value}_tid` || + fieldName === `${value}_status` || + fieldName === `${value}_target_id` || + fieldName === `${value}_uri` + ) { const fieldTable = `node__${data.field_name}.${fieldName}`; resolve(fieldTable); return; @@ -73,33 +79,43 @@ const getQuery = (connection: mysql.Connection, data: DrupalFieldData): Promise< }); return; } - + // For LINK fields only - get both URI and TITLE columns - connection.query(`SHOW COLUMNS FROM ${tableName}`, (error: any, columns: any) => { - if (error) { - console.error(`Error querying columns for link field ${data.field_name}:`, error); - resolve(''); - return; - } - - // Filter for link-specific columns only - const linkColumns = columns - .map((col: any) => col.Field) - .filter((field: string) => - (field === `${data.field_name}_uri` || field === `${data.field_name}_title`) && - field.startsWith(data.field_name) - ); - - if (linkColumns.length > 0) { - // Return both columns as MAX aggregations for link fields - const maxColumns = linkColumns.map((col: string) => `MAX(${tableName}.${col}) as ${col}`); - resolve(maxColumns.join(',')); - } else { - // Fallback to just URI if title doesn't exist - const uriColumn = `${data.field_name}_uri`; - resolve(`MAX(${tableName}.${uriColumn}) as ${uriColumn}`); + connection.query( + `SHOW COLUMNS FROM ${tableName}`, + (error: any, columns: any) => { + if (error) { + console.error( + `Error querying columns for link field ${data.field_name}:`, + error + ); + resolve(''); + return; + } + + // Filter for link-specific columns only + const linkColumns = columns + .map((col: any) => col.Field) + .filter( + (field: string) => + (field === `${data.field_name}_uri` || + field === `${data.field_name}_title`) && + field.startsWith(data.field_name) + ); + + if (linkColumns.length > 0) { + // Return both columns as MAX aggregations for link fields + const maxColumns = linkColumns.map( + (col: string) => `MAX(${tableName}.${col}) as ${col}` + ); + resolve(maxColumns.join(',')); + } else { + // Fallback to just URI if title doesn't exist + const uriColumn = `${data.field_name}_uri`; + resolve(`MAX(${tableName}.${uriColumn}) as ${uriColumn}`); + } } - }); + ); } catch (error) { console.error('Error in getQuery', error); resolve(''); // Resolve with empty string on error to continue process @@ -117,28 +133,37 @@ const generateQueriesForFields = async ( destination_stack_id: string ): Promise => { const srcFunc = 'generateQueriesForFields'; - + try { const select: { [contentType: string]: string } = {}; const countQuery: { [contentType: string]: string } = {}; - + // Group fields by content type - const contentTypes = [...new Set(fieldData.map(field => field.content_types))]; - + const contentTypes = [ + ...new Set(fieldData.map((field) => field.content_types)), + ]; + const message = `Processing ${contentTypes.length} content types for query generation...`; await customLogger(projectId, destination_stack_id, 'info', message); // Process each content type for (const contentType of contentTypes) { - const fieldsForType = fieldData.filter(field => field.content_types === contentType); + const fieldsForType = fieldData.filter( + (field) => field.content_types === contentType + ); const fieldCount = fieldsForType.length; const maxJoinLimit = 50; // Conservative limit to avoid MySQL's 61-table limit - + // Check if content type has too many fields for single query if (fieldCount > maxJoinLimit) { const warningMessage = `Content type '${contentType}' has ${fieldCount} fields (>${maxJoinLimit} limit). Using optimized base query only.`; - await customLogger(projectId, destination_stack_id, 'warn', warningMessage); - + await customLogger( + projectId, + destination_stack_id, + 'warn', + warningMessage + ); + // Generate simple base query without field JOINs to avoid MySQL limit const baseQuery = ` SELECT @@ -151,28 +176,38 @@ const generateQueriesForFields = async ( LEFT JOIN users ON users.uid = node.uid WHERE node.type = '${contentType}' GROUP BY node.nid - `.replace(/\s+/g, ' ').trim(); - + ` + .replace(/\s+/g, ' ') + .trim(); + const baseCountQuery = ` SELECT COUNT(DISTINCT node.nid) as countentry FROM node_field_data node WHERE node.type = '${contentType}' - `.replace(/\s+/g, ' ').trim(); - - select[contentType] = baseQuery + ` /* OPTIMIZED_NO_JOINS:${fieldCount} */`; + ` + .replace(/\s+/g, ' ') + .trim(); + + select[contentType] = + baseQuery + ` /* OPTIMIZED_NO_JOINS:${fieldCount} */`; countQuery[`${contentType}Count`] = baseCountQuery; - + const optimizedMessage = `Generated optimized base query for ${contentType} (avoiding ${fieldCount} JOINs)`; - await customLogger(projectId, destination_stack_id, 'info', optimizedMessage); - + await customLogger( + projectId, + destination_stack_id, + 'info', + optimizedMessage + ); + continue; // Skip to next content type } - + const tableJoins: string[] = []; const queries: Promise[] = []; // Collect all field queries (only for content types with manageable field count) - fieldsForType.forEach(fieldData => { + fieldsForType.forEach((fieldData) => { tableJoins.push(`node__${fieldData.field_name}`); queries.push(getQuery(connection, fieldData)); }); @@ -180,17 +215,17 @@ const generateQueriesForFields = async ( try { // Wait for all field queries to complete const results = await Promise.all(queries); - + // Filter out empty results - const validResults = results.filter(item => item); - + const validResults = results.filter((item) => item); + if (validResults.length === 0) { console.log(`No valid fields found for content type ${contentType}`); continue; } // Build the SELECT clause with proper handling for link fields - const modifiedResults = validResults.map(item => { + const modifiedResults = validResults.map((item) => { // Check if this is already a MAX aggregation (link fields) if (item.includes('MAX(') && item.includes(' as ')) { return item; // Link fields are already properly formatted @@ -201,14 +236,14 @@ const generateQueriesForFields = async ( // Build LEFT JOIN clauses const leftJoins = tableJoins.map( - table => `LEFT JOIN ${table} ON ${table}.entity_id = node.nid` + (table) => `LEFT JOIN ${table} ON ${table}.entity_id = node.nid` ); leftJoins.push('LEFT JOIN users ON users.uid = node.uid'); // Construct the complete query const selectClause = [ 'SELECT node.nid, MAX(node.title) AS title, MAX(node.langcode) AS langcode, MAX(node.created) as created, MAX(node.type) as type', - ...modifiedResults + ...modifiedResults, ].join(','); const fromClause = 'FROM node_field_data node'; @@ -218,29 +253,43 @@ const generateQueriesForFields = async ( // Final query construction const finalQuery = `${selectClause} ${fromClause} ${joinClause} ${whereClause} ${groupClause}`; - + // Clean up any double commas - select[contentType] = finalQuery.replace(/,,/g, ',').replace(/, ,/g, ','); + select[contentType] = finalQuery + .replace(/,,/g, ',') + .replace(/, ,/g, ','); // Build count query const countQueryStr = `SELECT count(distinct(node.nid)) as countentry ${fromClause} ${joinClause} ${whereClause}`; countQuery[`${contentType}Count`] = countQueryStr; const fieldMessage = `Generated queries for content type: ${contentType} with ${validResults.length} fields`; - await customLogger(projectId, destination_stack_id, 'info', fieldMessage); - + await customLogger( + projectId, + destination_stack_id, + 'info', + fieldMessage + ); } catch (error) { const errorMessage = `Error processing queries for content type: ${contentType}`; - await customLogger(projectId, destination_stack_id, 'error', errorMessage); - console.error('Error processing queries for content type:', contentType, error); + await customLogger( + projectId, + destination_stack_id, + 'error', + errorMessage + ); + console.error( + 'Error processing queries for content type:', + contentType, + error + ); } } return { page: select, - count: countQuery + count: countQuery, }; - } catch (error: any) { const errorMessage = `Error in generateQueriesForFields: ${error.message}`; await customLogger(projectId, destination_stack_id, 'error', errorMessage); @@ -254,9 +303,9 @@ const generateQueriesForFields = async ( */ /** * Validates that query configuration file exists (legacy compatibility) - * + * * NOTE: This function is for backward compatibility. - * The new dynamic query system uses createQuery() which generates queries + * The new dynamic query system uses createQuery() which generates queries * based on actual database field analysis. */ export const createQueryConfig = async ( @@ -269,10 +318,11 @@ export const createQueryConfig = async ( try { // Check if dynamic query file exists (should be created by createQuery service) await fs.promises.access(queryPath); - console.log(`โœ… Dynamic query configuration found at: ${queryPath}`); } catch (error) { // If no dynamic queries exist, this is an error since we removed hardcoded fallbacks - throw new Error(`โŒ No query configuration found at ${queryPath}. Dynamic queries must be generated first using createQuery() service.`); + throw new Error( + `โŒ No query configuration found at ${queryPath}. Dynamic queries must be generated first using createQuery() service.` + ); } }; @@ -285,13 +335,6 @@ export const createQuery = async ( let connection: mysql.Connection | null = null; try { - console.info('๐Ÿ” === DRUPAL QUERY SERVICE CONFIG ==='); - console.info('๐Ÿ“‹ Database Config:', JSON.stringify(dbConfig, null, 2)); - console.info('๐Ÿ“‹ Destination Stack ID:', destination_stack_id); - console.info('๐Ÿ“‹ Project ID:', projectId); - console.info('๐Ÿ“‹ Function:', srcFunc); - console.info('======================================'); - const queryDir = path.join(DATA, destination_stack_id, 'query'); const queryPath = path.join(queryDir, 'index.json'); @@ -302,13 +345,18 @@ export const createQuery = async ( await customLogger(projectId, destination_stack_id, 'info', message); // Create database connection - connection = await getDbConnection(dbConfig, projectId, destination_stack_id); + connection = await getDbConnection( + dbConfig, + projectId, + destination_stack_id + ); // SQL query to extract field configuration from Drupal - const configQuery = "SELECT *, CONVERT(data USING utf8) as data FROM config WHERE name LIKE '%field.field.node%'"; + const configQuery = + "SELECT *, CONVERT(data USING utf8) as data FROM config WHERE name LIKE '%field.field.node%'"; // Execute query using promise-based approach - const [rows] = await connection.promise().query(configQuery) as any[]; + const [rows] = (await connection.promise().query(configQuery)) as any[]; let fieldData: DrupalFieldData[] = []; @@ -317,12 +365,16 @@ export const createQuery = async ( try { const { unserialize } = await import('php-serialize'); const convDetails = unserialize(rows[i].data); - if (convDetails && typeof convDetails === 'object' && 'field_name' in convDetails) { + if ( + convDetails && + typeof convDetails === 'object' && + 'field_name' in convDetails + ) { fieldData.push({ field_name: convDetails.field_name, content_types: convDetails.bundle, type: convDetails.field_type, - content_handler: convDetails?.settings?.handler + content_handler: convDetails?.settings?.handler, }); } } catch (err: any) { @@ -338,23 +390,30 @@ export const createQuery = async ( await customLogger(projectId, destination_stack_id, 'info', fieldMessage); // Generate queries based on field data - const queryConfig = await generateQueriesForFields(connection, fieldData, projectId, destination_stack_id); + const queryConfig = await generateQueriesForFields( + connection, + fieldData, + projectId, + destination_stack_id + ); // Write query configuration to file - await fs.promises.writeFile(queryPath, JSON.stringify(queryConfig, null, 4), 'utf8'); + await fs.promises.writeFile( + queryPath, + JSON.stringify(queryConfig, null, 4), + 'utf8' + ); const successMessage = `Successfully generated and saved dynamic queries to: ${queryPath}`; await customLogger(projectId, destination_stack_id, 'info', successMessage); - - console.info(`โœ… Dynamic query configuration created at: ${queryPath}`); - console.info(`๐Ÿ“Š Generated queries for ${Object.keys(queryConfig.page).length} content types`); - } catch (error: any) { const errorMessage = `Failed to generate dynamic queries: ${error.message}`; await customLogger(projectId, destination_stack_id, 'error', errorMessage); - + console.error('โŒ Error generating dynamic queries:', error); - throw new Error(`Failed to connect to database or generate queries: ${error.message}`); + throw new Error( + `Failed to connect to database or generate queries: ${error.message}` + ); } finally { // Always close the connection when done if (connection) { diff --git a/ui/package.json b/ui/package.json index 862210464..994790048 100644 --- a/ui/package.json +++ b/ui/package.json @@ -36,6 +36,8 @@ "start": "react-scripts start", "build": "react-scripts build", "test": "react-scripts test", + "test:language-mapper": "node test-language-mapper.js", + "test:language-mapper-unit": "react-scripts test --testNamePattern='LoadLanguageMapper' --watchAll=false", "eject": "react-scripts eject", "prettify": "prettier --write .", "lint": "eslint .", diff --git a/ui/src/components/DestinationStack/Actions/LoadLanguageMapper.tsx b/ui/src/components/DestinationStack/Actions/LoadLanguageMapper.tsx index af8ab43c4..be3b893c1 100644 --- a/ui/src/components/DestinationStack/Actions/LoadLanguageMapper.tsx +++ b/ui/src/components/DestinationStack/Actions/LoadLanguageMapper.tsx @@ -76,12 +76,12 @@ const Mapper = ({ const [sourceoptions, setsourceoptions] = useState(sourceOptions); const newMigrationData = useSelector((state: RootState) => state?.migration?.newMigrationData); const dispatch = useDispatch(); - const [selectedStack, setSelectedStack] = useState(); + // const [selectedStack, setSelectedStack] = useState(); const [placeholder] = useState('Select language'); - useEffect(()=>{ - setSelectedStack(stack); - },[]); + // useEffect(()=>{ + // setSelectedStack(stack); + // },[]); useEffect(() => { const newMigrationDataObj: INewMigration = { @@ -287,7 +287,7 @@ const Mapper = ({ selectedValue: { label: string; value: string }, index: number, ) => { - const csLocaleKey = existingField?.[index]?.value; + // const csLocaleKey = existingField?.[index]?.value; const selectedLocaleKey = selectedValue?.value; const existingLabel = existingField?.[index]; //const selectedLocaleKey = selectedMappings[index]; @@ -381,7 +381,7 @@ const Mapper = ({ // Rebuild object with new sequential keys const updatedOptions = Object?.fromEntries( - entries?.map(([_, value], index) => [index, value]) + entries?.map(([, value], newIndex) => [newIndex, value]) ); //sourceLocale = updatedOptions[index]?.label; @@ -418,7 +418,15 @@ const Mapper = ({ return ( <> - {cmsLocaleOptions?.length > 0 ? ( + {/* ๐Ÿ”ง TC-11: Enhanced empty source locales handling */} + {!sourceOptions || sourceOptions.length === 0 ? ( + } + content="No source locales found. Please check your source configuration." + type="warning" + /> + ) : cmsLocaleOptions?.length > 0 ? ( cmsLocaleOptions?.map((locale: {label:string, value: string}, index: number) => (
@@ -503,7 +511,7 @@ const Mapper = ({ handleSelectedSourceLocale(data, index) } styles={{ - menuPortal: (base: any) => ({ ...base, zIndex: 9999 }) + menuPortal: (base: Record) => ({ ...base, zIndex: 9999 }) }} options={sourceoptions} placeholder={placeholder} @@ -565,11 +573,11 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { const [currentStack, setCurrentStack] = useState(stack); const [previousStack, setPreviousStack] = useState(); const [isStackChanged, setisStackChanged] = useState(false); - const [stackValue, setStackValue] = useState(stack?.value) + // const [stackValue, setStackValue] = useState(stack?.value) const [autoSelectedSourceLocale, setAutoSelectedSourceLocale] = useState<{ label: string; value: string } | null>(null); const [mapperLocaleState, setMapperLocaleState] = useState({}); - const prevStackRef:any = useRef(null); + const prevStackRef: React.MutableRefObject = useRef(null); useEffect(() => { if (prevStackRef?.current && stack && stack?.uid !== prevStackRef?.current?.uid) { @@ -641,6 +649,60 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { return fallback ? 'en-us' : availableContentstackLocales[0]?.value || sourceLocaleCode; }; + // ๐Ÿ†• Helper function to get next unmapped source locale for Add Language functionality + const getNextUnmappedSourceLocale = (): { label: string; value: string } | null => { + if (!sourceLocales || sourceLocales.length === 0) return null; + + // Get currently mapped source locales (values from localeMapping) + const mappedSourceLocales = Object.values(newMigrationData?.destination_stack?.localeMapping || {}) + .filter(Boolean) // Remove empty strings + .filter(locale => locale !== stack?.master_locale); // Exclude master locale + + // Find first unmapped source locale + const unmappedLocale = sourceLocales.find(source => + !mappedSourceLocales.includes(source.value) && + source.value !== stack?.master_locale // Don't suggest master locale again + ); + + return unmappedLocale || null; + }; + + // ๐Ÿ†• Helper function to check if source locale exists in destination + // Enhanced to handle duplicate/ambiguous locales with exact match preference + const findDestinationMatch = (sourceLocale: string): { label: string; value: string } | null => { + // ๐Ÿ†• STEP 1: Try exact match (case-insensitive) - HIGHEST PRIORITY + const exactMatch = options.find(dest => + dest.value.toLowerCase() === sourceLocale.toLowerCase() + ); + + if (exactMatch) { + console.info(`๐ŸŽฏ Exact match found for ${sourceLocale}: ${exactMatch.value}`); + return exactMatch; + } + + // ๐Ÿ†• STEP 2: Try smart mapping only if no exact match + // This handles cases like 'en' -> 'en-us' but prefers exact matches + const smartMatch = getSmartLocaleMapping(sourceLocale, options); + const smartMatchObj = options.find(dest => dest.value === smartMatch); + + if (smartMatchObj && smartMatchObj.value !== sourceLocale) { + console.info(`๐Ÿง  Smart match found for ${sourceLocale}: ${smartMatchObj.value}`); + } + + return smartMatchObj || null; + }; + + // ๐Ÿ†• Helper function to get unmapped destination locales for display + const getUnmappedDestinationLocales = (): { label: string; value: string }[] => { + const mappedDestinations = Object.keys(newMigrationData?.destination_stack?.localeMapping || {}) + .filter(key => !key.includes('-master_locale')); // Exclude master locale keys + + return options.filter(dest => + !mappedDestinations.includes(dest.value) && + dest.value !== stack?.master_locale // Don't include master locale + ); + }; + // ๐Ÿš€ UNIVERSAL LOCALE AUTO-MAPPING (All CMS platforms) // Handles both single and multiple locale scenarios useEffect(() => { @@ -697,8 +759,21 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { console.info(' Is Correct Step:', isCorrectStep); } - // โœ… EXISTING: Single locale auto-mapping (PRESERVED) - // Enhanced condition: Also trigger on stack changes for existing templates + // ๐Ÿ”ง TC-11: Handle empty source locales + if (!sourceLocale || sourceLocale.length === 0) { + console.warn('โš ๏ธ No source locales found - cannot proceed with mapping'); + return; // Exit early - will show "No languages configured" message + } + + // ๐Ÿ”ง TC-12: Handle empty destination locales (ensure master locale exists) + if (!allLocales || allLocales.length === 0) { + console.warn('โš ๏ธ No destination locales found - adding master locale as fallback'); + const masterLocaleObj = { label: stack?.master_locale || 'en-us', value: stack?.master_locale || 'en-us' }; + setoptions([masterLocaleObj]); + return; // Will re-trigger this effect with master locale added + } + + // โœ… EXISTING: Single locale auto-mapping (ENHANCED for TC-02) const shouldAutoMapSingle = (Object?.entries(newMigrationData?.destination_stack?.localeMapping || {})?.length === 0 || !keys || stackHasChanged); @@ -722,18 +797,31 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { newMigrationData?.project_current_step <= 2) { const singleSourceLocale = sourceLocale[0]; - const smartDestinationLocale = getSmartLocaleMapping(singleSourceLocale.value, allLocales); + + // ๐Ÿ”ง TC-02: Check for exact match first, only use smart mapping if exact match exists + const exactMatch = allLocales.find(dest => + dest.value.toLowerCase() === singleSourceLocale.value.toLowerCase() + ); + + let destinationLocale = ''; + if (exactMatch) { + destinationLocale = exactMatch.value; + console.info(`โœ… TC-02: Exact match found for single locale: ${singleSourceLocale.value} -> ${destinationLocale}`); + } else { + console.info(`๐Ÿ“ TC-02: No exact match for ${singleSourceLocale.value} - leaving destination empty for manual selection`); + // Leave destinationLocale empty as per TC-02 requirement + } // Set the auto-selected source locale for the Mapper component setAutoSelectedSourceLocale({ - label: singleSourceLocale.value, // Source locale (e.g., "en") - value: singleSourceLocale.value // Source locale for dropdown selection + label: singleSourceLocale.value, + value: singleSourceLocale.value }); // Set the mapping in Redux state const autoMapping = { [`${stack?.master_locale}-master_locale`]: stack?.master_locale, - [singleSourceLocale.value]: smartDestinationLocale + ...(destinationLocale ? { [destinationLocale]: singleSourceLocale.value } : {}) }; const newMigrationDataObj: INewMigration = { @@ -760,29 +848,141 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { stackHasChanged) && newMigrationData?.project_current_step <= 2) { - // console.info('๐Ÿš€ EXECUTING Multi-locale auto-mapping...'); - // console.info(' Source Locales for matching:', sourceLocale); - // console.info(' Available Destination Locales:', allLocales.slice(0, 10)); + console.info('๐Ÿš€ EXECUTING Multi-locale auto-mapping...'); + console.info(' Source Locales for matching:', sourceLocale); + console.info(' Available Destination Locales:', allLocales.slice(0, 10)); + + // ๐Ÿ†• CONDITION 2: Enhanced multi-locale logic with master locale priority + + // First, check if master locale from source matches destination (PRIORITY) + const masterLocaleFromSource = sourceLocale.find(source => + source.value.toLowerCase() === stack?.master_locale?.toLowerCase() + ); + + let hasAnyMatches = false; // Build auto-mapping for exact matches (case-insensitive) const autoMapping: Record = { [`${stack?.master_locale}-master_locale`]: stack?.master_locale }; + // ๐Ÿ†• STEP 1: Handle master locale priority first + if (masterLocaleFromSource) { + const masterDestMatch = allLocales.find(dest => + dest.value.toLowerCase() === masterLocaleFromSource.value.toLowerCase() + ); + if (masterDestMatch) { + autoMapping[masterLocaleFromSource.value] = masterDestMatch.value; + hasAnyMatches = true; + console.info(`๐Ÿ† Master locale priority mapping: ${masterLocaleFromSource.value} -> ${masterDestMatch.value}`); + } + } + + // ๐Ÿ†• STEP 2: Handle other locales with enhanced duplicate/ambiguous logic sourceLocale.forEach(source => { - // Case-insensitive exact matching only + // Skip if already mapped (master locale) + if (autoMapping[source.value]) return; + + // ๐Ÿ†• Enhanced matching: Prefer exact matches over partial/smart matches + // First try exact match const exactMatch = allLocales.find(dest => source.value.toLowerCase() === dest.value.toLowerCase() ); - // console.info(` Checking ${source.value} -> Found match:`, exactMatch?.value || 'No match'); - if (exactMatch) { autoMapping[source.value] = exactMatch.value; + hasAnyMatches = true; + console.info(`๐ŸŽฏ Exact match: ${source.value} -> ${exactMatch.value}`); + } else { + // Only try smart matching if no exact match exists + // This prevents 'en' from overriding 'en-us' -> 'en-us' mapping + const hasExactMatchInSource = sourceLocale.some(otherSource => + otherSource.value !== source.value && + allLocales.some(dest => dest.value.toLowerCase() === otherSource.value.toLowerCase()) + ); + + if (!hasExactMatchInSource) { + // Use smart mapping as fallback + const smartMatch = getSmartLocaleMapping(source.value, allLocales); + const smartMatchObj = allLocales.find(dest => dest.value === smartMatch); + + if (smartMatchObj && smartMatchObj.value !== source.value) { + autoMapping[source.value] = smartMatchObj.value; + hasAnyMatches = true; + console.info(`๐Ÿง  Smart match: ${source.value} -> ${smartMatchObj.value}`); + } else { + console.info(`โŒ No match found for: ${source.value}`); + } + } else { + console.info(`โญ๏ธ Skipping smart match for ${source.value} - exact matches take priority`); + } } }); - // console.info('๐ŸŽฏ Final Auto-mapping Result:', autoMapping); + // ๐Ÿ”ง TC-04 & TC-08: Enhanced no-match logic with master locale default + if (!hasAnyMatches) { + console.info('โš ๏ธ TC-04/TC-08: No matches found - setting up master locale default mapping'); + + // Auto-select destination master locale for first source locale as per TC-04/TC-08 + const firstSourceLocale = sourceLocale[0]; + const masterLocaleMapping = { + [`${stack?.master_locale}-master_locale`]: stack?.master_locale, + [stack?.master_locale || 'en-us']: firstSourceLocale.value // Map destination master to first source + }; + + const newMigrationDataObj: INewMigration = { + ...newMigrationData, + destination_stack: { + ...newMigrationData?.destination_stack, + localeMapping: masterLocaleMapping + } + }; + dispatch(updateNewMigrationData(newMigrationDataObj)); + + // Update the existingLocale state to show the mapping in UI + const updatedExistingLocale: ExistingFieldType = {}; + if (cmsLocaleOptions?.length > 0) { + // Find the master locale row + const masterRow = cmsLocaleOptions.find(locale => locale.value === 'master_locale'); + if (masterRow) { + updatedExistingLocale[masterRow.label] = { + label: firstSourceLocale.value, + value: firstSourceLocale.value + }; + } + } + setMapperLocaleState(prev => ({ ...prev, ...updatedExistingLocale })); + + console.info(`โœ… TC-04/TC-08: Auto-selected master locale ${stack?.master_locale} for first source ${firstSourceLocale.value}`); + + // Reset stack changed flag + if (isStackChanged) { + setisStackChanged(false); + } + return; // Exit early + } + + // ๐Ÿ”ง TC-03: Enhanced mapping for remaining locales + // If we have some matches but not all, try to map remaining locales to remaining destinations + const unmappedSources = sourceLocale.filter(source => !autoMapping[source.value]); + const unmappedDestinations = allLocales.filter(dest => + !Object.keys(autoMapping).includes(dest.value) && + dest.value !== stack?.master_locale + ); + + if (unmappedSources.length > 0 && unmappedDestinations.length > 0) { + console.info('๐Ÿ”ง TC-03: Mapping remaining locales to available destinations'); + + unmappedSources.forEach((source, index) => { + if (unmappedDestinations[index]) { + autoMapping[unmappedDestinations[index].value] = source.value; + console.info(`โœ… TC-03: Mapped remaining ${source.value} -> ${unmappedDestinations[index].value}`); + } + }); + } + + console.info('๐ŸŽฏ Final Auto-mapping Result:', autoMapping); + console.info(`โœ… Found ${Object.keys(autoMapping).length - 1} locale matches out of ${sourceLocale.length} source locales`); // Update Redux state with auto-mappings const newMigrationDataObj: INewMigration = { @@ -794,7 +994,7 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { }; dispatch(updateNewMigrationData(newMigrationDataObj)); - // console.info('โœ… Redux state updated with auto-mapping'); + console.info('โœ… Redux state updated with auto-mapping'); // ๐Ÿ”ฅ CRITICAL FIX: Update existingLocale state for dropdown display // The dropdown reads from existingLocale, not from Redux localeMapping @@ -926,15 +1126,81 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { // const fetchLocales = async () => { // return await getStackLocales(newMigrationData?.destination_stack?.selectedOrg?.value); // }; + + // ๐Ÿ†• CONDITION 3: Intelligent Add Language functionality with auto-suggestions const addRowComp = () => { setisStackChanged(false); + + // ๐Ÿ†• STEP 1: Get next unmapped source locale + const nextUnmappedSource = getNextUnmappedSourceLocale(); + + if (!nextUnmappedSource) { + console.info('โš ๏ธ No more unmapped source locales available'); + // Fallback to original behavior if no unmapped locales + setcmsLocaleOptions((prevList: { label: string; value: string }[]) => [ + ...prevList, + { + label: `${prevList.length}`, + value: '' + } + ]); + return; + } + + console.info(`๐Ÿ†• Adding language row for next unmapped source locale: ${nextUnmappedSource.value}`); + + // ๐Ÿ†• STEP 2: Check if source locale exists in destination + const destinationMatch = findDestinationMatch(nextUnmappedSource.value); + + // ๐Ÿ†• STEP 3 & 4: Auto-map if exists, leave empty if not + const newRowValue = destinationMatch ? destinationMatch.value : ''; + + console.info(` Destination match for ${nextUnmappedSource.value}:`, destinationMatch?.value || 'No match - leaving empty'); + + // Add new row with intelligent defaults setcmsLocaleOptions((prevList: { label: string; value: string }[]) => [ - ...prevList, // Keep existing elements + ...prevList, { - label: `${prevList.length}`, // Generate new label - value: '' + label: `${prevList.length}`, + value: newRowValue // Auto-map or leave empty based on match } ]); + + // ๐Ÿ†• STEP 5: Auto-select the source locale in the dropdown + // Update the mapping state to pre-select the source locale + setTimeout(() => { + const newRowIndex = cmsLocaleOptions.length; // This will be the index of the new row + + // Update existingLocale state to pre-select the source locale + setMapperLocaleState(prev => ({ + ...prev, + [`${newRowIndex}`]: { + label: nextUnmappedSource.value, + value: nextUnmappedSource.value + } + })); + + // If there's a destination match, also update the mapping + if (destinationMatch) { + const updatedMapping = { + ...newMigrationData?.destination_stack?.localeMapping, + [destinationMatch.value]: nextUnmappedSource.value + }; + + const newMigrationDataObj: INewMigration = { + ...newMigrationData, + destination_stack: { + ...newMigrationData?.destination_stack, + localeMapping: updatedMapping + } + }; + dispatch(updateNewMigrationData(newMigrationDataObj)); + + console.info(`โœ… Auto-mapped: ${nextUnmappedSource.value} -> ${destinationMatch.value}`); + } else { + console.info(`๐Ÿ“ Left empty for manual selection: ${nextUnmappedSource.value}`); + } + }, 100); // Small delay to ensure state updates properly }; const handleDeleteLocale = (id: number, locale: { label: string; value: string }) => { @@ -985,15 +1251,63 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { onClick={addRowComp} size="small" disabled={ - Object.keys(newMigrationData?.destination_stack?.localeMapping || {})?.length === - newMigrationData?.destination_stack?.sourceLocale?.length || - cmsLocaleOptions?.length === - newMigrationData?.destination_stack?.sourceLocale?.length || - newMigrationData?.project_current_step > 2 + // ๐Ÿ†• Enhanced disable logic: Check if all source locales are mapped or shown + (() => { + const totalSourceLocales = newMigrationData?.destination_stack?.sourceLocale?.length || 0; + const mappedLocalesCount = Object.keys(newMigrationData?.destination_stack?.localeMapping || {})?.length; + const visibleRowsCount = cmsLocaleOptions?.length || 0; + const isProjectCompleted = newMigrationData?.project_current_step > 2; + + // Disable if: all source locales are mapped OR all source locales have rows OR project is completed + const shouldDisable = mappedLocalesCount >= totalSourceLocales + 1 || // +1 for master locale + visibleRowsCount >= totalSourceLocales + 1 || // +1 for master locale + isProjectCompleted; + + console.info('๐Ÿ”ด Add Language Button Status:', { + totalSourceLocales, + mappedLocalesCount, + visibleRowsCount, + isProjectCompleted, + shouldDisable + }); + + return shouldDisable; + })() } > Add Language + + {/* ๐Ÿ†• ADDITIONAL SCENARIO: Display unmapped destination locales */} + {(() => { + const unmappedDestinations = getUnmappedDestinationLocales(); + + if (unmappedDestinations.length > 0 && newMigrationData?.project_current_step <= 2) { + return ( +
+ } + content={ +
+ Available destination locales not yet mapped: +
+ + {unmappedDestinations.map(locale => locale.value).join(', ')} + +
+ + You can manually connect these by clicking “Add Language” if needed. + +
+ } + type="light" + /> +
+ ); + } + return null; + })()} )}
diff --git a/upload-api/drupalMigrationData/drupalSchema/featured_content.json b/upload-api/drupalMigrationData/drupalSchema/featured_content.json deleted file mode 100644 index b2a795dee..000000000 --- a/upload-api/drupalMigrationData/drupalSchema/featured_content.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "title": "featured content", - "uid": "featured_content", - "schema": [ - { - "uid": "title", - "otherCmsField": "title", - "otherCmsType": "text", - "contentstackField": "title", - "contentstackFieldUid": "title", - "contentstackFieldType": "text", - "backupFieldType": "text", - "backupFieldUid": "title", - "advanced": { - "mandatory": true - } - }, - { - "uid": "url", - "otherCmsField": "url", - "otherCmsType": "text", - "contentstackField": "Url", - "contentstackFieldUid": "url", - "contentstackFieldType": "url", - "backupFieldType": "url", - "backupFieldUid": "url", - "advanced": { - "mandatory": true - } - }, - { - "uid": "body", - "otherCmsField": "Body", - "otherCmsType": "text_with_summary", - "contentstackField": "Body", - "contentstackFieldUid": "body", - "contentstackFieldType": "json", - "backupFieldType": "html", - "backupFieldUid": "body", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_featured_content", - "otherCmsField": "Featured Content", - "otherCmsType": "entity_reference_revisions", - "contentstackField": "Featured Content", - "contentstackFieldUid": "field_featured_content", - "contentstackFieldType": "single_line_text", - "backupFieldType": "single_line_text", - "backupFieldUid": "field_featured_content", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - } - ], - "description": "Schema for featured content", - "options": { - "is_page": true, - "singleton": false, - "sub_title": [], - "title": "title", - "url_pattern": "/:title", - "url_prefix": "/featuredcontent/" - } -} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/drupalSchema/file_tree.json b/upload-api/drupalMigrationData/drupalSchema/file_tree.json deleted file mode 100644 index bffd06702..000000000 --- a/upload-api/drupalMigrationData/drupalSchema/file_tree.json +++ /dev/null @@ -1,211 +0,0 @@ -{ - "title": "file tree", - "uid": "file_tree", - "schema": [ - { - "uid": "title", - "otherCmsField": "title", - "otherCmsType": "text", - "contentstackField": "title", - "contentstackFieldUid": "title", - "contentstackFieldType": "text", - "backupFieldType": "text", - "backupFieldUid": "title", - "advanced": { - "mandatory": true - } - }, - { - "uid": "url", - "otherCmsField": "url", - "otherCmsType": "text", - "contentstackField": "Url", - "contentstackFieldUid": "url", - "contentstackFieldType": "url", - "backupFieldType": "url", - "backupFieldUid": "url", - "advanced": { - "mandatory": true - } - }, - { - "uid": "field_content", - "otherCmsField": "Content", - "otherCmsType": "entity_reference_revisions", - "contentstackField": "Content", - "contentstackFieldUid": "field_content", - "contentstackFieldType": "single_line_text", - "backupFieldType": "single_line_text", - "backupFieldUid": "field_content", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_file", - "otherCmsField": "File", - "otherCmsType": "file", - "contentstackField": "File", - "contentstackFieldUid": "field_file_target_id", - "contentstackFieldType": "file", - "backupFieldType": "file", - "backupFieldUid": "field_file_target_id", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_formatted_title", - "otherCmsField": "Formatted Title", - "otherCmsType": "text_long", - "contentstackField": "Formatted Title", - "contentstackFieldUid": "field_formatted_title", - "contentstackFieldType": "json", - "backupFieldType": "html", - "backupFieldUid": "field_formatted_title", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_link", - "otherCmsField": "Link", - "otherCmsType": "link", - "contentstackField": "Link", - "contentstackFieldUid": "field_link", - "contentstackFieldType": "link", - "backupFieldType": "link", - "backupFieldUid": "field_link", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "taxonomies", - "otherCmsField": "Taxonomy", - "otherCmsType": "taxonomy", - "contentstackField": "Taxonomy", - "contentstackFieldUid": "taxonomies", - "contentstackFieldType": "taxonomy", - "backupFieldType": "taxonomy", - "backupFieldUid": "taxonomies", - "advanced": { - "taxonomies": [ - { - "taxonomy_uid": "centers", - "mandatory": false, - "multiple": true, - "non_localizable": false - }, - { - "taxonomy_uid": "eventtypes", - "mandatory": false, - "multiple": true, - "non_localizable": false - }, - { - "taxonomy_uid": "initiativetype", - "mandatory": false, - "multiple": true, - "non_localizable": false - }, - { - "taxonomy_uid": "issues", - "mandatory": false, - "multiple": true, - "non_localizable": false - }, - { - "taxonomy_uid": "metroarea", - "mandatory": false, - "multiple": true, - "non_localizable": false - }, - { - "taxonomy_uid": "people", - "mandatory": false, - "multiple": true, - "non_localizable": false - }, - { - "taxonomy_uid": "researchformat", - "mandatory": false, - "multiple": true, - "non_localizable": false - }, - { - "taxonomy_uid": "ricenewsgroup", - "mandatory": false, - "multiple": true, - "non_localizable": false - }, - { - "taxonomy_uid": "sponsors", - "mandatory": false, - "multiple": true, - "non_localizable": false - }, - { - "taxonomy_uid": "states", - "mandatory": false, - "multiple": true, - "non_localizable": false - }, - { - "taxonomy_uid": "tags", - "mandatory": false, - "multiple": true, - "non_localizable": false - }, - { - "taxonomy_uid": "urbanedgeposttypes", - "mandatory": false, - "multiple": true, - "non_localizable": false - } - ], - "mandatory": false, - "multiple": true, - "non_localizable": false, - "unique": false - } - } - ], - "description": "Schema for file tree", - "options": { - "is_page": true, - "singleton": false, - "sub_title": [], - "title": "title", - "url_pattern": "/:title", - "url_prefix": "/filetree/" - } -} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/drupalSchema/in_the_news.json b/upload-api/drupalMigrationData/drupalSchema/in_the_news.json deleted file mode 100644 index b30c52daf..000000000 --- a/upload-api/drupalMigrationData/drupalSchema/in_the_news.json +++ /dev/null @@ -1,123 +0,0 @@ -{ - "title": "in the news", - "uid": "in_the_news", - "schema": [ - { - "uid": "title", - "otherCmsField": "title", - "otherCmsType": "text", - "contentstackField": "title", - "contentstackFieldUid": "title", - "contentstackFieldType": "text", - "backupFieldType": "text", - "backupFieldUid": "title", - "advanced": { - "mandatory": true - } - }, - { - "uid": "url", - "otherCmsField": "url", - "otherCmsType": "text", - "contentstackField": "Url", - "contentstackFieldUid": "url", - "contentstackFieldType": "url", - "backupFieldType": "url", - "backupFieldUid": "url", - "advanced": { - "mandatory": true - } - }, - { - "uid": "field_external_link", - "otherCmsField": "External Link", - "otherCmsType": "string", - "contentstackField": "External Link", - "contentstackFieldUid": "field_external_link", - "contentstackFieldType": "single_line_text", - "backupFieldType": "multi_line_text", - "backupFieldUid": "field_external_link", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [ - "article", - "fact", - "featured_content", - "file_tree", - "institute", - "link", - "list", - "page", - "partners", - "profile" - ], - "description": "" - } - }, - { - "uid": "field_publication_date", - "otherCmsField": "Publication Date", - "otherCmsType": "datetime", - "contentstackField": "Publication Date", - "contentstackFieldUid": "field_publication_date", - "contentstackFieldType": "isodate", - "backupFieldType": "isodate", - "backupFieldUid": "field_publication_date", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_source", - "otherCmsField": "Source", - "otherCmsType": "string", - "contentstackField": "Source", - "contentstackFieldUid": "field_source", - "contentstackFieldType": "single_line_text", - "backupFieldType": "multi_line_text", - "backupFieldUid": "field_source", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [ - "article", - "fact", - "featured_content", - "file_tree", - "institute", - "link", - "list", - "page", - "partners", - "profile" - ], - "description": "" - } - } - ], - "description": "Schema for in the news", - "options": { - "is_page": true, - "singleton": false, - "sub_title": [], - "title": "title", - "url_pattern": "/:title", - "url_prefix": "/inthenews/" - } -} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/drupalSchema/partners.json b/upload-api/drupalMigrationData/drupalSchema/partners.json deleted file mode 100644 index 6a069a625..000000000 --- a/upload-api/drupalMigrationData/drupalSchema/partners.json +++ /dev/null @@ -1,141 +0,0 @@ -{ - "title": "partners", - "uid": "partners", - "schema": [ - { - "uid": "title", - "otherCmsField": "title", - "otherCmsType": "text", - "contentstackField": "title", - "contentstackFieldUid": "title", - "contentstackFieldType": "text", - "backupFieldType": "text", - "backupFieldUid": "title", - "advanced": { - "mandatory": true - } - }, - { - "uid": "url", - "otherCmsField": "url", - "otherCmsType": "text", - "contentstackField": "Url", - "contentstackFieldUid": "url", - "contentstackFieldType": "url", - "backupFieldType": "url", - "backupFieldUid": "url", - "advanced": { - "mandatory": true - } - }, - { - "uid": "field_formatted_title", - "otherCmsField": "Formatted Title", - "otherCmsType": "text_long", - "contentstackField": "Formatted Title", - "contentstackFieldUid": "field_formatted_title", - "contentstackFieldType": "json", - "backupFieldType": "html", - "backupFieldUid": "field_formatted_title", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_image", - "otherCmsField": "Image", - "otherCmsType": "image", - "contentstackField": "Image", - "contentstackFieldUid": "field_image_target_id", - "contentstackFieldType": "file", - "backupFieldType": "file", - "backupFieldUid": "field_image_target_id", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_link", - "otherCmsField": "Link", - "otherCmsType": "link", - "contentstackField": "Link", - "contentstackFieldUid": "field_link", - "contentstackFieldType": "link", - "backupFieldType": "link", - "backupFieldUid": "field_link", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_links", - "otherCmsField": "Links", - "otherCmsType": "link", - "contentstackField": "Links", - "contentstackFieldUid": "field_links", - "contentstackFieldType": "link", - "backupFieldType": "link", - "backupFieldUid": "field_links", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_summary", - "otherCmsField": "Summary", - "otherCmsType": "string_long", - "contentstackField": "Summary", - "contentstackFieldUid": "field_summary", - "contentstackFieldType": "json", - "backupFieldType": "html", - "backupFieldUid": "field_summary", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - } - ], - "description": "Schema for partners", - "options": { - "is_page": true, - "singleton": false, - "sub_title": [], - "title": "title", - "url_pattern": "/:title", - "url_prefix": "/partners/" - } -} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/drupalSchema/supporting_material.json b/upload-api/drupalMigrationData/drupalSchema/supporting_material.json deleted file mode 100644 index 586642a9b..000000000 --- a/upload-api/drupalMigrationData/drupalSchema/supporting_material.json +++ /dev/null @@ -1,163 +0,0 @@ -{ - "title": "supporting material", - "uid": "supporting_material", - "schema": [ - { - "uid": "title", - "otherCmsField": "title", - "otherCmsType": "text", - "contentstackField": "title", - "contentstackFieldUid": "title", - "contentstackFieldType": "text", - "backupFieldType": "text", - "backupFieldUid": "title", - "advanced": { - "mandatory": true - } - }, - { - "uid": "url", - "otherCmsField": "url", - "otherCmsType": "text", - "contentstackField": "Url", - "contentstackFieldUid": "url", - "contentstackFieldType": "url", - "backupFieldType": "url", - "backupFieldUid": "url", - "advanced": { - "mandatory": true - } - }, - { - "uid": "field_document_source", - "otherCmsField": "Document Source", - "otherCmsType": "link", - "contentstackField": "Document Source", - "contentstackFieldUid": "field_document_source", - "contentstackFieldType": "link", - "backupFieldType": "link", - "backupFieldUid": "field_document_source", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_document_upload", - "otherCmsField": "Document Upload", - "otherCmsType": "entity_reference", - "contentstackField": "Document Upload", - "contentstackFieldUid": "field_document_upload_target_id", - "contentstackFieldType": "reference", - "backupFieldType": "reference", - "backupFieldUid": "field_document_upload_target_id", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [ - "article", - "fact", - "featured_content", - "file_tree", - "institute", - "in_the_news", - "link", - "list", - "page", - "partners" - ], - "description": "" - } - }, - { - "uid": "field_media_type", - "otherCmsField": "Media Type", - "otherCmsType": "list_string", - "contentstackField": "Media Type", - "contentstackFieldUid": "field_media_type", - "contentstackFieldType": "single_line_text", - "backupFieldType": "multi_line_text", - "backupFieldUid": "field_media_type", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [ - "article", - "fact", - "featured_content", - "file_tree", - "institute", - "in_the_news", - "link", - "list", - "page", - "partners" - ], - "description": "" - } - }, - { - "uid": "field_press_kit", - "otherCmsField": "Press Kit", - "otherCmsType": "boolean", - "contentstackField": "Press Kit", - "contentstackFieldUid": "field_press_kit", - "contentstackFieldType": "boolean", - "backupFieldType": "boolean", - "backupFieldUid": "field_press_kit", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_publication_date", - "otherCmsField": "Publication Date", - "otherCmsType": "datetime", - "contentstackField": "Publication Date", - "contentstackFieldUid": "field_publication_date", - "contentstackFieldType": "isodate", - "backupFieldType": "isodate", - "backupFieldUid": "field_publication_date", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - } - ], - "description": "Schema for supporting material", - "options": { - "is_page": true, - "singleton": false, - "sub_title": [], - "title": "title", - "url_pattern": "/:title", - "url_prefix": "/supportingmaterial/" - } -} \ No newline at end of file diff --git a/upload-api/migration-drupal/libs/contentTypeMapper.js b/upload-api/migration-drupal/libs/contentTypeMapper.js index 72b09cd74..cb9836f5d 100644 --- a/upload-api/migration-drupal/libs/contentTypeMapper.js +++ b/upload-api/migration-drupal/libs/contentTypeMapper.js @@ -19,20 +19,16 @@ const loadTaxonomySchema = async (dbConfig = null) => { if (fs.existsSync(taxonomySchemaPath)) { const taxonomyData = fs.readFileSync(taxonomySchemaPath, 'utf8'); const taxonomies = JSON.parse(taxonomyData); - console.log(`โœ… Loaded ${taxonomies.length} taxonomies from schema file`); return taxonomies; } // If not found and dbConfig available, try to generate it if (dbConfig) { - console.log('โš ๏ธ Taxonomy schema not found, attempting to generate...'); const extractTaxonomy = require('./extractTaxonomy'); const taxonomies = await extractTaxonomy(dbConfig); - console.log(`โœ… Generated ${taxonomies.length} taxonomies`); return taxonomies; } - console.warn('โš ๏ธ Taxonomy schema not found and no dbConfig provided for generation'); return []; } catch (error) { @@ -258,7 +254,6 @@ const createTaxonomyFieldObject = (item, taxonomySchema, targetVocabularies = [] const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => { // Load taxonomy schema for taxonomy field processing const taxonomySchema = await loadTaxonomySchema(dbConfig); - console.log(`๐Ÿท๏ธ Loaded ${taxonomySchema.length} taxonomy vocabularies for field mapping`); // ๐Ÿท๏ธ Collect taxonomy fields for consolidation const collectedTaxonomies = []; @@ -273,7 +268,6 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => // Rich text with switching options: JSON RTE โ†’ HTML RTE โ†’ multiline โ†’ text const availableContentTypes = contentTypes?.filter(ct => ct !== item.content_types) || []; const referenceFields = availableContentTypes.slice(0, 10); - console.log(`๐Ÿ”„ Rich text field ${item.field_name} using backup content types:`, referenceFields); acc.push(createFieldObject(item, 'json', 'html', referenceFields)); break; } @@ -281,7 +275,6 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => // Single line with switching options: single_line โ†’ multiline โ†’ HTML RTE โ†’ JSON RTE const availableContentTypes = contentTypes?.filter(ct => ct !== item.content_types) || []; const referenceFields = availableContentTypes.slice(0, 10); - console.log(`๐Ÿ”„ Text field ${item.field_name} using backup content types:`, referenceFields); acc.push(createFieldObject(item, 'single_line_text', 'multi_line_text', referenceFields)); break; } @@ -290,7 +283,6 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => // Rich text with switching options: JSON RTE โ†’ HTML RTE โ†’ multiline โ†’ text const availableContentTypes = contentTypes?.filter(ct => ct !== item.content_types) || []; const referenceFields = availableContentTypes.slice(0, 10); - console.log(`๐Ÿ”„ Rich text field ${item.field_name} using backup content types:`, referenceFields); acc.push(createFieldObject(item, 'json', 'html', referenceFields)); break; } @@ -299,7 +291,6 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => // Single line with switching options: single_line โ†’ multiline โ†’ HTML RTE โ†’ JSON RTE const availableContentTypes = contentTypes?.filter(ct => ct !== item.content_types) || []; const referenceFields = availableContentTypes.slice(0, 10); - console.log(`๐Ÿ”„ Text field ${item.field_name} using backup content types:`, referenceFields); acc.push(createFieldObject(item, 'single_line_text', 'multi_line_text', referenceFields)); break; } @@ -310,7 +301,6 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => case 'taxonomy_term_reference': { // ๐Ÿท๏ธ Collect taxonomy field for consolidation instead of creating individual fields if (taxonomySchema && taxonomySchema.length > 0) { - console.log(`๐Ÿท๏ธ Collecting taxonomy field for consolidation: ${item.field_name}`); // Try to determine specific vocabularies this field references let targetVocabularies = []; @@ -318,7 +308,6 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => // Check if field has handler settings that specify target vocabularies if (item.handler_settings && item.handler_settings.target_bundles) { targetVocabularies = Object.keys(item.handler_settings.target_bundles); - console.log(`๐ŸŽฏ Found target vocabularies for ${item.field_name}:`, targetVocabularies); } // Add vocabularies to collection (avoid duplicates) @@ -338,7 +327,6 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => } } else { // Fallback to regular reference field if no taxonomy schema available - console.warn(`โš ๏ธ No taxonomy schema available for field: ${item.field_name}, using reference field`); acc.push(createFieldObject(item, 'reference', 'reference', ['taxonomy'])); } break; @@ -347,7 +335,6 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => // Check if this is a taxonomy field by handler if (item.handler === 'default:taxonomy_term') { // ๐Ÿท๏ธ Collect taxonomy field for consolidation instead of creating individual fields - console.log(`๐Ÿท๏ธ Collecting taxonomy field for consolidation: ${item.field_name}`); // Try to determine specific vocabularies this field references let targetVocabularies = []; @@ -355,7 +342,6 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => // Check if field has handler settings that specify target vocabularies if (item.reference) { targetVocabularies = Object.keys(item.reference); - console.log(`๐ŸŽฏ Found target vocabularies for ${item.field_name}:`, targetVocabularies); } if (taxonomySchema && taxonomySchema.length > 0) { @@ -376,7 +362,6 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => } } else { // Fallback to regular reference field if no taxonomy schema available - console.warn(`โš ๏ธ No taxonomy schema available for field: ${item.field_name}, using reference field`); // Use available content types instead of generic 'taxonomy' const availableContentTypes = contentTypes?.filter(ct => ct !== item.content_types) || []; const referenceFields = availableContentTypes.slice(0, 10); @@ -389,12 +374,10 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => if (item.reference && Object.keys(item.reference).length > 0) { // Use specific content types from field configuration referenceFields = Object.keys(item.reference); - console.log(`๐ŸŽฏ Found specific reference content types for ${item.field_name}:`, referenceFields); } else { // Backup: Use up to 10 content types from available content types const availableContentTypes = contentTypes?.filter(ct => ct !== item.content_types) || []; referenceFields = availableContentTypes.slice(0, 10); - console.log(`๐Ÿ”„ No specific content types found for ${item.field_name}, using backup content types:`, referenceFields); } acc.push(createFieldObject(item, 'reference', 'reference', referenceFields)); @@ -402,7 +385,6 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => // Handle other entity references - exclude taxonomy and limit to 10 content types const availableContentTypes = contentTypes?.filter(ct => ct !== item.content_types) || []; const referenceFields = availableContentTypes.slice(0, 10); - console.log(`๐Ÿ“‹ Using available content types for ${item.field_name}:`, referenceFields); acc.push(createFieldObject(item, 'reference', 'reference', referenceFields)); } break; @@ -485,7 +467,6 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => // ๐Ÿท๏ธ TAXONOMY CONSOLIDATION: Create single consolidated taxonomy field if any taxonomies were collected if (collectedTaxonomies.length > 0) { - console.log(`๐Ÿท๏ธ Creating consolidated taxonomy field with ${collectedTaxonomies.length} taxonomies:`, collectedTaxonomies); // Create consolidated taxonomy field with fixed properties const consolidatedTaxonomyField = { @@ -513,7 +494,6 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => // Add consolidated taxonomy field at the end of schema schemaArray.push(consolidatedTaxonomyField); - console.log(`โœ… Added consolidated taxonomy field 'taxonomies' with ${collectedTaxonomies.length} vocabularies`); } return schemaArray; diff --git a/upload-api/migration-drupal/libs/createInitialMapper.js b/upload-api/migration-drupal/libs/createInitialMapper.js index 9582b9781..bb4ebb05a 100644 --- a/upload-api/migration-drupal/libs/createInitialMapper.js +++ b/upload-api/migration-drupal/libs/createInitialMapper.js @@ -38,8 +38,6 @@ const createInitialMapper = async (systemConfig, prefix) => { if (!fs.existsSync(drupalFolderPath)) { await fsp.mkdir(drupalFolderPath, { recursive: true }); } - - console.log('Extracting content types from Drupal database...'); // Get database connection const connection = dbConnection(systemConfig); @@ -76,11 +74,9 @@ const createInitialMapper = async (systemConfig, prefix) => { } if (details_data.length === 0) { - console.log('No content types found to process'); return { contentTypes: [] }; } - console.log(`Processing ${details_data.length} content type entries...`); const initialMapper = []; const contentTypes = Object.keys(require('lodash').keyBy(details_data, 'content_types')); @@ -130,13 +126,8 @@ const createInitialMapper = async (systemConfig, prefix) => { ); } - console.log(`Successfully processed ${initialMapper.length} content types`); - console.log('Content type extraction completed successfully'); - // Close database connection connection.end(); - console.log('Database connection closed'); - return { contentTypes: initialMapper }; } catch (error) { console.error('Error in content type extraction:', error); diff --git a/upload-api/migration-drupal/libs/extractLocale.js b/upload-api/migration-drupal/libs/extractLocale.js index fea1e77f4..0b8dd68ca 100644 --- a/upload-api/migration-drupal/libs/extractLocale.js +++ b/upload-api/migration-drupal/libs/extractLocale.js @@ -3,94 +3,67 @@ /** * External module dependencies. */ -const { dbConnection} = require('../utils/helper'); +const { dbConnection } = require('../utils/helper'); /** * Apply locale transformation rules (same logic as API side) * - "und" alone โ†’ "en-us" - * - "und" + "en-us" โ†’ both become "en" - * - "en" + "en-us" โ†’ "en" becomes "und", "en-us" stays - * - All three โ†’ "en"+"und" become "und", "en-us" stays + * - "und" + "en-us" โ†’ "und" become "en", "en-us" stays + * - "en" + "und" โ†’ "und" becomes "en-us", "en" stays + * - All three "en" + "und" + "en-us" โ†’ all three stays + * - Apart from these, all other locales stay as is */ function applyLocaleTransformations(originalLocales) { - const locales = [...originalLocales]; // Copy to avoid mutation - const hasUnd = locales.includes('und'); - const hasEn = locales.includes('en'); - const hasEnUs = locales.includes('en-us'); - - console.log(`๐Ÿ” Locale Analysis: hasUnd=${hasUnd}, hasEn=${hasEn}, hasEnUs=${hasEnUs}`); - - const transformedSet = new Set(); - - // Apply transformation rules - locales.forEach(locale => { - if (locale === 'und') { - if (hasEn && hasEnUs) { - // If all three present, "und" stays as "und" - transformedSet.add('und'); - console.log(`๐Ÿ”„ "und" stays as "und" (all three present)`); - } else if (hasEnUs) { - // If "und" + "en-us", "und" becomes "en" - transformedSet.add('en'); - console.log(`๐Ÿ”„ Transforming "und" โ†’ "en" (en-us exists)`); - } else { - // If only "und", becomes "en-us" - transformedSet.add('en-us'); - console.log(`๐Ÿ”„ Transforming "und" โ†’ "en-us"`); - } - } else if (locale === 'en-us') { - if (hasUnd && !hasEn) { - // If "und" + "en-us" (no en), "und" becomes "en", so keep "en-us" - transformedSet.add('en-us'); - console.log(`๐Ÿ”„ "en-us" stays as "en-us" (und becomes en)`); - } else { - // Keep en-us as is in other cases - transformedSet.add('en-us'); - } - } else if (locale === 'en') { - if (hasEnUs && !hasUnd) { - // If "en" + "en-us" (no und), "en" becomes "und" - transformedSet.add('und'); - console.log(`๐Ÿ”„ Transforming "en" โ†’ "und" (en-us exists, no und)`); - } else { - // Keep "en" as is in other cases - transformedSet.add('en'); - } - } else { - // Keep other locales as is - transformedSet.add(locale); - } - }); - - return Array.from(transformedSet).sort(); + const locales = [...originalLocales]; // Copy to avoid mutation + const hasUnd = locales.includes('und'); + const hasEn = locales.includes('en'); + const hasEnUs = locales.includes('en-us'); + + // Start with all non-special locales (not und, en, en-us) + const result = locales.filter((locale) => !['und', 'en', 'en-us'].includes(locale)); + + // Apply transformation rules based on combinations + if (hasEn && hasUnd && hasEnUs) { + // Rule 4: All three "en" + "und" + "en-us" โ†’ all three stays + result.push('en', 'und', 'en-us'); + } else if (hasUnd && hasEnUs && !hasEn) { + // Rule 2: "und" + "en-us" โ†’ "und" become "en", "en-us" stays + result.push('en', 'en-us'); + } else if (hasEn && hasUnd && !hasEnUs) { + // Rule 3: "en" + "und" โ†’ "und" becomes "en-us", "en" stays + result.push('en', 'en-us'); + } else if (hasUnd && !hasEn && !hasEnUs) { + // Rule 1: "und" alone โ†’ "en-us" + result.push('en-us'); + } else { + // For any other combinations, keep locales as they are + if (hasEn) result.push('en'); + if (hasUnd) result.push('und'); + if (hasEnUs) result.push('en-us'); + } + + return Array.from(new Set(result)).sort(); } -const extractLocale = async ( systemConfig ) => -{ - let connection; - try - { +const extractLocale = async (systemConfig) => { + let connection; + try { // Get database connection - pass your MySQL config - connection = await dbConnection( systemConfig ); + connection = await dbConnection(systemConfig); + + // Simple query to get all unique language codes from content + const localeQuery = + "SELECT DISTINCT langcode FROM node_field_data WHERE langcode IS NOT NULL AND langcode != '' ORDER BY langcode"; + + const [localeRows] = await connection.promise().query(localeQuery); + const originalLocales = localeRows + .map((row) => row.langcode) + .filter((locale) => locale && locale.trim()); + + // Apply locale transformation rules for UI consistency + const transformedLocales = applyLocaleTransformations(originalLocales); - // DYNAMIC locale extraction - query directly for unique language codes - console.log('๐ŸŒ Extracting locales dynamically from Drupal database...'); - - // Simple query to get all unique language codes from content - const localeQuery = "SELECT DISTINCT langcode FROM node_field_data WHERE langcode IS NOT NULL AND langcode != '' ORDER BY langcode"; - - const [localeRows] = await connection.promise().query(localeQuery); - const originalLocales = localeRows.map(row => row.langcode).filter(locale => locale && locale.trim()); - - console.log(`๐Ÿ“ Found ${originalLocales.length} original locales:`, originalLocales); - - // ๐Ÿ”„ Apply locale transformation rules for UI consistency - const transformedLocales = applyLocaleTransformations(originalLocales); - - console.log(`โœ… Transformed to ${transformedLocales.length} locales for UI:`, transformedLocales); - - return transformedLocales; - + return transformedLocales; } catch (error) { console.error(`Error reading JSON file:`, error); return []; diff --git a/upload-api/migration-drupal/libs/extractTaxonomy.js b/upload-api/migration-drupal/libs/extractTaxonomy.js index 750174728..eca3c72ea 100644 --- a/upload-api/migration-drupal/libs/extractTaxonomy.js +++ b/upload-api/migration-drupal/libs/extractTaxonomy.js @@ -25,29 +25,26 @@ const generateSlug = (name) => { return name .toLowerCase() .replace(/[^a-z0-9\s-]/g, '') // Remove special characters - .replace(/\s+/g, '_') // Replace spaces with underscores - .replace(/-+/g, '_') // Replace hyphens with underscores - .replace(/_+/g, '_') // Replace multiple underscores with single - .replace(/^_|_$/g, ''); // Remove leading/trailing underscores + .replace(/\s+/g, '_') // Replace spaces with underscores + .replace(/-+/g, '_') // Replace hyphens with underscores + .replace(/_+/g, '_') // Replace multiple underscores with single + .replace(/^_|_$/g, ''); // Remove leading/trailing underscores }; /** * Extract taxonomy vocabularies (parent-level only) from Drupal database * and save them to drupalMigrationData/taxonomySchema/taxonomySchema.json - * + * * @param {Object} dbConfig - Database configuration * @returns {Promise} Array of vocabulary objects with uid and name */ const extractTaxonomy = async (dbConfig) => { - console.log('๐Ÿ” === EXTRACTING TAXONOMY VOCABULARIES ==='); - console.log('๐Ÿ“‹ Database Config:', JSON.stringify(dbConfig, null, 2)); - let connection; - + try { // Create database connection connection = mysql.createConnection(dbConfig); - + // Test connection await new Promise((resolve, reject) => { connection.connect((err) => { @@ -55,9 +52,7 @@ const extractTaxonomy = async (dbConfig) => { else resolve(); }); }); - - console.log('โœ… Database connection successful'); - + // Extract vocabularies from Drupal config table (Drupal 8+) // This gets only the parent vocabularies, not individual terms const vocabularyQuery = ` @@ -68,52 +63,42 @@ const extractTaxonomy = async (dbConfig) => { WHERE name LIKE 'taxonomy.vocabulary.%' AND data IS NOT NULL `; - - console.log('๐Ÿ” Executing vocabulary query...'); + const vocabularies = await executeQuery(connection, vocabularyQuery); - - console.log(`๐Ÿ“‹ Found ${vocabularies.length} vocabularies`); - + // Transform vocabularies to required format const taxonomySchema = []; - + for (const vocab of vocabularies) { try { if (vocab.vid && vocab.data) { // Unserialize the PHP data to get vocabulary details const vocabularyData = unserialize(vocab.data); - + if (vocabularyData && vocabularyData.name) { const uid = generateSlug(vocab.vid); // Use vid as base for uid const name = vocabularyData.name; - + taxonomySchema.push({ uid: uid, name: name }); - - console.log(`โœ… Added vocabulary: ${uid} (${name})`); } } } catch (parseError) { console.warn(`โš ๏ธ Failed to parse vocabulary data for ${vocab.vid}:`, parseError.message); } } - + // Create output directory const outputDir = path.join(__dirname, '..', '..', 'drupalMigrationData', 'taxonomySchema'); await fs.promises.mkdir(outputDir, { recursive: true }); - + // Save taxonomy schema to file const outputPath = path.join(outputDir, 'taxonomySchema.json'); await fs.promises.writeFile(outputPath, JSON.stringify(taxonomySchema, null, 2)); - - console.log(`โœ… Taxonomy schema saved to: ${outputPath}`); - console.log(`๐Ÿ“Š Total vocabularies extracted: ${taxonomySchema.length}`); - console.log('=========================================='); - + return taxonomySchema; - } catch (error) { console.error('โŒ Error extracting taxonomy:', error.message); throw error; diff --git a/upload-api/src/config/index.ts b/upload-api/src/config/index.ts index dc2d5cc35..9f4c210da 100644 --- a/upload-api/src/config/index.ts +++ b/upload-api/src/config/index.ts @@ -21,8 +21,8 @@ export default { port: '3306' }, drupalAssetsUrl: { - base_url: "", - public_path: "", + base_url: '', + public_path: '' }, - localPath: process.env.CONTAINER_PATH || 'sql', -}; \ No newline at end of file + localPath: process.env.CONTAINER_PATH || 'sql' +}; diff --git a/upload-api/src/routes/index.ts b/upload-api/src/routes/index.ts index 8d53af57f..84681cf70 100644 --- a/upload-api/src/routes/index.ts +++ b/upload-api/src/routes/index.ts @@ -89,69 +89,166 @@ router.post('/upload', upload.single('file'), async function (req: Request, res: }); // deepcode ignore NoRateLimitingForExpensiveWebOperation: -router.get('/validator', express.json(), fileOperationLimiter, async function (req: Request, res: Response) { - try { - const projectId: string | string[] = req?.headers?.projectid ?? ""; - const app_token: string | string[] = req?.headers?.app_token ?? ""; - const affix: string | string[] = req?.headers?.affix ?? "csm"; - const cmsType = config?.cmsType?.toLowerCase(); +router.get( + '/validator', + express.json(), + fileOperationLimiter, + async function (req: Request, res: Response) { + try { + const projectId: string | string[] = req?.headers?.projectid ?? ''; + const app_token: string | string[] = req?.headers?.app_token ?? ''; + const affix: string | string[] = req?.headers?.affix ?? 'csm'; + const cmsType = config?.cmsType?.toLowerCase(); - if (config?.isLocalPath) { - const fileName = path.basename(config?.localPath || ""); - //const fileName = config?.localPath?.replace(/\/$/, "")?.split?.('/')?.pop?.(); + if (config?.isLocalPath) { + const fileName = path.basename(config?.localPath || ''); + //const fileName = config?.localPath?.replace(/\/$/, "")?.split?.('/')?.pop?.(); - if (!fileName) { - res.send('Filename could not be determined from the local path.'); - } + if (!fileName) { + res.send('Filename could not be determined from the local path.'); + } - if (fileName) { - const name = fileName?.split?.('.')?.[0]; - const fileExt = fileName?.split('.')?.pop() ?? ''; - const bodyStream = createReadStream(config?.localPath?.replace(/\/$/, "")); - - bodyStream.on('error', (error: any) => { - console.error(error); - return res.status(500).json({ - status: "error", - message: "Error reading file.", - file_details: config - }); - }); - if (fileExt === 'xml') { - let xmlData = ''; + if (fileName) { + const name = fileName?.split?.('.')?.[0]; + const fileExt = fileName?.split('.')?.pop() ?? ''; + const bodyStream = createReadStream(config?.localPath?.replace(/\/$/, '')); - // Collect the data from the stream as a string - bodyStream.on('data', (chunk) => { - if (typeof chunk !== 'string' && !Buffer.isBuffer(chunk)) { - throw new Error('Expected chunk to be a string or a Buffer'); - } else { - // Convert chunk to string (if it's a Buffer) - xmlData += chunk.toString(); - } + bodyStream.on('error', (error: any) => { + console.error(error); + return res.status(500).json({ + status: 'error', + message: 'Error reading file.', + file_details: config + }); }); + if (fileExt === 'xml') { + let xmlData = ''; - // When the stream ends, process the XML data - bodyStream.on('end', async () => { - if (!xmlData) { - throw new Error('No data collected from the stream.'); - } + // Collect the data from the stream as a string + bodyStream.on('data', (chunk) => { + if (typeof chunk !== 'string' && !Buffer.isBuffer(chunk)) { + throw new Error('Expected chunk to be a string or a Buffer'); + } else { + // Convert chunk to string (if it's a Buffer) + xmlData += chunk.toString(); + } + }); - const data = await handleFileProcessing(fileExt, xmlData, cmsType, name); - res.status(data?.status || 200).json(data); - if (data?.status === 200) { - const filePath = path.join(__dirname, '..', '..', 'extracted_files', `${name}.json`); - createMapper(filePath, projectId, app_token, affix, config); - } - }); + // When the stream ends, process the XML data + bodyStream.on('end', async () => { + if (!xmlData) { + throw new Error('No data collected from the stream.'); + } + + const data = await handleFileProcessing(fileExt, xmlData, cmsType, name); + res.status(data?.status || 200).json(data); + if (data?.status === 200) { + const filePath = path.join( + __dirname, + '..', + '..', + 'extracted_files', + `${name}.json` + ); + createMapper(filePath, projectId, app_token, affix, config); + } + }); + } else { + // Create a writable stream to save the downloaded zip file + let zipBuffer = Buffer.alloc(0); + + // Collect the data from the stream into a buffer + bodyStream.on('data', (chunk) => { + if (!Buffer.isBuffer(chunk)) { + throw new Error('Expected chunk to be a Buffer'); + } else { + zipBuffer = Buffer.concat([zipBuffer, chunk]); + } + }); + + //buffer fully stremd + bodyStream.on('end', async () => { + if (!zipBuffer) { + throw new Error('No data collected from the stream.'); + } + const data = await handleFileProcessing(fileExt, zipBuffer, cmsType, name); + res.status(data?.status || 200).json(data); + if (data?.status === 200) { + let filePath = path.join(__dirname, '..', '..', 'extracted_files', name); + if (data?.file !== undefined) { + filePath = path.join(__dirname, '..', '..', 'extracted_files', name, data?.file); + } + createMapper(filePath, projectId, app_token, affix, config); + } + }); + } } - else { + } else { + if (config?.isSQL) { + const fileExt = 'sql'; + const name = 'sql'; + + // For SQL files, we don't need to read from S3, just validate the database connection + const result = await handleFileProcessing(fileExt, null, cmsType, name); + if (!result) { + console.error('File processing returned no result'); + return res.status(500).json({ + status: 500, + message: 'File processing failed to return a result', + file_details: config + }); + } + + // Only create mapper if validation was successful (status 200) + if (result.status === 200) { + const filePath = ''; + createMapper(filePath, projectId, app_token, affix, config); + } + + // Ensure we're sending back the complete file_details + const response = { + ...result, + file_details: { + ...result.file_details, + isSQL: config.isSQL, + mySQLDetails: config.mysql, // Changed from mysql to mySQLDetails + drupalAssetsUrl: config.drupalAssetsUrl + } + }; + + return res.status(result.status).json(response); + } else { + const params = { + Bucket: config?.awsData?.bucketName, + Key: config?.awsData?.bucketKey + }; + const getObjectCommand = new GetObjectCommand(params); + // Get the object from S3 + const s3File = await client.send(getObjectCommand); + //file Name From key + const fileName = params?.Key?.split?.('/')?.pop?.() ?? ''; + //file ext from fileName + const fileExt = fileName?.split?.('.')?.pop?.() ?? 'test'; + + if (!s3File?.Body) { + throw new Error('Empty response body from S3'); + } + + const bodyStream: Readable = s3File?.Body as Readable; + // Create a writable stream to save the downloaded zip file - let zipBuffer = Buffer.alloc(0); + const zipFileStream = createWriteStream(`${fileName}`); + + // // Pipe the S3 object's body to the writable stream + bodyStream.pipe(zipFileStream); + + // Create a writable stream to save the downloaded zip file + let zipBuffer: Buffer | null = null; // Collect the data from the stream into a buffer bodyStream.on('data', (chunk) => { - if (!Buffer.isBuffer(chunk)) { - throw new Error('Expected chunk to be a Buffer'); + if (zipBuffer === null) { + zipBuffer = chunk; } else { zipBuffer = Buffer.concat([zipBuffer, chunk]); } @@ -162,119 +259,31 @@ router.get('/validator', express.json(), fileOperationLimiter, async function (r if (!zipBuffer) { throw new Error('No data collected from the stream.'); } - const data = await handleFileProcessing(fileExt, zipBuffer, cmsType, name); - res.status(data?.status || 200).json(data); - if (data?.status === 200) { - let filePath = path.join(__dirname, '..', '..', 'extracted_files', name); - if (data?.file !== undefined) { - filePath = path.join(__dirname, '..', '..', 'extracted_files', name, data?.file); - } - createMapper(filePath, projectId, app_token, affix, config); + + const data = await handleFileProcessing(fileExt, zipBuffer, cmsType, fileName); + res.json(data); + res.send('file validated successfully.'); + let filePath = path.join(__dirname, '..', '..', 'extracted_files', fileName); + if (data?.file !== undefined) { + filePath = path.join(__dirname, '..', '..', 'extracted_files', fileName, data?.file); } + createMapper(filePath, projectId, app_token, affix, config); }); } } - } else { - if ( config?.isSQL ) - { - const fileExt = 'sql'; - const name = 'sql'; - - console.log('Processing SQL database connection'); - // For SQL files, we don't need to read from S3, just validate the database connection - const result = await handleFileProcessing(fileExt, null, cmsType, name); - if (!result) { - console.error('File processing returned no result'); - return res.status(500).json({ - status: 500, - message: 'File processing failed to return a result', - file_details: config - }); - } - - const filePath = ''; - createMapper(filePath, projectId, app_token, affix, config); - - // Ensure we're sending back the complete file_details - const response = { - ...result, - file_details: { - ...result.file_details, - isSQL: config.isSQL, - mySQLDetails: config.mysql, // Changed from mysql to mySQLDetails - drupalAssetsUrl: config.drupalAssetsUrl - } - }; - - console.log('Sending SQL validation response:', JSON.stringify(response, null, 2)); - return res.status(result.status).json(response); - } else - { - const params = { - Bucket: config?.awsData?.bucketName, - Key: config?.awsData?.bucketKey - }; - const getObjectCommand = new GetObjectCommand(params); - // Get the object from S3 - const s3File = await client.send(getObjectCommand); - //file Name From key - const fileName = params?.Key?.split?.('/')?.pop?.() ?? ''; - //file ext from fileName - const fileExt = fileName?.split?.('.')?.pop?.() ?? 'test'; - - if (!s3File?.Body) { - throw new Error('Empty response body from S3'); + } catch (err: any) { + console.error('๐Ÿš€ ~ router.get ~ err:', err); + // Only send error response if no response has been sent yet + if (!res.headersSent) { + res.status(500).json({ + status: 500, + message: 'Internal server error', + error: err.message + }); } - - const bodyStream: Readable = s3File?.Body as Readable; - - // Create a writable stream to save the downloaded zip file - const zipFileStream = createWriteStream(`${fileName}`); - - // // Pipe the S3 object's body to the writable stream - bodyStream.pipe(zipFileStream); - - // Create a writable stream to save the downloaded zip file - let zipBuffer: Buffer | null = null; - - // Collect the data from the stream into a buffer - bodyStream.on('data', (chunk) => { - if (zipBuffer === null) { - zipBuffer = chunk; - } else { - zipBuffer = Buffer.concat([zipBuffer, chunk]); - } - }); - - //buffer fully stremd - bodyStream.on('end', async () => { - if (!zipBuffer) { - throw new Error('No data collected from the stream.'); - } - - const data = await handleFileProcessing(fileExt, zipBuffer, cmsType, fileName); - res.json(data); - res.send('file validated successfully.'); - let filePath = path.join(__dirname, '..', '..', 'extracted_files', fileName); - if (data?.file !== undefined) { - filePath = path.join(__dirname, '..', '..', 'extracted_files', fileName, data?.file); - } - createMapper(filePath, projectId, app_token, affix, config); - }); } } - } catch (err: any) { - console.error('๐Ÿš€ ~ router.get ~ err:', err); - // Only send error response if no response has been sent yet - if (!res.headersSent) { - res.status(500).json({ - status: 500, - message: 'Internal server error', - error: err.message - }); - } - } -}); +); router.get('/config', async function (req: Request, res: Response) { res.json(config); diff --git a/upload-api/src/services/drupal/index.ts b/upload-api/src/services/drupal/index.ts index 627e43fcb..a2341d1b1 100644 --- a/upload-api/src/services/drupal/index.ts +++ b/upload-api/src/services/drupal/index.ts @@ -14,18 +14,13 @@ const createDrupalMapper = async ( affix: string | string[] ) => { try { - console.log('hey we are in createDrupalMapper'); - // this is to fetch the locales from the drupal database // const fetchedLocales:[]= await extractLocale(config) const localeData = await extractLocale(config); - console.log('๐Ÿ” DEBUG: Locale data from extractLocale:', localeData); // Extract taxonomy vocabularies and save to drupalMigrationData - console.log('๐Ÿท๏ธ Extracting taxonomy vocabularies...'); - const taxonomyData = await extractTaxonomy(config.mysql); - console.log(`โœ… Extracted ${taxonomyData.length} taxonomy vocabularies`); + await extractTaxonomy(config.mysql); const initialMapper = await createInitialMapper(config, affix); @@ -41,21 +36,15 @@ const createDrupalMapper = async ( }; const { data } = await axios.request(req); - console.log('๐Ÿš€ ~ createDrupalMapper ~ data:', data?.data); if (data?.data?.content_mapper?.length) { - console.log('Inside the if block of createDrupalMapper'); - logger.info('Validation success:', { status: HTTP_CODES?.OK, message: HTTP_TEXTS?.MAPPER_SAVED }); - } else { - console.log('Inside the else block of createDrupalMapper'); } const localeArray = Array.from(localeData); - console.log('๐Ÿ” DEBUG: Sending locales to API:', localeArray); - + const mapperConfig = { method: 'post', maxBodyLength: Infinity, @@ -77,7 +66,6 @@ const createDrupalMapper = async ( }); } } catch (err: any) { - console.error('๐Ÿš€ ~ createDrupalMapper ~ err:', err?.response?.data ?? err); logger.warn('Validation error:', { status: HTTP_CODES?.UNAUTHORIZED, message: HTTP_TEXTS?.VALIDATION_ERROR diff --git a/upload-api/src/services/fileProcessing.ts b/upload-api/src/services/fileProcessing.ts index 3c2d86273..ced293f34 100644 --- a/upload-api/src/services/fileProcessing.ts +++ b/upload-api/src/services/fileProcessing.ts @@ -68,52 +68,48 @@ const handleFileProcessing = async ( }; } } - }else if (fileExt === 'sql') { - console.log('SQL file processing'); + } else if (fileExt === 'sql') { try { - // Get database connection - const dbConnection = await getDbConnection(config.mysql); + // Validate SQL connection using our Drupal validator + const isValidConnection = await validator({ + data: config.mysql, + type: cmsType, + extension: fileExt + }); - if (dbConnection) { - await validator({ data: config, type: cmsType, extension: fileExt }); - logger.info('Database connection success:', { + if (isValidConnection) { + logger.info('Database validation success:', { status: HTTP_CODES?.OK, - message: 'Successfully connected to database' + message: 'File validated successfully' }); const successResponse = { status: HTTP_CODES?.OK, - message: 'Successfully connected to database', + message: 'File validated successfully', file_details: config }; - console.log('=== Sending response (sql success) ==='); - console.log('Response object:', JSON.stringify(successResponse, null, 2)); return successResponse; } else { - logger.warn('Database connection error:', { + logger.warn('Database validation failed:', { status: HTTP_CODES?.UNAUTHORIZED, - message: 'Failed to connect to database' + message: 'Failed to validate database connection or required tables are missing' }); - const authErrorResponse = { + const validationErrorResponse = { status: HTTP_CODES?.UNAUTHORIZED, - message: 'Failed to connect to database', + message: 'Failed to validate database connection or required tables are missing', file_details: config }; - console.log('=== Sending response (sql auth error) ==='); - console.log('Response object:', JSON.stringify(authErrorResponse, null, 2)); - return authErrorResponse; + return validationErrorResponse; } } catch (error) { - logger.error('Database connection error:', error); - console.log('=== Sending response (sql server error) ==='); - const errorResponse = { - status: HTTP_CODES?.SERVER_ERROR, - message: 'Failed to connect to database', - file_details: config - }; - console.log('Response object:', JSON.stringify(errorResponse, null, 2)); - return errorResponse; + logger.error('Database validation error:', error); + const errorResponse = { + status: HTTP_CODES?.SERVER_ERROR, + message: 'Database validation failed with error', + file_details: config + }; + return errorResponse; } - }else { + } else { // if file is not zip // Convert the buffer to a string assuming it's UTF-8 encoded const jsonString = Buffer?.from?.(zipBuffer)?.toString?.('utf8'); diff --git a/upload-api/src/validators/drupal/index.ts b/upload-api/src/validators/drupal/index.ts new file mode 100644 index 000000000..6b23e149c --- /dev/null +++ b/upload-api/src/validators/drupal/index.ts @@ -0,0 +1,142 @@ +import mysql from 'mysql2/promise'; +import logger from '../../utils/logger'; + +interface ValidatorProps { + data: { + host: string; + user: string; + password: string; + database: string; + port: number | string; + }; +} + +/** + * Validates Drupal SQL connection by testing a specific query + * Tests connection with: "SELECT *, CONVERT(data USING utf8) as data FROM config WHERE name LIKE '%field.field.node%'" + * @param data - Database configuration object containing connection details + * @returns Promise - true if connection successful and query returns results, false otherwise + */ +async function drupalValidator({ data }: ValidatorProps): Promise { + let connection: mysql.Connection | null = null; + + try { + // Debug: Log the received data structure + logger.info('Drupal validator: Received data structure', { + dataKeys: Object.keys(data || {}), + hasHost: !!data?.host, + hasUser: !!data?.user, + hasPassword: !!data?.password, + hasDatabase: !!data?.database, + host: data?.host, + user: data?.user, + database: data?.database, + port: data?.port + }); + + // Validate required connection parameters (password can be empty for local development) + if (!data?.host || !data?.user || !data?.database) { + logger.error('Drupal validator: Missing required database connection parameters', { + missingHost: !data?.host, + missingUser: !data?.user, + missingDatabase: !data?.database + }); + return false; + } + + // Create MySQL connection configuration + const connectionConfig: mysql.ConnectionOptions = { + host: data.host, + user: data.user, + password: data.password, + database: data.database, + port: Number(data.port) || 3306, + connectTimeout: 10000 // 10 seconds timeout + }; + + // Create the database connection + connection = await mysql.createConnection(connectionConfig); + + logger.info('Drupal validator: Database connection established successfully', { + host: data.host, + database: data.database, + port: Number(data.port) || 3306 + }); + + // Test connection and validate required Drupal tables exist + // Check for node_field_data table (this is the table that's missing in the error) + const nodeFieldDataQuery = 'SELECT COUNT(*) as count FROM node_field_data LIMIT 1'; + + try { + const [nodeRows] = await connection.execute(nodeFieldDataQuery); + logger.info('Drupal validator: node_field_data table exists and accessible'); + } catch (nodeError: any) { + logger.error('Drupal validator: node_field_data table check failed', { + error: nodeError.message, + code: nodeError.code, + sqlState: nodeError.sqlState + }); + return false; + } + + // Test with the specific Drupal config query + const configQuery = + "SELECT *, CONVERT(data USING utf8) as data FROM config WHERE name LIKE '%field.field.node%'"; + + try { + const [configRows] = await connection.execute(configQuery); + + // Check if config query returned any results + const hasConfigResults = Array.isArray(configRows) && configRows.length > 0; + + if (hasConfigResults) { + logger.info('Drupal validator: All validation checks passed successfully', { + nodeFieldDataExists: true, + configQueryResults: (configRows as any[]).length + }); + return true; + } else { + logger.warn('Drupal validator: Config query executed but returned no results', { + query: configQuery + }); + return false; + } + } catch (configError: any) { + logger.error('Drupal validator: Config table query failed', { + error: configError.message, + code: configError.code, + sqlState: configError.sqlState, + query: configQuery + }); + return false; + } + } catch (error: any) { + // Log specific error details for debugging + logger.error('Drupal validator: Database connection or query failed', { + error: error.message, + code: error.code, + sqlState: error.sqlState, + stack: error.stack, + host: data?.host, + database: data?.database, + port: data?.port + }); + + // Return false for any connection or query errors + return false; + } finally { + // Always close the connection if it was established + if (connection) { + try { + await connection.end(); + logger.info('Drupal validator: Database connection closed successfully'); + } catch (closeError: any) { + logger.warn('Drupal validator: Error closing database connection', { + error: closeError.message + }); + } + } + } +} + +export default drupalValidator; diff --git a/upload-api/src/validators/index.ts b/upload-api/src/validators/index.ts index b3e2d9c94..711e69a9b 100644 --- a/upload-api/src/validators/index.ts +++ b/upload-api/src/validators/index.ts @@ -2,6 +2,7 @@ import sitecoreValidator from './sitecore'; import contentfulValidator from './contentful'; import wordpressValidator from './wordpress'; import aemValidator from './aem'; +import drupalValidator from './drupal'; const validator = ({ data, type, extension }: { data: any; type: string; extension: string }) => { const CMSIdentifier = `${type}-${extension}`; @@ -22,6 +23,10 @@ const validator = ({ data, type, extension }: { data: any; type: string; extensi return aemValidator({ data }); } + case 'drupal-sql': { + return drupalValidator({ data }); + } + default: return false; } From 3823a2feb24cf1764a5f9fd9855985a782e5534d Mon Sep 17 00:00:00 2001 From: sauravraw Date: Mon, 22 Sep 2025 11:15:39 +0530 Subject: [PATCH 03/37] locale code update --- api/src/services/drupal/entries.service.ts | 80 ++- api/test-duplicate-fix-simple.js | 316 ++++++++++++ api/test-duplicate-fix-with-db.js | 476 ++++++++++++++++++ api/test-duplicate-removal.js | 123 +++++ .../Actions/LoadLanguageMapper.tsx | 289 +++++------ ui/src/pages/Migration/index.tsx | 34 +- 6 files changed, 1132 insertions(+), 186 deletions(-) create mode 100644 api/test-duplicate-fix-simple.js create mode 100644 api/test-duplicate-fix-with-db.js create mode 100644 api/test-duplicate-removal.js diff --git a/api/src/services/drupal/entries.service.ts b/api/src/services/drupal/entries.service.ts index 37216a944..02ef265d7 100644 --- a/api/src/services/drupal/entries.service.ts +++ b/api/src/services/drupal/entries.service.ts @@ -730,6 +730,9 @@ const processFieldData = async ( } else { processedData[dataKey] = isoDate.toISOString(); } + // Mark field as processed to avoid duplicate processing in second loop + processedFields.add(dataKey); + processedFields.add(matchingFieldConfig.field_name); continue; } } @@ -741,6 +744,9 @@ const processFieldData = async ( typeof value === 'number' ) { processedData[dataKey] = value === 1; + // Mark field as processed to avoid duplicate processing in second loop + processedFields.add(dataKey); + processedFields.add(matchingFieldConfig.field_name); continue; } } @@ -752,6 +758,9 @@ const processFieldData = async ( typeof value === 'number' ) { processedData[dataKey] = `${value}`; + // Mark field as processed to avoid duplicate processing in second loop + processedFields.add(dataKey); + processedFields.add(matchingFieldConfig.field_name); continue; } } @@ -840,17 +849,43 @@ const processFieldData = async ( }; } } else if (fieldName.endsWith('_value')) { + // Skip if this field was already processed in the main loop (avoid duplicates) + const baseFieldName = fieldName.replace('_value', ''); + if ( + processedFields.has(fieldName) || + processedFields.has(baseFieldName) + ) { + continue; + } + // Check if content contains HTML if (/<\/?[a-z][\s\S]*>/i.test(value)) { const dom = new JSDOM(value); const htmlDoc = dom.window.document.querySelector('body'); const jsonValue = htmlToJson(htmlDoc); - ctValue[fieldName.replace('_value', '')] = jsonValue; + ctValue[baseFieldName] = jsonValue; } else { - ctValue[fieldName.replace('_value', '')] = value; + ctValue[baseFieldName] = value; } + + // Mark both the original and base field as processed to avoid duplicates + processedFields.add(fieldName); + processedFields.add(baseFieldName); } else if (fieldName.endsWith('_status')) { - ctValue[fieldName.replace('_status', '')] = value; + // Skip if this field was already processed in the main loop (avoid duplicates) + const baseFieldName = fieldName.replace('_status', ''); + if ( + processedFields.has(fieldName) || + processedFields.has(baseFieldName) + ) { + continue; + } + + ctValue[baseFieldName] = value; + + // Mark both the original and base field as processed to avoid duplicates + processedFields.add(fieldName); + processedFields.add(baseFieldName); } else { // Check if content contains HTML if (typeof value === 'string' && /<\/?[a-z][\s\S]*>/i.test(value)) { @@ -864,14 +899,45 @@ const processFieldData = async ( } } - // Apply processed field data - Object.assign(ctValue, processedData); + // Apply processed field data, but prioritize ctValue (processed without suffixes) over processedData (with suffixes) + // This prevents duplicate fields like both 'body' and 'body_value' from appearing + const mergedData = { ...processedData, ...ctValue }; // Final cleanup: remove any null, undefined, or empty values from the final result + // Also remove duplicate fields where both suffixed and non-suffixed versions exist const cleanedEntry: any = {}; - for (const [key, val] of Object.entries(ctValue)) { + for (const [key, val] of Object.entries(mergedData)) { if (val !== null && val !== undefined && val !== '') { - cleanedEntry[key] = val; + // Check if this is a suffixed field (_value, _status, _uri) and if a non-suffixed version exists + const isValueField = key.endsWith('_value'); + const isStatusField = key.endsWith('_status'); + const isUriField = key.endsWith('_uri'); + + if (isValueField) { + const baseFieldName = key.replace('_value', ''); + // Only include the _value field if the base field doesn't exist + if (!mergedData.hasOwnProperty(baseFieldName)) { + cleanedEntry[key] = val; + } + // If base field exists, skip the _value field (base field takes priority) + } else if (isStatusField) { + const baseFieldName = key.replace('_status', ''); + // Only include the _status field if the base field doesn't exist + if (!mergedData.hasOwnProperty(baseFieldName)) { + cleanedEntry[key] = val; + } + // If base field exists, skip the _status field (base field takes priority) + } else if (isUriField) { + const baseFieldName = key.replace('_uri', ''); + // Only include the _uri field if the base field doesn't exist + if (!mergedData.hasOwnProperty(baseFieldName)) { + cleanedEntry[key] = val; + } + // If base field exists, skip the _uri field (base field takes priority) + } else { + // For non-suffixed fields, always include them + cleanedEntry[key] = val; + } } } diff --git a/api/test-duplicate-fix-simple.js b/api/test-duplicate-fix-simple.js new file mode 100644 index 000000000..e1af7d18e --- /dev/null +++ b/api/test-duplicate-fix-simple.js @@ -0,0 +1,316 @@ +#!/usr/bin/env node + +/** + * Simple test script to verify duplicate field fix + * Tests the core duplicate removal logic without complex imports + */ + +const fs = require('fs'); +const path = require('path'); + +console.log('๐Ÿงช Testing Duplicate Field Fix - Simple Version\n'); +console.log('===============================================\n'); + +// Simulate the exact duplicate removal logic from entries.service.ts +function testDuplicateRemovalLogic(mergedData) { + console.log('๐Ÿ“‹ Input Data (with potential duplicates):'); + console.log(JSON.stringify(mergedData, null, 2)); + console.log(''); + + // Apply the exact cleanup logic from the service + const cleanedEntry = {}; + for (const [key, val] of Object.entries(mergedData)) { + if (val !== null && val !== undefined && val !== '') { + // Check if this is a suffixed field (_value, _status, _uri) and if a non-suffixed version exists + const isValueField = key.endsWith('_value'); + const isStatusField = key.endsWith('_status'); + const isUriField = key.endsWith('_uri'); + + if (isValueField) { + const baseFieldName = key.replace('_value', ''); + // Only include the _value field if the base field doesn't exist + if (!mergedData.hasOwnProperty(baseFieldName)) { + cleanedEntry[key] = val; + console.log(`โœ… Keeping _value field: ${key} (no base field found)`); + } else { + console.log( + `๐Ÿ—‘๏ธ Removing _value field: ${key} (base field ${baseFieldName} exists)` + ); + } + // If base field exists, skip the _value field (base field takes priority) + } else if (isStatusField) { + const baseFieldName = key.replace('_status', ''); + // Only include the _status field if the base field doesn't exist + if (!mergedData.hasOwnProperty(baseFieldName)) { + cleanedEntry[key] = val; + console.log(`โœ… Keeping _status field: ${key} (no base field found)`); + } else { + console.log( + `๐Ÿ—‘๏ธ Removing _status field: ${key} (base field ${baseFieldName} exists)` + ); + } + // If base field exists, skip the _status field (base field takes priority) + } else if (isUriField) { + const baseFieldName = key.replace('_uri', ''); + // Only include the _uri field if the base field doesn't exist + if (!mergedData.hasOwnProperty(baseFieldName)) { + cleanedEntry[key] = val; + console.log(`โœ… Keeping _uri field: ${key} (no base field found)`); + } else { + console.log( + `๐Ÿ—‘๏ธ Removing _uri field: ${key} (base field ${baseFieldName} exists)` + ); + } + // If base field exists, skip the _uri field (base field takes priority) + } else { + // For non-suffixed fields, always include them + cleanedEntry[key] = val; + console.log(`โœ… Keeping base field: ${key}`); + } + } + } + + console.log('\n๐ŸŽฏ Output Data (duplicates removed):'); + console.log(JSON.stringify(cleanedEntry, null, 2)); + console.log(''); + + return cleanedEntry; +} + +// Test cases based on your actual entry structure +function runTestCases() { + const testCases = [ + { + name: 'Your Actual Entry Structure', + data: { + nid: 496, + title: 'Creativity: 13 Ways to Look at It', + langcode: 'en', + created: '2019-09-09T13:09:29.000Z', + type: 'article', + + // These are the duplicates from your actual file + body_value: { + type: 'doc', + uid: '025c76b8546a46b9aacfbea3a4ea15b3', + attrs: {}, + children: [ + { + type: 'p', + attrs: {}, + uid: 'f63185c9190d4f5aac92ada099e08d3c', + children: [ + { + text: 'Every day, we see the spark of imagination and invention...', + }, + ], + }, + ], + }, + body: { + type: 'doc', + uid: '0fa04056a606443f8e3417086b88cd38', + attrs: {}, + children: [ + { + type: 'p', + attrs: {}, + uid: '6e0be5417c9048d1a2c36ed42d242813', + children: [ + { + text: 'Every day, we see the spark of imagination and invention...', + }, + ], + }, + ], + }, + field_external_link_value: + 'https://magazine.rice.edu/2019/05/13-ways-of-looking-at-creativity/?utm_source=Hero&utm_medium=thirteen&utm_campaign=Hero%20Links', + field_external_link: + 'https://magazine.rice.edu/2019/05/13-ways-of-looking-at-creativity/?utm_source=Hero&utm_medium=thirteen&utm_campaign=Hero%20Links', + field_formatted_title_value: '

Creativity

\r\n', + field_formatted_title: { + type: 'doc', + uid: '2b5aac7ee1e44194a8d41ab806753677', + attrs: {}, + children: [ + { + type: 'p', + attrs: {}, + uid: 'cce203a33e0640d7a7ab389055638635', + children: [ + { text: 'Creativity', attrs: { style: {} }, bold: true }, + ], + }, + ], + }, + field_subtitle_value: '13 WAYS TO LOOK AT IT', + field_subtitle: '13 WAYS TO LOOK AT IT', + uid: 'content_type_entries_title_496', + locale: 'en', + }, + }, + { + name: 'Mixed Scenarios', + data: { + // Case 1: Both _value and base exist (should remove _value) + test_field_value: 'value version', + test_field: 'base version', + + // Case 2: Only _value exists (should keep _value) + only_value_field_value: 'only value exists', + + // Case 3: Only base exists (should keep base) + only_base_field: 'only base exists', + + // Case 4: Status fields + comment_status: 1, + comment: 'processed comment', + + // Case 5: URI fields + link_uri: 'https://example.com', + link: { title: 'Example', href: 'https://example.com' }, + + // Regular fields + title: 'Test Title', + nid: 123, + }, + }, + ]; + + console.log('๐Ÿงช Running Test Cases\n'); + + let allTestsPassed = true; + + testCases.forEach((testCase, index) => { + console.log(`TEST ${index + 1}: ${testCase.name}`); + console.log('='.repeat(testCase.name.length + 10)); + + const result = testDuplicateRemovalLogic(testCase.data); + + // Check for duplicates + const fieldNames = Object.keys(result); + const duplicates = []; + + fieldNames.forEach((field) => { + if (field.endsWith('_value')) { + const baseField = field.replace('_value', ''); + if (fieldNames.includes(baseField)) { + duplicates.push({ suffix: field, base: baseField }); + } + } else if (field.endsWith('_status')) { + const baseField = field.replace('_status', ''); + if (fieldNames.includes(baseField)) { + duplicates.push({ suffix: field, base: baseField }); + } + } else if (field.endsWith('_uri')) { + const baseField = field.replace('_uri', ''); + if (fieldNames.includes(baseField)) { + duplicates.push({ suffix: field, base: baseField }); + } + } + }); + + console.log('๐Ÿ“Š Test Results:'); + console.log(` Input fields: ${Object.keys(testCase.data).length}`); + console.log(` Output fields: ${fieldNames.length}`); + console.log(` Duplicates found: ${duplicates.length}`); + + if (duplicates.length === 0) { + console.log(' โœ… PASSED - No duplicates found!'); + } else { + console.log(' โŒ FAILED - Duplicates still exist:'); + duplicates.forEach((dup) => { + console.log(` - ${dup.suffix} + ${dup.base}`); + }); + allTestsPassed = false; + } + + console.log('\n'); + }); + + return allTestsPassed; +} + +// Test the specific case from your JSON file +function testYourSpecificCase() { + console.log('๐ŸŽฏ Testing Your Specific Case\n'); + console.log('=============================\n'); + + // This is exactly what should be in your JSON after the fix + const expectedCleanResult = { + nid: 496, + title: 'Creativity: 13 Ways to Look at It', + langcode: 'en', + created: '2019-09-09T13:09:29.000Z', + type: 'article', + uid: 'content_type_entries_title_496', + locale: 'en', + // Only these should remain (no _value versions) + body: { + /* processed content */ + }, + field_external_link: + 'https://magazine.rice.edu/2019/05/13-ways-of-looking-at-creativity/?utm_source=Hero&utm_medium=thirteen&utm_campaign=Hero%20Links', + field_formatted_title: { + /* processed content */ + }, + field_subtitle: '13 WAYS TO LOOK AT IT', + }; + + console.log( + 'โœ… Expected Clean Result (what you should see after migration):' + ); + console.log('Fields that should exist:'); + Object.keys(expectedCleanResult).forEach((field) => { + console.log(` - ${field}`); + }); + + console.log('\nโŒ Fields that should NOT exist:'); + console.log(' - body_value'); + console.log(' - field_external_link_value'); + console.log(' - field_formatted_title_value'); + console.log(' - field_subtitle_value'); + + console.log('\n๐Ÿ’ก After running migration, check your JSON file:'); + console.log( + ' /Users/saurav.upadhyay/Expert Service/Contentstack Migration/migration-v2/api/cmsMigrationData/blta4dadbc9c65d73cb/entries/article/en/en.json' + ); + console.log( + ' It should only contain the fields listed above, not the _value versions.' + ); +} + +// Main execution +function main() { + console.log('๐Ÿš€ Starting Duplicate Field Fix Tests\n'); + + const testResults = runTestCases(); + + testYourSpecificCase(); + + console.log('\n๐Ÿ“Š FINAL RESULTS'); + console.log('================'); + console.log(`All tests passed: ${testResults ? 'โœ… YES' : 'โŒ NO'}`); + + if (testResults) { + console.log( + '\n๐ŸŽ‰ SUCCESS: The duplicate removal logic is working correctly!' + ); + console.log('โœ… The fix should work when you run the full migration.'); + console.log('\n๐Ÿš€ Next Steps:'); + console.log(' 1. Run the migration from the UI'); + console.log(' 2. Check the generated JSON files'); + console.log( + ' 3. Verify no _value suffixes remain when base fields exist' + ); + } else { + console.log('\nโŒ FAILURE: The logic needs more work.'); + console.log('โš ๏ธ Do not run the full migration yet.'); + } + + process.exit(testResults ? 0 : 1); +} + +// Run the test +main(); diff --git a/api/test-duplicate-fix-with-db.js b/api/test-duplicate-fix-with-db.js new file mode 100644 index 000000000..fb7375850 --- /dev/null +++ b/api/test-duplicate-fix-with-db.js @@ -0,0 +1,476 @@ +#!/usr/bin/env node + +/** + * Test script to verify duplicate field fix using actual database and processing logic + * This will test the entries.service.ts fix without running the full UI migration + */ + +import fs from 'fs'; +import path from 'path'; +import mysql from 'mysql2'; +import { fileURLToPath } from 'url'; + +// Get current directory for ES modules +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +// Import the entries service (we'll need to adjust the import path) +const entriesServicePath = path.join( + __dirname, + 'src/services/drupal/entries.service.ts' +); + +console.log('๐Ÿงช Testing Duplicate Field Fix with Real Database Data\n'); +console.log('==================================================\n'); + +// Database configuration (you may need to adjust these) +const DB_CONFIG = { + host: process.env.DB_HOST || 'localhost', + user: process.env.DB_USER || 'root', + password: process.env.DB_PASSWORD || '', + database: process.env.DB_NAME || 'drupal_db', + port: process.env.DB_PORT || 3306, +}; + +console.log('๐Ÿ“‹ Database Configuration:'); +console.log(` Host: ${DB_CONFIG.host}`); +console.log(` Database: ${DB_CONFIG.database}`); +console.log(` User: ${DB_CONFIG.user}`); +console.log(''); + +// Create database connection +let connection; + +try { + connection = mysql.createConnection(DB_CONFIG); + console.log('โœ… Database connection established'); +} catch (error) { + console.error('โŒ Failed to connect to database:', error.message); + process.exit(1); +} + +// Test the processFieldData function directly +async function testProcessFieldData() { + console.log('\n๐Ÿ” Testing processFieldData function...\n'); + + // Sample Drupal entry data that would cause duplicates + const testEntryData = { + nid: 496, + title: 'Test Article', + langcode: 'en', + created: 1568032169, + type: 'article', + + // These fields should cause duplicates in the old logic + body_value: '

This is the body content from _value field

', + field_formatted_title_value: '

Formatted Title

', + field_external_link_value: 'https://example.com/test-link', + field_subtitle_value: 'Test Subtitle', + + // Some fields without _value counterparts + field_image_target_id: 123, + field_tags_target_id: 456, + }; + + console.log('๐Ÿ“‹ Input Entry Data:'); + console.log(JSON.stringify(testEntryData, null, 2)); + console.log(''); + + // Mock field configurations (simplified) + const mockFieldConfigs = [ + { field_name: 'body', field_type: 'text_with_summary' }, + { field_name: 'field_formatted_title', field_type: 'text' }, + { field_name: 'field_external_link', field_type: 'link' }, + { field_name: 'field_subtitle', field_type: 'text' }, + { field_name: 'field_image', field_type: 'image' }, + { field_name: 'field_tags', field_type: 'entity_reference' }, + ]; + + // Mock other required parameters + const mockAssetId = {}; + const mockReferenceId = {}; + const mockTaxonomyId = {}; + const mockTaxonomyFieldMapping = {}; + const mockReferenceFieldMapping = {}; + const mockAssetFieldMapping = {}; + const mockTaxonomyReferenceLookup = {}; + const contentType = 'article'; + + try { + // Import and test the processFieldData function + // Note: We'll need to create a simplified version since the actual import might be complex + const processedEntry = await simulateProcessFieldData( + testEntryData, + mockFieldConfigs, + mockAssetId, + mockReferenceId, + mockTaxonomyId, + mockTaxonomyFieldMapping, + mockReferenceFieldMapping, + mockAssetFieldMapping, + mockTaxonomyReferenceLookup, + contentType + ); + + console.log('๐ŸŽฏ Processed Entry Data:'); + console.log(JSON.stringify(processedEntry, null, 2)); + console.log(''); + + // Check for duplicates + const fieldNames = Object.keys(processedEntry); + const duplicateCheck = { + body: { + hasBase: fieldNames.includes('body'), + hasValue: fieldNames.includes('body_value'), + isDuplicate: + fieldNames.includes('body') && fieldNames.includes('body_value'), + }, + field_formatted_title: { + hasBase: fieldNames.includes('field_formatted_title'), + hasValue: fieldNames.includes('field_formatted_title_value'), + isDuplicate: + fieldNames.includes('field_formatted_title') && + fieldNames.includes('field_formatted_title_value'), + }, + field_external_link: { + hasBase: fieldNames.includes('field_external_link'), + hasValue: fieldNames.includes('field_external_link_value'), + isDuplicate: + fieldNames.includes('field_external_link') && + fieldNames.includes('field_external_link_value'), + }, + field_subtitle: { + hasBase: fieldNames.includes('field_subtitle'), + hasValue: fieldNames.includes('field_subtitle_value'), + isDuplicate: + fieldNames.includes('field_subtitle') && + fieldNames.includes('field_subtitle_value'), + }, + }; + + console.log('๐Ÿ” Duplicate Analysis:'); + console.log('====================='); + + let totalDuplicates = 0; + for (const [fieldName, check] of Object.entries(duplicateCheck)) { + console.log(`${fieldName}:`); + console.log(` Has base field: ${check.hasBase ? 'โœ…' : 'โŒ'}`); + console.log(` Has _value field: ${check.hasValue ? 'โœ…' : 'โŒ'}`); + console.log( + ` Is duplicate: ${ + check.isDuplicate ? 'โŒ YES (BAD)' : 'โœ… NO (GOOD)' + }` + ); + console.log(''); + + if (check.isDuplicate) { + totalDuplicates++; + } + } + + console.log('๐Ÿ“Š Summary:'); + console.log(` Total fields: ${fieldNames.length}`); + console.log(` Duplicate fields found: ${totalDuplicates}`); + console.log( + ` Fix working: ${totalDuplicates === 0 ? 'โœ… YES' : 'โŒ NO'}` + ); + + if (totalDuplicates === 0) { + console.log( + '\n๐ŸŽ‰ SUCCESS: No duplicate fields found! The fix is working correctly.' + ); + } else { + console.log( + '\nโŒ FAILURE: Duplicate fields still exist. The fix needs adjustment.' + ); + } + + return totalDuplicates === 0; + } catch (error) { + console.error('โŒ Error processing entry data:', error.message); + return false; + } +} + +// Simplified version of processFieldData for testing +async function simulateProcessFieldData( + entryData, + fieldConfigs, + assetId, + referenceId, + taxonomyId, + taxonomyFieldMapping, + referenceFieldMapping, + assetFieldMapping, + taxonomyReferenceLookup, + contentType +) { + const fieldNames = Object.keys(entryData); + const isoDate = new Date(); + const processedData = {}; + const skippedFields = new Set(); + const processedFields = new Set(); + + // First loop: Process special field types + for (const [dataKey, value] of Object.entries(entryData)) { + // Handle basic field processing (simplified) + if (value === null || value === undefined || value === '') { + continue; + } + + // Default case: copy field to processedData if it wasn't handled by special processing above + if (!(dataKey in processedData)) { + processedData[dataKey] = value; + } + } + + // Second loop: Process standard field transformations + const ctValue = {}; + + for (const fieldName of fieldNames) { + if (skippedFields.has(fieldName)) { + continue; + } + + const value = entryData[fieldName]; + + if (fieldName === 'created') { + ctValue[fieldName] = new Date(value * 1000).toISOString(); + } else if (fieldName === 'nid') { + ctValue.uid = `content_type_entries_title_${value}`; + } else if (fieldName === 'langcode') { + ctValue.locale = value || 'en-us'; + } else if (fieldName.endsWith('_value')) { + // Skip if this field was already processed in the main loop (avoid duplicates) + const baseFieldName = fieldName.replace('_value', ''); + if ( + processedFields.has(fieldName) || + processedFields.has(baseFieldName) + ) { + continue; + } + + // Process HTML content + if (/<\/?[a-z][\s\S]*>/i.test(value)) { + // Simulate HTML to JSON conversion (simplified) + ctValue[baseFieldName] = { + type: 'doc', + children: [ + { type: 'p', children: [{ text: value.replace(/<[^>]*>/g, '') }] }, + ], + }; + } else { + ctValue[baseFieldName] = value; + } + + // Mark both the original and base field as processed to avoid duplicates + processedFields.add(fieldName); + processedFields.add(baseFieldName); + } else if (fieldName.endsWith('_status')) { + const baseFieldName = fieldName.replace('_status', ''); + if ( + processedFields.has(fieldName) || + processedFields.has(baseFieldName) + ) { + continue; + } + + ctValue[baseFieldName] = value; + processedFields.add(fieldName); + processedFields.add(baseFieldName); + } else { + ctValue[fieldName] = value; + } + } + + // Apply processed field data, but prioritize ctValue over processedData + const mergedData = { ...processedData, ...ctValue }; + + // Final cleanup: remove duplicates and null values + const cleanedEntry = {}; + for (const [key, val] of Object.entries(mergedData)) { + if (val !== null && val !== undefined && val !== '') { + // Check if this is a suffixed field and if a non-suffixed version exists + const isValueField = key.endsWith('_value'); + const isStatusField = key.endsWith('_status'); + const isUriField = key.endsWith('_uri'); + + if (isValueField) { + const baseFieldName = key.replace('_value', ''); + // Only include the _value field if the base field doesn't exist + if (!mergedData.hasOwnProperty(baseFieldName)) { + cleanedEntry[key] = val; + } + } else if (isStatusField) { + const baseFieldName = key.replace('_status', ''); + if (!mergedData.hasOwnProperty(baseFieldName)) { + cleanedEntry[key] = val; + } + } else if (isUriField) { + const baseFieldName = key.replace('_uri', ''); + if (!mergedData.hasOwnProperty(baseFieldName)) { + cleanedEntry[key] = val; + } + } else { + // For non-suffixed fields, always include them + cleanedEntry[key] = val; + } + } + } + + return cleanedEntry; +} + +// Test with real database data +async function testWithRealDatabaseData() { + console.log('\n๐Ÿ” Testing with Real Database Data...\n'); + + return new Promise((resolve) => { + // Query to get a sample entry with _value fields + const query = ` + SELECT n.nid, n.title, n.langcode, n.created, n.type, + bd.body_value, bd.body_summary, + fft.field_formatted_title_value, + fel.field_external_link_value, + fs.field_subtitle_value + FROM node n + LEFT JOIN node__body bd ON n.nid = bd.entity_id + LEFT JOIN node__field_formatted_title fft ON n.nid = fft.entity_id + LEFT JOIN node__field_external_link fel ON n.nid = fel.entity_id + LEFT JOIN node__field_subtitle fs ON n.nid = fs.entity_id + WHERE n.type = 'article' + AND n.status = 1 + LIMIT 1 + `; + + connection.query(query, async (error, results) => { + if (error) { + console.error('โŒ Database query failed:', error.message); + resolve(false); + return; + } + + if (results.length === 0) { + console.log('โš ๏ธ No article entries found in database'); + resolve(false); + return; + } + + const dbEntry = results[0]; + console.log('๐Ÿ“‹ Raw Database Entry:'); + console.log(JSON.stringify(dbEntry, null, 2)); + console.log(''); + + // Process this real data through our function + try { + const processedEntry = await simulateProcessFieldData( + dbEntry, + [], // Empty field configs for simplicity + {}, + {}, + {}, + {}, + {}, + {}, + {}, + 'article' + ); + + console.log('๐ŸŽฏ Processed Real Entry:'); + console.log(JSON.stringify(processedEntry, null, 2)); + console.log(''); + + // Check for duplicates in real data + const fieldNames = Object.keys(processedEntry); + const duplicates = []; + + fieldNames.forEach((field) => { + if (field.endsWith('_value')) { + const baseField = field.replace('_value', ''); + if (fieldNames.includes(baseField)) { + duplicates.push({ suffix: field, base: baseField }); + } + } + }); + + console.log('๐Ÿ” Real Data Duplicate Check:'); + console.log(` Total fields: ${fieldNames.length}`); + console.log(` Duplicates found: ${duplicates.length}`); + + if (duplicates.length === 0) { + console.log(' โœ… No duplicates found in real data!'); + resolve(true); + } else { + console.log(' โŒ Duplicates found:'); + duplicates.forEach((dup) => { + console.log(` - ${dup.suffix} + ${dup.base}`); + }); + resolve(false); + } + } catch (error) { + console.error( + 'โŒ Error processing real database entry:', + error.message + ); + resolve(false); + } + }); + }); +} + +// Main test execution +async function runTests() { + console.log('๐Ÿš€ Starting Duplicate Field Fix Tests\n'); + + // Test 1: Simulated data + console.log('TEST 1: Simulated Entry Data'); + console.log('============================='); + const simulatedTest = await testProcessFieldData(); + + // Test 2: Real database data + console.log('\nTEST 2: Real Database Data'); + console.log('=========================='); + const realDataTest = await testWithRealDatabaseData(); + + // Summary + console.log('\n๐Ÿ“Š FINAL TEST RESULTS'); + console.log('====================='); + console.log( + `Simulated data test: ${simulatedTest ? 'โœ… PASSED' : 'โŒ FAILED'}` + ); + console.log( + `Real database test: ${realDataTest ? 'โœ… PASSED' : 'โŒ FAILED'}` + ); + + const allTestsPassed = simulatedTest && realDataTest; + console.log( + `\nOverall result: ${ + allTestsPassed ? '๐ŸŽ‰ ALL TESTS PASSED' : 'โŒ TESTS FAILED' + }` + ); + + if (allTestsPassed) { + console.log('\nโœ… The duplicate field fix is working correctly!'); + console.log('โœ… You can safely run the full migration from the UI.'); + } else { + console.log( + '\nโŒ The fix needs more work before running the full migration.' + ); + } + + // Close database connection + if (connection) { + connection.end(); + } + + process.exit(allTestsPassed ? 0 : 1); +} + +// Run the tests +runTests().catch((error) => { + console.error('โŒ Test execution failed:', error); + if (connection) { + connection.end(); + } + process.exit(1); +}); diff --git a/api/test-duplicate-removal.js b/api/test-duplicate-removal.js new file mode 100644 index 000000000..8bf227dc8 --- /dev/null +++ b/api/test-duplicate-removal.js @@ -0,0 +1,123 @@ +#!/usr/bin/env node + +/** + * Test script to verify duplicate field removal logic + */ + +console.log('๐Ÿงช Testing Duplicate Field Removal Logic\n'); + +// Simulate the mergedData that would contain duplicates +const testMergedData = { + // Duplicate case 1: both body_value and body exist + body_value: 'content from _value field', + body: 'content from base field', + + // Duplicate case 2: both field_formatted_title_value and field_formatted_title exist + field_formatted_title_value: 'title from _value field', + field_formatted_title: 'title from base field', + + // Duplicate case 3: both field_external_link_value and field_external_link exist + field_external_link_value: 'link from _value field', + field_external_link: 'link from base field', + + // Non-duplicate case: only _value field exists + field_only_value_exists_value: 'only value field', + + // Non-duplicate case: only base field exists + field_only_base_exists: 'only base field', + + // Other fields + title: 'Article Title', + nid: 496, + created: '2019-09-09T13:09:29.000Z', +}; + +console.log('๐Ÿ“‹ Input (with duplicates):'); +console.log(JSON.stringify(testMergedData, null, 2)); + +// Apply the duplicate removal logic +const cleanedEntry = {}; +for (const [key, val] of Object.entries(testMergedData)) { + if (val !== null && val !== undefined && val !== '') { + // Check if this is a suffixed field (_value, _status, _uri) and if a non-suffixed version exists + const isValueField = key.endsWith('_value'); + const isStatusField = key.endsWith('_status'); + const isUriField = key.endsWith('_uri'); + + if (isValueField) { + const baseFieldName = key.replace('_value', ''); + // Only include the _value field if the base field doesn't exist + if (!testMergedData.hasOwnProperty(baseFieldName)) { + cleanedEntry[key] = val; + console.log(`โœ… Keeping _value field: ${key} (no base field found)`); + } else { + console.log( + `๐Ÿ—‘๏ธ Removing _value field: ${key} (base field ${baseFieldName} exists)` + ); + } + // If base field exists, skip the _value field (base field takes priority) + } else if (isStatusField) { + const baseFieldName = key.replace('_status', ''); + // Only include the _status field if the base field doesn't exist + if (!testMergedData.hasOwnProperty(baseFieldName)) { + cleanedEntry[key] = val; + console.log(`โœ… Keeping _status field: ${key} (no base field found)`); + } else { + console.log( + `๐Ÿ—‘๏ธ Removing _status field: ${key} (base field ${baseFieldName} exists)` + ); + } + // If base field exists, skip the _status field (base field takes priority) + } else if (isUriField) { + const baseFieldName = key.replace('_uri', ''); + // Only include the _uri field if the base field doesn't exist + if (!testMergedData.hasOwnProperty(baseFieldName)) { + cleanedEntry[key] = val; + console.log(`โœ… Keeping _uri field: ${key} (no base field found)`); + } else { + console.log( + `๐Ÿ—‘๏ธ Removing _uri field: ${key} (base field ${baseFieldName} exists)` + ); + } + // If base field exists, skip the _uri field (base field takes priority) + } else { + // For non-suffixed fields, always include them + cleanedEntry[key] = val; + console.log(`โœ… Keeping base field: ${key}`); + } + } +} + +console.log('\n๐ŸŽฏ Output (duplicates removed):'); +console.log(JSON.stringify(cleanedEntry, null, 2)); + +console.log('\n๐Ÿ“Š Summary:'); +console.log(`Input fields: ${Object.keys(testMergedData).length}`); +console.log(`Output fields: ${Object.keys(cleanedEntry).length}`); +console.log( + `Removed duplicates: ${ + Object.keys(testMergedData).length - Object.keys(cleanedEntry).length + }` +); + +// Verify no duplicates remain +const hasBodyDuplicate = + cleanedEntry.hasOwnProperty('body') && + cleanedEntry.hasOwnProperty('body_value'); +const hasTitleDuplicate = + cleanedEntry.hasOwnProperty('field_formatted_title') && + cleanedEntry.hasOwnProperty('field_formatted_title_value'); +const hasLinkDuplicate = + cleanedEntry.hasOwnProperty('field_external_link') && + cleanedEntry.hasOwnProperty('field_external_link_value'); + +console.log(`\nโœ… Verification:`); +console.log( + `Body duplicate removed: ${!hasBodyDuplicate ? 'โœ… YES' : 'โŒ NO'}` +); +console.log( + `Title duplicate removed: ${!hasTitleDuplicate ? 'โœ… YES' : 'โŒ NO'}` +); +console.log( + `Link duplicate removed: ${!hasLinkDuplicate ? 'โœ… YES' : 'โŒ NO'}` +); diff --git a/ui/src/components/DestinationStack/Actions/LoadLanguageMapper.tsx b/ui/src/components/DestinationStack/Actions/LoadLanguageMapper.tsx index be3b893c1..22f5e725c 100644 --- a/ui/src/components/DestinationStack/Actions/LoadLanguageMapper.tsx +++ b/ui/src/components/DestinationStack/Actions/LoadLanguageMapper.tsx @@ -30,7 +30,6 @@ export type ExistingFieldType = { * @returns {JSX.Element | null} - Returns a JSX element if empty, otherwise null. */ const Mapper = ({ - key, uid, cmsLocaleOptions, handleLangugeDelete, @@ -63,7 +62,6 @@ const Mapper = ({ // ๐Ÿ”ฅ Sync with parent state when auto-mapping occurs useEffect(() => { if (parentLocaleState && Object.keys(parentLocaleState).length > 0) { - console.info('๐Ÿ”„ Syncing child existingLocale with parent state:', parentLocaleState); setexistingLocale(prev => ({ ...prev, ...parentLocaleState @@ -84,15 +82,37 @@ const Mapper = ({ // },[]); useEffect(() => { - const newMigrationDataObj: INewMigration = { - ...newMigrationData, - destination_stack: { - ...newMigrationData?.destination_stack, - localeMapping: selectedMappings + // ๐Ÿ”ง CRITICAL FIX: Merge selectedMappings with existing auto-mapping instead of overriding + const existingMapping = newMigrationData?.destination_stack?.localeMapping || {}; + const mergedMapping = { ...existingMapping, ...selectedMappings }; + + // Only update if there are actual changes to avoid infinite loops + const hasChanges = JSON.stringify(existingMapping) !== JSON.stringify(mergedMapping); + + if (hasChanges && Object.keys(selectedMappings).length > 0) { + console.info('๐Ÿ” DEBUG: Mapper updating localeMapping with selectedMappings:', selectedMappings); + console.info('๐Ÿ” DEBUG: Existing mapping:', existingMapping); + console.info('๐Ÿ” DEBUG: Merged mapping:', mergedMapping); + + // ๐Ÿ”ง CRITICAL CHECK: Don't override with empty values + const hasEmptyValues = Object.values(selectedMappings).some(value => value === ''); + if (hasEmptyValues) { + console.warn('โš ๏ธ WARNING: selectedMappings contains empty values, skipping update to preserve auto-mapping'); + console.warn(' Empty selectedMappings:', selectedMappings); + console.warn(' Preserving existing mapping:', existingMapping); + return; } - }; + + const newMigrationDataObj: INewMigration = { + ...newMigrationData, + destination_stack: { + ...newMigrationData?.destination_stack, + localeMapping: mergedMapping + } + }; - dispatch(updateNewMigrationData(newMigrationDataObj)); + dispatch(updateNewMigrationData(newMigrationDataObj)); + } }, [selectedMappings]); useEffect(() => { @@ -175,16 +195,22 @@ const Mapper = ({ setexistingLocale({}); setExistingField({}); + // ๐Ÿ”ง CRITICAL FIX: Don't override auto-mapping with empty values + // Check if auto-mapping already exists for this locale + const existingAutoMapping = newMigrationData?.destination_stack?.localeMapping?.[`${locale?.label}-master_locale`]; + updatedSelectedMappings = { - [`${locale?.label}-master_locale`]: '', + [`${locale?.label}-master_locale`]: existingAutoMapping || '', }; setSelectedMappings(updatedSelectedMappings); } else if ( !isLabelMismatch && !isStackChanged ) { const key = `${locale?.label}-master_locale` - updatedSelectedMappings = { - [key]: updatedSelectedMappings?.[`${locale?.label}-master_locale`] ? updatedSelectedMappings?.[`${locale?.label}-master_locale`] : '', + // ๐Ÿ”ง CRITICAL FIX: Use existing auto-mapping value instead of empty string + const existingAutoMapping = newMigrationData?.destination_stack?.localeMapping?.[key]; + updatedSelectedMappings = { + [key]: updatedSelectedMappings?.[key] || existingAutoMapping || '', }; setSelectedMappings(updatedSelectedMappings); } @@ -653,17 +679,21 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { const getNextUnmappedSourceLocale = (): { label: string; value: string } | null => { if (!sourceLocales || sourceLocales.length === 0) return null; - // Get currently mapped source locales (values from localeMapping) - const mappedSourceLocales = Object.values(newMigrationData?.destination_stack?.localeMapping || {}) - .filter(Boolean) // Remove empty strings - .filter(locale => locale !== stack?.master_locale); // Exclude master locale + // ๐Ÿ”ง CRITICAL FIX: Get currently mapped source locales from KEYS of localeMapping, not values + const localeMapping = newMigrationData?.destination_stack?.localeMapping || {}; + const mappedSourceLocales = Object.keys(localeMapping) + .filter(key => !key.includes('-master_locale')) // Exclude master locale entries + .filter(key => localeMapping[key] !== '' && localeMapping[key] !== null && localeMapping[key] !== undefined); // Only count valid mappings + + console.info('๐Ÿ” DEBUG: Currently mapped source locales:', mappedSourceLocales); + console.info('๐Ÿ” DEBUG: Available source locales:', sourceLocales.map(s => s.value)); // Find first unmapped source locale const unmappedLocale = sourceLocales.find(source => - !mappedSourceLocales.includes(source.value) && - source.value !== stack?.master_locale // Don't suggest master locale again + !mappedSourceLocales.includes(source.value) ); + console.info('๐Ÿ” DEBUG: Found unmapped locale:', unmappedLocale); return unmappedLocale || null; }; @@ -676,7 +706,6 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { ); if (exactMatch) { - console.info(`๐ŸŽฏ Exact match found for ${sourceLocale}: ${exactMatch.value}`); return exactMatch; } @@ -685,10 +714,6 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { const smartMatch = getSmartLocaleMapping(sourceLocale, options); const smartMatchObj = options.find(dest => dest.value === smartMatch); - if (smartMatchObj && smartMatchObj.value !== sourceLocale) { - console.info(`๐Ÿง  Smart match found for ${sourceLocale}: ${smartMatchObj.value}`); - } - return smartMatchObj || null; }; @@ -725,49 +750,14 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { // โœ… Declare keys before using it const keys = Object?.keys(newMigrationData?.destination_stack?.localeMapping || {})?.find( key => key === `${newMigrationData?.destination_stack?.selectedStack?.master_locale}-master_locale`); - - // ๐Ÿ” Debug logging to understand what's happening - if (sourceLocale && allLocales) { - console.info('๐Ÿ” Auto-mapping Debug Info:'); - console.info(' Source Locales:', sourceLocale); - console.info(' Destination Locales:', allLocales); - console.info(' Current Stack:', currentStack?.uid); - console.info(' Previous Stack:', previousStack?.uid); - console.info(' Is Stack Changed:', isStackChanged); - console.info(' Stack Has Changed (Improved):', stackHasChanged); - console.info(' Locale Mapping:', newMigrationData?.destination_stack?.localeMapping); - console.info(' Project Step:', newMigrationData?.project_current_step); - console.info(' Master Locale:', stack?.master_locale); - console.info(' Keys Found:', keys); - - // Check which condition will be met - const isSingleLocale = sourceLocale?.length === 1; - const isMultiLocale = sourceLocale?.length > 1; - const hasAllLocales = allLocales?.length > 0; - const hasCmsLocaleOptions = cmsLocaleOptions?.length > 0; - const shouldAutoMap = (Object?.entries(newMigrationData?.destination_stack?.localeMapping || {})?.length === 0 || - !keys || - stackHasChanged); - const isCorrectStep = newMigrationData?.project_current_step <= 2; - - console.info('๐Ÿ” Condition Check:'); - console.info(' Is Single Locale:', isSingleLocale); - console.info(' Is Multi Locale:', isMultiLocale); - console.info(' Has All Locales:', hasAllLocales); - console.info(' Has CMS Locale Options:', hasCmsLocaleOptions); - console.info(' Should Auto Map:', shouldAutoMap); - console.info(' Is Correct Step:', isCorrectStep); - } // ๐Ÿ”ง TC-11: Handle empty source locales if (!sourceLocale || sourceLocale.length === 0) { - console.warn('โš ๏ธ No source locales found - cannot proceed with mapping'); return; // Exit early - will show "No languages configured" message } // ๐Ÿ”ง TC-12: Handle empty destination locales (ensure master locale exists) if (!allLocales || allLocales.length === 0) { - console.warn('โš ๏ธ No destination locales found - adding master locale as fallback'); const masterLocaleObj = { label: stack?.master_locale || 'en-us', value: stack?.master_locale || 'en-us' }; setoptions([masterLocaleObj]); return; // Will re-trigger this effect with master locale added @@ -806,10 +796,6 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { let destinationLocale = ''; if (exactMatch) { destinationLocale = exactMatch.value; - console.info(`โœ… TC-02: Exact match found for single locale: ${singleSourceLocale.value} -> ${destinationLocale}`); - } else { - console.info(`๐Ÿ“ TC-02: No exact match for ${singleSourceLocale.value} - leaving destination empty for manual selection`); - // Leave destinationLocale empty as per TC-02 requirement } // Set the auto-selected source locale for the Mapper component @@ -820,10 +806,16 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { // Set the mapping in Redux state const autoMapping = { - [`${stack?.master_locale}-master_locale`]: stack?.master_locale, - ...(destinationLocale ? { [destinationLocale]: singleSourceLocale.value } : {}) + [`${singleSourceLocale.value}-master_locale`]: destinationLocale || stack?.master_locale, + ...(destinationLocale ? { [singleSourceLocale.value]: destinationLocale } : {}) }; + // ๐Ÿ” DEBUG: Log the mapping structure being created + console.info('๐Ÿ” DEBUG: Single locale autoMapping created:', JSON.stringify(autoMapping, null, 2)); + console.info('๐Ÿ” DEBUG: Source locale:', singleSourceLocale.value); + console.info('๐Ÿ” DEBUG: Destination locale:', destinationLocale); + console.info('๐Ÿ” DEBUG: Stack master locale:', stack?.master_locale); + const newMigrationDataObj: INewMigration = { ...newMigrationData, destination_stack: { @@ -833,6 +825,11 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { }; dispatch(updateNewMigrationData(newMigrationDataObj)); + // ๐Ÿ” DEBUG: Log after dispatch + console.info('๐Ÿ” DEBUG: Dispatched single locale mapping to Redux'); + + // ๐Ÿ” DEBUG: Auto-mapping completed for single locale + // Reset stack changed flag after auto-mapping if (isStackChanged) { setisStackChanged(false); @@ -848,10 +845,6 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { stackHasChanged) && newMigrationData?.project_current_step <= 2) { - console.info('๐Ÿš€ EXECUTING Multi-locale auto-mapping...'); - console.info(' Source Locales for matching:', sourceLocale); - console.info(' Available Destination Locales:', allLocales.slice(0, 10)); - // ๐Ÿ†• CONDITION 2: Enhanced multi-locale logic with master locale priority // First, check if master locale from source matches destination (PRIORITY) @@ -872,56 +865,24 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { dest.value.toLowerCase() === masterLocaleFromSource.value.toLowerCase() ); if (masterDestMatch) { - autoMapping[masterLocaleFromSource.value] = masterDestMatch.value; + // ๐Ÿ”ง CRITICAL FIX: Create both master locale entry AND regular mapping entry + autoMapping[`${masterLocaleFromSource.value}-master_locale`] = masterDestMatch.value; // For validation + autoMapping[masterLocaleFromSource.value] = masterDestMatch.value; // For regular mapping hasAnyMatches = true; - console.info(`๐Ÿ† Master locale priority mapping: ${masterLocaleFromSource.value} -> ${masterDestMatch.value}`); } } - // ๐Ÿ†• STEP 2: Handle other locales with enhanced duplicate/ambiguous logic - sourceLocale.forEach(source => { - // Skip if already mapped (master locale) - if (autoMapping[source.value]) return; - - // ๐Ÿ†• Enhanced matching: Prefer exact matches over partial/smart matches - // First try exact match - const exactMatch = allLocales.find(dest => - source.value.toLowerCase() === dest.value.toLowerCase() - ); - - if (exactMatch) { - autoMapping[source.value] = exactMatch.value; - hasAnyMatches = true; - console.info(`๐ŸŽฏ Exact match: ${source.value} -> ${exactMatch.value}`); - } else { - // Only try smart matching if no exact match exists - // This prevents 'en' from overriding 'en-us' -> 'en-us' mapping - const hasExactMatchInSource = sourceLocale.some(otherSource => - otherSource.value !== source.value && - allLocales.some(dest => dest.value.toLowerCase() === otherSource.value.toLowerCase()) - ); - - if (!hasExactMatchInSource) { - // Use smart mapping as fallback - const smartMatch = getSmartLocaleMapping(source.value, allLocales); - const smartMatchObj = allLocales.find(dest => dest.value === smartMatch); - - if (smartMatchObj && smartMatchObj.value !== source.value) { - autoMapping[source.value] = smartMatchObj.value; - hasAnyMatches = true; - console.info(`๐Ÿง  Smart match: ${source.value} -> ${smartMatchObj.value}`); - } else { - console.info(`โŒ No match found for: ${source.value}`); - } - } else { - console.info(`โญ๏ธ Skipping smart match for ${source.value} - exact matches take priority`); - } - } - }); + // ๐Ÿ”ง CRITICAL CHANGE: For multi-locale, only map the FIRST/MASTER locale initially + // Other locales will be handled by the "Add Language" functionality + console.info('๐Ÿ” DEBUG: Multi-locale detected - only mapping master locale initially'); + console.info(' Master locale from source:', masterLocaleFromSource?.value); + console.info(' Other locales will be available for "Add Language"'); + + // Skip auto-mapping all other locales to keep "Add Language" button enabled + // The "Add Language" functionality will handle mapping additional locales one by one // ๐Ÿ”ง TC-04 & TC-08: Enhanced no-match logic with master locale default if (!hasAnyMatches) { - console.info('โš ๏ธ TC-04/TC-08: No matches found - setting up master locale default mapping'); // Auto-select destination master locale for first source locale as per TC-04/TC-08 const firstSourceLocale = sourceLocale[0]; @@ -953,7 +914,6 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { } setMapperLocaleState(prev => ({ ...prev, ...updatedExistingLocale })); - console.info(`โœ… TC-04/TC-08: Auto-selected master locale ${stack?.master_locale} for first source ${firstSourceLocale.value}`); // Reset stack changed flag if (isStackChanged) { @@ -962,27 +922,13 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { return; // Exit early } - // ๐Ÿ”ง TC-03: Enhanced mapping for remaining locales - // If we have some matches but not all, try to map remaining locales to remaining destinations - const unmappedSources = sourceLocale.filter(source => !autoMapping[source.value]); - const unmappedDestinations = allLocales.filter(dest => - !Object.keys(autoMapping).includes(dest.value) && - dest.value !== stack?.master_locale - ); - - if (unmappedSources.length > 0 && unmappedDestinations.length > 0) { - console.info('๐Ÿ”ง TC-03: Mapping remaining locales to available destinations'); - - unmappedSources.forEach((source, index) => { - if (unmappedDestinations[index]) { - autoMapping[unmappedDestinations[index].value] = source.value; - console.info(`โœ… TC-03: Mapped remaining ${source.value} -> ${unmappedDestinations[index].value}`); - } - }); - } + // ๐Ÿ”ง REMOVED: TC-03 auto-mapping of remaining locales + // This was causing all locales to be mapped at once, disabling "Add Language" button + // Now remaining locales will be handled by "Add Language" functionality + console.info('๐Ÿ” DEBUG: Skipping auto-mapping of remaining locales to keep "Add Language" enabled'); - console.info('๐ŸŽฏ Final Auto-mapping Result:', autoMapping); - console.info(`โœ… Found ${Object.keys(autoMapping).length - 1} locale matches out of ${sourceLocale.length} source locales`); + const unmappedSources = sourceLocale.filter(source => !autoMapping[source.value]); + console.info(` Unmapped sources available for "Add Language": [${unmappedSources.map(s => s.value).join(', ')}]`); // Update Redux state with auto-mappings const newMigrationDataObj: INewMigration = { @@ -994,7 +940,7 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { }; dispatch(updateNewMigrationData(newMigrationDataObj)); - console.info('โœ… Redux state updated with auto-mapping'); + // ๐Ÿ” DEBUG: Auto-mapping completed for multi-locale // ๐Ÿ”ฅ CRITICAL FIX: Update existingLocale state for dropdown display // The dropdown reads from existingLocale, not from Redux localeMapping @@ -1129,39 +1075,47 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { // ๐Ÿ†• CONDITION 3: Intelligent Add Language functionality with auto-suggestions const addRowComp = () => { + console.info('๐Ÿ” DEBUG: Add Language button clicked'); setisStackChanged(false); // ๐Ÿ†• STEP 1: Get next unmapped source locale const nextUnmappedSource = getNextUnmappedSourceLocale(); + console.info('๐Ÿ” DEBUG: Next unmapped source locale:', nextUnmappedSource); if (!nextUnmappedSource) { - console.info('โš ๏ธ No more unmapped source locales available'); - // Fallback to original behavior if no unmapped locales + // Fallback: No more unmapped source locales available + console.warn('โš ๏ธ No more unmapped source locales available for "Add Language"'); setcmsLocaleOptions((prevList: { label: string; value: string }[]) => [ ...prevList, { - label: `${prevList.length}`, - value: '' + label: '', // ๐Ÿ”ง CRITICAL FIX: Empty label for manual selection + value: '' // Empty value for manual mapping } ]); return; } - console.info(`๐Ÿ†• Adding language row for next unmapped source locale: ${nextUnmappedSource.value}`); // ๐Ÿ†• STEP 2: Check if source locale exists in destination const destinationMatch = findDestinationMatch(nextUnmappedSource.value); + console.info('๐Ÿ” DEBUG: Destination match found:', destinationMatch); // ๐Ÿ†• STEP 3 & 4: Auto-map if exists, leave empty if not const newRowValue = destinationMatch ? destinationMatch.value : ''; + console.info(`๐Ÿ” DEBUG: Creating new row - Source: "${nextUnmappedSource.value}", Destination: "${newRowValue || 'empty'}"`); + + if (destinationMatch) { + console.info(`โœ… Auto-mapping: ${nextUnmappedSource.value} -> ${destinationMatch.value}`); + } else { + console.info(`โš ๏ธ No destination match for "${nextUnmappedSource.value}" - leaving destination empty for manual selection`); + } - console.info(` Destination match for ${nextUnmappedSource.value}:`, destinationMatch?.value || 'No match - leaving empty'); // Add new row with intelligent defaults setcmsLocaleOptions((prevList: { label: string; value: string }[]) => [ ...prevList, { - label: `${prevList.length}`, + label: nextUnmappedSource.value, // ๐Ÿ”ง CRITICAL FIX: Use actual source locale, not numeric index value: newRowValue // Auto-map or leave empty based on match } ]); @@ -1196,9 +1150,6 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { }; dispatch(updateNewMigrationData(newMigrationDataObj)); - console.info(`โœ… Auto-mapped: ${nextUnmappedSource.value} -> ${destinationMatch.value}`); - } else { - console.info(`๐Ÿ“ Left empty for manual selection: ${nextUnmappedSource.value}`); } }, 100); // Small delay to ensure state updates properly }; @@ -1254,23 +1205,36 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { // ๐Ÿ†• Enhanced disable logic: Check if all source locales are mapped or shown (() => { const totalSourceLocales = newMigrationData?.destination_stack?.sourceLocale?.length || 0; - const mappedLocalesCount = Object.keys(newMigrationData?.destination_stack?.localeMapping || {})?.length; + const localeMapping = newMigrationData?.destination_stack?.localeMapping || {}; const visibleRowsCount = cmsLocaleOptions?.length || 0; const isProjectCompleted = newMigrationData?.project_current_step > 2; - // Disable if: all source locales are mapped OR all source locales have rows OR project is completed - const shouldDisable = mappedLocalesCount >= totalSourceLocales + 1 || // +1 for master locale - visibleRowsCount >= totalSourceLocales + 1 || // +1 for master locale - isProjectCompleted; + // ๐Ÿ”ง CRITICAL FIX: Always disable for single locale - nothing more to add + if (totalSourceLocales <= 1) { + return true; // Single locale = disable "Add Language" button + } - console.info('๐Ÿ”ด Add Language Button Status:', { + // ๐Ÿ”ง CRITICAL FIX: Count only actual source locale mappings, not master locale entries + const actualMappedSourceLocales = Object.keys(localeMapping).filter(key => + !key.includes('-master_locale') && // Exclude master locale entries + localeMapping[key] !== '' && // Exclude empty mappings + localeMapping[key] !== null && // Exclude null mappings + localeMapping[key] !== undefined // Exclude undefined mappings + ).length; + + console.info('๐Ÿ” DEBUG: Add Language Button Logic:', { totalSourceLocales, - mappedLocalesCount, + actualMappedSourceLocales, visibleRowsCount, - isProjectCompleted, - shouldDisable + localeMapping }); + // Disable if: all source locales are mapped OR all source locales have visible rows OR project is completed + const shouldDisable = actualMappedSourceLocales >= totalSourceLocales || + visibleRowsCount >= totalSourceLocales || + isProjectCompleted; + + return shouldDisable; })() } @@ -1284,26 +1248,7 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { if (unmappedDestinations.length > 0 && newMigrationData?.project_current_step <= 2) { return ( -
- } - content={ -
- Available destination locales not yet mapped: -
- - {unmappedDestinations.map(locale => locale.value).join(', ')} - -
- - You can manually connect these by clicking “Add Language” if needed. - -
- } - type="light" - /> -
+ <> ); } return null; diff --git a/ui/src/pages/Migration/index.tsx b/ui/src/pages/Migration/index.tsx index 0b36c99dd..648b3180b 100644 --- a/ui/src/pages/Migration/index.tsx +++ b/ui/src/pages/Migration/index.tsx @@ -597,17 +597,37 @@ const Migration = () => { const handleOnClickDestinationStack = async (event: MouseEvent) => { setIsLoading(true); + // ๐Ÿ” DEBUG: Log the actual localeMapping being validated + console.info('๐Ÿ” DEBUG: About to validate localeMapping:', JSON.stringify(newMigrationData?.destination_stack?.localeMapping, null, 2)); + const hasNonEmptyMapping = newMigrationData?.destination_stack?.localeMapping && Object.entries(newMigrationData?.destination_stack?.localeMapping || {})?.every( - ([label, value]: [string, string]) => - Boolean(label?.trim()) && - value !== '' && - value !== null && - value !== undefined && - label !== 'undefined' && - isNaN(Number(label)) + ([label, value]: [string, string]) => { + const conditions = { + hasLabel: Boolean(label?.trim()), + notEmptyValue: value !== '', + notNullValue: value !== null, + notUndefinedValue: value !== undefined, + labelNotUndefined: label !== 'undefined', + labelNotNumeric: isNaN(Number(label)) + }; + + console.info(`๐Ÿ” DEBUG: Validating entry [${label}] = "${value}":`, conditions); + + const passes = conditions.hasLabel && + conditions.notEmptyValue && + conditions.notNullValue && + conditions.notUndefinedValue && + conditions.labelNotUndefined && + conditions.labelNotNumeric; + + console.info(`๐Ÿ” DEBUG: Entry result: ${passes ? 'โœ… PASS' : 'โŒ FAIL'}`); + return passes; + } ); + + console.info('๐Ÿ” DEBUG: Final hasNonEmptyMapping result:', hasNonEmptyMapping); const master_locale: LocalesType = {}; const locales: LocalesType = {}; From 5ccbeeaf6a8858af3278e038b504c9ec46db8ab7 Mon Sep 17 00:00:00 2001 From: sauravraw Date: Mon, 22 Sep 2025 11:16:08 +0530 Subject: [PATCH 04/37] locale code update --- api/test-drupal-locales.js | 150 ---------- api/test-duplicate-fix-simple.js | 316 -------------------- api/test-duplicate-fix-with-db.js | 476 ------------------------------ api/test-duplicate-removal.js | 123 -------- api/test-uid-format.js | 71 ----- 5 files changed, 1136 deletions(-) delete mode 100644 api/test-drupal-locales.js delete mode 100644 api/test-duplicate-fix-simple.js delete mode 100644 api/test-duplicate-fix-with-db.js delete mode 100644 api/test-duplicate-removal.js delete mode 100644 api/test-uid-format.js diff --git a/api/test-drupal-locales.js b/api/test-drupal-locales.js deleted file mode 100644 index 00ae177ec..000000000 --- a/api/test-drupal-locales.js +++ /dev/null @@ -1,150 +0,0 @@ -import { createLocale } from './src/services/drupal/locales.service.js'; -import mysql from 'mysql2/promise'; -import axios from 'axios'; -import fs from 'fs'; -import path from 'path'; - -// Test configuration using upload-api database (riceuniversity2) -const testProject = { - mysql: { - host: 'localhost', - user: 'root', - password: '', - database: 'riceuniversity2', - port: 3306 - } -}; - -const testDestinationStackId = 'test-drupal-locale-stack'; -const testProjectId = 'test-project-123'; - -async function testDrupalLocaleQueries() { - console.log('๐Ÿงช Testing Drupal Locale SQL Queries...'); - console.log('๐Ÿ“Š Database:', testProject.mysql.database); - - let connection; - - try { - // Create database connection - connection = await mysql.createConnection(testProject.mysql); - console.log('โœ… Database connection established'); - - // 1. Test master locale query - console.log('\n๐Ÿ” Testing Master Locale Query...'); - const masterLocaleQuery = ` - SELECT SUBSTRING_INDEX( - SUBSTRING_INDEX(CONVERT(data USING utf8), 'default_langcode";s:2:"', -1), - '"', 1 - ) as master_locale - FROM config - WHERE name = 'system.site' - `; - - const [masterRows] = await connection.execute(masterLocaleQuery); - const masterLocaleCode = masterRows[0]?.master_locale || 'en'; - console.log('โœ… Master Locale:', masterLocaleCode); - - // 2. Test all locales query - console.log('\n๐Ÿ” Testing All Locales Query...'); - const allLocalesQuery = ` - SELECT DISTINCT langcode - FROM node_field_data - WHERE langcode IS NOT NULL AND langcode != '' - ORDER BY langcode - `; - - const [allLocaleRows] = await connection.execute(allLocalesQuery); - const allLocaleCodes = allLocaleRows.map(row => row.langcode); - console.log('โœ… All Locales:', allLocaleCodes); - - // 3. Test non-master locales query - console.log('\n๐Ÿ” Testing Non-Master Locales Query...'); - const nonMasterLocalesQuery = ` - SELECT DISTINCT n.langcode - FROM node_field_data n - WHERE n.langcode IS NOT NULL - AND n.langcode != '' - AND n.langcode != ( - SELECT - SUBSTRING_INDEX( - SUBSTRING_INDEX(CONVERT(data USING utf8), 'default_langcode";s:2:"', -1), - '"', - 1 - ) - FROM config - WHERE name = 'system.site' - LIMIT 1 - ) - ORDER BY n.langcode - `; - - const [nonMasterRows] = await connection.execute(nonMasterLocalesQuery); - const nonMasterLocaleCodes = nonMasterRows.map(row => row.langcode); - console.log('โœ… Non-Master Locales:', nonMasterLocaleCodes); - - await connection.end(); - - // 4. Test Contentstack API - console.log('\n๐Ÿ” Testing Contentstack Locales API...'); - try { - const response = await axios.get('https://app.contentstack.com/api/v3/locales?include_all=true'); - const contentstackLocales = response.data?.locales || {}; - const localeCount = Object.keys(contentstackLocales).length; - console.log('โœ… Contentstack API Response:', `${localeCount} locales fetched`); - - // Test locale name lookup for found codes - console.log('\n๐Ÿ” Testing Locale Name Mapping...'); - allLocaleCodes.forEach(code => { - const name = contentstackLocales[code] || contentstackLocales[code.toLowerCase()] || 'Unknown'; - console.log(` ${code} โ†’ ${name}`); - }); - - } catch (apiError) { - console.error('โŒ Contentstack API Error:', apiError.message); - } - - // 5. Test the actual createLocale function - console.log('\n๐Ÿ” Testing createLocale Function...'); - try { - await createLocale(testDestinationStackId, testProjectId, testProject); - console.log('โœ… createLocale function executed successfully'); - - // Check if files were created - const localesDir = path.join('./cmsMigrationData', testDestinationStackId, 'locales'); - console.log('๐Ÿ“‚ Checking directory:', localesDir); - - if (fs.existsSync(localesDir)) { - const files = fs.readdirSync(localesDir); - console.log('๐Ÿ“ Created files:', files); - - // Read and display each file - files.forEach(file => { - const filePath = path.join(localesDir, file); - const content = JSON.parse(fs.readFileSync(filePath, 'utf8')); - console.log(`\n๐Ÿ“„ ${file}:`, JSON.stringify(content, null, 2)); - }); - } else { - console.log('โŒ Locales directory not found'); - } - - } catch (createError) { - console.error('โŒ createLocale function failed:', createError); - console.error('Stack trace:', createError.stack); - } - - console.log('\nโœ… All tests completed!'); - - } catch (error) { - console.error('โŒ Test failed:', error); - console.error('Stack trace:', error.stack); - } -} - -// Run the test -testDrupalLocaleQueries().then(() => { - console.log('\n๐Ÿ Test completed'); - process.exit(0); -}).catch((error) => { - console.error('๐Ÿ’ฅ Test crashed:', error); - process.exit(1); -}); diff --git a/api/test-duplicate-fix-simple.js b/api/test-duplicate-fix-simple.js deleted file mode 100644 index e1af7d18e..000000000 --- a/api/test-duplicate-fix-simple.js +++ /dev/null @@ -1,316 +0,0 @@ -#!/usr/bin/env node - -/** - * Simple test script to verify duplicate field fix - * Tests the core duplicate removal logic without complex imports - */ - -const fs = require('fs'); -const path = require('path'); - -console.log('๐Ÿงช Testing Duplicate Field Fix - Simple Version\n'); -console.log('===============================================\n'); - -// Simulate the exact duplicate removal logic from entries.service.ts -function testDuplicateRemovalLogic(mergedData) { - console.log('๐Ÿ“‹ Input Data (with potential duplicates):'); - console.log(JSON.stringify(mergedData, null, 2)); - console.log(''); - - // Apply the exact cleanup logic from the service - const cleanedEntry = {}; - for (const [key, val] of Object.entries(mergedData)) { - if (val !== null && val !== undefined && val !== '') { - // Check if this is a suffixed field (_value, _status, _uri) and if a non-suffixed version exists - const isValueField = key.endsWith('_value'); - const isStatusField = key.endsWith('_status'); - const isUriField = key.endsWith('_uri'); - - if (isValueField) { - const baseFieldName = key.replace('_value', ''); - // Only include the _value field if the base field doesn't exist - if (!mergedData.hasOwnProperty(baseFieldName)) { - cleanedEntry[key] = val; - console.log(`โœ… Keeping _value field: ${key} (no base field found)`); - } else { - console.log( - `๐Ÿ—‘๏ธ Removing _value field: ${key} (base field ${baseFieldName} exists)` - ); - } - // If base field exists, skip the _value field (base field takes priority) - } else if (isStatusField) { - const baseFieldName = key.replace('_status', ''); - // Only include the _status field if the base field doesn't exist - if (!mergedData.hasOwnProperty(baseFieldName)) { - cleanedEntry[key] = val; - console.log(`โœ… Keeping _status field: ${key} (no base field found)`); - } else { - console.log( - `๐Ÿ—‘๏ธ Removing _status field: ${key} (base field ${baseFieldName} exists)` - ); - } - // If base field exists, skip the _status field (base field takes priority) - } else if (isUriField) { - const baseFieldName = key.replace('_uri', ''); - // Only include the _uri field if the base field doesn't exist - if (!mergedData.hasOwnProperty(baseFieldName)) { - cleanedEntry[key] = val; - console.log(`โœ… Keeping _uri field: ${key} (no base field found)`); - } else { - console.log( - `๐Ÿ—‘๏ธ Removing _uri field: ${key} (base field ${baseFieldName} exists)` - ); - } - // If base field exists, skip the _uri field (base field takes priority) - } else { - // For non-suffixed fields, always include them - cleanedEntry[key] = val; - console.log(`โœ… Keeping base field: ${key}`); - } - } - } - - console.log('\n๐ŸŽฏ Output Data (duplicates removed):'); - console.log(JSON.stringify(cleanedEntry, null, 2)); - console.log(''); - - return cleanedEntry; -} - -// Test cases based on your actual entry structure -function runTestCases() { - const testCases = [ - { - name: 'Your Actual Entry Structure', - data: { - nid: 496, - title: 'Creativity: 13 Ways to Look at It', - langcode: 'en', - created: '2019-09-09T13:09:29.000Z', - type: 'article', - - // These are the duplicates from your actual file - body_value: { - type: 'doc', - uid: '025c76b8546a46b9aacfbea3a4ea15b3', - attrs: {}, - children: [ - { - type: 'p', - attrs: {}, - uid: 'f63185c9190d4f5aac92ada099e08d3c', - children: [ - { - text: 'Every day, we see the spark of imagination and invention...', - }, - ], - }, - ], - }, - body: { - type: 'doc', - uid: '0fa04056a606443f8e3417086b88cd38', - attrs: {}, - children: [ - { - type: 'p', - attrs: {}, - uid: '6e0be5417c9048d1a2c36ed42d242813', - children: [ - { - text: 'Every day, we see the spark of imagination and invention...', - }, - ], - }, - ], - }, - field_external_link_value: - 'https://magazine.rice.edu/2019/05/13-ways-of-looking-at-creativity/?utm_source=Hero&utm_medium=thirteen&utm_campaign=Hero%20Links', - field_external_link: - 'https://magazine.rice.edu/2019/05/13-ways-of-looking-at-creativity/?utm_source=Hero&utm_medium=thirteen&utm_campaign=Hero%20Links', - field_formatted_title_value: '

Creativity

\r\n', - field_formatted_title: { - type: 'doc', - uid: '2b5aac7ee1e44194a8d41ab806753677', - attrs: {}, - children: [ - { - type: 'p', - attrs: {}, - uid: 'cce203a33e0640d7a7ab389055638635', - children: [ - { text: 'Creativity', attrs: { style: {} }, bold: true }, - ], - }, - ], - }, - field_subtitle_value: '13 WAYS TO LOOK AT IT', - field_subtitle: '13 WAYS TO LOOK AT IT', - uid: 'content_type_entries_title_496', - locale: 'en', - }, - }, - { - name: 'Mixed Scenarios', - data: { - // Case 1: Both _value and base exist (should remove _value) - test_field_value: 'value version', - test_field: 'base version', - - // Case 2: Only _value exists (should keep _value) - only_value_field_value: 'only value exists', - - // Case 3: Only base exists (should keep base) - only_base_field: 'only base exists', - - // Case 4: Status fields - comment_status: 1, - comment: 'processed comment', - - // Case 5: URI fields - link_uri: 'https://example.com', - link: { title: 'Example', href: 'https://example.com' }, - - // Regular fields - title: 'Test Title', - nid: 123, - }, - }, - ]; - - console.log('๐Ÿงช Running Test Cases\n'); - - let allTestsPassed = true; - - testCases.forEach((testCase, index) => { - console.log(`TEST ${index + 1}: ${testCase.name}`); - console.log('='.repeat(testCase.name.length + 10)); - - const result = testDuplicateRemovalLogic(testCase.data); - - // Check for duplicates - const fieldNames = Object.keys(result); - const duplicates = []; - - fieldNames.forEach((field) => { - if (field.endsWith('_value')) { - const baseField = field.replace('_value', ''); - if (fieldNames.includes(baseField)) { - duplicates.push({ suffix: field, base: baseField }); - } - } else if (field.endsWith('_status')) { - const baseField = field.replace('_status', ''); - if (fieldNames.includes(baseField)) { - duplicates.push({ suffix: field, base: baseField }); - } - } else if (field.endsWith('_uri')) { - const baseField = field.replace('_uri', ''); - if (fieldNames.includes(baseField)) { - duplicates.push({ suffix: field, base: baseField }); - } - } - }); - - console.log('๐Ÿ“Š Test Results:'); - console.log(` Input fields: ${Object.keys(testCase.data).length}`); - console.log(` Output fields: ${fieldNames.length}`); - console.log(` Duplicates found: ${duplicates.length}`); - - if (duplicates.length === 0) { - console.log(' โœ… PASSED - No duplicates found!'); - } else { - console.log(' โŒ FAILED - Duplicates still exist:'); - duplicates.forEach((dup) => { - console.log(` - ${dup.suffix} + ${dup.base}`); - }); - allTestsPassed = false; - } - - console.log('\n'); - }); - - return allTestsPassed; -} - -// Test the specific case from your JSON file -function testYourSpecificCase() { - console.log('๐ŸŽฏ Testing Your Specific Case\n'); - console.log('=============================\n'); - - // This is exactly what should be in your JSON after the fix - const expectedCleanResult = { - nid: 496, - title: 'Creativity: 13 Ways to Look at It', - langcode: 'en', - created: '2019-09-09T13:09:29.000Z', - type: 'article', - uid: 'content_type_entries_title_496', - locale: 'en', - // Only these should remain (no _value versions) - body: { - /* processed content */ - }, - field_external_link: - 'https://magazine.rice.edu/2019/05/13-ways-of-looking-at-creativity/?utm_source=Hero&utm_medium=thirteen&utm_campaign=Hero%20Links', - field_formatted_title: { - /* processed content */ - }, - field_subtitle: '13 WAYS TO LOOK AT IT', - }; - - console.log( - 'โœ… Expected Clean Result (what you should see after migration):' - ); - console.log('Fields that should exist:'); - Object.keys(expectedCleanResult).forEach((field) => { - console.log(` - ${field}`); - }); - - console.log('\nโŒ Fields that should NOT exist:'); - console.log(' - body_value'); - console.log(' - field_external_link_value'); - console.log(' - field_formatted_title_value'); - console.log(' - field_subtitle_value'); - - console.log('\n๐Ÿ’ก After running migration, check your JSON file:'); - console.log( - ' /Users/saurav.upadhyay/Expert Service/Contentstack Migration/migration-v2/api/cmsMigrationData/blta4dadbc9c65d73cb/entries/article/en/en.json' - ); - console.log( - ' It should only contain the fields listed above, not the _value versions.' - ); -} - -// Main execution -function main() { - console.log('๐Ÿš€ Starting Duplicate Field Fix Tests\n'); - - const testResults = runTestCases(); - - testYourSpecificCase(); - - console.log('\n๐Ÿ“Š FINAL RESULTS'); - console.log('================'); - console.log(`All tests passed: ${testResults ? 'โœ… YES' : 'โŒ NO'}`); - - if (testResults) { - console.log( - '\n๐ŸŽ‰ SUCCESS: The duplicate removal logic is working correctly!' - ); - console.log('โœ… The fix should work when you run the full migration.'); - console.log('\n๐Ÿš€ Next Steps:'); - console.log(' 1. Run the migration from the UI'); - console.log(' 2. Check the generated JSON files'); - console.log( - ' 3. Verify no _value suffixes remain when base fields exist' - ); - } else { - console.log('\nโŒ FAILURE: The logic needs more work.'); - console.log('โš ๏ธ Do not run the full migration yet.'); - } - - process.exit(testResults ? 0 : 1); -} - -// Run the test -main(); diff --git a/api/test-duplicate-fix-with-db.js b/api/test-duplicate-fix-with-db.js deleted file mode 100644 index fb7375850..000000000 --- a/api/test-duplicate-fix-with-db.js +++ /dev/null @@ -1,476 +0,0 @@ -#!/usr/bin/env node - -/** - * Test script to verify duplicate field fix using actual database and processing logic - * This will test the entries.service.ts fix without running the full UI migration - */ - -import fs from 'fs'; -import path from 'path'; -import mysql from 'mysql2'; -import { fileURLToPath } from 'url'; - -// Get current directory for ES modules -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); - -// Import the entries service (we'll need to adjust the import path) -const entriesServicePath = path.join( - __dirname, - 'src/services/drupal/entries.service.ts' -); - -console.log('๐Ÿงช Testing Duplicate Field Fix with Real Database Data\n'); -console.log('==================================================\n'); - -// Database configuration (you may need to adjust these) -const DB_CONFIG = { - host: process.env.DB_HOST || 'localhost', - user: process.env.DB_USER || 'root', - password: process.env.DB_PASSWORD || '', - database: process.env.DB_NAME || 'drupal_db', - port: process.env.DB_PORT || 3306, -}; - -console.log('๐Ÿ“‹ Database Configuration:'); -console.log(` Host: ${DB_CONFIG.host}`); -console.log(` Database: ${DB_CONFIG.database}`); -console.log(` User: ${DB_CONFIG.user}`); -console.log(''); - -// Create database connection -let connection; - -try { - connection = mysql.createConnection(DB_CONFIG); - console.log('โœ… Database connection established'); -} catch (error) { - console.error('โŒ Failed to connect to database:', error.message); - process.exit(1); -} - -// Test the processFieldData function directly -async function testProcessFieldData() { - console.log('\n๐Ÿ” Testing processFieldData function...\n'); - - // Sample Drupal entry data that would cause duplicates - const testEntryData = { - nid: 496, - title: 'Test Article', - langcode: 'en', - created: 1568032169, - type: 'article', - - // These fields should cause duplicates in the old logic - body_value: '

This is the body content from _value field

', - field_formatted_title_value: '

Formatted Title

', - field_external_link_value: 'https://example.com/test-link', - field_subtitle_value: 'Test Subtitle', - - // Some fields without _value counterparts - field_image_target_id: 123, - field_tags_target_id: 456, - }; - - console.log('๐Ÿ“‹ Input Entry Data:'); - console.log(JSON.stringify(testEntryData, null, 2)); - console.log(''); - - // Mock field configurations (simplified) - const mockFieldConfigs = [ - { field_name: 'body', field_type: 'text_with_summary' }, - { field_name: 'field_formatted_title', field_type: 'text' }, - { field_name: 'field_external_link', field_type: 'link' }, - { field_name: 'field_subtitle', field_type: 'text' }, - { field_name: 'field_image', field_type: 'image' }, - { field_name: 'field_tags', field_type: 'entity_reference' }, - ]; - - // Mock other required parameters - const mockAssetId = {}; - const mockReferenceId = {}; - const mockTaxonomyId = {}; - const mockTaxonomyFieldMapping = {}; - const mockReferenceFieldMapping = {}; - const mockAssetFieldMapping = {}; - const mockTaxonomyReferenceLookup = {}; - const contentType = 'article'; - - try { - // Import and test the processFieldData function - // Note: We'll need to create a simplified version since the actual import might be complex - const processedEntry = await simulateProcessFieldData( - testEntryData, - mockFieldConfigs, - mockAssetId, - mockReferenceId, - mockTaxonomyId, - mockTaxonomyFieldMapping, - mockReferenceFieldMapping, - mockAssetFieldMapping, - mockTaxonomyReferenceLookup, - contentType - ); - - console.log('๐ŸŽฏ Processed Entry Data:'); - console.log(JSON.stringify(processedEntry, null, 2)); - console.log(''); - - // Check for duplicates - const fieldNames = Object.keys(processedEntry); - const duplicateCheck = { - body: { - hasBase: fieldNames.includes('body'), - hasValue: fieldNames.includes('body_value'), - isDuplicate: - fieldNames.includes('body') && fieldNames.includes('body_value'), - }, - field_formatted_title: { - hasBase: fieldNames.includes('field_formatted_title'), - hasValue: fieldNames.includes('field_formatted_title_value'), - isDuplicate: - fieldNames.includes('field_formatted_title') && - fieldNames.includes('field_formatted_title_value'), - }, - field_external_link: { - hasBase: fieldNames.includes('field_external_link'), - hasValue: fieldNames.includes('field_external_link_value'), - isDuplicate: - fieldNames.includes('field_external_link') && - fieldNames.includes('field_external_link_value'), - }, - field_subtitle: { - hasBase: fieldNames.includes('field_subtitle'), - hasValue: fieldNames.includes('field_subtitle_value'), - isDuplicate: - fieldNames.includes('field_subtitle') && - fieldNames.includes('field_subtitle_value'), - }, - }; - - console.log('๐Ÿ” Duplicate Analysis:'); - console.log('====================='); - - let totalDuplicates = 0; - for (const [fieldName, check] of Object.entries(duplicateCheck)) { - console.log(`${fieldName}:`); - console.log(` Has base field: ${check.hasBase ? 'โœ…' : 'โŒ'}`); - console.log(` Has _value field: ${check.hasValue ? 'โœ…' : 'โŒ'}`); - console.log( - ` Is duplicate: ${ - check.isDuplicate ? 'โŒ YES (BAD)' : 'โœ… NO (GOOD)' - }` - ); - console.log(''); - - if (check.isDuplicate) { - totalDuplicates++; - } - } - - console.log('๐Ÿ“Š Summary:'); - console.log(` Total fields: ${fieldNames.length}`); - console.log(` Duplicate fields found: ${totalDuplicates}`); - console.log( - ` Fix working: ${totalDuplicates === 0 ? 'โœ… YES' : 'โŒ NO'}` - ); - - if (totalDuplicates === 0) { - console.log( - '\n๐ŸŽ‰ SUCCESS: No duplicate fields found! The fix is working correctly.' - ); - } else { - console.log( - '\nโŒ FAILURE: Duplicate fields still exist. The fix needs adjustment.' - ); - } - - return totalDuplicates === 0; - } catch (error) { - console.error('โŒ Error processing entry data:', error.message); - return false; - } -} - -// Simplified version of processFieldData for testing -async function simulateProcessFieldData( - entryData, - fieldConfigs, - assetId, - referenceId, - taxonomyId, - taxonomyFieldMapping, - referenceFieldMapping, - assetFieldMapping, - taxonomyReferenceLookup, - contentType -) { - const fieldNames = Object.keys(entryData); - const isoDate = new Date(); - const processedData = {}; - const skippedFields = new Set(); - const processedFields = new Set(); - - // First loop: Process special field types - for (const [dataKey, value] of Object.entries(entryData)) { - // Handle basic field processing (simplified) - if (value === null || value === undefined || value === '') { - continue; - } - - // Default case: copy field to processedData if it wasn't handled by special processing above - if (!(dataKey in processedData)) { - processedData[dataKey] = value; - } - } - - // Second loop: Process standard field transformations - const ctValue = {}; - - for (const fieldName of fieldNames) { - if (skippedFields.has(fieldName)) { - continue; - } - - const value = entryData[fieldName]; - - if (fieldName === 'created') { - ctValue[fieldName] = new Date(value * 1000).toISOString(); - } else if (fieldName === 'nid') { - ctValue.uid = `content_type_entries_title_${value}`; - } else if (fieldName === 'langcode') { - ctValue.locale = value || 'en-us'; - } else if (fieldName.endsWith('_value')) { - // Skip if this field was already processed in the main loop (avoid duplicates) - const baseFieldName = fieldName.replace('_value', ''); - if ( - processedFields.has(fieldName) || - processedFields.has(baseFieldName) - ) { - continue; - } - - // Process HTML content - if (/<\/?[a-z][\s\S]*>/i.test(value)) { - // Simulate HTML to JSON conversion (simplified) - ctValue[baseFieldName] = { - type: 'doc', - children: [ - { type: 'p', children: [{ text: value.replace(/<[^>]*>/g, '') }] }, - ], - }; - } else { - ctValue[baseFieldName] = value; - } - - // Mark both the original and base field as processed to avoid duplicates - processedFields.add(fieldName); - processedFields.add(baseFieldName); - } else if (fieldName.endsWith('_status')) { - const baseFieldName = fieldName.replace('_status', ''); - if ( - processedFields.has(fieldName) || - processedFields.has(baseFieldName) - ) { - continue; - } - - ctValue[baseFieldName] = value; - processedFields.add(fieldName); - processedFields.add(baseFieldName); - } else { - ctValue[fieldName] = value; - } - } - - // Apply processed field data, but prioritize ctValue over processedData - const mergedData = { ...processedData, ...ctValue }; - - // Final cleanup: remove duplicates and null values - const cleanedEntry = {}; - for (const [key, val] of Object.entries(mergedData)) { - if (val !== null && val !== undefined && val !== '') { - // Check if this is a suffixed field and if a non-suffixed version exists - const isValueField = key.endsWith('_value'); - const isStatusField = key.endsWith('_status'); - const isUriField = key.endsWith('_uri'); - - if (isValueField) { - const baseFieldName = key.replace('_value', ''); - // Only include the _value field if the base field doesn't exist - if (!mergedData.hasOwnProperty(baseFieldName)) { - cleanedEntry[key] = val; - } - } else if (isStatusField) { - const baseFieldName = key.replace('_status', ''); - if (!mergedData.hasOwnProperty(baseFieldName)) { - cleanedEntry[key] = val; - } - } else if (isUriField) { - const baseFieldName = key.replace('_uri', ''); - if (!mergedData.hasOwnProperty(baseFieldName)) { - cleanedEntry[key] = val; - } - } else { - // For non-suffixed fields, always include them - cleanedEntry[key] = val; - } - } - } - - return cleanedEntry; -} - -// Test with real database data -async function testWithRealDatabaseData() { - console.log('\n๐Ÿ” Testing with Real Database Data...\n'); - - return new Promise((resolve) => { - // Query to get a sample entry with _value fields - const query = ` - SELECT n.nid, n.title, n.langcode, n.created, n.type, - bd.body_value, bd.body_summary, - fft.field_formatted_title_value, - fel.field_external_link_value, - fs.field_subtitle_value - FROM node n - LEFT JOIN node__body bd ON n.nid = bd.entity_id - LEFT JOIN node__field_formatted_title fft ON n.nid = fft.entity_id - LEFT JOIN node__field_external_link fel ON n.nid = fel.entity_id - LEFT JOIN node__field_subtitle fs ON n.nid = fs.entity_id - WHERE n.type = 'article' - AND n.status = 1 - LIMIT 1 - `; - - connection.query(query, async (error, results) => { - if (error) { - console.error('โŒ Database query failed:', error.message); - resolve(false); - return; - } - - if (results.length === 0) { - console.log('โš ๏ธ No article entries found in database'); - resolve(false); - return; - } - - const dbEntry = results[0]; - console.log('๐Ÿ“‹ Raw Database Entry:'); - console.log(JSON.stringify(dbEntry, null, 2)); - console.log(''); - - // Process this real data through our function - try { - const processedEntry = await simulateProcessFieldData( - dbEntry, - [], // Empty field configs for simplicity - {}, - {}, - {}, - {}, - {}, - {}, - {}, - 'article' - ); - - console.log('๐ŸŽฏ Processed Real Entry:'); - console.log(JSON.stringify(processedEntry, null, 2)); - console.log(''); - - // Check for duplicates in real data - const fieldNames = Object.keys(processedEntry); - const duplicates = []; - - fieldNames.forEach((field) => { - if (field.endsWith('_value')) { - const baseField = field.replace('_value', ''); - if (fieldNames.includes(baseField)) { - duplicates.push({ suffix: field, base: baseField }); - } - } - }); - - console.log('๐Ÿ” Real Data Duplicate Check:'); - console.log(` Total fields: ${fieldNames.length}`); - console.log(` Duplicates found: ${duplicates.length}`); - - if (duplicates.length === 0) { - console.log(' โœ… No duplicates found in real data!'); - resolve(true); - } else { - console.log(' โŒ Duplicates found:'); - duplicates.forEach((dup) => { - console.log(` - ${dup.suffix} + ${dup.base}`); - }); - resolve(false); - } - } catch (error) { - console.error( - 'โŒ Error processing real database entry:', - error.message - ); - resolve(false); - } - }); - }); -} - -// Main test execution -async function runTests() { - console.log('๐Ÿš€ Starting Duplicate Field Fix Tests\n'); - - // Test 1: Simulated data - console.log('TEST 1: Simulated Entry Data'); - console.log('============================='); - const simulatedTest = await testProcessFieldData(); - - // Test 2: Real database data - console.log('\nTEST 2: Real Database Data'); - console.log('=========================='); - const realDataTest = await testWithRealDatabaseData(); - - // Summary - console.log('\n๐Ÿ“Š FINAL TEST RESULTS'); - console.log('====================='); - console.log( - `Simulated data test: ${simulatedTest ? 'โœ… PASSED' : 'โŒ FAILED'}` - ); - console.log( - `Real database test: ${realDataTest ? 'โœ… PASSED' : 'โŒ FAILED'}` - ); - - const allTestsPassed = simulatedTest && realDataTest; - console.log( - `\nOverall result: ${ - allTestsPassed ? '๐ŸŽ‰ ALL TESTS PASSED' : 'โŒ TESTS FAILED' - }` - ); - - if (allTestsPassed) { - console.log('\nโœ… The duplicate field fix is working correctly!'); - console.log('โœ… You can safely run the full migration from the UI.'); - } else { - console.log( - '\nโŒ The fix needs more work before running the full migration.' - ); - } - - // Close database connection - if (connection) { - connection.end(); - } - - process.exit(allTestsPassed ? 0 : 1); -} - -// Run the tests -runTests().catch((error) => { - console.error('โŒ Test execution failed:', error); - if (connection) { - connection.end(); - } - process.exit(1); -}); diff --git a/api/test-duplicate-removal.js b/api/test-duplicate-removal.js deleted file mode 100644 index 8bf227dc8..000000000 --- a/api/test-duplicate-removal.js +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env node - -/** - * Test script to verify duplicate field removal logic - */ - -console.log('๐Ÿงช Testing Duplicate Field Removal Logic\n'); - -// Simulate the mergedData that would contain duplicates -const testMergedData = { - // Duplicate case 1: both body_value and body exist - body_value: 'content from _value field', - body: 'content from base field', - - // Duplicate case 2: both field_formatted_title_value and field_formatted_title exist - field_formatted_title_value: 'title from _value field', - field_formatted_title: 'title from base field', - - // Duplicate case 3: both field_external_link_value and field_external_link exist - field_external_link_value: 'link from _value field', - field_external_link: 'link from base field', - - // Non-duplicate case: only _value field exists - field_only_value_exists_value: 'only value field', - - // Non-duplicate case: only base field exists - field_only_base_exists: 'only base field', - - // Other fields - title: 'Article Title', - nid: 496, - created: '2019-09-09T13:09:29.000Z', -}; - -console.log('๐Ÿ“‹ Input (with duplicates):'); -console.log(JSON.stringify(testMergedData, null, 2)); - -// Apply the duplicate removal logic -const cleanedEntry = {}; -for (const [key, val] of Object.entries(testMergedData)) { - if (val !== null && val !== undefined && val !== '') { - // Check if this is a suffixed field (_value, _status, _uri) and if a non-suffixed version exists - const isValueField = key.endsWith('_value'); - const isStatusField = key.endsWith('_status'); - const isUriField = key.endsWith('_uri'); - - if (isValueField) { - const baseFieldName = key.replace('_value', ''); - // Only include the _value field if the base field doesn't exist - if (!testMergedData.hasOwnProperty(baseFieldName)) { - cleanedEntry[key] = val; - console.log(`โœ… Keeping _value field: ${key} (no base field found)`); - } else { - console.log( - `๐Ÿ—‘๏ธ Removing _value field: ${key} (base field ${baseFieldName} exists)` - ); - } - // If base field exists, skip the _value field (base field takes priority) - } else if (isStatusField) { - const baseFieldName = key.replace('_status', ''); - // Only include the _status field if the base field doesn't exist - if (!testMergedData.hasOwnProperty(baseFieldName)) { - cleanedEntry[key] = val; - console.log(`โœ… Keeping _status field: ${key} (no base field found)`); - } else { - console.log( - `๐Ÿ—‘๏ธ Removing _status field: ${key} (base field ${baseFieldName} exists)` - ); - } - // If base field exists, skip the _status field (base field takes priority) - } else if (isUriField) { - const baseFieldName = key.replace('_uri', ''); - // Only include the _uri field if the base field doesn't exist - if (!testMergedData.hasOwnProperty(baseFieldName)) { - cleanedEntry[key] = val; - console.log(`โœ… Keeping _uri field: ${key} (no base field found)`); - } else { - console.log( - `๐Ÿ—‘๏ธ Removing _uri field: ${key} (base field ${baseFieldName} exists)` - ); - } - // If base field exists, skip the _uri field (base field takes priority) - } else { - // For non-suffixed fields, always include them - cleanedEntry[key] = val; - console.log(`โœ… Keeping base field: ${key}`); - } - } -} - -console.log('\n๐ŸŽฏ Output (duplicates removed):'); -console.log(JSON.stringify(cleanedEntry, null, 2)); - -console.log('\n๐Ÿ“Š Summary:'); -console.log(`Input fields: ${Object.keys(testMergedData).length}`); -console.log(`Output fields: ${Object.keys(cleanedEntry).length}`); -console.log( - `Removed duplicates: ${ - Object.keys(testMergedData).length - Object.keys(cleanedEntry).length - }` -); - -// Verify no duplicates remain -const hasBodyDuplicate = - cleanedEntry.hasOwnProperty('body') && - cleanedEntry.hasOwnProperty('body_value'); -const hasTitleDuplicate = - cleanedEntry.hasOwnProperty('field_formatted_title') && - cleanedEntry.hasOwnProperty('field_formatted_title_value'); -const hasLinkDuplicate = - cleanedEntry.hasOwnProperty('field_external_link') && - cleanedEntry.hasOwnProperty('field_external_link_value'); - -console.log(`\nโœ… Verification:`); -console.log( - `Body duplicate removed: ${!hasBodyDuplicate ? 'โœ… YES' : 'โŒ NO'}` -); -console.log( - `Title duplicate removed: ${!hasTitleDuplicate ? 'โœ… YES' : 'โŒ NO'}` -); -console.log( - `Link duplicate removed: ${!hasLinkDuplicate ? 'โœ… YES' : 'โŒ NO'}` -); diff --git a/api/test-uid-format.js b/api/test-uid-format.js deleted file mode 100644 index c57ac56d5..000000000 --- a/api/test-uid-format.js +++ /dev/null @@ -1,71 +0,0 @@ -// Test the new UID format and JSON structure - -const testLocales = ['en', 'und', 'fr-fr', 'es-mx']; -const masterLocale = 'en'; - -console.log('๐Ÿงช Testing New UID Format and JSON Structure...\n'); - -// Test UID generation -console.log('๐Ÿ” UID Generation:'); -testLocales.forEach(langcode => { - const uid = `drupallocale_${langcode.toLowerCase().replace(/-/g, '_')}`; - console.log(` ${langcode} โ†’ ${uid}`); -}); - -console.log('\n๐Ÿ“„ Expected JSON Output:\n'); - -// Simulate the expected output -const msLocale = {}; -const allLocales = {}; -const localeList = {}; - -testLocales.forEach(langcode => { - const uid = `drupallocale_${langcode.toLowerCase().replace(/-/g, '_')}`; - const isMaster = langcode === masterLocale; - - // Apply transformation (simplified) - let code = langcode.toLowerCase(); - let name = ''; - - if (langcode === 'und') { - code = 'en-us'; - name = 'English - United States'; - } else if (langcode === 'en') { - name = 'English'; - } else if (langcode === 'fr-fr') { - name = 'French - France'; - } else if (langcode === 'es-mx') { - name = 'Spanish - Mexico'; - } - - const locale = { - code: code, - name: name, - fallback_locale: isMaster ? null : masterLocale.toLowerCase(), - uid: uid - }; - - if (isMaster) { - msLocale[uid] = locale; - } else { - allLocales[uid] = locale; - } - - localeList[uid] = locale; -}); - -console.log('โœ… master-locale.json:'); -console.log(JSON.stringify(msLocale, null, 2)); - -console.log('\nโœ… locales.json:'); -console.log(JSON.stringify(allLocales, null, 2)); - -console.log('\nโœ… language.json:'); -console.log(JSON.stringify(localeList, null, 2)); - -console.log('\n๐ŸŽฏ Key Features:'); -console.log(' โœ… UID format: drupallocale_{langcode}'); -console.log(' โœ… Hyphens replaced with underscores'); -console.log(' โœ… UID used as JSON key (not random UUID)'); -console.log(' โœ… Master locale has null fallback'); -console.log(' โœ… Non-master locales use master as fallback'); From c48802f483e0125d0d4baccb6969f521c970dcbe Mon Sep 17 00:00:00 2001 From: sauravraw Date: Tue, 23 Sep 2025 19:31:19 +0530 Subject: [PATCH 05/37] fixed published_detail issue --- .gitignore | 4 +- api/src/services/drupal/entries.service.ts | 36 +++++ test-drupal-locales-simple.js | 160 --------------------- test-drupal-locales.js | 75 ---------- test-drupal-locales.mjs | 76 ---------- 5 files changed, 39 insertions(+), 312 deletions(-) delete mode 100644 test-drupal-locales-simple.js delete mode 100644 test-drupal-locales.js delete mode 100644 test-drupal-locales.mjs diff --git a/.gitignore b/.gitignore index 25b20b358..e8732c99b 100644 --- a/.gitignore +++ b/.gitignore @@ -360,4 +360,6 @@ upload-api/extracted_files *copy* .qodo .vscode -*MigrationData* \ No newline at end of file +*MigrationData* +*.zip +*extracted_files* \ No newline at end of file diff --git a/api/src/services/drupal/entries.service.ts b/api/src/services/drupal/entries.service.ts index 02ef265d7..2f3d61614 100644 --- a/api/src/services/drupal/entries.service.ts +++ b/api/src/services/drupal/entries.service.ts @@ -565,6 +565,12 @@ const consolidateTaxonomyFields = ( const fieldsToRemove: string[] = []; const seenTermUids = new Set(); // Track unique term_uid values + console.log( + `๐Ÿท๏ธ Starting taxonomy consolidation for ${contentType} with ${ + Object.keys(processedEntry).length + } fields` + ); + // Iterate through all fields in the processed entry for (const [fieldKey, fieldValue] of Object.entries(processedEntry)) { // Extract field name from key (remove _target_id suffix) @@ -572,6 +578,9 @@ const consolidateTaxonomyFields = ( // Check if this is a taxonomy field using field analysis if (isTaxonomyField(fieldName, contentType, taxonomyFieldMapping)) { + console.log( + `๐Ÿท๏ธ Found taxonomy field in consolidation: ${fieldKey} -> ${fieldName}` + ); // Validate that field value is an array with taxonomy structure if (Array.isArray(fieldValue)) { for (const taxonomyItem of fieldValue) { @@ -655,6 +664,13 @@ const processFieldData = async ( .replace(/_status$/, '') .replace(/_uri$/, ''); + // Debug: Log all fields being processed + if (dataKey.endsWith('_target_id')) { + console.log( + `๐Ÿ” Processing _target_id field: ${dataKey} -> ${fieldName} (value: ${value}) for content type: ${contentType}` + ); + } + // Handle asset fields using field analysis if ( dataKey.endsWith('_target_id') && @@ -678,10 +694,14 @@ const processFieldData = async ( if (dataKey.endsWith('_target_id') && typeof value === 'number') { // Check if this is a taxonomy field using our field analysis if (isTaxonomyField(fieldName, contentType, taxonomyFieldMapping)) { + console.log( + `๐Ÿท๏ธ Processing taxonomy field: ${fieldName} (${dataKey}) with value: ${value} for content type: ${contentType}` + ); // Look up taxonomy reference using drupal_term_id const taxonomyRef = taxonomyReferenceLookup[value]; if (taxonomyRef) { + console.log(`๐Ÿท๏ธ Found taxonomy reference for ${value}:`, taxonomyRef); // Transform to array format with taxonomy_uid and term_uid (no drupal_term_id) processedData[dataKey] = [ { @@ -690,6 +710,9 @@ const processFieldData = async ( }, ]; } else { + console.log( + `โš ๏ธ No taxonomy reference found for drupal_term_id: ${value}` + ); // Fallback to numeric tid if lookup failed processedData[dataKey] = value; } @@ -1336,6 +1359,9 @@ const processEntries = async ( processedEntry = enhancedEntry; + // Add publish_details as an empty array to the end of entry creation + processedEntry.publish_details = []; + if (typeof entry.nid === 'number') { existingLocaleContent[`content_type_entries_title_${entry.nid}`] = processedEntry; @@ -1588,6 +1614,11 @@ export const createEntry = async ( assetFields: assetFieldMapping, } = await analyzeFieldTypes(dbConfig, destination_stack_id, projectId); + console.log( + `๐Ÿท๏ธ Taxonomy field mapping loaded:`, + JSON.stringify(taxonomyFieldMapping, null, 2) + ); + // Fetch field configurations const fieldConfigs = await fetchFieldConfigs( connection, @@ -1614,6 +1645,11 @@ export const createEntry = async ( const taxonomyReferenceLookup = await loadTaxonomyReferences( referencesSave ); + console.log( + `๐Ÿท๏ธ Loaded ${ + Object.keys(taxonomyReferenceLookup).length + } taxonomy reference mappings` + ); // Process each content type from query config (like original) const pageQuery = queryPageConfig.page; diff --git a/test-drupal-locales-simple.js b/test-drupal-locales-simple.js deleted file mode 100644 index 9b9fb35fe..000000000 --- a/test-drupal-locales-simple.js +++ /dev/null @@ -1,160 +0,0 @@ -const mysql = require('mysql2/promise'); -const axios = require('axios'); - -// Test configuration using upload-api database -const dbConfig = { - host: 'localhost', - user: 'root', - password: '', - database: 'riceuniversity2', - port: 3306 -}; - -async function testDrupalLocaleQueries() { - console.log('๐Ÿงช Testing Drupal Locale SQL Queries...'); - console.log('๐Ÿ“Š Database:', dbConfig.database); - - let connection; - - try { - // Create database connection - connection = await mysql.createConnection(dbConfig); - console.log('โœ… Database connection established'); - - // 1. Test master locale query - console.log('\n๐Ÿ” Testing Master Locale Query...'); - const masterLocaleQuery = ` - SELECT SUBSTRING_INDEX( - SUBSTRING_INDEX(CONVERT(data USING utf8), 'default_langcode";s:2:"', -1), - '"', 1 - ) as master_locale - FROM config - WHERE name = 'system.site' - `; - - const [masterRows] = await connection.execute(masterLocaleQuery); - const masterLocaleCode = masterRows[0]?.master_locale || 'en'; - console.log('โœ… Master Locale:', masterLocaleCode); - - // 2. Test all locales query - console.log('\n๐Ÿ” Testing All Locales Query...'); - const allLocalesQuery = ` - SELECT DISTINCT langcode - FROM node_field_data - WHERE langcode IS NOT NULL AND langcode != '' - ORDER BY langcode - `; - - const [allLocaleRows] = await connection.execute(allLocalesQuery); - const allLocaleCodes = allLocaleRows.map(row => row.langcode); - console.log('โœ… All Locales:', allLocaleCodes); - - // 3. Test non-master locales query - console.log('\n๐Ÿ” Testing Non-Master Locales Query...'); - const nonMasterLocalesQuery = ` - SELECT DISTINCT n.langcode - FROM node_field_data n - WHERE n.langcode IS NOT NULL - AND n.langcode != '' - AND n.langcode != ( - SELECT - SUBSTRING_INDEX( - SUBSTRING_INDEX(CONVERT(data USING utf8), 'default_langcode";s:2:"', -1), - '"', - 1 - ) - FROM config - WHERE name = 'system.site' - LIMIT 1 - ) - ORDER BY n.langcode - `; - - const [nonMasterRows] = await connection.execute(nonMasterLocalesQuery); - const nonMasterLocaleCodes = nonMasterRows.map(row => row.langcode); - console.log('โœ… Non-Master Locales:', nonMasterLocaleCodes); - - // 4. Test Contentstack API - console.log('\n๐Ÿ” Testing Contentstack Locales API...'); - try { - const response = await axios.get('https://app.contentstack.com/api/v3/locales?include_all=true'); - const contentstackLocales = response.data?.locales || {}; - const localeCount = Object.keys(contentstackLocales).length; - console.log('โœ… Contentstack API Response:', `${localeCount} locales fetched`); - - // Show sample locales - const sampleLocales = Object.entries(contentstackLocales).slice(0, 5); - console.log('๐Ÿ“‹ Sample Locales:', sampleLocales); - - // Test locale name lookup for found codes - console.log('\n๐Ÿ” Testing Locale Name Mapping...'); - allLocaleCodes.forEach(code => { - const name = contentstackLocales[code] || contentstackLocales[code.toLowerCase()] || 'Unknown'; - console.log(` ${code} โ†’ ${name}`); - }); - - } catch (apiError) { - console.error('โŒ Contentstack API Error:', apiError.message); - } - - // 5. Test transformation logic - console.log('\n๐Ÿ” Testing Transformation Logic...'); - const hasUnd = allLocaleCodes.includes('und'); - const hasEn = allLocaleCodes.includes('en'); - const hasEnUs = allLocaleCodes.includes('en-us'); - - console.log('๐Ÿ“Š Locale Analysis:'); - console.log(` Has "und": ${hasUnd}`); - console.log(` Has "en": ${hasEn}`); - console.log(` Has "en-us": ${hasEnUs}`); - console.log(` Master Locale: ${masterLocaleCode}`); - - // Apply transformation rules - console.log('\n๐Ÿ”„ Applying Transformation Rules...'); - allLocaleCodes.forEach(locale => { - let transformedCode = locale.toLowerCase(); - let transformedName = ''; - let isMaster = locale === masterLocaleCode; - - if (locale === 'und') { - if (hasEnUs) { - transformedCode = 'en'; - transformedName = 'English'; - } else { - transformedCode = 'en-us'; - transformedName = 'English - United States'; - } - } else if (locale === 'en-us') { - if (hasUnd) { - transformedCode = 'en'; - transformedName = 'English'; - } - } else if (locale === 'en' && hasEnUs) { - transformedCode = 'und'; - transformedName = 'Language Neutral'; - } - - console.log(` ${locale} โ†’ ${transformedCode} (${transformedName || 'API lookup'}) [${isMaster ? 'MASTER' : 'regular'}]`); - }); - - console.log('\nโœ… All tests completed successfully!'); - - } catch (error) { - console.error('โŒ Test failed:', error); - console.error('Stack trace:', error.stack); - } finally { - if (connection) { - await connection.end(); - console.log('๐Ÿ”Œ Database connection closed'); - } - } -} - -// Run the test -testDrupalLocaleQueries().then(() => { - console.log('\n๐Ÿ Test completed'); - process.exit(0); -}).catch((error) => { - console.error('๐Ÿ’ฅ Test crashed:', error); - process.exit(1); -}); diff --git a/test-drupal-locales.js b/test-drupal-locales.js deleted file mode 100644 index 0d389fa93..000000000 --- a/test-drupal-locales.js +++ /dev/null @@ -1,75 +0,0 @@ -const { createLocale } = await import('./api/src/services/drupal/locales.service.js'); -const path = await import('path'); -const fs = await import('fs'); - -// Test configuration using upload-api database -const testProject = { - mysql: { - host: 'localhost', - user: 'root', - password: '', - database: 'riceuniversity2', - port: '3306' - } -}; - -const testDestinationStackId = 'test-drupal-locale-stack'; -const testProjectId = 'test-project-123'; - -async function testDrupalLocales() { - console.log('๐Ÿงช Testing Drupal Locale System...'); - console.log('๐Ÿ“Š Database:', testProject.mysql.database); - console.log('๐ŸŽฏ Stack ID:', testDestinationStackId); - - try { - // Test the createLocale function - await createLocale(testDestinationStackId, testProjectId, testProject); - - console.log('โœ… Locale creation completed successfully!'); - - // Check if files were created - const localesDir = path.join('./api/cmsMigrationData', testDestinationStackId, 'locales'); - - console.log('\n๐Ÿ“ Checking created files:'); - - // Check master-locale.json - const masterLocalePath = path.join(localesDir, 'master-locale.json'); - if (fs.existsSync(masterLocalePath)) { - const masterLocale = JSON.parse(fs.readFileSync(masterLocalePath, 'utf8')); - console.log('โœ… master-locale.json:', JSON.stringify(masterLocale, null, 2)); - } else { - console.log('โŒ master-locale.json not found'); - } - - // Check locales.json - const localesPath = path.join(localesDir, 'locales.json'); - if (fs.existsSync(localesPath)) { - const locales = JSON.parse(fs.readFileSync(localesPath, 'utf8')); - console.log('โœ… locales.json:', JSON.stringify(locales, null, 2)); - } else { - console.log('โŒ locales.json not found'); - } - - // Check language.json - const languagePath = path.join(localesDir, 'language.json'); - if (fs.existsSync(languagePath)) { - const language = JSON.parse(fs.readFileSync(languagePath, 'utf8')); - console.log('โœ… language.json:', JSON.stringify(language, null, 2)); - } else { - console.log('โŒ language.json not found'); - } - - } catch (error) { - console.error('โŒ Test failed:', error); - console.error('Stack trace:', error.stack); - } -} - -// Run the test -testDrupalLocales().then(() => { - console.log('\n๐Ÿ Test completed'); - process.exit(0); -}).catch((error) => { - console.error('๐Ÿ’ฅ Test crashed:', error); - process.exit(1); -}); diff --git a/test-drupal-locales.mjs b/test-drupal-locales.mjs deleted file mode 100644 index 1ab0e3bd3..000000000 --- a/test-drupal-locales.mjs +++ /dev/null @@ -1,76 +0,0 @@ -import { createLocale } from './api/src/services/drupal/locales.service.js'; -import path from 'path'; -import fs from 'fs'; - -// Test configuration using upload-api database -const testProject = { - mysql: { - host: 'localhost', - user: 'root', - password: '', - database: 'riceuniversity2', - port: '3306' - } -}; - -const testDestinationStackId = 'test-drupal-locale-stack'; -const testProjectId = 'test-project-123'; - -async function testDrupalLocales() { - console.log('๐Ÿงช Testing Drupal Locale System...'); - console.log('๐Ÿ“Š Database:', testProject.mysql.database); - console.log('๐ŸŽฏ Stack ID:', testDestinationStackId); - - try { - // Test the createLocale function - await createLocale(testDestinationStackId, testProjectId, testProject); - - console.log('โœ… Locale creation completed successfully!'); - - // Check if files were created - const localesDir = path.join('./api/cmsMigrationData', testDestinationStackId, 'locales'); - - console.log('\n๐Ÿ“ Checking created files:'); - console.log('๐Ÿ“‚ Directory:', localesDir); - - // Check master-locale.json - const masterLocalePath = path.join(localesDir, 'master-locale.json'); - if (fs.existsSync(masterLocalePath)) { - const masterLocale = JSON.parse(fs.readFileSync(masterLocalePath, 'utf8')); - console.log('โœ… master-locale.json:', JSON.stringify(masterLocale, null, 2)); - } else { - console.log('โŒ master-locale.json not found at:', masterLocalePath); - } - - // Check locales.json - const localesPath = path.join(localesDir, 'locales.json'); - if (fs.existsSync(localesPath)) { - const locales = JSON.parse(fs.readFileSync(localesPath, 'utf8')); - console.log('โœ… locales.json:', JSON.stringify(locales, null, 2)); - } else { - console.log('โŒ locales.json not found at:', localesPath); - } - - // Check language.json - const languagePath = path.join(localesDir, 'language.json'); - if (fs.existsSync(languagePath)) { - const language = JSON.parse(fs.readFileSync(languagePath, 'utf8')); - console.log('โœ… language.json:', JSON.stringify(language, null, 2)); - } else { - console.log('โŒ language.json not found at:', languagePath); - } - - } catch (error) { - console.error('โŒ Test failed:', error); - console.error('Stack trace:', error.stack); - } -} - -// Run the test -testDrupalLocales().then(() => { - console.log('\n๐Ÿ Test completed'); - process.exit(0); -}).catch((error) => { - console.error('๐Ÿ’ฅ Test crashed:', error); - process.exit(1); -}); From 1f0f59068794e3c2cb0a6eba83787514ab2550d8 Mon Sep 17 00:00:00 2001 From: sauravraw Date: Fri, 26 Sep 2025 17:13:23 +0530 Subject: [PATCH 06/37] uidconnector added --- .../libs/contentTypeMapper.js | 191 +++++++++++------- .../libs/createInitialMapper.js | 71 +++++-- 2 files changed, 169 insertions(+), 93 deletions(-) diff --git a/upload-api/migration-drupal/libs/contentTypeMapper.js b/upload-api/migration-drupal/libs/contentTypeMapper.js index cb9836f5d..644d4403b 100644 --- a/upload-api/migration-drupal/libs/contentTypeMapper.js +++ b/upload-api/migration-drupal/libs/contentTypeMapper.js @@ -7,60 +7,89 @@ const path = require('path'); /** * Loads taxonomy schema from drupalMigrationData/taxonomySchema/taxonomySchema.json * If not found, attempts to generate it using extractTaxonomy - * + * * @param {Object} dbConfig - Database configuration for fallback taxonomy extraction * @returns {Array} Array of taxonomy vocabularies with uid and name */ const loadTaxonomySchema = async (dbConfig = null) => { try { - const taxonomySchemaPath = path.join(__dirname, '..', '..', 'drupalMigrationData', 'taxonomySchema', 'taxonomySchema.json'); - + const taxonomySchemaPath = path.join( + __dirname, + '..', + '..', + 'drupalMigrationData', + 'taxonomySchema', + 'taxonomySchema.json' + ); + // Check if taxonomy schema exists if (fs.existsSync(taxonomySchemaPath)) { const taxonomyData = fs.readFileSync(taxonomySchemaPath, 'utf8'); const taxonomies = JSON.parse(taxonomyData); return taxonomies; } - + // If not found and dbConfig available, try to generate it if (dbConfig) { const extractTaxonomy = require('./extractTaxonomy'); const taxonomies = await extractTaxonomy(dbConfig); return taxonomies; } - + return []; - } catch (error) { console.error('โŒ Error loading taxonomy schema:', error.message); return []; } }; +// Load restricted keywords +const idArray = require('../utils/restrictedKeyWords'); + /** - * Corrects the UID by applying a custom affix and sanitizing the string. - * - * @param {string} uid - The original UID that needs to be corrected. - * @param {string} affix - The affix to be prepended to the UID if it's restricted. - * @returns {string} The corrected UID with the affix (if applicable) and sanitized characters. - * - * @description - * This function checks if the provided `uid` is included in the `restrictedUid` list. If it is, the function will: - * 1. Prepend the provided `affix` to the `uid`. - * 2. Replace any non-alphanumeric characters in the `uid` with underscores. - * - * It then converts any uppercase letters to lowercase and prefixes them with an underscore (to match a typical snake_case format). - * - * If the `uid` is not restricted, the function simply returns it after converting uppercase letters to lowercase and adding an underscore before each uppercase letter. - * // Outputs: 'prefix_my_restricted_uid' + * Helper function to check if string starts with a number */ -const uidCorrector = (uid, affix) => { - let newId = uid; - if (restrictedUid?.includes?.(uid) || uid?.startsWith?.('_ids') || uid?.endsWith?.('_ids')) { - newId = uid?.replace?.(uid, `${affix}_${uid}`); - newId = newId?.replace?.(/[^a-zA-Z0-9]+/g, '_'); +function startsWithNumber(str) { + return /^\d/.test(str); +} + +/** + * Improved UID corrector based on Sitecore implementation but adapted for Drupal + * Handles all edge cases: restricted keywords, numbers, CamelCase, special characters + */ +const uidCorrector = (uid, prefix) => { + if (!uid || typeof uid !== 'string' || !prefix) { + return ''; + } + + let newUid = uid; + + // Handle restricted keywords + if (idArray.includes(uid) || uid.startsWith('_ids') || uid.endsWith('_ids')) { + newUid = `${prefix}_${uid}`; + } + + // Handle UIDs that start with numbers + if (startsWithNumber(newUid)) { + newUid = `${prefix}_${newUid}`; } - return newId.replace(/([A-Z])/g, (match) => `${match?.toLowerCase?.()}`); + + // Clean up the UID + newUid = newUid + .replace(/[ -]/g, '_') // Replace spaces and hyphens with underscores + .replace(/[^a-zA-Z0-9_]+/g, '_') // Replace non-alphanumeric characters (except underscore) + .replace(/\$/g, '') // Remove dollar signs + .toLowerCase() // Convert to lowercase + .replace(/([A-Z])/g, (match) => `_${match.toLowerCase()}`) // Handle camelCase + .replace(/_+/g, '_') // Replace multiple underscores with single + .replace(/^_|_$/g, ''); // Remove leading/trailing underscores + + // Ensure UID doesn't start with underscore (Contentstack requirement) + if (newUid.startsWith('_')) { + newUid = newUid.substring(1); + } + + return newUid; }; /** @@ -91,8 +120,12 @@ const extractAdvancedFields = (item, referenceFields = []) => { unique: false, nonLocalizable: false, validationErrorMessage: '', - embedObjects: referenceFields.length ? referenceFields : (referenceFields.length === 0 && Array.isArray(referenceFields) ? [] : undefined), - description: description, + embedObjects: referenceFields.length + ? referenceFields + : referenceFields.length === 0 && Array.isArray(referenceFields) + ? [] + : undefined, + description: description }; }; @@ -109,7 +142,7 @@ const extractAdvancedFields = (item, referenceFields = []) => { * This function generates a field object to be used in the context of a content management system (CMS), * specifically for fields that have both a primary and backup configuration. It extracts the necessary field * details from the provided `item` and augments it with additional information such as UID, field names, and field types. - * + * * The advanced field properties are extracted using the `extractAdvancedFields` function, including any reference fields, * field types, and other metadata related to the field configuration. * @@ -119,11 +152,11 @@ const createFieldObject = (item, contentstackFieldType, backupFieldType, referen // Add suffix for specific field types (following old migration pattern) const needsSuffix = ['reference', 'file'].includes(contentstackFieldType); const fieldNameWithSuffix = needsSuffix ? `${item?.field_name}_target_id` : item?.field_name; - + // ๐Ÿšซ For json and html fields, always use empty embedObjects array const shouldUseEmptyEmbedObjects = ['json', 'html'].includes(contentstackFieldType); const finalReferenceFields = shouldUseEmptyEmbedObjects ? [] : referenceFields; - + return { uid: item?.field_name, otherCmsField: item?.field_label, @@ -147,7 +180,7 @@ const createFieldObject = (item, contentstackFieldType, backupFieldType, referen * @description * This function generates a field object for dropdown or radio field types based on the provided item. * It ensures that the field's advanced properties are extracted from the item, including validation options. - * + * */ const createDropdownOrRadioFieldObject = (item, fieldType) => { return { @@ -161,7 +194,7 @@ const createDropdownOrRadioFieldObject = (item, fieldType) => { /** * Creates a taxonomy field object with specialized structure for Contentstack - * + * * @param {Object} item - The Drupal field item containing field details * @param {Array} taxonomySchema - Array of taxonomy vocabularies from taxonomySchema.json * @param {Array} targetVocabularies - Optional array of specific vocabularies this field references @@ -170,31 +203,32 @@ const createDropdownOrRadioFieldObject = (item, fieldType) => { const createTaxonomyFieldObject = (item, taxonomySchema, targetVocabularies = []) => { // Determine which taxonomies to include let taxonomiesToInclude = []; - + if (targetVocabularies && targetVocabularies.length > 0) { // If specific vocabularies are provided, use only those - taxonomiesToInclude = taxonomySchema.filter(taxonomy => - targetVocabularies.includes(taxonomy.uid) || targetVocabularies.includes(taxonomy.name) + taxonomiesToInclude = taxonomySchema.filter( + (taxonomy) => + targetVocabularies.includes(taxonomy.uid) || targetVocabularies.includes(taxonomy.name) ); } else { // If no specific vocabularies, include all available taxonomies taxonomiesToInclude = taxonomySchema; } - + // Build taxonomies array with default properties - const taxonomiesArray = taxonomiesToInclude.map(taxonomy => ({ + const taxonomiesArray = taxonomiesToInclude.map((taxonomy) => ({ taxonomy_uid: taxonomy.uid, mandatory: false, multiple: true, non_localizable: false })); - + // Get advanced field properties from the original field const advancedFields = extractAdvancedFields(item); - + // Add _target_id suffix for taxonomy fields (following old migration pattern) const fieldNameWithSuffix = `${item?.field_name}_target_id`; - + return { uid: item?.field_name, otherCmsField: item?.field_label, @@ -205,16 +239,16 @@ const createTaxonomyFieldObject = (item, taxonomySchema, targetVocabularies = [] backupFieldType: 'reference', backupFieldUid: uidCorrector(fieldNameWithSuffix, item?.prefix), advanced: { - data_type: "taxonomy", + data_type: 'taxonomy', display_name: item?.field_label || item?.field_name, uid: uidCorrector(fieldNameWithSuffix, item?.prefix), taxonomies: taxonomiesArray, - field_metadata: { - description: advancedFields?.field_metadata?.description || "", - default_value: advancedFields?.field_metadata?.default_value || "" + field_metadata: { + description: advancedFields?.field_metadata?.description || '', + default_value: advancedFields?.field_metadata?.default_value || '' }, - format: "", - error_messages: { format: "" }, + format: '', + error_messages: { format: '' }, mandatory: advancedFields?.mandatory || false, multiple: advancedFields?.multiple !== undefined ? advancedFields?.multiple : true, non_localizable: advancedFields?.non_localizable || false, @@ -235,14 +269,14 @@ const createTaxonomyFieldObject = (item, taxonomySchema, targetVocabularies = [] * @description * This function processes each Drupal field item from the input data and maps them to a specific schema structure. * It handles various Drupal field types and maps them to Contentstack field types with the following adaptations: - * + * * Field Type Mappings: * - Single line text โ†’ Multiline/HTML RTE/JSON RTE - * - Multiline text โ†’ HTML RTE/JSON RTE + * - Multiline text โ†’ HTML RTE/JSON RTE * - HTML RTE โ†’ JSON RTE * - JSON RTE โ†’ HTML RTE * - Taxonomy term references โ†’ Taxonomy fields with vocabulary mappings - * + * * The function supports processing of: * - Text fields with various widget types * - Rich text fields with associated reference fields @@ -254,26 +288,26 @@ const createTaxonomyFieldObject = (item, taxonomySchema, targetVocabularies = [] const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => { // Load taxonomy schema for taxonomy field processing const taxonomySchema = await loadTaxonomySchema(dbConfig); - + // ๐Ÿท๏ธ Collect taxonomy fields for consolidation const collectedTaxonomies = []; - + const schemaArray = data.reduce((acc, item) => { // Add prefix to item for UID correction item.prefix = prefix; - + switch (item.type) { case 'text_with_summary': case 'text_long': { // Rich text with switching options: JSON RTE โ†’ HTML RTE โ†’ multiline โ†’ text - const availableContentTypes = contentTypes?.filter(ct => ct !== item.content_types) || []; + const availableContentTypes = contentTypes?.filter((ct) => ct !== item.content_types) || []; const referenceFields = availableContentTypes.slice(0, 10); acc.push(createFieldObject(item, 'json', 'html', referenceFields)); break; } case 'text': { // Single line with switching options: single_line โ†’ multiline โ†’ HTML RTE โ†’ JSON RTE - const availableContentTypes = contentTypes?.filter(ct => ct !== item.content_types) || []; + const availableContentTypes = contentTypes?.filter((ct) => ct !== item.content_types) || []; const referenceFields = availableContentTypes.slice(0, 10); acc.push(createFieldObject(item, 'single_line_text', 'multi_line_text', referenceFields)); break; @@ -281,7 +315,7 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => case 'string_long': case 'comment': { // Rich text with switching options: JSON RTE โ†’ HTML RTE โ†’ multiline โ†’ text - const availableContentTypes = contentTypes?.filter(ct => ct !== item.content_types) || []; + const availableContentTypes = contentTypes?.filter((ct) => ct !== item.content_types) || []; const referenceFields = availableContentTypes.slice(0, 10); acc.push(createFieldObject(item, 'json', 'html', referenceFields)); break; @@ -289,7 +323,7 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => case 'string': case 'list_string': { // Single line with switching options: single_line โ†’ multiline โ†’ HTML RTE โ†’ JSON RTE - const availableContentTypes = contentTypes?.filter(ct => ct !== item.content_types) || []; + const availableContentTypes = contentTypes?.filter((ct) => ct !== item.content_types) || []; const referenceFields = availableContentTypes.slice(0, 10); acc.push(createFieldObject(item, 'single_line_text', 'multi_line_text', referenceFields)); break; @@ -301,25 +335,24 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => case 'taxonomy_term_reference': { // ๐Ÿท๏ธ Collect taxonomy field for consolidation instead of creating individual fields if (taxonomySchema && taxonomySchema.length > 0) { - // Try to determine specific vocabularies this field references let targetVocabularies = []; - + // Check if field has handler settings that specify target vocabularies if (item.handler_settings && item.handler_settings.target_bundles) { targetVocabularies = Object.keys(item.handler_settings.target_bundles); } - + // Add vocabularies to collection (avoid duplicates) if (targetVocabularies && targetVocabularies.length > 0) { - targetVocabularies.forEach(vocab => { + targetVocabularies.forEach((vocab) => { if (!collectedTaxonomies.includes(vocab)) { collectedTaxonomies.push(vocab); } }); } else { // Backup: Use all available taxonomies from taxonomySchema.json - taxonomySchema.forEach(taxonomy => { + taxonomySchema.forEach((taxonomy) => { if (!collectedTaxonomies.includes(taxonomy.uid)) { collectedTaxonomies.push(taxonomy.uid); } @@ -335,26 +368,26 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => // Check if this is a taxonomy field by handler if (item.handler === 'default:taxonomy_term') { // ๐Ÿท๏ธ Collect taxonomy field for consolidation instead of creating individual fields - + // Try to determine specific vocabularies this field references let targetVocabularies = []; - + // Check if field has handler settings that specify target vocabularies if (item.reference) { targetVocabularies = Object.keys(item.reference); } - + if (taxonomySchema && taxonomySchema.length > 0) { // Add vocabularies to collection (avoid duplicates) if (targetVocabularies && targetVocabularies.length > 0) { - targetVocabularies.forEach(vocab => { + targetVocabularies.forEach((vocab) => { if (!collectedTaxonomies.includes(vocab)) { collectedTaxonomies.push(vocab); } }); } else { // Backup: Use all available taxonomies from taxonomySchema.json - taxonomySchema.forEach(taxonomy => { + taxonomySchema.forEach((taxonomy) => { if (!collectedTaxonomies.includes(taxonomy.uid)) { collectedTaxonomies.push(taxonomy.uid); } @@ -363,27 +396,30 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => } else { // Fallback to regular reference field if no taxonomy schema available // Use available content types instead of generic 'taxonomy' - const availableContentTypes = contentTypes?.filter(ct => ct !== item.content_types) || []; + const availableContentTypes = + contentTypes?.filter((ct) => ct !== item.content_types) || []; const referenceFields = availableContentTypes.slice(0, 10); acc.push(createFieldObject(item, 'reference', 'reference', referenceFields)); } } else if (item.handler === 'default:node') { // Handle node reference fields - use specific content types from reference settings let referenceFields = []; - + if (item.reference && Object.keys(item.reference).length > 0) { // Use specific content types from field configuration referenceFields = Object.keys(item.reference); } else { // Backup: Use up to 10 content types from available content types - const availableContentTypes = contentTypes?.filter(ct => ct !== item.content_types) || []; + const availableContentTypes = + contentTypes?.filter((ct) => ct !== item.content_types) || []; referenceFields = availableContentTypes.slice(0, 10); } - + acc.push(createFieldObject(item, 'reference', 'reference', referenceFields)); } else { // Handle other entity references - exclude taxonomy and limit to 10 content types - const availableContentTypes = contentTypes?.filter(ct => ct !== item.content_types) || []; + const availableContentTypes = + contentTypes?.filter((ct) => ct !== item.content_types) || []; const referenceFields = availableContentTypes.slice(0, 10); acc.push(createFieldObject(item, 'reference', 'reference', referenceFields)); } @@ -434,8 +470,8 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => }, []); // Add default title and url fields if not present - const hasTitle = schemaArray.some(field => field.uid === 'title'); - const hasUrl = schemaArray.some(field => field.uid === 'url'); + const hasTitle = schemaArray.some((field) => field.uid === 'title'); + const hasUrl = schemaArray.some((field) => field.uid === 'url'); if (!hasTitle) { schemaArray.unshift({ @@ -467,7 +503,6 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => // ๐Ÿท๏ธ TAXONOMY CONSOLIDATION: Create single consolidated taxonomy field if any taxonomies were collected if (collectedTaxonomies.length > 0) { - // Create consolidated taxonomy field with fixed properties const consolidatedTaxonomyField = { uid: 'taxonomies', @@ -479,7 +514,7 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => backupFieldType: 'taxonomy', backupFieldUid: 'taxonomies', advanced: { - taxonomies: collectedTaxonomies.map(taxonomyUid => ({ + taxonomies: collectedTaxonomies.map((taxonomyUid) => ({ taxonomy_uid: taxonomyUid, mandatory: false, multiple: true, @@ -491,7 +526,7 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => unique: false } }; - + // Add consolidated taxonomy field at the end of schema schemaArray.push(consolidatedTaxonomyField); } diff --git a/upload-api/migration-drupal/libs/createInitialMapper.js b/upload-api/migration-drupal/libs/createInitialMapper.js index bb4ebb05a..e5d67bddd 100644 --- a/upload-api/migration-drupal/libs/createInitialMapper.js +++ b/upload-api/migration-drupal/libs/createInitialMapper.js @@ -4,8 +4,8 @@ /** * External module dependencies. */ -const fs = require('fs'); // for existsSync -const fsp = require('fs/promises'); // for async file operations +const fs = require('fs'); // for existsSync +const fsp = require('fs/promises'); // for async file operations const path = require('path'); const contentTypeMapper = require('./contentTypeMapper'); @@ -17,15 +17,49 @@ const config = require('../config'); const idArray = require('../utils/restrictedKeyWords'); /** - * Corrects the UID by adding a prefix and sanitizing the string if it is found in a specified list. + * Helper function to check if string starts with a number + */ +function startsWithNumber(str) { + return /^\d/.test(str); +} + +/** + * Improved UID corrector based on Sitecore implementation but adapted for Drupal + * Handles all edge cases: restricted keywords, numbers, CamelCase, special characters */ const uidCorrector = (uid, prefix) => { - let newId = uid; + if (!uid || typeof uid !== 'string' || !prefix) { + return ''; + } + + let newUid = uid; + + // Handle restricted keywords if (idArray.includes(uid) || uid.startsWith('_ids') || uid.endsWith('_ids')) { - newId = uid.replace(uid, `${prefix}_${uid}`); - newId = newId.replace(/[^a-zA-Z0-9]+/g, '_'); + newUid = `${prefix}_${uid}`; + } + + // Handle UIDs that start with numbers + if (startsWithNumber(newUid)) { + newUid = `${prefix}_${newUid}`; } - return newId.replace(/([A-Z])/g, (match) => `_${match.toLowerCase()}`); + + // Clean up the UID + newUid = newUid + .replace(/[ -]/g, '_') // Replace spaces and hyphens with underscores + .replace(/[^a-zA-Z0-9_]+/g, '_') // Replace non-alphanumeric characters (except underscore) + .replace(/\$/g, '') // Remove dollar signs + .toLowerCase() // Convert to lowercase + .replace(/([A-Z])/g, (match) => `_${match.toLowerCase()}`) // Handle camelCase + .replace(/_+/g, '_') // Replace multiple underscores with single + .replace(/^_|_$/g, ''); // Remove leading/trailing underscores + + // Ensure UID doesn't start with underscore (Contentstack requirement) + if (newUid.startsWith('_')) { + newUid = newUid.substring(1); + } + + return newUid; }; /** @@ -34,16 +68,17 @@ const uidCorrector = (uid, prefix) => { const createInitialMapper = async (systemConfig, prefix) => { try { const drupalFolderPath = path.resolve(config.data, config.drupal.drupal); - + if (!fs.existsSync(drupalFolderPath)) { await fsp.mkdir(drupalFolderPath, { recursive: true }); } - + // Get database connection const connection = dbConnection(systemConfig); - + // SQL query to get field configurations - const query = "SELECT *, CONVERT(data USING utf8) as data FROM config WHERE name LIKE '%field.field.node%'"; + const query = + "SELECT *, CONVERT(data USING utf8) as data FROM config WHERE name LIKE '%field.field.node%'"; // Execute query const [rows] = await connection.promise().query(query); @@ -55,7 +90,7 @@ const createInitialMapper = async (systemConfig, prefix) => { try { const { unserialize } = require('php-serialize'); const conv_details = unserialize(rows[i].data); - + details_data.push({ field_label: conv_details?.label, description: conv_details?.description, @@ -77,13 +112,14 @@ const createInitialMapper = async (systemConfig, prefix) => { return { contentTypes: [] }; } - const initialMapper = []; const contentTypes = Object.keys(require('lodash').keyBy(details_data, 'content_types')); // Process each content type for (const contentType of contentTypes) { - const contentTypeFields = require('lodash').filter(details_data, { content_types: contentType }); + const contentTypeFields = require('lodash').filter(details_data, { + content_types: contentType + }); const contenttypeTitle = contentType.split('_').join(' '); const contentTypeObject = { @@ -99,7 +135,12 @@ const createInitialMapper = async (systemConfig, prefix) => { }; // Map fields using contentTypeMapper - const contentstackFields = await contentTypeMapper(contentTypeFields, contentTypes, prefix, systemConfig.mysql); + const contentstackFields = await contentTypeMapper( + contentTypeFields, + contentTypes, + prefix, + systemConfig.mysql + ); contentTypeObject.fieldMapping = contentstackFields; initialMapper.push(contentTypeObject); From 41f3b9aac09d51d804ce424d72fc44deb83c0ab3 Mon Sep 17 00:00:00 2001 From: sauravraw Date: Tue, 30 Sep 2025 15:42:13 +0530 Subject: [PATCH 07/37] added dyanamic field support for drupal from UI --- .gitignore | 3 +- api/package-lock.json | 6 + api/package.json | 1 - api/src/services/drupal/entries.service.ts | 51 ++-- api/src/utils/content-type-creator.utils.ts | 93 ++++++- api/src/utils/entries-field-creator.utils.ts | 267 ++++++++++++------- 6 files changed, 291 insertions(+), 130 deletions(-) diff --git a/.gitignore b/.gitignore index e8732c99b..d83248aef 100644 --- a/.gitignore +++ b/.gitignore @@ -362,4 +362,5 @@ upload-api/extracted_files .vscode *MigrationData* *.zip -*extracted_files* \ No newline at end of file +*extracted_files* +*.tsbuildinfo \ No newline at end of file diff --git a/api/package-lock.json b/api/package-lock.json index 652c69600..52440406c 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -16,6 +16,7 @@ "axios": "^1.12.0", "chokidar": "^3.6.0", "cors": "^2.8.5", + "dayjs": "^1.11.18", "dotenv": "^16.3.1", "express": "^4.21.0", "express-validator": "^7.0.1", @@ -5710,6 +5711,11 @@ "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-1.30.1.tgz", "integrity": "sha512-hBSVCvSmWC+QypYObzwGOd9wqdDpOt+0wl0KbU+R+uuZBS1jN8VsD1ss3irQDknRj5NvxiTF6oj/nDRnN/UQNw==" }, + "node_modules/dayjs": { + "version": "1.11.18", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.18.tgz", + "integrity": "sha512-zFBQ7WFRvVRhKcWoUh+ZA1g2HVgUbsZm9sbddh8EC5iv93sui8DVVz1Npvz+r6meo9VKfa8NyLWBsQK1VvIKPA==" + }, "node_modules/debounce-fn": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/debounce-fn/-/debounce-fn-4.0.0.tgz", diff --git a/api/package.json b/api/package.json index 641e47c52..877b12d5a 100644 --- a/api/package.json +++ b/api/package.json @@ -8,7 +8,6 @@ "start": "NODE_ENV=production tsx ./src/server.ts", "start:prod": "NODE_ENV=production node dist/server.js", "dev": "NODE_ENV=production nodemon --exec tsx ./src/server.ts", - "test:drupal": "npx tsx test-drupal-services.js", "prettify": "prettier --write .", "windev": "SET NODE_ENV=production&& tsx watch ./src/server.ts", "winstart": "SET NODE_ENV=production&& node dist/server.js", diff --git a/api/src/services/drupal/entries.service.ts b/api/src/services/drupal/entries.service.ts index 2f3d61614..8080ae198 100644 --- a/api/src/services/drupal/entries.service.ts +++ b/api/src/services/drupal/entries.service.ts @@ -3,6 +3,7 @@ import path from 'path'; import mysql from 'mysql2'; import { v4 as uuidv4 } from 'uuid'; import { JSDOM } from 'jsdom'; +import _ from 'lodash'; import { htmlToJson, jsonToHtml, @@ -26,8 +27,33 @@ import { type AssetFieldMapping, } from './field-analysis.service.js'; import FieldFetcherService from './field-fetcher.service.js'; +import { + entriesFieldCreator, + unflatten, +} from '../../utils/entries-field-creator.utils.js'; // Dynamic import for phpUnserialize will be used in the function +// Local utility functions (extracted from entries-field-creator.utils.ts patterns) +const append = 'a'; + +function startsWithNumber(str: string) { + return /^\d/.test(str); +} + +const uidCorrector = ({ uid, id }: any) => { + const value = uid || id; + if (!value) return ''; + + if (startsWithNumber(value)) { + return `${append}_${_.replace( + value, + new RegExp('[ -]', 'g'), + '_' + )?.toLowerCase()}`; + } + return _.replace(value, new RegExp('[ -]', 'g'), '_')?.toLowerCase(); +}; + interface TaxonomyReference { drupal_term_id: number; taxonomy_uid: string; @@ -547,11 +573,7 @@ const processFieldByType = ( /** * Consolidates all taxonomy fields into a single 'taxonomies' field with unique term_uid validation - * - * @param processedEntry - The processed entry data - * @param contentType - The content type being processed - * @param taxonomyFieldMapping - Mapping of taxonomy fields from field analysis - * @returns Entry with consolidated taxonomy field + * Uses the same pattern as entries-field-creator.utils.ts */ const consolidateTaxonomyFields = ( processedEntry: any, @@ -581,6 +603,7 @@ const consolidateTaxonomyFields = ( console.log( `๐Ÿท๏ธ Found taxonomy field in consolidation: ${fieldKey} -> ${fieldName}` ); + // Validate that field value is an array with taxonomy structure if (Array.isArray(fieldValue)) { for (const taxonomyItem of fieldValue) { @@ -624,13 +647,6 @@ const consolidateTaxonomyFields = ( ); } - // Replace existing 'taxonomies' field if it exists (as per requirement) - if ('taxonomies' in processedEntry && consolidatedTaxonomies.length > 0) { - console.log( - `๐Ÿ”„ Replaced existing 'taxonomies' field with consolidated data for ${contentType}` - ); - } - return consolidatedEntry; }; @@ -819,7 +835,7 @@ const processFieldData = async ( } else if (fieldName.endsWith('_tid')) { ctValue[fieldName] = [value]; } else if (fieldName === 'nid') { - ctValue.uid = `content_type_entries_title_${value}`; + ctValue.uid = uidCorrector({ id: `content_type_entries_title_${value}` }); } else if (fieldName === 'langcode') { // Use the actual langcode from the entry for proper multilingual support ctValue.locale = value || 'en-us'; // fallback to en-us if langcode is empty @@ -1363,10 +1379,11 @@ const processEntries = async ( processedEntry.publish_details = []; if (typeof entry.nid === 'number') { - existingLocaleContent[`content_type_entries_title_${entry.nid}`] = - processedEntry; - allProcessedContent[`content_type_entries_title_${entry.nid}`] = - processedEntry; + const entryUid = uidCorrector({ + id: `content_type_entries_title_${entry.nid}`, + }); + existingLocaleContent[entryUid] = processedEntry; + allProcessedContent[entryUid] = processedEntry; } // Log each entry transformation diff --git a/api/src/utils/content-type-creator.utils.ts b/api/src/utils/content-type-creator.utils.ts index ec4f7466b..fb44ef892 100644 --- a/api/src/utils/content-type-creator.utils.ts +++ b/api/src/utils/content-type-creator.utils.ts @@ -122,7 +122,56 @@ const saveAppMapper = async ({ marketPlacePath, data, fileName }: any) => { } } -export const convertToSchemaFormate = ({ field, advanced = true, marketPlacePath, keyMapper }: any) => { +// Field type conversion validation to prevent data loss +export const validateFieldTypeConversion = (currentType: string, newType: string): { allowed: boolean; reason?: string } => { + // Define field type hierarchy (simple โ†’ complex) + const typeHierarchy = { + 'single_line_text': 1, + 'multi_line_text': 2, + 'html': 3, + 'json': 4 + }; + + const currentLevel = typeHierarchy[currentType as keyof typeof typeHierarchy] || 0; + const newLevel = typeHierarchy[newType as keyof typeof typeHierarchy] || 0; + + // Allow same type or upgrades (simple โ†’ complex) + if (currentLevel <= newLevel) { + return { allowed: true }; + } + + // Special case: Allow JSON RTE โ†’ HTML RTE (both are rich content types) + if (currentType === 'json' && newType === 'html') { + return { allowed: true }; + } + + // Block downgrades (complex โ†’ simple) to prevent data loss + const reasons = { + 'multi_line_text_to_single': 'Converting multi-line to single-line may lose line breaks and formatting', + 'html_to_text': 'Converting HTML RTE to text fields will lose rich content formatting', + 'json_to_text': 'Converting JSON RTE to text fields will lose structured data and assets' + }; + + let reason = 'This conversion may result in data loss'; + if (currentType === 'multi_line_text' && newType === 'single_line_text') { + reason = reasons.multi_line_text_to_single; + } else if (currentType === 'html' && ['single_line_text', 'multi_line_text'].includes(newType)) { + reason = reasons.html_to_text; + } else if (currentType === 'json' && ['single_line_text', 'multi_line_text'].includes(newType)) { + reason = reasons.json_to_text; + } + + return { allowed: false, reason }; +}; + +export const convertToSchemaFormate = ({ field, advanced = true, marketPlacePath, keyMapper, currentFieldType }: any) => { + // Validate field type conversion if currentFieldType is provided + if (currentFieldType && field?.contentstackFieldType) { + const validation = validateFieldTypeConversion(currentFieldType, field.contentstackFieldType); + if (!validation.allowed) { + throw new Error(`Field type conversion blocked: ${validation.reason}`); + } + } switch (field?.contentstackFieldType) { case 'single_line_text': { return { @@ -365,17 +414,43 @@ export const convertToSchemaFormate = ({ field, advanced = true, marketPlacePath } } - case "html": { + case "text": { + // Handle generic text field - determine if it should be single line or multi line based on metadata + const isMultiline = field?.advanced?.multiline === true; return { - "data_type": "html", + "data_type": "text", "display_name": field?.title, uid: field?.uid, "field_metadata": { description: "", default_value: field?.advanced?.default_value ?? '', - "allow_json_rte": true, - "embed_entry": false, - "rich_text_type": "advanced" + ...(isMultiline && { "multiline": true }) + }, + "format": field?.advanced?.validationRegex ?? '', + "error_messages": { + "format": field?.advanced?.validationErrorMessage ?? '', + }, + "multiple": field?.advanced?.multiple ?? false, + "mandatory": field?.advanced?.mandatory ?? false, + "unique": field?.advanced?.unique ?? false, + "non_localizable": field.advanced?.nonLocalizable ?? false + } + } + + case "html": { + return { + "data_type": "text", + "display_name": field?.title, + uid: field?.uid, + "field_metadata": { + "allow_rich_text": true, + "description": "", + default_value: field?.advanced?.default_value ?? '', + "multiline": false, + "rich_text_type": "advanced", + "options": [], + "ref_multiple_content_types": true, + "embed_entry": field?.advanced?.embedObjects?.length ? true : false }, "format": field?.advanced?.validationRegex ?? '', "error_messages": { @@ -465,10 +540,13 @@ export const convertToSchemaFormate = ({ field, advanced = true, marketPlacePath } case "reference": { + // Get reference fields from multiple possible sources + const referenceFields = field?.referenceTo || field?.refrenceTo || field?.advanced?.embedObjects || []; + return { data_type: "reference", display_name: field?.title, - reference_to: field?.refrenceTo?.map((item:string) => keyMapper?.[item] || item) ?? [], + reference_to: referenceFields?.map((item:string) => keyMapper?.[item] || item) ?? [], field_metadata: { ref_multiple: true, ref_multiple_content_types: true @@ -516,7 +594,6 @@ export const convertToSchemaFormate = ({ field, advanced = true, marketPlacePath "description": "", "multiline": false, "rich_text_type": "advanced", - "version": 3, "options": [], "ref_multiple_content_types": true, "embed_entry": field?.advanced?.embedObjects?.length ? true : false, diff --git a/api/src/utils/entries-field-creator.utils.ts b/api/src/utils/entries-field-creator.utils.ts index 6684f9d14..b8016d87f 100644 --- a/api/src/utils/entries-field-creator.utils.ts +++ b/api/src/utils/entries-field-creator.utils.ts @@ -1,10 +1,11 @@ -import _ from "lodash"; -import { JSDOM } from "jsdom"; +import _ from 'lodash'; +import { JSDOM } from 'jsdom'; import { htmlToJson } from '@contentstack/json-rte-serializer'; // @ts-ignore import { HTMLToJSON } from 'html-to-json-parser'; +import dayjs from 'dayjs'; -const append = "a"; +const append = 'a'; function startsWithNumber(str: string) { return /^\d/.test(str); @@ -12,18 +13,20 @@ function startsWithNumber(str: string) { const uidCorrector = ({ uid }: any) => { if (startsWithNumber(uid)) { - return `${append}_${_.replace(uid, new RegExp("[ -]", "g"), '_')?.toLowerCase()}` + return `${append}_${_.replace( + uid, + new RegExp('[ -]', 'g'), + '_' + )?.toLowerCase()}`; } - return _.replace(uid, new RegExp("[ -]", "g"), '_')?.toLowerCase() -} - - + return _.replace(uid, new RegExp('[ -]', 'g'), '_')?.toLowerCase(); +}; -const attachJsonRte = ({ content = "" }: any) => { +const attachJsonRte = ({ content = '' }: any) => { const dom = new JSDOM(content); - const htmlDoc = dom.window.document.querySelector("body"); + const htmlDoc = dom.window.document.querySelector('body'); return htmlToJson(htmlDoc); -} +}; type Table = { [key: string]: any }; @@ -33,23 +36,23 @@ export function unflatten(table: Table): any { for (const path in table) { let cursor: any = result; const length: number = path.length; - let property: string = ""; + let property: string = ''; let index: number = 0; while (index < length) { const char: string = path.charAt(index); - if (char === "[") { + if (char === '[') { const start: number = index + 1; - const end: number = path.indexOf("]", start); - cursor = (cursor[property] = cursor[property] || []); + const end: number = path.indexOf(']', start); + cursor = cursor[property] = cursor[property] || []; property = path.slice(start, end); index = end + 1; } else { - cursor = (cursor[property] = cursor[property] || {}); - const start: number = char === "." ? index + 1 : index; - const bracket: number = path.indexOf("[", start); - const dot: number = path.indexOf(".", start); + cursor = cursor[property] = cursor[property] || {}; + const start: number = char === '.' ? index + 1 : index; + const bracket: number = path.indexOf('[', start); + const dot: number = path.indexOf('.', start); let end: number; if (bracket < 0 && dot < 0) { @@ -69,20 +72,18 @@ export function unflatten(table: Table): any { cursor[property] = table[path]; } - return result[""]; + return result['']; } - - -const htmlConverter = async ({ content = "" }: any) => { +const htmlConverter = async ({ content = '' }: any) => { const dom = `
${content}
`; return await Promise.resolve(HTMLToJSON(dom, true)); -} +}; const getAssetsUid = ({ url }: any) => { if (url?.includes('/media')) { - if (url?.includes("?")) { - url = url?.split("?")?.[0]?.replace('.jpg', '') + if (url?.includes('?')) { + url = url?.split('?')?.[0]?.replace('.jpg', ''); } const newUrl = url?.match?.(/\/media\/(.*).ashx/)?.[1]; if (newUrl !== undefined) { @@ -93,7 +94,7 @@ const getAssetsUid = ({ url }: any) => { } else { return url; } -} +}; function flatten(data: any) { const result: any = {}; @@ -103,36 +104,40 @@ function flatten(data: any) { } else if (Array.isArray(cur)) { let l; for (let i = 0, l = cur?.length; i < l; i++) - recurse(cur?.[i], prop + "[" + i + "]"); + recurse(cur?.[i], prop + '[' + i + ']'); if (l == 0) result[prop] = []; } else { let isEmpty = true; for (const p in cur) { isEmpty = false; - recurse(cur[p], prop ? prop + "." + p : p); + recurse(cur[p], prop ? prop + '.' + p : p); } if (isEmpty && prop) result[prop] = {}; } } - recurse(data, ""); + recurse(data, ''); return result; } - -const findAssestInJsoRte = (jsonValue: any, allAssetJSON: any, idCorrector: any) => { +const findAssestInJsoRte = ( + jsonValue: any, + allAssetJSON: any, + idCorrector: any +) => { const flattenHtml = flatten(jsonValue); for (const [key, value] of Object.entries(flattenHtml)) { - if (value === "img") { - const newKey = key?.replace(".type", "") - const htmlData = _.get(jsonValue, newKey) - if (htmlData?.type === "img" && htmlData?.attrs) { - const urlRegex: any = /^(https?:\/\/)?([\da-z.-]+)\.([a-z.]{2,6})([\/\w.-]*)*\/?$/; + if (value === 'img') { + const newKey = key?.replace('.type', ''); + const htmlData = _.get(jsonValue, newKey); + if (htmlData?.type === 'img' && htmlData?.attrs) { + const urlRegex: any = + /^(https?:\/\/)?([\da-z.-]+)\.([a-z.]{2,6})([\/\w.-]*)*\/?$/; const uid = getAssetsUid({ url: htmlData?.attrs?.url }); if (!uid?.match(urlRegex)) { let asset: any = {}; if (uid?.includes('/')) { for (const value of Object.values(allAssetJSON)) { - if (value?.assetPath === `${uid}/`) { + if ((value as any)?.assetPath === `${uid}/`) { asset = value; } } @@ -142,41 +147,45 @@ const findAssestInJsoRte = (jsonValue: any, allAssetJSON: any, idCorrector: any) } if (asset?.uid) { const updated = { - "uid": htmlData?.uid, - "type": "reference", - "attrs": { - "display-type": "display", - "asset-uid": asset?.uid, - "content-type-uid": "sys_assets", - "asset-link": asset?.urlPath, - "asset-name": asset?.title, - "asset-type": asset?.content_type, - "type": "asset", - "class-name": "embedded-asset", - "inline": false + uid: htmlData?.uid, + type: 'reference', + attrs: { + 'display-type': 'display', + 'asset-uid': asset?.uid, + 'content-type-uid': 'sys_assets', + 'asset-link': asset?.urlPath, + 'asset-name': asset?.title, + 'asset-type': asset?.content_type, + type: 'asset', + 'class-name': 'embedded-asset', + inline: false, }, - "children": [ + children: [ { - "text": "" - } - ] - } - _.set(jsonValue, newKey, updated) + text: '', + }, + ], + }; + _.set(jsonValue, newKey, updated); } } else { - console.info('uid not found', uid) + console.info('uid not found', uid); } } } } return jsonValue; -} - - - - -export const entriesFieldCreator = async ({ field, content, idCorrector, allAssetJSON, contentTypes, entriesData, locale }: any) => { - +}; + +export const entriesFieldCreator = async ({ + field, + content, + idCorrector, + allAssetJSON, + contentTypes, + entriesData, + locale, +}: any) => { switch (field?.contentstackFieldType) { case 'multi_line_text': case 'single_line_text': @@ -185,26 +194,32 @@ export const entriesFieldCreator = async ({ field, content, idCorrector, allAsse } case 'json': { - const jsonData = attachJsonRte({ content }) + const jsonData = attachJsonRte({ content }); return findAssestInJsoRte(jsonData, allAssetJSON, idCorrector); } case 'dropdown': { - const isOptionPresent = field?.advanced?.options?.find((ops: any) => ops?.key === content || ops?.value === content); + const isOptionPresent = field?.advanced?.options?.find( + (ops: any) => ops?.key === content || ops?.value === content + ); if (isOptionPresent) { if (field?.advanced?.Multiple) { if (!isOptionPresent?.key) { - return isOptionPresent + return isOptionPresent; } return isOptionPresent; } return isOptionPresent?.value ?? null; } else { if (field?.advanced?.default_value) { - const isOptionDefaultValue = field?.advanced?.options?.find((ops: any) => ops?.key === field?.advanced?.default_value || ops?.value === field?.advanced?.default_value); + const isOptionDefaultValue = field?.advanced?.options?.find( + (ops: any) => + ops?.key === field?.advanced?.default_value || + ops?.value === field?.advanced?.default_value + ); if (field?.advanced?.Multiple) { if (!isOptionDefaultValue?.key) { - return isOptionDefaultValue + return isOptionDefaultValue; } return isOptionDefaultValue; } @@ -217,7 +232,7 @@ export const entriesFieldCreator = async ({ field, content, idCorrector, allAsse case 'number': { if (typeof content === 'string') { - return parseInt?.(content) + return parseInt?.(content); } return content; } @@ -226,10 +241,12 @@ export const entriesFieldCreator = async ({ field, content, idCorrector, allAsse const fileData = attachJsonRte({ content }); for (const item of fileData?.children ?? []) { if (item?.attrs?.['redactor-attributes']?.mediaid) { - const assetUid = idCorrector({ id: item?.attrs?.['redactor-attributes']?.mediaid }); + const assetUid = idCorrector({ + id: item?.attrs?.['redactor-attributes']?.mediaid, + }); return allAssetJSON?.[assetUid] ?? null; } else { - console.info('more', item?.attrs) + console.info('more', item?.attrs); } } return null; @@ -237,7 +254,7 @@ export const entriesFieldCreator = async ({ field, content, idCorrector, allAsse //need to change this case 'link': { - const linkType: any = await htmlConverter({ content }) + const linkType: any = await htmlConverter({ content }); let obj: any = { title: '', href: '' }; if (typeof linkType === 'string') { const parseData = JSON?.parse?.(linkType); @@ -246,10 +263,10 @@ export const entriesFieldCreator = async ({ field, content, idCorrector, allAsse if (item?.type === 'link') { obj = { title: item?.attributes?.id, - href: item?.attributes?.url ?? '' - } + href: item?.attributes?.url ?? '', + }; } - }) + }); } } return obj; @@ -257,31 +274,37 @@ export const entriesFieldCreator = async ({ field, content, idCorrector, allAsse case 'reference': { const refs: any = []; - if (field?.refrenceTo?.length) { + if (field?.refrenceTo?.length && content) { field?.refrenceTo?.forEach((entry: any) => { - const templatePresent = entriesData?.find((tel: any) => uidCorrector({ uid: tel?.template }) === entry); + const templatePresent = entriesData?.find( + (tel: any) => uidCorrector({ uid: tel?.template }) === entry + ); content?.split('|')?.forEach((id: string) => { - const entryid = templatePresent?.locale?.[locale]?.[idCorrector({ id })]; - if (entryid) { + if (id && id.trim()) { + const correctedId = idCorrector({ id: id.trim() }); + + // Check if entry exists in entriesData + const entryExists = + templatePresent?.locale?.[locale]?.[correctedId]; + + // For Drupal, we create references even if the entry doesn't exist yet + // as entries might be created in a different order refs?.push({ - "uid": idCorrector({ id }), - "_content_type_uid": entry - }) - } else { - // console.info("no entry for following id", id) + uid: correctedId, + _content_type_uid: entry, + }); } - }) - }) - } else { - console.info('test ====>'); + }); + }); } return refs; } - case 'global_field': { - const globalFieldsSchema = contentTypes?.find?.((gfd: any) => - gfd?.contentstackUid === field?.contentstackFieldUid && gfd?.type === 'global_field' + const globalFieldsSchema = contentTypes?.find?.( + (gfd: any) => + gfd?.contentstackUid === field?.contentstackFieldUid && + gfd?.type === 'global_field' ); if (globalFieldsSchema?.fieldMapping) { const mainSchema = []; @@ -290,9 +313,12 @@ export const entriesFieldCreator = async ({ field, content, idCorrector, allAsse if (item?.contentstackFieldType === 'group') { group[item?.contentstackFieldUid] = { ...item, fieldMapping: [] }; } else { - const groupSchema = group[item?.contentstackFieldUid?.split('.')?.[0]]; + const groupSchema = + group[item?.contentstackFieldUid?.split('.')?.[0]]; if (groupSchema) { - group?.[groupSchema?.contentstackFieldUid]?.fieldMapping?.push(item); + group?.[groupSchema?.contentstackFieldUid]?.fieldMapping?.push( + item + ); } else { mainSchema?.push(item); } @@ -302,22 +328,60 @@ export const entriesFieldCreator = async ({ field, content, idCorrector, allAsse const obj: any = {}; mainSchema?.forEach(async (field: any) => { if (field?.['uid']) { - obj[field?.contentstackFieldUid] = await entriesFieldCreator({ field, content }); + obj[field?.contentstackFieldUid] = await entriesFieldCreator({ + field, + content, + }); } else { Object?.values(field)?.forEach((item: any) => { if (item?.contentstackFieldType === 'group') { item?.fieldMapping?.forEach(async (ele: any) => { - obj[ele?.contentstackFieldUid] = await entriesFieldCreator({ field: ele, content }); - }) + obj[ele?.contentstackFieldUid] = await entriesFieldCreator({ + field: ele, + content, + }); + }); } - }) + }); } - }) + }); return await obj; } break; } + case 'isodate': { + // Handle Unix timestamp conversion to ISO date string + if ( + typeof content === 'number' || + (typeof content === 'string' && !isNaN(Number(content))) + ) { + const timestamp = + typeof content === 'string' ? parseInt(content) : content; + return dayjs.unix(timestamp).toISOString(); + } + // If it's already a date string, try to parse and convert + if (typeof content === 'string') { + const parsed = dayjs(content); + if (parsed.isValid()) { + return parsed.toISOString(); + } + } + return content; + } + + case 'taxonomy': { + // Handle taxonomy field - return as-is if it's already in the expected format + if (Array.isArray(content)) { + return content; + } + // If it's a single taxonomy object, wrap in array + if (content && typeof content === 'object' && content.taxonomy_uid) { + return [content]; + } + return content; + } + case 'boolean': { return typeof content === 'string' && content === '1' ? true : false; } @@ -327,7 +391,4 @@ export const entriesFieldCreator = async ({ field, content, idCorrector, allAsse return content; } } -} - - - +}; From 874f33e5dfa40e06eea09081eee9b5f5ea4360f8 Mon Sep 17 00:00:00 2001 From: sauravraw Date: Fri, 3 Oct 2025 19:49:07 +0530 Subject: [PATCH 08/37] test --- api/src/services/drupal.service.ts | 61 +- api/src/services/drupal/assets.service.ts | 10 - api/src/services/drupal/entries.service.ts | 153 ++--- api/src/services/migration.service.ts | 568 ++++++++++++++----- api/src/utils/entries-field-creator.utils.ts | 78 ++- 5 files changed, 548 insertions(+), 322 deletions(-) diff --git a/api/src/services/drupal.service.ts b/api/src/services/drupal.service.ts index 3daf94cee..58fd09af6 100644 --- a/api/src/services/drupal.service.ts +++ b/api/src/services/drupal.service.ts @@ -10,10 +10,10 @@ import { generateContentTypeSchemas } from './drupal/content-types.service.js'; /** * Drupal migration service with SQL-based data extraction. - * + * * All functions use direct database connections to extract data from Drupal * following the original migration patterns. - * + * * IMPORTANT: Run in this order for proper dependency resolution: * 1. createQuery - Generate dynamic queries from database analysis (MUST RUN FIRST) * 2. generateContentTypeSchemas - Convert upload-api schema to API content types (MUST RUN AFTER upload-api) @@ -28,36 +28,41 @@ export const drupalService = { createQuery, // Generate dynamic queries from database analysis (MUST RUN FIRST) createQueryConfig, // Helper: Create query configuration file for dynamic SQL generateContentTypeSchemas, // Convert upload-api schema to API content types (MUST RUN AFTER upload-api) - createAssets: (dbConfig: any, destination_stack_id: string, projectId: string, isTest = false, drupalAssetsConfig?: any) => { - console.info('๐Ÿ” === DRUPAL SERVICE - ASSETS WRAPPER ==='); - console.info('๐Ÿ“‹ Received DB Config from Migration Service:', JSON.stringify(dbConfig, null, 2)); - console.info('๐Ÿ“‹ Received Assets Config from Migration Service:', JSON.stringify(drupalAssetsConfig, null, 2)); - console.info('๐Ÿ“‹ Stack ID:', destination_stack_id); - console.info('๐Ÿ“‹ Project ID:', projectId); - console.info('๐Ÿ“‹ Is Test:', isTest); - console.info('=========================================='); + createAssets: ( + dbConfig: any, + destination_stack_id: string, + projectId: string, + isTest = false, + drupalAssetsConfig?: any + ) => { return createAssets( - dbConfig, - destination_stack_id, - projectId, - drupalAssetsConfig?.base_url || "", - drupalAssetsConfig?.public_path || "/sites/default/files/", + dbConfig, + destination_stack_id, + projectId, + drupalAssetsConfig?.base_url || '', + drupalAssetsConfig?.public_path || '/sites/default/files/', isTest ); }, - createRefrence, // Create reference mappings for relationships (run before entries) - createTaxonomy, // Extract and process Drupal taxonomies (vocabularies and terms) - createEntry: (dbConfig: any, destination_stack_id: string, projectId: string, isTest = false, masterLocale = 'en-us', contentTypeMapping: any[] = []) => { - console.info('๐Ÿ” === DRUPAL SERVICE - ENTRIES WRAPPER ==='); - console.info('๐Ÿ“‹ Received Config from Migration Service:', JSON.stringify(dbConfig, null, 2)); - console.info('๐Ÿ“‹ Stack ID:', destination_stack_id); - console.info('๐Ÿ“‹ Project ID:', projectId); - console.info('๐Ÿ“‹ Is Test:', isTest); - console.info('๐Ÿ“‹ Master Locale:', masterLocale); - console.info('๐Ÿ“‹ Content Type Mapping Count:', contentTypeMapping.length); - console.info('============================================'); - return createEntry(dbConfig, destination_stack_id, projectId, isTest, masterLocale, contentTypeMapping); + createRefrence, // Create reference mappings for relationships (run before entries) + createTaxonomy, // Extract and process Drupal taxonomies (vocabularies and terms) + createEntry: ( + dbConfig: any, + destination_stack_id: string, + projectId: string, + isTest = false, + masterLocale = 'en-us', + contentTypeMapping: any[] = [] + ) => { + return createEntry( + dbConfig, + destination_stack_id, + projectId, + isTest, + masterLocale, + contentTypeMapping + ); }, - createLocale, // Create locale configurations + createLocale, // Create locale configurations createVersionFile, // Create version metadata file }; diff --git a/api/src/services/drupal/assets.service.ts b/api/src/services/drupal/assets.service.ts index 70adc8c81..4bd16e1c5 100644 --- a/api/src/services/drupal/assets.service.ts +++ b/api/src/services/drupal/assets.service.ts @@ -338,16 +338,6 @@ export const createAssets = async ( let connection: mysql.Connection | null = null; try { - console.info('๐Ÿ” === DRUPAL ASSETS SERVICE CONFIG ==='); - console.info('๐Ÿ“‹ Database Config:', JSON.stringify(dbConfig, null, 2)); - console.info('๐Ÿ“‹ Destination Stack ID:', destination_stack_id); - console.info('๐Ÿ“‹ Project ID:', projectId); - console.info('๐Ÿ“‹ Base URL:', baseUrl); - console.info('๐Ÿ“‹ Public Path:', publicPath); - console.info('๐Ÿ“‹ Is Test Migration:', isTest); - console.info('๐Ÿ“‹ Function:', srcFunc); - console.info('========================================'); - const assetsSave = path.join(DATA, destination_stack_id, ASSETS_DIR_NAME); const assetMasterFolderPath = path.join( DATA, diff --git a/api/src/services/drupal/entries.service.ts b/api/src/services/drupal/entries.service.ts index 8080ae198..f09203a38 100644 --- a/api/src/services/drupal/entries.service.ts +++ b/api/src/services/drupal/entries.service.ts @@ -156,7 +156,7 @@ const loadTaxonomyReferences = async ( return lookup; } catch (error) { - console.warn('Could not load taxonomy references:', error); + console.error('Could not load taxonomy references:', error); return {}; } }; @@ -241,7 +241,7 @@ const fetchFieldConfigs = async ( fieldConfigs.push(configData as DrupalFieldConfig); } } catch (parseError) { - console.warn( + console.error( `Failed to parse field config for ${row.name}:`, parseError ); @@ -299,6 +299,7 @@ const isConversionAllowed = ( targetType: string ): boolean => { const conversionRules: { [key: string]: string[] } = { + // โœ… Single line can upgrade to multi-line, HTML RTE, or JSON RTE single_line: [ 'single_line_text', 'text', @@ -306,9 +307,12 @@ const isConversionAllowed = ( 'html', 'json', ], - multi_line: ['multi_line_text', 'text', 'html', 'json'], // Cannot convert to single_line_text - html_rte: ['html', 'json'], // Cannot convert to single_line_text or multi_line_text - json_rte: ['json', 'html'], // Cannot convert to single_line_text or multi_line_text + // โœ… Multi-line can upgrade to HTML RTE or JSON RTE (but not downgrade to single-line) + multi_line: ['multi_line_text', 'text', 'html', 'json'], + // โœ… HTML RTE can only convert to JSON RTE (no downgrades to text fields) + html_rte: ['html', 'json'], + // โœ… JSON RTE can only convert to HTML RTE (no downgrades to text fields) + json_rte: ['json', 'html'], }; return conversionRules[sourceType]?.includes(targetType) || false; @@ -334,7 +338,7 @@ const processFieldByType = ( // Check if conversion is allowed if (!isConversionAllowed(sourceType, targetType)) { - console.warn( + console.error( `Conversion not allowed: ${sourceType} โ†’ ${targetType}. Keeping original value.` ); return value; @@ -354,7 +358,7 @@ const processFieldByType = ( .trim(); return textContent; } catch (error) { - console.warn( + console.error( 'Failed to convert JSON RTE to single line text:', error ); @@ -396,7 +400,7 @@ const processFieldByType = ( }) || '' ); } catch (error) { - console.warn('Failed to convert JSON RTE to HTML:', error); + console.error('Failed to convert JSON RTE to HTML:', error); return String(value || ''); } } @@ -416,7 +420,7 @@ const processFieldByType = ( return htmlToJson(htmlDoc); } } catch (error) { - console.warn('Failed to convert HTML to JSON RTE:', error); + console.error('Failed to convert HTML to JSON RTE:', error); } } else if (typeof value === 'string') { // Plain text to JSON RTE @@ -427,7 +431,7 @@ const processFieldByType = ( return htmlToJson(htmlDoc); } } catch (error) { - console.warn('Failed to convert text to JSON RTE:', error); + console.error('Failed to convert text to JSON RTE:', error); } } // If already JSON RTE or conversion failed, return as-is @@ -454,11 +458,19 @@ const processFieldByType = ( }) || '

' ); } catch (error) { - console.warn('Failed to convert JSON RTE to HTML:', error); + console.error('Failed to convert JSON RTE to HTML:', error); return value; } + } else if (typeof value === 'string') { + // Check if it's already HTML + if (/<\/?[a-z][\s\S]*>/i.test(value)) { + // Already HTML, return as-is + return value; + } else { + // Plain text to HTML - wrap in paragraph tags + return `

${value}

`; + } } - // If already HTML or plain text, return as-is return typeof value === 'string' ? value : String(value || ''); } @@ -468,7 +480,7 @@ const processFieldByType = ( try { return jsonToMarkdown(value); } catch (error) { - console.warn('Failed to convert JSON RTE to Markdown:', error); + console.error('Failed to convert JSON RTE to Markdown:', error); return value; } } @@ -489,7 +501,7 @@ const processFieldByType = ( return assetReference; } - console.warn( + console.error( `Asset ${assetKey} not found or invalid, excluding from array` ); return null; @@ -507,7 +519,7 @@ const processFieldByType = ( return assetReference; } - console.warn(`Asset ${assetKey} not found or invalid, removing field`); + console.error(`Asset ${assetKey} not found or invalid, removing field`); return undefined; // Return undefined to indicate field should be removed } return value; @@ -587,12 +599,6 @@ const consolidateTaxonomyFields = ( const fieldsToRemove: string[] = []; const seenTermUids = new Set(); // Track unique term_uid values - console.log( - `๐Ÿท๏ธ Starting taxonomy consolidation for ${contentType} with ${ - Object.keys(processedEntry).length - } fields` - ); - // Iterate through all fields in the processed entry for (const [fieldKey, fieldValue] of Object.entries(processedEntry)) { // Extract field name from key (remove _target_id suffix) @@ -600,10 +606,6 @@ const consolidateTaxonomyFields = ( // Check if this is a taxonomy field using field analysis if (isTaxonomyField(fieldName, contentType, taxonomyFieldMapping)) { - console.log( - `๐Ÿท๏ธ Found taxonomy field in consolidation: ${fieldKey} -> ${fieldName}` - ); - // Validate that field value is an array with taxonomy structure if (Array.isArray(fieldValue)) { for (const taxonomyItem of fieldValue) { @@ -642,9 +644,6 @@ const consolidateTaxonomyFields = ( // Add consolidated taxonomy field if we have any taxonomies if (consolidatedTaxonomies.length > 0) { consolidatedEntry.taxonomies = consolidatedTaxonomies; - console.log( - `๐Ÿท๏ธ Consolidated ${fieldsToRemove.length} taxonomy fields into 'taxonomies' with ${consolidatedTaxonomies.length} unique terms for ${contentType}` - ); } return consolidatedEntry; @@ -680,13 +679,6 @@ const processFieldData = async ( .replace(/_status$/, '') .replace(/_uri$/, ''); - // Debug: Log all fields being processed - if (dataKey.endsWith('_target_id')) { - console.log( - `๐Ÿ” Processing _target_id field: ${dataKey} -> ${fieldName} (value: ${value}) for content type: ${contentType}` - ); - } - // Handle asset fields using field analysis if ( dataKey.endsWith('_target_id') && @@ -710,14 +702,10 @@ const processFieldData = async ( if (dataKey.endsWith('_target_id') && typeof value === 'number') { // Check if this is a taxonomy field using our field analysis if (isTaxonomyField(fieldName, contentType, taxonomyFieldMapping)) { - console.log( - `๐Ÿท๏ธ Processing taxonomy field: ${fieldName} (${dataKey}) with value: ${value} for content type: ${contentType}` - ); // Look up taxonomy reference using drupal_term_id const taxonomyRef = taxonomyReferenceLookup[value]; if (taxonomyRef) { - console.log(`๐Ÿท๏ธ Found taxonomy reference for ${value}:`, taxonomyRef); // Transform to array format with taxonomy_uid and term_uid (no drupal_term_id) processedData[dataKey] = [ { @@ -726,9 +714,6 @@ const processFieldData = async ( }, ]; } else { - console.log( - `โš ๏ธ No taxonomy reference found for drupal_term_id: ${value}` - ); // Fallback to numeric tid if lookup failed processedData[dataKey] = value; } @@ -1107,13 +1092,6 @@ const processEntries = async ( entriesByLocale[entryLocale].push(entry); }); - console.log( - `๐Ÿ“ Found entries in ${ - Object.keys(entriesByLocale).length - } locales for ${contentType}:`, - Object.keys(entriesByLocale) - ); - // ๐Ÿ”„ Apply locale folder transformation rules (same as locale service) // Rules: // - "und" alone โ†’ "en-us" @@ -1127,10 +1105,6 @@ const processEntries = async ( const hasEn = allLocales.includes('en'); const hasEnUs = allLocales.includes('en-us'); - console.log( - `๐Ÿ” Locale Analysis: hasUnd=${hasUnd}, hasEn=${hasEn}, hasEnUs=${hasEnUs}` - ); - // Transform locale folder names based on business rules Object.entries(entriesByLocale).forEach(([originalLocale, entries]) => { let targetFolder = originalLocale; @@ -1139,25 +1113,15 @@ const processEntries = async ( if (hasEn && hasEnUs) { // Rule 4: All three "en" + "und" + "en-us" โ†’ all three stays targetFolder = 'und'; - console.log(`๐Ÿ”„ "und" entries โ†’ "und" folder (all three present)`); } else if (hasEnUs && !hasEn) { // Rule 2: "und" + "en-us" โ†’ "und" become "en", "en-us" stays targetFolder = 'en'; - console.log( - `๐Ÿ”„ Transforming "und" entries โ†’ "en" folder (Rule 2: und+en-us)` - ); } else if (hasEn && !hasEnUs) { // Rule 3: "en" + "und" โ†’ "und" becomes "en-us", "en" stays targetFolder = 'en-us'; - console.log( - `๐Ÿ”„ Transforming "und" entries โ†’ "en-us" folder (Rule 3: en+und)` - ); } else if (!hasEn && !hasEnUs) { // Rule 1: "und" alone โ†’ "en-us" targetFolder = 'en-us'; - console.log( - `๐Ÿ”„ Transforming "und" entries โ†’ "en-us" folder (Rule 1: und alone)` - ); } else { // Keep as is for any other combinations targetFolder = 'und'; @@ -1165,11 +1129,9 @@ const processEntries = async ( } else if (originalLocale === 'en-us') { // "en-us" always stays as "en-us" in all rules targetFolder = 'en-us'; - console.log(`๐Ÿ”„ "en-us" entries โ†’ "en-us" folder (stays as is)`); } else if (originalLocale === 'en') { // "en" always stays as "en" in all rules (never transforms to "und") targetFolder = 'en'; - console.log(`๐Ÿ”„ "en" entries โ†’ "en" folder (stays as is)`); } // Merge entries if target folder already has entries @@ -1178,22 +1140,11 @@ const processEntries = async ( ...transformedEntriesByLocale[targetFolder], ...entries, ]; - console.log( - `๐Ÿ“ Merging ${originalLocale} entries into existing ${targetFolder} folder` - ); } else { transformedEntriesByLocale[targetFolder] = entries; - console.log( - `๐Ÿ“ Creating ${targetFolder} folder for ${originalLocale} entries` - ); } }); - console.log( - `๐Ÿ“‚ Final folder structure:`, - Object.keys(transformedEntriesByLocale) - ); - // Find content type mapping for field type switching const currentContentTypeMapping = contentTypeMapping.find( (ct) => @@ -1206,10 +1157,6 @@ const processEntries = async ( for (const [currentLocale, localeEntries] of Object.entries( transformedEntriesByLocale )) { - console.log( - `๐ŸŒ Processing ${localeEntries.length} entries for transformed locale: ${currentLocale}` - ); - // Create folder structure: entries/contentType/locale/ const contentTypeFolderPath = path.join( MIGRATION_DATA_CONFIG.DATA, @@ -1295,12 +1242,23 @@ const processEntries = async ( // Determine the proper field type based on schema configuration let targetFieldType = schemaField.data_type; - // Handle text fields with multiline metadata + // Handle HTML RTE fields (text with allow_rich_text: true) if ( + schemaField.data_type === 'text' && + schemaField.field_metadata?.allow_rich_text === true + ) { + targetFieldType = 'html'; // โœ… HTML RTE field + } + // Handle JSON RTE fields + else if (schemaField.data_type === 'json') { + targetFieldType = 'json'; // โœ… JSON RTE field + } + // Handle text fields with multiline metadata + else if ( schemaField.data_type === 'text' && schemaField.field_metadata?.multiline ) { - targetFieldType = 'multi_line_text'; // This will be handled as HTML in processFieldByType + targetFieldType = 'multi_line_text'; // โœ… Multi-line text field } // Create a mapping from schema field @@ -1310,13 +1268,9 @@ const processEntries = async ( backupFieldType: schemaField.data_type, advanced: schemaField, }; - - console.log( - `๐Ÿ“‹ Field mapping created for ${fieldName}: ${targetFieldType} (from schema)` - ); } } catch (error: any) { - console.warn( + console.error( `Failed to load content type schema for field ${fieldName}:`, error.message ); @@ -1431,10 +1385,6 @@ const processEntries = async ( // Create mandatory index.json file that maps to the locale file const indexData = { '1': localeFileName }; await writeFile(localeFolderPath, 'index.json', indexData); - - console.log( - `๐Ÿ“ Created mandatory index.json for ${contentType}/${currentLocale} โ†’ ${localeFileName}` - ); } } @@ -1587,14 +1537,6 @@ export const createEntry = async ( let connection: mysql.Connection | null = null; try { - console.info('๐Ÿ” === DRUPAL ENTRIES SERVICE CONFIG ==='); - console.info('๐Ÿ“‹ Database Config:', JSON.stringify(dbConfig, null, 2)); - console.info('๐Ÿ“‹ Destination Stack ID:', destination_stack_id); - console.info('๐Ÿ“‹ Project ID:', projectId); - console.info('๐Ÿ“‹ Is Test Migration:', isTest); - console.info('๐Ÿ“‹ Function:', srcFunc); - console.info('========================================='); - const entriesSave = path.join(DATA, destination_stack_id, ENTRIES_DIR_NAME); const assetsSave = path.join(DATA, destination_stack_id, ASSETS_DIR_NAME); const referencesSave = path.join( @@ -1631,11 +1573,6 @@ export const createEntry = async ( assetFields: assetFieldMapping, } = await analyzeFieldTypes(dbConfig, destination_stack_id, projectId); - console.log( - `๐Ÿท๏ธ Taxonomy field mapping loaded:`, - JSON.stringify(taxonomyFieldMapping, null, 2) - ); - // Fetch field configurations const fieldConfigs = await fetchFieldConfigs( connection, @@ -1646,9 +1583,6 @@ export const createEntry = async ( // Read supporting data - following original page.js pattern // Load assets from index.json (your new format) const assetId = (await readFile(assetsSave, 'index.json')) || {}; - console.log( - `๐Ÿ“ Loaded ${Object.keys(assetId).length} assets from index.json` - ); const referenceId = (await readFile(referencesSave, REFERENCES_FILE_NAME)) || {}; @@ -1662,11 +1596,6 @@ export const createEntry = async ( const taxonomyReferenceLookup = await loadTaxonomyReferences( referencesSave ); - console.log( - `๐Ÿท๏ธ Loaded ${ - Object.keys(taxonomyReferenceLookup).length - } taxonomy reference mappings` - ); // Process each content type from query config (like original) const pageQuery = queryPageConfig.page; diff --git a/api/src/services/migration.service.ts b/api/src/services/migration.service.ts index 7b319b74c..54e2c0238 100644 --- a/api/src/services/migration.service.ts +++ b/api/src/services/migration.service.ts @@ -13,7 +13,7 @@ import { LOCALE_MAPPER, STEPPER_STEPS, CMS, - GET_AUDIT_DATA + GET_AUDIT_DATA, } from '../constants/index.js'; import { BadRequestError, @@ -119,7 +119,12 @@ const createTestStack = async (req: Request): Promise => { `Generating dynamic queries for new test stack (${res?.data?.stack?.api_key})...`, token_payload ); - await customLogger(projectId, res?.data?.stack?.api_key, 'info', startMessage); + await customLogger( + projectId, + res?.data?.stack?.api_key, + 'info', + startMessage + ); // Get database configuration from project const dbConfig = { @@ -127,18 +132,27 @@ const createTestStack = async (req: Request): Promise => { user: project?.legacy_cms?.mySQLDetails?.user, password: project?.legacy_cms?.mySQLDetails?.password || '', database: project?.legacy_cms?.mySQLDetails?.database, - port: project?.legacy_cms?.mySQLDetails?.port || 3306 + port: project?.legacy_cms?.mySQLDetails?.port || 3306, }; // Generate dynamic queries for the new test stack - await drupalService.createQuery(dbConfig, res?.data?.stack?.api_key, projectId); + await drupalService.createQuery( + dbConfig, + res?.data?.stack?.api_key, + projectId + ); const successMessage = getLogMessage( srcFun, `Successfully generated queries for test stack (${res?.data?.stack?.api_key})`, token_payload ); - await customLogger(projectId, res?.data?.stack?.api_key, 'info', successMessage); + await customLogger( + projectId, + res?.data?.stack?.api_key, + 'info', + successMessage + ); } catch (error: any) { const errorMessage = getLogMessage( srcFun, @@ -146,7 +160,12 @@ const createTestStack = async (req: Request): Promise => { token_payload, error ); - await customLogger(projectId, res?.data?.stack?.api_key, 'error', errorMessage); + await customLogger( + projectId, + res?.data?.stack?.api_key, + 'error', + errorMessage + ); // Don't throw error - let test stack creation succeed even if query generation fails } } @@ -164,8 +183,9 @@ const createTestStack = async (req: Request): Promise => { return { data: { data: res.data, - url: `${config.CS_URL[token_payload?.region as keyof typeof config.CS_URL] - }/stack/${res.data.stack.api_key}/dashboard`, + url: `${ + config.CS_URL[token_payload?.region as keyof typeof config.CS_URL] + }/stack/${res.data.stack.api_key}/dashboard`, }, status: res.status, }; @@ -395,7 +415,7 @@ const startTestMigration = async (req: Request): Promise => { destinationStackId: project?.current_test_stack_id, projectId, keyMapper: project?.mapperKeys, - project + project, }); await siteCoreService?.createLocale( req, @@ -405,7 +425,7 @@ const startTestMigration = async (req: Request): Promise => { ); await siteCoreService?.createEnvironment( project?.current_test_stack_id - ) + ); await siteCoreService?.createVersionFile( project?.current_test_stack_id ); @@ -414,20 +434,102 @@ const startTestMigration = async (req: Request): Promise => { } case CMS.WORDPRESS: { if (packagePath) { - await wordpressService?.createLocale(req, project?.current_test_stack_id, projectId, project); - await wordpressService?.getAllAssets(file_path, packagePath, project?.current_test_stack_id, projectId) - await wordpressService?.createAssetFolderFile(file_path, project?.current_test_stack_id, projectId) - await wordpressService?.getAllreference(file_path, packagePath, project?.current_test_stack_id, projectId) - await wordpressService?.extractChunks(file_path, packagePath, project?.current_test_stack_id, projectId) - await wordpressService?.getAllAuthors(file_path, packagePath, project?.current_test_stack_id, projectId, contentTypes, project?.mapperKeys, project?.stackDetails?.master_locale, project) + await wordpressService?.createLocale( + req, + project?.current_test_stack_id, + projectId, + project + ); + await wordpressService?.getAllAssets( + file_path, + packagePath, + project?.current_test_stack_id, + projectId + ); + await wordpressService?.createAssetFolderFile( + file_path, + project?.current_test_stack_id, + projectId + ); + await wordpressService?.getAllreference( + file_path, + packagePath, + project?.current_test_stack_id, + projectId + ); + await wordpressService?.extractChunks( + file_path, + packagePath, + project?.current_test_stack_id, + projectId + ); + await wordpressService?.getAllAuthors( + file_path, + packagePath, + project?.current_test_stack_id, + projectId, + contentTypes, + project?.mapperKeys, + project?.stackDetails?.master_locale, + project + ); //await wordpressService?.extractContentTypes(projectId, project?.current_test_stack_id, contentTypes) - await wordpressService?.getAllTerms(file_path, packagePath, project?.current_test_stack_id, projectId, contentTypes, project?.mapperKeys, project?.stackDetails?.master_locale, project) - await wordpressService?.getAllTags(file_path, packagePath, project?.current_test_stack_id, projectId, contentTypes, project?.mapperKeys, project?.stackDetails?.master_locale, project) - await wordpressService?.getAllCategories(file_path, packagePath, project?.current_test_stack_id, projectId, contentTypes, project?.mapperKeys, project?.stackDetails?.master_locale, project) - await wordpressService?.extractPosts(packagePath, project?.current_test_stack_id, projectId, contentTypes, project?.mapperKeys, project?.stackDetails?.master_locale, project) - await wordpressService?.extractPages(packagePath, project?.current_test_stack_id, projectId, contentTypes, project?.mapperKeys, project?.stackDetails?.master_locale, project) - await wordpressService?.extractGlobalFields(project?.current_test_stack_id, projectId) - await wordpressService?.createVersionFile(project?.current_test_stack_id, projectId); + await wordpressService?.getAllTerms( + file_path, + packagePath, + project?.current_test_stack_id, + projectId, + contentTypes, + project?.mapperKeys, + project?.stackDetails?.master_locale, + project + ); + await wordpressService?.getAllTags( + file_path, + packagePath, + project?.current_test_stack_id, + projectId, + contentTypes, + project?.mapperKeys, + project?.stackDetails?.master_locale, + project + ); + await wordpressService?.getAllCategories( + file_path, + packagePath, + project?.current_test_stack_id, + projectId, + contentTypes, + project?.mapperKeys, + project?.stackDetails?.master_locale, + project + ); + await wordpressService?.extractPosts( + packagePath, + project?.current_test_stack_id, + projectId, + contentTypes, + project?.mapperKeys, + project?.stackDetails?.master_locale, + project + ); + await wordpressService?.extractPages( + packagePath, + project?.current_test_stack_id, + projectId, + contentTypes, + project?.mapperKeys, + project?.stackDetails?.master_locale, + project + ); + await wordpressService?.extractGlobalFields( + project?.current_test_stack_id, + projectId + ); + await wordpressService?.createVersionFile( + project?.current_test_stack_id, + projectId + ); } break; } @@ -482,34 +584,62 @@ const startTestMigration = async (req: Request): Promise => { user: project?.legacy_cms?.mySQLDetails?.user, password: project?.legacy_cms?.mySQLDetails?.password || '', database: project?.legacy_cms?.mySQLDetails?.database, - port: project?.legacy_cms?.mySQLDetails?.port || 3306 + port: project?.legacy_cms?.mySQLDetails?.port || 3306, }; // Get Drupal assets URL configuration from project const drupalAssetsConfig = { - base_url: project?.legacy_cms?.drupalAssetsUrl?.base_url || "", - public_path: project?.legacy_cms?.drupalAssetsUrl?.public_path || "/sites/default/files/" + base_url: project?.legacy_cms?.drupalAssetsUrl?.base_url || '', + public_path: + project?.legacy_cms?.drupalAssetsUrl?.public_path || + '/sites/default/files/', }; - console.info('๐Ÿ” === DRUPAL TEST MIGRATION CONFIG ==='); - console.info('๐Ÿ“‹ Database Config:', JSON.stringify(dbConfig, null, 2)); - console.info('๐Ÿ“‹ Drupal Assets Config:', JSON.stringify(drupalAssetsConfig, null, 2)); - console.info('๐Ÿ“‹ Project ID:', projectId); - console.info('๐Ÿ“‹ Test Stack ID:', project?.current_test_stack_id); - console.info('====================================='); - // Run Drupal migration services in proper order (following test-drupal-services sequence) // NOTE: Dynamic queries are generated during Step 2โ†’3 transition, no need for createQueryConfig - + // Generate content type schemas from upload-api (CRITICAL: Must run after upload-api generates schema) - await drupalService?.generateContentTypeSchemas(project?.current_test_stack_id, projectId); - - await drupalService?.createAssets(dbConfig, project?.current_test_stack_id, projectId, true, drupalAssetsConfig); - await drupalService?.createRefrence(dbConfig, project?.current_test_stack_id, projectId, true); - await drupalService?.createTaxonomy(dbConfig, project?.current_test_stack_id, projectId); - await drupalService?.createEntry(dbConfig, project?.current_test_stack_id, projectId, true, project?.stackDetails?.master_locale, project?.content_mapper || []); - await drupalService?.createLocale(dbConfig, project?.current_test_stack_id, projectId, project); - await drupalService?.createVersionFile(project?.current_test_stack_id, projectId); + await drupalService?.generateContentTypeSchemas( + project?.current_test_stack_id, + projectId + ); + + await drupalService?.createAssets( + dbConfig, + project?.current_test_stack_id, + projectId, + true, + drupalAssetsConfig + ); + await drupalService?.createRefrence( + dbConfig, + project?.current_test_stack_id, + projectId, + true + ); + await drupalService?.createTaxonomy( + dbConfig, + project?.current_test_stack_id, + projectId + ); + await drupalService?.createEntry( + dbConfig, + project?.current_test_stack_id, + projectId, + true, + project?.stackDetails?.master_locale, + project?.content_mapper || [] + ); + await drupalService?.createLocale( + dbConfig, + project?.current_test_stack_id, + projectId, + project + ); + await drupalService?.createVersionFile( + project?.current_test_stack_id, + projectId + ); break; } default: @@ -670,7 +800,7 @@ const startMigration = async (req: Request): Promise => { destinationStackId: project?.destination_stack_id, projectId, keyMapper: project?.mapperKeys, - project + project, }); await siteCoreService?.createLocale( req, @@ -686,20 +816,102 @@ const startMigration = async (req: Request): Promise => { } case CMS.WORDPRESS: { if (packagePath) { - await wordpressService?.createLocale(req, project?.current_test_stack_id, projectId, project); - await wordpressService?.getAllAssets(file_path, packagePath, project?.destination_stack_id, projectId,) - await wordpressService?.createAssetFolderFile(file_path, project?.destination_stack_id, projectId) - await wordpressService?.getAllreference(file_path, packagePath, project?.destination_stack_id, projectId) - await wordpressService?.extractChunks(file_path, packagePath, project?.destination_stack_id, projectId) - await wordpressService?.getAllAuthors(file_path, packagePath, project?.destination_stack_id, projectId, contentTypes, project?.mapperKeys, project?.stackDetails?.master_locale, project) + await wordpressService?.createLocale( + req, + project?.current_test_stack_id, + projectId, + project + ); + await wordpressService?.getAllAssets( + file_path, + packagePath, + project?.destination_stack_id, + projectId + ); + await wordpressService?.createAssetFolderFile( + file_path, + project?.destination_stack_id, + projectId + ); + await wordpressService?.getAllreference( + file_path, + packagePath, + project?.destination_stack_id, + projectId + ); + await wordpressService?.extractChunks( + file_path, + packagePath, + project?.destination_stack_id, + projectId + ); + await wordpressService?.getAllAuthors( + file_path, + packagePath, + project?.destination_stack_id, + projectId, + contentTypes, + project?.mapperKeys, + project?.stackDetails?.master_locale, + project + ); //await wordpressService?.extractContentTypes(projectId, project?.destination_stack_id) - await wordpressService?.getAllTerms(file_path, packagePath, project?.destination_stack_id, projectId, contentTypes, project?.mapperKeys, project?.stackDetails?.master_locale, project) - await wordpressService?.getAllTags(file_path, packagePath, project?.destination_stack_id, projectId, contentTypes, project?.mapperKeys, project?.stackDetails?.master_locale, project) - await wordpressService?.getAllCategories(file_path, packagePath, project?.destination_stack_id, projectId, contentTypes, project?.mapperKeys, project?.stackDetails?.master_locale, project) - await wordpressService?.extractPosts(packagePath, project?.destination_stack_id, projectId, contentTypes, project?.mapperKeys, project?.stackDetails?.master_locale, project) - await wordpressService?.extractPages(packagePath, project?.destination_stack_id, projectId, contentTypes, project?.mapperKeys, project?.stackDetails?.master_locale, project) - await wordpressService?.extractGlobalFields(project?.destination_stack_id, projectId) - await wordpressService?.createVersionFile(project?.destination_stack_id, projectId); + await wordpressService?.getAllTerms( + file_path, + packagePath, + project?.destination_stack_id, + projectId, + contentTypes, + project?.mapperKeys, + project?.stackDetails?.master_locale, + project + ); + await wordpressService?.getAllTags( + file_path, + packagePath, + project?.destination_stack_id, + projectId, + contentTypes, + project?.mapperKeys, + project?.stackDetails?.master_locale, + project + ); + await wordpressService?.getAllCategories( + file_path, + packagePath, + project?.destination_stack_id, + projectId, + contentTypes, + project?.mapperKeys, + project?.stackDetails?.master_locale, + project + ); + await wordpressService?.extractPosts( + packagePath, + project?.destination_stack_id, + projectId, + contentTypes, + project?.mapperKeys, + project?.stackDetails?.master_locale, + project + ); + await wordpressService?.extractPages( + packagePath, + project?.destination_stack_id, + projectId, + contentTypes, + project?.mapperKeys, + project?.stackDetails?.master_locale, + project + ); + await wordpressService?.extractGlobalFields( + project?.destination_stack_id, + projectId + ); + await wordpressService?.createVersionFile( + project?.destination_stack_id, + projectId + ); } break; } @@ -753,34 +965,61 @@ const startMigration = async (req: Request): Promise => { user: project?.legacy_cms?.mySQLDetails?.user, password: project?.legacy_cms?.mySQLDetails?.password || '', database: project?.legacy_cms?.mySQLDetails?.database, - port: project?.legacy_cms?.mySQLDetails?.port || 3306 + port: project?.legacy_cms?.mySQLDetails?.port || 3306, }; // Get Drupal assets URL configuration from project const drupalAssetsConfig = { - base_url: project?.legacy_cms?.drupalAssetsUrl?.base_url || "", - public_path: project?.legacy_cms?.drupalAssetsUrl?.public_path || "/sites/default/files/" + base_url: project?.legacy_cms?.drupalAssetsUrl?.base_url || '', + public_path: + project?.legacy_cms?.drupalAssetsUrl?.public_path || + '/sites/default/files/', }; - - console.info('๐Ÿ” === DRUPAL FINAL MIGRATION CONFIG ==='); - console.info('๐Ÿ“‹ Database Config:', JSON.stringify(dbConfig, null, 2)); - console.info('๐Ÿ“‹ Drupal Assets Config:', JSON.stringify(drupalAssetsConfig, null, 2)); - console.info('๐Ÿ“‹ Project ID:', projectId); - console.info('๐Ÿ“‹ Destination Stack ID:', project?.destination_stack_id); - console.info('====================================='); - // Run Drupal migration services in proper order (following test-drupal-services sequence) // NOTE: Dynamic queries are generated during Step 2โ†’3 transition, no need for createQueryConfig - + // Generate content type schemas from upload-api (CRITICAL: Must run after upload-api generates schema) - await drupalService?.generateContentTypeSchemas(project?.destination_stack_id, projectId); - - await drupalService?.createAssets(dbConfig, project?.destination_stack_id, projectId, false, drupalAssetsConfig); - await drupalService?.createRefrence(dbConfig, project?.destination_stack_id, projectId, false); - await drupalService?.createTaxonomy(dbConfig, project?.destination_stack_id, projectId); - await drupalService?.createLocale(dbConfig, project?.destination_stack_id, projectId, project); - await drupalService?.createEntry(dbConfig, project?.destination_stack_id, projectId, false, project?.stackDetails?.master_locale, project?.content_mapper || []); - await drupalService?.createVersionFile(project?.destination_stack_id, projectId); + await drupalService?.generateContentTypeSchemas( + project?.destination_stack_id, + projectId + ); + + await drupalService?.createAssets( + dbConfig, + project?.destination_stack_id, + projectId, + false, + drupalAssetsConfig + ); + await drupalService?.createRefrence( + dbConfig, + project?.destination_stack_id, + projectId, + false + ); + await drupalService?.createTaxonomy( + dbConfig, + project?.destination_stack_id, + projectId + ); + await drupalService?.createLocale( + dbConfig, + project?.destination_stack_id, + projectId, + project + ); + await drupalService?.createEntry( + dbConfig, + project?.destination_stack_id, + projectId, + false, + project?.stackDetails?.master_locale, + project?.content_mapper || [] + ); + await drupalService?.createVersionFile( + project?.destination_stack_id, + projectId + ); break; } default: @@ -805,24 +1044,36 @@ const getAuditData = async (req: Request): Promise => { const stopIndex = startIndex + limit; const searchText = req?.params?.searchText; const filter = req?.params?.filter; - const srcFunc = "getAuditData"; - if (projectId?.includes('..') || stackId?.includes('..') || moduleName?.includes('..')) { - throw new BadRequestError("Invalid projectId, stackId, or moduleName"); + const srcFunc = 'getAuditData'; + if ( + projectId?.includes('..') || + stackId?.includes('..') || + moduleName?.includes('..') + ) { + throw new BadRequestError('Invalid projectId, stackId, or moduleName'); } try { - const mainPath = process?.cwd() + const mainPath = process?.cwd(); const logsDir = path.join(mainPath, GET_AUDIT_DATA?.MIGRATION_DATA_DIR); const stackFolders = fs.readdirSync(logsDir); - const stackFolder = stackFolders?.find(folder => folder?.startsWith?.(stackId)); + const stackFolder = stackFolders?.find((folder) => + folder?.startsWith?.(stackId) + ); if (!stackFolder) { - throw new BadRequestError("Migration data not found for this stack"); + throw new BadRequestError('Migration data not found for this stack'); } - const auditLogPath = path?.resolve(logsDir, stackFolder, GET_AUDIT_DATA?.LOGS_DIR, GET_AUDIT_DATA?.AUDIT_DIR, GET_AUDIT_DATA?.AUDIT_REPORT); + const auditLogPath = path?.resolve( + logsDir, + stackFolder, + GET_AUDIT_DATA?.LOGS_DIR, + GET_AUDIT_DATA?.AUDIT_DIR, + GET_AUDIT_DATA?.AUDIT_REPORT + ); if (!fs.existsSync(auditLogPath)) { - throw new BadRequestError("Audit log path not found"); + throw new BadRequestError('Audit log path not found'); } const filePath = path?.resolve(auditLogPath, `${moduleName}.json`); let fileData; @@ -836,7 +1087,7 @@ const getAuditData = async (req: Request): Promise => { if (Array.isArray(parsed)) { combinedData = combinedData.concat(parsed); } else if (parsed && typeof parsed === 'object') { - Object.values(parsed).forEach(val => { + Object.values(parsed).forEach((val) => { if (Array.isArray(val)) { combinedData = combinedData.concat(val); } else if (val && typeof val === 'object') { @@ -852,14 +1103,20 @@ const getAuditData = async (req: Request): Promise => { throw new BadRequestError('Access to this file is not allowed.'); } - const fileContent = await fsPromises?.readFile(safeEntriesSelectFieldPath, 'utf8'); + const fileContent = await fsPromises?.readFile( + safeEntriesSelectFieldPath, + 'utf8' + ); try { if (typeof fileContent === 'string') { const parsed = JSON?.parse(fileContent); addToCombined(parsed); } } catch (error) { - logger.error(`Error parsing JSON from file ${entriesSelectFieldPath}:`, error); + logger.error( + `Error parsing JSON from file ${entriesSelectFieldPath}:`, + error + ); throw new BadRequestError('Invalid JSON format in audit file'); } } @@ -893,7 +1150,9 @@ const getAuditData = async (req: Request): Promise => { safeFilePath.includes('..') || !safeFilePath.startsWith(auditLogPath) ) { - throw new BadRequestError('Path traversal detected or access to this file is not allowed.'); + throw new BadRequestError( + 'Path traversal detected or access to this file is not allowed.' + ); } const fileContent = await fsPromises?.readFile(safeFilePath, 'utf8'); try { @@ -908,27 +1167,32 @@ const getAuditData = async (req: Request): Promise => { } if (!fileData) { - throw new BadRequestError(`No audit data found for module: ${moduleName}`); + throw new BadRequestError( + `No audit data found for module: ${moduleName}` + ); } let transformedData = transformAndFlattenData(fileData); if (moduleName === 'Entries_Select_feild') { if (filter != GET_AUDIT_DATA?.FILTERALL) { - const filters = filter?.split("-"); + const filters = filter?.split('-'); transformedData = transformedData?.filter((log) => { return filters?.some((filter) => { return ( - log?.display_type?.toLowerCase()?.includes(filter?.toLowerCase()) || + log?.display_type + ?.toLowerCase() + ?.includes(filter?.toLowerCase()) || log?.data_type?.toLowerCase()?.includes(filter?.toLowerCase()) ); }); }); } - if (searchText && searchText !== null && searchText !== "null") { + if (searchText && searchText !== null && searchText !== 'null') { transformedData = transformedData?.filter((item) => { - return Object?.values(item)?.some(value => - value && - typeof value === 'string' && - value?.toLowerCase?.()?.includes(searchText?.toLowerCase()) + return Object?.values(item)?.some( + (value) => + value && + typeof value === 'string' && + value?.toLowerCase?.()?.includes(searchText?.toLowerCase()) ); }); } @@ -936,25 +1200,24 @@ const getAuditData = async (req: Request): Promise => { return { data: finalData, totalCount: transformedData?.length, - status: HTTP_CODES?.OK + status: HTTP_CODES?.OK, }; } if (filter != GET_AUDIT_DATA?.FILTERALL) { - const filters = filter?.split("-"); + const filters = filter?.split('-'); transformedData = transformedData?.filter((log) => { return filters?.some((filter) => { - return ( - log?.data_type?.toLowerCase()?.includes(filter?.toLowerCase()) - ); + return log?.data_type?.toLowerCase()?.includes(filter?.toLowerCase()); }); }); } - if (searchText && searchText !== null && searchText !== "null") { + if (searchText && searchText !== null && searchText !== 'null') { transformedData = transformedData?.filter((item: any) => { - return Object?.values(item)?.some(value => - value && - typeof value === 'string' && - value?.toLowerCase?.()?.includes(searchText?.toLowerCase()) + return Object?.values(item)?.some( + (value) => + value && + typeof value === 'string' && + value?.toLowerCase?.()?.includes(searchText?.toLowerCase()) ); }); } @@ -963,9 +1226,8 @@ const getAuditData = async (req: Request): Promise => { return { data: paginatedData, totalCount: transformedData?.length, - status: HTTP_CODES?.OK + status: HTTP_CODES?.OK, }; - } catch (error: any) { logger.error( getLogMessage( @@ -984,14 +1246,16 @@ const getAuditData = async (req: Request): Promise => { * Transforms and flattens nested data structure into an array of items * with sequential tuid values */ -const transformAndFlattenData = (data: any): Array<{ [key: string]: any, id: number }> => { +const transformAndFlattenData = ( + data: any +): Array<{ [key: string]: any; id: number }> => { try { const flattenedItems: Array<{ [key: string]: any }> = []; if (Array.isArray(data)) { data?.forEach((item, index) => { flattenedItems?.push({ - ...item ?? {}, - uid: item?.uid || `item-${index}` + ...(item ?? {}), + uid: item?.uid || `item-${index}`, }); }); } else if (typeof data === 'object' && data !== null) { @@ -999,24 +1263,24 @@ const transformAndFlattenData = (data: any): Array<{ [key: string]: any, id: num if (Array.isArray(value)) { value?.forEach((item, index) => { flattenedItems?.push({ - ...item ?? {}, + ...(item ?? {}), parentKey: key, - uid: item?.uid || `${key}-${index}` + uid: item?.uid || `${key}-${index}`, }); }); } else if (typeof value === 'object' && value !== null) { flattenedItems?.push({ ...value, key, - uid: (value as any)?.uid || key + uid: (value as any)?.uid || key, }); } }); } return flattenedItems?.map((item, index) => ({ - ...item ?? {}, - id: index + 1 + ...(item ?? {}), + id: index + 1, })); } catch (error) { console.error('Error transforming data:', error); @@ -1024,41 +1288,47 @@ const transformAndFlattenData = (data: any): Array<{ [key: string]: any, id: num } }; const getLogs = async (req: Request): Promise => { - const projectId = req?.params?.projectId ? path?.basename(req.params.projectId) : ""; - const stackId = req?.params?.stackId ? path?.basename(req.params.stackId) : ""; + const projectId = req?.params?.projectId + ? path?.basename(req.params.projectId) + : ''; + const stackId = req?.params?.stackId + ? path?.basename(req.params.stackId) + : ''; const limit = req?.params?.limit ? parseInt(req.params.limit) : 10; - const startIndex = req?.params?.startIndex ? parseInt(req.params.startIndex) : 0; + const startIndex = req?.params?.startIndex + ? parseInt(req.params.startIndex) + : 0; const stopIndex = startIndex + limit; const searchText = req?.params?.searchText ?? null; - const filter = req?.params?.filter ?? "all"; - const srcFunc = "getLogs"; + const filter = req?.params?.filter ?? 'all'; + const srcFunc = 'getLogs'; if ( !projectId || !stackId || - projectId?.includes("..") || - stackId?.includes("..") + projectId?.includes('..') || + stackId?.includes('..') ) { - throw new BadRequestError("Invalid projectId or stackId"); + throw new BadRequestError('Invalid projectId or stackId'); } try { const mainPath = process?.cwd(); if (!mainPath) { - throw new BadRequestError("Invalid application path"); + throw new BadRequestError('Invalid application path'); } - const logsDir = path?.join(mainPath, "logs"); + const logsDir = path?.join(mainPath, 'logs'); const loggerPath = path?.join(logsDir, projectId, `${stackId}.log`); const absolutePath = path?.resolve(loggerPath); if (!absolutePath?.startsWith(logsDir)) { - throw new BadRequestError("Access to this file is not allowed."); + throw new BadRequestError('Access to this file is not allowed.'); } if (fs.existsSync(absolutePath)) { let index = 0; - const logs = await fs?.promises?.readFile?.(absolutePath, "utf8"); + const logs = await fs?.promises?.readFile?.(absolutePath, 'utf8'); let logEntries = logs - ?.split("\n") + ?.split('\n') ?.map((line) => { try { - const parsedLine = JSON?.parse(line) + const parsedLine = JSON?.parse(line); parsedLine && (parsedLine['id'] = index); ++index; @@ -1069,23 +1339,34 @@ const getLogs = async (req: Request): Promise => { }) ?.filter?.((entry) => entry !== null); if (!logEntries?.length) { - return { logs: [], total: 0, filterOptions: [], status: HTTP_CODES?.OK }; + return { + logs: [], + total: 0, + filterOptions: [], + status: HTTP_CODES?.OK, + }; } - const filterOptions = Array?.from(new Set(logEntries?.map((log) => log?.level))); - const auditStartIndex = logEntries?.findIndex?.(log => log?.message?.includes("Starting audit process")); - const auditEndIndex = logEntries?.findIndex?.(log => log?.message?.includes("Audit process completed")); + const filterOptions = Array?.from( + new Set(logEntries?.map((log) => log?.level)) + ); + const auditStartIndex = logEntries?.findIndex?.((log) => + log?.message?.includes('Starting audit process') + ); + const auditEndIndex = logEntries?.findIndex?.((log) => + log?.message?.includes('Audit process completed') + ); logEntries = logEntries?.slice?.(1, logEntries?.length - 2); - if (filter !== "all") { - const filters = filter?.split("-") ?? []; + if (filter !== 'all') { + const filters = filter?.split('-') ?? []; logEntries = logEntries?.filter((log) => { return filters?.some((filter) => { return log?.level ?.toLowerCase() - ?.includes?.(filter?.toLowerCase() ?? ""); + ?.includes?.(filter?.toLowerCase() ?? ''); }); }); } - if (searchText && searchText !== "null") { + if (searchText && searchText !== 'null') { logEntries = logEntries?.filter?.((log) => matchesSearchText(log, searchText) ); @@ -1095,7 +1376,7 @@ const getLogs = async (req: Request): Promise => { logs: paginatedLogs, total: logEntries?.length ?? 0, filterOptions: filterOptions, - status: HTTP_CODES?.OK + status: HTTP_CODES?.OK, }; } else { logger.error(getLogMessage(srcFunc, HTTP_TEXTS?.LOGS_NOT_FOUND)); @@ -1120,13 +1401,6 @@ const getLogs = async (req: Request): Promise => { export const createSourceLocales = async (req: Request) => { const projectId = req?.params?.projectId; const locales = req?.body?.locale; - - console.log('๐Ÿ” DEBUG: createSourceLocales received:', { - projectId, - locales, - localesType: typeof locales, - localesArray: Array.isArray(locales) - }); try { // Find the project with the specified projectId @@ -1217,5 +1491,5 @@ export const migrationService = { getLogs, createSourceLocales, updateLocaleMapper, - getAuditData + getAuditData, }; diff --git a/api/src/utils/entries-field-creator.utils.ts b/api/src/utils/entries-field-creator.utils.ts index b8016d87f..75fa3121d 100644 --- a/api/src/utils/entries-field-creator.utils.ts +++ b/api/src/utils/entries-field-creator.utils.ts @@ -199,35 +199,63 @@ export const entriesFieldCreator = async ({ } case 'dropdown': { - const isOptionPresent = field?.advanced?.options?.find( - (ops: any) => ops?.key === content || ops?.value === content - ); - if (isOptionPresent) { - if (field?.advanced?.Multiple) { - if (!isOptionPresent?.key) { - return isOptionPresent; - } - return isOptionPresent; - } - return isOptionPresent?.value ?? null; - } else { - if (field?.advanced?.default_value) { - const isOptionDefaultValue = field?.advanced?.options?.find( - (ops: any) => - ops?.key === field?.advanced?.default_value || - ops?.value === field?.advanced?.default_value + // Handle dropdown, radio, and checkbox fields following CSV scenarios + const choices = + field?.advanced?.enum?.choices || field?.advanced?.options || []; + const isMultiple = field?.advanced?.multiple || false; + const displayType = field?.advanced?.display_type || 'dropdown'; + + // Handle multiple values (arrays) - for checkboxes + if (Array.isArray(content)) { + const matchedValues = []; + for (const item of content) { + const match = choices.find( + (choice: any) => + choice?.key === item || + choice?.value === item || + String(choice?.key) === String(item) || + String(choice?.value) === String(item) ); - if (field?.advanced?.Multiple) { - if (!isOptionDefaultValue?.key) { - return isOptionDefaultValue; - } - return isOptionDefaultValue; + if (match) { + // For checkboxes (multiple=true), return the value + // For single fields, return just the value + matchedValues.push(match.value); } - return isOptionDefaultValue?.value ?? null; - } else { - return field?.advanced?.default_value; } + return matchedValues.length > 0 + ? isMultiple + ? matchedValues + : matchedValues[0] + : null; } + + // Handle single values - for dropdown and radio + const match = choices.find( + (choice: any) => + choice?.key === content || + choice?.value === content || + String(choice?.key) === String(content) || + String(choice?.value) === String(content) + ); + + if (match) { + // Always return the value, not the whole choice object + return match.value; + } + + // Fallback to default value + const defaultValue = + field?.advanced?.default_value || + field?.advanced?.field_metadata?.default_value; + if (defaultValue) { + const defaultMatch = choices.find( + (choice: any) => + choice?.key === defaultValue || choice?.value === defaultValue + ); + return defaultMatch ? defaultMatch.value : defaultValue; + } + + return null; } case 'number': { From 73ede1cdb3eb230ad534d8e5297d59b77d5f31f6 Mon Sep 17 00:00:00 2001 From: sauravraw Date: Tue, 7 Oct 2025 13:02:26 +0530 Subject: [PATCH 09/37] Remove drupalMigrationData from Git tracking and update .gitignore --- .gitignore | 3 +- .../drupalSchema/article.json | 263 ------------ .../drupalSchema/fact.json | 81 ---- .../drupalSchema/institute.json | 141 ------- .../drupalSchema/link.json | 151 ------- .../drupalSchema/list.json | 151 ------- .../drupalSchema/page.json | 381 ------------------ .../drupalSchema/profile.json | 251 ------------ .../drupalSchema/school.json | 161 -------- .../drupalSchema/tour.json | 111 ----- .../drupalSchema/video.json | 161 -------- .../taxonomySchema/taxonomySchema.json | 22 - 12 files changed, 2 insertions(+), 1875 deletions(-) delete mode 100644 upload-api/drupalMigrationData/drupalSchema/article.json delete mode 100644 upload-api/drupalMigrationData/drupalSchema/fact.json delete mode 100644 upload-api/drupalMigrationData/drupalSchema/institute.json delete mode 100644 upload-api/drupalMigrationData/drupalSchema/link.json delete mode 100644 upload-api/drupalMigrationData/drupalSchema/list.json delete mode 100644 upload-api/drupalMigrationData/drupalSchema/page.json delete mode 100644 upload-api/drupalMigrationData/drupalSchema/profile.json delete mode 100644 upload-api/drupalMigrationData/drupalSchema/school.json delete mode 100644 upload-api/drupalMigrationData/drupalSchema/tour.json delete mode 100644 upload-api/drupalMigrationData/drupalSchema/video.json delete mode 100644 upload-api/drupalMigrationData/taxonomySchema/taxonomySchema.json diff --git a/.gitignore b/.gitignore index d83248aef..94ef576e6 100644 --- a/.gitignore +++ b/.gitignore @@ -363,4 +363,5 @@ upload-api/extracted_files *MigrationData* *.zip *extracted_files* -*.tsbuildinfo \ No newline at end of file +*.tsbuildinfo +*drupalMigrationData* \ No newline at end of file diff --git a/upload-api/drupalMigrationData/drupalSchema/article.json b/upload-api/drupalMigrationData/drupalSchema/article.json deleted file mode 100644 index a4d3d674e..000000000 --- a/upload-api/drupalMigrationData/drupalSchema/article.json +++ /dev/null @@ -1,263 +0,0 @@ -{ - "title": "article", - "uid": "article", - "schema": [ - { - "uid": "title", - "otherCmsField": "title", - "otherCmsType": "text", - "contentstackField": "title", - "contentstackFieldUid": "title", - "contentstackFieldType": "text", - "backupFieldType": "text", - "backupFieldUid": "title", - "advanced": { - "mandatory": true - } - }, - { - "uid": "url", - "otherCmsField": "url", - "otherCmsType": "text", - "contentstackField": "Url", - "contentstackFieldUid": "url", - "contentstackFieldType": "url", - "backupFieldType": "url", - "backupFieldUid": "url", - "advanced": { - "mandatory": true - } - }, - { - "uid": "body", - "otherCmsField": "Body", - "otherCmsType": "text_with_summary", - "contentstackField": "Body", - "contentstackFieldUid": "body", - "contentstackFieldType": "json", - "backupFieldType": "html", - "backupFieldUid": "body", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_author", - "otherCmsField": "Author", - "otherCmsType": "entity_reference", - "contentstackField": "Author", - "contentstackFieldUid": "field_author_target_id", - "contentstackFieldType": "reference", - "backupFieldType": "reference", - "backupFieldUid": "field_author_target_id", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [ - "profile" - ], - "description": "" - } - }, - { - "uid": "field_external_link", - "otherCmsField": "External Link", - "otherCmsType": "string", - "contentstackField": "External Link", - "contentstackFieldUid": "field_external_link", - "contentstackFieldType": "single_line_text", - "backupFieldType": "multi_line_text", - "backupFieldUid": "field_external_link", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [ - "fact", - "institute", - "link", - "list", - "page", - "profile", - "school", - "tour", - "video" - ], - "description": "" - } - }, - { - "uid": "field_formatted_subtitle", - "otherCmsField": "Formatted Subtitle", - "otherCmsType": "text", - "contentstackField": "Formatted Subtitle", - "contentstackFieldUid": "field_formatted_subtitle", - "contentstackFieldType": "single_line_text", - "backupFieldType": "multi_line_text", - "backupFieldUid": "field_formatted_subtitle", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [ - "fact", - "institute", - "link", - "list", - "page", - "profile", - "school", - "tour", - "video" - ], - "description": "" - } - }, - { - "uid": "field_formatted_title", - "otherCmsField": "Formatted Title", - "otherCmsType": "text_long", - "contentstackField": "Formatted Title", - "contentstackFieldUid": "field_formatted_title", - "contentstackFieldType": "json", - "backupFieldType": "html", - "backupFieldUid": "field_formatted_title", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_image", - "otherCmsField": "Image", - "otherCmsType": "image", - "contentstackField": "Image", - "contentstackFieldUid": "field_image_target_id", - "contentstackFieldType": "file", - "backupFieldType": "file", - "backupFieldUid": "field_image_target_id", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_note", - "otherCmsField": "Note", - "otherCmsType": "string_long", - "contentstackField": "Note", - "contentstackFieldUid": "field_note", - "contentstackFieldType": "json", - "backupFieldType": "html", - "backupFieldUid": "field_note", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_subtitle", - "otherCmsField": "Subtitle", - "otherCmsType": "string", - "contentstackField": "Subtitle", - "contentstackFieldUid": "field_subtitle", - "contentstackFieldType": "single_line_text", - "backupFieldType": "multi_line_text", - "backupFieldUid": "field_subtitle", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [ - "fact", - "institute", - "link", - "list", - "page", - "profile", - "school", - "tour", - "video" - ], - "description": "" - } - }, - { - "uid": "taxonomies", - "otherCmsField": "Taxonomy", - "otherCmsType": "taxonomy", - "contentstackField": "Taxonomy", - "contentstackFieldUid": "taxonomies", - "contentstackFieldType": "taxonomy", - "backupFieldType": "taxonomy", - "backupFieldUid": "taxonomies", - "advanced": { - "taxonomies": [ - { - "taxonomy_uid": "news", - "mandatory": false, - "multiple": true, - "non_localizable": false - }, - { - "taxonomy_uid": "tags", - "mandatory": false, - "multiple": true, - "non_localizable": false - } - ], - "mandatory": false, - "multiple": true, - "non_localizable": false, - "unique": false - } - } - ], - "description": "Schema for article", - "options": { - "is_page": true, - "singleton": false, - "sub_title": [], - "title": "title", - "url_pattern": "/:title", - "url_prefix": "/article/" - } -} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/drupalSchema/fact.json b/upload-api/drupalMigrationData/drupalSchema/fact.json deleted file mode 100644 index 3352891b6..000000000 --- a/upload-api/drupalMigrationData/drupalSchema/fact.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "title": "fact", - "uid": "fact", - "schema": [ - { - "uid": "title", - "otherCmsField": "title", - "otherCmsType": "text", - "contentstackField": "title", - "contentstackFieldUid": "title", - "contentstackFieldType": "text", - "backupFieldType": "text", - "backupFieldUid": "title", - "advanced": { - "mandatory": true - } - }, - { - "uid": "url", - "otherCmsField": "url", - "otherCmsType": "text", - "contentstackField": "Url", - "contentstackFieldUid": "url", - "contentstackFieldType": "url", - "backupFieldType": "url", - "backupFieldUid": "url", - "advanced": { - "mandatory": true - } - }, - { - "uid": "field_body", - "otherCmsField": "Body", - "otherCmsType": "text_long", - "contentstackField": "Body", - "contentstackFieldUid": "field_body", - "contentstackFieldType": "json", - "backupFieldType": "html", - "backupFieldUid": "field_body", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_image", - "otherCmsField": "Image", - "otherCmsType": "image", - "contentstackField": "Image", - "contentstackFieldUid": "field_image_target_id", - "contentstackFieldType": "file", - "backupFieldType": "file", - "backupFieldUid": "field_image_target_id", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - } - ], - "description": "Schema for fact", - "options": { - "is_page": true, - "singleton": false, - "sub_title": [], - "title": "title", - "url_pattern": "/:title", - "url_prefix": "/fact/" - } -} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/drupalSchema/institute.json b/upload-api/drupalMigrationData/drupalSchema/institute.json deleted file mode 100644 index e096500d8..000000000 --- a/upload-api/drupalMigrationData/drupalSchema/institute.json +++ /dev/null @@ -1,141 +0,0 @@ -{ - "title": "institute", - "uid": "institute", - "schema": [ - { - "uid": "title", - "otherCmsField": "title", - "otherCmsType": "text", - "contentstackField": "title", - "contentstackFieldUid": "title", - "contentstackFieldType": "text", - "backupFieldType": "text", - "backupFieldUid": "title", - "advanced": { - "mandatory": true - } - }, - { - "uid": "url", - "otherCmsField": "url", - "otherCmsType": "text", - "contentstackField": "Url", - "contentstackFieldUid": "url", - "contentstackFieldType": "url", - "backupFieldType": "url", - "backupFieldUid": "url", - "advanced": { - "mandatory": true - } - }, - { - "uid": "field_formatted_title", - "otherCmsField": "Formatted Title", - "otherCmsType": "text_long", - "contentstackField": "Formatted Title", - "contentstackFieldUid": "field_formatted_title", - "contentstackFieldType": "json", - "backupFieldType": "html", - "backupFieldUid": "field_formatted_title", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_image", - "otherCmsField": "Image", - "otherCmsType": "image", - "contentstackField": "Image", - "contentstackFieldUid": "field_image_target_id", - "contentstackFieldType": "file", - "backupFieldType": "file", - "backupFieldUid": "field_image_target_id", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_link", - "otherCmsField": "Link", - "otherCmsType": "link", - "contentstackField": "Link", - "contentstackFieldUid": "field_link", - "contentstackFieldType": "link", - "backupFieldType": "link", - "backupFieldUid": "field_link", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_links", - "otherCmsField": "Links", - "otherCmsType": "link", - "contentstackField": "Links", - "contentstackFieldUid": "field_links", - "contentstackFieldType": "link", - "backupFieldType": "link", - "backupFieldUid": "field_links", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_summary", - "otherCmsField": "Summary", - "otherCmsType": "string_long", - "contentstackField": "Summary", - "contentstackFieldUid": "field_summary", - "contentstackFieldType": "json", - "backupFieldType": "html", - "backupFieldUid": "field_summary", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - } - ], - "description": "Schema for institute", - "options": { - "is_page": true, - "singleton": false, - "sub_title": [], - "title": "title", - "url_pattern": "/:title", - "url_prefix": "/institute/" - } -} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/drupalSchema/link.json b/upload-api/drupalMigrationData/drupalSchema/link.json deleted file mode 100644 index 0baa3108c..000000000 --- a/upload-api/drupalMigrationData/drupalSchema/link.json +++ /dev/null @@ -1,151 +0,0 @@ -{ - "title": "link", - "uid": "link", - "schema": [ - { - "uid": "title", - "otherCmsField": "title", - "otherCmsType": "text", - "contentstackField": "title", - "contentstackFieldUid": "title", - "contentstackFieldType": "text", - "backupFieldType": "text", - "backupFieldUid": "title", - "advanced": { - "mandatory": true - } - }, - { - "uid": "url", - "otherCmsField": "url", - "otherCmsType": "text", - "contentstackField": "Url", - "contentstackFieldUid": "url", - "contentstackFieldType": "url", - "backupFieldType": "url", - "backupFieldUid": "url", - "advanced": { - "mandatory": true - } - }, - { - "uid": "field_formatted_title", - "otherCmsField": "Formatted Title", - "otherCmsType": "text_long", - "contentstackField": "Formatted Title", - "contentstackFieldUid": "field_formatted_title", - "contentstackFieldType": "json", - "backupFieldType": "html", - "backupFieldUid": "field_formatted_title", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_image", - "otherCmsField": "Image", - "otherCmsType": "image", - "contentstackField": "Image", - "contentstackFieldUid": "field_image_target_id", - "contentstackFieldType": "file", - "backupFieldType": "file", - "backupFieldUid": "field_image_target_id", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_link", - "otherCmsField": "Link", - "otherCmsType": "link", - "contentstackField": "Link", - "contentstackFieldUid": "field_link", - "contentstackFieldType": "link", - "backupFieldType": "link", - "backupFieldUid": "field_link", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_link_order", - "otherCmsField": "Link Order", - "otherCmsType": "integer", - "contentstackField": "Link Order", - "contentstackFieldUid": "field_link_order", - "contentstackFieldType": "number", - "backupFieldType": "number", - "backupFieldUid": "field_link_order", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "taxonomies", - "otherCmsField": "Taxonomy", - "otherCmsType": "taxonomy", - "contentstackField": "Taxonomy", - "contentstackFieldUid": "taxonomies", - "contentstackFieldType": "taxonomy", - "backupFieldType": "taxonomy", - "backupFieldUid": "taxonomies", - "advanced": { - "taxonomies": [ - { - "taxonomy_uid": "pages", - "mandatory": false, - "multiple": true, - "non_localizable": false - }, - { - "taxonomy_uid": "tags", - "mandatory": false, - "multiple": true, - "non_localizable": false - } - ], - "mandatory": false, - "multiple": true, - "non_localizable": false, - "unique": false - } - } - ], - "description": "Schema for link", - "options": { - "is_page": true, - "singleton": false, - "sub_title": [], - "title": "title", - "url_pattern": "/:title", - "url_prefix": "/link/" - } -} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/drupalSchema/list.json b/upload-api/drupalMigrationData/drupalSchema/list.json deleted file mode 100644 index 010b3c35b..000000000 --- a/upload-api/drupalMigrationData/drupalSchema/list.json +++ /dev/null @@ -1,151 +0,0 @@ -{ - "title": "list", - "uid": "list", - "schema": [ - { - "uid": "title", - "otherCmsField": "title", - "otherCmsType": "text", - "contentstackField": "title", - "contentstackFieldUid": "title", - "contentstackFieldType": "text", - "backupFieldType": "text", - "backupFieldUid": "title", - "advanced": { - "mandatory": true - } - }, - { - "uid": "url", - "otherCmsField": "url", - "otherCmsType": "text", - "contentstackField": "Url", - "contentstackFieldUid": "url", - "contentstackFieldType": "url", - "backupFieldType": "url", - "backupFieldUid": "url", - "advanced": { - "mandatory": true - } - }, - { - "uid": "field_formatted_title", - "otherCmsField": "Formatted Title", - "otherCmsType": "text_long", - "contentstackField": "Formatted Title", - "contentstackFieldUid": "field_formatted_title", - "contentstackFieldType": "json", - "backupFieldType": "html", - "backupFieldUid": "field_formatted_title", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_link", - "otherCmsField": "Link", - "otherCmsType": "link", - "contentstackField": "Link", - "contentstackFieldUid": "field_link", - "contentstackFieldType": "link", - "backupFieldType": "link", - "backupFieldUid": "field_link", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_links", - "otherCmsField": "Links", - "otherCmsType": "link", - "contentstackField": "Links", - "contentstackFieldUid": "field_links", - "contentstackFieldType": "link", - "backupFieldType": "link", - "backupFieldUid": "field_links", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_list_order", - "otherCmsField": "List Order", - "otherCmsType": "integer", - "contentstackField": "List Order", - "contentstackFieldUid": "field_list_order", - "contentstackFieldType": "number", - "backupFieldType": "number", - "backupFieldUid": "field_list_order", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "taxonomies", - "otherCmsField": "Taxonomy", - "otherCmsType": "taxonomy", - "contentstackField": "Taxonomy", - "contentstackFieldUid": "taxonomies", - "contentstackFieldType": "taxonomy", - "backupFieldType": "taxonomy", - "backupFieldUid": "taxonomies", - "advanced": { - "taxonomies": [ - { - "taxonomy_uid": "list", - "mandatory": false, - "multiple": true, - "non_localizable": false - }, - { - "taxonomy_uid": "tags", - "mandatory": false, - "multiple": true, - "non_localizable": false - } - ], - "mandatory": false, - "multiple": true, - "non_localizable": false, - "unique": false - } - } - ], - "description": "Schema for list", - "options": { - "is_page": true, - "singleton": false, - "sub_title": [], - "title": "title", - "url_pattern": "/:title", - "url_prefix": "/list/" - } -} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/drupalSchema/page.json b/upload-api/drupalMigrationData/drupalSchema/page.json deleted file mode 100644 index f826794aa..000000000 --- a/upload-api/drupalMigrationData/drupalSchema/page.json +++ /dev/null @@ -1,381 +0,0 @@ -{ - "title": "page", - "uid": "page", - "schema": [ - { - "uid": "title", - "otherCmsField": "title", - "otherCmsType": "text", - "contentstackField": "title", - "contentstackFieldUid": "title", - "contentstackFieldType": "text", - "backupFieldType": "text", - "backupFieldUid": "title", - "advanced": { - "mandatory": true - } - }, - { - "uid": "url", - "otherCmsField": "url", - "otherCmsType": "text", - "contentstackField": "Url", - "contentstackFieldUid": "url", - "contentstackFieldType": "url", - "backupFieldType": "url", - "backupFieldUid": "url", - "advanced": { - "mandatory": true - } - }, - { - "uid": "field_body", - "otherCmsField": "Body", - "otherCmsType": "text_long", - "contentstackField": "Body", - "contentstackFieldUid": "field_body", - "contentstackFieldType": "json", - "backupFieldType": "html", - "backupFieldUid": "field_body", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_external_link", - "otherCmsField": "External Link", - "otherCmsType": "string", - "contentstackField": "External Link", - "contentstackFieldUid": "field_external_link", - "contentstackFieldType": "single_line_text", - "backupFieldType": "multi_line_text", - "backupFieldUid": "field_external_link", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [ - "article", - "fact", - "institute", - "link", - "list", - "profile", - "school", - "tour", - "video" - ], - "description": "" - } - }, - { - "uid": "field_formatted_title", - "otherCmsField": "Formatted Title", - "otherCmsType": "text_long", - "contentstackField": "Formatted Title", - "contentstackFieldUid": "field_formatted_title", - "contentstackFieldType": "json", - "backupFieldType": "html", - "backupFieldUid": "field_formatted_title", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_hero_body", - "otherCmsField": "Hero Body", - "otherCmsType": "text_long", - "contentstackField": "Hero Body", - "contentstackFieldUid": "field_hero_body", - "contentstackFieldType": "json", - "backupFieldType": "html", - "backupFieldUid": "field_hero_body", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_hero_image", - "otherCmsField": "Hero Image", - "otherCmsType": "image", - "contentstackField": "Hero Image", - "contentstackFieldUid": "field_hero_image_target_id", - "contentstackFieldType": "file", - "backupFieldType": "file", - "backupFieldUid": "field_hero_image_target_id", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_hero_intro", - "otherCmsField": "Hero Intro", - "otherCmsType": "string", - "contentstackField": "Hero Intro", - "contentstackFieldUid": "field_hero_intro", - "contentstackFieldType": "single_line_text", - "backupFieldType": "multi_line_text", - "backupFieldUid": "field_hero_intro", - "advanced": { - "default_value": "ABBREVIATED INTRODUCTORY", - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [ - "article", - "fact", - "institute", - "link", - "list", - "profile", - "school", - "tour", - "video" - ], - "description": "" - } - }, - { - "uid": "field_hero_title", - "otherCmsField": "Hero Title", - "otherCmsType": "text_long", - "contentstackField": "Hero Title", - "contentstackFieldUid": "field_hero_title", - "contentstackFieldType": "json", - "backupFieldType": "html", - "backupFieldUid": "field_hero_title", - "advanced": { - "default_value": "

Page Title Here

\r\n", - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_image", - "otherCmsField": "Image", - "otherCmsType": "image", - "contentstackField": "Image", - "contentstackFieldUid": "field_image_target_id", - "contentstackFieldType": "file", - "backupFieldType": "file", - "backupFieldUid": "field_image_target_id", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_link", - "otherCmsField": "Link", - "otherCmsType": "link", - "contentstackField": "Link", - "contentstackFieldUid": "field_link", - "contentstackFieldType": "link", - "backupFieldType": "link", - "backupFieldUid": "field_link", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_meta_tags", - "otherCmsField": "Meta Tags", - "otherCmsType": "metatag", - "contentstackField": "Meta Tags", - "contentstackFieldUid": "field_meta_tags", - "contentstackFieldType": "single_line_text", - "backupFieldType": "single_line_text", - "backupFieldUid": "field_meta_tags", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_paragraphs", - "otherCmsField": "Paragraphs", - "otherCmsType": "entity_reference_revisions", - "contentstackField": "Paragraphs", - "contentstackFieldUid": "field_paragraphs", - "contentstackFieldType": "single_line_text", - "backupFieldType": "single_line_text", - "backupFieldUid": "field_paragraphs", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_quick_links", - "otherCmsField": "Quick Links", - "otherCmsType": "link", - "contentstackField": "Quick Links", - "contentstackFieldUid": "field_quick_links", - "contentstackFieldType": "link", - "backupFieldType": "link", - "backupFieldUid": "field_quick_links", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_subtitle", - "otherCmsField": "Subtitle", - "otherCmsType": "string", - "contentstackField": "Subtitle", - "contentstackFieldUid": "field_subtitle", - "contentstackFieldType": "single_line_text", - "backupFieldType": "multi_line_text", - "backupFieldUid": "field_subtitle", - "advanced": { - "default_value": "LEARN MORE", - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [ - "article", - "fact", - "institute", - "link", - "list", - "profile", - "school", - "tour", - "video" - ], - "description": "" - } - }, - { - "uid": "field_summary", - "otherCmsField": "Summary", - "otherCmsType": "string_long", - "contentstackField": "Summary", - "contentstackFieldUid": "field_summary", - "contentstackFieldType": "json", - "backupFieldType": "html", - "backupFieldUid": "field_summary", - "advanced": { - "default_value": "Fostering diversity and an intellectual environment, Rice University is a comprehensive research university located on a 300-acre tree-lined campus in Houston, Texas. Rice produces the next generation of leaders and advances tomorrowโ€™s thinking.", - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "taxonomies", - "otherCmsField": "Taxonomy", - "otherCmsType": "taxonomy", - "contentstackField": "Taxonomy", - "contentstackFieldUid": "taxonomies", - "contentstackFieldType": "taxonomy", - "backupFieldType": "taxonomy", - "backupFieldUid": "taxonomies", - "advanced": { - "taxonomies": [ - { - "taxonomy_uid": "pages", - "mandatory": false, - "multiple": true, - "non_localizable": false - }, - { - "taxonomy_uid": "tags", - "mandatory": false, - "multiple": true, - "non_localizable": false - } - ], - "mandatory": false, - "multiple": true, - "non_localizable": false, - "unique": false - } - } - ], - "description": "Schema for page", - "options": { - "is_page": true, - "singleton": false, - "sub_title": [], - "title": "title", - "url_pattern": "/:title", - "url_prefix": "/page/" - } -} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/drupalSchema/profile.json b/upload-api/drupalMigrationData/drupalSchema/profile.json deleted file mode 100644 index 4c8160fad..000000000 --- a/upload-api/drupalMigrationData/drupalSchema/profile.json +++ /dev/null @@ -1,251 +0,0 @@ -{ - "title": "profile", - "uid": "profile", - "schema": [ - { - "uid": "title", - "otherCmsField": "title", - "otherCmsType": "text", - "contentstackField": "title", - "contentstackFieldUid": "title", - "contentstackFieldType": "text", - "backupFieldType": "text", - "backupFieldUid": "title", - "advanced": { - "mandatory": true - } - }, - { - "uid": "url", - "otherCmsField": "url", - "otherCmsType": "text", - "contentstackField": "Url", - "contentstackFieldUid": "url", - "contentstackFieldType": "url", - "backupFieldType": "url", - "backupFieldUid": "url", - "advanced": { - "mandatory": true - } - }, - { - "uid": "body", - "otherCmsField": "Body", - "otherCmsType": "text_with_summary", - "contentstackField": "Body", - "contentstackFieldUid": "body", - "contentstackFieldType": "json", - "backupFieldType": "html", - "backupFieldUid": "body", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_address", - "otherCmsField": "Address", - "otherCmsType": "string", - "contentstackField": "Address", - "contentstackFieldUid": "field_address", - "contentstackFieldType": "single_line_text", - "backupFieldType": "multi_line_text", - "backupFieldUid": "field_address", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [ - "article", - "fact", - "institute", - "link", - "list", - "page", - "school", - "tour", - "video" - ], - "description": "" - } - }, - { - "uid": "field_email", - "otherCmsField": "Email", - "otherCmsType": "email", - "contentstackField": "Email", - "contentstackFieldUid": "field_email", - "contentstackFieldType": "text", - "backupFieldType": "text", - "backupFieldUid": "field_email", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_hours", - "otherCmsField": "Hours", - "otherCmsType": "string", - "contentstackField": "Hours", - "contentstackFieldUid": "field_hours", - "contentstackFieldType": "single_line_text", - "backupFieldType": "multi_line_text", - "backupFieldUid": "field_hours", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [ - "article", - "fact", - "institute", - "link", - "list", - "page", - "school", - "tour", - "video" - ], - "description": "" - } - }, - { - "uid": "field_image", - "otherCmsField": "Image", - "otherCmsType": "image", - "contentstackField": "Image", - "contentstackFieldUid": "field_image_target_id", - "contentstackFieldType": "file", - "backupFieldType": "file", - "backupFieldUid": "field_image_target_id", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_links", - "otherCmsField": "Links", - "otherCmsType": "link", - "contentstackField": "Links", - "contentstackFieldUid": "field_links", - "contentstackFieldType": "link", - "backupFieldType": "link", - "backupFieldUid": "field_links", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_phone", - "otherCmsField": "Phone", - "otherCmsType": "string", - "contentstackField": "Phone", - "contentstackFieldUid": "field_phone", - "contentstackFieldType": "single_line_text", - "backupFieldType": "multi_line_text", - "backupFieldUid": "field_phone", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [ - "article", - "fact", - "institute", - "link", - "list", - "page", - "school", - "tour", - "video" - ], - "description": "" - } - }, - { - "uid": "field_summary", - "otherCmsField": "Summary", - "otherCmsType": "string_long", - "contentstackField": "Summary", - "contentstackFieldUid": "field_summary", - "contentstackFieldType": "json", - "backupFieldType": "html", - "backupFieldUid": "field_summary", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_title", - "otherCmsField": "Title", - "otherCmsType": "string_long", - "contentstackField": "Title", - "contentstackFieldUid": "field_title", - "contentstackFieldType": "json", - "backupFieldType": "html", - "backupFieldUid": "field_title", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - } - ], - "description": "Schema for profile", - "options": { - "is_page": true, - "singleton": false, - "sub_title": [], - "title": "title", - "url_pattern": "/:title", - "url_prefix": "/profile/" - } -} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/drupalSchema/school.json b/upload-api/drupalMigrationData/drupalSchema/school.json deleted file mode 100644 index 6cd733e64..000000000 --- a/upload-api/drupalMigrationData/drupalSchema/school.json +++ /dev/null @@ -1,161 +0,0 @@ -{ - "title": "school", - "uid": "school", - "schema": [ - { - "uid": "title", - "otherCmsField": "title", - "otherCmsType": "text", - "contentstackField": "title", - "contentstackFieldUid": "title", - "contentstackFieldType": "text", - "backupFieldType": "text", - "backupFieldUid": "title", - "advanced": { - "mandatory": true - } - }, - { - "uid": "url", - "otherCmsField": "url", - "otherCmsType": "text", - "contentstackField": "Url", - "contentstackFieldUid": "url", - "contentstackFieldType": "url", - "backupFieldType": "url", - "backupFieldUid": "url", - "advanced": { - "mandatory": true - } - }, - { - "uid": "field_formatted_title", - "otherCmsField": "Formatted Title", - "otherCmsType": "text_long", - "contentstackField": "Formatted Title", - "contentstackFieldUid": "field_formatted_title", - "contentstackFieldType": "json", - "backupFieldType": "html", - "backupFieldUid": "field_formatted_title", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_image", - "otherCmsField": "Image", - "otherCmsType": "image", - "contentstackField": "Image", - "contentstackFieldUid": "field_image_target_id", - "contentstackFieldType": "file", - "backupFieldType": "file", - "backupFieldUid": "field_image_target_id", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_link", - "otherCmsField": "Link", - "otherCmsType": "link", - "contentstackField": "Link", - "contentstackFieldUid": "field_link", - "contentstackFieldType": "link", - "backupFieldType": "link", - "backupFieldUid": "field_link", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_links", - "otherCmsField": "Links", - "otherCmsType": "link", - "contentstackField": "Links", - "contentstackFieldUid": "field_links", - "contentstackFieldType": "link", - "backupFieldType": "link", - "backupFieldUid": "field_links", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_school_order", - "otherCmsField": "School Order", - "otherCmsType": "integer", - "contentstackField": "School Order", - "contentstackFieldUid": "field_school_order", - "contentstackFieldType": "number", - "backupFieldType": "number", - "backupFieldUid": "field_school_order", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_summary", - "otherCmsField": "Summary", - "otherCmsType": "string_long", - "contentstackField": "Summary", - "contentstackFieldUid": "field_summary", - "contentstackFieldType": "json", - "backupFieldType": "html", - "backupFieldUid": "field_summary", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - } - ], - "description": "Schema for school", - "options": { - "is_page": true, - "singleton": false, - "sub_title": [], - "title": "title", - "url_pattern": "/:title", - "url_prefix": "/school/" - } -} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/drupalSchema/tour.json b/upload-api/drupalMigrationData/drupalSchema/tour.json deleted file mode 100644 index 4d6be850c..000000000 --- a/upload-api/drupalMigrationData/drupalSchema/tour.json +++ /dev/null @@ -1,111 +0,0 @@ -{ - "title": "tour", - "uid": "tour", - "schema": [ - { - "uid": "title", - "otherCmsField": "title", - "otherCmsType": "text", - "contentstackField": "title", - "contentstackFieldUid": "title", - "contentstackFieldType": "text", - "backupFieldType": "text", - "backupFieldUid": "title", - "advanced": { - "mandatory": true - } - }, - { - "uid": "url", - "otherCmsField": "url", - "otherCmsType": "text", - "contentstackField": "Url", - "contentstackFieldUid": "url", - "contentstackFieldType": "url", - "backupFieldType": "url", - "backupFieldUid": "url", - "advanced": { - "mandatory": true - } - }, - { - "uid": "field_external_link", - "otherCmsField": "External Link", - "otherCmsType": "string", - "contentstackField": "External Link", - "contentstackFieldUid": "field_external_link", - "contentstackFieldType": "single_line_text", - "backupFieldType": "multi_line_text", - "backupFieldUid": "field_external_link", - "advanced": { - "default_value": "https://www.rice.edu/virtualtours/graduatestudies/tourfiles/flash/index.html?utm_source=Tour&utm_medium=Virtual&utm_campaign=GraduateWindow", - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [ - "article", - "fact", - "institute", - "link", - "list", - "page", - "profile", - "school", - "video" - ], - "description": "" - } - }, - { - "uid": "field_formatted_title", - "otherCmsField": "Formatted Title", - "otherCmsType": "text_long", - "contentstackField": "Formatted Title", - "contentstackFieldUid": "field_formatted_title", - "contentstackFieldType": "json", - "backupFieldType": "html", - "backupFieldUid": "field_formatted_title", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_image", - "otherCmsField": "Image", - "otherCmsType": "image", - "contentstackField": "Image", - "contentstackFieldUid": "field_image_target_id", - "contentstackFieldType": "file", - "backupFieldType": "file", - "backupFieldUid": "field_image_target_id", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - } - ], - "description": "Schema for tour", - "options": { - "is_page": true, - "singleton": false, - "sub_title": [], - "title": "title", - "url_pattern": "/:title", - "url_prefix": "/tour/" - } -} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/drupalSchema/video.json b/upload-api/drupalMigrationData/drupalSchema/video.json deleted file mode 100644 index d59d47866..000000000 --- a/upload-api/drupalMigrationData/drupalSchema/video.json +++ /dev/null @@ -1,161 +0,0 @@ -{ - "title": "video", - "uid": "video", - "schema": [ - { - "uid": "title", - "otherCmsField": "title", - "otherCmsType": "text", - "contentstackField": "title", - "contentstackFieldUid": "title", - "contentstackFieldType": "text", - "backupFieldType": "text", - "backupFieldUid": "title", - "advanced": { - "mandatory": true - } - }, - { - "uid": "url", - "otherCmsField": "url", - "otherCmsType": "text", - "contentstackField": "Url", - "contentstackFieldUid": "url", - "contentstackFieldType": "url", - "backupFieldType": "url", - "backupFieldUid": "url", - "advanced": { - "mandatory": true - } - }, - { - "uid": "field_formatted_title", - "otherCmsField": "Formatted Title", - "otherCmsType": "text_long", - "contentstackField": "Formatted Title", - "contentstackFieldUid": "field_formatted_title", - "contentstackFieldType": "json", - "backupFieldType": "html", - "backupFieldUid": "field_formatted_title", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_image", - "otherCmsField": "Image", - "otherCmsType": "image", - "contentstackField": "Image", - "contentstackFieldUid": "field_image_target_id", - "contentstackFieldType": "file", - "backupFieldType": "file", - "backupFieldUid": "field_image_target_id", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "field_video", - "otherCmsField": "Video", - "otherCmsType": "string", - "contentstackField": "Video", - "contentstackFieldUid": "field_video", - "contentstackFieldType": "single_line_text", - "backupFieldType": "multi_line_text", - "backupFieldUid": "field_video", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [ - "article", - "fact", - "institute", - "link", - "list", - "page", - "profile", - "school", - "tour" - ], - "description": "" - } - }, - { - "uid": "field_video_order", - "otherCmsField": "Video Order", - "otherCmsType": "integer", - "contentstackField": "Video Order", - "contentstackFieldUid": "field_video_order", - "contentstackFieldType": "number", - "backupFieldType": "number", - "backupFieldUid": "field_video_order", - "advanced": { - "default_value": null, - "mandatory": false, - "multiple": false, - "unique": false, - "nonLocalizable": false, - "validationErrorMessage": "", - "embedObjects": [], - "description": "" - } - }, - { - "uid": "taxonomies", - "otherCmsField": "Taxonomy", - "otherCmsType": "taxonomy", - "contentstackField": "Taxonomy", - "contentstackFieldUid": "taxonomies", - "contentstackFieldType": "taxonomy", - "backupFieldType": "taxonomy", - "backupFieldUid": "taxonomies", - "advanced": { - "taxonomies": [ - { - "taxonomy_uid": "videos", - "mandatory": false, - "multiple": true, - "non_localizable": false - }, - { - "taxonomy_uid": "tags", - "mandatory": false, - "multiple": true, - "non_localizable": false - } - ], - "mandatory": false, - "multiple": true, - "non_localizable": false, - "unique": false - } - } - ], - "description": "Schema for video", - "options": { - "is_page": true, - "singleton": false, - "sub_title": [], - "title": "title", - "url_pattern": "/:title", - "url_prefix": "/video/" - } -} \ No newline at end of file diff --git a/upload-api/drupalMigrationData/taxonomySchema/taxonomySchema.json b/upload-api/drupalMigrationData/taxonomySchema/taxonomySchema.json deleted file mode 100644 index ebc3fe8aa..000000000 --- a/upload-api/drupalMigrationData/taxonomySchema/taxonomySchema.json +++ /dev/null @@ -1,22 +0,0 @@ -[ - { - "uid": "list", - "name": "List" - }, - { - "uid": "news", - "name": "News" - }, - { - "uid": "pages", - "name": "Pages" - }, - { - "uid": "tags", - "name": "Tags" - }, - { - "uid": "videos", - "name": "Videos" - } -] \ No newline at end of file From 76c561983de25cef0ad1f9765bcf56d67e1d6545 Mon Sep 17 00:00:00 2001 From: sauravraw Date: Wed, 8 Oct 2025 18:21:12 +0530 Subject: [PATCH 10/37] Update package dependencies and refactor asset configuration handling in Drupal services - Added `mysql2` package to dependencies for improved database interactions. - Refactored asset configuration handling in various services to use a unified `assetsConfig` structure instead of `drupalAssetsUrl`. - Enhanced validation processes to include asset accessibility checks. - Updated related service methods to accommodate new asset configuration structure. - Improved logging for better traceability during asset validation and database interactions. --- api/src/services/drupal.service.ts | 6 +- api/src/services/drupal/assets.service.ts | 525 +++++++++++++++--- .../services/drupal/content-types.service.ts | 33 +- api/src/services/drupal/entries.service.ts | 83 ++- api/src/services/marketplace.service.ts | 91 +-- api/src/services/migration.service.ts | 37 +- api/src/services/projects.service.ts | 286 +++++----- package-lock.json | 127 +++++ package.json | 1 + ui/src/cmsData/legacyCms.json | 4 +- .../LegacyCms/Actions/LoadSelectCms.tsx | 76 ++- .../LegacyCms/Actions/LoadUploadFile.tsx | 65 ++- ui/src/context/app/app.interface.ts | 10 +- .../libs/contentTypeMapper.js | 271 ++++++++- .../libs/createInitialMapper.js | 2 +- .../migration-drupal/libs/extractTaxonomy.js | 2 +- upload-api/src/config/index.ts | 6 +- upload-api/src/models/types.ts | 4 +- upload-api/src/routes/index.ts | 3 +- upload-api/src/services/createMapper.ts | 3 +- upload-api/src/services/fileProcessing.ts | 4 +- upload-api/src/validators/drupal/index.ts | 276 ++++++++- upload-api/src/validators/index.ts | 14 +- 23 files changed, 1535 insertions(+), 394 deletions(-) diff --git a/api/src/services/drupal.service.ts b/api/src/services/drupal.service.ts index 58fd09af6..39770b6f6 100644 --- a/api/src/services/drupal.service.ts +++ b/api/src/services/drupal.service.ts @@ -33,14 +33,14 @@ export const drupalService = { destination_stack_id: string, projectId: string, isTest = false, - drupalAssetsConfig?: any + assetsConfig?: any ) => { return createAssets( dbConfig, destination_stack_id, projectId, - drupalAssetsConfig?.base_url || '', - drupalAssetsConfig?.public_path || '/sites/default/files/', + assetsConfig?.base_url || '', + assetsConfig?.public_path || '', isTest ); }, diff --git a/api/src/services/drupal/assets.service.ts b/api/src/services/drupal/assets.service.ts index 4bd16e1c5..163d135a1 100644 --- a/api/src/services/drupal/assets.service.ts +++ b/api/src/services/drupal/assets.service.ts @@ -67,28 +67,344 @@ async function writeFile(dirPath: string, filename: string, data: any) { } /** - * Constructs the full URL for Drupal assets, handling public:// and private:// schemes + * Executes SQL query and returns results as Promise */ +const executeQuery = ( + connection: mysql.Connection, + query: string +): Promise => { + return new Promise((resolve, reject) => { + connection.query(query, (error, results) => { + if (error) { + reject(error); + } else { + resolve(results as any[]); + } + }); + }); +}; + +const publicPathCache = new Map(); + +// AUTO-DETECT PUBLIC PATH FROM DATABASE +const detectPublicPath = async ( + connection: mysql.Connection, + baseUrl: string, + projectId: string, + destination_stack_id: string +): Promise => { + const srcFunc = 'detectPublicPath'; + + try { + // Try to get public file path from Drupal's system table + const configQuery = ` + SELECT value + FROM config + WHERE name = 'system.file' + LIMIT 1 + `; + + try { + const configResults = await executeQuery(connection, configQuery); + if (configResults.length > 0) { + const config = JSON.parse(configResults[0].value); + if (config.path && config.path.public) { + const detectedPath = config.path.public; + console.log(`โœ… Detected public path from config: ${detectedPath}`); + return detectedPath.endsWith('/') ? detectedPath : `${detectedPath}/`; + } + } + } catch (configErr) { + console.log( + 'โš ๏ธ Config table not found or not readable, trying web detection...' + ); + } + + // Final fallback: Try to detect from an actual file by testing URLs + const sampleFileQuery = ` + SELECT uri, filename + FROM file_managed + WHERE uri LIKE 'public://%' + LIMIT 5 + `; + + const sampleResults = await executeQuery(connection, sampleFileQuery); + if (sampleResults.length > 0) { + // Try common Drupal paths with the user-provided baseUrl + const commonPaths = [ + '/sites/default/files/', + '/sites/all/files/', + '/sites/g/files/bxs2566/files/', // Rice University specific + '/files/', + ]; + + // Also try to extract path patterns from the database URIs + for (const sampleFile of sampleResults) { + const sampleUri = sampleFile.uri; + + for (const testPath of commonPaths) { + const testUrl = `${baseUrl}${testPath}${sampleUri.replace( + 'public://', + '' + )}`; + try { + const response = await axios.get(testUrl, { + timeout: 5000, + maxContentLength: 1024, // Only download first 1KB to test + headers: { + 'User-Agent': 'Contentstack-Drupal-Migration/1.0', + }, + }); + if (response.status === 200) { + console.log( + `โœ… Detected public path by testing URLs: ${testPath}` + ); + const message = getLogMessage( + srcFunc, + `Auto-detected public path: ${testPath}`, + {} + ); + await customLogger( + projectId, + destination_stack_id, + 'info', + message + ); + return testPath; + } + } catch (err) { + // Continue to next path + } + } + } + + // If common paths don't work, try to extract from URI patterns + // Look for patterns like /sites/[site]/files/ in the database + const uriPatternQuery = ` + SELECT DISTINCT uri + FROM file_managed + WHERE uri LIKE 'public://%' + LIMIT 10 + `; + + const uriResults = await executeQuery(connection, uriPatternQuery); + const pathPatterns = new Set(); + + uriResults.forEach((row) => { + const uri = row.uri; + // Extract potential path patterns from URIs + const matches = uri.match(/public:\/\/(?:sites\/([^\/]+)\/)?files\//); + if (matches) { + pathPatterns.add(`/sites/${matches[1]}/files/`); + } + }); + + // Test extracted patterns + for (const pattern of pathPatterns) { + const patternStr = pattern as string; + for (const sampleFile of sampleResults.slice(0, 2)) { + // Test with fewer files + const testUrl = `${baseUrl}${patternStr}${sampleFile.uri.replace( + 'public://', + '' + )}`; + try { + const response = await axios.get(testUrl, { + timeout: 5000, + maxContentLength: 1024, // Only download first 1KB to test + headers: { + 'User-Agent': 'Contentstack-Drupal-Migration/1.0', + }, + }); + if (response.status === 200) { + console.log( + `โœ… Detected public path from URI patterns: ${patternStr}` + ); + const message = getLogMessage( + srcFunc, + `Auto-detected public path from patterns: ${patternStr}`, + {} + ); + await customLogger( + projectId, + destination_stack_id, + 'info', + message + ); + return patternStr; + } + } catch (err) { + // Continue to next pattern + } + } + } + } + + // Ultimate fallback + console.log( + `โš ๏ธ Could not auto-detect path for baseUrl=${baseUrl}. Using default: /sites/default/files/` + ); + + const message = getLogMessage( + srcFunc, + `Could not auto-detect public path. Using default: /sites/default/files/`, + {} + ); + await customLogger(projectId, destination_stack_id, 'warn', message); + return '/sites/default/files/'; + } catch (error: any) { + const message = getLogMessage( + srcFunc, + `Error detecting public path: ${error.message}. Using default.`, + {}, + error + ); + await customLogger(projectId, destination_stack_id, 'warn', message); + return '/sites/default/files/'; + } +}; + +// URL VALIDATION AND NORMALIZATION +const normalizeUrlConfig = ( + baseUrl: string, + publicPath: string +): { baseUrl: string; publicPath: string } => { + // Validate inputs - allow empty values for auto-detection + if (!baseUrl && !publicPath) { + throw new Error( + `Invalid URL configuration: Both baseUrl and publicPath are empty. At least one must be provided.` + ); + } + + // Normalize baseUrl (handle empty case) + let normalizedBaseUrl = baseUrl ? baseUrl.trim() : ''; + + if (normalizedBaseUrl) { + // Remove trailing slash from baseUrl + normalizedBaseUrl = normalizedBaseUrl.replace(/\/+$/, ''); + + // Ensure baseUrl has protocol + if ( + !normalizedBaseUrl.startsWith('http://') && + !normalizedBaseUrl.startsWith('https://') + ) { + normalizedBaseUrl = `https://${normalizedBaseUrl}`; + } + + // Validate baseUrl format + try { + new URL(normalizedBaseUrl); + } catch (error) { + throw new Error( + `Invalid baseUrl format: "${baseUrl}" โ†’ "${normalizedBaseUrl}". Please provide a valid URL.` + ); + } + } + + // Normalize publicPath (handle empty case) + let normalizedPublicPath = publicPath ? publicPath.trim() : ''; + + if (normalizedPublicPath) { + // Ensure publicPath starts with / + if (!normalizedPublicPath.startsWith('/')) { + normalizedPublicPath = `/${normalizedPublicPath}`; + } + + // Ensure publicPath ends with / + if (!normalizedPublicPath.endsWith('/')) { + normalizedPublicPath = `${normalizedPublicPath}/`; + } + + // Remove duplicate slashes + normalizedPublicPath = normalizedPublicPath.replace(/\/+/g, '/'); + + // Validate publicPath doesn't contain invalid characters + if ( + normalizedPublicPath.includes('..') || + normalizedPublicPath.includes('//') + ) { + throw new Error( + `Invalid publicPath format: "${publicPath}" โ†’ "${normalizedPublicPath}". Path contains invalid characters.` + ); + } + } + + console.log(`๐Ÿ”ง URL Normalization:`); + console.log( + ` Original baseUrl: "${baseUrl}" โ†’ Normalized: "${normalizedBaseUrl}"` + ); + console.log( + ` Original publicPath: "${publicPath}" โ†’ Normalized: "${normalizedPublicPath}"` + ); + + return { + baseUrl: normalizedBaseUrl, + publicPath: normalizedPublicPath, + }; +}; + +// DYNAMIC URL CONSTRUCTION - HANDLES MULTIPLE PATH FORMATS const constructAssetUrl = ( uri: string, baseUrl: string, publicPath: string ): string => { - let url = uri; - const replaceValue = baseUrl + publicPath; + try { + // Normalize the input URLs first + const { baseUrl: cleanBaseUrl, publicPath: cleanPublicPath } = + normalizeUrlConfig(baseUrl, publicPath); - if (!url.startsWith('http')) { - url = url.replace('public://', replaceValue); - url = url.replace('private://', replaceValue); - } + // Already a full URL - return as is + if (uri.startsWith('http://') || uri.startsWith('https://')) { + return uri; + } + + // Handle public:// scheme + if (uri.startsWith('public://')) { + const relativePath = uri.replace('public://', ''); - return encodeURI(url); + // Check if we have valid baseUrl and publicPath + if (!cleanBaseUrl || !cleanPublicPath) { + throw new Error( + `Cannot construct URL: baseUrl="${cleanBaseUrl}", publicPath="${cleanPublicPath}". Both are required for public:// URIs.` + ); + } + + const fullUrl = `${cleanBaseUrl}${cleanPublicPath}${relativePath}`; + console.log(`๐Ÿ”— Constructed URL: ${fullUrl} (from URI: ${uri})`); + return fullUrl; + } + + // Handle private:// scheme + if (uri.startsWith('private://')) { + const relativePath = uri.replace('private://', ''); + + if (!cleanBaseUrl) { + throw new Error( + `Cannot construct URL: baseUrl="${cleanBaseUrl}". Base URL is required for private:// URIs.` + ); + } + + return `${cleanBaseUrl}/system/files/${relativePath}`; + } + + // Handle relative paths + const path = uri.startsWith('/') ? uri : `/${uri}`; + + if (!cleanBaseUrl) { + throw new Error( + `Cannot construct URL: baseUrl="${cleanBaseUrl}". Base URL is required for relative paths.` + ); + } + + return `${cleanBaseUrl}${path}`; + } catch (error: any) { + console.error(`โŒ URL Construction Error: ${error.message}`); + throw new Error(`Failed to construct asset URL: ${error.message}`); + } }; -/** - * Saves an asset to the destination stack directory for Drupal migration. - * Based on the original Drupal v8 migration logic. - */ +// IMPROVED SAVE ASSET WITH BETTER ERROR HANDLING const saveAsset = async ( assets: DrupalAsset, failedJSON: any, @@ -98,31 +414,49 @@ const saveAsset = async ( destination_stack_id: string, baseUrl: string = '', publicPath: string = '/sites/default/files/', - retryCount = 0 + retryCount = 0, + authHeaders: any = {} // NEW: Support for authentication ): Promise => { try { const srcFunc = 'saveAsset'; - const assetsSave = path.join(DATA, destination_stack_id, ASSETS_DIR_NAME); + const assetsSave = path.join( + DATA, + destination_stack_id, + ASSETS_DIR_NAME, + 'files' + ); - // Use Drupal-specific field names const assetId = `assets_${assets.fid}`; const fileName = assets.filename; + console.log( + `๐Ÿ” Processing asset: ${fileName} (FID: ${assets.fid}, URI: ${assets.uri})` + ); const fileUrl = constructAssetUrl(assets.uri, baseUrl, publicPath); + console.log(`๐Ÿ“ฅ Final download URL: ${fileUrl}`); // Check if asset already exists if (fs.existsSync(path.resolve(assetsSave, assetId, fileName))) { - return assetId; // Asset already exists + console.log(`โญ๏ธ Skipping existing asset: ${fileName}`); + return assetId; } try { + console.log(`๐Ÿ“ฅ Downloading: ${fileName} (FID: ${assets.fid})`); + const response = await axios.get(fileUrl, { responseType: 'arraybuffer', - timeout: 30000, + timeout: 120000, // Increased to 2 minutes + maxContentLength: 500 * 1024 * 1024, // 500MB max + headers: { + 'User-Agent': 'Contentstack-Drupal-Migration/1.0', + ...authHeaders, // Spread any authentication headers + }, + validateStatus: (status) => status === 200, // Only accept 200 }); const assetPath = path.resolve(assetsSave, assetId); - // Create asset data following Drupal migration pattern + // Create asset data assetData[assetId] = { uid: assetId, urlPath: `/assets/${assetId}`, @@ -139,12 +473,6 @@ const saveAsset = async ( publish_details: [], }; - const message = getLogMessage( - srcFunc, - `Asset "${fileName}" with id ${assets.fid} has been successfully transformed.`, - {} - ); - await fs.promises.mkdir(assetPath, { recursive: true }); await fs.promises.writeFile( path.join(assetPath, fileName), @@ -159,16 +487,30 @@ const saveAsset = async ( metadata.push({ uid: assetId, url: fileUrl, filename: fileName }); - // Remove from failed assets if it was previously failed if (failedJSON[assetId]) { delete failedJSON[assetId]; } + const message = getLogMessage( + srcFunc, + `โœ… Asset "${fileName}" (${assets.fid}) downloaded successfully.`, + {} + ); await customLogger(projectId, destination_stack_id, 'info', message); + return assetId; } catch (err: any) { - if (retryCount < 1) { - // Retry once + // Retry logic with exponential backoff + if (retryCount < 3) { + // Increased to 3 retries + const delay = Math.pow(2, retryCount) * 1000; // 1s, 2s, 4s + console.log( + `โณ Retrying ${fileName} in ${delay}ms... (Attempt ${ + retryCount + 1 + }/3)` + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + return await saveAsset( assets, failedJSON, @@ -178,70 +520,69 @@ const saveAsset = async ( destination_stack_id, baseUrl, publicPath, - retryCount + 1 + retryCount + 1, + authHeaders ); } else { - // Mark as failed after retry + // Failed after retries + const errorDetails = { + status: err.response?.status, + statusText: err.response?.statusText, + message: err.message, + url: fileUrl, + }; + failedJSON[assetId] = { failedUid: assets.fid, name: fileName, url: fileUrl, file_size: assets.filesize, - reason_for_error: err?.message, + reason_for_error: JSON.stringify(errorDetails), }; const message = getLogMessage( srcFunc, - `Failed to download asset "${fileName}" with id ${assets.fid}: ${err.message}`, + `โŒ Failed to download "${fileName}" (${assets.fid}): ${err.message}`, {}, err ); await customLogger(projectId, destination_stack_id, 'error', message); + return assetId; } } } catch (error) { - console.error('Error in saveAsset:', error); + console.error('โŒ Error in saveAsset:', error); return `assets_${assets.fid}`; } }; -/** - * Executes SQL query and returns results as Promise - */ -const executeQuery = ( - connection: mysql.Connection, - query: string -): Promise => { - return new Promise((resolve, reject) => { - connection.query(query, (error, results) => { - if (error) { - reject(error); - } else { - resolve(results as any[]); - } - }); - }); -}; - -/** - * Fetches assets from database using SQL query - */ +// ASSETS QUERY - FETCH ALL FILES const fetchAssetsFromDB = async ( connection: mysql.Connection, projectId: string, destination_stack_id: string ): Promise => { const srcFunc = 'fetchAssetsFromDB'; - const assetsQuery = - 'SELECT a.fid, a.filename, a.uri, a.filesize, a.filemime FROM file_managed a'; + + // Query to fetch ALL files from Drupal + const assetsQuery = ` + SELECT + fm.fid, + fm.filename, + fm.uri, + fm.filesize, + fm.filemime + FROM file_managed fm + ORDER BY fm.fid ASC + `; try { const results = await executeQuery(connection, assetsQuery); const message = getLogMessage( srcFunc, - `Fetched ${results.length} assets from database.`, + `Fetched ${results.length} total assets from database.`, {} ); await customLogger(projectId, destination_stack_id, 'info', message); @@ -271,7 +612,8 @@ const retryFailedAssets = async ( projectId: string, destination_stack_id: string, baseUrl: string = '', - publicPath: string = '/sites/default/files/' + publicPath: string = '/sites/default/files/', + authHeaders: any = {} ): Promise => { const srcFunc = 'retryFailedAssets'; @@ -297,7 +639,9 @@ const retryFailedAssets = async ( projectId, destination_stack_id, baseUrl, - publicPath + publicPath, + 0, + authHeaders ) ) ); @@ -322,22 +666,44 @@ const retryFailedAssets = async ( } }; -/** - * Creates and processes assets from Drupal database for migration to Contentstack. - * Based on the original Drupal v8 migration logic with direct SQL queries. - */ +// UPDATED createAssets WITH AUTO-DETECTION export const createAssets = async ( dbConfig: any, destination_stack_id: string, projectId: string, baseUrl: string = '', - publicPath: string = '/sites/default/files/', + publicPath: string = '', // Now optional - will auto-detect if empty isTest = false ) => { const srcFunc = 'createAssets'; let connection: mysql.Connection | null = null; try { + // Create database connection first + connection = await getDbConnection( + dbConfig, + projectId, + destination_stack_id + ); + + // Auto-detect public path if not provided or empty + let detectedPublicPath = publicPath; + if (!publicPath || publicPath.trim() === '') { + console.log('๐Ÿ” Auto-detecting public path (no path provided)...'); + detectedPublicPath = await detectPublicPath( + connection, + baseUrl, + projectId, + destination_stack_id + ); + } else { + console.log(`โœ… Using provided public path: ${publicPath}`); + } + + console.log(`๐Ÿ“ Using public path: ${detectedPublicPath}`); + console.log(`๐ŸŒ Using base URL: ${baseUrl}`); + console.log(`๐Ÿ”ง Original publicPath parameter: "${publicPath}"`); + const assetsSave = path.join(DATA, destination_stack_id, ASSETS_DIR_NAME); const assetMasterFolderPath = path.join( DATA, @@ -346,26 +712,21 @@ export const createAssets = async ( ASSETS_DIR_NAME ); - // Initialize directories and files await fs.promises.mkdir(assetsSave, { recursive: true }); await fs.promises.mkdir(assetMasterFolderPath, { recursive: true }); - // Initialize data structures const failedJSON: any = {}; const assetData: any = {}; const metadata: AssetMetaData[] = []; const fileMeta = { '1': ASSETS_SCHEMA_FILE }; const failedAssetIds: string[] = []; - const message = getLogMessage(srcFunc, `Exporting assets...`, {}); - await customLogger(projectId, destination_stack_id, 'info', message); - - // Create database connection - connection = await getDbConnection( - dbConfig, - projectId, - destination_stack_id + const message = getLogMessage( + srcFunc, + `Exporting assets using base URL: ${baseUrl} and public path: ${detectedPublicPath}`, + {} ); + await customLogger(projectId, destination_stack_id, 'info', message); // Fetch assets from database const assetsData = await fetchAssetsFromDB( @@ -380,8 +741,7 @@ export const createAssets = async ( assets = assets.slice(0, 10); } - // Use batch processing for large datasets to prevent EMFILE errors - const batchSize = assets.length > 10000 ? 100 : 1000; // Smaller batches for very large datasets + const batchSize = assets.length > 10000 ? 100 : 1000; const results = await processBatches( assets, async (asset: DrupalAsset) => { @@ -394,7 +754,9 @@ export const createAssets = async ( projectId, destination_stack_id, baseUrl, - publicPath + detectedPublicPath, // Use detected path + 0, + {} // authHeaders ); } catch (error) { failedAssetIds.push(asset.fid.toString()); @@ -403,11 +765,10 @@ export const createAssets = async ( }, { batchSize, - concurrency: 1, // Process one at a time to prevent file handle exhaustion - delayBetweenBatches: 200, // 200ms delay between batches to allow file handles to close + concurrency: 5, // Increased from 1 for better performance + delayBetweenBatches: 200, }, (batchIndex, totalBatches, batchResults) => { - // Periodically save progress for very large datasets if (batchIndex % 10 === 0) { console.log( `๐Ÿ’พ Progress: ${batchIndex}/${totalBatches} batches completed (${( @@ -419,7 +780,7 @@ export const createAssets = async ( } ); - // Retry failed assets if any + // Retry failed assets if (failedAssetIds.length > 0) { await retryFailedAssets( connection, @@ -430,11 +791,10 @@ export const createAssets = async ( projectId, destination_stack_id, baseUrl, - publicPath + detectedPublicPath // Use detected path ); } - // Write files following the original pattern await writeFile(assetsSave, ASSETS_SCHEMA_FILE, assetData); await writeFile(assetsSave, ASSETS_FILE_NAME, fileMeta); @@ -472,7 +832,6 @@ export const createAssets = async ( await customLogger(projectId, destination_stack_id, 'error', message); throw err; } finally { - // Close database connection if (connection) { connection.end(); } diff --git a/api/src/services/drupal/content-types.service.ts b/api/src/services/drupal/content-types.service.ts index 3941e1b01..16be69cc7 100644 --- a/api/src/services/drupal/content-types.service.ts +++ b/api/src/services/drupal/content-types.service.ts @@ -5,7 +5,8 @@ import { convertToSchemaFormate } from '../../utils/content-type-creator.utils.j import { getLogMessage } from '../../utils/index.js'; import customLogger from '../../utils/custom-logger.utils.js'; -const { DATA, CONTENT_TYPES_DIR_NAME } = MIGRATION_DATA_CONFIG; +const { DATA, CONTENT_TYPES_DIR_NAME, CONTENT_TYPES_SCHEMA_FILE } = + MIGRATION_DATA_CONFIG; /** * Generates API content types from upload-api drupal schema @@ -62,6 +63,9 @@ export const generateContentTypeSchemas = async ( ); } + // Build complete schema array (NO individual files) + const allApiSchemas = []; + for (const schemaFile of schemaFiles) { try { const uploadApiSchemaFilePath = path.join( @@ -75,13 +79,8 @@ export const generateContentTypeSchemas = async ( // Convert upload-api schema to API format const apiSchema = convertUploadApiSchemaToApiSchema(uploadApiSchema); - // Write API schema file - const apiSchemaFilePath = path.join(apiContentTypesPath, schemaFile); - await fs.promises.writeFile( - apiSchemaFilePath, - JSON.stringify(apiSchema, null, 2), - 'utf8' - ); + // Add to combined schema array (NO individual files) + allApiSchemas.push(apiSchema); const fieldMessage = getLogMessage( srcFunc, @@ -112,6 +111,21 @@ export const generateContentTypeSchemas = async ( } } + // Write ONLY the combined schema.json file + const combinedSchemaPath = path.join( + apiContentTypesPath, + CONTENT_TYPES_SCHEMA_FILE + ); + await fs.promises.writeFile( + combinedSchemaPath, + JSON.stringify(allApiSchemas, null, 2), + 'utf8' + ); + + console.log( + `โœ… Generated schema.json with ${allApiSchemas.length} content types (NO individual files)` + ); + const successMessage = getLogMessage( srcFunc, `Successfully generated ${schemaFiles.length} content type schemas from upload-api`, @@ -260,3 +274,6 @@ function mapFieldTypeToDataType(fieldType: string): string { return fieldTypeMap[fieldType] || 'text'; } + +// Removed regenerateCombinedSchemaFromIndividualFiles function +// We now generate ONLY schema.json directly, no individual files diff --git a/api/src/services/drupal/entries.service.ts b/api/src/services/drupal/entries.service.ts index f09203a38..0aa55ae69 100644 --- a/api/src/services/drupal/entries.service.ts +++ b/api/src/services/drupal/entries.service.ts @@ -699,37 +699,88 @@ const processFieldData = async ( } // Handle entity references (taxonomy and node references) using field analysis - if (dataKey.endsWith('_target_id') && typeof value === 'number') { + // NOTE: value can be a number (single reference) or string (GROUP_CONCAT comma-separated IDs) + if ( + dataKey.endsWith('_target_id') && + (typeof value === 'number' || typeof value === 'string') + ) { // Check if this is a taxonomy field using our field analysis if (isTaxonomyField(fieldName, contentType, taxonomyFieldMapping)) { - // Look up taxonomy reference using drupal_term_id - const taxonomyRef = taxonomyReferenceLookup[value]; - - if (taxonomyRef) { - // Transform to array format with taxonomy_uid and term_uid (no drupal_term_id) - processedData[dataKey] = [ - { + // Handle both single ID (number) and GROUP_CONCAT result (comma-separated string) + const targetIds = + typeof value === 'string' + ? value + .split(',') + .map((id) => parseInt(id.trim())) + .filter((id) => !isNaN(id)) + : [value]; + + const transformedTaxonomies: Array<{ + taxonomy_uid: string; + term_uid: string; + }> = []; + + for (const tid of targetIds) { + // Look up taxonomy reference using drupal_term_id + const taxonomyRef = taxonomyReferenceLookup[tid]; + + if (taxonomyRef) { + transformedTaxonomies.push({ taxonomy_uid: taxonomyRef.taxonomy_uid, term_uid: taxonomyRef.term_uid, - }, - ]; + }); + } else { + console.warn( + `โš ๏ธ Taxonomy term ${tid} not found in reference lookup for field ${fieldName}` + ); + } + } + + if (transformedTaxonomies.length > 0) { + processedData[dataKey] = transformedTaxonomies; } else { - // Fallback to numeric tid if lookup failed + // Fallback to original value if no lookups succeeded processedData[dataKey] = value; } + + // Mark field as processed so it doesn't get overwritten by ctValue loop + processedFields.add(dataKey); + skippedFields.add(dataKey); // Also skip in ctValue loop + continue; // Skip further processing for this field } else if ( isReferenceField(fieldName, contentType, referenceFieldMapping) ) { // Handle node reference fields using field analysis - const referenceKey = `content_type_entries_title_${value}`; - if (referenceKey in referenceId) { - // Transform to array format with proper reference structure - processedData[dataKey] = [referenceId[referenceKey]]; + // Handle both single ID (number) and GROUP_CONCAT result (comma-separated string) + const targetIds = + typeof value === 'string' + ? value + .split(',') + .map((id) => parseInt(id.trim())) + .filter((id) => !isNaN(id)) + : [value]; + + const transformedReferences: any[] = []; + + for (const nid of targetIds) { + const referenceKey = `content_type_entries_title_${nid}`; + if (referenceKey in referenceId) { + transformedReferences.push(referenceId[referenceKey]); + } + } + + if (transformedReferences.length > 0) { + processedData[dataKey] = transformedReferences; } else { - // If reference not found, mark field as skipped + // If no references found, mark field as skipped skippedFields.add(dataKey); } + + // Mark field as processed so it doesn't get overwritten by ctValue loop + processedFields.add(dataKey); + skippedFields.add(dataKey); // Also skip in ctValue loop + continue; // Skip further processing for this field } } diff --git a/api/src/services/marketplace.service.ts b/api/src/services/marketplace.service.ts index e7f5301a1..15de3d06b 100644 --- a/api/src/services/marketplace.service.ts +++ b/api/src/services/marketplace.service.ts @@ -1,18 +1,16 @@ import path from 'path'; import fs from 'fs'; -import getAuthtoken from "../utils/auth.utils.js"; +import getAuthtoken from '../utils/auth.utils.js'; import { MIGRATION_DATA_CONFIG, KEYTOREMOVE } from '../constants/index.js'; import { getAppManifestAndAppConfig } from '../utils/market-app.utils.js'; -import { v4 as uuidv4 } from "uuid"; - +import { v4 as uuidv4 } from 'uuid'; const { EXTENSIONS_MAPPER_DIR_NAME, MARKETPLACE_APPS_DIR_NAME, - MARKETPLACE_APPS_FILE_NAME + MARKETPLACE_APPS_FILE_NAME, } = MIGRATION_DATA_CONFIG; - const groupByAppUid = (data: any) => { return data?.reduce?.((acc: any, item: any) => { if (!acc[item.appUid]) { @@ -21,22 +19,27 @@ const groupByAppUid = (data: any) => { acc[item.appUid].push(item.extensionUid); return acc; }, {}); -} +}; const removeKeys = (obj: any, keysToRemove: any) => { return Object.fromEntries( Object.entries(obj).filter(([key]) => !keysToRemove.includes(key)) ); -} +}; const writeManifestFile = async ({ destinationStackId, appManifest }: any) => { - const dirPath = path.join(process.cwd(), MIGRATION_DATA_CONFIG.DATA, destinationStackId, MARKETPLACE_APPS_DIR_NAME); + const dirPath = path.join( + process.cwd(), + MIGRATION_DATA_CONFIG.DATA, + destinationStackId, + MARKETPLACE_APPS_DIR_NAME + ); try { await fs.promises.access(dirPath); } catch (err) { try { await fs.promises.mkdir(dirPath, { recursive: true }); } catch (mkdirErr) { - console.error("๐Ÿš€ ~ fs.mkdir ~ err:", mkdirErr); + console.error('๐Ÿš€ ~ fs.mkdir ~ err:', mkdirErr); return; } } @@ -44,23 +47,37 @@ const writeManifestFile = async ({ destinationStackId, appManifest }: any) => { const filePath = path.join(dirPath, MARKETPLACE_APPS_FILE_NAME); await fs.promises.writeFile(filePath, JSON.stringify(appManifest, null, 2)); } catch (writeErr) { - console.error("๐Ÿš€ ~ fs.writeFile ~ err:", writeErr); + console.error('๐Ÿš€ ~ fs.writeFile ~ err:', writeErr); } -} +}; - - -const createAppManifest = async ({ destinationStackId, region, userId, orgId }: any) => { +const createAppManifest = async ({ + destinationStackId, + region, + userId, + orgId, +}: any) => { const authtoken = await getAuthtoken(region, userId); - const marketPlacePath = path.join(MIGRATION_DATA_CONFIG.DATA, destinationStackId, EXTENSIONS_MAPPER_DIR_NAME); - const AppMapper: any = await fs.promises.readFile(marketPlacePath, "utf-8").catch(async () => { }); + const marketPlacePath = path.join( + MIGRATION_DATA_CONFIG.DATA, + destinationStackId, + EXTENSIONS_MAPPER_DIR_NAME + ); + const AppMapper: any = await fs.promises + .readFile(marketPlacePath, 'utf-8') + .catch(async () => {}); if (AppMapper !== undefined) { const appManifest: any = []; const groupUids: any = groupByAppUid(JSON.parse(AppMapper)); for await (const [key, value] of Object?.entries?.(groupUids) || {}) { - const data: any = await getAppManifestAndAppConfig({ organizationUid: orgId, authtoken, region, manifestUid: key }); + const data: any = await getAppManifestAndAppConfig({ + organizationUid: orgId, + authtoken, + region, + manifestUid: key, + }); data.manifest = removeKeys(data, KEYTOREMOVE); - const extensionUids: any = new Set(value) ?? []; + const extensionUids: any = new Set(value as any) ?? []; const locations: any = []; for (const ext of extensionUids ?? []) { const seprateUid = ext?.split?.('-'); @@ -68,38 +85,48 @@ const createAppManifest = async ({ destinationStackId, region, userId, orgId }: const extUid: string = seprateUid?.[0]; for (const loc of data?.ui_location?.locations ?? []) { if (loc?.type === type) { - const isPresent = locations?.meta?.findIndex((item: any) => item?.extension_uid === extUid); + const isPresent = locations?.meta?.findIndex( + (item: any) => item?.extension_uid === extUid + ); if (isPresent === undefined) { locations?.push({ type, - meta: [{ ...(loc?.meta?.[0] || {}), extension_uid: extUid }] - }) + meta: [{ ...(loc?.meta?.[0] || {}), extension_uid: extUid }], + }); } } } } - const configData = data?.ui_location?.locations?.find((ele: any) => ele?.type === 'cs.cm.stack.config'); + const configData = data?.ui_location?.locations?.find( + (ele: any) => ele?.type === 'cs.cm.stack.config' + ); if (configData) { locations?.push({ type: configData?.type, - meta: [{ ...(configData?.meta?.[0] || {}), name: 'Config', extension_uid: uuidv4() }] - }) + meta: [ + { + ...(configData?.meta?.[0] || {}), + name: 'Config', + extension_uid: uuidv4(), + }, + ], + }); } data.ui_location.locations = locations; - data.status = "installed"; + data.status = 'installed'; data.target = { - "type": "stack", - "uid": destinationStackId + type: 'stack', + uid: destinationStackId, }; data.installation_uid = data?.uid; - data.configuration = ""; - data.server_configuration = ""; + data.configuration = ''; + data.server_configuration = ''; appManifest?.push(removeKeys(data, KEYTOREMOVE)); } await writeManifestFile({ destinationStackId, appManifest }); } -} +}; export const marketPlaceAppService = { - createAppManifest -} \ No newline at end of file + createAppManifest, +}; diff --git a/api/src/services/migration.service.ts b/api/src/services/migration.service.ts index 54e2c0238..5798c7ac5 100644 --- a/api/src/services/migration.service.ts +++ b/api/src/services/migration.service.ts @@ -587,14 +587,24 @@ const startTestMigration = async (req: Request): Promise => { port: project?.legacy_cms?.mySQLDetails?.port || 3306, }; - // Get Drupal assets URL configuration from project + // Get Drupal assets URL configuration from project or request body + // Fallback to empty strings if not provided (auto-detection will be used) const drupalAssetsConfig = { - base_url: project?.legacy_cms?.drupalAssetsUrl?.base_url || '', + base_url: + project?.legacy_cms?.assetsConfig?.base_url || + req.body?.assetsConfig?.base_url || + '', public_path: - project?.legacy_cms?.drupalAssetsUrl?.public_path || - '/sites/default/files/', + project?.legacy_cms?.assetsConfig?.public_path || + req.body?.assetsConfig?.public_path || + '', }; + console.log( + '๐Ÿ”ง Migration service - drupalAssetsConfig:', + drupalAssetsConfig + ); + // Run Drupal migration services in proper order (following test-drupal-services sequence) // NOTE: Dynamic queries are generated during Step 2โ†’3 transition, no need for createQueryConfig @@ -968,13 +978,24 @@ const startMigration = async (req: Request): Promise => { port: project?.legacy_cms?.mySQLDetails?.port || 3306, }; - // Get Drupal assets URL configuration from project + // Get Drupal assets URL configuration from project or request body + // Fallback to empty strings if not provided (auto-detection will be used) const drupalAssetsConfig = { - base_url: project?.legacy_cms?.drupalAssetsUrl?.base_url || '', + base_url: + project?.legacy_cms?.assetsConfig?.base_url || + req.body?.assetsConfig?.base_url || + '', public_path: - project?.legacy_cms?.drupalAssetsUrl?.public_path || - '/sites/default/files/', + project?.legacy_cms?.assetsConfig?.public_path || + req.body?.assetsConfig?.public_path || + '', }; + + console.log( + '๐Ÿ”ง Migration service (production) - drupalAssetsConfig:', + drupalAssetsConfig + ); + // Run Drupal migration services in proper order (following test-drupal-services sequence) // NOTE: Dynamic queries are generated during Step 2โ†’3 transition, no need for createQueryConfig diff --git a/api/src/services/projects.service.ts b/api/src/services/projects.service.ts index 9121bb4fd..5c73e0863 100644 --- a/api/src/services/projects.service.ts +++ b/api/src/services/projects.service.ts @@ -1,30 +1,30 @@ -import { Request } from "express"; -import ProjectModelLowdb from "../models/project-lowdb.js"; -import ContentTypesMapperModelLowdb from "../models/contentTypesMapper-lowdb.js"; -import FieldMapperModel from "../models/FieldMapper.js"; -import { drupalService } from "./drupal.service.js"; +import { Request } from 'express'; +import ProjectModelLowdb from '../models/project-lowdb.js'; +import ContentTypesMapperModelLowdb from '../models/contentTypesMapper-lowdb.js'; +import FieldMapperModel from '../models/FieldMapper.js'; +import { drupalService } from './drupal.service.js'; import { BadRequestError, ExceptionFunction, NotFoundError, -} from "../utils/custom-errors.utils.js"; +} from '../utils/custom-errors.utils.js'; import { HTTP_TEXTS, HTTP_CODES, STEPPER_STEPS, NEW_PROJECT_STATUS, CMS, -} from "../constants/index.js"; -import { config } from "../config/index.js"; -import { getLogMessage, isEmpty, safePromise } from "../utils/index.js"; -import getAuthtoken from "../utils/auth.utils.js"; -import https from "../utils/https.utils.js"; -import getProjectUtil from "../utils/get-project.utils.js"; -import logger from "../utils/logger.js"; -import customLogger from "../utils/custom-logger.utils.js"; +} from '../constants/index.js'; +import { config } from '../config/index.js'; +import { getLogMessage, isEmpty, safePromise } from '../utils/index.js'; +import getAuthtoken from '../utils/auth.utils.js'; +import https from '../utils/https.utils.js'; +import getProjectUtil from '../utils/get-project.utils.js'; +import logger from '../utils/logger.js'; +import customLogger from '../utils/custom-logger.utils.js'; // import { contentMapperService } from "./contentMapper.service.js"; -import { v4 as uuidv4 } from "uuid"; +import { v4 as uuidv4 } from 'uuid'; /** * Retrieves all projects based on the provided request object. @@ -37,16 +37,16 @@ const getAllProjects = async (req: Request) => { const orgId = req?.params?.orgId; const decodedToken = req.body.token_payload; - const { user_id = "", region = "" } = decodedToken; + const { user_id = '', region = '' } = decodedToken; await ProjectModelLowdb.read(); const projects = ProjectModelLowdb.chain - .get("projects") + .get('projects') .filter({ org_id: orgId, region, owner: user_id, - isDeleted: false + isDeleted: false, }) .value(); @@ -64,7 +64,7 @@ const getProject = async (req: Request) => { const orgId = req?.params?.orgId; const projectId = req?.params?.projectId; const decodedToken = req.body.token_payload; - const { user_id = "", region = "" } = decodedToken; + const { user_id = '', region = '' } = decodedToken; // Find the project based on both orgId and projectId, region, owner const project = await getProjectUtil( projectId, @@ -74,7 +74,7 @@ const getProject = async (req: Request) => { region: region, owner: user_id, }, - "getProject" + 'getProject' ); return project; @@ -90,8 +90,8 @@ const createProject = async (req: Request) => { const orgId = req?.params?.orgId; const { name, description } = req.body; const decodedToken = req.body.token_payload; - const { user_id = "", region = "" } = decodedToken; - const srcFunc = "createProject"; + const { user_id = '', region = '' } = decodedToken; + const srcFunc = 'createProject'; const projectData = { id: uuidv4(), region, @@ -102,26 +102,26 @@ const createProject = async (req: Request) => { description, status: NEW_PROJECT_STATUS[0], current_step: STEPPER_STEPS.LEGACY_CMS, - destination_stack_id: "", + destination_stack_id: '', test_stacks: [], - current_test_stack_id: "", + current_test_stack_id: '', legacy_cms: { is_fileValid: false, awsDetails: { - awsRegion: "", - bucketName: "", - buketKey: "", + awsRegion: '', + bucketName: '', + buketKey: '', }, is_sql: false, mySQLDetails: { - host: "", - user: "", - database:"" + host: '', + user: '', + database: '', + }, + assetsConfig: { + base_url: '', + public_path: '', }, - drupalAssetsUrl: { - base_url: "", - public_path: "" - } }, content_mapper: [], execution_log: [], @@ -130,18 +130,18 @@ const createProject = async (req: Request) => { created_at: new Date().toISOString(), isDeleted: false, isNewStack: false, - newStackId: "", + newStackId: '', stackDetails: { uid: '', label: '', master_locale: '', created_at: '', - isNewStack: false + isNewStack: false, }, mapperKeys: {}, isMigrationStarted: false, - isMigrationCompleted:false, - migration_execution:false, + isMigrationCompleted: false, + migration_execution: false, }; try { @@ -160,8 +160,8 @@ const createProject = async (req: Request) => { ) ); return { - status: "success", - message: "Project created successfully", + status: 'success', + message: 'Project created successfully', project: { name: projectData.name, id: projectData.id, @@ -198,8 +198,8 @@ const updateProject = async (req: Request) => { const projectId = req?.params?.projectId; const updateData = req?.body; const decodedToken = req.body.token_payload; - const { user_id = "", region = "" } = decodedToken; - const srcFunc = "updateProject"; + const { user_id = '', region = '' } = decodedToken; + const srcFunc = 'updateProject'; let project: any; // Find the project based on both orgId and projectId @@ -243,8 +243,8 @@ const updateProject = async (req: Request) => { ) ); return { - status: "success", - message: "Project updated successfully", + status: 'success', + message: 'Project updated successfully', project: { name: updateData?.name, description: updateData?.description, @@ -282,7 +282,7 @@ const updateProject = async (req: Request) => { const updateLegacyCMS = async (req: Request) => { const { orgId, projectId } = req.params; const { token_payload, legacy_cms } = req.body; - const srcFunc = "updateLegacyCMS"; + const srcFunc = 'updateLegacyCMS'; await ProjectModelLowdb.read(); const projectIndex = (await getProjectUtil( @@ -364,7 +364,7 @@ const updateLegacyCMS = async (req: Request) => { * @returns An object with the status and data properties. */ const updateAffix = async (req: Request) => { - const srcFunc = "updateAffix"; + const srcFunc = 'updateAffix'; const { orgId, projectId } = req.params; const { token_payload, affix } = req.body; @@ -401,7 +401,7 @@ const updateAffix = async (req: Request) => { * @returns An object with the status and data properties. */ const affixConfirmation = async (req: Request) => { - const srcFunc = "affixConfirmation"; + const srcFunc = 'affixConfirmation'; const { orgId, projectId } = req.params; const { token_payload, affix_confirmation } = req.body; @@ -450,9 +450,10 @@ const updateFileFormat = async (req: Request) => { awsDetails, is_sql, mySQLDetails, - drupalAssetsUrl, + assetsConfig, } = req.body; - const srcFunc = "updateFileFormat"; + + const srcFunc = 'updateFileFormat'; const projectIndex = (await getProjectUtil( projectId, { @@ -507,17 +508,17 @@ const updateFileFormat = async (req: Request) => { awsDetails.bucketName; data.projects[projectIndex].legacy_cms.awsDetails.buketKey = awsDetails.buketKey; - data.projects[ projectIndex ].legacy_cms.is_sql = is_sql; + data.projects[projectIndex].legacy_cms.is_sql = is_sql; data.projects[projectIndex].legacy_cms.mySQLDetails.host = mySQLDetails.host; data.projects[projectIndex].legacy_cms.mySQLDetails.user = mySQLDetails.user; data.projects[projectIndex].legacy_cms.mySQLDetails.database = mySQLDetails.database; - data.projects[projectIndex].legacy_cms.drupalAssetsUrl.base_url = - drupalAssetsUrl?.base_url || ""; - data.projects[projectIndex].legacy_cms.drupalAssetsUrl.public_path = - drupalAssetsUrl?.public_path || ""; + data.projects[projectIndex].legacy_cms.assetsConfig.base_url = + assetsConfig?.base_url || ''; + data.projects[projectIndex].legacy_cms.assetsConfig.public_path = + assetsConfig?.public_path || ''; }); logger.info( @@ -555,7 +556,7 @@ const updateFileFormat = async (req: Request) => { * @returns An object with the status and data properties. */ const fileformatConfirmation = async (req: Request) => { - const srcFunc = "fileformat"; + const srcFunc = 'fileformat'; const { orgId, projectId } = req.params; const { token_payload, fileformat_confirmation } = req.body; @@ -599,7 +600,7 @@ const fileformatConfirmation = async (req: Request) => { const updateDestinationStack = async (req: Request) => { const { orgId, projectId } = req.params; const { token_payload, stack_api_key } = req.body; - const srcFunc = "updateDestinationStack"; + const srcFunc = 'updateDestinationStack'; await ProjectModelLowdb.read(); const projectIndex = (await getProjectUtil( @@ -648,7 +649,7 @@ const updateDestinationStack = async (req: Request) => { try { const [err, res] = await safePromise( https({ - method: "GET", + method: 'GET', url: `${config.CS_API[ token_payload?.region as keyof typeof config.CS_API ]!}/stacks`, @@ -747,7 +748,7 @@ const generateQueriesWithRetry = async ( user: project?.legacy_cms?.mySQLDetails?.user, password: project?.legacy_cms?.mySQLDetails?.password || '', database: project?.legacy_cms?.mySQLDetails?.database, - port: project?.legacy_cms?.mySQLDetails?.port || 3306 + port: project?.legacy_cms?.mySQLDetails?.port || 3306, }; const logMessage = getLogMessage( @@ -806,8 +807,8 @@ const generateQueriesWithRetry = async ( token_payload ); await customLogger(projectId, stackId, 'info', retryMessage); - - await new Promise(resolve => setTimeout(resolve, retryDelay)); + + await new Promise((resolve) => setTimeout(resolve, retryDelay)); } } @@ -824,9 +825,7 @@ const generateQueriesWithRetry = async ( const updateCurrentStep = async (req: Request) => { const { orgId, projectId } = req.params; const token_payload = req.body.token_payload; - const srcFunc = "updateCurrentStep"; - - + const srcFunc = 'updateCurrentStep'; try { await ProjectModelLowdb.read(); @@ -847,7 +846,6 @@ const updateCurrentStep = async (req: Request) => { const isStepCompleted = project?.legacy_cms?.cms && project?.legacy_cms?.file_format; - switch (project.current_step) { case STEPPER_STEPS.LEGACY_CMS: { if (project.status !== NEW_PROJECT_STATUS[0] || !isStepCompleted) { @@ -865,7 +863,8 @@ const updateCurrentStep = async (req: Request) => { data.projects[projectIndex].current_step = STEPPER_STEPS.DESTINATION_STACK; data.projects[projectIndex].status = - project.current_step <= STEPPER_STEPS.CONTENT_MAPPING ? NEW_PROJECT_STATUS[0] + project.current_step <= STEPPER_STEPS.CONTENT_MAPPING + ? NEW_PROJECT_STATUS[0] : NEW_PROJECT_STATUS[1]; data.projects[projectIndex].updated_at = new Date().toISOString(); }); @@ -896,7 +895,12 @@ const updateCurrentStep = async (req: Request) => { `Generating dynamic queries for Drupal project before proceeding to Content Mapping...`, token_payload ); - await customLogger(projectId, project?.destination_stack_id, 'info', startMessage); + await customLogger( + projectId, + project?.destination_stack_id, + 'info', + startMessage + ); // Generate queries for destination stack const destinationSuccess = await generateQueriesWithRetry( @@ -924,17 +928,13 @@ const updateCurrentStep = async (req: Request) => { const failedStacks = []; if (!destinationSuccess) failedStacks.push('destination'); if (!testSuccess) failedStacks.push('test'); - - const errorMessage = `Query generation failed for ${failedStacks.join(' and ')} stack(s). Something went wrong. Please try again.`; - - logger.error( - getLogMessage( - srcFunc, - errorMessage, - token_payload - ) - ); - + + const errorMessage = `Query generation failed for ${failedStacks.join( + ' and ' + )} stack(s). Something went wrong. Please try again.`; + + logger.error(getLogMessage(srcFunc, errorMessage, token_payload)); + throw new BadRequestError(errorMessage); } @@ -943,7 +943,12 @@ const updateCurrentStep = async (req: Request) => { `Dynamic queries successfully generated for all stacks. Proceeding to Content Mapping step.`, token_payload ); - await customLogger(projectId, project?.destination_stack_id, 'info', completeMessage); + await customLogger( + projectId, + project?.destination_stack_id, + 'info', + completeMessage + ); } await ProjectModelLowdb.update((data: any) => { @@ -956,7 +961,6 @@ const updateCurrentStep = async (req: Request) => { break; } case STEPPER_STEPS.CONTENT_MAPPING: { - if ( project.status === NEW_PROJECT_STATUS[0] || !isStepCompleted || @@ -970,14 +974,11 @@ const updateCurrentStep = async (req: Request) => { token_payload ) ); - throw new BadRequestError( - HTTP_TEXTS.CANNOT_PROCEED_CONTENT_MAPPING - ); + throw new BadRequestError(HTTP_TEXTS.CANNOT_PROCEED_CONTENT_MAPPING); } await ProjectModelLowdb.update((data: any) => { - data.projects[projectIndex].current_step = - STEPPER_STEPS.TESTING; + data.projects[projectIndex].current_step = STEPPER_STEPS.TESTING; data.projects[projectIndex].status = NEW_PROJECT_STATUS[4]; data.projects[projectIndex].updated_at = new Date().toISOString(); }); @@ -985,13 +986,13 @@ const updateCurrentStep = async (req: Request) => { } case STEPPER_STEPS.TESTING: { if ( - (project.status === NEW_PROJECT_STATUS[0] || + project.status === NEW_PROJECT_STATUS[0] || !isStepCompleted || !project?.destination_stack_id || project?.content_mapper?.length === 0 || - !project?.current_test_stack_id) || !project?.migration_execution + !project?.current_test_stack_id || + !project?.migration_execution ) { - logger.error( getLogMessage( srcFunc, @@ -999,14 +1000,11 @@ const updateCurrentStep = async (req: Request) => { token_payload ) ); - throw new BadRequestError( - HTTP_TEXTS.CANNOT_PROCEED_TEST_MIGRATION - ); + throw new BadRequestError(HTTP_TEXTS.CANNOT_PROCEED_TEST_MIGRATION); } - + await ProjectModelLowdb.update((data: any) => { - data.projects[projectIndex].current_step = - STEPPER_STEPS.MIGRATION; + data.projects[projectIndex].current_step = STEPPER_STEPS.MIGRATION; data.projects[projectIndex].status = NEW_PROJECT_STATUS[4]; data.projects[projectIndex].updated_at = new Date().toISOString(); }); @@ -1028,14 +1026,11 @@ const updateCurrentStep = async (req: Request) => { token_payload ) ); - throw new BadRequestError( - HTTP_TEXTS.CANNOT_PROCEED_MIGRATION - ); + throw new BadRequestError(HTTP_TEXTS.CANNOT_PROCEED_MIGRATION); } await ProjectModelLowdb.update((data: any) => { - data.projects[projectIndex].current_step = - STEPPER_STEPS.MIGRATION; + data.projects[projectIndex].current_step = STEPPER_STEPS.MIGRATION; data.projects[projectIndex].status = NEW_PROJECT_STATUS[5]; data.projects[projectIndex].updated_at = new Date().toISOString(); }); @@ -1074,8 +1069,8 @@ const updateCurrentStep = async (req: Request) => { const deleteProject = async (req: Request) => { const { orgId, projectId } = req.params; const decodedToken = req.body.token_payload; - const { user_id = "", region = "" } = decodedToken; - const srcFunc = "deleteProject"; + const { user_id = '', region = '' } = decodedToken; + const srcFunc = 'deleteProject'; await ProjectModelLowdb.read(); const projectIndex = (await getProjectUtil( @@ -1101,7 +1096,7 @@ const deleteProject = async (req: Request) => { if (!isEmpty(content_mapper_id)) { content_mapper_id.map(async (item: any) => { const contentMapperData = ContentTypesMapperModelLowdb.chain - .get("ContentTypesMappers") + .get('ContentTypesMappers') .find({ id: item, projectId: projectId }) .value(); @@ -1111,7 +1106,7 @@ const deleteProject = async (req: Request) => { if (!isEmpty(fieldMappingIds)) { (fieldMappingIds || []).forEach((field: any) => { const fieldIndex = FieldMapperModel.chain - .get("field_mapper") + .get('field_mapper') .findIndex({ id: field, projectId: projectId }) .value(); if (fieldIndex > -1) { @@ -1123,7 +1118,7 @@ const deleteProject = async (req: Request) => { } //delete all content Mapper which is related to Project const contentMapperID = ContentTypesMapperModelLowdb.chain - .get("ContentTypesMappers") + .get('ContentTypesMappers') .findIndex({ id: item, projectId: projectId }) .value(); await ContentTypesMapperModelLowdb.update((Cdata: any) => { @@ -1166,8 +1161,8 @@ const deleteProject = async (req: Request) => { const revertProject = async (req: Request) => { const { orgId, projectId } = req?.params ?? {}; const decodedToken = req.body.token_payload; - const { user_id = "", region = "" } = decodedToken; - const srcFunc = "revertProject"; + const { user_id = '', region = '' } = decodedToken; + const srcFunc = 'revertProject'; await ProjectModelLowdb.read(); const projectIndex = (await getProjectUtil( @@ -1200,7 +1195,7 @@ const revertProject = async (req: Request) => { status: HTTP_CODES.OK, data: { message: HTTP_TEXTS.PROJECT_REVERT, - Project: projects + Project: projects, }, }; } @@ -1217,7 +1212,7 @@ const revertProject = async (req: Request) => { const updateStackDetails = async (req: Request) => { const { orgId, projectId } = req.params; const { token_payload, stack_details } = req.body; - const srcFunc = "updateStackDetails"; + const srcFunc = 'updateStackDetails'; await ProjectModelLowdb.read(); const projectIndex = (await getProjectUtil( @@ -1278,8 +1273,7 @@ const updateStackDetails = async (req: Request) => { const updateContentMapper = async (req: Request) => { const { orgId, projectId } = req.params; const { token_payload, content_mapper } = req.body; - const srcFunc = "updateContentMapper"; - + const srcFunc = 'updateContentMapper'; await ProjectModelLowdb.read(); const projectIndex = (await getProjectUtil( @@ -1335,26 +1329,25 @@ const updateContentMapper = async (req: Request) => { */ const updateMigrationExecution = async (req: Request) => { const { orgId, projectId } = req.params; // Extract organization and project IDs from the route parameters - const { token_payload, stack_details } = req.body; // Extract token payload and stack details from the request body - const srcFunc = "updateMigrationExecutionKey"; + const { token_payload, stack_details } = req.body; // Extract token payload and stack details from the request body + const srcFunc = 'updateMigrationExecutionKey'; - // Ensure the `ProjectModelLowdb` database is ready to be read - await ProjectModelLowdb.read(); + // Ensure the `ProjectModelLowdb` database is ready to be read + await ProjectModelLowdb.read(); - // Retrieve the project index using the `getProjectUtil` helper - const projectIndex = (await getProjectUtil( - projectId, - { - id: projectId, - org_id: orgId, - region: token_payload?.region, - owner: token_payload?.user_id, - }, - srcFunc, - true - )) as number; + // Retrieve the project index using the `getProjectUtil` helper + const projectIndex = (await getProjectUtil( + projectId, + { + id: projectId, + org_id: orgId, + region: token_payload?.region, + owner: token_payload?.user_id, + }, + srcFunc, + true + )) as number; try { - // Update the project in the `ProjectModelLowdb` database await ProjectModelLowdb.update((data: any) => { data.projects[projectIndex].migration_execution = true; // Set migration execution to true @@ -1372,12 +1365,11 @@ const updateMigrationExecution = async (req: Request) => { // Return success response return { - status: HTTP_CODES.OK, + status: HTTP_CODES.OK, data: { - message: HTTP_TEXTS.MIGRATION_EXECUTION_KEY_UPDATED, + message: HTTP_TEXTS.MIGRATION_EXECUTION_KEY_UPDATED, }, }; - } catch (error: any) { // Log error message logger.error( @@ -1391,13 +1383,12 @@ const updateMigrationExecution = async (req: Request) => { // Throw a custom exception with the error details throw new ExceptionFunction( - error?.message || HTTP_TEXTS.INTERNAL_ERROR, - error?.statusCode || error?.status || HTTP_CODES.SERVER_ERROR + error?.message || HTTP_TEXTS.INTERNAL_ERROR, + error?.statusCode || error?.status || HTTP_CODES.SERVER_ERROR ); } }; - /** * get the destination_stack_id of completed projects. * @@ -1405,25 +1396,26 @@ const updateMigrationExecution = async (req: Request) => { * @returns An object with the status and data of the update operation. * @throws ExceptionFunction if an error occurs during the process. */ -const getMigratedStacks = async(req: Request) => { - - const { token_payload } = req.body; - const srcFunc = "getMigratedStacks"; +const getMigratedStacks = async (req: Request) => { + const { token_payload } = req.body; + const srcFunc = 'getMigratedStacks'; try { await ProjectModelLowdb.read(); const projects = ProjectModelLowdb.data?.projects || []; // Map through projects to extract `destinationstack` key - const destinationStacks = projects.filter((project: any) => project?.status === 5 && project.current_step === 5) - .map((project: any) => project.destination_stack_id); + const destinationStacks = projects + .filter( + (project: any) => project?.status === 5 && project.current_step === 5 + ) + .map((project: any) => project.destination_stack_id); return { - status: HTTP_CODES.OK, - destinationStacks + status: HTTP_CODES.OK, + destinationStacks, }; - - } catch (error:any) { + } catch (error: any) { // Log error message logger.error( getLogMessage( @@ -1436,13 +1428,11 @@ const getMigratedStacks = async(req: Request) => { // Throw a custom exception with the error details throw new ExceptionFunction( - error?.message || HTTP_TEXTS.INTERNAL_ERROR, - error?.statusCode || error?.status || HTTP_CODES.SERVER_ERROR + error?.message || HTTP_TEXTS.INTERNAL_ERROR, + error?.statusCode || error?.status || HTTP_CODES.SERVER_ERROR ); - } - -} +}; export const projectService = { getAllProjects, @@ -1461,5 +1451,5 @@ export const projectService = { updateStackDetails, updateContentMapper, updateMigrationExecution, - getMigratedStacks + getMigratedStacks, }; diff --git a/package-lock.json b/package-lock.json index 1b0684463..faad6a926 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "@types/estree": "^1.0.7", "@types/express": "^5.0.1", "husky": "^4.3.8", + "mysql2": "^3.15.1", "prettier": "^2.4.1", "rimraf": "^3.0.2", "validate-branch-name": "^1.3.0", @@ -463,6 +464,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/aws-ssl-profiles": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/aws-ssl-profiles/-/aws-ssl-profiles-1.1.2.tgz", + "integrity": "sha512-NZKeq9AfyQvEeNlN0zSYAaWrmBffJh3IELMZfRpJVWgrpEbtEpnjvzqBPf+mxoI287JohRDoa+/nsfqqiZmF6g==", + "dev": true, + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/axios": { "version": "1.11.0", "resolved": "https://registry.npmjs.org/axios/-/axios-1.11.0.tgz", @@ -1071,6 +1081,15 @@ "node": ">=0.4.0" } }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "dev": true, + "engines": { + "node": ">=0.10" + } + }, "node_modules/dot-prop": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-6.0.1.tgz", @@ -1534,6 +1553,15 @@ "node": ">= 0.6.0" } }, + "node_modules/generate-function": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.3.1.tgz", + "integrity": "sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ==", + "dev": true, + "dependencies": { + "is-property": "^1.0.2" + } + }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -2609,6 +2637,12 @@ "node": ">=8" } }, + "node_modules/is-property": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", + "integrity": "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g==", + "dev": true + }, "node_modules/is-regex": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.1.tgz", @@ -2912,6 +2946,36 @@ "node": ">= 12.0.0" } }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "dev": true + }, + "node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "dev": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/lru.min": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/lru.min/-/lru.min-1.1.2.tgz", + "integrity": "sha512-Nv9KddBcQSlQopmBHXSsZVY5xsdlZkdH/Iey0BlcBYggMd4two7cZnKOK9vmy3nY0O5RGH99z1PCeTpPqszUYg==", + "dev": true, + "engines": { + "bun": ">=1.0.0", + "deno": ">=1.30.0", + "node": ">=8.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wellwelwel" + } + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -2990,6 +3054,54 @@ "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==" }, + "node_modules/mysql2": { + "version": "3.15.1", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.15.1.tgz", + "integrity": "sha512-WZMIRZstT2MFfouEaDz/AGFnGi1A2GwaDe7XvKTdRJEYiAHbOrh4S3d8KFmQeh11U85G+BFjIvS1Di5alusZsw==", + "dev": true, + "dependencies": { + "aws-ssl-profiles": "^1.1.1", + "denque": "^2.1.0", + "generate-function": "^2.3.1", + "iconv-lite": "^0.7.0", + "long": "^5.2.1", + "lru.min": "^1.0.0", + "named-placeholders": "^1.1.3", + "seq-queue": "^0.0.5", + "sqlstring": "^2.3.2" + }, + "engines": { + "node": ">= 8.0" + } + }, + "node_modules/mysql2/node_modules/iconv-lite": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.0.tgz", + "integrity": "sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ==", + "dev": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/named-placeholders": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.3.tgz", + "integrity": "sha512-eLoBxg6wE/rZkJPhU/xRX1WTpkFEwDJEN96oxFrTsqBdbT5ec295Q+CoHrL9IT0DipqKhmGcaZmwOt8OON5x1w==", + "dev": true, + "dependencies": { + "lru-cache": "^7.14.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/object-inspect": { "version": "1.13.4", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", @@ -3713,6 +3825,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/seq-queue": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/seq-queue/-/seq-queue-0.0.5.tgz", + "integrity": "sha512-hr3Wtp/GZIc/6DAGPDcV4/9WoZhjrkXsi5B/07QgX8tsdc6ilr7BFM6PM6rbdAX1kFSDYeZGLipIZZKyQP0O5Q==", + "dev": true + }, "node_modules/set-blocking": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", @@ -3984,6 +4102,15 @@ "node": ">=6" } }, + "node_modules/sqlstring": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.3.tgz", + "integrity": "sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/stack-trace": { "version": "0.0.10", "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", diff --git a/package.json b/package.json index 73b1cbf93..a663ac948 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "@types/estree": "^1.0.7", "@types/express": "^5.0.1", "husky": "^4.3.8", + "mysql2": "^3.15.1", "prettier": "^2.4.1", "rimraf": "^3.0.2", "validate-branch-name": "^1.3.0", diff --git a/ui/src/cmsData/legacyCms.json b/ui/src/cmsData/legacyCms.json index d4f169925..bb844acf2 100644 --- a/ui/src/cmsData/legacyCms.json +++ b/ui/src/cmsData/legacyCms.json @@ -93,8 +93,8 @@ ] }, { - "cms_id": "drupal v8+", - "title": "Drupal v8+", + "cms_id": "drupal", + "title": "Drupal", "description": "", "group_name": "lightning", "doc_url": { diff --git a/ui/src/components/LegacyCms/Actions/LoadSelectCms.tsx b/ui/src/components/LegacyCms/Actions/LoadSelectCms.tsx index 0b6d15c86..507486e92 100644 --- a/ui/src/components/LegacyCms/Actions/LoadSelectCms.tsx +++ b/ui/src/components/LegacyCms/Actions/LoadSelectCms.tsx @@ -50,6 +50,7 @@ const LoadSelectCms = (props: LoadSelectCmsProps) => { //const [setErrorMessage] = useState(''); const [isError, setIsError] = useState(false); const [isLoading, setIsLoading] = useState(false); + const [configDetails, setConfigDetails] = useState(null); // Store config details (mysql, assetsConfig) /**** ALL METHODS HERE ****/ @@ -82,6 +83,18 @@ const LoadSelectCms = (props: LoadSelectCmsProps) => { const { data } = await getConfig(); // api call to get cms type from upload service + console.info('๐Ÿ” CONFIG API Response:', data); + console.info('๐Ÿ” data.mysql:', data?.mysql); + console.info('๐Ÿ” data.assetsConfig:', data?.assetsConfig); + + // Store config details to display in UI + setConfigDetails({ + mySQLDetails: data?.mysql, + assetsConfig: data?.assetsConfig, + isSQL: data?.isSQL, + cmsType: data?.cmsType + }); + const cms = data?.cmsType?.toLowerCase(); if (isEmptyString(cmsType?.cms_id)) { @@ -96,22 +109,47 @@ const LoadSelectCms = (props: LoadSelectCmsProps) => { (cms: ICMSType) => cms?.parent?.toLowerCase() === cmstype?.toLowerCase() ); } + + // Store config data (mysql, assetsConfig) in Redux for later use + // First, log what's currently in Redux + console.info('๐Ÿ” BEFORE UPDATE - Current file_details in Redux:', newMigrationData?.legacy_cms?.uploadedFile?.file_details); + + // Determine which CMS to set as selected + let newSelectedCard: ICMSType | undefined; + if (filteredCmsData?.length === 1) { + newSelectedCard = filteredCmsData[0]; + } else { + newSelectedCard = DEFAULT_CMS_TYPE; + } + const newMigrationDataObj = { ...newMigrationData, legacy_cms: { ...newMigrationData?.legacy_cms, - selectedFileFormat: filteredCmsData[0].allowed_file_formats[0] + selectedCms: newSelectedCard, // Include selectedCms in this dispatch + selectedFileFormat: filteredCmsData[0].allowed_file_formats[0], + uploadedFile: { + ...newMigrationData?.legacy_cms?.uploadedFile, + file_details: { + ...newMigrationData?.legacy_cms?.uploadedFile?.file_details, + mySQLDetails: data?.mysql, // Store mysql as mySQLDetails + assetsConfig: data?.assetsConfig, // Store assetsConfig + isSQL: data?.isSQL, + cmsType: data?.cmsType, + localPath: data?.localPath, + awsData: data?.awsData + } + } } }; - - // Debug logging for selectedFileFormat setting - console.info('๐Ÿ”ง === LOAD SELECT CMS DEBUG ==='); - console.info('๐Ÿ“‹ filteredCmsData[0]:', filteredCmsData[0]); - console.info('๐Ÿ“‹ allowed_file_formats[0]:', filteredCmsData[0]?.allowed_file_formats[0]); - console.info('๐Ÿ“‹ selectedFileFormat being set:', newMigrationDataObj.legacy_cms.selectedFileFormat); - console.info('================================'); - - //dispatch(updateNewMigrationData(newMigrationDataObj)); + + console.info('๐Ÿ” Updated newMigrationDataObj with config:', newMigrationDataObj); + console.info('๐Ÿ” file_details after mapping:', newMigrationDataObj?.legacy_cms?.uploadedFile?.file_details); + console.info('๐Ÿ” mySQLDetails after mapping:', newMigrationDataObj?.legacy_cms?.uploadedFile?.file_details?.mySQLDetails); + + dispatch(updateNewMigrationData(newMigrationDataObj)); // Dispatch to save config to Redux + + console.info('โœ… DISPATCHED to Redux - newMigrationDataObj dispatched'); setCmsData(filteredCmsData); @@ -127,13 +165,6 @@ const LoadSelectCms = (props: LoadSelectCmsProps) => { setCmsData(_filterCmsData); - let newSelectedCard: ICMSType | undefined; - - if (filteredCmsData?.length === 1) { - newSelectedCard = filteredCmsData[0]; - } else { - newSelectedCard = DEFAULT_CMS_TYPE; - } setIsLoading(false); if (!isEmptyString(newSelectedCard?.title)) { @@ -141,15 +172,10 @@ const LoadSelectCms = (props: LoadSelectCmsProps) => { //setErrorMessage(''); setIsError(false); - const newMigrationDataObj: INewMigration = { - ...newMigrationData, - legacy_cms: { - ...newMigrationData?.legacy_cms, - selectedCms: newSelectedCard - } - }; + // The dispatch already happened above (line 150) with all the data including selectedCms + // No need to dispatch again here + //await updateLegacyCMSData(selectedOrganisation.value, projectId, { legacy_cms: newSelectedCard?.cms_id }); - dispatch(updateNewMigrationData(newMigrationDataObj)); props?.handleStepChange(props?.currentStep); } } catch (error) { diff --git a/ui/src/components/LegacyCms/Actions/LoadUploadFile.tsx b/ui/src/components/LegacyCms/Actions/LoadUploadFile.tsx index 1528815a6..3cfe02e5d 100644 --- a/ui/src/components/LegacyCms/Actions/LoadUploadFile.tsx +++ b/ui/src/components/LegacyCms/Actions/LoadUploadFile.tsx @@ -37,15 +37,25 @@ interface UploadState { const FileComponent = ({ fileDetails }: Props) => { + return (
- {fileDetails?.isLocalPath && - (!isEmptyString(fileDetails?.localPath) || - !isEmptyString(fileDetails?.awsData?.awsRegion)) ? ( + {fileDetails?.isLocalPath ? ( + // โœ… Case 1: Local file path
+ ) : fileDetails?.isSQL ? ( + // โœ… Case 2: MySQL details + fileDetails?.mySQLDetails && ( +
+

Host: {fileDetails?.mySQLDetails?.host}

+

Database: {fileDetails?.mySQLDetails?.database}

+

User: {fileDetails?.mySQLDetails?.user}

+
+ ) ) : ( + // โœ… Case 3: AWS details

AWS Region: {fileDetails?.awsData?.awsRegion}

Bucket Name: {fileDetails?.awsData?.bucketName}

@@ -56,6 +66,7 @@ const FileComponent = ({ fileDetails }: Props) => { ); }; + const saveStateToLocalStorage = (state: UploadState, projectId: string) => { sessionStorage.setItem(`uploadProgressState_${projectId}`, JSON.stringify(state)); }; @@ -84,7 +95,8 @@ const LoadUploadFile = (props: LoadUploadFileProps) => { ); const [isConfigLoading, setIsConfigLoading] = useState(false); const [cmsType, setCmsType]= useState(''); - const [fileDetails, setFileDetails] = useState(newMigrationDataRef?.current?.legacy_cms?.uploadedFile?.file_details); + // Use newMigrationData directly from Redux, not the ref, so it updates when Redux changes + const [fileDetails, setFileDetails] = useState(newMigrationData?.legacy_cms?.uploadedFile?.file_details); const [fileExtension, setFileExtension] = useState(''); const [progressPercentage, setProgressPercentage] = useState(0); const [showProgress, setShowProgress] = useState(false); @@ -102,6 +114,7 @@ const LoadUploadFile = (props: LoadUploadFileProps) => { //Handle further action on file is uploaded to server const handleOnFileUploadCompletion = async () => { try { + setIsValidationAttempted(false); setValidationMessage(''); setIsLoading(true); @@ -146,9 +159,9 @@ const LoadUploadFile = (props: LoadUploadFileProps) => { database: data?.file_details?.mySQLDetails?.database, port: data?.file_details?.mySQLDetails?.port }, - drupalAssetsUrl: { - base_url: data?.file_details?.drupalAssetsUrl?.base_url, - public_path: data?.file_details?.drupalAssetsUrl?.public_path + assetsConfig: { + base_url: data?.file_details?.assetsConfig?.base_url, + public_path: data?.file_details?.assetsConfig?.public_path } }, cmsType: data?.cmsType @@ -158,8 +171,6 @@ const LoadUploadFile = (props: LoadUploadFileProps) => { // For Drupal SQL files, ensure selectedFileFormat is set in the same update if (status === 200 && data?.file_details?.isSQL && data?.file_details?.cmsType === 'drupal') { - console.info('๐Ÿ”ง === LOAD UPLOAD FILE DEBUG ==='); - console.info('๐Ÿ“‹ Setting selectedFileFormat for Drupal SQL in combined update'); // Add selectedFileFormat to the existing newMigrationDataObj newMigrationDataObj.legacy_cms.selectedFileFormat = { @@ -169,18 +180,17 @@ const LoadUploadFile = (props: LoadUploadFileProps) => { group_name: 'sql', isactive: true }; - - console.info('๐Ÿ“‹ Combined newMigrationDataObj:', newMigrationDataObj); - console.info('================================'); } - console.info('๐Ÿ“‹ Dispatching combined newMigrationDataObj:', newMigrationDataObj); - console.info('๐Ÿ” DEBUG: uploadedFile.isValidated in dispatch:', newMigrationDataObj.legacy_cms.uploadedFile.isValidated); dispatch(updateNewMigrationData(newMigrationDataObj)); if (status === 200) { setIsValidated(true); - setValidationMessage('File validated successfully.'); + setValidationMessage( + data?.file_details?.isSQL + ? 'Connection established successfully.' + : 'File validated successfully.' + ); setIsDisabled(true); @@ -196,7 +206,11 @@ const LoadUploadFile = (props: LoadUploadFileProps) => { } } else if (status === 500) { setIsValidated(false); - setValidationMessage('File not found'); + setValidationMessage( + data?.file_details?.isSQL + ? 'Connection failed' + : 'File not found' + ); setIsValidationAttempted(true); setProgressPercentage(100); } else if (status === 429) { @@ -318,6 +332,17 @@ const LoadUploadFile = (props: LoadUploadFileProps) => { } }; + // Update fileDetails whenever Redux state changes + useEffect(() => { + const latestFileDetails = newMigrationData?.legacy_cms?.uploadedFile?.file_details; + + if (latestFileDetails) { + setFileDetails(latestFileDetails); + } else { + console.warn('โš ๏ธ latestFileDetails is empty, not updating'); + } + }, [newMigrationData?.legacy_cms?.uploadedFile?.file_details]); + useEffect(() => { getConfigDetails(); }, []); @@ -389,7 +414,11 @@ const LoadUploadFile = (props: LoadUploadFileProps) => { ) { setIsValidated(true); setShowMessage(true); - setValidationMessage('File validated successfully.'); + setValidationMessage( + fileDetails?.isSQL + ? 'Connection established successfully.' + : 'File validated successfully.' + ); setIsDisabled(true); !isEmptyString(newMigrationData?.legacy_cms?.affix) || !isEmptyString(newMigrationData?.legacy_cms?.selectedCms?.cms_id) || @@ -486,7 +515,7 @@ const LoadUploadFile = (props: LoadUploadFileProps) => { version="v2" disabled={!(reValidate || (!isDisabled && !isEmptyString(newMigrationData?.legacy_cms?.affix)))} > - Validate File + {fileDetails?.isSQL ? 'Check Connection' : 'Validate File'}
diff --git a/ui/src/context/app/app.interface.ts b/ui/src/context/app/app.interface.ts index a589a1591..b6fb0bf3e 100644 --- a/ui/src/context/app/app.interface.ts +++ b/ui/src/context/app/app.interface.ts @@ -64,7 +64,7 @@ export interface FileDetails { database?: string; port?: number; }; - drupalAssetsUrl?: { + assetsConfig?: { base_url?: string; public_path?: string; }; @@ -81,7 +81,7 @@ export interface IFile { isValidated: boolean; reValidate: boolean; cmsType: string; - buttonClicked:boolean + buttonClicked: boolean; } export interface ICMSType extends ICardType { @@ -217,7 +217,7 @@ export interface INewMigration { stackDetails: IDropDown; migration_execution: IMigrationExecutionStep; project_current_step: number; - settings:ISetting; + settings: ISetting; } export interface TestStacks { @@ -334,7 +334,7 @@ export const DEFAULT_FILE: IFile = { database: '', port: 0 }, - drupalAssetsUrl: { + assetsConfig: { base_url: '', public_path: '' }, @@ -418,7 +418,7 @@ export const DEFAULT_NEW_MIGRATION: INewMigration = { migration_execution: DEFAULT_MIGRATION_EXECUTION_STEP, project_current_step: 0, settings: DEFAULT_SETTING, - isContentMapperGenerated: false, + isContentMapperGenerated: false }; export const DEFAULT_URL_TYPE: IURLType = { diff --git a/upload-api/migration-drupal/libs/contentTypeMapper.js b/upload-api/migration-drupal/libs/contentTypeMapper.js index 644d4403b..3f747a052 100644 --- a/upload-api/migration-drupal/libs/contentTypeMapper.js +++ b/upload-api/migration-drupal/libs/contentTypeMapper.js @@ -116,7 +116,7 @@ const extractAdvancedFields = (item, referenceFields = []) => { return { default_value: item?.default_value || null, mandatory: item?.required || false, - multiple: item?.max > 1 || false, + multiple: item?.max > 1 || item?.cardinality === -1 || false, unique: false, nonLocalizable: false, validationErrorMessage: '', @@ -171,23 +171,206 @@ const createFieldObject = (item, contentstackFieldType, backupFieldType, referen }; /** - * Creates a field object for dropdown or radio field types with appropriate options and validations. + * Creates a field object for boolean field types with proper display_type. * - * @param {Object} item - The Drupal field item that includes field details like `type`, etc. - * @param {string} fieldType - The type of field being created (e.g., 'dropdown', 'radio'). + * @param {Object} item - The Drupal field item that includes field details like `type`, `widget`, etc. + * @returns {Object} A field object that includes the field configuration for boolean fields. + */ +const createBooleanFieldObject = (item) => { + const fieldNameWithSuffix = item?.field_name; + const advancedFields = extractAdvancedFields(item); + + // Determine display type based on widget + let displayType; + const widgetType = item.widget?.type || item.widget; + + if (widgetType === 'boolean_checkbox') { + displayType = 'checkbox'; + } + // Add other boolean widget types as needed + + return { + uid: item?.field_name, + otherCmsField: item?.field_label, + otherCmsType: item?.type, + contentstackField: item?.field_label, + contentstackFieldUid: uidCorrector(fieldNameWithSuffix, item?.prefix), + contentstackFieldType: 'boolean', + backupFieldType: 'boolean', + backupFieldUid: uidCorrector(fieldNameWithSuffix, item?.prefix), + advanced: { + ...advancedFields, + data_type: 'boolean', + display_type: displayType, + field_metadata: { + description: advancedFields?.description || '', + default_value: false + } + } + }; +}; + +/** + * Creates a field object for dropdown, radio, or checkbox field types following CSV scenarios. + * Maps Drupal field configurations to Contentstack format based on widget and cardinality. + * + * @param {Object} item - The Drupal field item that includes field details like `type`, `widget`, etc. + * @param {string} baseFieldType - The base field type ('dropdown', 'radio', 'checkbox'). + * @param {string} dataType - The data type ('text' for list_string, 'number' for list_integer/list_float). * @returns {Object} A field object that includes the field configuration and validation options. * * @description - * This function generates a field object for dropdown or radio field types based on the provided item. - * It ensures that the field's advanced properties are extracted from the item, including validation options. + * This function generates a field object for list field types based on CSV scenarios: + * - Dropdown (options_select) โ†’ display_type: "dropdown", multiple: false + * - Radio (options_buttons + cardinality=1) โ†’ display_type: "radio", multiple: false + * - Checkboxes (options_buttons + cardinality=-1) โ†’ display_type: "checkbox", multiple: true + */ +const createDropdownOrRadioFieldObject = ( + item, + baseFieldType, + dataType = 'text', + numericType = null +) => { + // Determine display type and multiple based on CSV scenarios + let displayType = 'dropdown'; + let multiple = false; + + // Map based on CSV scenarios from drupal_field_mapping.csv + const widgetType = item.widget?.type || item.widget; + if (widgetType) { + switch (widgetType) { + case 'options_select': + // Dropdown scenario + displayType = 'dropdown'; + multiple = false; + break; + case 'options_buttons': + // Radio or Checkbox based on cardinality + if (item.cardinality === -1 || item.max === -1 || item.max > 1) { + // Checkboxes (cardinality=-1) + displayType = 'checkbox'; + multiple = true; + } else { + // Radio buttons (cardinality=1) + displayType = 'radio'; + multiple = false; + } + break; + default: + // Fallback to dropdown + displayType = 'dropdown'; + multiple = false; + } + } else { + // Fallback based on field configuration + displayType = baseFieldType; + multiple = item.max > 1 || item.cardinality === -1; + } + + const fieldNameWithSuffix = item?.field_name; + const advancedFields = extractAdvancedFields(item); + + // Extract actual choices from field settings (allowed_values) + let actualChoices = []; + + if (item.settings && item.settings.allowed_values) { + // Convert allowed_values object to choices array + // Drupal format: { stored_key: display_label } -> Contentstack format: [{ value: display_label, key: stored_key }] + actualChoices = Object.entries(item.settings.allowed_values).map(([key, value]) => { + let processedKey = key; + + // For numeric fields, ensure the key (stored value) is properly typed + if (dataType === 'number') { + if (numericType === 'float') { + // For float fields, preserve decimal precision in the key + processedKey = isNaN(key) ? key : parseFloat(key); + } else if (numericType === 'integer') { + // For integer fields, convert key to integer + processedKey = isNaN(key) ? key : parseInt(key, 10); + } else { + // Default number handling for key + processedKey = isNaN(key) ? key : Number(key); + } + } + + return { + value: value, // Display label (e.g., "1.5 Stars", "$10.50") + key: processedKey // Stored value (e.g., 1.5, 10.50) + }; + }); + } else { + // Fallback: generate minimal choices if no allowed_values found + if (dataType === 'number') { + if (numericType === 'float') { + actualChoices = [ + { value: 1.0, key: 'option_1' }, + { value: 2.0, key: 'option_2' } + ]; + } else { + actualChoices = [ + { value: 1, key: 'option_1' }, + { value: 2, key: 'option_2' } + ]; + } + } else { + actualChoices = [ + { value: 'Option 1', key: 'option_1' }, + { value: 'Option 2', key: 'option_2' } + ]; + } + } + + return { + uid: item?.field_name, + otherCmsField: item?.field_label, + otherCmsType: item?.type, + contentstackField: item?.field_label, + contentstackFieldUid: uidCorrector(fieldNameWithSuffix, item?.prefix), + contentstackFieldType: baseFieldType, + backupFieldType: baseFieldType, + backupFieldUid: uidCorrector(fieldNameWithSuffix, item?.prefix), + advanced: { + ...advancedFields, + data_type: dataType, + display_type: displayType, + multiple: multiple, + enum: { + advanced: true, + choices: actualChoices + }, + field_metadata: { + description: advancedFields?.description || '', + default_value: '', + default_key: '' + } + } + }; +}; + +/** + * Creates a date range field object with specialized structure for Contentstack * + * @param {Object} item - The Drupal field item containing field details + * @returns {Object} A date range field object with date_range: true metadata */ -const createDropdownOrRadioFieldObject = (item, fieldType) => { +const createDateRangeFieldObject = (item) => { + const fieldNameWithSuffix = item?.field_name; + const advancedFields = extractAdvancedFields(item); + return { - ...createFieldObject(item, fieldType, fieldType), + uid: item?.field_name, + otherCmsField: item?.field_label, + otherCmsType: item?.type, + contentstackField: item?.field_label, + contentstackFieldUid: uidCorrector(fieldNameWithSuffix, item?.prefix), + contentstackFieldType: 'isodate', + backupFieldType: 'isodate', + backupFieldUid: uidCorrector(fieldNameWithSuffix, item?.prefix), advanced: { - ...extractAdvancedFields(item), - options: [{ value: 'value', key: 'key' }] + ...advancedFields, + date_range: true, // This enables start/end date functionality + description: advancedFields?.description || '', + default_value: {} } }; }; @@ -306,30 +489,40 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => break; } case 'text': { - // Single line with switching options: single_line โ†’ multiline โ†’ HTML RTE โ†’ JSON RTE - const availableContentTypes = contentTypes?.filter((ct) => ct !== item.content_types) || []; - const referenceFields = availableContentTypes.slice(0, 10); - acc.push(createFieldObject(item, 'single_line_text', 'multi_line_text', referenceFields)); + // Multi Line Text Fields + acc.push(createFieldObject(item, 'multi_line_text', 'multi_line_text')); + break; + } + case 'string_long': { + // Multi Line Text Fields + acc.push(createFieldObject(item, 'multi_line_text', 'multi_line_text')); break; } - case 'string_long': case 'comment': { - // Rich text with switching options: JSON RTE โ†’ HTML RTE โ†’ multiline โ†’ text - const availableContentTypes = contentTypes?.filter((ct) => ct !== item.content_types) || []; - const referenceFields = availableContentTypes.slice(0, 10); - acc.push(createFieldObject(item, 'json', 'html', referenceFields)); + // Comment Field - multiline + acc.push(createFieldObject(item, 'multi_line_text', 'multi_line_text')); break; } - case 'string': - case 'list_string': { + case 'string': { // Single line with switching options: single_line โ†’ multiline โ†’ HTML RTE โ†’ JSON RTE const availableContentTypes = contentTypes?.filter((ct) => ct !== item.content_types) || []; const referenceFields = availableContentTypes.slice(0, 10); acc.push(createFieldObject(item, 'single_line_text', 'multi_line_text', referenceFields)); break; } + case 'telephone': { + // Telephone field - number field + acc.push(createFieldObject(item, 'number', 'number')); + break; + } case 'email': { - acc.push(createFieldObject(item, 'text', 'text')); + // Email field - single line text + acc.push(createFieldObject(item, 'single_line_text', 'single_line_text')); + break; + } + case 'list_string': { + // Select/Dropdown field for string values (supports radio/checkbox/dropdown) + acc.push(createDropdownOrRadioFieldObject(item, 'dropdown', 'text')); break; } case 'taxonomy_term_reference': { @@ -365,8 +558,11 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => break; } case 'entity_reference': { - // Check if this is a taxonomy field by handler - if (item.handler === 'default:taxonomy_term') { + // Check if this is a media field by handler + if (item.handler === 'default:media') { + // Media entity references should be treated as file fields + acc.push(createFieldObject(item, 'file', 'file')); + } else if (item.handler === 'default:taxonomy_term') { // ๐Ÿท๏ธ Collect taxonomy field for consolidation instead of creating individual fields // Try to determine specific vocabularies this field references @@ -432,7 +628,7 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => } case 'list_boolean': case 'boolean': { - acc.push(createFieldObject(item, 'boolean', 'boolean')); + acc.push(createBooleanFieldObject(item)); break; } case 'datetime': @@ -440,24 +636,39 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => acc.push(createFieldObject(item, 'isodate', 'isodate')); break; } + case 'daterange': { + // Date range field - isodate with date_range: true + acc.push(createDateRangeFieldObject(item)); + break; + } case 'integer': case 'decimal': - case 'float': - case 'list_integer': - case 'list_float': { + case 'float': { acc.push(createFieldObject(item, 'number', 'number')); break; } + case 'list_integer': { + // Select/Dropdown field for integer values (supports radio/checkbox/dropdown) + acc.push(createDropdownOrRadioFieldObject(item, 'dropdown', 'number', 'integer')); + break; + } + case 'list_float': { + // Select/Dropdown field for float values (supports radio/checkbox/dropdown) + acc.push(createDropdownOrRadioFieldObject(item, 'dropdown', 'number', 'float')); + break; + } case 'link': { acc.push(createFieldObject(item, 'link', 'link')); break; } case 'list_text': { - acc.push(createDropdownOrRadioFieldObject(item, 'dropdown')); + // Select/Dropdown field for text values (supports radio/checkbox/dropdown) + acc.push(createDropdownOrRadioFieldObject(item, 'dropdown', 'text')); break; } case 'list_number': { - acc.push(createDropdownOrRadioFieldObject(item, 'dropdown')); + // Select/Dropdown field for number values (supports radio/checkbox/dropdown) + acc.push(createDropdownOrRadioFieldObject(item, 'dropdown', 'number')); break; } default: { diff --git a/upload-api/migration-drupal/libs/createInitialMapper.js b/upload-api/migration-drupal/libs/createInitialMapper.js index e5d67bddd..d5fb51727 100644 --- a/upload-api/migration-drupal/libs/createInitialMapper.js +++ b/upload-api/migration-drupal/libs/createInitialMapper.js @@ -104,7 +104,7 @@ const createInitialMapper = async (systemConfig, prefix) => { default_value: conv_details?.default_value?.[0]?.value }); } catch (error) { - console.warn(`Couldn't parse row ${i}:`, error.message); + console.error(`Couldn't parse row ${i}:`, error.message); } } diff --git a/upload-api/migration-drupal/libs/extractTaxonomy.js b/upload-api/migration-drupal/libs/extractTaxonomy.js index eca3c72ea..1a4aee754 100644 --- a/upload-api/migration-drupal/libs/extractTaxonomy.js +++ b/upload-api/migration-drupal/libs/extractTaxonomy.js @@ -86,7 +86,7 @@ const extractTaxonomy = async (dbConfig) => { } } } catch (parseError) { - console.warn(`โš ๏ธ Failed to parse vocabulary data for ${vocab.vid}:`, parseError.message); + console.error(`โš ๏ธ Failed to parse vocabulary data for ${vocab.vid}:`, parseError.message); } } diff --git a/upload-api/src/config/index.ts b/upload-api/src/config/index.ts index 9f4c210da..bb837f711 100644 --- a/upload-api/src/config/index.ts +++ b/upload-api/src/config/index.ts @@ -20,9 +20,9 @@ export default { database: 'riceuniversity1', port: '3306' }, - drupalAssetsUrl: { - base_url: '', - public_path: '' + assetsConfig: { + base_url: 'https://www.rice.edu/', // Dynamic: Can be any domain, with/without trailing slash + public_path: 'sites/g/files/bxs2566/files' // Dynamic: Can be any path, with/without slashes }, localPath: process.env.CONTAINER_PATH || 'sql' }; diff --git a/upload-api/src/models/types.ts b/upload-api/src/models/types.ts index 228b03fca..bea27b72d 100644 --- a/upload-api/src/models/types.ts +++ b/upload-api/src/models/types.ts @@ -23,8 +23,8 @@ export interface Config { database: string; port: string; }; - drupalAssetsUrl: { + assetsConfig: { base_url: string; public_path: string; }; -} \ No newline at end of file +} diff --git a/upload-api/src/routes/index.ts b/upload-api/src/routes/index.ts index 84681cf70..95ad20e9a 100644 --- a/upload-api/src/routes/index.ts +++ b/upload-api/src/routes/index.ts @@ -212,7 +212,7 @@ router.get( ...result.file_details, isSQL: config.isSQL, mySQLDetails: config.mysql, // Changed from mysql to mySQLDetails - drupalAssetsUrl: config.drupalAssetsUrl + assetsConfig: config.assetsConfig } }; @@ -286,6 +286,7 @@ router.get( ); router.get('/config', async function (req: Request, res: Response) { + console.log('๐Ÿ” UPLOAD-API Response config:', config); res.json(config); }); diff --git a/upload-api/src/services/createMapper.ts b/upload-api/src/services/createMapper.ts index ee4f5ab79..44db0ee49 100644 --- a/upload-api/src/services/createMapper.ts +++ b/upload-api/src/services/createMapper.ts @@ -12,8 +12,7 @@ const createMapper = async ( config: Config ) => { const CMSIdentifier = config?.cmsType?.toLowerCase(); - console.log('CMSIdentifier', CMSIdentifier); - + switch (CMSIdentifier) { case 'sitecore': { return await createSitecoreMapper(filePath, projectId, app_token, affix, config); diff --git a/upload-api/src/services/fileProcessing.ts b/upload-api/src/services/fileProcessing.ts index ced293f34..e14ee13f6 100644 --- a/upload-api/src/services/fileProcessing.ts +++ b/upload-api/src/services/fileProcessing.ts @@ -71,10 +71,12 @@ const handleFileProcessing = async ( } else if (fileExt === 'sql') { try { // Validate SQL connection using our Drupal validator + // Also validate assets configuration if provided const isValidConnection = await validator({ data: config.mysql, type: cmsType, - extension: fileExt + extension: fileExt, + assetsConfig: config.assetsConfig // Pass assetsConfig for validation }); if (isValidConnection) { diff --git a/upload-api/src/validators/drupal/index.ts b/upload-api/src/validators/drupal/index.ts index 6b23e149c..b5cb3c94d 100644 --- a/upload-api/src/validators/drupal/index.ts +++ b/upload-api/src/validators/drupal/index.ts @@ -1,4 +1,5 @@ import mysql from 'mysql2/promise'; +import axios from 'axios'; import logger from '../../utils/logger'; interface ValidatorProps { @@ -9,15 +10,264 @@ interface ValidatorProps { database: string; port: number | string; }; + assetsConfig?: { + base_url?: string; + public_path?: string; + }; } /** - * Validates Drupal SQL connection by testing a specific query + * Normalizes and validates URL configuration + */ +function normalizeUrlConfig( + baseUrl: string, + publicPath: string +): { baseUrl: string; publicPath: string } { + let normalizedBaseUrl = baseUrl ? baseUrl.trim() : ''; + let normalizedPublicPath = publicPath ? publicPath.trim() : ''; + + if (normalizedBaseUrl) { + // Remove trailing slash from baseUrl + normalizedBaseUrl = normalizedBaseUrl.replace(/\/+$/, ''); + + // Ensure baseUrl has protocol + if (!normalizedBaseUrl.startsWith('http://') && !normalizedBaseUrl.startsWith('https://')) { + normalizedBaseUrl = `https://${normalizedBaseUrl}`; + } + } + + if (normalizedPublicPath) { + // Ensure publicPath starts with / + if (!normalizedPublicPath.startsWith('/')) { + normalizedPublicPath = `/${normalizedPublicPath}`; + } + + // Ensure publicPath ends with / + if (!normalizedPublicPath.endsWith('/')) { + normalizedPublicPath = `${normalizedPublicPath}/`; + } + + // Remove duplicate slashes + normalizedPublicPath = normalizedPublicPath.replace(/\/+/g, '/'); + } + + return { + baseUrl: normalizedBaseUrl, + publicPath: normalizedPublicPath + }; +} + +/** + * Constructs asset URL from URI, baseUrl, and publicPath + */ +function constructAssetUrl(uri: string, baseUrl: string, publicPath: string): string { + const { baseUrl: cleanBaseUrl, publicPath: cleanPublicPath } = normalizeUrlConfig( + baseUrl, + publicPath + ); + + // Handle public:// scheme + if (uri.startsWith('public://')) { + const relativePath = uri.replace('public://', ''); + return `${cleanBaseUrl}${cleanPublicPath}${relativePath}`; + } + + // Handle private:// scheme + if (uri.startsWith('private://')) { + const relativePath = uri.replace('private://', ''); + return `${cleanBaseUrl}/system/files/${relativePath}`; + } + + // Handle absolute URLs + if (uri.startsWith('http://') || uri.startsWith('https://')) { + return uri; + } + + // Handle relative paths + const path = uri.startsWith('/') ? uri : `/${uri}`; + return `${cleanBaseUrl}${path}`; +} + +/** + * Validates assets configuration by testing if assets are accessible + * @param connection - Active MySQL connection + * @param assetsConfig - Assets configuration with base_url and public_path + * @returns Promise - true if at least one asset returns 200, false otherwise + */ +async function validateAssetsConfig( + connection: mysql.Connection, + assetsConfig: { base_url?: string; public_path?: string } +): Promise { + try { + const baseUrl = assetsConfig.base_url || ''; + const publicPath = assetsConfig.public_path || ''; + + if (!baseUrl) { + logger.warn('Assets validator: No base_url provided, skipping asset validation'); + return true; // Skip validation if no base_url provided + } + + logger.info('Assets validator: Starting asset URL validation', { + baseUrl, + publicPath + }); + + // Fetch up to 10 assets from the database + const assetsQuery = ` + SELECT + fm.fid, + fm.filename, + fm.uri, + fm.filesize, + fm.filemime + FROM file_managed fm + WHERE fm.uri LIKE 'public://%' + ORDER BY fm.fid ASC + LIMIT 10 + `; + + const [assets] = await connection.execute(assetsQuery); + + if (!Array.isArray(assets) || assets.length === 0) { + logger.warn('Assets validator: No assets found in database'); + return true; // No assets to validate + } + + logger.info(`Assets validator: Found ${assets.length} assets to test`); + + // Test each asset URL + let successCount = 0; + const testResults: Array<{ + uri: string; + url: string; + status: number | string; + success: boolean; + }> = []; + + for (const asset of assets as any[]) { + const assetUrl = constructAssetUrl(asset.uri, baseUrl, publicPath); + + try { + const response = await axios.head(assetUrl, { + timeout: 5000, + maxRedirects: 5, + validateStatus: (status) => status === 200, + headers: { + 'User-Agent': 'Contentstack-Drupal-Migration-Validator/1.0' + } + }); + + if (response.status === 200) { + // โœ… CHECK CONTENT-TYPE: Ensure it's an actual asset, not an HTML page + const contentType = response.headers['content-type'] || ''; + + // Valid asset content types (not HTML) + const isValidAsset = + contentType.includes('image/') || // Images: image/jpeg, image/png, etc. + contentType.includes('application/pdf') || // PDFs + contentType.includes('application/zip') || // Archives + contentType.includes('video/') || // Videos + contentType.includes('audio/') || // Audio + contentType.includes('application/octet-stream') || // Generic binary + contentType.includes('application/msword') || // Word docs + contentType.includes('application/vnd.') || // Office docs (Excel, PowerPoint, etc.) + contentType.includes('text/plain') || // Text files + contentType.includes('text/csv'); // CSV files + + // Reject HTML pages (common for error pages or redirects) + const isHtmlPage = + contentType.includes('text/html') || contentType.includes('application/xhtml'); + + if (isValidAsset && !isHtmlPage) { + successCount++; + testResults.push({ + uri: asset.uri, + url: assetUrl, + status: 200, + success: true + }); + + logger.info(`Assets validator: Valid asset found`, { + uri: asset.uri, + url: assetUrl, + status: 200, + contentType: contentType + }); + + // If we found at least one working asset, validation passes + if (successCount >= 1) { + logger.info( + `Assets validator: Validation successful (${successCount}/${assets.length} assets accessible)`, + { + successCount, + totalTested: testResults.length, + baseUrl, + publicPath + } + ); + return true; + } + } else { + // Status 200 but wrong content type (likely HTML error page) + testResults.push({ + uri: asset.uri, + url: assetUrl, + status: `200 but invalid content-type: ${contentType}`, + success: false + }); + + logger.warn(`Assets validator: URL returns 200 but not a valid asset`, { + uri: asset.uri, + url: assetUrl, + contentType: contentType, + isHtmlPage: isHtmlPage + }); + } + } + } catch (error: any) { + testResults.push({ + uri: asset.uri, + url: assetUrl, + status: error.response?.status || error.message, + success: false + }); + + logger.debug('Assets validator: Asset not accessible', { + uri: asset.uri, + url: assetUrl, + status: error.response?.status, + error: error.message + }); + } + } + + // If no assets were accessible, validation fails + logger.error('Assets validator: No accessible assets found', { + totalTested: testResults.length, + successCount, + baseUrl, + publicPath, + failedUrls: testResults.filter((r) => !r.success).map((r) => r.url) + }); + + return false; + } catch (error: any) { + logger.error('Assets validator: Error during asset validation', { + error: error.message, + stack: error.stack + }); + return false; + } +} + +/** + * Validates Drupal SQL connection and optionally validates assets configuration * Tests connection with: "SELECT *, CONVERT(data USING utf8) as data FROM config WHERE name LIKE '%field.field.node%'" * @param data - Database configuration object containing connection details + * @param assetsConfig - Optional assets configuration to validate * @returns Promise - true if connection successful and query returns results, false otherwise */ -async function drupalValidator({ data }: ValidatorProps): Promise { +async function drupalValidator({ data, assetsConfig }: ValidatorProps): Promise { let connection: mysql.Connection | null = null; try { @@ -90,10 +340,30 @@ async function drupalValidator({ data }: ValidatorProps): Promise { const hasConfigResults = Array.isArray(configRows) && configRows.length > 0; if (hasConfigResults) { - logger.info('Drupal validator: All validation checks passed successfully', { + logger.info('Drupal validator: Database validation checks passed', { nodeFieldDataExists: true, configQueryResults: (configRows as any[]).length }); + + // If assetsConfig is provided, validate assets accessibility + if (assetsConfig && (assetsConfig.base_url || assetsConfig.public_path)) { + logger.info('Drupal validator: Starting assets configuration validation'); + + const assetsValid = await validateAssetsConfig(connection, assetsConfig); + + if (!assetsValid) { + logger.error('Drupal validator: Assets validation failed', { + baseUrl: assetsConfig.base_url, + publicPath: assetsConfig.public_path + }); + return false; + } + + logger.info('Drupal validator: All validation checks (DB + Assets) passed successfully'); + } else { + logger.info('Drupal validator: No assetsConfig provided, skipping asset validation'); + } + return true; } else { logger.warn('Drupal validator: Config query executed but returned no results', { diff --git a/upload-api/src/validators/index.ts b/upload-api/src/validators/index.ts index 711e69a9b..367244a42 100644 --- a/upload-api/src/validators/index.ts +++ b/upload-api/src/validators/index.ts @@ -4,7 +4,17 @@ import wordpressValidator from './wordpress'; import aemValidator from './aem'; import drupalValidator from './drupal'; -const validator = ({ data, type, extension }: { data: any; type: string; extension: string }) => { +const validator = ({ + data, + type, + extension, + assetsConfig +}: { + data: any; + type: string; + extension: string; + assetsConfig?: { base_url?: string; public_path?: string }; +}) => { const CMSIdentifier = `${type}-${extension}`; switch (CMSIdentifier) { case 'sitecore-zip': { @@ -24,7 +34,7 @@ const validator = ({ data, type, extension }: { data: any; type: string; extensi } case 'drupal-sql': { - return drupalValidator({ data }); + return drupalValidator({ data, assetsConfig }); } default: From 2f75331420997e7c13bf3d11fb4130f7adfe5c1a Mon Sep 17 00:00:00 2001 From: sauravraw Date: Tue, 14 Oct 2025 11:31:42 +0530 Subject: [PATCH 11/37] Refactor content mapper and enhance taxonomy handling - Updated import statements to use single quotes for consistency. - Refactored functions in `projects.contentMapper.controller.ts` for improved readability. - Added `getExistingTaxonomies` function to retrieve taxonomies from both source and destination. - Enhanced `putTestData` in `contentMapper.service.ts` to store taxonomies in the project database. - Updated UI components to support taxonomy selection and display. - Improved logging for better traceability during taxonomy operations. --- .../projects.contentMapper.controller.ts | 50 +++-- api/src/models/project-lowdb.ts | 13 +- api/src/routes/contentMapper.routes.ts | 36 ++- api/src/services/contentMapper.service.ts | 211 ++++++++++++++++++ .../services/drupal/content-types.service.ts | 148 ++++++++++-- ui/src/components/AdvancePropertise/index.tsx | 198 +++++++++++++++- .../ContentMapper/contentMapper.interface.ts | 7 + ui/src/services/api/migration.service.ts | 97 +++++--- upload-api/src/services/drupal/index.ts | 44 +++- 9 files changed, 724 insertions(+), 80 deletions(-) diff --git a/api/src/controllers/projects.contentMapper.controller.ts b/api/src/controllers/projects.contentMapper.controller.ts index af49095fa..3e3ede696 100644 --- a/api/src/controllers/projects.contentMapper.controller.ts +++ b/api/src/controllers/projects.contentMapper.controller.ts @@ -1,5 +1,5 @@ -import { Request, Response } from "express"; -import { contentMapperService } from "../services/contentMapper.service.js"; +import { Request, Response } from 'express'; +import { contentMapperService } from '../services/contentMapper.service.js'; /** * Handles the PUT request to update test data. * @@ -132,22 +132,43 @@ const getSingleContentTypes = async ( * @param res - The response object. * @returns A Promise that resolves to void. */ -const getSingleGlobalField = async(req: Request, res: Response): Promise => { +const getSingleGlobalField = async ( + req: Request, + res: Response +): Promise => { const resp = await contentMapperService.getSingleGlobalField(req); res.status(201).json(resp); -} +}; -/** -* update content mapping details a project. -* -* @param req - The request object. -* @param res - The response object. -* @returns A Promise that resolves to void. -*/ -const updateContentMapper = async (req: Request, res: Response): Promise => { +/** + * update content mapping details a project. + * + * @param req - The request object. + * @param res - The response object. + * @returns A Promise that resolves to void. + */ +const updateContentMapper = async ( + req: Request, + res: Response +): Promise => { const project = await contentMapperService.updateContentMapper(req); res.status(project.status).json(project); - } +}; + +/** + * Retrieves existing taxonomies from both source and destination. + * + * @param {Request} req - The request object. + * @param {Response} res - The response object. + * @returns {Promise} - A promise that resolves when the operation is complete. + */ +const getExistingTaxonomies = async ( + req: Request, + res: Response +): Promise => { + const resp = await contentMapperService.getExistingTaxonomies(req); + res.status(resp?.status || 200).json(resp); +}; export const contentMapperController = { getContentTypes, @@ -158,8 +179,9 @@ export const contentMapperController = { resetContentType, // removeMapping, getSingleContentTypes, + getExistingTaxonomies, removeContentMapper, updateContentMapper, getExistingGlobalFields, - getSingleGlobalField + getSingleGlobalField, }; diff --git a/api/src/models/project-lowdb.ts b/api/src/models/project-lowdb.ts index bb58f37e3..2ea80b78c 100644 --- a/api/src/models/project-lowdb.ts +++ b/api/src/models/project-lowdb.ts @@ -1,6 +1,6 @@ import path from 'path'; -import { JSONFile } from "lowdb/node"; -import LowWithLodash from "../utils/lowdb-lodash.utils.js"; +import { JSONFile } from 'lowdb/node'; +import LowWithLodash from '../utils/lowdb-lodash.utils.js'; /** * Represents the LegacyCMS object. @@ -82,8 +82,9 @@ interface Project { mapperKeys: {}; extract_path: string; isMigrationStarted: boolean; - isMigrationCompleted:boolean; + isMigrationCompleted: boolean; migration_execution: boolean; + taxonomies?: any[]; // Taxonomies from source CMS } interface ProjectDocument { @@ -96,8 +97,10 @@ const defaultData: ProjectDocument = { projects: [] }; * Represents the database instance for the project. */ const db = new LowWithLodash( - new JSONFile(path.join(process.cwd(), "database", "project.json")), + new JSONFile( + path.join(process.cwd(), 'database', 'project.json') + ), defaultData ); -export default db; \ No newline at end of file +export default db; diff --git a/api/src/routes/contentMapper.routes.ts b/api/src/routes/contentMapper.routes.ts index d9fbfd8c1..f2464d6fb 100644 --- a/api/src/routes/contentMapper.routes.ts +++ b/api/src/routes/contentMapper.routes.ts @@ -1,6 +1,6 @@ -import express from "express"; -import { contentMapperController } from "../controllers/projects.contentMapper.controller.js"; -import { asyncRouter } from "../utils/async-router.utils.js"; +import express from 'express'; +import { contentMapperController } from '../controllers/projects.contentMapper.controller.js'; +import { asyncRouter } from '../utils/async-router.utils.js'; const router = express.Router({ mergeParams: true }); @@ -9,7 +9,7 @@ const router = express.Router({ mergeParams: true }); * @route POST /createDummyData/:projectId */ router.post( - "/createDummyData/:projectId", + '/createDummyData/:projectId', asyncRouter(contentMapperController.putTestData) ); @@ -18,7 +18,7 @@ router.post( * @route GET /contentTypes/:projectId/:skip/:limit/:searchText? */ router.get( - "/contentTypes/:projectId/:skip/:limit/:searchText?", + '/contentTypes/:projectId/:skip/:limit/:searchText?', asyncRouter(contentMapperController.getContentTypes) ); @@ -27,7 +27,7 @@ router.get( * @route GET /fieldMapping/:contentTypeId/:skip/:limit/:searchText? */ router.get( - "/fieldMapping/:projectId/:contentTypeId/:skip/:limit/:searchText?", + '/fieldMapping/:projectId/:contentTypeId/:skip/:limit/:searchText?', asyncRouter(contentMapperController.getFieldMapping) ); @@ -36,7 +36,7 @@ router.get( * @route GET /:projectId */ router.get( - "/:projectId/contentTypes/:contentTypeUid?", + '/:projectId/contentTypes/:contentTypeUid?', asyncRouter(contentMapperController.getExistingContentTypes) ); @@ -45,16 +45,25 @@ router.get( * @route GET /:projectId */ router.get( - "/:projectId/globalFields/:globalFieldUid?", + '/:projectId/globalFields/:globalFieldUid?', asyncRouter(contentMapperController.getExistingGlobalFields) ); +/** + * Get Existing Taxonomies from source and destination + * @route GET /:projectId/taxonomies + */ +router.get( + '/:projectId/taxonomies', + asyncRouter(contentMapperController.getExistingTaxonomies) +); + /** * Update FieldMapping or contentType * @route PUT /contentTypes/:orgId/:projectId/:contentTypeId */ router.put( - "/contentTypes/:orgId/:projectId/:contentTypeId", + '/contentTypes/:orgId/:projectId/:contentTypeId', asyncRouter(contentMapperController.putContentTypeFields) ); @@ -63,7 +72,7 @@ router.put( * @route PUT /resetFields/:orgId/:projectId/:contentTypeId */ router.put( - "/resetFields/:orgId/:projectId/:contentTypeId", + '/resetFields/:orgId/:projectId/:contentTypeId', asyncRouter(contentMapperController.resetContentType) ); @@ -81,7 +90,7 @@ router.put( * @route GET /:orgId/:projectId/content-mapper */ router.get( - "/:orgId/:projectId/content-mapper", + '/:orgId/:projectId/content-mapper', asyncRouter(contentMapperController.removeContentMapper) ); @@ -89,7 +98,10 @@ router.get( * Update content mapper * @route GET /:orgId/:projectId */ -router.patch("/:orgId/:projectId/mapper_keys", asyncRouter(contentMapperController.updateContentMapper)); +router.patch( + '/:orgId/:projectId/mapper_keys', + asyncRouter(contentMapperController.updateContentMapper) +); /** * Get Single Global Field data diff --git a/api/src/services/contentMapper.service.ts b/api/src/services/contentMapper.service.ts index 2e2a0b4df..551d8c919 100644 --- a/api/src/services/contentMapper.service.ts +++ b/api/src/services/contentMapper.service.ts @@ -1,4 +1,6 @@ import { Request } from 'express'; +import fs from 'fs'; +import path from 'path'; import { getLogMessage, isEmpty, safePromise } from '../utils/index.js'; import { BadRequestError, @@ -11,6 +13,7 @@ import { NEW_PROJECT_STATUS, CONTENT_TYPE_STATUS, VALIDATION_ERRORS, + MIGRATION_DATA_CONFIG, } from '../constants/index.js'; import logger from '../utils/logger.js'; import { config } from '../config/index.js'; @@ -76,12 +79,39 @@ const putTestData = async (req: Request) => { : uuidv4(); field.id = id; fieldIds.push(id); + + // Initialize referenceTo from advanced data (upload-api) + let referenceTo: string[] = []; + if (field?.backupFieldType === 'reference') { + // Reference fields use embedObjects OR reference_to + referenceTo = + field?.advanced?.embedObjects || + field?.advanced?.reference_to || + []; + console.log( + ` ๐Ÿ“ Initializing referenceTo for reference field "${field?.contentstackFieldUid}":`, + referenceTo + ); + } else if ( + field?.backupFieldType === 'taxonomy' && + field?.advanced?.taxonomies + ) { + referenceTo = field.advanced.taxonomies.map( + (t: any) => t.taxonomy_uid || t + ); + console.log( + ` ๐Ÿ“ Initializing referenceTo for taxonomy field "${field?.contentstackFieldUid}":`, + referenceTo + ); + } + return { id, projectId, contentTypeId: type?.id, isDeleted: false, ...field, + referenceTo, // Initialize referenceTo field }; }) : []; @@ -135,6 +165,25 @@ const putTestData = async (req: Request) => { ).mySQLDetails = req.body.mySQLDetails; } + // Store taxonomies if provided + if (req?.body?.taxonomies && Array.isArray(req.body.taxonomies)) { + ProjectModelLowdb.data.projects[index].taxonomies = req.body.taxonomies; + logger.info( + `โœ“ Stored ${req.body.taxonomies.length} taxonomies for project ${projectId}` + ); + console.log('๐Ÿ’พ Taxonomies stored in project database:', { + projectId, + count: req.body.taxonomies.length, + taxonomies: req.body.taxonomies, + }); + } else { + console.warn('โš ๏ธ No taxonomies provided in request body:', { + hasTaxonomies: !!req.body.taxonomies, + isArray: Array.isArray(req.body.taxonomies), + taxonomiesValue: req.body.taxonomies, + }); + } + await ProjectModelLowdb.write(); } else { throw new BadRequestError(HTTP_TEXTS.CONTENT_TYPE_NOT_FOUND); @@ -1278,6 +1327,167 @@ const updateContentMapper = async (req: Request) => { } }; +/** + * Retrieves existing taxonomies from the destination Contentstack stack + * and source taxonomy data from migration files. + * @param req - The request object containing the project ID and token payload. + * @returns An object containing source taxonomies and destination taxonomies. + */ +const getExistingTaxonomies = async (req: Request) => { + const projectId = req?.params?.projectId; + const { token_payload } = req.body; + + try { + // Get project details + await ProjectModelLowdb.read(); + const project = ProjectModelLowdb.chain + .get('projects') + .find({ id: projectId }) + .value(); + + if (!project) { + return { + data: 'Project not found', + status: 404, + }; + } + + const stackId = project?.destination_stack_id; + + // Step 1: Get source taxonomies from project database (sent by upload-api) + let sourceTaxonomies: any[] = []; + + if (project?.taxonomies && Array.isArray(project.taxonomies)) { + // Taxonomies stored in project database (sent from upload-api during validation) + sourceTaxonomies = project.taxonomies.map((taxonomy: any) => ({ + uid: taxonomy.uid, + name: taxonomy.name || taxonomy.uid, + description: taxonomy.description || '', + source: 'source_cms', + })); + logger.info( + `โœ“ Found ${sourceTaxonomies.length} source taxonomies in project database` + ); + console.log( + '๐Ÿ“Š Source Taxonomies from DB:', + JSON.stringify(sourceTaxonomies, null, 2) + ); + } else { + // Fallback: Try reading from migration-data files + logger.warn( + 'No taxonomies found in project database, checking fallback paths...' + ); + console.warn( + 'โš ๏ธ project.taxonomies is empty or not an array:', + project?.taxonomies + ); + + // Path 1: Check api/migration-data (processed taxonomies) + const apiMigrationDataPath = path.join( + MIGRATION_DATA_CONFIG.DATA, + stackId, + MIGRATION_DATA_CONFIG.TAXONOMIES_DIR_NAME, + MIGRATION_DATA_CONFIG.TAXONOMIES_FILE_NAME + ); + + try { + if (fs.existsSync(apiMigrationDataPath)) { + const taxonomiesData = await fs.promises.readFile( + apiMigrationDataPath, + 'utf8' + ); + const taxonomiesObject = JSON.parse(taxonomiesData); + + // Convert object to array with proper structure + const apiTaxonomies = Object.entries(taxonomiesObject).map( + ([uid, data]: [string, any]) => ({ + uid: data.uid || uid, + name: data.name || uid, + description: data.description || '', + source: 'source_cms', + }) + ); + sourceTaxonomies.push(...apiTaxonomies); + logger.info( + `โœ“ Found ${apiTaxonomies.length} taxonomies in migration-data (fallback)` + ); + } + } catch (fileError: any) { + logger.error( + `Error reading migration-data taxonomies: ${fileError.message}` + ); + } + } + + // Step 2: Get destination taxonomies from Contentstack (if stack exists) + let destinationTaxonomies: any[] = []; + + if (token_payload?.region && token_payload?.user_id && stackId) { + try { + const authtoken = await getAuthtoken( + token_payload.region, + token_payload.user_id + ); + + const baseUrl = `${config.CS_API[ + token_payload?.region as keyof typeof config.CS_API + ]!}/taxonomies`; + + const headers = { + api_key: stackId, + authtoken, + }; + + // Fetch taxonomies from Contentstack + const taxonomies = await fetchAllPaginatedData( + baseUrl, + headers, + 100, + 'getExistingTaxonomies', + 'taxonomies' + ); + + destinationTaxonomies = taxonomies.map((taxonomy: any) => ({ + uid: taxonomy.uid, + name: taxonomy.name, + description: taxonomy.description || '', + source: 'destination_stack', + })); + console.log( + '๐ŸŽฏ Destination Taxonomies from Contentstack:', + JSON.stringify(destinationTaxonomies, null, 2) + ); + } catch (apiError: any) { + logger.error( + `Error fetching destination taxonomies: ${apiError.message}` + ); + console.error('โŒ Failed to fetch destination taxonomies:', apiError); + } + } + + const response = { + sourceTaxonomies, + destinationTaxonomies, + status: 201, + }; + + console.log('๐Ÿ“ค Returning taxonomy response:', { + sourceTaxonomiesCount: sourceTaxonomies.length, + destinationTaxonomiesCount: destinationTaxonomies.length, + totalCount: sourceTaxonomies.length + destinationTaxonomies.length, + }); + + return response; + } catch (error: any) { + logger.error(`Error in getExistingTaxonomies: ${error.message}`); + console.error('โŒ Error in getExistingTaxonomies:', error); + return { + data: error.message, + status: error.status || 500, + }; + } +}; + export const contentMapperService = { putTestData, getContentTypes, @@ -1292,4 +1502,5 @@ export const contentMapperService = { updateContentMapper, getExistingGlobalFields, getSingleGlobalField, + getExistingTaxonomies, }; diff --git a/api/src/services/drupal/content-types.service.ts b/api/src/services/drupal/content-types.service.ts index 16be69cc7..ef32f653c 100644 --- a/api/src/services/drupal/content-types.service.ts +++ b/api/src/services/drupal/content-types.service.ts @@ -4,6 +4,8 @@ import { MIGRATION_DATA_CONFIG } from '../../constants/index.js'; import { convertToSchemaFormate } from '../../utils/content-type-creator.utils.js'; import { getLogMessage } from '../../utils/index.js'; import customLogger from '../../utils/custom-logger.utils.js'; +import FieldMapperModel from '../../models/FieldMapper.js'; +import ContentTypesMapperModelLowdb from '../../models/contentTypesMapper-lowdb.js'; const { DATA, CONTENT_TYPES_DIR_NAME, CONTENT_TYPES_SCHEMA_FILE } = MIGRATION_DATA_CONFIG; @@ -63,6 +65,18 @@ export const generateContentTypeSchemas = async ( ); } + // Load saved field mappings from database to get UI selections + await FieldMapperModel.read(); + await ContentTypesMapperModelLowdb.read(); + + const savedFieldMappings = FieldMapperModel.data.field_mapper.filter( + (field: any) => field && field.projectId === projectId + ); + + console.log( + `๐Ÿ“Š Loaded ${savedFieldMappings.length} saved field mappings with UI selections from database` + ); + // Build complete schema array (NO individual files) const allApiSchemas = []; @@ -76,8 +90,12 @@ export const generateContentTypeSchemas = async ( fs.readFileSync(uploadApiSchemaFilePath, 'utf8') ); - // Convert upload-api schema to API format - const apiSchema = convertUploadApiSchemaToApiSchema(uploadApiSchema); + // Convert upload-api schema to API format WITH saved field mappings from UI + const apiSchema = convertUploadApiSchemaToApiSchema( + uploadApiSchema, + savedFieldMappings, + projectId + ); // Add to combined schema array (NO individual files) allApiSchemas.push(apiSchema); @@ -151,8 +169,13 @@ export const generateContentTypeSchemas = async ( /** * Converts upload-api drupal schema format to API content type format * This preserves the original field types and user selections from the upload-api + * AND applies user's UI selections from Content Mapper for reference/taxonomy fields */ -function convertUploadApiSchemaToApiSchema(uploadApiSchema: any): any { +function convertUploadApiSchemaToApiSchema( + uploadApiSchema: any, + savedFieldMappings: any[] = [], + projectId?: string +): any { const apiSchema = { title: uploadApiSchema.title, uid: uploadApiSchema.uid, @@ -163,6 +186,10 @@ function convertUploadApiSchemaToApiSchema(uploadApiSchema: any): any { return apiSchema; } + console.log(`\n๐Ÿ”„ Converting content type: ${uploadApiSchema.uid}`); + console.log(` ๐Ÿ“‹ Fields in upload-api: ${uploadApiSchema.schema.length}`); + console.log(` ๐Ÿ’พ Saved mappings available: ${savedFieldMappings.length}`); + // Convert each field from upload-api format to API format for (const uploadField of uploadApiSchema.schema) { try { @@ -194,19 +221,112 @@ function convertUploadApiSchemaToApiSchema(uploadApiSchema: any): any { }); if (apiField) { - // Preserve additional metadata from upload-api - if ( - uploadField.contentstackFieldType === 'reference' && - uploadField.advanced?.reference_to - ) { - apiField.reference_to = uploadField.advanced.reference_to; + // Find saved field mapping from database (user's UI selections) + // Try multiple matching strategies to find the right field + const savedMapping = savedFieldMappings.find( + (mapping: any) => + mapping.contentstackFieldUid === uploadField.contentstackFieldUid || + mapping.contentstackFieldUid === uploadField.uid || + mapping.uid === uploadField.contentstackFieldUid || + mapping.uid === uploadField.uid + ); + + console.log( + ` ๐Ÿ” Checking field "${uploadField.contentstackFieldUid}" (${uploadField.contentstackFieldType})` + ); + console.log( + ` Upload-API data:`, + uploadField.advanced?.reference_to || + uploadField.advanced?.taxonomies || + 'none' + ); + console.log(` Saved mapping found:`, savedMapping ? 'YES' : 'NO'); + if (savedMapping) { + console.log(` Saved referenceTo:`, savedMapping.referenceTo); } - if ( - uploadField.contentstackFieldType === 'taxonomy' && - uploadField.advanced?.taxonomies - ) { - apiField.taxonomies = uploadField.advanced.taxonomies; + // Use UI selections if available, otherwise fall back to upload-api data + if (uploadField.contentstackFieldType === 'reference') { + if ( + savedMapping?.referenceTo && + Array.isArray(savedMapping.referenceTo) && + savedMapping.referenceTo.length > 0 + ) { + // MERGE: Combine old upload-api data with new UI selections (no duplicates) + // Check both embedObjects AND reference_to + const oldReferences = + uploadField.advanced?.embedObjects || + uploadField.advanced?.reference_to || + []; + const newReferences = savedMapping.referenceTo || []; + const mergedReferences = [ + ...new Set([...oldReferences, ...newReferences]), + ]; + + apiField.reference_to = mergedReferences; + console.log( + ` โœ… Reference field "${ + uploadField.contentstackFieldUid + }": MERGED [${mergedReferences.join(', ')}]` + ); + console.log( + ` (Old: [${oldReferences.join( + ', ' + )}] + New: [${newReferences.join(', ')}])` + ); + } else { + // Fall back to upload-api data only (check both embedObjects and reference_to) + const fallbackReferences = + uploadField.advanced?.embedObjects || + uploadField.advanced?.reference_to; + if (fallbackReferences) { + apiField.reference_to = fallbackReferences; + console.log( + ` โš ๏ธ Reference field "${uploadField.contentstackFieldUid}": Using upload-api data only (no UI selection)` + ); + } + } + } + + if (uploadField.contentstackFieldType === 'taxonomy') { + if ( + savedMapping?.referenceTo && + Array.isArray(savedMapping.referenceTo) && + savedMapping.referenceTo.length > 0 + ) { + // MERGE: Combine old upload-api taxonomies with new UI selections (no duplicates) + const oldTaxonomyUIDs = ( + uploadField.advanced?.taxonomies || [] + ).map((t: any) => t.taxonomy_uid || t); + const newTaxonomyUIDs = savedMapping.referenceTo || []; + const mergedTaxonomyUIDs = [ + ...new Set([...oldTaxonomyUIDs, ...newTaxonomyUIDs]), + ]; + + // Convert UIDs to taxonomy format + apiField.taxonomies = mergedTaxonomyUIDs.map((taxUid: string) => ({ + taxonomy_uid: taxUid, + mandatory: uploadField.advanced?.mandatory || false, + multiple: uploadField.advanced?.multiple !== false, // Default to true for taxonomies + non_localizable: uploadField.advanced?.non_localizable || false, + })); + console.log( + ` โœ… Taxonomy field "${ + uploadField.contentstackFieldUid + }": MERGED [${mergedTaxonomyUIDs.join(', ')}]` + ); + console.log( + ` (Old: [${oldTaxonomyUIDs.join( + ', ' + )}] + New: [${newTaxonomyUIDs.join(', ')}])` + ); + } else if (uploadField.advanced?.taxonomies) { + // Fall back to upload-api data only + apiField.taxonomies = uploadField.advanced.taxonomies; + console.log( + ` โš ๏ธ Taxonomy field "${uploadField.contentstackFieldUid}": Using upload-api data only (no UI selection)` + ); + } } // Preserve field metadata for proper field type conversion diff --git a/ui/src/components/AdvancePropertise/index.tsx b/ui/src/components/AdvancePropertise/index.tsx index a33a25540..b8cb1de22 100644 --- a/ui/src/components/AdvancePropertise/index.tsx +++ b/ui/src/components/AdvancePropertise/index.tsx @@ -12,10 +12,11 @@ import { Select, Radio, Button, + InstructionText, } from '@contentstack/venus-components'; // Service -import { getContentTypes } from '../../services/api/migration.service'; +import { getContentTypes, getExistingTaxonomies } from '../../services/api/migration.service'; // Utilities import { validateArray } from '../../utilities/functions'; @@ -32,6 +33,13 @@ interface ContentTypeOption { value: string; } +interface Taxonomy { + uid: string; + name: string; + description?: string; + source?: string; +} + /** * Component for displaying advanced properties. * @param props - The schema properties. @@ -79,6 +87,9 @@ const AdvancePropertise = (props: SchemaProps) => { props?.value?.embedObjects ); const [referencedCT, setReferencedCT] = useState(referencedItems || null); + const [sourceTaxonomies, setSourceTaxonomies] = useState([]); + const [destinationTaxonomies, setDestinationTaxonomies] = useState([]); + const [referencedTaxonomies, setReferencedTaxonomies] = useState(referencedItems || null); const [showOptions, setShowOptions] = useState>({}); const [showIcon, setShowIcon] = useState(); const filterRef = useRef(null); @@ -94,9 +105,96 @@ const AdvancePropertise = (props: SchemaProps) => { setShowIcon(defaultIndex); } }, []); + useEffect(() => { + console.info('๐Ÿ” AdvancePropertise Props Debug:', { + fieldtype: props?.fieldtype, + referenceTo: props?.data?.referenceTo, + 'advanced.reference_to': props?.data?.advanced?.reference_to, + 'advanced.taxonomies': props?.data?.advanced?.taxonomies, + advancedFull: props?.data?.advanced, + fullData: props?.data + }); + fetchContentTypes(''); + fetchTaxonomies(); }, []); + + // Update referenced CT when content types are fetched (only for Reference fields) + useEffect(() => { + if (props?.fieldtype === 'Reference' && contentTypes.length > 0) { + // Merge old (upload-api) and new (UI) selections + // Reference fields can use embedObjects OR reference_to + const oldReferences = props?.data?.advanced?.embedObjects || props?.data?.advanced?.reference_to || []; + const newReferences = props?.data?.referenceTo || []; + const allReferenceUIDs = Array.from(new Set([...oldReferences, ...newReferences])); + + console.info('๐Ÿ” Reference field - Loading selections:', { + 'advanced.embedObjects': props?.data?.advanced?.embedObjects, + 'advanced.reference_to': props?.data?.advanced?.reference_to, + oldFromUploadApi: oldReferences, + newFromUI: newReferences, + merged: allReferenceUIDs + }); + + if (allReferenceUIDs.length > 0) { + const matchedCTs = allReferenceUIDs + .map((uid: string) => { + const ct = contentTypes.find((c: ContentType) => c.contentstackUid === uid); + return ct ? { label: ct.contentstackTitle, value: ct.contentstackUid } : null; + }) + .filter(Boolean) as ContentTypeOption[]; + + console.info('โœ… Matched Referenced CTs (OLD + NEW):', { + fieldtype: props.fieldtype, + totalCount: matchedCTs.length, + matchedCTs + }); + + if (matchedCTs.length > 0) { + setReferencedCT(matchedCTs); + } + } + } + }, [contentTypes, props?.data?.referenceTo, props?.data?.advanced, props?.fieldtype]); + + // Update referenced taxonomies when taxonomies are fetched (only for Taxonomy fields) + useEffect(() => { + if (props?.fieldtype === 'Taxonomy' && (sourceTaxonomies.length > 0 || destinationTaxonomies.length > 0)) { + const allTaxonomies = [...sourceTaxonomies, ...destinationTaxonomies]; + + // Merge old (upload-api) and new (UI) selections + const oldTaxonomies = (props?.data?.advanced?.taxonomies || []).map((t: { taxonomy_uid?: string } | string) => (typeof t === 'string' ? t : t.taxonomy_uid || '')); + const newTaxonomies = props?.data?.referenceTo || []; + const allTaxonomyUIDs = Array.from(new Set([...oldTaxonomies, ...newTaxonomies])); + + console.info('๐Ÿ” Taxonomy field - Loading selections:', { + oldFromUploadApi: oldTaxonomies, + newFromUI: newTaxonomies, + merged: allTaxonomyUIDs + }); + + if (allTaxonomyUIDs.length > 0) { + const matchedTaxonomies = allTaxonomyUIDs + .map((uid: string) => { + const taxonomy = allTaxonomies.find((t: Taxonomy) => t.uid === uid); + return taxonomy ? { label: taxonomy.name || taxonomy.uid, value: taxonomy.uid } : null; + }) + .filter(Boolean) as ContentTypeOption[]; + + console.info('โœ… Matched Referenced Taxonomies (OLD + NEW):', { + fieldtype: props.fieldtype, + totalCount: matchedTaxonomies.length, + allTaxonomiesCount: allTaxonomies.length, + matchedTaxonomies + }); + + if (matchedTaxonomies.length > 0) { + setReferencedTaxonomies(matchedTaxonomies); + } + } + } + }, [sourceTaxonomies, destinationTaxonomies, props?.data?.referenceTo, props?.data?.advanced, props?.fieldtype]); /** * Fetches the content types list. * @param searchText - The search text. @@ -111,6 +209,35 @@ const AdvancePropertise = (props: SchemaProps) => { } }; + /** + * Fetches taxonomies from both source CMS and destination stack. + */ + const fetchTaxonomies = async () => { + try { + console.info('๐Ÿ”„ Fetching taxonomies for project:', props?.projectId); + const { data } = await getExistingTaxonomies(props?.projectId ?? ''); + + console.info('๐Ÿ“ฅ Received taxonomy data from API:', { + rawData: data, + sourceTaxonomiesCount: data?.sourceTaxonomies?.length || 0, + destinationTaxonomiesCount: data?.destinationTaxonomies?.length || 0, + sourceTaxonomies: data?.sourceTaxonomies, + destinationTaxonomies: data?.destinationTaxonomies + }); + + setSourceTaxonomies(data?.sourceTaxonomies || []); + setDestinationTaxonomies(data?.destinationTaxonomies || []); + + console.info('โœ… Taxonomies state updated:', { + sourceCount: data?.sourceTaxonomies?.length || 0, + destinationCount: data?.destinationTaxonomies?.length || 0 + }); + } catch (error) { + console.error('โŒ Error fetching taxonomies:', error); + return error; + } + }; + /** * Handles the change event for input fields. * @param field - The field name. @@ -588,6 +715,75 @@ const AdvancePropertise = (props: SchemaProps) => { )} + {props?.fieldtype === 'Taxonomy' && ( + + + Referenced Taxonomies + + { - console.info('๐ŸŽฏ Taxonomy selection changed:', selectedOptions); + setReferencedTaxonomies(selectedOptions); const taxonomyArray = selectedOptions?.map((item: optionsType) => item?.value); - console.info('๐Ÿ’พ Saving taxonomy references:', taxonomyArray); + props?.updateFieldSettings( props?.rowId, @@ -758,29 +726,27 @@ const AdvancePropertise = (props: SchemaProps) => { value: taxonomy?.uid })); - console.info('๐ŸŽจ Dropdown options generated:', { - totalOptions: dropdownOptions.length, - sourceCount: sourceTaxonomies?.length || 0, - destinationCount: destinationTaxonomies?.length || 0, - uniqueCount: uniqueTaxonomiesMap.size, - fullOptions: dropdownOptions - }); + return dropdownOptions; })() } - placeholder="Add Taxonomy(ies)" + placeholder={isTaxonomiesLoading ? "Loading..." : "Add Taxonomy(ies)"} version="v2" isSearchable={true} isClearable={true} width="350px" maxMenuHeight={200} /> - {sourceTaxonomies?.length === 0 && destinationTaxonomies?.length === 0 && ( + {isTaxonomiesLoading ? ( + + Loading taxonomies... + + ) : sourceTaxonomies?.length === 0 && destinationTaxonomies?.length === 0 ? ( No taxonomies found. Please upload source data or create taxonomies in your destination stack. - )} + ) : null} )} diff --git a/ui/src/components/DestinationStack/Actions/LoadLanguageMapper.tsx b/ui/src/components/DestinationStack/Actions/LoadLanguageMapper.tsx index 75f5c4a92..414286814 100644 --- a/ui/src/components/DestinationStack/Actions/LoadLanguageMapper.tsx +++ b/ui/src/components/DestinationStack/Actions/LoadLanguageMapper.tsx @@ -90,16 +90,10 @@ const Mapper = ({ const hasChanges = JSON.stringify(existingMapping) !== JSON.stringify(mergedMapping); if (hasChanges && Object.keys(selectedMappings).length > 0) { - console.info('๐Ÿ” DEBUG: Mapper updating localeMapping with selectedMappings:', selectedMappings); - console.info('๐Ÿ” DEBUG: Existing mapping:', existingMapping); - console.info('๐Ÿ” DEBUG: Merged mapping:', mergedMapping); // ๐Ÿ”ง CRITICAL CHECK: Don't override with empty values const hasEmptyValues = Object.values(selectedMappings).some(value => value === ''); if (hasEmptyValues) { - console.warn('โš ๏ธ WARNING: selectedMappings contains empty values, skipping update to preserve auto-mapping'); - console.warn(' Empty selectedMappings:', selectedMappings); - console.warn(' Preserving existing mapping:', existingMapping); return; } @@ -685,15 +679,11 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { .filter(key => !key.includes('-master_locale')) // Exclude master locale entries .filter(key => localeMapping[key] !== '' && localeMapping[key] !== null && localeMapping[key] !== undefined); // Only count valid mappings - console.info('๐Ÿ” DEBUG: Currently mapped source locales:', mappedSourceLocales); - console.info('๐Ÿ” DEBUG: Available source locales:', sourceLocales.map(s => s.value)); - // Find first unmapped source locale const unmappedLocale = sourceLocales.find(source => !mappedSourceLocales.includes(source.value) ); - console.info('๐Ÿ” DEBUG: Found unmapped locale:', unmappedLocale); return unmappedLocale || null; }; @@ -810,12 +800,6 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { ...(destinationLocale ? { [singleSourceLocale.value]: destinationLocale } : {}) }; - // ๐Ÿ” DEBUG: Log the mapping structure being created - console.info('๐Ÿ” DEBUG: Single locale autoMapping created:', JSON.stringify(autoMapping, null, 2)); - console.info('๐Ÿ” DEBUG: Source locale:', singleSourceLocale.value); - console.info('๐Ÿ” DEBUG: Destination locale:', destinationLocale); - console.info('๐Ÿ” DEBUG: Stack master locale:', stack?.master_locale); - const newMigrationDataObj: INewMigration = { ...newMigrationData, destination_stack: { @@ -825,11 +809,6 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { }; dispatch(updateNewMigrationData(newMigrationDataObj)); - // ๐Ÿ” DEBUG: Log after dispatch - console.info('๐Ÿ” DEBUG: Dispatched single locale mapping to Redux'); - - // ๐Ÿ” DEBUG: Auto-mapping completed for single locale - // Reset stack changed flag after auto-mapping if (isStackChanged) { setisStackChanged(false); @@ -872,15 +851,6 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { } } - // ๐Ÿ”ง CRITICAL CHANGE: For multi-locale, only map the FIRST/MASTER locale initially - // Other locales will be handled by the "Add Language" functionality - console.info('๐Ÿ” DEBUG: Multi-locale detected - only mapping master locale initially'); - console.info(' Master locale from source:', masterLocaleFromSource?.value); - console.info(' Other locales will be available for "Add Language"'); - - // Skip auto-mapping all other locales to keep "Add Language" button enabled - // The "Add Language" functionality will handle mapping additional locales one by one - // ๐Ÿ”ง TC-04 & TC-08: Enhanced no-match logic with master locale default if (!hasAnyMatches) { @@ -925,10 +895,8 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { // ๐Ÿ”ง REMOVED: TC-03 auto-mapping of remaining locales // This was causing all locales to be mapped at once, disabling "Add Language" button // Now remaining locales will be handled by "Add Language" functionality - console.info('๐Ÿ” DEBUG: Skipping auto-mapping of remaining locales to keep "Add Language" enabled'); const unmappedSources = sourceLocale.filter(source => !autoMapping[source.value]); - console.info(` Unmapped sources available for "Add Language": [${unmappedSources.map(s => s.value).join(', ')}]`); // Update Redux state with auto-mappings const newMigrationDataObj: INewMigration = { @@ -940,8 +908,6 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { }; dispatch(updateNewMigrationData(newMigrationDataObj)); - // ๐Ÿ” DEBUG: Auto-mapping completed for multi-locale - // ๐Ÿ”ฅ CRITICAL FIX: Update existingLocale state for dropdown display // The dropdown reads from existingLocale, not from Redux localeMapping const updatedExistingLocale: ExistingFieldType = {}; @@ -1075,16 +1041,13 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { // ๐Ÿ†• CONDITION 3: Intelligent Add Language functionality with auto-suggestions const addRowComp = () => { - console.info('๐Ÿ” DEBUG: Add Language button clicked'); setisStackChanged(false); // ๐Ÿ†• STEP 1: Get next unmapped source locale const nextUnmappedSource = getNextUnmappedSourceLocale(); - console.info('๐Ÿ” DEBUG: Next unmapped source locale:', nextUnmappedSource); if (!nextUnmappedSource) { // Fallback: No more unmapped source locales available - console.warn('โš ๏ธ No more unmapped source locales available for "Add Language"'); setcmsLocaleOptions((prevList: { label: string; value: string }[]) => [ ...prevList, { @@ -1098,18 +1061,9 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { // ๐Ÿ†• STEP 2: Check if source locale exists in destination const destinationMatch = findDestinationMatch(nextUnmappedSource.value); - console.info('๐Ÿ” DEBUG: Destination match found:', destinationMatch); // ๐Ÿ†• STEP 3 & 4: Auto-map if exists, leave empty if not const newRowValue = destinationMatch ? destinationMatch.value : ''; - console.info(`๐Ÿ” DEBUG: Creating new row - Source: "${nextUnmappedSource.value}", Destination: "${newRowValue || 'empty'}"`); - - if (destinationMatch) { - console.info(`โœ… Auto-mapping: ${nextUnmappedSource.value} -> ${destinationMatch.value}`); - } else { - console.info(`โš ๏ธ No destination match for "${nextUnmappedSource.value}" - leaving destination empty for manual selection`); - } - // Add new row with intelligent defaults setcmsLocaleOptions((prevList: { label: string; value: string }[]) => [ @@ -1222,13 +1176,6 @@ const LanguageMapper = ({stack, uid} :{ stack : IDropDown, uid : string}) => { localeMapping[key] !== undefined // Exclude undefined mappings ).length; - console.info('๐Ÿ” DEBUG: Add Language Button Logic:', { - totalSourceLocales, - actualMappedSourceLocales, - visibleRowsCount, - localeMapping - }); - // Disable if: all source locales are mapped OR all source locales have visible rows OR project is completed const shouldDisable = actualMappedSourceLocales >= totalSourceLocales || visibleRowsCount >= totalSourceLocales || diff --git a/ui/src/components/LegacyCms/Actions/LoadSelectCms.tsx b/ui/src/components/LegacyCms/Actions/LoadSelectCms.tsx index 790ce3fd9..6d8338052 100644 --- a/ui/src/components/LegacyCms/Actions/LoadSelectCms.tsx +++ b/ui/src/components/LegacyCms/Actions/LoadSelectCms.tsx @@ -83,11 +83,6 @@ const LoadSelectCms = (props: LoadSelectCmsProps) => { setIsLoading(true); const { data } = await getConfig(); // api call to get cms type from upload service - - console.info('๐Ÿ” CONFIG API Response:', data); - console.info('๐Ÿ” data.mysql:', data?.mysql); - console.info('๐Ÿ” data.assetsConfig:', data?.assetsConfig); - // Store config details to display in UI setConfigDetails({ mySQLDetails: data?.mysql, @@ -129,11 +124,7 @@ const LoadSelectCms = (props: LoadSelectCmsProps) => { setCmsData([]); } } - - // Store config data (mysql, assetsConfig) in Redux for later use - // First, log what's currently in Redux - console.info('๐Ÿ” BEFORE UPDATE - Current file_details in Redux:', newMigrationData?.legacy_cms?.uploadedFile?.file_details); - + // Determine which CMS to set as selected let newSelectedCard: ICMSType | undefined; if (filteredCmsData?.length === 1) { @@ -163,14 +154,9 @@ const LoadSelectCms = (props: LoadSelectCmsProps) => { } }; - console.info('๐Ÿ” Updated newMigrationDataObj with config:', newMigrationDataObj); - console.info('๐Ÿ” file_details after mapping:', newMigrationDataObj?.legacy_cms?.uploadedFile?.file_details); - console.info('๐Ÿ” mySQLDetails after mapping:', newMigrationDataObj?.legacy_cms?.uploadedFile?.file_details?.mySQLDetails); dispatch(updateNewMigrationData(newMigrationDataObj)); // Dispatch to save config to Redux - console.info('โœ… DISPATCHED to Redux - newMigrationDataObj dispatched'); - // setCmsData(filteredCmsData); //Normal Search diff --git a/ui/src/components/LegacyCms/index.tsx b/ui/src/components/LegacyCms/index.tsx index b61cf1557..90e425fb4 100644 --- a/ui/src/components/LegacyCms/index.tsx +++ b/ui/src/components/LegacyCms/index.tsx @@ -211,37 +211,17 @@ const LegacyCMSComponent = forwardRef(({ legacyCMSData, isCompleted, handleOnAll },[newMigrationData]); useEffect(()=>{ - // Debug logging for completion check - console.info('๐Ÿ” === LEGACY CMS COMPLETION CHECK ==='); - console.info('๐Ÿ“Š Current state:', { - affix: newMigrationData?.legacy_cms?.affix, - selectedFileFormat: newMigrationData?.legacy_cms?.selectedFileFormat, - selectedCms: newMigrationData?.legacy_cms?.selectedCms, - uploadedFile: newMigrationData?.legacy_cms?.uploadedFile, - isValidated: newMigrationData?.legacy_cms?.uploadedFile?.isValidated - }); - - console.info('โœ… Individual conditions:'); - console.info(' - affix exists:', !isEmptyString(newMigrationData?.legacy_cms?.affix)); - console.info(' - selectedFileFormat.title exists:', !isEmptyString(newMigrationData?.legacy_cms?.selectedFileFormat?.title)); - console.info(' - selectedCms.title exists:', !isEmptyString(newMigrationData?.legacy_cms?.selectedCms?.title)); - console.info(' - uploadedFile.isValidated:', newMigrationData?.legacy_cms?.uploadedFile?.isValidated); - const allConditionsMet = !isEmptyString(newMigrationData?.legacy_cms?.affix) && !isEmptyString(newMigrationData?.legacy_cms?.selectedFileFormat?.title) && ! isEmptyString(newMigrationData?.legacy_cms?.selectedCms?.title) && newMigrationData?.legacy_cms?.uploadedFile?.isValidated; - console.info('๐ŸŽฏ All conditions met:', allConditionsMet); - console.info('====================================='); if(allConditionsMet){ - console.info('โœ… Setting all steps completed to TRUE'); setIsAllStepsCompleted(true); handleAllStepsComplete(true); } else{ - console.info('โŒ Setting all steps completed to FALSE'); setIsAllStepsCompleted(false); handleAllStepsComplete(false); } diff --git a/ui/src/components/MigrationFlowHeader/index.tsx b/ui/src/components/MigrationFlowHeader/index.tsx index f245212d0..e7f697fa3 100644 --- a/ui/src/components/MigrationFlowHeader/index.tsx +++ b/ui/src/components/MigrationFlowHeader/index.tsx @@ -65,7 +65,6 @@ const MigrationFlowHeader = ({ // Only navigate if there's a mismatch and we have valid data if (urlStep !== dbStep && dbStep && params?.projectId) { - console.info(`๐Ÿ”„ Step mismatch detected: URL shows step ${urlStep}, DB shows step ${dbStep}. Redirecting...`); const url = `/projects/${params?.projectId}/migration/steps/${dbStep}`; navigate(url, { replace: true }); } diff --git a/ui/src/pages/Migration/index.tsx b/ui/src/pages/Migration/index.tsx index 5228451d7..eed87c5ee 100644 --- a/ui/src/pages/Migration/index.tsx +++ b/ui/src/pages/Migration/index.tsx @@ -137,7 +137,6 @@ const Migration = () => { value !== null && value !== undefined ); - //console.info("legacyCMSRef?.current ", legacyCMSRef?.current,legacyCMSRef?.current?.getInternalActiveStepIndex()) if(legacyCMSRef?.current && newMigrationData?.project_current_step === 1 && legacyCMSRef?.current?.getInternalActiveStepIndex() > -1){ setIsSaved(true); } @@ -522,10 +521,6 @@ const Migration = () => { mySQLDetails: newMigrationData?.legacy_cms?.uploadedFile?.file_details?.mySQLDetails }; - console.info('๐Ÿ” === FILE FORMAT API CALL DEBUG ==='); - console.info('๐Ÿ“‹ Sending fileFormatData:', fileFormatData); - console.info('๐Ÿ“‹ mySQLDetails:', newMigrationData?.legacy_cms?.uploadedFile?.file_details?.mySQLDetails); - console.info('====================================='); await updateFileFormatData(selectedOrganisation?.value, projectId, fileFormatData); @@ -533,14 +528,12 @@ const Migration = () => { const currentProjectData = await getMigrationData(selectedOrganisation?.value, projectId); const currentStep = currentProjectData?.data?.current_step; - console.info(`๐Ÿ” Current project step: ${currentStep}, attempting to proceed...`); // Only call updateCurrentStepData if we're at step 1 (to avoid 400 error) let res; if (currentStep === 1) { res = await updateCurrentStepData(selectedOrganisation.value, projectId); } else { - console.info(`โš ๏ธ Project already at step ${currentStep}, skipping step update`); res = { status: 200 }; // Simulate success to continue flow } @@ -601,8 +594,6 @@ const Migration = () => { const handleOnClickDestinationStack = async (event: MouseEvent) => { setIsLoading(true); - // ๐Ÿ” DEBUG: Log the actual localeMapping being validated - console.info('๐Ÿ” DEBUG: About to validate localeMapping:', JSON.stringify(newMigrationData?.destination_stack?.localeMapping, null, 2)); const hasNonEmptyMapping = newMigrationData?.destination_stack?.localeMapping && @@ -617,8 +608,6 @@ const Migration = () => { labelNotNumeric: isNaN(Number(label)) }; - console.info(`๐Ÿ” DEBUG: Validating entry [${label}] = "${value}":`, conditions); - const passes = conditions.hasLabel && conditions.notEmptyValue && conditions.notNullValue && @@ -626,12 +615,9 @@ const Migration = () => { conditions.labelNotUndefined && conditions.labelNotNumeric; - console.info(`๐Ÿ” DEBUG: Entry result: ${passes ? 'โœ… PASS' : 'โŒ FAIL'}`); return passes; } ); - - console.info('๐Ÿ” DEBUG: Final hasNonEmptyMapping result:', hasNonEmptyMapping); const master_locale: LocalesType = {}; const locales: LocalesType = {}; From 1a1226cc0016fd851c3823fbcc9243e8bcffc74d Mon Sep 17 00:00:00 2001 From: sauravraw Date: Tue, 14 Oct 2025 12:13:20 +0530 Subject: [PATCH 13/37] Refactor ContentMapper and LogScreen components for improved performance and readability - Added useCallback to handleSelectedEntries in ContentMapper for better performance. - Updated pluralization logic in ContentMapper for cleaner code. - Enhanced log parsing in LogScreen components to skip empty lines and silently handle malformed entries, improving robustness. --- ui/src/components/ContentMapper/index.tsx | 8 ++++---- ui/src/components/LogScreen/MigrationLogViewer.tsx | 8 +++++++- ui/src/components/LogScreen/index.tsx | 8 +++++++- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/ui/src/components/ContentMapper/index.tsx b/ui/src/components/ContentMapper/index.tsx index f073bf929..297de4df4 100644 --- a/ui/src/components/ContentMapper/index.tsx +++ b/ui/src/components/ContentMapper/index.tsx @@ -1,5 +1,5 @@ // Libraries -import { useEffect, useState, useRef, useImperativeHandle, forwardRef } from 'react'; +import { useEffect, useState, useRef, useImperativeHandle, forwardRef, useCallback } from 'react'; import { useDispatch, useSelector } from 'react-redux'; import { useNavigate, useParams } from 'react-router-dom'; import { @@ -1128,7 +1128,7 @@ const ContentMapper = forwardRef(({ handleStepChange }: contentMapperProps, ref: * @param singleSelectedRowIds - The single selected row IDs * @returns void */ - const handleSelectedEntries = (singleSelectedRowIds: string[]) => { + const handleSelectedEntries = useCallback((singleSelectedRowIds: string[]) => { const selectedObj: UidMap = {}; const previousRowIds: UidMap = { ...rowIds as UidMap }; @@ -1220,7 +1220,7 @@ const ContentMapper = forwardRef(({ handleStepChange }: contentMapperProps, ref: setRowIds(selectedObj); setSelectedEntries(updatedTableData); - }; + }, [rowIds, selectedEntries, tableData]); // Method for change select value const handleValueChange = (value: FieldTypes, rowIndex: string, rowContentstackFieldUid: string) => { @@ -2659,7 +2659,7 @@ const ContentMapper = forwardRef(({ handleStepChange }: contentMapperProps, ref: rowSelectCheckboxProp={{ key: '_canSelect', value: true }} name={{ singular: '', - plural: `${totalCounts === 0 ? 'Count' : ''}` + plural: totalCounts === 0 ? 'Count' : '' }} />
diff --git a/ui/src/components/LogScreen/MigrationLogViewer.tsx b/ui/src/components/LogScreen/MigrationLogViewer.tsx index 4ca2acc5c..d8224c74c 100644 --- a/ui/src/components/LogScreen/MigrationLogViewer.tsx +++ b/ui/src/components/LogScreen/MigrationLogViewer.tsx @@ -91,6 +91,11 @@ const MigrationLogViewer = ({ serverPath }: LogsType) => { const logArray = newLogs?.split('\n'); logArray?.forEach((logLine) => { + // Skip empty or whitespace-only lines + if (!logLine || !logLine.trim()) { + return; + } + try { //parse each log entry as a JSON object const parsedLog = JSON?.parse(logLine); @@ -103,7 +108,8 @@ const MigrationLogViewer = ({ serverPath }: LogsType) => { }; parsedLogsArray.push(plogs); } catch (error) { - console.error('error in parsing logs : ', error); + // Silently skip malformed log entries + // console.error('error in parsing logs : ', error); } }); diff --git a/ui/src/components/LogScreen/index.tsx b/ui/src/components/LogScreen/index.tsx index 8dd06f900..4cfcb6b20 100644 --- a/ui/src/components/LogScreen/index.tsx +++ b/ui/src/components/LogScreen/index.tsx @@ -88,6 +88,11 @@ const TestMigrationLogViewer = ({ serverPath, sendDataToParent, projectId }: Log const logArray = newLogs?.split('\n'); logArray?.forEach((logLine) => { + // Skip empty or whitespace-only lines + if (!logLine || !logLine.trim()) { + return; + } + try { // parse each log entry as a JSON object const parsedLog = JSON.parse(logLine); @@ -99,7 +104,8 @@ const TestMigrationLogViewer = ({ serverPath, sendDataToParent, projectId }: Log }; parsedLogsArray.push(plogs); } catch (error) { - console.error('error in parsing logs : ', error); + // Silently skip malformed log entries + // console.error('error in parsing logs : ', error); } }); setLogs((prevLogs) => [ From 0a1db74694f0ea66f7464fde0c63f38ba310e37a Mon Sep 17 00:00:00 2001 From: sauravraw Date: Tue, 14 Oct 2025 12:48:38 +0530 Subject: [PATCH 14/37] Refactor logging and improve field type handling in contentMapper service and UI - Removed unnecessary console logs from contentMapper.service.ts to enhance performance and readability. - Updated field type handling in content-types.service.ts to prioritize user-selected types over defaults. - Added logging for field type changes in the ContentMapper component to track modifications sent to the backend. - Cleaned up code by eliminating redundant logging statements across various functions. --- api/src/services/contentMapper.service.ts | 41 -------- .../services/drupal/content-types.service.ts | 99 +++++-------------- ui/src/components/ContentMapper/index.tsx | 11 +++ 3 files changed, 38 insertions(+), 113 deletions(-) diff --git a/api/src/services/contentMapper.service.ts b/api/src/services/contentMapper.service.ts index 551d8c919..85d33067a 100644 --- a/api/src/services/contentMapper.service.ts +++ b/api/src/services/contentMapper.service.ts @@ -88,10 +88,6 @@ const putTestData = async (req: Request) => { field?.advanced?.embedObjects || field?.advanced?.reference_to || []; - console.log( - ` ๐Ÿ“ Initializing referenceTo for reference field "${field?.contentstackFieldUid}":`, - referenceTo - ); } else if ( field?.backupFieldType === 'taxonomy' && field?.advanced?.taxonomies @@ -99,10 +95,6 @@ const putTestData = async (req: Request) => { referenceTo = field.advanced.taxonomies.map( (t: any) => t.taxonomy_uid || t ); - console.log( - ` ๐Ÿ“ Initializing referenceTo for taxonomy field "${field?.contentstackFieldUid}":`, - referenceTo - ); } return { @@ -171,17 +163,7 @@ const putTestData = async (req: Request) => { logger.info( `โœ“ Stored ${req.body.taxonomies.length} taxonomies for project ${projectId}` ); - console.log('๐Ÿ’พ Taxonomies stored in project database:', { - projectId, - count: req.body.taxonomies.length, - taxonomies: req.body.taxonomies, - }); } else { - console.warn('โš ๏ธ No taxonomies provided in request body:', { - hasTaxonomies: !!req.body.taxonomies, - isArray: Array.isArray(req.body.taxonomies), - taxonomiesValue: req.body.taxonomies, - }); } await ProjectModelLowdb.write(); @@ -1368,19 +1350,11 @@ const getExistingTaxonomies = async (req: Request) => { logger.info( `โœ“ Found ${sourceTaxonomies.length} source taxonomies in project database` ); - console.log( - '๐Ÿ“Š Source Taxonomies from DB:', - JSON.stringify(sourceTaxonomies, null, 2) - ); } else { // Fallback: Try reading from migration-data files logger.warn( 'No taxonomies found in project database, checking fallback paths...' ); - console.warn( - 'โš ๏ธ project.taxonomies is empty or not an array:', - project?.taxonomies - ); // Path 1: Check api/migration-data (processed taxonomies) const apiMigrationDataPath = path.join( @@ -1408,9 +1382,6 @@ const getExistingTaxonomies = async (req: Request) => { }) ); sourceTaxonomies.push(...apiTaxonomies); - logger.info( - `โœ“ Found ${apiTaxonomies.length} taxonomies in migration-data (fallback)` - ); } } catch (fileError: any) { logger.error( @@ -1453,15 +1424,10 @@ const getExistingTaxonomies = async (req: Request) => { description: taxonomy.description || '', source: 'destination_stack', })); - console.log( - '๐ŸŽฏ Destination Taxonomies from Contentstack:', - JSON.stringify(destinationTaxonomies, null, 2) - ); } catch (apiError: any) { logger.error( `Error fetching destination taxonomies: ${apiError.message}` ); - console.error('โŒ Failed to fetch destination taxonomies:', apiError); } } @@ -1471,16 +1437,9 @@ const getExistingTaxonomies = async (req: Request) => { status: 201, }; - console.log('๐Ÿ“ค Returning taxonomy response:', { - sourceTaxonomiesCount: sourceTaxonomies.length, - destinationTaxonomiesCount: destinationTaxonomies.length, - totalCount: sourceTaxonomies.length + destinationTaxonomies.length, - }); - return response; } catch (error: any) { logger.error(`Error in getExistingTaxonomies: ${error.message}`); - console.error('โŒ Error in getExistingTaxonomies:', error); return { data: error.message, status: error.status || 500, diff --git a/api/src/services/drupal/content-types.service.ts b/api/src/services/drupal/content-types.service.ts index ef32f653c..17291e715 100644 --- a/api/src/services/drupal/content-types.service.ts +++ b/api/src/services/drupal/content-types.service.ts @@ -73,10 +73,17 @@ export const generateContentTypeSchemas = async ( (field: any) => field && field.projectId === projectId ); - console.log( - `๐Ÿ“Š Loaded ${savedFieldMappings.length} saved field mappings with UI selections from database` + // Log fields with UI changes + const fieldsWithTypeChanges = savedFieldMappings.filter( + (field: any) => + field.contentstackFieldType && + field.backupFieldType !== field.contentstackFieldType ); + if (fieldsWithTypeChanges.length > 0) { + fieldsWithTypeChanges.forEach((field: any) => {}); + } + // Build complete schema array (NO individual files) const allApiSchemas = []; @@ -140,20 +147,12 @@ export const generateContentTypeSchemas = async ( 'utf8' ); - console.log( - `โœ… Generated schema.json with ${allApiSchemas.length} content types (NO individual files)` - ); - const successMessage = getLogMessage( srcFunc, `Successfully generated ${schemaFiles.length} content type schemas from upload-api`, {} ); await customLogger(projectId, destination_stack_id, 'info', successMessage); - - console.log( - `๐ŸŽ‰ Successfully converted ${schemaFiles.length} content type schemas` - ); } catch (error: any) { const errorMessage = getLogMessage( srcFunc, @@ -186,19 +185,29 @@ function convertUploadApiSchemaToApiSchema( return apiSchema; } - console.log(`\n๐Ÿ”„ Converting content type: ${uploadApiSchema.uid}`); - console.log(` ๐Ÿ“‹ Fields in upload-api: ${uploadApiSchema.schema.length}`); - console.log(` ๐Ÿ’พ Saved mappings available: ${savedFieldMappings.length}`); - // Convert each field from upload-api format to API format for (const uploadField of uploadApiSchema.schema) { try { + // Find saved field mapping from database FIRST to get user's field type selection + const savedMapping = savedFieldMappings.find( + (mapping: any) => + mapping.contentstackFieldUid === uploadField.contentstackFieldUid || + mapping.contentstackFieldUid === uploadField.uid || + mapping.uid === uploadField.contentstackFieldUid || + mapping.uid === uploadField.uid + ); + + // Use UI-selected field type if available, otherwise use upload-api type + const fieldType = + savedMapping?.contentstackFieldType || + uploadField.contentstackFieldType; + // Map upload-api field to API format using convertToSchemaFormate const apiField = convertToSchemaFormate({ field: { title: uploadField.contentstackField || uploadField.otherCmsField, uid: uploadField.contentstackFieldUid, - contentstackFieldType: uploadField.contentstackFieldType, + contentstackFieldType: fieldType, // Use UI selection if available advanced: { ...uploadField.advanced, mandatory: uploadField.advanced?.mandatory || false, @@ -221,32 +230,9 @@ function convertUploadApiSchemaToApiSchema( }); if (apiField) { - // Find saved field mapping from database (user's UI selections) - // Try multiple matching strategies to find the right field - const savedMapping = savedFieldMappings.find( - (mapping: any) => - mapping.contentstackFieldUid === uploadField.contentstackFieldUid || - mapping.contentstackFieldUid === uploadField.uid || - mapping.uid === uploadField.contentstackFieldUid || - mapping.uid === uploadField.uid - ); - - console.log( - ` ๐Ÿ” Checking field "${uploadField.contentstackFieldUid}" (${uploadField.contentstackFieldType})` - ); - console.log( - ` Upload-API data:`, - uploadField.advanced?.reference_to || - uploadField.advanced?.taxonomies || - 'none' - ); - console.log(` Saved mapping found:`, savedMapping ? 'YES' : 'NO'); - if (savedMapping) { - console.log(` Saved referenceTo:`, savedMapping.referenceTo); - } - // Use UI selections if available, otherwise fall back to upload-api data - if (uploadField.contentstackFieldType === 'reference') { + // Check against the FINAL field type (which may have been changed in UI) + if (fieldType === 'reference') { if ( savedMapping?.referenceTo && Array.isArray(savedMapping.referenceTo) && @@ -264,16 +250,6 @@ function convertUploadApiSchemaToApiSchema( ]; apiField.reference_to = mergedReferences; - console.log( - ` โœ… Reference field "${ - uploadField.contentstackFieldUid - }": MERGED [${mergedReferences.join(', ')}]` - ); - console.log( - ` (Old: [${oldReferences.join( - ', ' - )}] + New: [${newReferences.join(', ')}])` - ); } else { // Fall back to upload-api data only (check both embedObjects and reference_to) const fallbackReferences = @@ -281,14 +257,11 @@ function convertUploadApiSchemaToApiSchema( uploadField.advanced?.reference_to; if (fallbackReferences) { apiField.reference_to = fallbackReferences; - console.log( - ` โš ๏ธ Reference field "${uploadField.contentstackFieldUid}": Using upload-api data only (no UI selection)` - ); } } } - if (uploadField.contentstackFieldType === 'taxonomy') { + if (fieldType === 'taxonomy') { if ( savedMapping?.referenceTo && Array.isArray(savedMapping.referenceTo) && @@ -310,22 +283,9 @@ function convertUploadApiSchemaToApiSchema( multiple: uploadField.advanced?.multiple !== false, // Default to true for taxonomies non_localizable: uploadField.advanced?.non_localizable || false, })); - console.log( - ` โœ… Taxonomy field "${ - uploadField.contentstackFieldUid - }": MERGED [${mergedTaxonomyUIDs.join(', ')}]` - ); - console.log( - ` (Old: [${oldTaxonomyUIDs.join( - ', ' - )}] + New: [${newTaxonomyUIDs.join(', ')}])` - ); } else if (uploadField.advanced?.taxonomies) { // Fall back to upload-api data only apiField.taxonomies = uploadField.advanced.taxonomies; - console.log( - ` โš ๏ธ Taxonomy field "${uploadField.contentstackFieldUid}": Using upload-api data only (no UI selection)` - ); } } @@ -338,11 +298,6 @@ function convertUploadApiSchemaToApiSchema( apiSchema.schema.push(apiField); } } catch (error: any) { - console.warn( - `Failed to convert field ${uploadField.uid}:`, - error.message - ); - // Fallback: create basic field structure apiSchema.schema.push({ display_name: diff --git a/ui/src/components/ContentMapper/index.tsx b/ui/src/components/ContentMapper/index.tsx index 297de4df4..3335f4cec 100644 --- a/ui/src/components/ContentMapper/index.tsx +++ b/ui/src/components/ContentMapper/index.tsx @@ -1224,6 +1224,11 @@ const ContentMapper = forwardRef(({ handleStepChange }: contentMapperProps, ref: // Method for change select value const handleValueChange = (value: FieldTypes, rowIndex: string, rowContentstackFieldUid: string) => { + // Find the field being changed + const changedField = selectedEntries?.find?.( + (row) => row?.uid === rowIndex && row?.contentstackFieldUid === rowContentstackFieldUid + ); + setIsDropDownChanged(true); setFieldValue(value); const updatedRows: FieldMapType[] = selectedEntries?.map?.((row) => { @@ -1905,6 +1910,12 @@ const ContentMapper = forwardRef(({ handleStepChange }: contentMapperProps, ref: } }; + // Log field type changes being sent to backend + const changedFields = selectedEntries?.filter(field => { + const original = tableData?.find(t => t.uid === field.uid); + return original && original.contentstackFieldType !== field.contentstackFieldType; + }); + try { const { data } = await updateContentType( orgId, From fdc5bc3eb84607c49329a45df87278995cfa4a75 Mon Sep 17 00:00:00 2001 From: sauravraw Date: Tue, 14 Oct 2025 13:19:12 +0530 Subject: [PATCH 15/37] Enhance field type mapping logic in entries.service.ts - Introduced a priority system for field type mapping, first checking the generated content type schema for user-selected field types. - Improved error handling when loading content type schemas, ensuring fallback mechanisms are in place. - Cleaned up redundant code related to field type inference, streamlining the process for better performance and readability. --- api/src/services/drupal/entries.service.ts | 123 ++++++++++----------- 1 file changed, 60 insertions(+), 63 deletions(-) diff --git a/api/src/services/drupal/entries.service.ts b/api/src/services/drupal/entries.service.ts index 0aa55ae69..180da878b 100644 --- a/api/src/services/drupal/entries.service.ts +++ b/api/src/services/drupal/entries.service.ts @@ -1252,8 +1252,67 @@ const processEntries = async ( for (const [fieldName, fieldValue] of Object.entries(processedEntry)) { let fieldMapping = null; - // First try to find mapping from UI content type mapping + // PRIORITY 1: Read from generated content type schema (has UI-selected field types) + // This is checked FIRST because it contains the final field types after user's UI changes + // Load the content type schema to get user's field type selections + try { + const contentTypeSchemaPath = path.join( + MIGRATION_DATA_CONFIG.DATA, + destination_stack_id, + 'content_types', + `${contentType}.json` + ); + const contentTypeSchema = JSON.parse( + await fs.promises.readFile(contentTypeSchemaPath, 'utf8') + ); + + // Find field in schema + const schemaField = contentTypeSchema.schema?.find( + (field: any) => + field.uid === fieldName || + field.uid === fieldName.replace(/_target_id$/, '') || + field.uid === fieldName.replace(/_value$/, '') || + fieldName.includes(field.uid) + ); + + if (schemaField) { + // Determine the proper field type based on schema configuration + let targetFieldType = schemaField.data_type; + + // Handle HTML RTE fields (text with allow_rich_text: true) + if ( + schemaField.data_type === 'text' && + schemaField.field_metadata?.allow_rich_text === true + ) { + targetFieldType = 'html'; // โœ… HTML RTE field + } + // Handle JSON RTE fields + else if (schemaField.data_type === 'json') { + targetFieldType = 'json'; // โœ… JSON RTE field + } + // Handle text fields with multiline metadata + else if ( + schemaField.data_type === 'text' && + schemaField.field_metadata?.multiline + ) { + targetFieldType = 'multi_line_text'; // โœ… Multi-line text field + } + + // Create a mapping from schema field + fieldMapping = { + uid: fieldName, + contentstackFieldType: targetFieldType, + backupFieldType: schemaField.data_type, + advanced: schemaField, + }; + } + } catch (error: any) { + // Schema not found, will try fallback below + } + + // FALLBACK: If schema not found, try UI content type mapping if ( + !fieldMapping && currentContentTypeMapping && currentContentTypeMapping.fieldMapping ) { @@ -1266,68 +1325,6 @@ const processEntries = async ( ); } - // If no UI mapping found, try to infer from content type schema - if (!fieldMapping) { - // Load the content type schema to get user's field type selections - try { - const contentTypeSchemaPath = path.join( - MIGRATION_DATA_CONFIG.DATA, - destination_stack_id, - 'content_types', - `${contentType}.json` - ); - const contentTypeSchema = JSON.parse( - await fs.promises.readFile(contentTypeSchemaPath, 'utf8') - ); - - // Find field in schema - const schemaField = contentTypeSchema.schema?.find( - (field: any) => - field.uid === fieldName || - field.uid === fieldName.replace(/_target_id$/, '') || - field.uid === fieldName.replace(/_value$/, '') || - fieldName.includes(field.uid) - ); - - if (schemaField) { - // Determine the proper field type based on schema configuration - let targetFieldType = schemaField.data_type; - - // Handle HTML RTE fields (text with allow_rich_text: true) - if ( - schemaField.data_type === 'text' && - schemaField.field_metadata?.allow_rich_text === true - ) { - targetFieldType = 'html'; // โœ… HTML RTE field - } - // Handle JSON RTE fields - else if (schemaField.data_type === 'json') { - targetFieldType = 'json'; // โœ… JSON RTE field - } - // Handle text fields with multiline metadata - else if ( - schemaField.data_type === 'text' && - schemaField.field_metadata?.multiline - ) { - targetFieldType = 'multi_line_text'; // โœ… Multi-line text field - } - - // Create a mapping from schema field - fieldMapping = { - uid: fieldName, - contentstackFieldType: targetFieldType, - backupFieldType: schemaField.data_type, - advanced: schemaField, - }; - } - } catch (error: any) { - console.error( - `Failed to load content type schema for field ${fieldName}:`, - error.message - ); - } - } - if (fieldMapping) { // Apply field type processing based on user's selection const processedValue = processFieldByType( From 5fe0ac6aeabd94d99becb09d14c16aff866f0de4 Mon Sep 17 00:00:00 2001 From: sauravraw Date: Tue, 14 Oct 2025 17:00:43 +0530 Subject: [PATCH 16/37] Enhance asset configuration handling in project models and services - Added optional assetsConfig field to LegacyCMS interface for better asset management. - Updated putTestData function to conditionally update assetsConfig only if provided, preventing overwrites with empty values. - Refactored updateFileFormat to ensure existing asset configurations are preserved when updating. - Removed unnecessary console logs in createDrupalMapper for cleaner code and improved performance. --- api/src/models/project-lowdb.ts | 4 ++++ api/src/services/contentMapper.service.ts | 9 +++++++++ api/src/services/projects.service.ts | 18 ++++++++++++++---- upload-api/src/services/drupal/index.ts | 12 ------------ 4 files changed, 27 insertions(+), 16 deletions(-) diff --git a/api/src/models/project-lowdb.ts b/api/src/models/project-lowdb.ts index 2ea80b78c..b4f6c54a0 100644 --- a/api/src/models/project-lowdb.ts +++ b/api/src/models/project-lowdb.ts @@ -30,6 +30,10 @@ interface LegacyCMS { database: string; port?: number; }; + assetsConfig?: { + base_url?: string; + public_path?: string; + }; is_sql: boolean; file_path: string; is_fileValid: boolean; diff --git a/api/src/services/contentMapper.service.ts b/api/src/services/contentMapper.service.ts index 85d33067a..c8ba5238a 100644 --- a/api/src/services/contentMapper.service.ts +++ b/api/src/services/contentMapper.service.ts @@ -146,6 +146,7 @@ const putTestData = async (req: Request) => { ( ProjectModelLowdb.data.projects[index].legacy_cms as any ).assetsConfig = req.body.assetsConfig; + } else { } if ( @@ -167,6 +168,14 @@ const putTestData = async (req: Request) => { } await ProjectModelLowdb.write(); + + // Re-read from disk to verify persistence + await ProjectModelLowdb.read(); + const verifyIndex = ProjectModelLowdb.chain + .get('projects') + .findIndex({ id: projectId }) + .value(); + const verifyProject = ProjectModelLowdb.data.projects[verifyIndex]; } else { throw new BadRequestError(HTTP_TEXTS.CONTENT_TYPE_NOT_FOUND); } diff --git a/api/src/services/projects.service.ts b/api/src/services/projects.service.ts index d0004bd4e..63e89d46b 100644 --- a/api/src/services/projects.service.ts +++ b/api/src/services/projects.service.ts @@ -515,10 +515,20 @@ const updateFileFormat = async (req: Request) => { mySQLDetails.user; data.projects[projectIndex].legacy_cms.mySQLDetails.database = mySQLDetails.database; - data.projects[projectIndex].legacy_cms.assetsConfig.base_url = - assetsConfig?.base_url || ''; - data.projects[projectIndex].legacy_cms.assetsConfig.public_path = - assetsConfig?.public_path || ''; + + // Only update assetsConfig if it's provided and has values + // Don't overwrite existing config with empty strings + if (assetsConfig && (assetsConfig.base_url || assetsConfig.public_path)) { + data.projects[projectIndex].legacy_cms.assetsConfig.base_url = + assetsConfig.base_url || + data.projects[projectIndex].legacy_cms.assetsConfig.base_url || + ''; + data.projects[projectIndex].legacy_cms.assetsConfig.public_path = + assetsConfig.public_path || + data.projects[projectIndex].legacy_cms.assetsConfig.public_path || + ''; + } else { + } }); logger.info( diff --git a/upload-api/src/services/drupal/index.ts b/upload-api/src/services/drupal/index.ts index 40f0754e8..b4a6d72e9 100644 --- a/upload-api/src/services/drupal/index.ts +++ b/upload-api/src/services/drupal/index.ts @@ -38,17 +38,11 @@ const createDrupalMapper = async ( 'taxonomySchema', 'taxonomySchema.json' ); - console.log('๐Ÿ” Looking for taxonomies at path:', taxonomyPath); if (fs.existsSync(taxonomyPath)) { const taxonomyData = await fs.promises.readFile(taxonomyPath, 'utf8'); taxonomies = JSON.parse(taxonomyData); logger.info(`โœ“ Loaded ${taxonomies.length} taxonomies to send to API`); - console.log('๐Ÿ“ฆ Taxonomies loaded from file:', { - path: taxonomyPath, - count: taxonomies.length, - taxonomies: taxonomies - }); } else { console.warn('โš ๏ธ Taxonomy file not found at:', taxonomyPath); } @@ -65,12 +59,6 @@ const createDrupalMapper = async ( taxonomies: taxonomies // Add taxonomies to payload }; - console.log('๐Ÿ“ค Sending payload to API with:', { - contentTypesCount: initialMapper?.contentTypes?.length || 0, - taxonomiesCount: taxonomies.length, - hasTaxonomies: taxonomies.length > 0 - }); - const req = { method: 'post', maxBodyLength: Infinity, From 8ea963a010a0f5422fe65c360b53c43fdd228d4e Mon Sep 17 00:00:00 2001 From: sauravraw Date: Tue, 14 Oct 2025 18:12:41 +0530 Subject: [PATCH 17/37] Enhance logging for reference and taxonomy fields in contentMapper service - Added logging to putTestData for initializing reference and taxonomy fields with their respective referenceTo values. - Implemented logging in updateContentType to track updates for reference and taxonomy fields, improving traceability during field mapping operations. --- api/src/services/contentMapper.service.ts | 32 +++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/api/src/services/contentMapper.service.ts b/api/src/services/contentMapper.service.ts index c8ba5238a..4d3ee6892 100644 --- a/api/src/services/contentMapper.service.ts +++ b/api/src/services/contentMapper.service.ts @@ -97,6 +97,17 @@ const putTestData = async (req: Request) => { ); } + if ( + (field?.backupFieldType === 'reference' || + field?.backupFieldType === 'taxonomy') && + referenceTo.length > 0 + ) { + console.log( + `๐Ÿ“Œ putTestData: Initializing ${field?.backupFieldType} field "${field?.contentstackField}" with referenceTo:`, + referenceTo + ); + } + return { id, projectId, @@ -700,6 +711,27 @@ const updateContentType = async (req: Request) => { if (Array?.isArray?.(fieldMapping) && !isEmpty(fieldMapping)) { await FieldMapperModel.read(); + + // Log reference/taxonomy fields being updated + const refTaxFields = fieldMapping.filter( + (f: any) => + (f.backupFieldType === 'reference' || + f.backupFieldType === 'taxonomy') && + f.referenceTo && + f.referenceTo.length > 0 + ); + if (refTaxFields.length > 0) { + console.log( + '\n๐Ÿ’พ updateContentType: Saving reference/taxonomy fields:' + ); + refTaxFields.forEach((f: any) => { + console.log( + ` โ€ข ${f.contentstackField} (${f.backupFieldType}): referenceTo =`, + f.referenceTo + ); + }); + } + fieldMapping.forEach((field: any) => { const fieldIndex = FieldMapperModel.data.field_mapper.findIndex( (f: any) => From 1aff4abab976505509e33db79457a1e88cdc625f Mon Sep 17 00:00:00 2001 From: sauravraw Date: Mon, 27 Oct 2025 11:13:44 +0530 Subject: [PATCH 18/37] refactor: filter out profile fields in content mapping and improve related logic - Updated various services and components to exclude 'profile' fields from processing and mapping. - Enhanced the content type mapper to filter out profile references and prevent their inclusion in generated files. - Fixed typos in reference handling across components for consistency. - Added new test scripts for profile reference validation in both API and upload-api packages. --- .gitignore | 4 +- api/package.json | 1 + api/src/services/contentMapper.service.ts | 7 ++- .../services/drupal/content-types.service.ts | 11 ++-- api/src/services/drupal/query.service.ts | 11 ++-- ui/src/components/AdvancePropertise/index.tsx | 2 +- .../ContentMapper/contentMapper.interface.ts | 2 +- ui/src/components/ContentMapper/index.tsx | 4 +- upload-api/migration-drupal/index.js | 4 +- .../libs/contentTypeMapper.js | 28 ++++++--- .../libs/createInitialMapper.js | 60 +++++++++++++++++-- upload-api/package.json | 1 + upload-api/src/services/drupal/index.ts | 4 +- 13 files changed, 105 insertions(+), 34 deletions(-) diff --git a/.gitignore b/.gitignore index 967eba801..3464aef47 100644 --- a/.gitignore +++ b/.gitignore @@ -365,4 +365,6 @@ app.json *.zip *extracted_files* *.tsbuildinfo -*drupalMigrationData* \ No newline at end of file +*drupalMigrationData* +# Snyk Security Extension - AI Rules (auto-generated) +.cursor/rules/snyk_rules.mdc diff --git a/api/package.json b/api/package.json index 7d139258e..39ef1cf90 100644 --- a/api/package.json +++ b/api/package.json @@ -8,6 +8,7 @@ "start": "NODE_ENV=production tsx ./src/server.ts", "start:prod": "NODE_ENV=production node dist/server.js", "dev": "NODE_ENV=production nodemon --exec tsx ./src/server.ts", + "test:profile-refs": "node test-profile-references.js", "prettify": "prettier --write .", "windev": "SET NODE_ENV=production&& tsx watch ./src/server.ts", "winstart": "SET NODE_ENV=production&& node dist/server.js", diff --git a/api/src/services/contentMapper.service.ts b/api/src/services/contentMapper.service.ts index 4d3ee6892..a498d6086 100644 --- a/api/src/services/contentMapper.service.ts +++ b/api/src/services/contentMapper.service.ts @@ -83,11 +83,14 @@ const putTestData = async (req: Request) => { // Initialize referenceTo from advanced data (upload-api) let referenceTo: string[] = []; if (field?.backupFieldType === 'reference') { - // Reference fields use embedObjects OR reference_to - referenceTo = + // Reference fields use embedObjects OR reference_to (filter out profile) + const rawReferences = field?.advanced?.embedObjects || field?.advanced?.reference_to || []; + referenceTo = rawReferences.filter( + (ref: string) => ref && ref.toLowerCase() !== 'profile' + ); } else if ( field?.backupFieldType === 'taxonomy' && field?.advanced?.taxonomies diff --git a/api/src/services/drupal/content-types.service.ts b/api/src/services/drupal/content-types.service.ts index 17291e715..f61453392 100644 --- a/api/src/services/drupal/content-types.service.ts +++ b/api/src/services/drupal/content-types.service.ts @@ -247,15 +247,18 @@ function convertUploadApiSchemaToApiSchema( const newReferences = savedMapping.referenceTo || []; const mergedReferences = [ ...new Set([...oldReferences, ...newReferences]), - ]; + ].filter((ref) => ref && ref.toLowerCase() !== 'profile'); // Filter out profile apiField.reference_to = mergedReferences; } else { // Fall back to upload-api data only (check both embedObjects and reference_to) - const fallbackReferences = + const fallbackReferences = ( uploadField.advanced?.embedObjects || - uploadField.advanced?.reference_to; - if (fallbackReferences) { + uploadField.advanced?.reference_to || + [] + ).filter((ref: string) => ref && ref.toLowerCase() !== 'profile'); // Filter out profile + + if (fallbackReferences && fallbackReferences.length > 0) { apiField.reference_to = fallbackReferences; } } diff --git a/api/src/services/drupal/query.service.ts b/api/src/services/drupal/query.service.ts index e36b55926..424a61a7a 100644 --- a/api/src/services/drupal/query.service.ts +++ b/api/src/services/drupal/query.service.ts @@ -138,10 +138,10 @@ const generateQueriesForFields = async ( const select: { [contentType: string]: string } = {}; const countQuery: { [contentType: string]: string } = {}; - // Group fields by content type + // Group fields by content type and filter out profile const contentTypes = [ ...new Set(fieldData.map((field) => field.content_types)), - ]; + ].filter((contentType) => contentType !== 'profile'); const message = `Processing ${contentTypes.length} content types for query generation...`; await customLogger(projectId, destination_stack_id, 'info', message); @@ -242,7 +242,7 @@ const generateQueriesForFields = async ( // Construct the complete query const selectClause = [ - 'SELECT node.nid, MAX(node.title) AS title, MAX(node.langcode) AS langcode, MAX(node.created) as created, MAX(node.type) as type', + 'SELECT node.nid, MAX(node.title) AS title, MAX(node.langcode) AS langcode, MAX(node.type) as type', ...modifiedResults, ].join(','); @@ -368,7 +368,8 @@ export const createQuery = async ( if ( convDetails && typeof convDetails === 'object' && - 'field_name' in convDetails + 'field_name' in convDetails && + convDetails.bundle !== 'profile' // Filter out profile fields ) { fieldData.push({ field_name: convDetails.field_name, @@ -386,7 +387,7 @@ export const createQuery = async ( throw new Error('No field configuration found in Drupal database'); } - const fieldMessage = `Found ${fieldData.length} field configurations in database`; + const fieldMessage = `Found ${fieldData.length} field configurations in database (profile fields filtered out)`; await customLogger(projectId, destination_stack_id, 'info', fieldMessage); // Generate queries based on field data diff --git a/ui/src/components/AdvancePropertise/index.tsx b/ui/src/components/AdvancePropertise/index.tsx index 754aa2623..93936f4e3 100644 --- a/ui/src/components/AdvancePropertise/index.tsx +++ b/ui/src/components/AdvancePropertise/index.tsx @@ -75,7 +75,7 @@ const AdvancePropertise = (props: SchemaProps) => { value: item })); - const referencedItems = props?.data?.refrenceTo?.map((item: string) => ({ + const referencedItems = props?.data?.referenceTo?.map((item: string) => ({ label: item, value: item })); diff --git a/ui/src/components/ContentMapper/contentMapper.interface.ts b/ui/src/components/ContentMapper/contentMapper.interface.ts index a3135e21b..06d06a8b7 100644 --- a/ui/src/components/ContentMapper/contentMapper.interface.ts +++ b/ui/src/components/ContentMapper/contentMapper.interface.ts @@ -67,7 +67,7 @@ export interface FieldMapType { contentstackUid: string; _invalid?: boolean; backupFieldUid: string; - refrenceTo: string[]; + referenceTo: string[]; } export interface Advanced { diff --git a/ui/src/components/ContentMapper/index.tsx b/ui/src/components/ContentMapper/index.tsx index 8f4b51dce..3335f4cec 100644 --- a/ui/src/components/ContentMapper/index.tsx +++ b/ui/src/components/ContentMapper/index.tsx @@ -338,7 +338,7 @@ const ContentMapper = forwardRef(({ handleStepChange }: contentMapperProps, ref: // Make title and url field non editable useEffect(() => { tableData?.forEach((field) => { - if(field?.backupFieldType === 'reference' && field?.refrenceTo?.length === 0) { + if(field?.backupFieldType === 'reference' && field?.referenceTo?.length === 0) { field._canSelect = false; }else if(field?.backupFieldType === 'taxonomy' && field?.referenceTo?.length === 0) { field._canSelect = false; @@ -867,7 +867,7 @@ const ContentMapper = forwardRef(({ handleStepChange }: contentMapperProps, ref: if (row?.uid === rowId && row?.contentstackFieldUid === rowContentstackFieldUid) { const updatedRow = { ...row, - refrenceTo: updatedSettings?.referenedItems, + referenceTo: updatedSettings?.referenedItems, advanced: { ...row?.advanced, ...updatedSettings } }; return updatedRow; diff --git a/upload-api/migration-drupal/index.js b/upload-api/migration-drupal/index.js index 395c84b50..ac588e200 100644 --- a/upload-api/migration-drupal/index.js +++ b/upload-api/migration-drupal/index.js @@ -1,8 +1,8 @@ /* eslint-disable @typescript-eslint/no-var-requires */ const extractTaxonomy = require('./libs/extractTaxonomy'); -const createInitialMapper = require( './libs/createInitialMapper' ); -const extractLocale= require('./libs/extractLocale'); +const createInitialMapper = require('./libs/createInitialMapper'); +const extractLocale = require('./libs/extractLocale'); module.exports = { // extractContentTypes, diff --git a/upload-api/migration-drupal/libs/contentTypeMapper.js b/upload-api/migration-drupal/libs/contentTypeMapper.js index 3f747a052..658b33785 100644 --- a/upload-api/migration-drupal/libs/contentTypeMapper.js +++ b/upload-api/migration-drupal/libs/contentTypeMapper.js @@ -1,6 +1,6 @@ /* eslint-disable @typescript-eslint/no-var-requires */ +/* eslint-disable operator-linebreak */ -const restrictedUid = require('../utils/restrictedKeyWords'); const fs = require('fs'); const path = require('path'); @@ -53,6 +53,18 @@ function startsWithNumber(str) { return /^\d/.test(str); } +/** + * Helper function to filter out profile from reference fields + * @param {Array} referenceFields - Array of content type UIDs + * @returns {Array} Filtered array without profile + */ +function filterOutProfile(referenceFields) { + if (!Array.isArray(referenceFields)) { + return referenceFields; + } + return referenceFields.filter((ref) => ref && ref.toLowerCase() !== 'profile'); +} + /** * Improved UID corrector based on Sitecore implementation but adapted for Drupal * Handles all edge cases: restricted keywords, numbers, CamelCase, special characters @@ -484,7 +496,7 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => case 'text_long': { // Rich text with switching options: JSON RTE โ†’ HTML RTE โ†’ multiline โ†’ text const availableContentTypes = contentTypes?.filter((ct) => ct !== item.content_types) || []; - const referenceFields = availableContentTypes.slice(0, 10); + const referenceFields = filterOutProfile(availableContentTypes.slice(0, 10)); acc.push(createFieldObject(item, 'json', 'html', referenceFields)); break; } @@ -506,7 +518,7 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => case 'string': { // Single line with switching options: single_line โ†’ multiline โ†’ HTML RTE โ†’ JSON RTE const availableContentTypes = contentTypes?.filter((ct) => ct !== item.content_types) || []; - const referenceFields = availableContentTypes.slice(0, 10); + const referenceFields = filterOutProfile(availableContentTypes.slice(0, 10)); acc.push(createFieldObject(item, 'single_line_text', 'multi_line_text', referenceFields)); break; } @@ -594,7 +606,7 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => // Use available content types instead of generic 'taxonomy' const availableContentTypes = contentTypes?.filter((ct) => ct !== item.content_types) || []; - const referenceFields = availableContentTypes.slice(0, 10); + const referenceFields = filterOutProfile(availableContentTypes.slice(0, 10)); acc.push(createFieldObject(item, 'reference', 'reference', referenceFields)); } } else if (item.handler === 'default:node') { @@ -602,13 +614,13 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => let referenceFields = []; if (item.reference && Object.keys(item.reference).length > 0) { - // Use specific content types from field configuration - referenceFields = Object.keys(item.reference); + // Use specific content types from field configuration (filter out profile) + referenceFields = filterOutProfile(Object.keys(item.reference)); } else { // Backup: Use up to 10 content types from available content types const availableContentTypes = contentTypes?.filter((ct) => ct !== item.content_types) || []; - referenceFields = availableContentTypes.slice(0, 10); + referenceFields = filterOutProfile(availableContentTypes.slice(0, 10)); } acc.push(createFieldObject(item, 'reference', 'reference', referenceFields)); @@ -616,7 +628,7 @@ const contentTypeMapper = async (data, contentTypes, prefix, dbConfig = null) => // Handle other entity references - exclude taxonomy and limit to 10 content types const availableContentTypes = contentTypes?.filter((ct) => ct !== item.content_types) || []; - const referenceFields = availableContentTypes.slice(0, 10); + const referenceFields = filterOutProfile(availableContentTypes.slice(0, 10)); acc.push(createFieldObject(item, 'reference', 'reference', referenceFields)); } break; diff --git a/upload-api/migration-drupal/libs/createInitialMapper.js b/upload-api/migration-drupal/libs/createInitialMapper.js index d5fb51727..b0265f427 100644 --- a/upload-api/migration-drupal/libs/createInitialMapper.js +++ b/upload-api/migration-drupal/libs/createInitialMapper.js @@ -12,7 +12,7 @@ const contentTypeMapper = require('./contentTypeMapper'); /** * Internal module dependencies. */ -const { dbConnection, writeFile } = require('../utils/helper'); +const { dbConnection } = require('../utils/helper'); const config = require('../config'); const idArray = require('../utils/restrictedKeyWords'); @@ -86,11 +86,28 @@ const createInitialMapper = async (systemConfig, prefix) => { const details_data = []; // Process each row to extract field data + let profileFieldsFiltered = 0; for (let i = 0; i < rows.length; i++) { try { const { unserialize } = require('php-serialize'); const conv_details = unserialize(rows[i].data); + // Filter out profile content type fields (case-insensitive) + const bundleName = conv_details?.bundle?.toLowerCase(); + if (bundleName === 'profile') { + profileFieldsFiltered++; + console.log( + `๐Ÿšซ Filtered out profile field: ${conv_details?.field_name} (bundle: ${conv_details?.bundle})` + ); + continue; // Skip profile fields + } + + // Double check - don't add if content_types would be 'profile' + if (conv_details?.bundle?.toLowerCase() === 'profile') { + console.log(`โš ๏ธ Skipping profile data that slipped through first filter`); + continue; + } + details_data.push({ field_label: conv_details?.label, description: conv_details?.description, @@ -108,15 +125,37 @@ const createInitialMapper = async (systemConfig, prefix) => { } } + if (profileFieldsFiltered > 0) { + console.log(`โœ… Filtered out ${profileFieldsFiltered} profile fields`); + } + if (details_data.length === 0) { return { contentTypes: [] }; } const initialMapper = []; - const contentTypes = Object.keys(require('lodash').keyBy(details_data, 'content_types')); + const allContentTypes = Object.keys(require('lodash').keyBy(details_data, 'content_types')); + // Aggressive filter: remove profile (case-insensitive) and any null/undefined + const contentTypes = allContentTypes.filter( + (contentType) => contentType && contentType.toLowerCase() !== 'profile' + ); + + console.log(`๐Ÿ“‹ Content Types Found: ${allContentTypes.length}`); + console.log(` All: ${allContentTypes.join(', ')}`); + console.log(` After filtering profile: ${contentTypes.join(', ')}`); + + if (allContentTypes.some((ct) => ct && ct.toLowerCase() === 'profile')) { + console.log('โš ๏ธ WARNING: Profile was in content types list but has been filtered out'); + } // Process each content type for (const contentType of contentTypes) { + // Extra safety check - skip if contentType is profile (case-insensitive) + if (!contentType || contentType.toLowerCase() === 'profile') { + console.log(`๐Ÿšซ Skipping profile content type in processing loop`); + continue; + } + const contentTypeFields = require('lodash').filter(details_data, { content_types: contentType }); @@ -161,12 +200,21 @@ const createInitialMapper = async (systemConfig, prefix) => { } }; - await fsp.writeFile( - path.join(drupalFolderPath, `${contentType}.json`), - JSON.stringify(main, null, 4) - ); + // Final safety check before writing file - NEVER write profile.json + if (contentType.toLowerCase() === 'profile') { + console.log(`๐Ÿ›‘ BLOCKED: Attempted to create profile.json - skipping!`); + continue; + } + + const filePath = path.join(drupalFolderPath, `${contentType}.json`); + await fsp.writeFile(filePath, JSON.stringify(main, null, 4)); + console.log(`โœ“ Created: ${contentType}.json`); } + console.log( + `\nโœ… Successfully created ${contentTypes.length} content type files (profile excluded)` + ); + // Close database connection connection.end(); return { contentTypes: initialMapper }; diff --git a/upload-api/package.json b/upload-api/package.json index d429490d5..dc94e7827 100644 --- a/upload-api/package.json +++ b/upload-api/package.json @@ -6,6 +6,7 @@ "scripts": { "build": "npm --prefix migration-aem run build && tsc", "test": "echo \"Error: no test specified\" && exit 1", + "test:profile": "node test-profile-removal.js", "start": "PORT=4002 nodemon --exec tsx src/index.ts --ignore drupalMigrationData --ignore migration-* --ignore build --ignore *.log --ignore extracted_files", "start:prod": "tsc && node build/index.js", "validation": "tsc && node build/main.js", diff --git a/upload-api/src/services/drupal/index.ts b/upload-api/src/services/drupal/index.ts index b4a6d72e9..bde9471ee 100644 --- a/upload-api/src/services/drupal/index.ts +++ b/upload-api/src/services/drupal/index.ts @@ -51,9 +51,9 @@ const createDrupalMapper = async ( console.error('โŒ Error reading taxonomy file:', error); } - // Include assetsConfig, mySQLDetails, and taxonomies with the mapper data + // Build the complete mapper payload with all content types const mapperPayload = { - ...initialMapper, + contentTypes: initialMapper.contentTypes, // All content types (no profile) assetsConfig: config.assetsConfig, mySQLDetails: config.mysql, taxonomies: taxonomies // Add taxonomies to payload From 984c01d21c75246fa0798398be3a1c0e5efd36be Mon Sep 17 00:00:00 2001 From: sauravraw Date: Mon, 27 Oct 2025 15:47:47 +0530 Subject: [PATCH 19/37] refactor: enhance drupal service and locale mapping logic - Added 'project' parameter to various service functions to support dynamic locale mapping. - Implemented a new utility function, mapDrupalLocales, to facilitate mapping of source locales to destination locales based on user-selected configurations. - Updated processEntries and createLocale functions to utilize the new mapping logic, improving flexibility in locale transformations. - Enhanced overall code readability and maintainability by restructuring locale handling processes. --- api/src/services/drupal.service.ts | 6 +- api/src/services/drupal/entries.service.ts | 70 +++++---- api/src/services/drupal/locales.service.ts | 158 ++++++++++++++------- api/src/services/migration.service.ts | 6 +- 4 files changed, 161 insertions(+), 79 deletions(-) diff --git a/api/src/services/drupal.service.ts b/api/src/services/drupal.service.ts index 39770b6f6..9cd1d5f2b 100644 --- a/api/src/services/drupal.service.ts +++ b/api/src/services/drupal.service.ts @@ -52,7 +52,8 @@ export const drupalService = { projectId: string, isTest = false, masterLocale = 'en-us', - contentTypeMapping: any[] = [] + contentTypeMapping: any[] = [], + project: any = null ) => { return createEntry( dbConfig, @@ -60,7 +61,8 @@ export const drupalService = { projectId, isTest, masterLocale, - contentTypeMapping + contentTypeMapping, + project ); }, createLocale, // Create locale configurations diff --git a/api/src/services/drupal/entries.service.ts b/api/src/services/drupal/entries.service.ts index 180da878b..4c373fcbb 100644 --- a/api/src/services/drupal/entries.service.ts +++ b/api/src/services/drupal/entries.service.ts @@ -31,6 +31,7 @@ import { entriesFieldCreator, unflatten, } from '../../utils/entries-field-creator.utils.js'; +import { mapDrupalLocales } from './locales.service.js'; // Dynamic import for phpUnserialize will be used in the function // Local utility functions (extracted from entries-field-creator.utils.ts patterns) @@ -1039,7 +1040,8 @@ const processEntries = async ( destination_stack_id: string, masterLocale: string, contentTypeMapping: any[] = [], - isTest: boolean = false + isTest: boolean = false, + project: any = null ): Promise<{ [key: string]: any } | null> => { const srcFunc = 'processEntries'; @@ -1143,56 +1145,68 @@ const processEntries = async ( entriesByLocale[entryLocale].push(entry); }); - // ๐Ÿ”„ Apply locale folder transformation rules (same as locale service) - // Rules: - // - "und" alone โ†’ "en-us" - // - "und" + "en-us" โ†’ "und" become "en", "en-us" stays - // - "en" + "und" โ†’ "und" becomes "en-us", "en" stays - // - All three "en" + "und" + "en-us" โ†’ all three stays - // - Apart from these, all other locales stay as is + // Map source locales to destination locales using user-selected mapping from UI + // This replaces the old hardcoded transformation rules with dynamic user mapping const transformedEntriesByLocale: { [locale: string]: any[] } = {}; const allLocales = Object.keys(entriesByLocale); const hasUnd = allLocales.includes('und'); const hasEn = allLocales.includes('en'); const hasEnUs = allLocales.includes('en-us'); - // Transform locale folder names based on business rules + // Get locale mapping configuration from project + const localeMapping = project?.localeMapping || {}; + const localesFromProject = { + masterLocale: project?.master_locale || {}, + ...(project?.locales || {}), + }; + + // Apply source locale transformation rules first (und โ†’ en-us, etc.) + // Then map the transformed source locale to destination locale using user's selection Object.entries(entriesByLocale).forEach(([originalLocale, entries]) => { - let targetFolder = originalLocale; + // Step 1: Apply Drupal-specific transformation rules (same as before) + let transformedSourceLocale = originalLocale; if (originalLocale === 'und') { if (hasEn && hasEnUs) { // Rule 4: All three "en" + "und" + "en-us" โ†’ all three stays - targetFolder = 'und'; + transformedSourceLocale = 'und'; } else if (hasEnUs && !hasEn) { // Rule 2: "und" + "en-us" โ†’ "und" become "en", "en-us" stays - targetFolder = 'en'; + transformedSourceLocale = 'en'; } else if (hasEn && !hasEnUs) { // Rule 3: "en" + "und" โ†’ "und" becomes "en-us", "en" stays - targetFolder = 'en-us'; + transformedSourceLocale = 'en-us'; } else if (!hasEn && !hasEnUs) { // Rule 1: "und" alone โ†’ "en-us" - targetFolder = 'en-us'; + transformedSourceLocale = 'en-us'; } else { // Keep as is for any other combinations - targetFolder = 'und'; + transformedSourceLocale = 'und'; } } else if (originalLocale === 'en-us') { // "en-us" always stays as "en-us" in all rules - targetFolder = 'en-us'; + transformedSourceLocale = 'en-us'; } else if (originalLocale === 'en') { // "en" always stays as "en" in all rules (never transforms to "und") - targetFolder = 'en'; + transformedSourceLocale = 'en'; } - // Merge entries if target folder already has entries - if (transformedEntriesByLocale[targetFolder]) { - transformedEntriesByLocale[targetFolder] = [ - ...transformedEntriesByLocale[targetFolder], + // Step 2: Map transformed source locale to destination locale using user's mapping + const destinationLocale = mapDrupalLocales({ + masterLocale, + locale: transformedSourceLocale, + locales: localesFromProject, + localeMapping, + }); + + // Merge entries if destination locale already has entries + if (transformedEntriesByLocale[destinationLocale]) { + transformedEntriesByLocale[destinationLocale] = [ + ...transformedEntriesByLocale[destinationLocale], ...entries, ]; } else { - transformedEntriesByLocale[targetFolder] = entries; + transformedEntriesByLocale[destinationLocale] = entries; } }); @@ -1468,7 +1482,8 @@ const processContentType = async ( destination_stack_id: string, masterLocale: string, contentTypeMapping: any[] = [], - isTest: boolean = false + isTest: boolean = false, + project: any = null ): Promise => { const srcFunc = 'processContentType'; @@ -1519,7 +1534,8 @@ const processContentType = async ( destination_stack_id, masterLocale, contentTypeMapping, - isTest + isTest, + project ); // If no entries returned, break the loop @@ -1579,7 +1595,8 @@ export const createEntry = async ( projectId: string, isTest = false, masterLocale = 'en-us', - contentTypeMapping: any[] = [] + contentTypeMapping: any[] = [], + project: any = null ): Promise => { const srcFunc = 'createEntry'; let connection: mysql.Connection | null = null; @@ -1668,7 +1685,8 @@ export const createEntry = async ( destination_stack_id, masterLocale, contentTypeMapping, - isTest + isTest, + project ); } diff --git a/api/src/services/drupal/locales.service.ts b/api/src/services/drupal/locales.service.ts index 0cf2edb88..63410cebe 100644 --- a/api/src/services/drupal/locales.service.ts +++ b/api/src/services/drupal/locales.service.ts @@ -6,6 +6,7 @@ import { Locale } from '../../models/types.js'; import { getAllLocales, getLogMessage } from '../../utils/index.js'; import customLogger from '../../utils/custom-logger.utils.js'; import { createDbConnection } from '../../helper/index.js'; +import { v4 as uuidv4 } from 'uuid'; const { DATA: MIGRATION_DATA_PATH, @@ -38,6 +39,59 @@ function getKeyByValue( return Object.entries(obj).find(([_, value]) => value === targetValue)?.[0]; } +/** + * Maps source locale to destination locale based on user-selected mapping from UI. + * Similar to WordPress/Contentful/Sitecore mapLocales function. + * + * @param masterLocale - The master locale code from the stack + * @param locale - The source locale code from Drupal + * @param locales - The locale mapping object from project (contains master_locale and locales) + * @param localeMapping - The direct locale mapping from UI (optional, takes precedence) + * @returns The mapped destination locale code + */ +export function mapDrupalLocales({ + masterLocale, + locale, + locales, + localeMapping, +}: { + masterLocale: string; + locale: string; + locales?: any; + localeMapping?: Record; +}): string { + // Priority 1: Check direct locale mapping from UI (format: { "en-master_locale": "fr-fr", "es": "es-es" }) + if (localeMapping) { + // Check if this is a master locale mapping + const masterKey = `${locale}-master_locale`; + if (localeMapping[masterKey]) { + return localeMapping[masterKey]; + } + + // Check direct mapping + if (localeMapping[locale]) { + return localeMapping[locale]; + } + } + + // Priority 2: Check if source locale matches master locale in mapping + if (locales?.masterLocale?.[masterLocale] === locale) { + return Object.keys(locales.masterLocale)?.[0] || masterLocale; + } + + // Priority 3: Check regular locales mapping + if (locales) { + for (const [key, value] of Object.entries(locales)) { + if (typeof value !== 'object' && value === locale) { + return key; + } + } + } + + // Priority 4: Return locale as-is (lowercase) + return locale?.toLowerCase?.() || locale; +} + /** * Fetches locale names from Contentstack API */ @@ -139,9 +193,9 @@ function applyLocaleTransformations( * This function: * 1. Fetches master locale from Drupal system.site config * 2. Fetches all locales from node_field_data - * 3. Fetches non-master locales separately - * 4. Gets locale names from Contentstack API - * 5. Applies special transformation rules for "und", "en", "en-us" + * 3. Uses user-selected locale mapping from UI (project.localeMapping) + * 4. Maps source locales to destination locales based on user selection + * 5. Sets master locale based on user selection * 6. Creates 3 JSON files: master-locale.json, locales.json, language.json */ export const createLocale = async ( @@ -198,7 +252,7 @@ export const createLocale = async ( `; const masterRows: any = await executeQuery(masterLocaleQuery); - const masterLocaleCode = masterRows[0]?.master_locale || 'en'; + const sourceMasterLocale = masterRows[0]?.master_locale || 'en'; // 2. Get all locales from node_field_data const allLocalesQuery = ` @@ -209,64 +263,70 @@ export const createLocale = async ( `; const allLocaleRows: any = await executeQuery(allLocalesQuery); - const allLocaleCodes = allLocaleRows.map((row: any) => row.langcode); - - // 3. Get non-master locales - const nonMasterLocalesQuery = ` - SELECT DISTINCT n.langcode - FROM node_field_data n - WHERE n.langcode IS NOT NULL - AND n.langcode != '' - AND n.langcode != ( - SELECT - SUBSTRING_INDEX( - SUBSTRING_INDEX(CONVERT(data USING utf8), 'default_langcode";s:2:"', -1), - '"', - 1 - ) - FROM config - WHERE name = 'system.site' - LIMIT 1 - ) - ORDER BY n.langcode - `; - - const nonMasterRows: any = await executeQuery(nonMasterLocalesQuery); - const nonMasterLocaleCodes = nonMasterRows.map((row: any) => row.langcode); + const sourceLocaleCodes = allLocaleRows.map((row: any) => row.langcode); // Close database connection connection.end(); - // 4. Fetch locale names from Contentstack API - const contentstackLocales = await fetchContentstackLocales(); + // 3. Get user-selected locale mapping from UI (project.localeMapping or project.locales/master_locale) + // localeMapping format: { "en-master_locale": "fr-fr", "es": "es-es", ... } + const localeMapping = project?.localeMapping || {}; + const masterLocaleFromProject = project?.master_locale || {}; + const localesFromProject = project?.locales || {}; - // 5. Apply special transformation rules + // 4. Fetch locale names from Contentstack API + const [err, localesApiResponse] = await getAllLocales(); + const contentstackLocales = localesApiResponse?.data?.locales || {}; + + // 5. Map source locales to destination locales using user selection + // Find the destination master locale based on source master locale + const masterLocaleKey = `${sourceMasterLocale}-master_locale`; + let destinationMasterLocale = + localeMapping[masterLocaleKey] || + Object.keys(masterLocaleFromProject)?.[0] || + project?.stackDetails?.master_locale || + 'en-us'; + + // Process transformed locales first (handle und, en, en-us) const transformedLocales = applyLocaleTransformations( - allLocaleCodes, - masterLocaleCode + sourceLocaleCodes, + sourceMasterLocale ); - // 6. Process each locale - transformedLocales.forEach((localeInfo, index) => { - const { code, name, isMaster } = localeInfo; + // Map each transformed source locale to destination locale + transformedLocales.forEach((localeInfo) => { + const { code: sourceCode, isMaster } = localeInfo; + + // Find destination locale from mapping + let destinationCode: string; + + if (isMaster) { + // For master locale, use the mapped master locale + destinationCode = destinationMasterLocale; + } else { + // For non-master locales, check mapping or use as-is + destinationCode = + localeMapping[sourceCode] || + localesFromProject[sourceCode] || + sourceCode; + } - // Create UID using original langcode from database - const originalLangcode = allLocaleCodes[index]; // Get original langcode from database - const uid = `drupallocale_${originalLangcode - .toLowerCase() - .replace(/-/g, '_')}`; + // Create UID + const uid = uuidv4(); - // Get name from Contentstack API or use transformed name + // Get name from Contentstack API const localeName = - name || - contentstackLocales[code] || - contentstackLocales[code.toLowerCase()] || + contentstackLocales[destinationCode] || + contentstackLocales[destinationCode.toLowerCase()] || + contentstackLocales[sourceCode] || 'Unknown Language'; const newLocale: Locale = { - code: code.toLowerCase(), + code: destinationCode.toLowerCase(), name: localeName, - fallback_locale: isMaster ? null : masterLocaleCode.toLowerCase(), + fallback_locale: isMaster + ? null + : destinationMasterLocale.toLowerCase(), uid: uid, }; @@ -284,14 +344,14 @@ export const createLocale = async ( const finalAllLocales = Object.keys(allLocales).length > 0 ? allLocales : {}; - // 7. Write locale files (same structure as Contentful) + // 6. Write locale files (same structure as Contentful/WordPress) await writeFile(localeSave, LOCALE_FILE_NAME, finalAllLocales); // locales.json (non-master only) await writeFile(localeSave, LOCALE_MASTER_LOCALE, msLocale); // master-locale.json (master only) await writeFile(localeSave, LOCALE_CF_LANGUAGE, localeList); // language.json (all locales) const message = getLogMessage( srcFunc, - `Drupal locales have been successfully transformed. Master: ${masterLocaleCode}, Total: ${allLocaleCodes.length}, Non-master: ${nonMasterLocaleCodes.length}`, + `Drupal locales have been successfully transformed. Source Master: ${sourceMasterLocale}, Destination Master: ${destinationMasterLocale}, Total: ${sourceLocaleCodes.length}`, {} ); await customLogger(projectId, destination_stack_id, 'info', message); diff --git a/api/src/services/migration.service.ts b/api/src/services/migration.service.ts index bc6573867..ebb711435 100644 --- a/api/src/services/migration.service.ts +++ b/api/src/services/migration.service.ts @@ -672,7 +672,8 @@ const startTestMigration = async (req: Request): Promise => { projectId, true, project?.stackDetails?.master_locale, - project?.content_mapper || [] + project?.content_mapper || [], + project ); await drupalService?.createLocale( dbConfig, @@ -1103,7 +1104,8 @@ const startMigration = async (req: Request): Promise => { projectId, false, project?.stackDetails?.master_locale, - project?.content_mapper || [] + project?.content_mapper || [], + project ); await drupalService?.createVersionFile( project?.destination_stack_id, From e73cc8ccca524469537c56cd22e0d2b3fafa77bc Mon Sep 17 00:00:00 2001 From: sauravraw Date: Mon, 27 Oct 2025 15:56:56 +0530 Subject: [PATCH 20/37] feat: implement asset URL tracking in Drupal asset service - Added AssetUrlTracker interface to monitor successful and failed asset downloads. - Enhanced saveAsset function to utilize URL tracking, logging both successful and failed attempts. - Implemented fallback path logic for asset downloads, improving reliability in case of primary path failures. - Updated createAssets function to initialize and write asset URL summary to assets_url.json, providing better visibility into asset processing outcomes. --- api/src/services/drupal/assets.service.ts | 148 ++++++++++++++++++++-- 1 file changed, 139 insertions(+), 9 deletions(-) diff --git a/api/src/services/drupal/assets.service.ts b/api/src/services/drupal/assets.service.ts index 163d135a1..992a69b53 100644 --- a/api/src/services/drupal/assets.service.ts +++ b/api/src/services/drupal/assets.service.ts @@ -36,6 +36,23 @@ interface DrupalAsset { count?: string | number; // For file_usage table } +/** + * Interface to track asset download URLs and their status + */ +interface AssetUrlTracker { + success: Array<{ + uid: string; + url: string; + filename: string; + }>; + failed: Array<{ + uid: string; + url: string; + filename: string; + reason: string; + }>; +} + /** * Writes data to a specified file, ensuring the target directory exists. */ @@ -415,7 +432,9 @@ const saveAsset = async ( baseUrl: string = '', publicPath: string = '/sites/default/files/', retryCount = 0, - authHeaders: any = {} // NEW: Support for authentication + authHeaders: any = {}, // Support for authentication + urlTracker?: AssetUrlTracker, // Track successful and failed URLs + userProvidedPublicPath?: string // Original user-provided path (for failed URL tracking) ): Promise => { try { const srcFunc = 'saveAsset'; @@ -487,6 +506,15 @@ const saveAsset = async ( metadata.push({ uid: assetId, url: fileUrl, filename: fileName }); + // Track successful download + if (urlTracker) { + urlTracker.success.push({ + uid: assetId, + url: fileUrl, + filename: fileName, + }); + } + if (failedJSON[assetId]) { delete failedJSON[assetId]; } @@ -521,10 +549,69 @@ const saveAsset = async ( baseUrl, publicPath, retryCount + 1, - authHeaders + authHeaders, + urlTracker, + userProvidedPublicPath ); } else { - // Failed after retries + // After 3 retries failed, try fallback paths (if not already tried) + const commonPaths = [ + '/sites/default/files/', + '/sites/all/files/', + '/sites/g/files/bxs2566/files/', // Rice University specific + 'sites/default/files/', + 'sites/all/files/', + ]; + + // Only try fallback if current path is the user-provided path + const isUserProvidedPath = + publicPath === (userProvidedPublicPath || publicPath); + + if (isUserProvidedPath) { + console.log( + `๐Ÿ”„ Primary path failed. Trying fallback paths for ${fileName}...` + ); + + // Try each common path + for (const fallbackPath of commonPaths) { + // Skip if already tried + if (fallbackPath === publicPath) { + continue; + } + + console.log(`๐Ÿ” Trying fallback path: ${fallbackPath}`); + + try { + // Attempt download with fallback path (reset retry count) + const result = await saveAsset( + assets, + failedJSON, + assetData, + metadata, + projectId, + destination_stack_id, + baseUrl, + fallbackPath, + 0, // Reset retry count for fallback attempt + authHeaders, + urlTracker, + userProvidedPublicPath || publicPath // Keep original user path for tracking + ); + + // Check if asset was actually saved (exists in assetData) + if (assetData[assetId]) { + console.log(`โœ… Success with fallback path: ${fallbackPath}`); + return result; // Successfully downloaded with fallback path + } + } catch (fallbackErr) { + // Continue to next fallback path + console.log(`โŒ Fallback path ${fallbackPath} also failed`); + continue; + } + } + } + + // All attempts failed - log failure const errorDetails = { status: err.response?.status, statusText: err.response?.statusText, @@ -532,17 +619,36 @@ const saveAsset = async ( url: fileUrl, }; + // Use user-provided public path for the failed URL + const failedUrl = constructAssetUrl( + assets.uri, + baseUrl, + userProvidedPublicPath || publicPath + ); + failedJSON[assetId] = { failedUid: assets.fid, name: fileName, - url: fileUrl, + url: failedUrl, file_size: assets.filesize, reason_for_error: JSON.stringify(errorDetails), }; + // Track failed download with user-provided URL + if (urlTracker) { + urlTracker.failed.push({ + uid: assetId, + url: failedUrl, + filename: fileName, + reason: `${err.response?.status || 'Network error'}: ${ + err.message + }`, + }); + } + const message = getLogMessage( srcFunc, - `โŒ Failed to download "${fileName}" (${assets.fid}): ${err.message}`, + `โŒ Failed to download "${fileName}" (${assets.fid}) after all attempts: ${err.message}`, {}, err ); @@ -613,7 +719,9 @@ const retryFailedAssets = async ( destination_stack_id: string, baseUrl: string = '', publicPath: string = '/sites/default/files/', - authHeaders: any = {} + authHeaders: any = {}, + urlTracker?: AssetUrlTracker, + userProvidedPublicPath?: string ): Promise => { const srcFunc = 'retryFailedAssets'; @@ -641,7 +749,9 @@ const retryFailedAssets = async ( baseUrl, publicPath, 0, - authHeaders + authHeaders, + urlTracker, + userProvidedPublicPath ) ) ); @@ -721,6 +831,12 @@ export const createAssets = async ( const fileMeta = { '1': ASSETS_SCHEMA_FILE }; const failedAssetIds: string[] = []; + // Initialize URL tracker for assets_url.json + const urlTracker: AssetUrlTracker = { + success: [], + failed: [], + }; + const message = getLogMessage( srcFunc, `Exporting assets using base URL: ${baseUrl} and public path: ${detectedPublicPath}`, @@ -756,7 +872,9 @@ export const createAssets = async ( baseUrl, detectedPublicPath, // Use detected path 0, - {} // authHeaders + {}, // authHeaders + urlTracker, // Pass URL tracker + publicPath || detectedPublicPath // Use original user-provided path for tracking ); } catch (error) { failedAssetIds.push(asset.fid.toString()); @@ -791,7 +909,10 @@ export const createAssets = async ( projectId, destination_stack_id, baseUrl, - detectedPublicPath // Use detected path + detectedPublicPath, // Use detected path + {}, // authHeaders + urlTracker, // Pass URL tracker + publicPath || detectedPublicPath // User-provided path for tracking ); } @@ -802,6 +923,15 @@ export const createAssets = async ( await writeFile(assetMasterFolderPath, ASSETS_FAILED_FILE, failedJSON); } + // Write assets_url.json with successful and failed URLs + await writeFile(assetsSave, 'assets_url.json', urlTracker); + + console.log(`๐Ÿ“Š Asset URL Summary:`); + console.log( + ` โœ… Successfully downloaded: ${urlTracker.success.length}` + ); + console.log(` โŒ Failed downloads: ${urlTracker.failed.length}`); + const successMessage = getLogMessage( srcFunc, `Successfully processed ${ From d71296839bf76588cab4f5a14bfd76c58d00fadb Mon Sep 17 00:00:00 2001 From: sauravraw Date: Mon, 27 Oct 2025 16:50:18 +0530 Subject: [PATCH 21/37] fix: update file format handling and improve SQL connection detection - Changed the title of the SQL file format in legacyCms.json to "ApiTokens". - Refactored LoadFileFormat component to enhance SQL connection detection using multiple indicators. - Improved file format validation logic for Drupal SQL connections, ensuring correct handling and icon display. - Updated Redux state management for selected file formats to reflect changes in SQL connection handling. --- ui/src/cmsData/legacyCms.json | 4 +- .../LegacyCms/Actions/LoadFileFormat.tsx | 169 +++++++++++------- upload-api/src/config/index.ts | 2 +- 3 files changed, 111 insertions(+), 64 deletions(-) diff --git a/ui/src/cmsData/legacyCms.json b/ui/src/cmsData/legacyCms.json index 433f6389c..2578f53d3 100644 --- a/ui/src/cmsData/legacyCms.json +++ b/ui/src/cmsData/legacyCms.json @@ -106,7 +106,7 @@ "allowed_file_formats": [ { "fileformat_id": "sql", - "title": "SQL", + "title": "ApiTokens", "description": "", "group_name": "sql", "isactive": true, @@ -323,4 +323,4 @@ "restricted_keyword_checkbox_text": "Please acknowledge that you have referred to the Contentstack restricted keywords", "affix_cta": "Continue", "file_format_cta": "Continue" -} \ No newline at end of file +} diff --git a/ui/src/components/LegacyCms/Actions/LoadFileFormat.tsx b/ui/src/components/LegacyCms/Actions/LoadFileFormat.tsx index 291e22f56..174255509 100644 --- a/ui/src/components/LegacyCms/Actions/LoadFileFormat.tsx +++ b/ui/src/components/LegacyCms/Actions/LoadFileFormat.tsx @@ -32,7 +32,7 @@ const LoadFileFormat = (props: LoadFileFormatProps) => { const [isCheckedBoxChecked] = useState( newMigrationData?.legacy_cms?.isFileFormatCheckboxChecked || true ); - const [fileIcon, setFileIcon] = useState(newMigrationDataRef?.current?.legacy_cms?.selectedFileFormat?.title); + const [fileIcon, setFileIcon] = useState(''); const [isError, setIsError] = useState(false); const [error, setError] = useState(''); @@ -57,74 +57,104 @@ const LoadFileFormat = (props: LoadFileFormatProps) => { const handleFileFormat = async() =>{ try { - - const cmsType = !isEmptyString(newMigrationData?.legacy_cms?.selectedCms?.parent) ? newMigrationData?.legacy_cms?.selectedCms?.parent : newMigrationData?.legacy_cms?.uploadedFile?.cmsType; + + const cmsType = !isEmptyString(newMigrationData?.legacy_cms?.selectedCms?.parent) + ? newMigrationData?.legacy_cms?.selectedCms?.parent + : newMigrationData?.legacy_cms?.uploadedFile?.cmsType; const filePath = newMigrationData?.legacy_cms?.uploadedFile?.file_details?.localPath?.toLowerCase(); + // Check if this is a SQL connection with multiple fallback indicators + const isSQLFromFlag = newMigrationData?.legacy_cms?.uploadedFile?.file_details?.isSQL === true; + const isSQLFromName = newMigrationData?.legacy_cms?.uploadedFile?.name?.toLowerCase() === 'sql'; + const isSQLFromPath = filePath === 'sql'; + const isSQLFromAwsData = (newMigrationData?.legacy_cms?.uploadedFile?.file_details?.awsData as any)?.mysql !== undefined; + + // SQL connection is true if ANY of these indicators are true + const isSQLConnection = isSQLFromFlag || isSQLFromName || isSQLFromPath || isSQLFromAwsData; + // Get file format from selectedFileFormat or detect from upload response let fileFormat: string = newMigrationData?.legacy_cms?.selectedFileFormat?.title?.toLowerCase(); // If fileFormat is not set, try to detect from upload response - if (!fileFormat && newMigrationData?.legacy_cms?.uploadedFile?.file_details?.isSQL) { + if (!fileFormat && isSQLConnection) { fileFormat = 'sql'; } - if(! isEmptyString(selectedCard?.fileformat_id) && selectedCard?.fileformat_id !== fileFormat && newMigrationData?.project_current_step > 1){ - setFileIcon(selectedCard?.title); + + const { all_cms = [] } = migrationData?.legacyCMSData || {}; + let filteredCmsData: ICMSType[] = all_cms; + if (cmsType) { + filteredCmsData = all_cms?.filter( + (cms) => cms?.parent?.toLowerCase() === cmsType?.toLowerCase() + ); + } + // Special handling for Drupal SQL format + const isDrupal = cmsType?.toLowerCase() === 'drupal'; + + let isFormatValid = false; + + // KEY FIX: Check isSQLConnection instead of comparing fileFormat string + if (isDrupal && isSQLConnection) { + // For Drupal SQL connections, automatically accept the format + isFormatValid = true; } else { - const { all_cms = [] } = migrationData?.legacyCMSData || {}; - let filteredCmsData: ICMSType[] = all_cms; - if (cmsType) { - filteredCmsData = all_cms?.filter( - (cms) => cms?.parent?.toLowerCase() === cmsType?.toLowerCase() - ); - } - - // Special handling for Drupal SQL format - const isDrupal = cmsType?.toLowerCase() === 'drupal'; - const isSQLFormat = fileFormat?.toLowerCase() === 'sql'; - - let isFormatValid = false; - - if (isDrupal && isSQLFormat) { - // For Drupal, automatically accept SQL format - isFormatValid = true; - } else { - // For other CMS types, use the original validation logic - const foundFormat = filteredCmsData[0]?.allowed_file_formats?.find( - (format: ICardType) => { - const isValid = format?.fileformat_id?.toLowerCase() === fileFormat?.toLowerCase(); - return isValid; - } - ); - isFormatValid = !!foundFormat; - } - - if (!isFormatValid) { - setIsError(true); - setError('File format does not support, please add the correct file format.'); - } else { - // Clear any previous errors - setIsError(false); - setError(''); - } - - const selectedFileFormatObj = { - description: '', - fileformat_id: fileFormat, - group_name: fileFormat, - isactive: true, - title: fileFormat === 'zip' ? fileFormat?.charAt?.(0)?.toUpperCase() + fileFormat?.slice?.(1) : fileFormat?.toUpperCase() - } - - // Set file icon based on format - if (isDrupal && isSQLFormat) { - setFileIcon('SQL'); - } else { - setFileIcon(fileFormat === 'zip' ? fileFormat?.charAt?.(0).toUpperCase() + fileFormat?.slice?.(1) : fileFormat === 'directory' ? 'Folder' : fileFormat?.toUpperCase()); - } + // For other CMS types, use the original validation logic + const foundFormat = filteredCmsData[0]?.allowed_file_formats?.find( + (format: ICardType) => { + const isValid = format?.fileformat_id?.toLowerCase() === fileFormat?.toLowerCase(); + return isValid; + } + ); + isFormatValid = !!foundFormat; + } + if (!isFormatValid) { + setIsError(true); + setError('File format does not support, please add the correct file format.'); + } else { + // Clear any previous errors + setIsError(false); + setError(''); } - } catch (error) { + + // For SQL connections, use 'sql' as the format identifier + const displayFormat = isSQLConnection ? 'sql' : fileFormat; + + const selectedFileFormatObj = { + description: '', + fileformat_id: displayFormat, + group_name: displayFormat, + isactive: true, + title: displayFormat === 'zip' + ? displayFormat?.charAt?.(0)?.toUpperCase() + displayFormat?.slice?.(1) + : displayFormat?.toUpperCase() + } + + // Update Redux state with the correct file format + if (isDrupal && isSQLConnection) { + dispatch( + updateNewMigrationData({ + ...newMigrationData, + legacy_cms: { + ...newMigrationData?.legacy_cms, + selectedFileFormat: selectedFileFormatObj + } + }) + ); + } + + // Set file icon - use SQL for SQL connections + if (isDrupal && isSQLConnection) { + setFileIcon('SQL'); + } else { + const iconValue = displayFormat === 'zip' + ? displayFormat?.charAt?.(0).toUpperCase() + displayFormat?.slice?.(1) + : displayFormat === 'directory' + ? 'Folder' + : displayFormat?.toUpperCase(); + setFileIcon(iconValue); + } + } catch (error) { + console.error('โŒ Error in handleFileFormat:', error); return error; } }; @@ -136,6 +166,18 @@ const LoadFileFormat = (props: LoadFileFormatProps) => { useEffect(() => { newMigrationDataRef.current = newMigrationData; + + // Check if SQL connection and update icon immediately + const isSQLFromFlag = newMigrationData?.legacy_cms?.uploadedFile?.file_details?.isSQL === true; + const isSQLFromName = newMigrationData?.legacy_cms?.uploadedFile?.name?.toLowerCase() === 'sql'; + const isSQLFromPath = newMigrationData?.legacy_cms?.uploadedFile?.file_details?.localPath?.toLowerCase() === 'sql'; + const isSQLFromAwsData = (newMigrationData?.legacy_cms?.uploadedFile?.file_details?.awsData as any)?.mysql !== undefined; + const isSQLConnection = isSQLFromFlag || isSQLFromName || isSQLFromPath || isSQLFromAwsData; + const isDrupal = newMigrationData?.legacy_cms?.selectedCms?.parent?.toLowerCase() === 'drupal'; + + if (isDrupal && isSQLConnection) { + setFileIcon('SQL'); + } }, [newMigrationData]); @@ -144,7 +186,12 @@ const LoadFileFormat = (props: LoadFileFormatProps) => {