Merge branch 'bugfix/446462_upload_large_files' into 'develop'

bugfix/446462_upload_large_files

See merge request upm-inesdata/inesdata-connector-interface!39

Author: ralconada-gmv

package-lock.json

@@ -27,11 +27,11 @@
     "@angular/router": "^17.1.2",
     "@auth0/angular-jwt": "^5.2.0",
     "@ckeditor/ckeditor5-angular": "^7.0.1",
-    "@ckeditor/ckeditor5-build-classic": "^41.4.2",
-    "@ckeditor/ckeditor5-core": "^41.4.2",
-    "@ckeditor/ckeditor5-engine": "^41.4.2",
-    "@ckeditor/ckeditor5-utils": "^41.4.2",
-    "@ckeditor/ckeditor5-watchdog": "^41.4.2",
+    "@ckeditor/ckeditor5-build-classic": "^43.1.0",
+    "@ckeditor/ckeditor5-core": "^43.1.0",
+    "@ckeditor/ckeditor5-engine": "^43.1.0",
+    "@ckeditor/ckeditor5-utils": "^43.1.0",
+    "@ckeditor/ckeditor5-watchdog": "^43.1.0",
     "@jsonforms/angular": "^3.2.1",
     "@jsonforms/angular-material": "^3.2.1",
     "@jsonforms/core": "^3.2.1",

package.json

@@ -59,3 +59,16 @@ mat-card-actions {
 :host ::ng-deep .mat-expansion-indicator {
   display: none;
 }
+
+.toggle {
+  margin-bottom: 22px;
+  --mat-mdc-form-field-floating-label-scale: 0.75;
+  display: inline-flex;
+  flex-direction: column;
+  min-width: 0;
+  text-align: left;
+}
+
+:host ::ng-deep .toggle .mdc-form-field{
+  font-size: 16px;
+}

src/app/pages/assets/asset-create/asset-create.component.html

@@ -1,6 +1,5 @@
 import { Component, Inject, OnInit } from '@angular/core';
-import { HttpDataAddress, DataAddress, AssetInput } from '@think-it-labs/edc-connector-client';
-import { MatDialogRef } from "@angular/material/dialog";
+import { HttpDataAddress, DataAddress } from '@think-it-labs/edc-connector-client';
 import { JsonDoc } from "../../../shared/models/json-doc";
 import { StorageType } from "../../../shared/models/storage-type";
 import { AmazonS3DataAddress } from "../../../shared/models/amazon-s3-data-address";
@@ -89,7 +88,7 @@ export class AssetCreateComponent implements OnInit {
     type: 'InesDataStore'
   };
 
-  assetType:any;
+  assetType: any;
   assetTypes = Object.entries(ASSET_TYPES);
   defaultForms: JsonFormData[]
   selectedForms: JsonFormData[]
@@ -101,7 +100,7 @@ export class AssetCreateComponent implements OnInit {
   config = CKEDITOR_CONFIG
   selectedAssetTypeVocabularies: Vocabulary[]
 
-  urlPattern: RegExp = /^(file|ftp|http|https|imap|irc|nntp|acap|icap|mtqp|wss):\/\/(localhost|([a-z\d]([a-z\d-]*[a-z\d])*)|(([a-z\d]([a-z\d-]*[a-z\d])*)\.)+[a-z]{2,}|((\d{1,3}\.){3}\d{1,3}))(:\d+)?(\/[-a-z\d%_.~+]*)*(\?[;&a-z\d%_.~+=-]*)?(\#[-a-z\d_]*)?$/i;
+  urlPattern: RegExp = /^https?:\/\/(www\.)?[-a-zA-Z0-9@:%._+~#=]{1,256}\.[a-zA-Z]{2,6}\b([-a-zA-Z0-9()@:%_+.~#?&//=]*)$/;
 
   private fetch$ = new BehaviorSubject(null);
 
@@ -122,7 +121,7 @@ export class AssetCreateComponent implements OnInit {
         }
         if (this.selectedVocabularies?.length > 0) {
           this.selectedVocabularies.forEach(s => {
-            if(this.selectedAssetTypeVocabularies.find(satv=> satv['@id'] === s['@id'])){
+            if (this.selectedAssetTypeVocabularies.find(satv => satv['@id'] === s['@id'])) {
               this.initVocabularyForm(s, false)
             }
           })
@@ -168,7 +167,7 @@ export class AssetCreateComponent implements OnInit {
     const forms: JsonFormData[] = [...this.defaultForms, ...this.selectedForms]
 
     let assetDataProperty: any = {}
-    forms.forEach(async f=>{
+    forms.forEach(async f => {
       if (f.schema && f.schema.hasOwnProperty("@context")) {
         // Add context if it is provided in the Json Schema
         const jsonSchema: JsonDoc = f.schema as JsonDoc;
@@ -197,7 +196,7 @@ export class AssetCreateComponent implements OnInit {
       dataAddress = this.amazonS3DataAddress;
     } else if (this.storageTypeId === DATA_ADDRESS_TYPES.httpData) {
       dataAddress = this.httpDataAddress;
-    }  else if (this.storageTypeId === DATA_ADDRESS_TYPES.inesDataStore) {
+    } else if (this.storageTypeId === DATA_ADDRESS_TYPES.inesDataStore) {
       dataAddress = this.inesDataStoreAddress;
     } else {
       this.notificationService.showError("Incorrect destination value");
@@ -213,6 +212,7 @@ export class AssetCreateComponent implements OnInit {
     };
 
     if (this.storageTypeId === DATA_ADDRESS_TYPES.inesDataStore && this.inesDataStoreAddress?.file) {
+      this.loadingService.showLoading('Processing the file...');
       const file = this.inesDataStoreAddress?.file;
 
       const chunkSize = 1024 * 1024;
@@ -229,7 +229,7 @@ export class AssetCreateComponent implements OnInit {
       assetInput.blob = new Blob(chunks);
     }
 
-    this.createAsset(assetInput)
+    await this.createAsset(assetInput)
   }
   addInfoProperties(properties: JsonDoc) {
     // Add default information
@@ -245,7 +245,7 @@ export class AssetCreateComponent implements OnInit {
     this.addKeywords(properties);
   }
 
-  addKeywords(properties: JsonDoc){
+  addKeywords(properties: JsonDoc) {
     const parsedKeywords: string[] = [];
     this.keywords.split(",").forEach(keyword => parsedKeywords.push(keyword.trim()));
     properties["dcat:keyword"] = parsedKeywords;
@@ -286,7 +286,7 @@ export class AssetCreateComponent implements OnInit {
     if (!this.id || !this.storageTypeId || !this.name || !this.version || !this.description || !this.keywords || !this.shortDescription || !this.assetType) {
       return false;
     } else {
-      if (this.storageTypeId === DATA_ADDRESS_TYPES.httpData && (!this.httpDataAddress.name || !this.httpDataAddress.baseUrl  || !this.validateUrl())) {
+      if (this.storageTypeId === DATA_ADDRESS_TYPES.httpData && (!this.httpDataAddress.name || !this.httpDataAddress.baseUrl || !this.validateUrl())) {
         return false;
       }
       if (this.storageTypeId === DATA_ADDRESS_TYPES.amazonS3 && !this.amazonS3DataAddress.region) {
@@ -328,7 +328,7 @@ export class AssetCreateComponent implements OnInit {
 
     if (this.selectedVocabularies.length > 0) {
       this.selectedVocabularies.forEach(s => {
-        if(this.selectedAssetTypeVocabularies.find(satv=> satv['@id'] === s['@id'])){
+        if (this.selectedAssetTypeVocabularies.find(satv => satv['@id'] === s['@id'])) {
           this.initVocabularyForm(s, false)
         }
       })
@@ -339,24 +339,24 @@ export class AssetCreateComponent implements OnInit {
    * Transform to text asset type value
    * @returns asset type text
    */
-  getAssetTypeText(){
-    return this.assetType?ASSET_TYPES[this.assetType as keyof typeof ASSET_TYPES]:'';
+  getAssetTypeText() {
+    return this.assetType ? ASSET_TYPES[this.assetType as keyof typeof ASSET_TYPES] : '';
   }
 
-  setFiles(event:File[]){
-    if(event?.length>0){
+  setFiles(event: File[]) {
+    if (event?.length > 0) {
       this.inesDataStoreAddress.file = event[0]
-    }else{
+    } else {
       delete this.inesDataStoreAddress.file
     }
   }
 
-  onSelectionChangeVocabulary(){
+  onSelectionChangeVocabulary() {
     this.selectedForms = []
 
     if (this.selectedVocabularies.length > 0) {
       this.selectedVocabularies.forEach(s => {
-        if(this.selectedAssetTypeVocabularies.find(satv=> satv['@id'] === s['@id'])){
+        if (this.selectedAssetTypeVocabularies.find(satv => satv['@id'] === s['@id'])) {
           this.initVocabularyForm(s, false)
         }
       })
@@ -366,46 +366,76 @@ export class AssetCreateComponent implements OnInit {
   validateUrl(): boolean {
     const regex = new RegExp(this.urlPattern);
     return regex.test(this.httpDataAddress.baseUrl);
-}
+  }
 
 
-private createAsset(asset: AssetInput){
-    const newAsset = asset;
-    if (newAsset) {
-      if (newAsset.dataAddress.type !== DATA_ADDRESS_TYPES.inesDataStore) {
-        this.assetService.createAsset(newAsset).subscribe({
-          next: () => this.fetch$.next(null),
-          error: err => this.showError(err, "This asset cannot be created"),
-          complete: () => {
-            this.navigateToAsset()
-            this.notificationService.showInfo("Successfully created");
-            this.loadingService.hideLoading();
-          }
-        })
-      } else {
-        this.assetService.createStorageAsset(newAsset).subscribe({
-          next: () => this.fetch$.next(null),
-          error: err => this.showError(err, "This asset cannot be created"),
-          complete: () => {
-            this.navigateToAsset()
-            this.notificationService.showInfo("Successfully created");
-            this.loadingService.hideLoading();
+  async createAsset(assetInput: any) {
+    if (this.storageTypeId === DATA_ADDRESS_TYPES.inesDataStore && this.inesDataStoreAddress.file) {
+      const file = this.inesDataStoreAddress.file;
+      const chunkSize = 50 * 1024 * 1024; // 50 MB
+      const totalChunks = Math.ceil(file.size / chunkSize);
+      const fileName = file.name;
+      const maxRetries = 3;
+
+      for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++) {
+        const start = chunkIndex * chunkSize;
+        const chunk = file.slice(start, start + chunkSize);
+
+        let attempt = 0;
+        let success = false;
+
+        const progressPercentage = Math.floor(((chunkIndex + 1) / totalChunks) * 100);
+
+        while (attempt < maxRetries && !success) {
+          try {
+            this.loadingService.updateMessage(`Uploading file: ${progressPercentage}% completed`);
+
+            await this.assetService.uploadChunk(assetInput, chunk, fileName, chunkIndex, totalChunks);
+            success = true;
+          } catch (error) {
+            attempt++;
+            if (attempt >= maxRetries) {
+              this.loadingService.hideLoading();
+              this.notificationService.showError(`Error uploading chunk ${chunkIndex + 1}. Maximum retries reached.`);
+              return;
+            }
           }
-        })
+        }
       }
 
+      try {
+        await this.assetService.finalizeUpload(assetInput, fileName);
+        this.loadingService.hideLoading();
+        this.notificationService.showInfo('Asset created successfully');
+        this.navigateToAsset();
+      } catch (error: any) {
+        this.loadingService.hideLoading();
+        this.notificationService.showError('Error finalizing the asset creation: ' + error.error[0].message);
+      }
+    } else {
+      this.assetService.createAsset(assetInput).subscribe({
+        next: () => this.fetch$.next(null),
+        error: (err) => {
+          this.loadingService.hideLoading();
+          this.showError(err, "Error creating the asset: " + err.error[0].message);
+        },
+        complete: () => {
+          this.loadingService.hideLoading();
+          this.notificationService.showInfo('Asset created successfully');
+          this.navigateToAsset();
+        },
+      });
     }
   }
 
 
-
   private showError(error: string, errorMessage: string) {
     this.notificationService.showError(errorMessage);
     console.error(error);
     this.loadingService.hideLoading();
   }
 
-  navigateToAsset(){
+  navigateToAsset() {
     this.router.navigate(['assets'])
   }
 }

src/app/pages/assets/asset-create/asset-create.component.scss

@@ -1,15 +1,12 @@
 import { Component, OnInit } from '@angular/core';
 import { BehaviorSubject } from 'rxjs';
-import { first } from 'rxjs/operators';
 import { MatDialog } from '@angular/material/dialog';
-import { AssetInput, Asset } from "../../../shared/models/edc-connector-entities";
+import { Asset } from "../../../shared/models/edc-connector-entities";
 import { AssetService } from "../../../shared/services/asset.service";
 import { ConfirmationDialogComponent, ConfirmDialogModel } from "../../../shared/components/confirmation-dialog/confirmation-dialog.component";
 import { NotificationService } from "../../../shared/services/notification.service";
-import { CONTEXTS, DATA_ADDRESS_TYPES } from 'src/app/shared/utils/app.constants';
 import { PageEvent } from '@angular/material/paginator';
-import { EDC_CONTEXT, QuerySpec, DataAddress } from '@think-it-labs/edc-connector-client';
-import { ContractOffersViewerComponent } from '../../catalog/contract-offers-viewer/contract-offers-viewer.component';
+import { EDC_CONTEXT, QuerySpec } from '@think-it-labs/edc-connector-client';
 import { compact } from 'jsonld';
 import { Router } from '@angular/router';
 
@@ -67,7 +64,7 @@ export class AssetViewerComponent implements OnInit {
         if (res) {
           this.assetService.removeAsset(asset.id).subscribe({
             next: () => this.fetch$.next(null),
-            error: err => this.showError(err, "This asset cannot be deleted"),
+            error: err => this.showError(err, "This asset cannot be deleted: " + err.error[0].message),
             complete: () => {
               this.countAssets();
               this.loadAssets(this.currentPage);

src/app/pages/assets/asset-create/asset-create.component.ts

@@ -1,12 +1,9 @@
-import { Component, Inject, OnInit } from '@angular/core';
+import { Component, OnInit } from '@angular/core';
 import { MatDialog } from '@angular/material/dialog';
 import { CatalogBrowserService } from "../../../shared/services/catalog-browser.service";
 import { DataOffer } from 'src/app/shared/models/data-offer';
-import { ContractOffersViewerComponent } from '../contract-offers-viewer/contract-offers-viewer.component';
-import { Policy } from 'src/app/shared/models/edc-connector-entities';
 import { PageEvent } from '@angular/material/paginator';
 import { QuerySpec } from '@think-it-labs/edc-connector-client';
-import { ContractOffer } from 'src/app/shared/models/contract-offer';
 import { Router } from '@angular/router';
 
 

src/app/pages/assets/asset-viewer/asset-viewer.component.ts

@@ -21,6 +21,8 @@ mat-list-item:hover .property-title,
 .property-value {
   font-size: 14px;
   color: #d7d7d7;
+  overflow-wrap: anywhere;
+  white-space: pre-wrap;
 }
 
 .property-icon {
@@ -102,11 +104,6 @@ ul {
   padding: 20px;
 }
 
-.property-value {
-  overflow-wrap: anywhere;
-  white-space: pre-wrap;
-}
-
 .grey {
   margin-bottom: 15px;
 }

src/app/pages/catalog/catalog-browser/catalog-browser.component.ts

@@ -1,5 +1,5 @@
 import { Component, Inject } from '@angular/core';
-import { MatDialog, MatDialogRef, MAT_DIALOG_DATA } from '@angular/material/dialog';
+import { MatDialog } from '@angular/material/dialog';
 import { TransferProcessStates } from "../../../shared/models/transfer-process-states";
 import { NegotiationResult } from "../../../shared/models/negotiation-result";
 import { ContractNegotiation, ContractNegotiationRequest, Policy } from "../../../shared/models/edc-connector-entities";
@@ -8,7 +8,6 @@ import { NotificationService } from 'src/app/shared/services/notification.servic
 import { StorageType } from 'src/app/shared/models/storage-type';
 import { PolicyCard } from '../../../shared/models/policy/policy-card';
 import { DATA_ADDRESS_TYPES } from '../../../shared/utils/app.constants';
-import { ContractOffer } from 'src/app/shared/models/contract-offer';
 import { PolicyCardBuilder } from 'src/app/shared/models/policy/policy-card-builder';
 import { JsonDialogData } from '../../json-dialog/json-dialog/json-dialog.data';
 import { JsonDialogComponent } from '../../json-dialog/json-dialog/json-dialog.component'
@@ -218,9 +217,13 @@ export class ContractOffersViewerComponent {
           if (finishedNegotiationStates.includes(updatedNegotiation.state)) {
             let offerId = negotiation.offerId;
             this.runningNegotiations.delete(offerId);
+            const errorDetail = updatedNegotiation.optionalValue("edc", "errorDetail");
             if (updatedNegotiation["state"] === "VERIFIED" || updatedNegotiation["state"] === "FINALIZED") {
               this.finishedNegotiations.set(offerId, updatedNegotiation);
               this.notificationService.showInfo("Contract Negotiation complete!");
+            } else if (updatedNegotiation["state"] === "TERMINATED" && typeof errorDetail === 'string' &&  errorDetail.includes("Contract offer is not valid")) {
+              this.finishedNegotiations.set(offerId, updatedNegotiation);
+              this.notificationService.showError("Contract offer is not valid.");
             }
           }
 

src/app/pages/catalog/contract-offers-viewer/contract-offers-viewer.component.scss

@@ -1,5 +1,4 @@
-import { Component, Inject, OnInit } from '@angular/core';
-import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog';
+import { Component, OnInit } from '@angular/core';
 import { AssetService } from "../../../shared/services/asset.service";
 import { PolicyService } from "../../../shared/services/policy.service";
 import { Asset, PolicyDefinition, ContractDefinitionInput } from "../../../shared/models/edc-connector-entities";