♻️ rework tests structure

Author: ianardee

.github/workflows/_test-integrations.yml

@@ -12,7 +12,6 @@ jobs:
     name: Run Integration Tests
     timeout-minutes: 30
     strategy:
-      max-parallel: 4
       matrix:
         os:
           - "ubuntu-latest"

.github/workflows/_test-units.yml

@@ -10,7 +10,6 @@ jobs:
   run-tests:
     name: Run Tests
     strategy:
-      max-parallel: 4
       matrix:
         os:
           - "ubuntu-latest"

tests/data

@@ -4,11 +4,11 @@ import path from "path";
 import { InvoiceSplitterV1 } from "../../src/product";
 import { extractInvoices } from "../../src/imageOperations";
 import { PathInput } from "../../src";
-import { RESOURCE_PATH } from "../index";
+import { V1_PRODUCT_PATH } from "../index";
 
 const dataPath = {
-  complete: path.join(RESOURCE_PATH, "products/invoice_splitter/response_v1/complete.json"),
-  fileSample: path.join(RESOURCE_PATH, "products/invoice_splitter/invoice_5p.pdf"),
+  complete: path.join(V1_PRODUCT_PATH, "invoice_splitter/response_v1/complete.json"),
+  fileSample: path.join(V1_PRODUCT_PATH, "invoice_splitter/invoice_5p.pdf"),
 };
 
 describe("A multi-page invoice document", () => {

tests/imageOperations/invoiceSplitterExtractor.spec.ts

@@ -4,13 +4,13 @@ import path from "path";
 import { MultiReceiptsDetectorV1 } from "../../src/product";
 import { extractReceipts } from "../../src/imageOperations";
 import { PathInput } from "../../src";
-import { RESOURCE_PATH } from "../index";
+import { V1_PRODUCT_PATH } from "../index";
 
 const dataPath = {
-  complete: path.join(RESOURCE_PATH, "products/multi_receipts_detector/response_v1/complete.json"),
-  fileSample: path.join(RESOURCE_PATH, "products/multi_receipts_detector/default_sample.jpg"),
-  completeMultiPage: path.join(RESOURCE_PATH, "products/multi_receipts_detector/response_v1/multipage_sample.json"),
-  multiPageSample: path.join(RESOURCE_PATH, "products/multi_receipts_detector/multipage_sample.pdf"),
+  complete: path.join(V1_PRODUCT_PATH, "multi_receipts_detector/response_v1/complete.json"),
+  fileSample: path.join(V1_PRODUCT_PATH, "multi_receipts_detector/default_sample.jpg"),
+  completeMultiPage: path.join(V1_PRODUCT_PATH, "multi_receipts_detector/response_v1/multipage_sample.json"),
+  multiPageSample: path.join(V1_PRODUCT_PATH, "multi_receipts_detector/multipage_sample.pdf"),
 };
 
 describe("A single-page multi-receipts document", () => {

tests/imageOperations/multiReceiptsExtractor.spec.ts

@@ -0,0 +1,77 @@
+const fs = require("fs");
+const path = require("path");
+const axios = require("axios");
+const FormData = require("form-data");
+
+async function sendFileWithPolling(
+  filePath,
+  modelId,
+  apiKey,
+  maxRetries = 30,
+  pollingInterval = 2
+) {
+  const fileName = path.basename(filePath);
+  const headers = {
+    "Authorization": apiKey
+  };
+
+  const formData = new FormData();
+  formData.append("model_id", modelId);
+  formData.append("rag", "false");
+  formData.append("file", fs.createReadStream(filePath), {
+    filename: fileName,
+    contentType: "application/octet-stream"
+  });
+
+  console.log(`Enqueuing file: ${filePath}`);
+  const response = await axios.post(
+    "https://api-v2.mindee.net/v2/inferences/enqueue",
+    formData,
+    {headers: { ...headers, ...formData.getHeaders() }}
+  );
+
+  const jobData = response.data.job;
+  const pollingUrl = jobData.polling_url;
+
+  await new Promise(resolve => setTimeout(resolve, 3000));
+
+  for (let attempt = 0; attempt < maxRetries; attempt++) {
+    console.log(`Polling on: ${pollingUrl}`);
+
+    const pollResponse = await axios.get(pollingUrl, {
+      headers,
+      maxRedirects: 0,
+      validateStatus: status => status >= 200 && status < 400
+    });
+
+    const pollData = pollResponse.data;
+    const jobStatus = pollData.job?.status;
+
+    if (pollResponse.status === 302 || jobStatus === "Processed") {
+      const resultUrl = pollData.job?.result_url;
+      console.log(`Get result from: ${resultUrl}`);
+
+      const resultResponse = await axios.get(resultUrl, { headers });
+      return resultResponse.data;
+    }
+
+    await new Promise(resolve => setTimeout(resolve, pollingInterval * 1000));
+  }
+
+  throw new Error(`Polling timed out after ${maxRetries} attempts`);
+}
+
+async function main() {
+  try {
+    const result = await sendFileWithPolling(
+      "/home/ianardee/Downloads/Invoice-5Z69WRLL-0003.pdf",
+      "3b87142f-0eb4-4e6d-92a8-1ed01f097052",
+      "md_xcygjjg8ehfaf2jfeyzomxghvhzh4kzs"
+    );
+    console.log(result);
+  } catch (error) {
+    console.error("Error:", error.message);
+  }
+}
+
+main();

tests/in.js

@@ -1,5 +1,9 @@
 import path from "node:path";
 
 export const RESOURCE_PATH = path.join(__dirname, "data");
+
 export const V1_RESOURCE_PATH = path.join(RESOURCE_PATH, "v1");
+export const V1_PRODUCT_PATH = path.join(V1_RESOURCE_PATH, "products");
+
 export const V2_RESOURCE_PATH = path.join(RESOURCE_PATH, "v2");
+export const V2_PRODUCT_PATH = path.join(V2_RESOURCE_PATH, "products");

tests/index.ts

@@ -19,7 +19,7 @@ import { compressImage } from "../../src/imageOperations";
 import { compressPdf } from "../../src/pdf";
 import { extractTextFromPdf } from "../../src/pdf/pdfUtils";
 import { logger } from "../../src/logger";
-import { RESOURCE_PATH } from "../index";
+import { RESOURCE_PATH, V1_PRODUCT_PATH } from "../index";
 
 describe("Test different types of input", () => {
   const outputPath = path.join(RESOURCE_PATH, "output");
@@ -55,12 +55,12 @@ describe("Test different types of input", () => {
 
   it("should accept JPEG files from a path", async () => {
     const inputSource = new PathInput({
-      inputPath: path.join(RESOURCE_PATH, "products/expense_receipts/default_sample.jpg"),
+      inputPath: path.join(V1_PRODUCT_PATH, "expense_receipts/default_sample.jpg"),
     });
     await inputSource.init();
 
     const expectedResult = await fs.promises.readFile(
-      path.join(RESOURCE_PATH, "products/expense_receipts/default_sample.jpg")
+      path.join(V1_PRODUCT_PATH, "expense_receipts/default_sample.jpg")
     );
     expect(inputSource.inputType).to.equals(INPUT_TYPE_PATH);
     expect(inputSource.filename).to.equals("default_sample.jpg");
@@ -103,7 +103,7 @@ describe("Test different types of input", () => {
   });
 
   it("should accept read streams", async () => {
-    const filePath = path.join(RESOURCE_PATH, "products/expense_receipts/default_sample.jpg");
+    const filePath = path.join(V1_PRODUCT_PATH, "expense_receipts/default_sample.jpg");
     const stream = fs.createReadStream(filePath);
     const filename = "default_sample.jpg";
     const inputSource = new StreamInput({
@@ -121,7 +121,7 @@ describe("Test different types of input", () => {
   });
 
   it("should accept raw bytes", async () => {
-    const filePath = path.join(RESOURCE_PATH, "products/expense_receipts/default_sample.jpg");
+    const filePath = path.join(V1_PRODUCT_PATH, "expense_receipts/default_sample.jpg");
     const inputBytes = await fs.promises.readFile(filePath);
     // don't provide an extension to see if we can detect MIME
     // type based on contents
@@ -144,7 +144,7 @@ describe("Test different types of input", () => {
     const filename = "invoice_01.pdf";
     const buffer = Buffer.from(
       await fs.promises.readFile(
-        path.join(RESOURCE_PATH, "products/invoices/invoice_10p.pdf")
+        path.join(V1_PRODUCT_PATH, "invoices/invoice_10p.pdf")
       )
     );
     const inputSource = new BufferInput({
@@ -257,7 +257,7 @@ describe("Test different types of input", () => {
 
   it("PDF Compress From InputSource", async () => {
     const pdfResizeInput = new PathInput(
-      { inputPath: path.join(RESOURCE_PATH, "products/invoice_splitter/default_sample.pdf") }
+      { inputPath: path.join(V1_PRODUCT_PATH, "invoice_splitter/default_sample.pdf") }
     );
     await pdfResizeInput.init();
 
@@ -267,10 +267,7 @@ describe("Test different types of input", () => {
     await fs.promises.writeFile(path.join(outputPath, "resize_indirect.pdf"), compressedPdf);
 
     const initialFileStats = await fs.promises.stat(
-      path.join(
-        RESOURCE_PATH,
-        "products/invoice_splitter/default_sample.pdf"
-      )
+      path.join(V1_PRODUCT_PATH, "invoice_splitter/default_sample.pdf")
     );
     const renderedFileStats = await fs.promises.stat(
       path.join(outputPath, "resize_indirect.pdf")
@@ -280,7 +277,7 @@ describe("Test different types of input", () => {
 
   it("PDF Compress From Compressor", async () => {
     const pdfResizeInput = new PathInput(
-      { inputPath: path.join(RESOURCE_PATH, "products/invoice_splitter/default_sample.pdf") }
+      { inputPath: path.join(V1_PRODUCT_PATH, "invoice_splitter/default_sample.pdf") }
     );
     await pdfResizeInput.init();
 
@@ -297,7 +294,7 @@ describe("Test different types of input", () => {
     }
 
     const initialFileStats = await fs.promises.stat(
-      path.join(RESOURCE_PATH, "products/invoice_splitter/default_sample.pdf")
+      path.join(V1_PRODUCT_PATH, "invoice_splitter/default_sample.pdf")
     );
     const renderedFileStats = await Promise.all(
       fileNames.map(fileName => fs.promises.stat(path.join(outputPath, fileName)))

tests/input/sources.spec.ts

@@ -5,7 +5,7 @@ import * as pdf from "../../src/pdf";
 import { PageOptions } from "../../src/input";
 import { PageOptionsOperation } from "../../src";
 import * as fs from "node:fs";
-import {RESOURCE_PATH} from "../index";
+import { RESOURCE_PATH } from "../index";
 
 describe("Test pdf lib", () => {
   let client: mindee.Client;

tests/pdf/pdfTypes.spec.ts

@@ -10,7 +10,7 @@ import { RESOURCE_PATH } from "../../index";
 describe("MindeeV1 - Asynchronous API predict response", () => {
   it("should parse a successful enqueue", async () => {
     const jsonData = await fs.readFile(
-      path.join(RESOURCE_PATH, "async/post_success.json")
+      path.join(RESOURCE_PATH, "v1/async/post_success.json")
     );
     const httpResponse: StringDict = {
       data: JSON.parse(jsonData.toString()),
@@ -26,7 +26,7 @@ describe("MindeeV1 - Asynchronous API predict response", () => {
 
   it("should parse a failed enqueue", async () => {
     const jsonData = await fs.readFile(
-      path.join(RESOURCE_PATH, "async/post_fail_forbidden.json")
+      path.join(RESOURCE_PATH, "v1/async/post_fail_forbidden.json")
     );
     const httpResponse: StringDict = {
       data: JSON.parse(jsonData.toString()),
@@ -36,7 +36,7 @@ describe("MindeeV1 - Asynchronous API predict response", () => {
 
   it("should parse a failed job", async () => {
     const jsonData = await fs.readFile(
-      path.join(RESOURCE_PATH, "async/get_failed_job_error.json")
+      path.join(RESOURCE_PATH, "v1/async/get_failed_job_error.json")
     );
     const httpResponse: StringDict = {
       data: JSON.parse(jsonData.toString()),
@@ -46,7 +46,7 @@ describe("MindeeV1 - Asynchronous API predict response", () => {
 
   it("should parse a job in progress", async () => {
     const jsonData = await fs.readFile(
-      path.join(RESOURCE_PATH, "async/get_processing.json")
+      path.join(RESOURCE_PATH, "v1/async/get_processing.json")
     );
     const httpResponse: StringDict = {
       data: JSON.parse(jsonData.toString()),
@@ -62,7 +62,7 @@ describe("MindeeV1 - Asynchronous API predict response", () => {
 
   it("should parse a completed job", async () => {
     const jsonData = await fs.readFile(
-      path.join(RESOURCE_PATH, "async/get_completed.json")
+      path.join(RESOURCE_PATH, "v1/async/get_completed.json")
     );
     const httpResponse: StringDict = {
       data: JSON.parse(jsonData.toString()),