Refactor Readability parameters to use named arguments and enhance regex support in domain filtering; add tests for regex functionality with demo HTML fixture

Claudio-Emmolo · Claudio-Emmolo · commit 100f21a992f6 · 2025-07-04T16:46:31.000+02:00
diff --git a/README.md b/README.md
@@ -64,8 +64,8 @@ use The3LabsTeam\LaravelReadability\Readability as ReadabilityClass;
 $html = '<html>...</html>';
 $parse = (new ReadabilityClass($html))
 ->getSourceList(
-    $domainWhitelist = ['example.com', 'another-example.com/some-path'],
-    $tagsToExtract = ['a', 'iframe', 'text'] // Optional, default is ['a', 'iframe']
+    domainWhitelist: ['example.com', 'another-example.com/some-path'],
+    tagsToExtract: ['a', 'iframe', 'text'] // Optional, default is ['a', 'iframe']
 )
 ->parse();
 $content = $parsed->getContent();
diff --git a/src/Readability.php b/src/Readability.php
@@ -132,7 +132,7 @@ public function getDirection(): string
     /**
      * Estrae i link da tag specifici e dal testo, filtrando per dominio
      *
-     * @param array $domainWhitelist
+     * @param array $domainWhitelist - List of domains to whitelist, can be regex patterns or simple strings
      * @param array $tagsToExtract ['a', 'iframe', 'img', 'text']
      * @return $this
      */
@@ -172,18 +172,23 @@ public function addSourceList(array $domainWhitelist = [], array $tagsToExtract
             }
         }
 
-        //Rimuovi i link che non hanno https o http
-        $links = array_filter($links, fn($link) =>
-            preg_match('#^(https?://|www\.)#i', $link)
-        );
-
         if (!empty($domainWhitelist)) {
             $filtered = [];
             foreach ($links as $href) {
                 foreach ($domainWhitelist as $domain) {
-                    if (stripos($href, $domain) !== false) {
-                        $filtered[] = $href;
-                        break;
+                    // Check if it's a regex pattern (starts and ends with /)
+                    if (preg_match('/^\/.*\/$/', $domain)) {
+                        // It's a regex pattern
+                        if (preg_match($domain, $href)) {
+                            $filtered[] = $href;
+                            break;
+                        }
+                    } else {
+                        // It's a regular domain string
+                        if (stripos($href, $domain) !== false) {
+                            $filtered[] = $href;
+                            break;
+                        }
                     }
                 }
             }
diff --git a/tests/ReadabilityTest.php b/tests/ReadabilityTest.php
@@ -3,6 +3,8 @@
 use The3LabsTeam\Readability\Facades\Readability;
 use The3LabsTeam\Readability\Readability as ReadabilityClass;
 
+//./vendor/bin/pest tests/ReadabilityTest.php
+
 it('cannot parse a non html', function () {
     $html = 'This is a test';
     expect(function () use ($html) {
@@ -157,4 +159,12 @@
     expect($content)->toContain('https://satoshi.nakamotoinstitute.org');
     expect($content)->toContain('https://www.facebook.com/v3.2/plugins/post');
     expect($content)->toContain('https://www.bitcoin.com/wp-content/uploads/2020/10/bitcoin-whitepaper-featured-image.jpg');
+});
+
+it('can parse and get the content with source list (regex)', function () {
+    $html = file_get_contents(__DIR__.'/fixtures/demo-with-iframe.html');
+    $content = (new ReadabilityClass($html))->addSourceList(['/twitter\.com\/.*\/status\/\d+/'])->parse()->getContent();
+    expect($content)->not->toBeNull();
+    expect($content)->toContain('https://twitter.com/TomsHWItalia/status/1927702682380149224?ref_src=twsrc%5Etfw');
+    expect($content)->not->toContain('https://twitter.com/DA_NON_PRENDERE');
 });
diff --git a/tests/fixtures/demo-with-iframe.html b/tests/fixtures/demo-with-iframe.html
@@ -128,6 +128,10 @@ <h4>October 31, 2008</h4>
             <iframe class="" style="border: none; width: 100%; height: 478px; visibility: visible;" title="fb:post Facebook Social Plugin" src="https://www.facebook.com/v3.2/plugins/post.php?app_id=&amp;channel=https%3A%2F%2Fstaticxx.facebook.com%2Fx%2Fconnect%2Fxd_arbiter%2F%3Fversion%3D46%23cb%3Df59d0d8549941a612%26domain%3Dtomshardware.test%26is_canvas%3Dfalse%26origin%3Dhttp%253A%252F%252Ftomshardware.test%252Ff882f8e4004a0c055%26relation%3Dparent.parent&amp;container_width=834&amp;href=https%3A%2F%2Fwww.facebook.com%2Ftomshardware.it%2Fposts%2Fpfbid09kerg3CPZNbDVZvTJeqBKVjuDYuXwNTRNtV2zycQD1BrJ5tvbVy6FLPLkaz4FR5nl&amp;lazy=true&amp;locale=en_US&amp;sdk=joey&amp;show_text=true&amp;width=auto" name="f1b548f199a063ba9" height="1000px" frameborder="0" scrolling="no" allow="encrypted-media" allowfullscreen="allowfullscreen" loading="lazy" data-testid="fb:post Facebook Social Plugin"></iframe>
 
             <iframe class="" src="https://www.facebook.com/posts/18dwdwxcowmkd"></iframe>
+            <iframe class="" src="https://twitter.com/TomsHWItalia/status/1927702682380149224?ref_src=twsrc%5Etfw"></iframe>
+            <iframe class="" src="https://twitter.com/DA_NON_PRENDERE?ref_src=twsrc%5Etfw"></iframe>
+
+
 
             <img class="img-fluid d-block mx-auto my-4" src="https://www.bitcoin.com/wp-content/uploads/2020/10/bitcoin-whitepaper-featured-image.jpg">
 
