Merge pull request #43 from FriendsOfREDAXO/socket_to_httpclient

rex_socket to http client - mehr debugging bei fehlern

Author: dergel

composer.json

@@ -1,12 +1,13 @@
 {
     "license": "MIT",
     "require": {
-        "jfcherng/php-diff": "^6.15",
+        "jfcherng/php-diff": "^6",
         "html2text/html2text": "^4.3",
         "voku/simple_html_dom": "^4.8",
-        "laminas/laminas-feed": "^2.20",
+        "laminas/laminas-feed": "^2",
         "mkalkbrenner/php-htmldiff-advanced": "^0.0.8",
-        "ezyang/htmlpurifier": "^4.16"
+        "ezyang/htmlpurifier": "*",
+        "symfony/http-client": "^7.3"
     },
     "replace": {
         "psr/container": "*",

composer.lock

@@ -15,6 +15,7 @@
     ->ensureColumn(new rex_sql_column('categories', 'text', false, ''))
     ->ensureColumn(new rex_sql_column('status', 'tinyint', false, '0'))
     ->ensureColumn(new rex_sql_column('interval', 'int(10) unsigned', false, '1440'))
+    ->ensureColumn(new rex_sql_column('last_message', 'text', false, ''))
     ->ensureColumn(new rex_sql_column('http_auth_login', 'VARCHAR(100)', true))
     ->ensureColumn(new rex_sql_column('http_auth_password', 'VARCHAR(100)', true))
     ->ensureColumn(new rex_sql_column('last_scan', 'datetime', true))

install.php

@@ -71,6 +71,7 @@ diff_detect_snapshot_error = Snapshot von `{0}` konnte nicht erstellt werden. Fe
 back_to_snapshots = Zurück zu Snapshots
 last_scan = Letzter Scanversuch
 last_snapshot = Letzter Snapshot
+last_message = Letzte Scaninfo
 
 interval_in_min_5 = 5 Minuten
 interval_in_min_15 = 15 Minuten

lang/de_de.lang

@@ -0,0 +1,73 @@
+<?php
+
+namespace FriendsOfRedaxo\DiffDetect\Command;
+
+use FriendsOfRedaxo\DiffDetect\Index;
+use FriendsOfRedaxo\DiffDetect\Url;
+use rex;
+use rex_console_command;
+use rex_exception;
+use rex_sql;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+use function count;
+
+class Console extends rex_console_command
+{
+    protected function execute(InputInterface $input, OutputInterface $output): int
+    {
+        $io = $this->getStyle($input, $output);
+        $io->title('DiffDetect Console Command');
+
+        $sql = rex_sql::factory();
+        $URLs = $sql->getArray(
+            '
+            SELECT      u.*
+                        , i.createdate AS last_index_date
+            FROM        ' . rex::getTable('diff_detect_url') . ' u
+            LEFT JOIN   (
+                SELECT  url_id, MAX(createdate) createdate
+                FROM    ' . rex::getTable('diff_detect_index') . '
+                GROUP BY url_id
+                ) i
+            ON          u.id = i.url_id
+            WHERE       u.status = 1
+            AND         (
+                u.last_scan IS NULL
+                OR  u.last_scan < DATE_SUB(:datetime, INTERVAL u.interval MINUTE)
+            )
+            order by u.last_scan
+            LIMIT 5
+        ',
+            [
+                'datetime' => date(rex_sql::FORMAT_DATETIME),
+            ],
+        );
+
+        foreach ($URLs as $URLArray) {
+            $Url = Url::get($URLArray['id']);
+            try {
+                if (Index::createSnapshot($Url)) {
+                    $io->success('Snapshot created for URL: ' . $Url->getName() . ' / ' . $Url->getUrl() . ' [' . $Url->getId() . ']');
+                } else {
+                    $io->success('Snapshot NOT created for URL: ' . $Url->getName() . ' / ' . $Url->getUrl() . ' [' . $Url->getId() . ']');
+                }
+            } catch (rex_exception $e) {
+                $io->error('Snapshot error for URL: ' . $Url->getName() . ' / ' . $Url->getUrl() . ' [' . $Url->getId() . ']');
+                break;
+            }
+        }
+
+        if (0 === count($URLs)) {
+            $io->info('no snapshots');
+        }
+
+        $io->text('Total URLs processed: ' . count($URLs));
+        $io->text('');
+
+        Index::cleanUpSnapshots();
+
+        return 1;
+    }
+}

lib/Command/Console.php

@@ -2,6 +2,7 @@
 
 namespace FriendsOfRedaxo\DiffDetect;
 
+use Exception;
 use Html2Text\Html2Text;
 use InvalidArgumentException;
 use rex;
@@ -10,7 +11,9 @@
 use rex_instance_pool_trait;
 use rex_sql;
 use rex_sql_exception;
-use voku\helper\HtmlDomParser;
+
+use function is_array;
+use function sprintf;
 
 final class Index
 {
@@ -27,7 +30,7 @@ private function __construct(int $id)
     }
 
     /**
-     * @return null|static
+     * @return static|null
      */
     public static function get(int $id): ?self
     {
@@ -103,18 +106,28 @@ private static function fromSqlData(array $data): self
     public static function createSnapshot(Url $url): bool
     {
         $url->setLastScan();
-        $response = $url->getContent();
-        $content = $response->getBody();
+        $content = '';
+        $headers = [];
+        try {
+            $response = $url->getResponse();
+            $content = $response['Content'] ?? '';
+            $headers = $response['Headers'] ?? [];
+            $statusCode = $response['StatusCode'] ?? 0;
+            $statusMessage = '[' . $statusCode . '] OK';
+        } catch (Exception $e) {
+            $statusCode = $e->getCode();
+            $statusMessage = '[' . $statusCode . '] ' . $e->getMessage();
+        }
+
+        $url->setLastMessage($statusMessage);
+
+        $headers = self::flattenArray($headers);
 
         if ('HTML' === $url->getType()) {
-            $content = preg_replace('/<script\b[^>]*>(.*?)<\/script>/is', '', $content);
-            $content = preg_replace('/<style\b[^>]*>(.*?)<\/style>/is', '', $content);
-            $content = preg_replace('/<noscript\b[^>]*>(.*?)<\/noscript>/is', '', $content);
-            $content = strip_tags($content, ['img', 'video']);
+            $content = (new Html2Text($content))->getText();
         }
 
         $hash = md5($content);
-
         $sql = rex_sql::factory();
         $sql->setTable(rex::getTable('diff_detect_index'));
         $sql->setWhere('url_id = ? ORDER BY createdate DESC LIMIT 1', [$url->getId()]);
@@ -132,11 +145,11 @@ public static function createSnapshot(Url $url): bool
         $sql->addGlobalCreateFields();
         $sql->addGlobalUpdateFields();
         $sql->setValue('url_id', $url->getId());
-        $sql->setValue('content', $response->getBody());
+        $sql->setValue('content', $content);
         $sql->setValue('hash', $hash);
-        $sql->setValue('header', $response->getHeader());
-        $sql->setValue('statusCode', $response->getStatusCode());
-        $sql->setValue('statusMessage', $response->getStatusMessage());
+        $sql->setValue('header', implode(',', $headers));
+        $sql->setValue('statusCode', $statusCode);
+        $sql->setValue('statusMessage', $statusMessage);
         $sql->insert();
 
         return true;
@@ -167,6 +180,8 @@ public static function cleanUpSnapshots(): void
                 WHERE
                 url_id = :url_id
                 AND createdate < DATE_SUB(:datetime, INTERVAL :interval MINUTE)
+                ORDER BY createdate ASC
+                LIMIT 100
             ', [
                 'url_id' => $URL->getId(),
                 'datetime' => date(rex_sql::FORMAT_DATETIME),
@@ -176,6 +191,7 @@ public static function cleanUpSnapshots(): void
             foreach ($indeces as $Index) {
                 $Index = self::fromSqlData($Index);
                 $Index->delete();
+                echo ' Deleted index with ID: ' . $Index->getId() . ' for URL: ' . $URL->getName() . "\n";
             }
         }
     }
@@ -194,14 +210,7 @@ public function getUrl(): ?Url
 
     public function getContent(): string
     {
-        if ('RSS' === $this->url?->getType()) {
-            return $this->getValue('content');
-        }
-
-        $content = $this->getValue('content');
-        // $content = HtmlDomParser::str_get_html($content)->findOne('#content')->innerHtml();
-        $content = (new Html2Text($content))->getText();
-        return $content;
+        return $this->getValue('content');
     }
 
     public function delete(): void
@@ -220,4 +229,21 @@ public function delete(): void
             throw new rex_exception($sql->getError());
         }
     }
+
+    private static function flattenArray($array, $prefix = '')
+    {
+        $result = [];
+
+        foreach ($array as $key => $value) {
+            $newKey = '' === $prefix ? $key : $prefix . '.' . $key;
+
+            if (is_array($value)) {
+                $result = array_merge($result, self::flattenArray($value, $newKey));
+            } else {
+                $result[$newKey] = $value;
+            }
+        }
+
+        return $result;
+    }
 }

lib/Index.php

@@ -42,7 +42,7 @@ public function calculate(): string
         $itemsAfter = $this->getItems($this->after);
 
         $output = '';
-        /** @var \Laminas\Feed\Reader\Entry\Rss $item */
+        /** @var Rss $item */
         foreach ($itemsBefore as $id => $item) {
             if (array_key_exists($id, $itemsAfter)) {
                 $diff = HtmlDiffAdvanced::create($this->renderItem($item), $this->renderItem($itemsAfter[$id]));

lib/RssDiff.php

@@ -2,13 +2,15 @@
 
 namespace FriendsOfRedaxo\DiffDetect;
 
+use Exception;
 use InvalidArgumentException;
 use rex;
 use rex_instance_pool_trait;
-use rex_socket;
-use rex_socket_response;
 use rex_sql;
 use rex_sql_exception;
+use Symfony\Component\HttpClient\HttpClient;
+
+use function sprintf;
 
 final class Url
 {
@@ -26,7 +28,7 @@ private function __construct(int $id)
     }
 
     /**
-     * @return null|static
+     * @return static|null
      */
     public static function get(int $id): ?self
     {
@@ -115,31 +117,60 @@ private static function fromSqlData(array $data): self
         return $dataset;
     }
 
-    public function getContent(): rex_socket_response
+    public function getResponse(): array
     {
-        $socket = rex_socket::factoryUrl($this->getValue('url'));
-        $socket->acceptCompression();
-        $socket->followRedirects(self::$maxRedirects);
-        $socket->setTimeout(self::$timeout);
+        $Options = [
+            'timeout' => self::$timeout,
+            'max_redirects' => self::$maxRedirects,
+            'headers' => [
+                'User-Agent' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+                // 'Accept-Encoding' => 'gzip, deflate, br',
+            ],
+        ];
 
         $login = $this->getValue('http_auth_login');
         $password = $this->getValue('http_auth_password');
+        if ('' !== $login && '' !== $password) {
+            $Options['headers']['Authorization'] = 'Basic ' . base64_encode($login . ':' . $password);
+        }
+
+        $client = HttpClient::create();
+        $response = $client->request('GET', $this->getValue('url'), $Options);
+
+        $stream = $client->stream($response);
 
-        if ('' === $login && '' === $password) {
-            $socket->addBasicAuthorization($login, $password);
+        $content = '';
+        foreach ($stream as $chunk) {
+            $content .= $chunk->getContent();
         }
 
-        $response = $socket->doGet();
-        $cookie = $response->getHeader('Set-Cookie');
+        $headers = $response->getHeaders();
+        $statusCode = $response->getStatusCode();
 
-        if (null !== $cookie) {
-            // separate cookie value from optional attributes
-            [$cookieValue] = explode(';', $cookie);
-            $socket->addHeader('Cookie', $cookieValue);
-            $response = $socket->doGet();
+        // $cookie = $response->getHeader('Set-Cookie');
+        //
+        // if (null !== $cookie) {
+        //     // separate cookie value from optional attributes
+        //     [$cookieValue] = explode(';', $cookie);
+        //     $socket->addHeader('Cookie', $cookieValue);
+        //     $response = $socket->doGet();
+        // }
+
+        if (200 !== $response->getStatusCode()) {
+            throw new Exception(sprintf('Failed to fetch content from URL "%s". HTTP status code: %d', $this->getValue('url'), $response->getStatusCode()));
+        }
+
+        if ('' === $content) {
+            throw new Exception(sprintf('No content received from URL "%s".', $this->getValue('url')));
         }
 
-        return $response->decompressContent(true);
+        unset($response);
+
+        return [
+            'Content' => $content,
+            'Headers' => $headers,
+            'StatusCode' => $statusCode,
+        ];
     }
 
     public function getType(): string
@@ -178,4 +209,15 @@ public function setLastScan(): void
             ],
         );
     }
+
+    public function setLastMessage(string $message): void
+    {
+        rex_sql::factory()->setQuery(
+            'update ' . rex::getTable('diff_detect_url') . ' set last_message = :last_message where id = :id',
+            [
+                'id' => $this->getId(),
+                'last_message' => $message,
+            ],
+        );
+    }
 }