diff --git a/.github/workflows/macos_l11.yml b/.github/workflows/macos_l11.yml index 5f6d164..8db3eff 100644 --- a/.github/workflows/macos_l11.yml +++ b/.github/workflows/macos_l11.yml @@ -25,6 +25,9 @@ jobs: with: php-version: '8.2' + - name: "Homebrew: Install GNU Core Utilities" + run: yes | brew install coreutils + - name: Validate composer.json and composer.lock run: composer validate --strict diff --git a/README.md b/README.md index 9085514..fd4eca8 100644 --- a/README.md +++ b/README.md @@ -13,13 +13,17 @@ Utilize Laravel Processes to run PHP code asynchronously. - Restrictions from `laravel/serializable-closure` apply (see [their README](https://github.com/laravel/serializable-closure)) - Hands-off execution: no built-in result-checking, check the results yourself (e.g. via database, file cache, etc) +This library internally uses an Artisan command to run the async code, which is similar to Laravel 11 [Concurrency](https://laravel.com/docs/11.x/concurrency). + ## Why should I want this? This library is very helpful for these cases: -- You want a minimal-setup async for easy vertical scaling +- You want a cross-platform minimal-setup async for easy vertical scaling - You want to start quick-and-dirty async tasks right now (e.g. prefetching resources, pinging remote, etc.) - Best is if your task only has very few lines of code - Laravel 11 [Concurrency](https://laravel.com/docs/11.x/concurrency) is too limiting; e.g.: - You want to do something else while waiting for results + - You want to conveniently limit the max (real) execution time of the concurrent tasks +- And perhaps more! Of course, if you are considering extreme scaling (e.g. Redis queues in Laravel, multi-worker clusters, etc.) or guaranteed task execution, then this library is obviously not for you. @@ -32,6 +36,12 @@ composer require vectorial1024/laravel-process-async This library supports Unix and Windows; see the Testing section for more details. +### Extra requirements for Unix +If you are on Unix, check that you also have the following: +- GNU Core Utilities (`coreutils`) + - MacOS do `brew install coreutils`! + - Other Unix distros should check if `coreutils` is preinstalled + ## Change log Please see `CHANGELOG.md`. @@ -59,6 +69,24 @@ $task->start(); // the task is now run in another PHP process, and will not report back to this PHP process. ``` +### Task time limits +You can set task time limits before you start them, but you cannot change them after the tasks are started. When the time limit is reached, the async task is killed. + +The default time limit is 30 real seconds. You can also choose to not set any time limit, in this case the (CLI) PHP `max_execution_time` directive will control the time limit. + +Note: `AsyncTaskInterface` contains an implementable method `handleTimeout` for you to define timeout-related cleanups (e.g. write to some log that the task has timed out). This method is still called when the PHP `max_execution_time` directive is triggered. + +```php +// start with the default time limit... +$task->start(); + +// start task with a different time limit... +$task->withTimeLimit(15)->start(); + +// ...or not have any limits at all (beware of orphaned processes!) +$task->withoutTimeLimit()->start(); +``` + ## Testing PHPUnit via Composer script: ```sh diff --git a/src/AsyncTask.php b/src/AsyncTask.php index 9a8346b..a0ed917 100644 --- a/src/AsyncTask.php +++ b/src/AsyncTask.php @@ -1,12 +1,16 @@ laravelStartVal = defined(self::LARAVEL_START) ? constant("LARAVEL_START") : null; + + // install a timeout detector + // this single function checks all kinds of timeouts + register_shutdown_function([$this, 'shutdownCheckTaskTimeout']); + if (OsInfo::isWindows()) { + // windows can just use PHP's time limit + set_time_limit($this->timeLimit); + } else { + // assume anything not Windows to be Unix + // we already set it to kill this task after the timeout, so we just need to install a listener to catch the signal and exit gracefully + pcntl_async_signals(true); + pcntl_signal(SIGINT, function () { + // sicne we are already running with nohup, we can use SIGINT to indicate that a timeout has occurred. + // exit asap so that our error checking inside shutdown functions can take place outside of the usual max_execution_time limit + $this->hasSigInt = true; + exit(); + }); + + // and we also need to see the command name of our parent, to correctly track time + $this->timerProcID = getmypid(); + $parentPid = posix_getppid(); + $parentCmd = exec("ps -p $parentPid -o comm="); + if ($parentCmd === "timeout" || $parentCmd === "gtimeout") { + // we should use the parent instead to time this task + $this->timerProcID = $parentPid; + } + } // then, execute the task itself if ($this->theTask instanceof SerializableClosure) { @@ -77,12 +169,32 @@ public function start(): void if (OsInfo::isWindows()) { // basically, in windows, it is too tedioous to check whether we are in cmd or ps, // but we require cmd (ps won't work here), so might as well force cmd like this + // windows has real max time limit $this->runnerProcess = Process::quietly()->start("cmd /c start /b $baseCommand"); return; } // assume anything not windows to be unix // unix use nohup - $this->runnerProcess = Process::quietly()->start("nohup $baseCommand"); + // check time limit settings + $timeoutClause = ""; + if ($this->timeLimit > 0) { + // do we really have timeout here? + if (static::$hasGnuCoreUtils === null) { + // haven't checked before; check + $tmpOut = exec("command -v timeout || command -v gtimeout"); + $cmdName = !empty($tmpOut) ? $tmpOut : null; + unset($tmpOut); + static::$hasGnuCoreUtils = $cmdName !== null; + static::$timeoutCmdName = $cmdName; + } + if (static::$hasGnuCoreUtils === false) { + // can't do anything without GNU coreutils! + throw new RuntimeException("AsyncTask time limit requires GNU coreutils, but GNU coreutils was not installed"); + } + // 2 is INT signal + $timeoutClause = static::$timeoutCmdName . " -s 2 {$this->timeLimit}"; + } + $this->runnerProcess = Process::quietly()->start("nohup $timeoutClause $baseCommand >/dev/null 2>&1"); } /** @@ -126,4 +238,125 @@ public static function fromBase64Serial(string $serial): ?static return null; } } + + /** + * Returns the maximum real time this task is allowed to run. This also includes time spent on sleeping and waiting! + * + * Null indicates unlimited time. + * @return int|null The time limit in seconds. + */ + public function getTimeLimit(): int|null + { + return $this->timeLimit; + } + + /** + * Sets the maximum real time this task is allowed to run. Chainable. + * + * When the task reaches the time limit, the timeout handler (if exists) will be called. + * @param int $seconds The time limit in seconds. + * @return AsyncTask $this for chaining. + */ + public function withTimeLimit(int $seconds): static + { + if ($seconds == 0) { + throw new LogicException("AsyncTask time limit must be positive (hint: use withoutTimeLimit() for no time limits)"); + } + if ($seconds < 0) { + throw new LogicException("AsyncTask time limit must be positive"); + } + $this->timeLimit = $seconds; + return $this; + } + + /** + * Sets this task to run with no time limit (PHP INI `max_execution_time` may apply). Chainable. + * @return AsyncTask $this for chaining. + */ + public function withoutTimeLimit(): static + { + $this->timeLimit = null; + return $this; + } + + /** + * A shutdown function. + * + * Upon shutdown, checks whether this is due to the task timing out, and if so, triggers the timeout handler. + * @return void + */ + protected function shutdownCheckTaskTimeout(): void + { + if (!$this->hasTimedOut()) { + // shutdown due to other reasons; skip + return; + } + + // timeout! + // trigger the timeout handler + if ($this->theTask instanceof AsyncTaskInterface) { + $this->theTask->handleTimeout(); + } + } + + /** + * During shutdown, checks whether this shutdown satisfies the "task timed out shutdown" condition. + * @return bool True if this task is timed out according to our specifications. + */ + private function hasTimedOut(): bool + { + // we perform a series of checks to see if this task has timed out + + // dedicated SIGINT indicates a timeout + if ($this->hasSigInt) { + return true; + } + + // runtime timeout triggers a PHP fatal error + // this can happen on Windows by our specification, or on Unix when the actual CLI PHP time limit is smaller than the time limit of this task + $lastError = error_get_last(); + if ($lastError !== null && ($lastError['type'] & self::FATAL_ERROR_BITMASK)) { + // has fatal error; is it our timeout error? + return str_contains($lastError['message'], "Maximum execution time"); + } + unset($lastError); + + // the remaining checks use the time-limit variable, so if it is unset, then there is nothing to check + if ($this->timeLimit <= 0) { + return false; + } + + // not a runtime timeout; one of the following: + // it ended within the time limit; or + // on Unix, it ran out of time so it is getting a SIGTERM from us; or + // it somehow ran out of time, and is being manually detected and killed + if ($this->laravelStartVal !== null) { + // this is very important; in some test cases, this is being run directly by PHPUnit, and so LARAVEL_START will be null + // in this case, we have no idea when this task has started running, so we cannot deduce any timeout statuses + + // check LARAVEL_START with microtime + $timeElapsed = microtime(true) - $this->laravelStartVal; + if ($timeElapsed >= $this->timeLimit) { + // yes + return true; + } + + // if we are on Unix, and when we have set a task time limit, then the LARAVEL_START value is inaccurate + // because there will always be a small but significant delay between `timeout` start time and PHP start time. + // in this case, we will look at the pre-determined timer PID to ask about the actual elapsed time through the kernel's proc data + // this method should be slower than the microtime method + if (OsInfo::isUnix()) { + // get time elapsed in seconds + $tempOut = exec("ps -p {$this->timerProcID} -o etimes="); + // this must exist (we are still running!), otherwise it indicates the kernel is broken and we can go grab a chicken dinner instead + $timeElapsed = (int) $tempOut; + unset($tempOut); + // it seems like etimes can get random off-by-1 inaccuracies (e.g. timeout supposed to be 7, but etimes sees 6.99999... and prints "6") + return $timeElapsed >= $this->timeLimit; + } + } + + // didn't see anything; assume is no + return false; + } } diff --git a/src/AsyncTaskInterface.php b/src/AsyncTaskInterface.php index 086ca83..a76d4ed 100644 --- a/src/AsyncTaskInterface.php +++ b/src/AsyncTaskInterface.php @@ -1,5 +1,7 @@ null); + + // test +ve + $timeLimit = random_int(1, 100); + $task->withTimeLimit($timeLimit); + $this->assertEquals($timeLimit, $task->getTimeLimit()); + + // test 0; 0 makes no sense here + $this->expectException(LogicException::class); + $task->withTimeLimit(0); + + // test -ve; also makes no sense here + $this->expectException(LogicException::class); + $task->withTimeLimit(-1); + } + + public function testConfigNoTimeLimit() + { + $task = new AsyncTask(fn() => null); + $task->withoutTimeLimit(); + $this->assertNull($task->getTimeLimit()); + } + // --------- // integration test with the cli artisan via a mocked artisan file, which tests various features of this library @@ -50,6 +82,7 @@ public function testAsyncBasic() { // tests that we can dispatch async tasks to the cli artisan $testFileName = $this->getStoragePath("testAsyncBasic.txt"); + @unlink($testFileName); $message = "Hello world!"; $task = new AsyncTask(function () use ($testFileName, $message) { $fp = fopen($testFileName, "w"); @@ -60,10 +93,11 @@ public function testAsyncBasic() $task->start(); // sleep a bit to wait for the async - sleep(1); + $this->sleep(1); $this->assertFileExists($testFileName, "The async task probably did not run because its output file cannot be found."); $this->assertStringEqualsFile($testFileName, $message); + $this->assertNoNohupFile(); unlink($testFileName); } @@ -85,4 +119,93 @@ public function testAsyncBackground() $timeElapsed = $timeAfter - $timeBefore; $this->assertLessThan($sleepDuration, $timeElapsed, "The async task probably did not start in the background because the time taken to start it was too long."); } + + public function testAsyncSilence() + { + // test that the async runner is really silent: e.g. it should not generate any nohup.out files + // the intended way of using this library is to send all kinds of debug output to the Laravel logs + // alternatively, laravel will simply catch exceptions and send them to the laravel logs + $task = new AsyncTask(function () { + // randomly throw exception + throw new RuntimeException("random testing exception"); + }); + $task->start(); + $this->sleep(1); + $this->assertNoNohupFile(); + } + + public function testAsyncTimeout() + { + // test that we can trigger the async task timeout + $message = "timeout occured"; + $textFilePath = $this->getStoragePath("testAsyncTimeout.txt"); + $timeoutTask = new TestTimeoutNormalTask($message, $textFilePath); + @unlink($textFilePath); + $task = new AsyncTask($timeoutTask); + $task->withTimeLimit(1)->start(); + // we wait for it to timeout + $this->sleep(2); + // should have timed out + $this->assertFileExists($textFilePath, "The async task probably did not trigger its timeout handler because its timeout output file is not found."); + $this->assertStringEqualsFile($textFilePath, $message); + $this->assertNoNohupFile(); + } + + public function testAsyncTimeoutIgnoreErrors() + { + // test that the async timeout handler is not triggered due to other fatal errors + $message = "timeout occured"; + $textFilePath = $this->getStoragePath("testAsyncTimeoutIgnoreErrors.txt"); + @unlink($textFilePath); + $timeoutTask = new TestTimeoutErrorTask($message, $textFilePath); + $task = new AsyncTask($timeoutTask); + $task->withTimeLimit(2)->start(); + // we wait for it to timeout + $this->sleep(2); + // should have timed out + $this->assertFileDoesNotExist($textFilePath, "The async task timeout handler was inappropriately triggered (PHP fatal errors should not trigger timeouts)."); + $this->assertNoNohupFile(); + } + + public function testAsyncTimeoutIgnoreNoProblem() + { + // test that the async timeout handler is not triggered when nothing happened + $message = "timeout occured"; + $textFilePath = $this->getStoragePath("testAsyncTimeoutIgnoreNoProblem.txt"); + @unlink($textFilePath); + $timeoutTask = new TestTimeoutNoOpTask($message, $textFilePath); + $task = new AsyncTask($timeoutTask); + $task->withTimeLimit(2)->start(); + // we wait for it to timeout + $this->sleep(2); + // should have timed out + $this->assertFileDoesNotExist($textFilePath, "The async task timeout handler was inappropriately triggered (finishing a task before the time limit should not trigger timeouts)."); + $this->assertNoNohupFile(); + + // repeat with no time limit + @unlink($textFilePath); + $task = new AsyncTask($timeoutTask); + $task->withoutTimeLimit()->start(); + // we wait for it to timeout + $this->sleep(0.5); + // should have timed out + $this->assertFileDoesNotExist($textFilePath, "The async task timeout handler was inappropriately triggered (tasks without time limits should not trigger timeouts)."); + $this->assertNoNohupFile(); + } + + public function testAsyncTimeoutIgnoreENotice() + { + // test that the async timeout handler is not triggered when there is an E_NOTICE error + $message = "timeout occured"; + $textFilePath = $this->getStoragePath("testAsyncTimeoutIgnoreENotice.txt"); + @unlink($textFilePath); + $timeoutTask = new TestTimeoutENoticeTask($message, $textFilePath); + $task = new AsyncTask($timeoutTask); + $task->withTimeLimit(2)->start(); + // we wait for it to timeout + $this->sleep(2); + // should have timed out + $this->assertFileDoesNotExist($textFilePath, "The async task timeout handler was inappropriately triggered (E_NOTICE should not trigger timeouts)."); + $this->assertNoNohupFile(); + } } diff --git a/tests/BaseTestCase.php b/tests/BaseTestCase.php index 651acb5..3b41e65 100644 --- a/tests/BaseTestCase.php +++ b/tests/BaseTestCase.php @@ -20,6 +20,15 @@ protected function getPackageProviders($app) // --- + /** + * Returns the base path of this project (i.e., the directory of composer.json). + * @return string + */ + protected function getBasePath(): string + { + return dirname(__FILE__, 2); + } + /** * Returns the path for mocking the Laravel storage path. * @param string $fileName @@ -27,7 +36,36 @@ protected function getPackageProviders($app) */ protected function getStoragePath(string $fileName): string { - return dirname(__FILE__, 2) . "/storage/$fileName"; + return $this->getBasePath() . "/storage/$fileName"; + } + + /** + * Sleeps for some time. + * @param float $seconds The number of seconds. + * @return void + */ + protected function sleep(float $seconds): void + { + $wholeSeconds = (int) $seconds; + $fractionalSeconds = $seconds - $wholeSeconds; + if ($wholeSeconds > 0) { + sleep($wholeSeconds); + } + if ($fractionalSeconds > 0) { + usleep($fractionalSeconds * 1000000); + } + } + + /** + * Asserts that the nohup.out file is not found in our project while running CI/CD. This checks that we are truly silencing the output of the task runner. + * + * Applicable only in Unix systems; Windows systems will get a vacuous successful assertion on this. + * @return void + */ + protected function assertNoNohupFile(string $message = ''): void + { + $nohupFilePath = $this->getBasePath() . "/nohup.out"; + $this->assertFileDoesNotExist($nohupFilePath, "The async task did not run silently since the nohup.out file can be found."); } // --- diff --git a/tests/DummyAsyncTask.php b/tests/Tasks/DummyAsyncTask.php similarity index 76% rename from tests/DummyAsyncTask.php rename to tests/Tasks/DummyAsyncTask.php index 1cbfe94..270f07f 100644 --- a/tests/DummyAsyncTask.php +++ b/tests/Tasks/DummyAsyncTask.php @@ -1,6 +1,6 @@ targetFilePath, "w"); + fwrite($fp, $this->message); + fflush($fp); + fclose($fp); + } +} diff --git a/tests/Tasks/TestTimeoutErrorTask.php b/tests/Tasks/TestTimeoutErrorTask.php new file mode 100644 index 0000000..24117ea --- /dev/null +++ b/tests/Tasks/TestTimeoutErrorTask.php @@ -0,0 +1,30 @@ +targetFilePath, "w"); + fwrite($fp, $this->message); + fflush($fp); + fclose($fp); + } +} diff --git a/tests/Tasks/TestTimeoutNoOpTask.php b/tests/Tasks/TestTimeoutNoOpTask.php new file mode 100644 index 0000000..538ca62 --- /dev/null +++ b/tests/Tasks/TestTimeoutNoOpTask.php @@ -0,0 +1,30 @@ +targetFilePath, "w"); + fwrite($fp, $this->message); + fflush($fp); + fclose($fp); + } +} diff --git a/tests/Tasks/TestTimeoutNormalTask.php b/tests/Tasks/TestTimeoutNormalTask.php new file mode 100644 index 0000000..e10b8af --- /dev/null +++ b/tests/Tasks/TestTimeoutNormalTask.php @@ -0,0 +1,32 @@ +targetFilePath, "w"); + fwrite($fp, $this->message); + fflush($fp); + fclose($fp); + } +}