Merge pull request #1502 from acelaya-forks/feature/cli-tests

Feature/cli tests
2025-03-14 04:00:57 +03:00 · 2022-08-10 17:39:27 +02:00 · 2022-08-10 17:39:27 +02:00 · 4bd3fa74d1
commit 4bd3fa74d1
parent af9ea13933 71553988d5
11 changed files with 295 additions and 21 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -34,13 +34,16 @@ jobs:
    strategy:
      matrix:
        php-version: ['8.1']
-        test-group: ['unit', 'api']
+        test-group: ['unit', 'api', 'cli']
    steps:
      - name: Checkout code
        uses: actions/checkout@v2
-      - name: Start database server
+      - name: Start postgres database server
        if: ${{ matrix.test-group == 'api' }}
        run: docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_postgres
+      - name: Start maria database server
+        if: ${{ matrix.test-group == 'cli' }}
+        run: docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_maria
      - name: Use PHP
        uses: shivammathur/setup-php@v2
        with:
@ -109,7 +112,7 @@ jobs:
    strategy:
      matrix:
        php-version: ['8.1']
-        test-group: ['unit', 'db', 'api']
+        test-group: ['unit', 'db', 'api', 'cli']
    steps:
      - name: Checkout code
        uses: actions/checkout@v2
@ -156,6 +159,7 @@ jobs:
      - run: mv build/coverage-unit/coverage-unit.cov build/coverage-unit.cov
      - run: mv build/coverage-db/coverage-db.cov build/coverage-db.cov
      - run: mv build/coverage-api/coverage-api.cov build/coverage-api.cov
+      - run: mv build/coverage-cli/coverage-cli.cov build/coverage-cli.cov
      - run: wget https://phar.phpunit.de/phpcov-8.2.1.phar
      - run: php phpcov-8.2.1.phar merge build --clover build/clover.xml
      - name: Publish coverage
@ -175,6 +179,7 @@ jobs:
            coverage-unit
            coverage-db
            coverage-api
+            coverage-cli

  build-docker-image:
    runs-on: ubuntu-22.04
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -4,6 +4,23 @@ All notable changes to this project will be documented in this file.

 The format is based on [Keep a Changelog](https://keepachangelog.com), and this project adheres to [Semantic Versioning](https://semver.org).

+## [Unreleased]
+### Added
+* *Nothing*
+
+### Changed
+* [#1339](https://github.com/shlinkio/shlink/issues/1339) Added new test suite for CLI E2E tests.
+
+### Deprecated
+* *Nothing*
+
+### Removed
+* *Nothing*
+
+### Fixed
+* *Nothing*
+
+
 ## [3.2.1] - 2022-08-08
 ### Added
 * *Nothing*
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@ -102,7 +102,9 @@ In order to ensure stability and no regressions are introduced while developing

    Since the app instance is run on a process different from the one running the tests, when a test fails it might not be obvious why. To help debugging that, the app will dump all its logs inside `data/log/api-tests`, where you will find the `shlink.log` and `access.log` files.

-* **CLI tests**: *TBD. Once included, its purpose will be the same as API tests, but running through the command line*
+* **CLI tests**: These are E2E tests too, but they test console commands instead of REST endpoints.
+
+    They use Maria DB as the database engine, and include the same fixtures as the API tests, that ensure the same data exists at the beginning of the execution.

 Depending on the kind of contribution, maybe not all kinds of tests are needed, but the more you provide, the better.

@ -119,6 +121,7 @@ Depending on the kind of contribution, maybe not all kinds of tests are needed,
    For example, `test:db:postgres`.

 * Run `./indocker composer test:api` to run API E2E tests. For these, the Postgres database engine is used.
+* Run `./indocker composer test:cli` to run CLI E2E tests. For these, the Maria DB database engine is used.
 * Run `./indocker composer infect:test` to run both unit and database tests (over sqlite) and then apply mutations to them with [infection](https://infection.github.io/).
 * Run `./indocker composer ci` to run all previous commands together. This command is run during the project's continuous integration.
 * Run `./indocker composer ci:parallel` to do the same as in previous case, but parallelizing non-conflicting tasks as much as possible.
--- a/composer.json
+++ b/composer.json
@ -58,7 +58,7 @@
    "require-dev": {
        "cebe/php-openapi": "^1.7",
        "devster/ubench": "^2.1",
-        "dms/phpunit-arraysubset-asserts": "^0.3.0",
+        "dms/phpunit-arraysubset-asserts": "^0.4.0",
        "infection/infection": "^0.26.5",
        "openswoole/ide-helper": "~4.11.1",
        "phpspec/prophecy-phpunit": "^2.0",
@ -87,6 +87,7 @@
    "autoload-dev": {
        "psr-4": {
            "ShlinkioTest\\Shlink\\CLI\\": "module/CLI/test",
+            "ShlinkioCliTest\\Shlink\\CLI\\": "module/CLI/test-cli",
            "ShlinkioTest\\Shlink\\Rest\\": "module/Rest/test",
            "ShlinkioApiTest\\Shlink\\Rest\\": "module/Rest/test-api",
            "ShlinkioTest\\Shlink\\Core\\": "module/Core/test",
@ -106,7 +107,7 @@
        ],
        "ci:parallel": [
            "@parallel cs stan swagger:validate test:unit:ci test:db:sqlite:ci test:db:mysql test:db:maria test:db:postgres test:db:ms",
-            "@parallel infect:test:api infect:ci:unit infect:ci:db"
+            "@parallel infect:test:api infect:test:cli infect:ci:unit infect:ci:db"
        ],
        "cs": "phpcs",
        "cs:fix": "phpcbf",
@ -114,12 +115,14 @@
        "test": [
            "@test:unit",
            "@test:db",
-            "@test:api"
+            "@test:api",
+            "@test:cli"
        ],
        "test:ci": [
            "@test:unit:ci",
            "@test:db",
-            "@test:api:ci"
+            "@test:api:ci",
+            "@test:cli:ci"
        ],
        "test:unit": "@php vendor/bin/phpunit --order-by=random --colors=always --coverage-php build/coverage-unit.cov --testdox",
        "test:unit:ci": "@test:unit --coverage-xml=build/coverage-unit/coverage-xml --log-junit=build/coverage-unit/junit.xml",
@ -133,11 +136,14 @@
        "test:db:ms": "DB_DRIVER=mssql composer test:db:sqlite",
        "test:api": "bin/test/run-api-tests.sh",
        "test:api:ci": "GENERATE_COVERAGE=yes composer test:api",
+        "test:cli": "APP_ENV=test DB_DRIVER=maria TEST_ENV=cli php vendor/bin/phpunit --order-by=random --colors=always --testdox -c phpunit-cli.xml",
+        "test:cli:ci": "GENERATE_COVERAGE=yes composer test:cli -- --log-junit=build/coverage-cli/junit.xml",
        "infect:ci:base": "infection --threads=4 --log-verbosity=default --only-covered --only-covering-test-cases --skip-initial-tests",
        "infect:ci:unit": "@infect:ci:base --coverage=build/coverage-unit --min-msi=84",
        "infect:ci:db": "@infect:ci:base --coverage=build/coverage-db --min-msi=95 --configuration=infection-db.json",
        "infect:ci:api": "@infect:ci:base --coverage=build/coverage-api --min-msi=80 --configuration=infection-api.json",
-        "infect:ci": "@parallel infect:ci:unit infect:ci:db infect:ci:api",
+        "infect:ci:cli": "@infect:ci:base --coverage=build/coverage-cli --min-msi=80 --configuration=infection-cli.json",
+        "infect:ci": "@parallel infect:ci:unit infect:ci:db infect:ci:api infect:ci:cli",
        "infect:test": [
            "@parallel test:unit:ci test:db:sqlite:ci test:api:ci",
            "@infect:ci"
@ -150,6 +156,10 @@
            "@test:api:ci",
            "@infect:ci:api"
        ],
+        "infect:test:cli": [
+            "@test:cli:ci",
+            "@infect:ci:cli"
+        ],
        "swagger:validate": "php-openapi validate docs/swagger/swagger.json",
        "swagger:inline": "php-openapi inline docs/swagger/swagger.json docs/swagger/swagger-inlined.json",
        "clean:dev": "rm -f data/database.sqlite && rm -f config/params/generated_config.php"
--- a/config/test/bootstrap_cli_tests.php
+++ b/config/test/bootstrap_cli_tests.php
@ -0,0 +1,28 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\TestUtils;
+
+use Doctrine\ORM\EntityManager;
+use Psr\Container\ContainerInterface;
+
+use function file_exists;
+use function unlink;
+
+/** @var ContainerInterface $container */
+$container = require __DIR__ . '/../container.php';
+$testHelper = $container->get(Helper\TestHelper::class);
+$config = $container->get('config');
+$em = $container->get(EntityManager::class);
+
+// Delete old coverage in PHP, to avoid merging older executions with current one
+$covFile = __DIR__ . '/../../build/coverage-cli.cov';
+if (file_exists($covFile)) {
+    unlink($covFile);
+}
+
+$testHelper->createTestDb(['bin/cli', 'db:create'], ['bin/cli', 'db:migrate']);
+CliTest\CliTestCase::setSeedFixturesCallback(
+    static fn () => $testHelper->seedFixtures($em, $config['data_fixtures'] ?? []),
+);
--- a/config/test/test_config.global.php
+++ b/config/test/test_config.global.php
@ -8,8 +8,10 @@ use GuzzleHttp\Client;
 use Laminas\ConfigAggregator\ConfigAggregator;
 use Laminas\Diactoros\Response\EmptyResponse;
 use Laminas\ServiceManager\Factory\InvokableFactory;
+use League\Event\EventDispatcher;
 use Monolog\Level;
 use PHPUnit\Runner\Version;
+use Psr\Container\ContainerInterface;
 use Psr\Http\Message\ResponseInterface;
 use Psr\Http\Message\ServerRequestInterface;
 use Psr\Http\Server\RequestHandlerInterface;
@ -20,7 +22,12 @@ use SebastianBergmann\CodeCoverage\Report\Html\Facade as Html;
 use SebastianBergmann\CodeCoverage\Report\PHP;
 use SebastianBergmann\CodeCoverage\Report\Xml\Facade as Xml;
 use Shlinkio\Shlink\Common\Logger\LoggerType;
+use Symfony\Component\Console\Application;
+use Symfony\Component\Console\Event\ConsoleCommandEvent;
+use Symfony\Component\Console\Event\ConsoleTerminateEvent;
+use Symfony\Contracts\EventDispatcher\EventDispatcherInterface;

+use function file_exists;
 use function Laminas\Stratigility\middleware;
 use function Shlinkio\Shlink\Config\env;
 use function sprintf;
@ -30,14 +37,40 @@ use const ShlinkioTest\Shlink\SWOOLE_TESTING_HOST;
 use const ShlinkioTest\Shlink\SWOOLE_TESTING_PORT;

 $isApiTest = env('TEST_ENV') === 'api';
+$isCliTest = env('TEST_ENV') === 'cli';
+$isE2eTest = $isApiTest || $isCliTest;
 $generateCoverage = env('GENERATE_COVERAGE') === 'yes';
-if ($isApiTest && $generateCoverage) {
+
+$coverage = null;
+if ($isE2eTest && $generateCoverage) {
    $filter = new Filter();
    $filter->includeDirectory(__DIR__ . '/../../module/Core/src');
-    $filter->includeDirectory(__DIR__ . '/../../module/Rest/src');
+    $filter->includeDirectory(__DIR__ . '/../../module/' . ($isApiTest ? 'Rest' : 'CLI') . '/src');
    $coverage = new CodeCoverage((new Selector())->forLineCoverage($filter), $filter);
 }

+/**
+ * @param 'api'|'cli' $type
+ */
+$exportCoverage = static function (string $type = 'api') use (&$coverage): void {
+    if ($coverage === null) {
+        return;
+    }
+
+    $basePath = __DIR__ . '/../../build/coverage-' . $type;
+    $covPath = $basePath . '.cov';
+
+    // Every CLI test runs on its own process and dumps the coverage afterwards.
+    // Try to load it and merge it, so that we end up with the whole coverage at the end.
+    if ($type === 'cli' && file_exists($covPath)) {
+        $coverage->merge(require $covPath);
+    }
+
+    (new PHP())->process($coverage, $covPath);
+    (new Xml(Version::getVersionString()))->process($coverage, $basePath . '/coverage-xml');
+    (new Html())->process($coverage, $basePath . '/coverage-html');
+};
+
 $buildDbConnection = static function (): array {
    $driver = env('DB_DRIVER', 'sqlite');
    $isCi = env('CI', false);
@ -113,17 +146,10 @@ return [
        [
            'name' => 'dump_coverage',
            'path' => '/api-tests/stop-coverage',
-            'middleware' => middleware(static function () use (&$coverage) {
+            'middleware' => middleware(static function () use ($exportCoverage) {
                // TODO I have tried moving this block to a listener so that it's invoked automatically,
                //      but then the coverage is generated empty ¯\_(ツ)_/¯
-                if ($coverage) { // @phpstan-ignore-line
-                    $basePath = __DIR__ . '/../../build/coverage-api';
-
-                    (new PHP())->process($coverage, $basePath . '.cov');
-                    (new Xml(Version::getVersionString()))->process($coverage, $basePath . '/coverage-xml');
-                    (new Html())->process($coverage, $basePath . '/coverage-html');
-                }
-
+                $exportCoverage();
                return new EmptyResponse();
            }),
            'allowed_methods' => ['GET'],
@ -164,6 +190,62 @@ return [
        'factories' => [
            TestUtils\Helper\TestHelper::class => InvokableFactory::class,
        ],
+        'delegators' => $isCliTest ? [
+            Application::class => [
+                static function (
+                    ContainerInterface $c,
+                    string $serviceName,
+                    callable $callback,
+                ) use (
+                    &$coverage,
+                    $exportCoverage,
+                ) {
+                    /** @var Application $app */
+                    $app = $callback();
+                    $wrappedEventDispatcher = new EventDispatcher();
+
+                    // When the command starts, start collecting coverage
+                    $wrappedEventDispatcher->subscribeTo(
+                        ConsoleCommandEvent::class,
+                        static function () use (&$coverage): void {
+                            $id = env('COVERAGE_ID');
+                            if ($id === null) {
+                                return;
+                            }
+
+                            $coverage?->start($id);
+                        },
+                    );
+                    // When the command ends, stop collecting coverage
+                    $wrappedEventDispatcher->subscribeTo(
+                        ConsoleTerminateEvent::class,
+                        static function () use (&$coverage, $exportCoverage): void {
+                            $id = env('COVERAGE_ID');
+                            if ($id === null) {
+                                return;
+                            }
+
+                            $coverage?->stop();
+                            $exportCoverage('cli');
+                        },
+                    );
+
+                    $app->setDispatcher(new class ($wrappedEventDispatcher) implements EventDispatcherInterface {
+                        public function __construct(private EventDispatcher $wrappedDispatcher)
+                        {
+                        }
+
+                        public function dispatch(object $event, ?string $eventName = null): object
+                        {
+                            $this->wrappedDispatcher->dispatch($event);
+                            return $event;
+                        }
+                    });
+
+                    return $app;
+                },
+            ],
+        ] : [],
    ],

    'entity_manager' => [
@ -172,6 +254,7 @@ return [

    'data_fixtures' => [
        'paths' => [
+            // TODO These are used for CLI tests too, so maybe should be somewhere else
            __DIR__ . '/../../module/Rest/test-api/Fixtures',
        ],
    ],
--- a/infection-cli.json
+++ b/infection-cli.json
@ -0,0 +1,24 @@
+{
+    "source": {
+        "directories": [
+            "module/*/src"
+        ]
+    },
+    "timeout": 5,
+    "logs": {
+        "text": "build/infection-cli/infection-log.txt",
+        "html": "build/infection-cli/infection-log.html",
+        "summary": "build/infection-cli/summary-log.txt",
+        "debug": "build/infection-cli/debug-log.txt"
+    },
+    "tmpDir": "build/infection-cli/temp",
+    "phpUnit": {
+        "configDir": "."
+    },
+    "testFrameworkOptions": "--configuration=phpunit-cli.xml",
+    "mutators": {
+        "@default": true,
+        "IdenticalEqual": false,
+        "NotIdenticalNotEqual": false
+    }
+}
--- a/module/CLI/test-cli/Command/GenerateApiKeyTest.php
+++ b/module/CLI/test-cli/Command/GenerateApiKeyTest.php
@ -0,0 +1,21 @@
+<?php
+
+declare(strict_types=1);
+
+namespace ShlinkioCliTest\Shlink\CLI\Command;
+
+use Shlinkio\Shlink\CLI\Command\Api\GenerateKeyCommand;
+use Shlinkio\Shlink\CLI\Util\ExitCodes;
+use Shlinkio\Shlink\TestUtils\CliTest\CliTestCase;
+
+class GenerateApiKeyTest extends CliTestCase
+{
+    /** @test */
+    public function outputIsCorrect(): void
+    {
+        [$output, $exitCode] = $this->exec([GenerateKeyCommand::NAME]);
+
+        self::assertStringContainsString('[OK] Generated API key', $output);
+        self::assertEquals(ExitCodes::EXIT_SUCCESS, $exitCode);
+    }
+}
--- a/module/CLI/test-cli/Command/ListApiKeysTest.php
+++ b/module/CLI/test-cli/Command/ListApiKeysTest.php
@ -0,0 +1,63 @@
+<?php
+
+declare(strict_types=1);
+
+namespace ShlinkioCliTest\Shlink\CLI\Command;
+
+use Cake\Chronos\Chronos;
+use Shlinkio\Shlink\CLI\Command\Api\ListKeysCommand;
+use Shlinkio\Shlink\CLI\Util\ExitCodes;
+use Shlinkio\Shlink\TestUtils\CliTest\CliTestCase;
+
+class ListApiKeysTest extends CliTestCase
+{
+    /**
+     * @test
+     * @dataProvider provideFlags
+     */
+    public function generatesExpectedOutput(array $flags, string $expectedOutput): void
+    {
+        [$output, $exitCode] = $this->exec([ListKeysCommand::NAME, ...$flags]);
+
+        self::assertEquals($expectedOutput, $output);
+        self::assertEquals(ExitCodes::EXIT_SUCCESS, $exitCode);
+    }
+
+    public function provideFlags(): iterable
+    {
+        $expiredApiKeyDate = Chronos::now()->subDay()->startOfDay()->toAtomString();
+        $enabledOnlyOutput = <<<OUT
+        +-----------------+------+---------------------------+--------------------------+
+        | Key             | Name | Expiration date           | Roles                    |
+        +-----------------+------+---------------------------+--------------------------+
+        | valid_api_key   | -    | -                         | Admin                    |
+        +-----------------+------+---------------------------+--------------------------+
+        | expired_api_key | -    | {$expiredApiKeyDate} | Admin                    |
+        +-----------------+------+---------------------------+--------------------------+
+        | author_api_key  | -    | -                         | Author only              |
+        +-----------------+------+---------------------------+--------------------------+
+        | domain_api_key  | -    | -                         | Domain only: example.com |
+        +-----------------+------+---------------------------+--------------------------+
+
+        OUT;
+
+        yield 'no flags' => [[], <<<OUT
+            +------------------+------+------------+---------------------------+--------------------------+
+            | Key              | Name | Is enabled | Expiration date           | Roles                    |
+            +------------------+------+------------+---------------------------+--------------------------+
+            | valid_api_key    | -    | +++        | -                         | Admin                    |
+            +------------------+------+------------+---------------------------+--------------------------+
+            | disabled_api_key | -    | ---        | -                         | Admin                    |
+            +------------------+------+------------+---------------------------+--------------------------+
+            | expired_api_key  | -    | ---        | {$expiredApiKeyDate} | Admin                    |
+            +------------------+------+------------+---------------------------+--------------------------+
+            | author_api_key   | -    | +++        | -                         | Author only              |
+            +------------------+------+------------+---------------------------+--------------------------+
+            | domain_api_key   | -    | +++        | -                         | Domain only: example.com |
+            +------------------+------+------------+---------------------------+--------------------------+
+
+            OUT];
+        yield '-e' => [['-e'], $enabledOnlyOutput];
+        yield '--enabled-only' => [['--enabled-only'], $enabledOnlyOutput];
+    }
+}
--- a/module/Rest/test-api/Fixtures/ApiKeyFixture.php
+++ b/module/Rest/test-api/Fixtures/ApiKeyFixture.php
@ -25,7 +25,7 @@ class ApiKeyFixture extends AbstractFixture implements DependentFixtureInterface
    {
        $manager->persist($this->buildApiKey('valid_api_key', true));
        $manager->persist($this->buildApiKey('disabled_api_key', false));
-        $manager->persist($this->buildApiKey('expired_api_key', true, Chronos::now()->subDay()));
+        $manager->persist($this->buildApiKey('expired_api_key', true, Chronos::now()->subDay()->startOfDay()));

        $authorApiKey = $this->buildApiKey('author_api_key', true);
        $authorApiKey->registerRole(RoleDefinition::forAuthoredShortUrls());
--- a/phpunit-cli.xml
+++ b/phpunit-cli.xml
@ -0,0 +1,20 @@
+<?xml version="1.0"?>
+<phpunit
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:noNamespaceSchemaLocation="vendor/phpunit/phpunit/phpunit.xsd"
+    bootstrap="./config/test/bootstrap_cli_tests.php"
+    colors="true"
+>
+    <testsuites>
+        <testsuite name="Shlink CLI tests">
+            <directory>./module/*/test-cli</directory>
+        </testsuite>
+    </testsuites>
+
+    <coverage processUncoveredFiles="true">
+        <include>
+            <directory suffix=".php">./module/CLI/src</directory>
+            <directory suffix=".php">./module/Core/src</directory>
+        </include>
+    </coverage>
+</phpunit>