mirrors/shlink
1
0
Fork 0
mirror of https://github.com/shlinkio/shlink.git synced 2024-11-27 08:18:24 +03:00
Code Issues Projects Releases Packages Wiki Activity

Basic short-úrl import implementation

Browse source
This commit is contained in:
Alejandro Celaya 2020-10-24 13:55:54 +02:00
parent 554d9b092f
commit ec3e7212b2
6 changed files with 100 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
composer.json
View file

@ -53,7 +53,7 @@
"shlinkio/shlink-common": "^3.2.0",
"shlinkio/shlink-config": "^1.0",
"shlinkio/shlink-event-dispatcher": "^1.4",
"shlinkio/shlink-importer": "^1.0",
"shlinkio/shlink-importer": "^1.0.1",
"shlinkio/shlink-installer": "^5.1.0",
"shlinkio/shlink-ip-geolocation": "^1.5",
"symfony/console": "^5.1",

4
module/Core/src/Entity/ShortUrl.php
View file

@ -15,7 +15,7 @@ use Shlinkio\Shlink\Core\Exception\ShortCodeCannotBeRegeneratedException;
use Shlinkio\Shlink\Core\Model\ShortUrlEdit;
use Shlinkio\Shlink\Core\Model\ShortUrlMeta;
use Shlinkio\Shlink\Core\Validation\ShortUrlMetaInputFilter;
use Shlinkio\Shlink\Importer\Model\ShlinkUrl;
use Shlinkio\Shlink\Importer\Model\ImportedShlinkUrl;
use function count;
use function Shlinkio\Shlink\Core\generateRandomShortCode;
@ -58,7 +58,7 @@ class ShortUrl extends AbstractEntity
}
public static function fromImport(
ShlinkUrl $url,
ImportedShlinkUrl $url,
string $source,
bool $importShortCode,
?DomainResolverInterface $domainResolver = null

29
module/Core/src/Importer/ImportedLinksProcessor.php
View file

@ -7,9 +7,11 @@ namespace Shlinkio\Shlink\Core\Importer;
use Doctrine\ORM\EntityManagerInterface;
use Shlinkio\Shlink\Core\Domain\Resolver\DomainResolverInterface;
use Shlinkio\Shlink\Core\Entity\ShortUrl;
use Shlinkio\Shlink\Core\Repository\ShortUrlRepositoryInterface;
use Shlinkio\Shlink\Core\Util\DoctrineBatchIterator;
use Shlinkio\Shlink\Core\Util\TagManagerTrait;
use Shlinkio\Shlink\Importer\ImportedLinksProcessorInterface;
use Shlinkio\Shlink\Importer\Model\ShlinkUrl;
use Shlinkio\Shlink\Importer\Model\ImportedShlinkUrl;
class ImportedLinksProcessor implements ImportedLinksProcessorInterface
{
@ -25,31 +27,28 @@ class ImportedLinksProcessor implements ImportedLinksProcessorInterface
}
/**
* @param ShlinkUrl[] $shlinkUrls
* @param iterable|ImportedShlinkUrl[] $shlinkUrls
*/
public function process(iterable $shlinkUrls, string $source, array $params): void
{
/** @var ShortUrlRepositoryInterface $shortUrlRepo */
$shortUrlRepo = $this->em->getRepository(ShortUrl::class);
$importShortCodes = $params['import_short_codes'];
$count = 0;
$persistBlock = 100;
$iterable = new DoctrineBatchIterator($shlinkUrls, $this->em, 100);
foreach ($shlinkUrls as $url) {
$count++;
/** @var ImportedShlinkUrl $url */
foreach ($iterable as $url) {
// Skip already imported URLs
if ($shortUrlRepo->importedUrlExists($url, $source, $importShortCodes)) {
continue;
}
$shortUrl = ShortUrl::fromImport($url, $source, $importShortCodes, $this->domainResolver);
$shortUrl->setTags($this->tagNamesToEntities($this->em, $url->tags()));
// TODO Handle errors while creating short URLs, to avoid making the whole process fail
// * Duplicated short code
$this->em->persist($shortUrl);
// Flush and clear after X iterations
if ($count % $persistBlock === 0) {
$this->em->flush();
$this->em->clear();
}
}
$this->em->flush();
$this->em->clear();
}
}

13
module/Core/src/Repository/ShortUrlRepository.php
View file

@ -11,6 +11,7 @@ use Shlinkio\Shlink\Common\Util\DateRange;
use Shlinkio\Shlink\Core\Entity\ShortUrl;
use Shlinkio\Shlink\Core\Model\ShortUrlMeta;
use Shlinkio\Shlink\Core\Model\ShortUrlsOrdering;
use Shlinkio\Shlink\Importer\Model\ImportedShlinkUrl;
use function array_column;
use function array_key_exists;
@ -254,4 +255,16 @@ DQL;
return $qb->getQuery()->getOneOrNullResult();
}
public function importedUrlExists(ImportedShlinkUrl $url, string $source, bool $importShortCodes): bool
{
$findConditions = ['importSource' => $source];
if ($importShortCodes) {
$findConditions['shortCode'] = $url->shortCode();
} else {
$findConditions['longUrl'] = $url->longUrl();
}
return $this->count($findConditions) > 0;
}
}

3
module/Core/src/Repository/ShortUrlRepositoryInterface.php
View file

@ -9,6 +9,7 @@ use Shlinkio\Shlink\Common\Util\DateRange;
use Shlinkio\Shlink\Core\Entity\ShortUrl;
use Shlinkio\Shlink\Core\Model\ShortUrlMeta;
use Shlinkio\Shlink\Core\Model\ShortUrlsOrdering;
use Shlinkio\Shlink\Importer\Model\ImportedShlinkUrl;
interface ShortUrlRepositoryInterface extends ObjectRepository
{
@ -30,4 +31,6 @@ interface ShortUrlRepositoryInterface extends ObjectRepository
public function shortCodeIsInUse(string $slug, ?string $domain): bool;
public function findOneMatching(string $url, array $tags, ShortUrlMeta $meta): ?ShortUrl;
public function importedUrlExists(ImportedShlinkUrl $url, string $source, bool $importShortCodes): bool;
}

67
module/Core/src/Util/DoctrineBatchIterator.php Normal file
View file

@ -0,0 +1,67 @@
<?php
declare(strict_types=1);
namespace Shlinkio\Shlink\Core\Util;
use Doctrine\ORM\EntityManagerInterface;
use IteratorAggregate;
use Throwable;
/**
* Inspired by ocramius/doctrine-batch-utils https://github.com/Ocramius/DoctrineBatchUtils
*/
class DoctrineBatchIterator implements IteratorAggregate
{
private iterable $resultSet;
private EntityManagerInterface $em;
private int $batchSize;
public function __construct(iterable $resultSet, EntityManagerInterface $em, int $batchSize)
{
$this->resultSet = $resultSet;
$this->em = $em;
$this->batchSize = $batchSize;
}
/**
* @throws Throwable
*/
public function getIterator(): iterable
{
$iteration = 0;
$resultSet = $this->resultSet;
$this->em->beginTransaction();
try {
foreach ($resultSet as $key => $value) {
$iteration++;
yield $key => $value;
$this->flushAndClearBatch($iteration);
}
} catch (Throwable $e) {
$this->em->rollback();
throw $e;
}
$this->flushAndClearEntityManager();
$this->em->commit();
}
private function flushAndClearBatch(int $iteration): void
{
if ($iteration % $this->batchSize) {
return;
}
$this->flushAndClearEntityManager();
}
private function flushAndClearEntityManager(): void
{
$this->em->flush();
$this->em->clear();
}
}