feat: add limit options to the slowest bridges

bridges/DarkReadingBridge.php

@@ -33,7 +33,8 @@ class DarkReadingBridge extends FeedExpander {
 				'Insider Threats' => '663_Insider%20Threats',
 				'Vulnerability Management' => '664_Vulnerability%20Management',
 			)
-		)
+		),
+		'limit' => self::LIMIT,
 	));
 
 	public function collectData(){
@@ -48,7 +49,8 @@ class DarkReadingBridge extends FeedExpander {
 		if ($feed_id != '000') {
 			$feed_url .= '?f_n=' . $feed_id . '&f_ln=' . $feed_name;
 		}
-		$this->collectExpandableDatas($feed_url, 20);
+		$limit = $this->getInput('limit') ?? 10;
+		$this->collectExpandableDatas($feed_url, $limit);
 	}
 
 	protected function parseItem($newsItem){

bridges/FindACrewBridge.php

@@ -26,7 +26,8 @@ class FindACrewBridge extends BridgeAbstract {
 			'distance' => array(
 				'name' => 'Limit boundary of search in KM',
 				'title' => 'Boundary of the search in kilometers when using longitude and latitude'
-			)
+			),
+			'limit' => self::LIMIT,
 		)
 	);
 
@@ -59,7 +60,8 @@ class FindACrewBridge extends BridgeAbstract {
 		$html = getSimpleHTMLDOM($url, $header, $opts) or returnClientError('No results for this query.');
 
 		$annonces = $html->find('.css_SrhRst');
-		foreach ($annonces as $annonce) {
+		$limit = $this->getInput('limit') ?? 10;
+		foreach (array_slice($annonces, 0, $limit) as $annonce) {
 			$item = array();
 
 			$link = parent::getURI() . $annonce->find('.lstsum-btn-con a', 0)->href;

bridges/FolhaDeSaoPauloBridge.php

@@ -63,6 +63,7 @@ class FolhaDeSaoPauloBridge extends FeedExpander {
 			$feed_url = self::URI . '/' . $this->getInput('feed');
 		}
 		Debug::log('URL: ' . $feed_url);
-		$this->collectExpandableDatas($feed_url, $this->getInput('amount'));
+		$limit = $this->getInput('amount');
+		$this->collectExpandableDatas($feed_url, $limit);
 	}
 }

bridges/GQMagazineBridge.php

@@ -32,6 +32,7 @@ class GQMagazineBridge extends BridgeAbstract
 			'required' => true,
 			'exampleValue' => 'sexe/news'
 		),
+		'limit' => self::LIMIT,
 	));
 
 	const REPLACED_ATTRIBUTES = array(
@@ -76,7 +77,12 @@ class GQMagazineBridge extends BridgeAbstract
 
 		// Since GQ don't want simple class scrapping, let's do it the hard way and ... discover content !
 		$main = $html->find('main', 0);
+		$limit = $this->getInput('limit') ?? 10;
 		foreach ($main->find('a') as $link) {
+			if (count($this->items) >= $limit) {
+				break;
+			}
+
 			$uri = $link->href;
 			$date = $link->parent()->find('time', 0);
 

bridges/InternetArchiveBridge.php

@@ -23,7 +23,8 @@ class InternetArchiveBridge extends BridgeAbstract {
 					'Web Archives' => 'web-archive',
 				),
 				'defaultValue' => 'uploads',
-			)
+			),
+			'limit' => self::LIMIT,
 		)
 	);
 
@@ -72,7 +73,8 @@ class InternetArchiveBridge extends BridgeAbstract {
 		if ($this->getInput('content') !== 'posts') {
 			$detailsDivNumber = 0;
 
-			foreach ($html->find('div.results > div[data-id]') as $index => $result) {
+			$results = $html->find('div.results > div[data-id]');
+			foreach ($results as $index => $result) {
 				$item = array();
 
 				if (in_array($result->class, $this->skipClasses)) {
@@ -110,6 +112,11 @@ class InternetArchiveBridge extends BridgeAbstract {
 				}
 
 				$detailsDivNumber++;
+
+				$limit = $this->getInput('limit') ?? 10;
+				if (count($this->items) >= $limit) {
+					break;
+				}
 			}
 		}
 
@@ -302,7 +309,7 @@ EOD;
 
 			$items[] = $item;
 
-			if (count($items) >= 10) {
+			if (count($items) >= $this->getInput('limit') ?? 10) {
 				break;
 			}
 		}

bridges/NextInpactBridge.php

@@ -55,7 +55,8 @@ class NextInpactBridge extends FeedExpander {
 				'Hide Brief' => '1',
 				'Only Brief' => '2'
 			)
-		)
+		),
+		'limit' => self::LIMIT,
 	));
 
 	public function collectData(){
@@ -80,7 +81,9 @@ class NextInpactBridge extends FeedExpander {
 			$feed = 'params';
 		}
 
-		$this->collectExpandableDatas($base_uri . 'rss/' . $feed . '.xml' . $args);
+		$url = sprintf('%srss/%s.xml%s', $base_uri, $feed, $args);
+		$limit = $this->getInput('limit') ?? 10;
+		$this->collectExpandableDatas($url, $limit);
 	}
 
 	protected function parseItem($newsItem){

bridges/OpenlyBridge.php

@@ -130,7 +130,9 @@ class OpenlyBridge extends BridgeAbstract {
 			$this->feedTitle = $html->find('a.tooltipitem', 0)->plaintext;
 		}
 
-		foreach($html->find('div.item') as $div) {
+		$items = $html->find('div.item');
+		$limit = 5;
+		foreach(array_slice($items, 0, $limit) as $div) {
 			$this->items[] = $this->getArticle($div->find('a', 0)->href);
 
 			if (count($this->items) >= $this->itemLimit) {

bridges/PcGamerBridge.php

@@ -7,11 +7,18 @@ class PcGamerBridge extends BridgeAbstract
 		updates and news on all your favorite PC gaming franchises.';
 	const MAINTAINER = 'IceWreck, mdemoss';
 
+	const PARAMETERS = [
+		[
+			'limit' => self::LIMIT,
+		]
+	];
+
 	public function collectData()
 	{
 		$html = getSimpleHTMLDOMCached($this->getURI(), 300);
 		$stories = $html->find('a.article-link');
-		foreach ($stories as $element) {
+		$limit = $this->getInput('limit') ?? 10;
+		foreach (array_slice($stories, 0, $limit) as $element) {
 			$item = array();
 			$item['uri'] = $element->href;
 			$articleHtml = getSimpleHTMLDOMCached($item['uri']);

bridges/UnogsBridge.php

@@ -17,7 +17,8 @@ class UnogsBridge extends BridgeAbstract {
 					'What\'s New' => 'new last 7 days',
 					'Expiring' => 'expiring'
 				)
-			)
+			),
+			'limit' => self::LIMIT,
 		),
 		'Global' => array(),
 		'Country' => array(
@@ -160,8 +161,17 @@ EOD;
 				break;
 		}
 
-		$api_url = self::URI . '/api/search?query=' . urlencode($feed)
+		$limit = $this->getInput('limit') ?? 30;
-				. ($country_code ? '&countrylist=' . $country_code : '') . '&limit=30';
+
+		// https://rapidapi.com/unogs/api/unogsng/details
+		$api_url = sprintf(
+			'%s/api/search?query=%s%s&limit=%s',
+			self::URI,
+			urlencode($feed),
+			$country_code ? '&countrylist=' . $country_code : '',
+			$limit
+		);
+
 		$json_data = $this->getJSON($api_url);
 		$movies = $json_data['results'];
 

bridges/WeLiveSecurityBridge.php

@@ -5,6 +5,11 @@ class WeLiveSecurityBridge extends FeedExpander {
 	const NAME = 'We Live Security';
 	const URI = 'https://www.welivesecurity.com/';
 	const DESCRIPTION = 'Returns the newest articles.';
+	const PARAMETERS = [
+		[
+			'limit' => self::LIMIT,
+		],
+	];
 
 	protected function parseItem($item){
 		$item = parent::parseItem($item);
@@ -27,6 +32,7 @@ class WeLiveSecurityBridge extends FeedExpander {
 
 	public function collectData(){
 		$feed = static::URI . 'feed/';
-		$this->collectExpandableDatas($feed);
+		$limit = $this->getInput('limit') ?? 10;
+		$this->collectExpandableDatas($feed, $limit);
 	}
 }

bridges/WiredBridge.php

@@ -22,7 +22,8 @@ class WiredBridge extends FeedExpander {
 				'WIRED Guides' => 'wired-guide',		// /feed/tag/wired-guide/latest/rss
 				'Photo' => 'photo'						// /feed/category/photo/latest/rss
 			)
-		)
+		),
+		'limit' => self::LIMIT,
 	));
 
 	public function collectData(){
@@ -42,7 +43,8 @@ class WiredBridge extends FeedExpander {
 		}
 		$feed_url .= 'rss';
 
-		$this->collectExpandableDatas($feed_url);
+		$limit = $this->getInput('limit') ?? -1;
+		$this->collectExpandableDatas($feed_url, $limit);
 	}
 
 	protected function parseItem($newsItem){

bridges/ZDNetBridge.php

@@ -156,7 +156,8 @@ class ZDNetBridge extends FeedExpander {
 					'ZDNet Government' => 'blog/government'
 				)
 			)
-		)
+		),
+		'limit' => self::LIMIT,
 	));
 
 	public function collectData(){
@@ -167,7 +168,8 @@ class ZDNetBridge extends FeedExpander {
 			$baseUri = str_replace('www.', 'downloads.', $baseUri);
 		}
 		$url = $baseUri . trim($feed, '/') . '/rss.xml';
-		$this->collectExpandableDatas($url);
+		$limit = $this->getInput('limit') ?? 10;
+		$this->collectExpandableDatas($url, $limit);
 	}
 
 	protected function parseItem($item){

lib/BridgeAbstract.php

@@ -87,6 +87,16 @@ abstract class BridgeAbstract implements BridgeInterface {
 	 */
 	const TEST_DETECT_PARAMETERS = array();
 
+	/**
+	 * This is a convenient const for the limit option in bridge contexts.
+	 * Can be inlined and modified if necessary.
+	 */
+	protected const LIMIT = [
+		'name'          => 'Limit',
+		'type'          => 'number',
+		'title'         => 'Maximum number of items to return',
+	];
+
 	/**
 	 * Holds the list of items collected by the bridge
 	 *