diff --git a/bridges/TwitterBridge.php b/bridges/TwitterBridge.php
index 0d8b0243..6372def6 100644
--- a/bridges/TwitterBridge.php
+++ b/bridges/TwitterBridge.php
@@ -2,6 +2,7 @@
class TwitterBridge extends BridgeAbstract {
const NAME = 'Twitter Bridge';
const URI = 'https://twitter.com/';
+ const API_URI = 'https://api.twitter.com';
const CACHE_TIMEOUT = 300; // 5min
const DESCRIPTION = 'returns tweets';
const MAINTAINER = 'pmaziere';
@@ -168,6 +169,27 @@ EOD
}
}
+ public function getApiURI() {
+ switch($this->queriedContext) {
+ case 'By keyword or hashtag':
+ return self::API_URI
+ . '/2/search/adaptive.json?q='
+ . urlencode($this->getInput('q'))
+ . '&tweet_mode=extended';
+ case 'By username':
+ return self::API_URI
+ . '/2/timeline/profile/'
+ . $this->getRestId($this->getInput('u'))
+ . '.json?tweet_mode=extended';
+ case 'By list':
+ return self::API_URI
+ . '/2/timeline/list.json?list_id='
+ . $this->getListId($this->getInput('user'), $this->getInput('list'))
+ . '&tweet_mode=extended';
+ default: returnServerError('Invalid query context !');
+ }
+ }
+
public function collectData(){
$html = '';
$page = $this->getURI();
@@ -176,14 +198,9 @@ EOD
'User-Agent: Mozilla/5.0 (Windows NT 9.0; WOW64; Trident/7.0; rv:11.0) like Gecko'
);
- if(php_sapi_name() === 'cli' && empty(ini_get('curl.cainfo'))) {
- $cookies = $this->getCookies($page);
- $html = getSimpleHTMLDOM($page, array_merge($header, array("Cookie: $cookies")));
- } else {
- $html = getSimpleHTMLDOM($page, $header, array(CURLOPT_COOKIEFILE => ''));
- }
+ $data = json_decode($this->getApiContents($this->getApiURI()));
- if(!$html) {
+ if(!$data) {
switch($this->queriedContext) {
case 'By keyword or hashtag':
returnServerError('No results for this query.');
@@ -196,75 +213,33 @@ EOD
$hidePictures = $this->getInput('nopic');
- foreach($html->find('div.js-stream-tweet') as $tweet) {
+ foreach($data->globalObjects->tweets as $tweet) {
// Skip retweets?
if($this->getInput('noretweet')
- && $tweet->find('div.context span.js-retweet-text a', 0)) {
- continue;
- }
-
- // remove 'invisible' content
- foreach($tweet->find('.invisible') as $invisible) {
- $invisible->outertext = '';
- }
-
- // Skip protmoted tweets
- $heading = $tweet->previousSibling();
- if(!is_null($heading) &&
- $heading->getAttribute('class') === 'promoted-tweet-heading'
- ) {
+ && isset($tweet->retweeted_status_id_str)) {
continue;
}
$item = array();
// extract username and sanitize
- $item['username'] = htmlspecialchars_decode($tweet->getAttribute('data-screen-name'), ENT_QUOTES);
- // extract fullname (pseudonym)
- $item['fullname'] = htmlspecialchars_decode($tweet->getAttribute('data-name'), ENT_QUOTES);
- // get author
+ $user_info = $this->getUserInformation($tweet->user_id_str, $data->globalObjects);
+
+ $item['username'] = $user_info->name;
+ $item['fullname'] = $user_info->screen_name;
$item['author'] = $item['fullname'] . ' (@' . $item['username'] . ')';
- if($rt = $tweet->find('div.context span.js-retweet-text a', 0)) {
- $item['author'] .= ' RT: @' . $rt->plaintext;
- }
- // get avatar link
- $item['avatar'] = $tweet->find('img', 0)->src;
- // get TweetID
- $item['id'] = $tweet->getAttribute('data-tweet-id');
- // get tweet link
- $item['uri'] = self::URI . substr($tweet->find('a.js-permalink', 0)->getAttribute('href'), 1);
+ $item['avatar'] = $user_info->profile_image_url_https;
+
+ $item['id'] = $tweet->id_str;
+ $item['uri'] = self::URI . $tweet->user_id_str . '/status/' . $item['id'];
// extract tweet timestamp
- $item['timestamp'] = $tweet->find('span.js-short-timestamp', 0)->getAttribute('data-time');
+ $item['timestamp'] = $tweet->created_at;
+
// generate the title
- $item['title'] = strip_tags($this->fixAnchorSpacing(htmlspecialchars_decode(
- $tweet->find('p.js-tweet-text', 0), ENT_QUOTES), ''));
+ $item['title'] = $tweet->full_text;
+ $cleanedTweet = $tweet->full_text;
- switch($this->queriedContext) {
- case 'By list':
- // Check if filter applies to list (using raw content)
- if($this->getInput('filter')) {
- if(stripos($tweet->find('p.js-tweet-text', 0)->plaintext, $this->getInput('filter')) === false) {
- continue 2; // switch + for-loop!
- }
- }
- break;
- default:
- }
-
- $this->processContentLinks($tweet);
- $this->processEmojis($tweet);
-
- // get tweet text
- $cleanedTweet = str_replace(
- 'href="/',
- 'href="' . self::URI,
- $tweet->find('p.js-tweet-text', 0)->innertext
- );
-
- // fix anchors missing spaces in-between
- $cleanedTweet = $this->fixAnchorSpacing($cleanedTweet);
-
- // Add picture to content
+ // Add avatar
$picture_html = '';
if(!$hidePictures) {
$picture_html = <<getImageURI($tweet);
- if(!$this->getInput('noimg') && !is_null($images)) {
-
- foreach ($images as $image) {
-
- // Set image scaling
- $image_orig = $this->getInput('noimgscaling') ? $image : $image . ':orig';
- $image_thumb = $this->getInput('noimgscaling') ? $image : $image . ':thumb';
-
+ if(isset($tweet->extended_entities->media) && !$this->getInput('noimg')) {
+ foreach($tweet->extended_entities->media as $media) {
+ $image = $media->media_url_https;
+ $display_image = $media->display_url;
// add enclosures
- $item['enclosures'][] = $image_orig;
+ $item['enclosures'][] = $image;
$image_html .= <<
+
+ src="{$display_image}" />
EOD;
}
}
- // add content
+ switch($this->queriedContext) {
+ case 'By list':
+ // Check if filter applies to list (using raw content)
+ if($this->getInput('filter')) {
+ if(stripos($cleanedTweet, $this->getInput('filter')) === false) {
+ continue 2; // switch + for-loop!
+ }
+ }
+ break;
+ default:
+ }
+
$item['content'] = <<
{$picture_html}
@@ -315,151 +296,85 @@ EOD;
EOD;
- // add quoted tweet
- $quotedTweet = $tweet->find('div.QuoteTweet', 0);
- if($quotedTweet) {
- // get tweet text
- $cleanedQuotedTweet = str_replace(
- 'href="/',
- 'href="' . self::URI,
- $quotedTweet->find('div.tweet-text', 0)->innertext
- );
-
- $this->processContentLinks($quotedTweet);
- $this->processEmojis($quotedTweet);
-
- // Add embeded image to content
- $quotedImage_html = '';
- $quotedImages = $this->getQuotedImageURI($tweet);
-
- if(!$this->getInput('noimg') && !is_null($quotedImages)) {
-
- foreach ($quotedImages as $image) {
-
- // Set image scaling
- $image_orig = $this->getInput('noimgscaling') ? $image : $image . ':orig';
- $image_thumb = $this->getInput('noimgscaling') ? $image : $image . ':thumb';
-
- // add enclosures
- $item['enclosures'][] = $image_orig;
-
- $quotedImage_html .= <<
-
-
-EOD;
- }
- }
-
- $item['content'] = <<
-
-
{$cleanedQuotedTweet}
-
-
-EOD;
- }
$item['content'] = htmlspecialchars_decode($item['content'], ENT_QUOTES);
// put out
$this->items[] = $item;
}
+
+ usort($this->items, array('TwitterBridge', 'compareTweetId'));
}
- private function processEmojis($tweet){
- // process emojis (reduce size)
- foreach($tweet->find('img.Emoji') as $img) {
- $img->style .= ' height: 1em;';
+ private static function compareTweetId($tweet1, $tweet2) {
+ return (intval($tweet1['id']) < intval($tweet2['id']) ? 1 : -1);
+ }
+
+ //The aim of this function is to get an API key and a guest token
+ //This function takes 2 requests, and therefore is cached
+ private function getApiKey() {
+
+ $cacheFac = new CacheFactory();
+ $cacheFac->setWorkingDir(PATH_LIB_CACHES);
+ $cache = $cacheFac->create(Configuration::getConfig('cache', 'type'));
+ $cache->setScope(get_called_class());
+ $cache->setKey(array('api_key'));
+ $data = $cache->loadData();
+
+ if($data === null) {
+ $twitterPage = getContents('https://twitter.com');
+ $jsMainRegex = '/(https:\/\/abs\.twimg\.com\/responsive-web\/web\/main\.[^\.]+\.js)/m';
+ preg_match_all($jsMainRegex, $twitterPage, $jsMainMatches, PREG_SET_ORDER, 0);
+ $jsLink = $jsMainMatches[0][0];
+ $guestTokenRegex = '/gt=([0-9]*)/m';
+ preg_match_all($guestTokenRegex, $twitterPage, $guestTokenMatches, PREG_SET_ORDER, 0);
+ $guestToken = $guestTokenMatches[0][1];
+
+ $jsContent = getContents($jsLink);
+ $apiKeyRegex = '/([a-zA-Z0-9]{59}%[a-zA-Z0-9]{44})/m';
+ preg_match_all($apiKeyRegex, $jsContent, $apiKeyMatches, PREG_SET_ORDER, 0);
+ $apiKey = $apiKeyMatches[0][0];
+ $cache->saveData(array($apiKey, $guestToken));
+ return array($apiKey, $guestToken);
}
+
+ return $data;
+
}
- private function processContentLinks($tweet){
- // processing content links
- foreach($tweet->find('a') as $link) {
- if($link->hasAttribute('data-expanded-url')) {
- $link->href = $link->getAttribute('data-expanded-url');
- }
- $link->removeAttribute('data-expanded-url');
- $link->removeAttribute('data-query-source');
- $link->removeAttribute('rel');
- $link->removeAttribute('class');
- $link->removeAttribute('target');
- $link->removeAttribute('title');
- }
+ private function getApiContents($uri) {
+ $apiKeys = $this->getApiKey();
+ $headers = array('authorization: Bearer ' . $apiKeys[0],
+ 'x-guest-token: ' . $apiKeys[1],
+ );
+ return getContents($uri, $headers);
}
- private function fixAnchorSpacing($content){
- // fix anchors missing spaces in-between
- return str_replace(
- 'getApiContents($searchURL);
+ $searchResult = json_decode($searchResult);
+ return $searchResult->data->user->rest_id;
}
- private function getImageURI($tweet){
- // Find media in tweet
- $images = array();
+ private function getListId($username, $listName) {
+ $searchparams = urlencode('{"screenName":"'
+ . strtolower($username)
+ . '", "listSlug": "'
+ . $listName
+ . '", "withHighlightedLabel":false}');
+ $searchURL = self::API_URI . '/graphql/ErWsz9cObLel1BF-HjuBlA/ListBySlug?variables=' . $searchparams;
+ $searchResult = $this->getApiContents($searchURL);
+ $searchResult = json_decode($searchResult);
+ return $searchResult->data->user_by_screen_name->list->id_str;
+ }
- $container = $tweet->find('div.AdaptiveMedia-container', 0);
-
- if($container && $container->find('img', 0)) {
- foreach ($container->find('img') as $img) {
- $images[] = $img->src;
+ private function getUserInformation($userId, $apiData) {
+ foreach($apiData->users as $user) {
+ if($user->id_str == $userId) {
+ return $user;
}
}
-
- if (!empty($images)) {
- return $images;
- }
-
- return null;
}
- private function getQuotedImageURI($tweet){
- // Find media in tweet
- $images = array();
-
- $container = $tweet->find('div.QuoteMedia-container', 0);
-
- if($container && $container->find('img', 0)) {
- foreach ($container->find('img') as $img) {
- $images[] = $img->src;
- }
- }
-
- if (!empty($images)) {
- return $images;
- }
-
- return null;
- }
-
- private function getCookies($pageURL){
-
- $ctx = stream_context_create(array(
- 'http' => array(
- 'follow_location' => false
- )
- )
- );
- $a = file_get_contents($pageURL, 0, $ctx);
-
- //First request to get the cookie
- $cookies = '';
- foreach($http_response_header as $hdr) {
- if(stripos($hdr, 'Set-Cookie') !== false) {
- $cLine = explode(':', $hdr)[1];
- $cLine = explode(';', $cLine)[0];
- $cookies .= ';' . $cLine;
- }
- }
-
- return substr($cookies, 2);
- }
}
diff --git a/cache/pages/.gitkeep b/cache/pages/.gitkeep
deleted file mode 100644
index e69de29b..00000000
diff --git a/cache/server/.gitkeep b/cache/server/.gitkeep
deleted file mode 100644
index e69de29b..00000000