[TwitterBridge] Fix search, user, list ID (#3566)

* Add ability to fetch user, list tweet

* Fix user, search, list ID although list still broke

* clear whitespace

* Revert CACHE_TIMEOUT

* clear whitespace, change single quote

* Clear PHP warning, add ability to get full-text if truncated

* Clear PHP warning

* clear warning

* clear whitespace

* Add check condition for mediaDetails.

* Add whitespace

* Add try catch exception for get full-text tweet

* clear warning

* clear warning

bridges/TwitterBridge.php

@@ -210,6 +210,16 @@ EOD
         }
     }
 
+    private function getFullText($id)
+    {
+        $url = sprintf(
+            'https://cdn.syndication.twimg.com/tweet-result?id=%s&lang=en',
+            $id
+        );
+
+        return json_decode(getContents($url), false);
+    }
+
     public function collectData()
     {
         // $data will contain an array of all found tweets (unfiltered)
@@ -220,13 +230,11 @@ EOD
         $tweets = [];
 
         // Get authentication information
-
+        $cache = RssBridge::getCache();
+        $api = new TwitterClient($cache);
         // Try to get all tweets
         switch ($this->queriedContext) {
             case 'By username':
-                $cache = RssBridge::getCache();
-                $api = new TwitterClient($cache);
-
                 $screenName = $this->getInput('u');
                 $screenName = trim($screenName);
                 $screenName = ltrim($screenName, '@');
@@ -243,30 +251,40 @@ EOD
                     'tweet_search_mode' => 'live',
                 ];
 
-                $data = $this->makeApiCall('/1.1/search/tweets.json', $params)->statuses;
+                $tweets = $api->search($params)->statuses;
+                $data = (object) [
+                    'tweets' => $tweets
+                ];
                 break;
 
             case 'By list':
                 // Does not work with the recent twitter changes
-                $params = [
+                // $params = [
-                'slug'              => strtolower($this->getInput('list')),
+                // 'slug'              => strtolower($this->getInput('list')),
-                'owner_screen_name' => strtolower($this->getInput('user')),
+                // 'owner_screen_name' => strtolower($this->getInput('user')),
-                'tweet_mode'        => 'extended',
+                // 'tweet_mode'        => 'extended',
+                // ];
+                $query = [
+                    'screenName' => strtolower($this->getInput('user')),
+                    'listSlug' => strtolower($this->getInput('list'))
                 ];
 
-                $data = $this->makeApiCall('/1.1/lists/statuses.json', $params);
+                $data = $api->fetchListTweets($query, $this->queriedContext);
                 break;
 
             case 'By list ID':
                 // Does not work with the recent twitter changes
-                $params = [
+                // $params = [
-                'list_id'           => $this->getInput('listid'),
+                // 'list_id'           => $this->getInput('listid'),
-                'tweet_mode'        => 'extended',
+                // 'tweet_mode'        => 'extended',
+                // ];
+
+                $query = [
+                    'listId' => $this->getInput('listid')
                 ];
 
-                $data = $this->makeApiCall('/1.1/lists/statuses.json', $params);
+                $data = $api->fetchListTweets($query, $this->queriedContext);
                 break;
-
             default:
                 returnServerError('Invalid query context !');
         }
@@ -314,6 +332,7 @@ EOD
             $this->feedIconUrl = $data->user_info->legacy->profile_image_url_https ?? null;
         }
 
+        $i = 0;
         foreach ($tweets as $tweet) {
             // Skip own Retweets...
             if (isset($tweet->retweeted_status) && $tweet->retweeted_status->user->id_str === $tweet->user->id_str) {
@@ -325,14 +344,6 @@ EOD
                 continue;
             }
 
-            switch ($this->queriedContext) {
-                case 'By username':
-                    if ($this->getInput('norep') && isset($tweet->in_reply_to_status_id)) {
-                        continue 2;
-                    }
-                    break;
-            }
-
             $item = [];
 
             $realtweet = $tweet;
@@ -341,11 +352,40 @@ EOD
                 $realtweet = $tweet->retweeted_status;
             }
 
+            if (isset($realtweet->truncated) && $realtweet->truncated) {
+                try {
+                    $realtweet = $this->getFullText($realtweet->id_str);
+                } catch (HttpException $e) {
+                    $realtweet = $tweet;
+                }
+            }
+
+            switch ($this->queriedContext) {
+                case 'By username':
+                    if ($this->getInput('norep') && isset($tweet->in_reply_to_status_id)) {
+                        continue 2;
+                    }
                     $item['username']  = $data->user_info->legacy->screen_name;
                     $item['fullname']  = $data->user_info->legacy->name;
                     $item['avatar']    = $data->user_info->legacy->profile_image_url_https;
-            $item['timestamp'] = $realtweet->created_at;
                     $item['id']        = $realtweet->id_str;
+                    break;
+                case 'By list':
+                case 'By list ID':
+                    $item['username']  = $data->userIds[$i]->legacy->screen_name;
+                    $item['fullname']  = $data->userIds[$i]->legacy->name;
+                    $item['avatar']    = $data->userIds[$i]->legacy->profile_image_url_https;
+                    $item['id']        = $realtweet->conversation_id_str;
+                    break;
+                case 'By keyword or hashtag':
+                    $item['username']  = $realtweet->user->screen_name;
+                    $item['fullname']  = $realtweet->user->name;
+                    $item['avatar']    = $realtweet->user->profile_image_url_https;
+                    $item['id']        = $realtweet->id_str;
+                    break;
+            }
+
+            $item['timestamp'] = $realtweet->created_at;
             $item['uri']       = self::URI . $item['username'] . '/status/' . $item['id'];
             $item['author']    = (isset($tweet->retweeted_status) ? 'RT: ' : '')
                          . $item['fullname']
@@ -353,7 +393,11 @@ EOD
                          . $item['username'] . ')';
 
             // Convert plain text URLs into HTML hyperlinks
+            if (isset($realtweet->full_text)) {
                 $fulltext = $realtweet->full_text;
+            } else {
+                $fulltext = $realtweet->text;
+            }
             $cleanedTweet = $fulltext;
 
             $foundUrls = false;
@@ -385,7 +429,7 @@ EOD
             if ($foundUrls === false) {
                 // fallback to regex'es
                 $reg_ex = '/(http|https|ftp|ftps)\:\/\/[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(\/\S*)?/';
-                if (preg_match($reg_ex, $realtweet->full_text, $url)) {
+                if (preg_match($reg_ex, $fulltext, $url)) {
                     $cleanedTweet = preg_replace(
                         $reg_ex,
                         "<a href='{$url[0]}' target='_blank'>{$url[0]}</a> ",
@@ -410,10 +454,17 @@ EOD
 EOD;
             }
 
+            $medias = [];
+            if (isset($realtweet->extended_entities->media)) {
+                $medias = $realtweet->extended_entities->media;
+            } else if (isset($realtweet->mediaDetails)) {
+                $medias = $realtweet->mediaDetails;
+            }
+
             // Get images
             $media_html = '';
-            if (isset($realtweet->extended_entities->media) && !$this->getInput('noimg')) {
+            if (!$this->getInput('noimg')) {
-                foreach ($realtweet->extended_entities->media as $media) {
+                foreach ($medias as $media) {
                     switch ($media->type) {
                         case 'photo':
                             $image = $media->media_url_https . '?name=orig';
@@ -496,6 +547,7 @@ EOD;
 EOD;
 
             // put out
+            $i++;
             $this->items[] = $item;
         }
 

lib/TwitterClient.php

@@ -20,6 +20,88 @@ class TwitterClient
         $this->authorization = 'AAAAAAAAAAAAAAAAAAAAAGHtAgAAAAAA%2Bx7ILXNILCqkSGIzy6faIHZ9s3Q%3DQy97w6SIrzE7lQwPJEYQBsArEE2fC25caFwRBvAGi456G09vGR';
     }
 
+    private function extractTweetAndUsersFromGraphQL($timeline)
+    {
+        if (isset($timeline->data->user)) {
+            $result = $timeline->data->user->result;
+            $instructions = $result->timeline_v2->timeline->instructions;
+        } else {
+            $result = $timeline->data->list->timeline_response;
+            $instructions = $result->timeline->instructions;
+        }
+        if (isset($result->__typename) && $result->__typename === 'UserUnavailable') {
+            throw new \Exception('UserUnavailable');
+        }
+        $instructionTypes = [
+            'TimelineAddEntries',
+            'TimelineClearCache',
+            'TimelinePinEntry', // unclear purpose, maybe pinned tweet?
+        ];
+        if (!isset($instructions[1]) && isset($timeline->data->user)) {
+            throw new \Exception('The account exists but has not tweeted yet?');
+        }
+
+        $entries = null;
+        foreach ($instructions as $instruction) {
+            $instructionType = '';
+            if (isset($instruction->type)) {
+                $instructionType = $instruction->type;
+            } else {
+                $instructionType = $instruction->__typename;
+            }
+
+            if ($instructionType === 'TimelineAddEntries') {
+                $entries = $instruction->entries;
+                break;
+            }
+        }
+        if (!$entries) {
+            throw new \Exception(sprintf('Unable to find time line tweets in: %s', implode(',', array_column($instructions, 'type'))));
+        }
+
+        $tweets = [];
+        $userIds = [];
+        foreach ($entries as $entry) {
+            $entryType = '';
+
+            if (isset($entry->content->entryType)) {
+                $entryType = $entry->content->entryType;
+            } else {
+                $entryType = $entry->content->__typename;
+            }
+
+            if ($entryType !== 'TimelineTimelineItem') {
+                continue;
+            }
+
+            if (isset($timeline->data->user)) {
+                if (!isset($entry->content->itemContent->tweet_results->result->legacy)) {
+                    continue;
+                }
+                $tweets[] = $entry->content->itemContent->tweet_results->result->legacy;
+
+                $userIds[] = $entry->content->itemContent->tweet_results->result->core->user_results->result;
+            } else {
+                if (!isset($entry->content->content->tweetResult->result->legacy)) {
+                    continue;
+                }
+                $tweets[] = $entry->content->content->tweetResult->result->legacy;
+
+                $userIds[] = $entry->content->content->tweetResult->result->core->user_result->result;
+            }
+        }
+
+        return (object) [
+            'userIds' => $userIds,
+            'tweets' => $tweets,
+        ];
+    }
+
+    private function extractTweetFromSearch($searchResult)
+    {
+        return $searchResult->statuses;
+    }
+
     public function fetchUserTweets(string $screenName): \stdClass
     {
         $this->fetchGuestToken();
@@ -36,58 +118,66 @@ class TwitterClient
         }
 
         try {
-            $timeline = $this->fetchTimeline($userInfo->rest_id);
+            $timeline = $this->fetchTimelineUsingSearch($screenName);
         } catch (HttpException $e) {
             if ($e->getCode() === 403) {
                 $this->data['guest_token'] = null;
                 $this->fetchGuestToken();
-                $timeline = $this->fetchTimeline($userInfo->rest_id);
+                $timeline = $this->fetchTimelineUsingSearch($screenName);
             } else {
                 throw $e;
             }
         }
 
-        $result = $timeline->data->user->result;
+        $tweets = $this->extractTweetFromSearch($timeline);
-        if ($result->__typename === 'UserUnavailable') {
-            throw new \Exception('UserUnavailable');
-        }
-        $instructionTypes = [
-            'TimelineAddEntries',
-            'TimelineClearCache',
-            'TimelinePinEntry', // unclear purpose, maybe pinned tweet?
-        ];
-        $instructions = $result->timeline_v2->timeline->instructions;
-        if (!isset($instructions[1])) {
-            throw new \Exception('The account exists but has not tweeted yet?');
-        }
 
-        $entries = null;
-        foreach ($instructions as $instruction) {
-            if ($instruction->type === 'TimelineAddEntries') {
-                $entries = $instruction->entries;
-                break;
-            }
-        }
-        if (!$entries) {
-            throw new \Exception(sprintf('Unable to find time line tweets in: %s', implode(',', array_column($instructions, 'type'))));
-        }
-
-        $tweets = [];
-        foreach ($entries as $entry) {
-            if ($entry->content->entryType !== 'TimelineTimelineItem') {
-                continue;
-            }
-            if (!isset($entry->content->itemContent->tweet_results->result->legacy)) {
-                continue;
-            }
-            $tweets[] = $entry->content->itemContent->tweet_results->result->legacy;
-        }
         return (object) [
             'user_info' => $userInfo,
             'tweets' => $tweets,
         ];
     }
 
+    public function fetchListTweets($query, $operation = '')
+    {
+        $id = '';
+        $this->fetchGuestToken();
+        if ($operation == 'By list') {
+            try {
+                $listInfo = $this->fetchListInfoBySlug($query['screenName'], $query['listSlug']);
+                $id = $listInfo->id_str;
+            } catch (HttpException $e) {
+                if ($e->getCode() === 403) {
+                    $this->data['guest_token'] = null;
+                    $this->fetchGuestToken();
+                    $listInfo = $this->fetchListInfoBySlug($query['screenName'], $query['listSlug']);
+                    $id = $listInfo->id_str;
+                } else {
+                    throw $e;
+                }
+            }
+        } else if ($operation == 'By list ID') {
+            $id = $query['listId'];
+        } else {
+            throw new \Exception('Unknown operation to make list tweets');
+        }
+
+        try {
+            $timeline = $this->fetchListTimeline($id);
+        } catch (HttpException $e) {
+            if ($e->getCode() === 403) {
+                $this->data['guest_token'] = null;
+                $this->fetchGuestToken();
+                $timeline = $this->fetchListTimeline($id);
+            } else {
+                throw $e;
+            }
+        }
+
+        $data = $this->extractTweetAndUsersFromGraphQL($timeline);
+
+        return $data;
+    }
+
     private function fetchGuestToken(): void
     {
         if (isset($this->data['guest_token'])) {
@@ -173,6 +263,158 @@ class TwitterClient
         return $response;
     }
 
+    private function fetchTimelineUsingSearch($screenName)
+    {
+        $params = [
+            'q' => 'from:' . $screenName,
+            'modules' => 'status',
+            'result_type' => 'recent'
+        ];
+        $response = $this->search($params);
+        return $response;
+    }
+
+    public function search($queryParam)
+    {
+         $url = sprintf(
+             'https://api.twitter.com/1.1/search/tweets.json?%s',
+             http_build_query($queryParam)
+         );
+        $response = Json::decode(getContents($url, $this->createHttpHeaders()), false);
+        return $response;
+    }
+
+    private function fetchListInfoBySlug($screenName, $listSlug)
+    {
+        if (isset($this->data[$screenName . '-' . $listSlug])) {
+            return $this->data[$screenName . '-' . $listSlug];
+        }
+
+        $features = [
+            'android_graphql_skip_api_media_color_palette' => false,
+            'blue_business_profile_image_shape_enabled' => false,
+            'creator_subscriptions_subscription_count_enabled' => false,
+            'creator_subscriptions_tweet_preview_api_enabled' => true,
+            'freedom_of_speech_not_reach_fetch_enabled' => false,
+            'graphql_is_translatable_rweb_tweet_is_translatable_enabled' => false,
+            'hidden_profile_likes_enabled' => false,
+            'highlights_tweets_tab_ui_enabled' => false,
+            'interactive_text_enabled' => false,
+            'longform_notetweets_consumption_enabled' => true,
+            'longform_notetweets_inline_media_enabled' => false,
+            'longform_notetweets_richtext_consumption_enabled' => true,
+            'longform_notetweets_rich_text_read_enabled' => false,
+            'responsive_web_edit_tweet_api_enabled' => false,
+            'responsive_web_enhance_cards_enabled' => false,
+            'responsive_web_graphql_exclude_directive_enabled' => true,
+            'responsive_web_graphql_skip_user_profile_image_extensions_enabled' => false,
+            'responsive_web_graphql_timeline_navigation_enabled' => false,
+            'responsive_web_media_download_video_enabled' => false,
+            'responsive_web_text_conversations_enabled' => false,
+            'responsive_web_twitter_article_tweet_consumption_enabled' => false,
+            'responsive_web_twitter_blue_verified_badge_is_enabled' => true,
+            'rweb_lists_timeline_redesign_enabled' => true,
+            'spaces_2022_h2_clipping' => true,
+            'spaces_2022_h2_spaces_communities' => true,
+            'standardized_nudges_misinfo' => false,
+            'subscriptions_verification_info_enabled' => true,
+            'subscriptions_verification_info_reason_enabled' => true,
+            'subscriptions_verification_info_verified_since_enabled' => true,
+            'super_follow_badge_privacy_enabled' => false,
+            'super_follow_exclusive_tweet_notifications_enabled' => false,
+            'super_follow_tweet_api_enabled' => false,
+            'super_follow_user_api_enabled' => false,
+            'tweet_awards_web_tipping_enabled' => false,
+            'tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled' => false,
+            'tweetypie_unmention_optimization_enabled' => false,
+            'unified_cards_ad_metadata_container_dynamic_card_content_query_enabled' => false,
+            'verified_phone_label_enabled' => false,
+            'vibe_api_enabled' => false,
+            'view_counts_everywhere_api_enabled' => false
+        ];
+        $variables = [
+            'screenName' => $screenName,
+            'listSlug' => $listSlug
+        ];
+
+        $url = sprintf(
+            'https://twitter.com/i/api/graphql/-kmqNvm5Y-cVrfvBy6docg/ListBySlug?variables=%s&features=%s',
+            urlencode(json_encode($variables)),
+            urlencode(json_encode($features))
+        );
+
+        $response = Json::decode(getContents($url, $this->createHttpHeaders()), false);
+        if (isset($response->errors)) {
+            // Grab the first error message
+            throw new \Exception(sprintf('From twitter api: "%s"', $response->errors[0]->message));
+        }
+        $listInfo = $response->data->user_by_screen_name->list;
+        $this->data[$screenName . '-' . $listSlug] = $listInfo;
+
+        $this->cache->setScope('twitter');
+        $this->cache->setKey(['cache']);
+        $this->cache->saveData($this->data);
+        return $listInfo;
+    }
+
+    private function fetchListTimeline($listId)
+    {
+        $features = [
+            'android_graphql_skip_api_media_color_palette' => false,
+            'blue_business_profile_image_shape_enabled' => false,
+            'creator_subscriptions_subscription_count_enabled' => false,
+            'creator_subscriptions_tweet_preview_api_enabled' => true,
+            'freedom_of_speech_not_reach_fetch_enabled' => false,
+            'graphql_is_translatable_rweb_tweet_is_translatable_enabled' => false,
+            'hidden_profile_likes_enabled' => false,
+            'highlights_tweets_tab_ui_enabled' => false,
+            'interactive_text_enabled' => false,
+            'longform_notetweets_consumption_enabled' => true,
+            'longform_notetweets_inline_media_enabled' => false,
+            'longform_notetweets_richtext_consumption_enabled' => true,
+            'longform_notetweets_rich_text_read_enabled' => false,
+            'responsive_web_edit_tweet_api_enabled' => false,
+            'responsive_web_enhance_cards_enabled' => false,
+            'responsive_web_graphql_exclude_directive_enabled' => true,
+            'responsive_web_graphql_skip_user_profile_image_extensions_enabled' => false,
+            'responsive_web_graphql_timeline_navigation_enabled' => false,
+            'responsive_web_media_download_video_enabled' => false,
+            'responsive_web_text_conversations_enabled' => false,
+            'responsive_web_twitter_article_tweet_consumption_enabled' => false,
+            'responsive_web_twitter_blue_verified_badge_is_enabled' => true,
+            'rweb_lists_timeline_redesign_enabled' => true,
+            'spaces_2022_h2_clipping' => true,
+            'spaces_2022_h2_spaces_communities' => true,
+            'standardized_nudges_misinfo' => false,
+            'subscriptions_verification_info_enabled' => true,
+            'subscriptions_verification_info_reason_enabled' => true,
+            'subscriptions_verification_info_verified_since_enabled' => true,
+            'super_follow_badge_privacy_enabled' => false,
+            'super_follow_exclusive_tweet_notifications_enabled' => false,
+            'super_follow_tweet_api_enabled' => false,
+            'super_follow_user_api_enabled' => false,
+            'tweet_awards_web_tipping_enabled' => false,
+            'tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled' => false,
+            'tweetypie_unmention_optimization_enabled' => false,
+            'unified_cards_ad_metadata_container_dynamic_card_content_query_enabled' => false,
+            'verified_phone_label_enabled' => false,
+            'vibe_api_enabled' => false,
+            'view_counts_everywhere_api_enabled' => false
+        ];
+        $variables = [
+            'rest_id' => $listId,
+            'count' => 20
+        ];
+
+        $url = sprintf(
+            'https://twitter.com/i/api/graphql/BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline?variables=%s&features=%s',
+            urlencode(json_encode($variables)),
+            urlencode(json_encode($features))
+        );
+        $response = Json::decode(getContents($url, $this->createHttpHeaders()), false);
+        return $response;
+    }
+
     private function createHttpHeaders(): array
     {
         $headers = [