mirrors/rss-bridge
1
0
Fork 0
mirror of https://github.com/RSS-Bridge/rss-bridge.git synced 2024-11-22 01:25:28 +03:00
Code Issues Projects Releases Packages Wiki Activity

[SitemapBridge] Add SitemapBridge (#3602)

Browse source 
* [SitemapBridge] Add SitemapBridge

This bridge is a variant of CssSelectorBridge.
Instead of retrieving article list from home page,
retrieves article list from SEO sitemap.xml.
Requires CssSelectorBridge to be installed.

* [SitemapBridge] Code linting
This commit is contained in:
ORelio 2023-08-08 15:02:01 +02:00 committed by GitHub
parent 43ec82179b
commit b86ee5778b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 152 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

152
bridges/SitemapBridge.php Normal file
View file

@ -0,0 +1,152 @@
<?php
class SitemapBridge extends CssSelectorBridge
{
const MAINTAINER = 'ORelio';
const NAME = 'Sitemap Bridge';
const URI = 'https://github.com/RSS-Bridge/rss-bridge/';
const DESCRIPTION = 'Convert any site to RSS feed using SEO Sitemap and CSS selectors (Advanced Users)';
const PARAMETERS = [
[
'home_page' => [
'name' => 'Site URL: Home page with latest articles',
'title' => <<<EOT
The bridge will analyze the site like a search engine does.
The URL specified here determines the feed title and URL.
EOT,
'exampleValue' => 'https://example.com/blog/',
'required' => true
],
'url_pattern' => [
'name' => 'Pattern for site URLs to take in feed',
'title' => 'Select items by applying a regular expression on their URL',
'exampleValue' => 'https://example.com/article/.*',
'required' => true
],
'content_selector' => [
'name' => 'Selector for each article content',
'title' => <<<EOT
This bridge works using CSS selectors, e.g. "div.article" will match <div class="article">.
Everything inside that element becomes feed item content.
EOT,
'exampleValue' => 'article.content',
'required' => true
],
'content_cleanup' => [
'name' => '[Optional] Content cleanup: List of items to remove',
'title' => 'Selector for unnecessary elements to remove inside article contents.',
'exampleValue' => 'div.ads, div.comments',
],
'title_cleanup' => [
'name' => '[Optional] Text to remove from article title',
'title' => 'Specify here some text from page title that need to be removed, e.g. " | BlogName".',
'exampleValue' => ' | BlogName',
],
'site_map' => [
'name' => '[Optional] sitemap.xml URL',
'title' => <<<EOT
By default, the bridge will analyze robots.txt to find out URL for sitemap.xml.
Alternatively, you can specify here the direct URL for sitemap XML.
The sitemap.xml file must have <loc> and <lastmod> fields for the bridge to work:
Eg. <url><loc>https://article/url</loc><lastmod>2000-12-31T23:59Z</lastmod></url>
<loc> is feed item URL, <lastmod> for selecting the most recent entries.
EOT,
'exampleValue' => 'https://example.com/sitemap.xml',
],
'limit' => self::LIMIT
]
];
public function collectData()
{
$url = $this->getInput('home_page');
$url_pattern = $this->getInput('url_pattern');
$content_selector = $this->getInput('content_selector');
$content_cleanup = $this->getInput('content_cleanup');
$title_cleanup = $this->getInput('title_cleanup');
$site_map = $this->getInput('site_map');
$limit = $this->getInput('limit');
$this->feedName = $this->getPageTitle($url, $title_cleanup);
$sitemap_url = empty($site_map) ? $url : $site_map;
$sitemap_xml = $this->getSitemapXml($sitemap_url, !empty($site_map));
$links = $this->sitemapXmlToList($sitemap_xml, $url_pattern, empty($limit) ? 10 : $limit);
if (empty($links) && empty(sitemapXmlToList($sitemap_xml))) {
returnClientError('Could not retrieve URLs with Timestamps from Sitemap: ' . $sitemap_url);
}
foreach ($links as $link) {
$this->items[] = $this->expandEntryWithSelector($link, $content_selector, $content_cleanup, $title_cleanup);
}
}
/**
* Retrieve site map from specified URL
* @param string $url URL pointing to any page of the site, e.g. "https://example.com/blog" OR directly to the site map e.g. "https://example.com/sitemap.xml"
* @param string $is_site_map TRUE if the specified URL points directly to the sitemap XML
* @return object Sitemap DOM (from parsed XML)
*/
protected function getSitemapXml(&$url, $is_site_map = false)
{
if (!$is_site_map) {
$robots_txt = getSimpleHTMLDOM(urljoin($url, '/robots.txt'))->outertext;
preg_match('/Sitemap: ([^ ]+)/', $robots_txt, $matches);
if (empty($matches)) {
returnClientError('Failed to determine Sitemap from robots.txt. Try setting it manually.');
}
$url = $matches[1];
}
return getSimpleHTMLDOM($url);
}
/**
* Retrieve N most recent URLs from Site Map
* @param object $sitemap Site map XML DOM
* @param string $url_pattern Optional pattern to look for in URLs
* @param int $limit Optional maximum amount of URLs to return
* @param bool $keep_date TRUE to keep dates (url => date array instead of url array)
* @return array Array of URLs
*/
protected function sitemapXmlToList($sitemap, $url_pattern = '', $limit = 0, $keep_date = false)
{
$links = [];
foreach ($sitemap->find('sitemap') as $nested_sitemap) {
$url = $nested_sitemap->find('loc');
if (!empty($url)) {
$url = $url[0]->plaintext;
if (str_ends_with(strtolower($url), '.xml')) {
$nested_sitemap_xml = $this->getSitemapXml($url, true);
$nested_sitemap_links = $this->sitemapXmlToList($nested_sitemap_xml, $url_pattern, null, true);
$links = array_merge($links, $nested_sitemap_links);
}
}
}
if (!empty($url_pattern)) {
$url_pattern = str_replace('/', '\/', $url_pattern);
}
foreach ($sitemap->find('url') as $item) {
$url = $item->find('loc');
$lastmod = $item->find('lastmod');
if (!empty($url) && !empty($lastmod)) {
$url = $url[0]->plaintext;
$lastmod = $lastmod[0]->plaintext;
$timestamp = strtotime($lastmod);
if (empty($url_pattern) || preg_match('/' . $url_pattern . '/', $url) === 1) {
$links[$url] = $timestamp;
}
}
}
arsort($links);
if ($limit > 0 && count($links) > $limit) {
$links = array_slice($links, 0, $limit);
}
return $keep_date ? $links : array_keys($links);
}
}