Compare commits

...

No commits in common. "gh-pages" and "master" have entirely different histories.

817 changed files with 97471 additions and 14122 deletions

8
.devcontainer/Dockerfile Normal file
View File

@ -0,0 +1,8 @@
FROM rssbridge/rss-bridge:latest
RUN apt-get update && \
apt-get install --yes --no-install-recommends \
git && \
pecl install xdebug && \
pear install PHP_CodeSniffer && \
docker-php-ext-enable xdebug

View File

@ -0,0 +1,27 @@
{
"name": "rss-bridge dev",
"build": { "dockerfile": "Dockerfile" },
"customizations": {
// Configure properties specific to VS Code.
"vscode": {
// Set *default* container specific settings.json values on container create.
"settings": {
"php.validate.executablePath": "/usr/local/bin/php",
"phpSniffer.executablesFolder": "/usr/local/bin/",
"phpcs.executablePath": "/usr/local/bin/phpcs",
"phpcs.lintOnType": false
},
// Add the IDs of extensions you want installed when the container is created.
"extensions": [
"xdebug.php-debug",
"bmewburn.vscode-intelephense-client",
"philfontaine.autolaunch",
"eamodio.gitlens",
"shevaua.phpcs"
]
}
},
"forwardPorts": [3100, 9000, 9003],
"postCreateCommand": "cp .devcontainer/nginx.conf /etc/nginx/conf.d/default.conf && cp .devcontainer/xdebug.ini /usr/local/etc/php/conf.d/xdebug.ini && mkdir .vscode && cp .devcontainer/launch.json .vscode && echo '*' > whitelist.txt && chmod a+x \"$(pwd)\" && rm -rf /var/www/html && ln -s \"$(pwd)\" /var/www/html && nginx && php-fpm -D"
}

49
.devcontainer/launch.json Normal file
View File

@ -0,0 +1,49 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Listen for Xdebug",
"type": "php",
"request": "launch",
"port": 9003,
"auto": true
},
{
"name": "Launch currently open script",
"type": "php",
"request": "launch",
"program": "${file}",
"cwd": "${fileDirname}",
"port": 0,
"runtimeArgs": [
"-dxdebug.start_with_request=yes"
],
"env": {
"XDEBUG_MODE": "debug,develop",
"XDEBUG_CONFIG": "client_port=${port}"
}
},
{
"name": "Launch Built-in web server",
"type": "php",
"request": "launch",
"runtimeArgs": [
"-dxdebug.mode=debug",
"-dxdebug.start_with_request=yes",
"-S",
"localhost:0"
],
"program": "",
"cwd": "${workspaceRoot}",
"port": 9003,
"serverReadyAction": {
"pattern": "Development Server \\(http://localhost:([0-9]+)\\) started",
"uriFormat": "http://localhost:%s",
"action": "openExternally"
}
}
]
}

17
.devcontainer/nginx.conf Normal file
View File

@ -0,0 +1,17 @@
server {
listen 3100 default_server;
root /workspaces/rss-bridge;
access_log /var/log/nginx/rssbridge.access.log;
error_log /var/log/nginx/rssbridge.error.log;
index index.php;
location ~ /(\.|vendor|tests) {
deny all;
return 403; # Forbidden
}
location ~ \.php$ {
include snippets/fastcgi-php.conf;
fastcgi_pass unix:/var/run/php/php8.2-fpm.sock;
}
}

7
.devcontainer/xdebug.ini Normal file
View File

@ -0,0 +1,7 @@
[xdebug]
xdebug.mode=develop,debug
xdebug.client_host=localhost
xdebug.client_port=9003
xdebug.start_with_request=yes
xdebug.discover_client_host=false
xdebug.log='/var/www/html/xdebug.log'

16
.dockerignore Normal file
View File

@ -0,0 +1,16 @@
.git
!.git/HEAD
!.git/refs/heads/*
.gitattributes
.github/*
.travis.yml
cache/*
CONTRIBUTING.md
DEBUG
Dockerfile
phpcompatibility.xml
phpcs.xml
phpcs.xml
scalingo.json
tests/*
whitelist.txt

4
.git-blame-ignore-revs Normal file
View File

@ -0,0 +1,4 @@
# Reformat code base to PSR12
4f75591060d95208a301bc6bf460d875631b29cc
# Fix coding style missed by phpbcf
951092eef374db048b77bac85e75e3547bfac702

68
.gitattributes vendored Normal file
View File

@ -0,0 +1,68 @@
# Auto detect text files and perform LF normalization
* text=auto
*.sh text eol=lf
# Custom for Visual Studio
*.cs diff=csharp
*.sln merge=union
*.csproj merge=union
*.vbproj merge=union
*.fsproj merge=union
*.dbproj merge=union
# Standard to msysgit
*.doc diff=astextplain
*.DOC diff=astextplain
*.docx diff=astextplain
*.DOCX diff=astextplain
*.dot diff=astextplain
*.DOT diff=astextplain
*.pdf diff=astextplain
*.PDF diff=astextplain
*.rtf diff=astextplain
*.RTF diff=astextplain
# Ignore files in git archive (i.e. GitHub release builds)
## Docker
Dockerfile export-ignore
.dockerignore export-ignore
## Travis
.travis.yml export-ignore
## GitHub
.github/ export-ignore
## Git
.gitattributes export-ignore
.gitignore export-ignore
## Scalingo
scalingo.json export-ignore
## RSS-Bridge
phpunit.xml export-ignore
phpcs.xml export-ignore
phpcompatibility.xml export-ignore
tests/ export-ignore
cache/.gitkeep export-ignore
## Composer
#
# Keep the following lines commented out. Heroku does
# not function if the composer files are ignored during
# export. For more information see
# https://github.com/rss-bridge/rss-bridge/issues/1165
#
# composer.json export-ignore
# composer.lock export-ignore
## Heroku
#
# Keep the following line commented out. Heroku does
# not function if app.json is ignored during export.
# For more information see
# https://github.com/rss-bridge/rss-bridge/issues/1165
#
# app.json export-ignore

7
.github/.gitignore vendored Normal file
View File

@ -0,0 +1,7 @@
# Visual Studio Code
.vscode/*
# Generated files
comment*.md
comment*.txt
*.html

7
.github/CONTRIBUTING.md vendored Normal file
View File

@ -0,0 +1,7 @@
### Pull request policy
See the [Pull request policy page on the documentation](https://rss-bridge.github.io/rss-bridge/For_Developers/Pull_Request_policy.html) for more information on the pull request policy.
### Coding style
See the [Coding style policy page on the documentation](https://rss-bridge.github.io/rss-bridge/For_Developers/Coding_style_policy.html) for more information on the coding style of the project.

View File

@ -0,0 +1,64 @@
---
name: Bridge request
about: Use this template for requesting a new bridge
title: Bridge request for ...
labels: Bridge-Request
assignees: ''
---
# Bridge request
<!--
This is a bridge request. Start by adding a descriptive title (i.e. `Bridge request for GitHub`). Use the "Preview" button to see a preview of your request. Make sure your request is complete before submitting!
Notice: This comment is only visible to you while you work on your request. Please do not remove any of the lines in the template (you may add your own outside the "<!--" and "- ->" lines!)
-->
## General information
<!--
Please describe what you expect from the bridge. Whenever possible provide sample links and screenshots (you can just paste them here) to express your expectations and help others understand your request. If possible, mark relevant areas in your screenshot. Use the following questions for reference:
-->
- _Host URI for the bridge_ (i.e. `https://github.com`):
- Which information would you like to see?
- How should the information be displayed/formatted?
- Which of the following parameters do you expect?
- [X] Title
- [X] URI (link to the original article)
- [ ] Author
- [ ] Timestamp
- [X] Content (the content of the article)
- [ ] Enclosures (pictures, videos, etc...)
- [ ] Categories (categories, tags, etc...)
## Options
<!--Select options from the list below. Add your own option if one is missing:-->
- [ ] Limit number of returned items
- _Default limit_: 5
- [ ] Load full articles
- _Cache articles_ (articles are stored in a local cache on first request): yes
- _Cache timeout_ : 24 hours
- [X] Balance requests (RSS-Bridge uses cached versions to reduce bandwith usage)
- _Timeout_ (default = 5 minutes): 5 minutes
<!--Be aware that some options might not be available for your specific request due to technical limitations!-->
<!--
## Additional notes
Keep in mind that opening a request does not guarantee the bridge being implemented! That depends entirely on the interest and time of others to make the bridge for you.
You can also implement your own bridge (with support of the community if needed). Find more information in the [RSS-Bridge Documentation](https://rss-bridge.github.io/rss-bridge/For_Developers/index.html) developer section.
-->

38
.github/ISSUE_TEMPLATE/bug_report.md vendored Normal file
View File

@ -0,0 +1,38 @@
---
name: Bug report
about: Create a report to help us improve
title: ''
labels: Bug-Report
assignees: ''
---
**Describe the bug**
A clear and concise description of what the bug is.
**To Reproduce**
Steps to reproduce the behavior:
1. Go to '...'
2. Click on '....'
3. Scroll down to '....'
4. See error
**Expected behavior**
A clear and concise description of what you expected to happen.
**Screenshots**
If applicable, add screenshots to help explain your problem.
**Desktop (please complete the following information):**
- OS: [e.g. iOS]
- Browser [e.g. chrome, safari]
- Version [e.g. 22]
**Smartphone (please complete the following information):**
- Device: [e.g. iPhone6]
- OS: [e.g. iOS8.1]
- Browser [e.g. stock browser, safari]
- Version [e.g. 22]
**Additional context**
Add any other context about the problem here.

View File

@ -0,0 +1,20 @@
---
name: Feature request
about: Suggest an idea for this project
title: ''
labels: Feature-Request
assignees: ''
---
**Is your feature request related to a problem? Please describe.**
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
**Describe the solution you'd like**
A clear and concise description of what you want to happen.
**Describe alternatives you've considered**
A clear and concise description of any alternative solutions or features you've considered.
**Additional context**
Add any other context or screenshots about the feature request here.

2
.github/prtester-requirements.txt vendored Normal file
View File

@ -0,0 +1,2 @@
beautifulsoup4>=4.10.0
requests>=2.26.0

208
.github/prtester.py vendored Normal file
View File

@ -0,0 +1,208 @@
import argparse
import requests
import re
from bs4 import BeautifulSoup
from datetime import datetime
from typing import Iterable
import os
import glob
import urllib
# This script is specifically written to be used in automation for https://github.com/RSS-Bridge/rss-bridge
#
# This will scrape the whitelisted bridges in the current state (port 3000) and the PR state (port 3001) of
# RSS-Bridge, generate a feed for each of the bridges and save the output as html files.
# It also add a <base> tag with the url of em's public instance, so viewing
# the HTML file locally will actually work as designed.
ARTIFACT_FILE_EXTENSION = '.html'
class Instance:
name = ''
url = ''
def main(instances: Iterable[Instance], with_upload: bool, with_reduced_upload: bool, title: str, output_file: str):
start_date = datetime.now()
prid = os.getenv('PR')
artifact_base_url = f'https://rss-bridge.github.io/rss-bridge-tests/prs/{prid}'
artifact_directory = os.getcwd()
for file in glob.glob(f'*{ARTIFACT_FILE_EXTENSION}', root_dir=artifact_directory):
os.remove(file)
table_rows = []
for instance in instances:
page = requests.get(instance.url) # Use python requests to grab the rss-bridge main page
soup = BeautifulSoup(page.content, "html.parser") # use bs4 to turn the page into soup
bridge_cards = soup.select('.bridge-card') # get a soup-formatted list of all bridges on the rss-bridge page
table_rows += testBridges(
instance=instance,
bridge_cards=bridge_cards,
with_upload=with_upload,
with_reduced_upload=with_reduced_upload,
artifact_directory=artifact_directory,
artifact_base_url=artifact_base_url) # run the main scraping code with the list of bridges
with open(file=output_file, mode='w+', encoding='utf-8') as file:
table_rows_value = '\n'.join(sorted(table_rows))
file.write(f'''
## {title}
| Bridge | Context | Status |
| - | - | - |
{table_rows_value}
*last change: {start_date.strftime("%A %Y-%m-%d %H:%M:%S")}*
'''.strip())
def testBridges(instance: Instance, bridge_cards: Iterable, with_upload: bool, with_reduced_upload: bool, artifact_directory: str, artifact_base_url: str) -> Iterable:
instance_suffix = ''
if instance.name:
instance_suffix = f' ({instance.name})'
table_rows = []
for bridge_card in bridge_cards:
bridgeid = bridge_card.get('id')
bridgeid = bridgeid.split('-')[1] # this extracts a readable bridge name from the bridge metadata
print(f'{bridgeid}{instance_suffix}')
bridge_name = bridgeid.replace('Bridge', '')
context_forms = bridge_card.find_all("form")
form_number = 1
for context_form in context_forms:
# a bridge can have multiple contexts, named 'forms' in html
# this code will produce a fully working url that should create a working feed when called
# this will create an example feed for every single context, to test them all
context_parameters = {}
error_messages = []
context_name = '*untitled*'
context_name_element = context_form.find_previous_sibling('h5')
if context_name_element and context_name_element.text.strip() != '':
context_name = context_name_element.text
parameters = context_form.find_all("input")
lists = context_form.find_all("select")
# this for/if mess cycles through all available input parameters, checks if it required, then pulls
# the default or examplevalue and then combines it all together into the url parameters
# if an example or default value is missing for a required attribute, it will throw an error
# any non-required fields are not tested!!!
for parameter in parameters:
parameter_type = parameter.get('type')
parameter_name = parameter.get('name')
if parameter_type == 'hidden':
context_parameters[parameter_name] = parameter.get('value')
if parameter_type == 'number' or parameter_type == 'text':
if parameter.has_attr('required'):
if parameter.get('placeholder') == '':
if parameter.get('value') == '':
error_messages.append(f'Missing example or default value for parameter "{parameter_name}"')
else:
context_parameters[parameter_name] = parameter.get('value')
else:
context_parameters[parameter_name] = parameter.get('placeholder')
# same thing, just for checkboxes. If a checkbox is checked per default, it gets added to the url parameters
if parameter_type == 'checkbox':
if parameter.has_attr('checked'):
context_parameters[parameter_name] = 'on'
for listing in lists:
selectionvalue = ''
listname = listing.get('name')
cleanlist = []
options = listing.find_all('option')
for option in options:
if 'optgroup' in option.name:
cleanlist.extend(option)
else:
cleanlist.append(option)
firstselectionentry = 1
for selectionentry in cleanlist:
if firstselectionentry:
selectionvalue = selectionentry.get('value')
firstselectionentry = 0
else:
if 'selected' in selectionentry.attrs:
selectionvalue = selectionentry.get('value')
break
context_parameters[listname] = selectionvalue
artifact_url = 'about:blank'
if error_messages:
status = '<br>'.join(map(lambda m: f'❌ `{m}`', error_messages))
else:
# if all example/default values are present, form the full request url, run the request, add a <base> tag with
# the url of em's public instance to the response text (so that relative paths work, e.g. to the static css file) and
# then save it to a html file.
context_parameters.update({
'action': 'display',
'bridge': bridgeid,
'format': 'Html',
})
request_url = f'{instance.url}/?{urllib.parse.urlencode(context_parameters)}'
response = requests.get(request_url)
page_text = response.text.replace('<head>','<head><base href="https://rss-bridge.org/bridge01/" target="_blank">')
page_text = page_text.encode("utf_8")
soup = BeautifulSoup(page_text, "html.parser")
status_messages = []
if response.status_code != 200:
status_messages += [f'❌ `HTTP status {response.status_code} {response.reason}`']
else:
feed_items = soup.select('.feeditem')
feed_items_length = len(feed_items)
if feed_items_length <= 0:
status_messages += [f'⚠️ `The feed has no items`']
elif feed_items_length == 1 and len(soup.select('.error')) > 0:
status_messages += [f'❌ `{getFirstLine(feed_items[0].text)}`']
status_messages += map(lambda e: f'❌ `{getFirstLine(e.text)}`', soup.select('.error .error-type') + soup.select('.error .error-message'))
for item_element in soup.select('.feeditem'): # remove all feed items to not accidentally selected <pre> tags from item content
item_element.decompose()
status_messages += map(lambda e: f'⚠️ `{getFirstLine(e.text)}`', soup.find_all('pre'))
status_messages = list(dict.fromkeys(status_messages)) # remove duplicates
status = '<br>'.join(status_messages)
status_is_ok = status == '';
if status_is_ok:
status = '✔️'
if with_upload and (not with_reduced_upload or not status_is_ok):
filename = f'{bridge_name} {form_number}{instance_suffix}{ARTIFACT_FILE_EXTENSION}'
filename = re.sub(r'[^a-z0-9 \_\-\.]', '', filename, flags=re.I).replace(' ', '_')
with open(file=f'{artifact_directory}/{filename}', mode='wb') as file:
file.write(page_text)
artifact_url = f'{artifact_base_url}/{filename}'
table_rows.append(f'| {bridge_name} | [{form_number} {context_name}{instance_suffix}]({artifact_url}) | {status} |')
form_number += 1
return table_rows
def getFirstLine(value: str) -> str:
# trim whitespace and remove text that can break the table or is simply unnecessary
clean_value = re.sub(r'^\[[^\]]+\]\s*rssbridge\.|[\|`]', '', value.strip())
first_line = next(iter(clean_value.splitlines()), '')
max_length = 250
if (len(first_line) > max_length):
first_line = first_line[:max_length] + '...'
return first_line
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--instances', nargs='+')
parser.add_argument('--no-upload', action='store_true')
parser.add_argument('--reduced-upload', action='store_true')
parser.add_argument('--title', default='Pull request artifacts')
parser.add_argument('--output-file', default=os.getcwd() + '/comment.txt')
args = parser.parse_args()
instances = []
if args.instances:
for instance_arg in args.instances:
instance_arg_parts = instance_arg.split('::')
instance = Instance()
instance.name = instance_arg_parts[1].strip() if len(instance_arg_parts) >= 2 else ''
instance.url = instance_arg_parts[0].strip().rstrip("/")
instances.append(instance)
else:
instance = Instance()
instance.name = 'current'
instance.url = 'http://localhost:3000'
instances.append(instance)
instance = Instance()
instance.name = 'pr'
instance.url = 'http://localhost:3001'
instances.append(instance)
main(
instances=instances,
with_upload=not args.no_upload,
with_reduced_upload=args.reduced_upload and not args.no_upload,
title=args.title,
output_file=args.output_file
);

61
.github/workflows/dockerbuild.yml vendored Normal file
View File

@ -0,0 +1,61 @@
name: Build Image on Commit and Release
on:
push:
branches:
- 'master'
tags:
- '20*'
env:
DOCKERHUB_SLUG: rssbridge/rss-bridge
GHCR_SLUG: ghcr.io/rss-bridge/rss-bridge
jobs:
bake:
runs-on: ubuntu-latest
steps:
-
name: Checkout
uses: actions/checkout@v3
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@v5
with:
images: |
${{ env.DOCKERHUB_SLUG }}
${{ env.GHCR_SLUG }}
tags: |
type=raw,value=latest
type=sha
type=ref,event=tag,enable=${{ startsWith(github.ref, 'refs/tags/20') }}
type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/20') }}
-
name: Set up QEMU
uses: docker/setup-qemu-action@v3
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
-
name: Login to DockerHub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
-
name: Build and push
uses: docker/bake-action@v5
with:
files: |
./docker-bake.hcl
${{ steps.docker_meta.outputs.bake-file }}
targets: image-all
push: true

27
.github/workflows/documentation.yml vendored Normal file
View File

@ -0,0 +1,27 @@
name: Documentation
on:
push:
paths:
- 'docs/**'
jobs:
documentation:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: Setup PHP
uses: shivammathur/setup-php@v2
with:
php-version: 8.0
- name: Install dependencies
run: composer global require daux/daux.io
- name: Generate documentation
run: daux generate
- name: Deploy same repository 🚀
uses: JamesIves/github-pages-deploy-action@v4
with:
folder: "static"
branch: gh-pages

49
.github/workflows/lint.yml vendored Normal file
View File

@ -0,0 +1,49 @@
name: Lint
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
jobs:
phpcs:
runs-on: ubuntu-20.04
strategy:
matrix:
php-versions: ['7.4']
steps:
- uses: actions/checkout@v4
- uses: shivammathur/setup-php@v2
with:
php-version: ${{ matrix.php-versions }}
tools: phpcs
- run: phpcs . --standard=phpcs.xml --warning-severity=0 --extensions=php -p
phpcompatibility:
runs-on: ubuntu-20.04
strategy:
matrix:
php-versions: ['7.4']
steps:
- uses: actions/checkout@v4
- uses: shivammathur/setup-php@v2
with:
php-version: ${{ matrix.php-versions }}
- run: composer global config --no-plugins allow-plugins.dealerdirect/phpcodesniffer-composer-installer true
- run: composer global require dealerdirect/phpcodesniffer-composer-installer
- run: composer global require phpcompatibility/php-compatibility
- run: ~/.composer/vendor/bin/phpcs . --standard=phpcompatibility.xml --warning-severity=0 --extensions=php -p
executable_php_files_check:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v4
- run: |
if find -name "*.php" -executable -type f -print -exec false {} +
then
echo 'Good, no executable php scripts found'
else
echo 'Please unmark php scripts above as non-executable'
exit 1
fi

126
.github/workflows/prhtmlgenerator.yml vendored Normal file
View File

@ -0,0 +1,126 @@
name: 'PR Testing'
on:
pull_request_target:
branches: [ master ]
jobs:
check-bridges:
name: Check if bridges were changed
runs-on: ubuntu-latest
outputs:
BRIDGES: ${{ steps.check1.outputs.BRIDGES }}
steps:
- name: Check number of bridges
id: check1
run: |
PR=${{github.event.number}};
wget https://patch-diff.githubusercontent.com/raw/$GITHUB_REPOSITORY/pull/$PR.patch;
bridgeamount=$(cat $PR.patch | grep "\bbridges/[A-Za-z0-9]*Bridge\.php\b" | sed "s=.*\bbridges/\([A-Za-z0-9]*\)Bridge\.php\b.*=\1=g" | sort | uniq | wc -l);
echo "BRIDGES=$bridgeamount" >> "$GITHUB_OUTPUT"
test-pr:
name: Generate HTML
runs-on: ubuntu-latest
needs: check-bridges
if: needs.check-bridges.outputs.BRIDGES > 0
env:
PYTHONUNBUFFERED: 1
# Needs additional permissions https://github.com/actions/first-interaction/issues/10#issuecomment-1041402989
steps:
- name: Check out self
uses: actions/checkout@v4
with:
ref: ${{github.event.pull_request.head.ref}}
repository: ${{github.event.pull_request.head.repo.full_name}}
- name: Check out rss-bridge
run: |
PR=${{github.event.number}};
wget -O requirements.txt https://raw.githubusercontent.com/$GITHUB_REPOSITORY/${{ github.event.pull_request.base.ref }}/.github/prtester-requirements.txt;
wget https://raw.githubusercontent.com/$GITHUB_REPOSITORY/${{ github.event.pull_request.base.ref }}/.github/prtester.py;
wget https://patch-diff.githubusercontent.com/raw/$GITHUB_REPOSITORY/pull/$PR.patch;
touch DEBUG;
cat $PR.patch | grep "\bbridges/[A-Za-z0-9]*Bridge\.php\b" | sed "s=.*\bbridges/\([A-Za-z0-9]*\)Bridge\.php\b.*=\1=g" | sort | uniq > whitelist.txt
- name: Start Docker - Current
run: |
docker run -d -v $GITHUB_WORKSPACE/whitelist.txt:/app/whitelist.txt -v $GITHUB_WORKSPACE/DEBUG:/app/DEBUG -p 3000:80 ghcr.io/rss-bridge/rss-bridge:latest
- name: Start Docker - PR
run: |
docker build -t prbuild .;
docker run -d -v $GITHUB_WORKSPACE/whitelist.txt:/app/whitelist.txt -v $GITHUB_WORKSPACE/DEBUG:/app/DEBUG -p 3001:80 prbuild
- name: Setup python
uses: actions/setup-python@v5
with:
python-version: '3.13'
cache: 'pip'
- name: Install requirements
run: |
cd $GITHUB_WORKSPACE
pip install -r requirements.txt
- name: Run bridge tests
id: testrun
run: |
mkdir results;
python prtester.py;
body="$(cat comment.txt)";
body="${body//'%'/'%25'}";
body="${body//$'\n'/'%0A'}";
body="${body//$'\r'/'%0D'}";
echo "bodylength=${#body}" >> $GITHUB_OUTPUT
env:
PR: ${{ github.event.number }}
- name: Upload generated tests
uses: actions/upload-artifact@v4
id: upload-generated-tests
with:
name: tests
path: '*.html'
- name: Find Comment
if: ${{ steps.testrun.outputs.bodylength > 130 }}
uses: peter-evans/find-comment@v3
id: fc
with:
issue-number: ${{ github.event.pull_request.number }}
comment-author: 'github-actions[bot]'
body-includes: Pull request artifacts
- name: Create or update comment
if: ${{ steps.testrun.outputs.bodylength > 130 }}
uses: peter-evans/create-or-update-comment@v4
with:
comment-id: ${{ steps.fc.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body-file: comment.txt
edit-mode: replace
upload_tests:
name: Upload tests
runs-on: ubuntu-latest
needs: test-pr
steps:
- uses: actions/checkout@v4
with:
repository: 'RSS-Bridge/rss-bridge-tests'
ref: 'main'
token: ${{ secrets.RSSTESTER_ACTION }}
- name: Setup git config
run: |
git config --global user.name "GitHub Actions"
git config --global user.email "<>"
- name: Download tests
uses: actions/download-artifact@v4
with:
name: tests
- name: Move tests
run: |
cd prs
mkdir -p ${{github.event.number}}
cd ${{github.event.number}}
mv -f $GITHUB_WORKSPACE/*.html .
- name: Commit and push generated tests
run: |
export COMMIT_MESSAGE="Added tests for PR ${{github.event.number}}"
git add .
git commit -m "$COMMIT_MESSAGE"
git push

23
.github/workflows/tests.yml vendored Normal file
View File

@ -0,0 +1,23 @@
name: Tests
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
jobs:
phpunit8:
runs-on: ubuntu-20.04
strategy:
matrix:
php-versions: ['7.4', '8.0', '8.1']
steps:
- uses: actions/checkout@v4
- uses: shivammathur/setup-php@v2
with:
php-version: ${{ matrix.php-versions }}
env:
update: true
- run: composer install
- run: composer test

249
.gitignore vendored Normal file
View File

@ -0,0 +1,249 @@
#################
## Eclipse
#################
vendor/*
data/
*.pydevproject
.project
.metadata
tmp/
*.tmp
*.bak
*.swp
*~.nib
local.properties
.classpath
.settings/
.loadpath
# External tool builders
.externalToolBuilders/
# Locally stored "Eclipse launch configurations"
*.launch
# CDT-specific
.cproject
# PDT-specific
.buildpath
#################
## Visual Studio
#################
## Ignore Visual Studio temporary files, build results, and
## files generated by popular Visual Studio add-ons.
# User-specific files
*.suo
*.user
*.sln.docstates
# Build results
[Dd]ebug/
[Rr]elease/
x64/
build/
[Bb]in/
[Oo]bj/
# MSTest test Results
[Tt]est[Rr]esult*/
[Bb]uild[Ll]og.*
*_i.c
*_p.c
*.ilk
*.meta
*.obj
*.pch
*.pdb
*.pgc
*.pgd
*.rsp
*.sbr
*.tlb
*.tli
*.tlh
*.tmp
*.tmp_proj
*.log
*.vspscc
*.vssscc
.builds
*.pidb
*.log
*.scc
# Visual C++ cache files
ipch/
*.aps
*.ncb
*.opensdf
*.sdf
*.cachefile
# Visual Studio profiler
*.psess
*.vsp
*.vspx
# Guidance Automation Toolkit
*.gpState
# ReSharper is a .NET coding add-in
_ReSharper*/
*.[Rr]e[Ss]harper
# TeamCity is a build add-in
_TeamCity*
# DotCover is a Code Coverage Tool
*.dotCover
# NCrunch
*.ncrunch*
.*crunch*.local.xml
# Installshield output folder
[Ee]xpress/
# DocProject is a documentation generator add-in
DocProject/buildhelp/
DocProject/Help/*.HxT
DocProject/Help/*.HxC
DocProject/Help/*.hhc
DocProject/Help/*.hhk
DocProject/Help/*.hhp
DocProject/Help/Html2
DocProject/Help/html
# Click-Once directory
publish/
# Publish Web Output
*.Publish.xml
*.pubxml
# NuGet Packages Directory
## TODO: If you have NuGet Package Restore enabled, uncomment the next line
#packages/
# Windows Azure Build Output
csx
*.build.csdef
# Windows Store app package directory
AppPackages/
# Others
sql/
*.Cache
ClientBin/
[Ss]tyle[Cc]op.*
~$*
*~
*.dbmdl
*.[Pp]ublish.xml
*.pfx
*.publishsettings
# RIA/Silverlight projects
Generated_Code/
# Backup & report files from converting an old project file to a newer
# Visual Studio version. Backup files are not needed, because we have git ;-)
_UpgradeReport_Files/
Backup*/
UpgradeLog*.XML
UpgradeLog*.htm
# SQL Server files
App_Data/*.mdf
App_Data/*.ldf
#################
## Other ide stuff
#################
.idea/*
[#]*[#]
#############
## Windows detritus
#############
# Windows image file caches
Thumbs.db
ehthumbs.db
# Folder config file
Desktop.ini
# Recycle Bin used on file shares
$RECYCLE.BIN/
# Mac crap
.DS_Store
#############
## Python
#############
*.py[co]
# Packages
*.egg
*.egg-info
dist/
build/
eggs/
parts/
var/
sdist/
develop-eggs/
.installed.cfg
# Installer logs
pip-log.txt
# Unit test / coverage reports
.coverage
.phpunit.result.cache
.tox
#Translations
*.mo
#Mr Developer
.mr.developer.cfg
##############
## RSS-Bridge
##############
/cache
/whitelist.txt
DEBUG
config.ini.php
config/*
!config/nginx.conf
!config/php-fpm.conf
!config/php.ini
######################
## VisualStudioCode ##
######################
.vscode/*
#Builder
.buildconfig
#Auth
.htaccess
.htpasswd
#Crawler
robots.txt

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

225
CONTRIBUTORS.md Normal file
View File

@ -0,0 +1,225 @@
# Contributors
* [16mhz](https://github.com/16mhz)
* [adamchainz](https://github.com/adamchainz)
* [Ahiles3005](https://github.com/Ahiles3005)
* [akirk](https://github.com/akirk)
* [Albirew](https://github.com/Albirew)
* [aledeg](https://github.com/aledeg)
* [alex73](https://github.com/alex73)
* [alexAubin](https://github.com/alexAubin)
* [Alkarex](https://github.com/Alkarex)
* [AmauryCarrade](https://github.com/AmauryCarrade)
* [arnd-s](https://github.com/arnd-s)
* [ArthurHoaro](https://github.com/ArthurHoaro)
* [Astalaseven](https://github.com/Astalaseven)
* [Astyan-42](https://github.com/Astyan-42)
* [austinhuang0131](https://github.com/austinhuang0131)
* [axor-mst](https://github.com/axor-mst)
* [ayacoo](https://github.com/ayacoo)
* [az5he6ch](https://github.com/az5he6ch)
* [b1nj](https://github.com/b1nj)
* [benasse](https://github.com/benasse)
* [Binnette](https://github.com/Binnette)
* [BoboTiG](https://github.com/BoboTiG)
* [Bockiii](https://github.com/Bockiii)
* [brtsos](https://github.com/brtsos)
* [captn3m0](https://github.com/captn3m0)
* [chemel](https://github.com/chemel)
* [Chouchen](https://github.com/Chouchen)
* [ckiw](https://github.com/ckiw)
* [cn-tools](https://github.com/cn-tools)
* [cnlpete](https://github.com/cnlpete)
* [corenting](https://github.com/corenting)
* [couraudt](https://github.com/couraudt)
* [csisoap](https://github.com/csisoap)
* [da2x](https://github.com/da2x)
* [dabenzel](https://github.com/dabenzel)
* [Daiyousei](https://github.com/Daiyousei)
* [dawidsowa](https://github.com/dawidsowa)
* [DevonHess](https://github.com/DevonHess)
* [dhuschde](https://github.com/dhuschde)
* [disk0x](https://github.com/disk0x)
* [DJCrashdummy](https://github.com/DJCrashdummy)
* [Djuuu](https://github.com/Djuuu)
* [DnAp](https://github.com/DnAp)
* [dominik-th](https://github.com/dominik-th)
* [Draeli](https://github.com/Draeli)
* [Dreckiger-Dan](https://github.com/Dreckiger-Dan)
* [drego85](https://github.com/drego85)
* [drklee3](https://github.com/drklee3)
* [DRogueRonin](https://github.com/DRogueRonin)
* [dvikan](https://github.com/dvikan)
* [eggwhalefrog](https://github.com/eggwhalefrog)
* [em92](https://github.com/em92)
* [eMerzh](https://github.com/eMerzh)
* [EtienneM](https://github.com/EtienneM)
* [f0086](https://github.com/f0086)
* [fanch317](https://github.com/fanch317)
* [fatuuse](https://github.com/fatuuse)
* [fivefilters](https://github.com/fivefilters)
* [floviolleau](https://github.com/floviolleau)
* [fluffy-critter](https://github.com/fluffy-critter)
* [fmachen](https://github.com/fmachen)
* [Frenzie](https://github.com/Frenzie)
* [fulmeek](https://github.com/fulmeek)
* [ggiessen](https://github.com/ggiessen)
* [gileri](https://github.com/gileri)
* [Ginko-Aloe](https://github.com/Ginko-Aloe)
* [girlpunk](https://github.com/girlpunk)
* [Glandos](https://github.com/Glandos)
* [gloony](https://github.com/gloony)
* [GregThib](https://github.com/GregThib)
* [griffaurel](https://github.com/griffaurel)
* [Grummfy](https://github.com/Grummfy)
* [gsantner](https://github.com/gsantner)
* [guigot](https://github.com/guigot)
* [hollowleviathan](https://github.com/hollowleviathan)
* [hpacleb](https://github.com/hpacleb)
* [hunhejj](https://github.com/hunhejj)
* [husim0](https://github.com/husim0)
* [IceWreck](https://github.com/IceWreck)
* [imagoiq](https://github.com/imagoiq)
* [j0k3r](https://github.com/j0k3r)
* [JackNUMBER](https://github.com/JackNUMBER)
* [jacquesh](https://github.com/jacquesh)
* [jakubvalenta](https://github.com/jakubvalenta)
* [JasonGhent](https://github.com/JasonGhent)
* [jcgoette](https://github.com/jcgoette)
* [jdesgats](https://github.com/jdesgats)
* [jdigilio](https://github.com/jdigilio)
* [JeremyRand](https://github.com/JeremyRand)
* [JimDog546](https://github.com/JimDog546)
* [jNullj](https://github.com/jNullj)
* [Jocker666z](https://github.com/Jocker666z)
* [johnnygroovy](https://github.com/johnnygroovy)
* [johnpc](https://github.com/johnpc)
* [joni1993](https://github.com/joni1993)
* [jtojnar](https://github.com/jtojnar)
* [KamaleiZestri](https://github.com/KamaleiZestri)
* [kkoyung](https://github.com/kkoyung)
* [klimplant](https://github.com/klimplant)
* [KN4CK3R](https://github.com/KN4CK3R)
* [kolarcz](https://github.com/kolarcz)
* [kranack](https://github.com/kranack)
* [kraoc](https://github.com/kraoc)
* [krisu5](https://github.com/krisu5)
* [l1n](https://github.com/l1n)
* [laBecasse](https://github.com/laBecasse)
* [lagaisse](https://github.com/lagaisse)
* [lalannev](https://github.com/lalannev)
* [langfingaz](https://github.com/langfingaz)
* [lassana](https://github.com/lassana)
* [ldidry](https://github.com/ldidry)
* [Leomaradan](https://github.com/Leomaradan)
* [leyrer](https://github.com/leyrer)
* [liamka](https://github.com/liamka)
* [Limero](https://github.com/Limero)
* [LogMANOriginal](https://github.com/LogMANOriginal)
* [lorenzos](https://github.com/lorenzos)
* [lukasklinger](https://github.com/lukasklinger)
* [m0zes](https://github.com/m0zes)
* [Mar-Koeh](https://github.com/Mar-Koeh)
* [marcus-at-localhost](https://github.com/marcus-at-localhost)
* [marius8510000-bot](https://github.com/marius8510000-bot)
* [matthewseal](https://github.com/matthewseal)
* [mcbyte-it](https://github.com/mcbyte-it)
* [mdemoss](https://github.com/mdemoss)
* [melangue](https://github.com/melangue)
* [metaMMA](https://github.com/metaMMA)
* [mibe](https://github.com/mibe)
* [mickaelBert](https://github.com/mickaelBert)
* [mightymt](https://github.com/mightymt)
* [mitsukarenai](https://github.com/mitsukarenai)
* [Monocularity](https://github.com/Monocularity)
* [MonsieurPoutounours](https://github.com/MonsieurPoutounours)
* [mr-flibble](https://github.com/mr-flibble)
* [mro](https://github.com/mro)
* [mschwld](https://github.com/mschwld)
* [muekoeff](https://github.com/muekoeff)
* [mw80](https://github.com/mw80)
* [mxmehl](https://github.com/mxmehl)
* [Mynacol](https://github.com/Mynacol)
* [nel50n](https://github.com/nel50n)
* [niawag](https://github.com/niawag)
* [Niehztog](https://github.com/Niehztog)
* [NikNikYkt](https://github.com/NikNikYkt)
* [Nono-m0le](https://github.com/Nono-m0le)
* [NotsoanoNimus](https://github.com/NotsoanoNimus)
* [obsiwitch](https://github.com/obsiwitch)
* [Ololbu](https://github.com/Ololbu)
* [ORelio](https://github.com/ORelio)
* [otakuf](https://github.com/otakuf)
* [Park0](https://github.com/Park0)
* [Paroleen](https://github.com/Paroleen)
* [Patricol](https://github.com/Patricol)
* [paulchen](https://github.com/paulchen)
* [PaulVayssiere](https://github.com/PaulVayssiere)
* [pellaeon](https://github.com/pellaeon)
* [PeterDaveHello](https://github.com/PeterDaveHello)
* [Peterr-K](https://github.com/Peterr-K)
* [Piranhaplant](https://github.com/Piranhaplant)
* [pirnz](https://github.com/pirnz)
* [pit-fgfjiudghdf](https://github.com/pit-fgfjiudghdf)
* [pitchoule](https://github.com/pitchoule)
* [pmaziere](https://github.com/pmaziere)
* [Pofilo](https://github.com/Pofilo)
* [prysme01](https://github.com/prysme01)
* [pubak42](https://github.com/pubak42)
* [Qluxzz](https://github.com/Qluxzz)
* [quentinus95](https://github.com/quentinus95)
* [quickwick](https://github.com/quickwick)
* [rakoo](https://github.com/rakoo)
* [RawkBob](https://github.com/RawkBob)
* [regisenguehard](https://github.com/regisenguehard)
* [Riduidel](https://github.com/Riduidel)
* [rogerdc](https://github.com/rogerdc)
* [Roliga](https://github.com/Roliga)
* [ronansalmon](https://github.com/ronansalmon)
* [rremizov](https://github.com/rremizov)
* [s0lesurviv0r](https://github.com/s0lesurviv0r)
* [sal0max](https://github.com/sal0max)
* [sebsauvage](https://github.com/sebsauvage)
* [shutosg](https://github.com/shutosg)
* [simon816](https://github.com/simon816)
* [Simounet](https://github.com/Simounet)
* [somini](https://github.com/somini)
* [SpangleLabs](https://github.com/SpangleLabs)
* [SqrtMinusOne](https://github.com/SqrtMinusOne)
* [squeek502](https://github.com/squeek502)
* [StelFux](https://github.com/StelFux)
* [stjohnjohnson](https://github.com/stjohnjohnson)
* [Stopka](https://github.com/Stopka)
* [Strubbl](https://github.com/Strubbl)
* [sublimz](https://github.com/sublimz)
* [sunchaserinfo](https://github.com/sunchaserinfo)
* [SuperSandro2000](https://github.com/SuperSandro2000)
* [sysadminstory](https://github.com/sysadminstory)
* [t0stiman](https://github.com/t0stiman)
* [tameroski](https://github.com/tameroski)
* [teromene](https://github.com/teromene)
* [tgkenney](https://github.com/tgkenney)
* [thefranke](https://github.com/thefranke)
* [TheRadialActive](https://github.com/TheRadialActive)
* [theScrabi](https://github.com/theScrabi)
* [thezeroalpha](https://github.com/thezeroalpha)
* [thibaultcouraud](https://github.com/thibaultcouraud)
* [timendum](https://github.com/timendum)
* [TitiTestScalingo](https://github.com/TitiTestScalingo)
* [tomaszkane](https://github.com/tomaszkane)
* [tomershvueli](https://github.com/tomershvueli)
* [TotalCaesar659](https://github.com/TotalCaesar659)
* [tpikonen](https://github.com/tpikonen)
* [TReKiE](https://github.com/TReKiE)
* [triatic](https://github.com/triatic)
* [User123698745](https://github.com/User123698745)
* [VerifiedJoseph](https://github.com/VerifiedJoseph)
* [vitkabele](https://github.com/vitkabele)
* [WalterBarrett](https://github.com/WalterBarrett)
* [wtuuju](https://github.com/wtuuju)
* [xurxof](https://github.com/xurxof)
* [yamanq](https://github.com/yamanq)
* [yardenac](https://github.com/yardenac)
* [ymeister](https://github.com/ymeister)
* [yue-dongchen](https://github.com/yue-dongchen)
* [ZeNairolf](https://github.com/ZeNairolf)

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

72
Dockerfile Normal file
View File

@ -0,0 +1,72 @@
FROM debian:12-slim AS rssbridge
LABEL description="RSS-Bridge is a PHP project capable of generating RSS and Atom feeds for websites that don't have one."
LABEL repository="https://github.com/RSS-Bridge/rss-bridge"
LABEL website="https://github.com/RSS-Bridge/rss-bridge"
ARG DEBIAN_FRONTEND=noninteractive
RUN set -xe && \
apt-get update && \
apt-get install --yes --no-install-recommends \
ca-certificates \
nginx \
nss-plugin-pem \
php-curl \
php-fpm \
php-intl \
# php-json is enabled by default with PHP 8.2 in Debian 12
php-mbstring \
php-memcached \
# php-opcache is enabled by default with PHP 8.2 in Debian 12
# php-openssl is enabled by default with PHP 8.2 in Debian 12
php-sqlite3 \
php-xml \
php-zip \
# php-zlib is enabled by default with PHP 8.2 in Debian 12
# for downloading libcurl-impersonate
curl \
&& \
# install curl-impersonate library
curlimpersonate_version=0.6.0 && \
{ \
{ \
[ $(arch) = 'aarch64' ] && \
archive="libcurl-impersonate-v${curlimpersonate_version}.aarch64-linux-gnu.tar.gz" && \
sha512sum="d04b1eabe71f3af06aa1ce99b39a49c5e1d33b636acedcd9fad163bc58156af5c3eb3f75aa706f335515791f7b9c7a6c40ffdfa47430796483ecef929abd905d" \
; } \
|| { \
[ $(arch) = 'armv7l' ] && \
archive="libcurl-impersonate-v${curlimpersonate_version}.arm-linux-gnueabihf.tar.gz" && \
sha512sum="05906b4efa1a6ed8f3b716fd83d476b6eea6bfc68e3dbc5212d65a2962dcaa7bd1f938c9096a7535252b11d1d08fb93adccc633585ff8cb8cec5e58bfe969bc9" \
; } \
|| { \
[ $(arch) = 'x86_64' ] && \
archive="libcurl-impersonate-v${curlimpersonate_version}.x86_64-linux-gnu.tar.gz" && \
sha512sum="480bbe9452cd9aff2c0daaaf91f1057b3a96385f79011628a9237223757a9b0d090c59cb5982dc54ea0d07191657299ea91ca170a25ced3d7d410fcdff130ace" \
; } \
} && \
curl -LO "https://github.com/lwthiker/curl-impersonate/releases/download/v${curlimpersonate_version}/${archive}" && \
echo "$sha512sum $archive" | sha512sum -c - && \
mkdir -p /usr/local/lib/curl-impersonate && \
tar xaf "$archive" -C /usr/local/lib/curl-impersonate --wildcards 'libcurl-impersonate-ff.so*' && \
rm "$archive" && \
apt-get purge --assume-yes curl && \
rm -rf /var/lib/apt/lists/*
ENV LD_PRELOAD /usr/local/lib/curl-impersonate/libcurl-impersonate-ff.so
ENV CURL_IMPERSONATE ff91esr
# logs should go to stdout / stderr
RUN ln -sfT /dev/stderr /var/log/nginx/error.log; \
ln -sfT /dev/stdout /var/log/nginx/access.log; \
chown -R --no-dereference www-data:adm /var/log/nginx/
COPY ./config/nginx.conf /etc/nginx/sites-available/default
COPY ./config/php-fpm.conf /etc/php/8.2/fpm/pool.d/rss-bridge.conf
COPY ./config/php.ini /etc/php/8.2/fpm/conf.d/90-rss-bridge.ini
COPY --chown=www-data:www-data ./ /app/
EXPOSE 80
ENTRYPOINT ["/app/docker-entrypoint.sh"]

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

536
README.md Normal file
View File

@ -0,0 +1,536 @@
# RSS-Bridge
![RSS-Bridge](static/logo_600px.png)
RSS-Bridge is a PHP web application.
It generates web feeds for websites that don't have one.
Officially hosted instance: https://rss-bridge.org/bridge01/
IRC channel #rssbridge at https://libera.chat/
[Full documentation](https://rss-bridge.github.io/rss-bridge/index.html)
Alternatively find another
[public instance](https://rss-bridge.github.io/rss-bridge/General/Public_Hosts.html).
Requires minimum PHP 7.4.
[![LICENSE](https://img.shields.io/badge/license-UNLICENSE-blue.svg)](UNLICENSE)
[![GitHub release](https://img.shields.io/github/release/rss-bridge/rss-bridge.svg?logo=github)](https://github.com/rss-bridge/rss-bridge/releases/latest)
[![irc.libera.chat](https://img.shields.io/badge/irc.libera.chat-%23rssbridge-blue.svg)](https://web.libera.chat/#rssbridge)
[![Actions Status](https://img.shields.io/github/actions/workflow/status/RSS-Bridge/rss-bridge/tests.yml?branch=master&label=GitHub%20Actions&logo=github)](https://github.com/RSS-Bridge/rss-bridge/actions)
|||
|:-:|:-:|
|![Screenshot #1](/static/screenshot-1.png?raw=true)|![Screenshot #2](/static/screenshot-2.png?raw=true)|
|![Screenshot #3](/static/screenshot-3.png?raw=true)|![Screenshot #4](/static/screenshot-4.png?raw=true)|
|![Screenshot #5](/static/screenshot-5.png?raw=true)|![Screenshot #6](/static/screenshot-6.png?raw=true)|
## A subset of bridges (15/447)
* `CssSelectorBridge`: [Scrape out a feed using CSS selectors](https://rss-bridge.org/bridge01/#bridge-CssSelectorBridge)
* `FeedMergeBridge`: [Combine multiple feeds into one](https://rss-bridge.org/bridge01/#bridge-FeedMergeBridge)
* `FeedReducerBridge`: [Reduce a noisy feed by some percentage](https://rss-bridge.org/bridge01/#bridge-FeedReducerBridge)
* `FilterBridge`: [Filter a feed by excluding/including items by keyword](https://rss-bridge.org/bridge01/#bridge-FilterBridge)
* `GettrBridge`: [Fetches the latest posts from a GETTR user](https://rss-bridge.org/bridge01/#bridge-GettrBridge)
* `MastodonBridge`: [Fetches statuses from a Mastodon (ActivityPub) instance](https://rss-bridge.org/bridge01/#bridge-MastodonBridge)
* `RedditBridge`: [Fetches posts from a user/subredit (with filtering options)](https://rss-bridge.org/bridge01/#bridge-RedditBridge)
* `RumbleBridge`: [Fetches channel/user videos](https://rss-bridge.org/bridge01/#bridge-RumbleBridge)
* `SoundcloudBridge`: [Fetches music by username](https://rss-bridge.org/bridge01/#bridge-SoundcloudBridge)
* `TelegramBridge`: [Fetches posts from a public channel](https://rss-bridge.org/bridge01/#bridge-TelegramBridge)
* `ThePirateBayBridge:` [Fetches torrents by search/user/category](https://rss-bridge.org/bridge01/#bridge-ThePirateBayBridge)
* `TikTokBridge`: [Fetches posts by username](https://rss-bridge.org/bridge01/#bridge-TikTokBridge)
* `TwitchBridge`: [Fetches videos from channel](https://rss-bridge.org/bridge01/#bridge-TwitchBridge)
* `XPathBridge`: [Scrape out a feed using XPath expressions](https://rss-bridge.org/bridge01/#bridge-XPathBridge)
* `YoutubeBridge`: [Fetches videos by username/channel/playlist/search](https://rss-bridge.org/bridge01/#bridge-YoutubeBridge)
* `YouTubeCommunityTabBridge`: [Fetches posts from a channel's community tab](https://rss-bridge.org/bridge01/#bridge-YouTubeCommunityTabBridge)
## Tutorial
### How to install on traditional shared web hosting
RSS-Bridge can basically be unzipped into a web folder. Should be working instantly.
Latest zip:
https://github.com/RSS-Bridge/rss-bridge/archive/refs/heads/master.zip (2MB)
### How to install on Debian 12 (nginx + php-fpm)
These instructions have been tested on a fresh Debian 12 VM from Digital Ocean (1vcpu-512mb-10gb, 5 USD/month).
```shell
timedatectl set-timezone Europe/Oslo
apt install git nginx php8.2-fpm php-mbstring php-simplexml php-curl php-intl
# Create a user account
useradd --shell /bin/bash --create-home rss-bridge
cd /var/www
# Create folder and change its ownership to rss-bridge
mkdir rss-bridge && chown rss-bridge:rss-bridge rss-bridge/
# Become rss-bridge
su rss-bridge
# Clone master branch into existing folder
git clone https://github.com/RSS-Bridge/rss-bridge.git rss-bridge/
cd rss-bridge
# Copy over the default config (OPTIONAL)
cp -v config.default.ini.php config.ini.php
# Recursively give full permissions to user/owner
chmod 700 --recursive ./
# Give read and execute to others on folder ./static
chmod o+rx ./ ./static
# Recursively give give read to others on folder ./static
chmod o+r --recursive ./static
```
Nginx config:
```nginx
# /etc/nginx/sites-enabled/rss-bridge.conf
server {
listen 80;
# TODO: change to your own server name
server_name example.com;
access_log /var/log/nginx/rss-bridge.access.log;
error_log /var/log/nginx/rss-bridge.error.log;
log_not_found off;
# Intentionally not setting a root folder
# Static content only served here
location /static/ {
alias /var/www/rss-bridge/static/;
}
# Pass off to php-fpm only when location is EXACTLY == /
location = / {
root /var/www/rss-bridge/;
include snippets/fastcgi-php.conf;
fastcgi_read_timeout 45s;
fastcgi_pass unix:/run/php/rss-bridge.sock;
}
# Reduce log noise
location = /favicon.ico {
access_log off;
}
# Reduce log noise
location = /robots.txt {
access_log off;
}
}
```
PHP FPM pool config:
```ini
; /etc/php/8.2/fpm/pool.d/rss-bridge.conf
[rss-bridge]
user = rss-bridge
group = rss-bridge
listen = /run/php/rss-bridge.sock
listen.owner = www-data
listen.group = www-data
; Create 10 workers standing by to serve requests
pm = static
pm.max_children = 10
; Respawn worker after 500 requests (workaround for memory leaks etc.)
pm.max_requests = 500
```
PHP ini config:
```ini
; /etc/php/8.2/fpm/conf.d/30-rss-bridge.ini
max_execution_time = 15
memory_limit = 64M
```
Restart fpm and nginx:
```shell
# Lint and restart php-fpm
php-fpm8.2 -t && systemctl restart php8.2-fpm
# Lint and restart nginx
nginx -t && systemctl restart nginx
```
### How to install from Composer
Install the latest release.
```shell
cd /var/www
composer create-project -v --no-dev --no-scripts rss-bridge/rss-bridge
```
### How to install with Caddy
TODO. See https://github.com/RSS-Bridge/rss-bridge/issues/3785
### Install from Docker Hub:
Install by downloading the docker image from Docker Hub:
```bash
# Create container
docker create --name=rss-bridge --publish 3000:80 --volume $(pwd)/config:/config rssbridge/rss-bridge
```
You can put custom `config.ini.php` and bridges into `./config`.
**You must restart container for custom changes to take effect.**
See `docker-entrypoint.sh` for details.
```bash
# Start container
docker start rss-bridge
```
Browse http://localhost:3000/
### Install by locally building from Dockerfile
```bash
# Build image from Dockerfile
docker build -t rss-bridge .
# Create container
docker create --name rss-bridge --publish 3000:80 --volume $(pwd)/config:/config rss-bridge
```
You can put custom `config.ini.php` and bridges into `./config`.
**You must restart container for custom changes to take effect.**
See `docker-entrypoint.sh` for details.
```bash
# Start container
docker start rss-bridge
```
Browse http://localhost:3000/
### Install with docker-compose (using Docker Hub)
You can put custom `config.ini.php` and bridges into `./config`.
**You must restart container for custom changes to take effect.**
See `docker-entrypoint.sh` for details.
```bash
docker-compose up
```
Browse http://localhost:3000/
### Other installation methods
[![Deploy on Scalingo](https://cdn.scalingo.com/deploy/button.svg)](https://my.scalingo.com/deploy?source=https://github.com/sebsauvage/rss-bridge)
[![Deploy to Heroku](https://www.herokucdn.com/deploy/button.svg)](https://heroku.com/deploy)
[![Deploy to Cloudron](https://cloudron.io/img/button.svg)](https://www.cloudron.io/store/com.rssbridgeapp.cloudronapp.html)
[![Run on PikaPods](https://www.pikapods.com/static/run-button.svg)](https://www.pikapods.com/pods?run=rssbridge)
The Heroku quick deploy currently does not work. It might work if you fork this repo and
modify the `repository` in `scalingo.json`. See https://github.com/RSS-Bridge/rss-bridge/issues/2688
Learn more in
[Installation](https://rss-bridge.github.io/rss-bridge/For_Hosts/Installation.html).
## How-to
### How to fix "Access denied."
Output is from php-fpm. It is unable to read index.php.
chown rss-bridge:rss-bridge /var/www/rss-bridge/index.php
### How to password-protect the instance (token)
Modify `config.ini.php`:
[authentication]
token = "hunter2"
### How to remove all cache items
As current user:
bin/cache-clear
As user rss-bridge:
sudo -u rss-bridge bin/cache-clear
As root:
sudo bin/cache-clear
### How to remove all expired cache items
bin/cache-prune
### How to fix "PHP Fatal error: Uncaught Exception: The FileCache path is not writable"
```shell
# Give rss-bridge ownership
chown rss-bridge:rss-bridge -R /var/www/rss-bridge/cache
# Or, give www-data ownership
chown www-data:www-data -R /var/www/rss-bridge/cache
# Or, give everyone write permission
chmod 777 -R /var/www/rss-bridge/cache
# Or last ditch effort (CAREFUL)
rm -rf /var/www/rss-bridge/cache/ && mkdir /var/www/rss-bridge/cache/
```
### How to fix "attempt to write a readonly database"
The sqlite files (db, wal and shm) are not writeable.
chown -v rss-bridge:rss-bridge cache/*
### How to fix "Unable to prepare statement: 1, no such table: storage"
rm cache/*
### How to create a new bridge from scratch
Create the new bridge in e.g. `bridges/BearBlogBridge.php`:
```php
<?php
class BearBlogBridge extends BridgeAbstract
{
const NAME = 'BearBlog (bearblog.dev)';
public function collectData()
{
$dom = getSimpleHTMLDOM('https://herman.bearblog.dev/blog/');
foreach ($dom->find('.blog-posts li') as $li) {
$a = $li->find('a', 0);
$this->items[] = [
'title' => $a->plaintext,
'uri' => 'https://herman.bearblog.dev' . $a->href,
];
}
}
}
```
Learn more in [bridge api](https://rss-bridge.github.io/rss-bridge/Bridge_API/index.html).
### How to enable all bridges
enabled_bridges[] = *
### How to enable some bridges
```
enabled_bridges[] = TwitchBridge
enabled_bridges[] = GettrBridge
```
### How to enable debug mode
The
[debug mode](https://rss-bridge.github.io/rss-bridge/For_Developers/Debug_mode.html)
disables the majority of caching operations.
enable_debug_mode = true
### How to switch to memcached as cache backend
```
[cache]
; Cache backend: file (default), sqlite, memcached, null
type = "memcached"
```
### How to switch to sqlite3 as cache backend
type = "sqlite"
### How to disable bridge errors (as feed items)
When a bridge fails, RSS-Bridge will produce a feed with a single item describing the error.
This way, feed readers pick it up and you are notified.
If you don't want this behaviour, switch the error output to `http`:
[error]
; Defines how error messages are returned by RSS-Bridge
;
; "feed" = As part of the feed (default)
; "http" = As HTTP error message
; "none" = No errors are reported
output = "http"
### How to accumulate errors before finally reporting it
Modify `report_limit` so that an error must occur 3 times before it is reported.
; Defines how often an error must occur before it is reported to the user
report_limit = 3
The report count is reset to 0 each day.
### How to password-protect the instance (HTTP Basic Auth)
[authentication]
enable = true
username = "alice"
password = "cat"
Will typically require feed readers to be configured with the credentials.
It may also be possible to manually include the credentials in the URL:
https://alice:cat@rss-bridge.org/bridge01/?action=display&bridge=FabriceBellardBridge&format=Html
### How to create a new output format
See `formats/PlaintextFormat.php` for an example.
### How to run unit tests and linter
These commands require that you have installed the dev dependencies in `composer.json`.
Run all tests:
./vendor/bin/phpunit
Run a single test class:
./vendor/bin/phpunit --filter UrlTest
Run linter:
./vendor/bin/phpcs --standard=phpcs.xml --warning-severity=0 --extensions=php -p ./
https://github.com/squizlabs/PHP_CodeSniffer/wiki
### How to spawn a minimal development environment
php -S 127.0.0.1:9001
http://127.0.0.1:9001/
## Explanation
We are RSS-Bridge community, a group of developers continuing the project initiated by sebsauvage,
webmaster of
[sebsauvage.net](https://sebsauvage.net), author of
[Shaarli](https://sebsauvage.net/wiki/doku.php?id=php:shaarli) and
[ZeroBin](https://sebsauvage.net/wiki/doku.php?id=php:zerobin).
See [CONTRIBUTORS.md](CONTRIBUTORS.md)
RSS-Bridge uses caching to prevent services from banning your server for repeatedly updating feeds.
The specific cache duration can be different between bridges.
RSS-Bridge allows you to take full control over which bridges are displayed to the user.
That way you can host your own RSS-Bridge service with your favorite collection of bridges!
Current maintainers (as of 2024): @dvikan and @Mynacol #2519
## Reference
### Feed item structure
This is the feed item structure that bridges are expected to produce.
```php
$item = [
'uri' => 'https://example.com/blog/hello',
'title' => 'Hello world',
// Publication date in unix timestamp
'timestamp' => 1668706254,
'author' => 'Alice',
'content' => 'Here be item content',
'enclosures' => [
'https://example.com/foo.png',
'https://example.com/bar.png'
],
'categories' => [
'news',
'tech',
],
// Globally unique id
'uid' => 'e7147580c8747aad',
]
```
### Output formats
* `Atom`: Atom feed, for use in feed readers
* `Html`: Simple HTML page
* `Json`: JSON, for consumption by other applications
* `Mrss`: MRSS feed, for use in feed readers
* `Plaintext`: Raw text, for consumption by other applications
* `Sfeed`: Text, TAB separated
### Cache backends
* `File`
* `SQLite`
* `Memcached`
* `Array`
* `Null`
### Licenses
The source code for RSS-Bridge is [Public Domain](UNLICENSE).
RSS-Bridge uses third party libraries with their own license:
* [`Parsedown`](https://github.com/erusev/parsedown) licensed under the [MIT License](https://opensource.org/licenses/MIT)
* [`PHP Simple HTML DOM Parser`](https://simplehtmldom.sourceforge.io/docs/1.9/index.html) licensed under the [MIT License](https://opensource.org/licenses/MIT)
* [`php-urljoin`](https://github.com/fluffy-critter/php-urljoin) licensed under the [MIT License](https://opensource.org/licenses/MIT)
* [`Laravel framework`](https://github.com/laravel/framework/) licensed under the [MIT License](https://opensource.org/licenses/MIT)
## Rant
*Dear so-called "social" websites.*
Your catchword is "share", but you don't want us to share. You want to keep us within your walled gardens. That's why you've been removing RSS links from webpages, hiding them deep on your website, or removed feeds entirely, replacing it with crippled or demented proprietary API. **FUCK YOU.**
You're not social when you hamper sharing by removing feeds. You're happy to have customers creating content for your ecosystem, but you don't want this content out - a content you do not even own. Google Takeout is just a gimmick. We want our data to flow, we want RSS or Atom feeds.
We want to share with friends, using open protocols: RSS, Atom, XMPP, whatever. Because no one wants to have *your* service with *your* applications using *your* API force-feeding them. Friends must be free to choose whatever software and service they want.
We are rebuilding bridges you have willfully destroyed.
Get your shit together: Put RSS/Atom back in.

File diff suppressed because one or more lines are too long

25
UNLICENSE Normal file
View File

@ -0,0 +1,25 @@
This is free and unencumbered software released into the public domain.
Anyone is free to copy, modify, publish, use, compile, sell, or
distribute this software, either in source code form or as a compiled
binary, for any purpose, commercial or non-commercial, and by any
means.
In jurisdictions that recognize copyright laws, the author or authors
of this software dedicate any and all copyright interest in the
software to the public domain. We make this dedication for the benefit
of the public at large and to the detriment of our heirs and
successors. We intend this dedication to be an overt act of
relinquishment in perpetuity of all present and future rights to this
software under copyright law.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
For more information, please refer to <http://unlicense.org>

View File

@ -0,0 +1,67 @@
<?php
/**
* Checks if the website for a given bridge is reachable.
*
* **Remarks**
* - This action is only available in debug mode.
* - Returns the bridge status as Json-formatted string.
* - Returns an error if the bridge is not whitelisted.
* - Returns a responsive web page that automatically checks all whitelisted
* bridges (using JavaScript) if no bridge is specified.
*/
class ConnectivityAction implements ActionInterface
{
private BridgeFactory $bridgeFactory;
public function __construct(
BridgeFactory $bridgeFactory
) {
$this->bridgeFactory = $bridgeFactory;
}
public function __invoke(Request $request): Response
{
if (!Debug::isEnabled()) {
return new Response('This action is only available in debug mode!', 403);
}
$bridgeName = $request->get('bridge');
if (!$bridgeName) {
return new Response(render_template('connectivity.html.php'));
}
$bridgeClassName = $this->bridgeFactory->createBridgeClassName($bridgeName);
if (!$bridgeClassName) {
return new Response('Bridge not found', 404);
}
return $this->reportBridgeConnectivity($bridgeClassName);
}
private function reportBridgeConnectivity($bridgeClassName)
{
if (!$this->bridgeFactory->isEnabled($bridgeClassName)) {
throw new \Exception('Bridge is not whitelisted!');
}
$bridge = $this->bridgeFactory->create($bridgeClassName);
$curl_opts = [
CURLOPT_CONNECTTIMEOUT => 5,
CURLOPT_FOLLOWLOCATION => true,
];
$result = [
'bridge' => $bridgeClassName,
'successful' => false,
'http_code' => null,
];
try {
$response = getContents($bridge::URI, [], $curl_opts, true);
$result['http_code'] = $response->getCode();
if (in_array($result['http_code'], [200])) {
$result['successful'] = true;
}
} catch (\Exception $e) {
}
return new Response(Json::encode($result), 200, ['content-type' => 'text/json']);
}
}

51
actions/DetectAction.php Normal file
View File

@ -0,0 +1,51 @@
<?php
class DetectAction implements ActionInterface
{
private BridgeFactory $bridgeFactory;
public function __construct(
BridgeFactory $bridgeFactory
) {
$this->bridgeFactory = $bridgeFactory;
}
public function __invoke(Request $request): Response
{
$url = $request->get('url');
$format = $request->get('format');
if (!$url) {
return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'You must specify a url']));
}
if (!$format) {
return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'You must specify a format']));
}
foreach ($this->bridgeFactory->getBridgeClassNames() as $bridgeClassName) {
if (!$this->bridgeFactory->isEnabled($bridgeClassName)) {
continue;
}
$bridge = $this->bridgeFactory->create($bridgeClassName);
$bridgeParams = $bridge->detectParameters($url);
if (!$bridgeParams) {
continue;
}
$query = [
'action' => 'display',
'bridge' => $bridgeClassName,
'format' => $format,
];
$query = array_merge($query, $bridgeParams);
return new Response('', 301, ['location' => '?' . http_build_query($query)]);
}
return new Response(render(__DIR__ . '/../templates/error.html.php', [
'message' => 'No bridge found for given URL: ' . $url,
]));
}
}

231
actions/DisplayAction.php Normal file
View File

@ -0,0 +1,231 @@
<?php
class DisplayAction implements ActionInterface
{
private CacheInterface $cache;
private Logger $logger;
private BridgeFactory $bridgeFactory;
public function __construct(
CacheInterface $cache,
Logger $logger,
BridgeFactory $bridgeFactory
) {
$this->cache = $cache;
$this->logger = $logger;
$this->bridgeFactory = $bridgeFactory;
}
public function __invoke(Request $request): Response
{
$bridgeName = $request->get('bridge');
$format = $request->get('format');
$noproxy = $request->get('_noproxy');
if (!$bridgeName) {
return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'Missing bridge name parameter']), 400);
}
$bridgeClassName = $this->bridgeFactory->createBridgeClassName($bridgeName);
if (!$bridgeClassName) {
return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'Bridge not found']), 404);
}
if (!$format) {
return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'You must specify a format']), 400);
}
if (!$this->bridgeFactory->isEnabled($bridgeClassName)) {
return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'This bridge is not whitelisted']), 400);
}
// Disable proxy (if enabled and per user's request)
if (
Configuration::getConfig('proxy', 'url')
&& Configuration::getConfig('proxy', 'by_bridge')
&& $noproxy
) {
// This const is only used once in getContents()
define('NOPROXY', true);
}
$cacheKey = 'http_' . json_encode($request->toArray());
$bridge = $this->bridgeFactory->create($bridgeClassName);
$response = $this->createResponse($request, $bridge, $format);
if ($response->getCode() === 200) {
$ttl = $request->get('_cache_timeout');
if (Configuration::getConfig('cache', 'custom_timeout') && $ttl) {
$ttl = (int) $ttl;
} else {
$ttl = $bridge->getCacheTimeout();
}
$this->cache->set($cacheKey, $response, $ttl);
}
return $response;
}
private function createResponse(Request $request, BridgeAbstract $bridge, string $format)
{
$items = [];
try {
$bridge->loadConfiguration();
// Remove parameters that don't concern bridges
$remove = [
'token',
'action',
'bridge',
'format',
'_noproxy',
'_cache_timeout',
'_error_time',
'_', // Some RSS readers add a cache-busting parameter (_=<timestamp>) to feed URLs, detect and ignore them.
];
$requestArray = $request->toArray();
$input = array_diff_key($requestArray, array_fill_keys($remove, ''));
$bridge->setInput($input);
$bridge->collectData();
$items = $bridge->getItems();
} catch (\Throwable $e) {
if ($e instanceof RateLimitException) {
// These are internally generated by bridges
$this->logger->info(sprintf('RateLimitException in DisplayAction(%s): %s', $bridge->getShortName(), create_sane_exception_message($e)));
return new Response(render(__DIR__ . '/../templates/exception.html.php', ['e' => $e]), 429);
}
if ($e instanceof HttpException) {
if (in_array($e->getCode(), [429, 503])) {
// Log with debug, immediately reproduce and return
$this->logger->debug(sprintf('Exception in DisplayAction(%s): %s', $bridge->getShortName(), create_sane_exception_message($e)));
return new Response(render(__DIR__ . '/../templates/exception.html.php', ['e' => $e]), $e->getCode());
}
// Some other status code which we let fail normally (but don't log it)
} else {
// Log error if it's not an HttpException
$this->logger->error(sprintf('Exception in DisplayAction(%s)', $bridge->getShortName()), ['e' => $e]);
}
$errorOutput = Configuration::getConfig('error', 'output');
$reportLimit = Configuration::getConfig('error', 'report_limit');
$errorCount = 1;
if ($reportLimit > 1) {
$errorCount = $this->logBridgeError($bridge->getName(), $e->getCode());
}
// Let clients know about the error if we are passed the report limit
if ($errorCount >= $reportLimit) {
if ($errorOutput === 'feed') {
// Render the exception as a feed item
$items = [$this->createFeedItemFromException($e, $bridge)];
} elseif ($errorOutput === 'http') {
return new Response(render(__DIR__ . '/../templates/exception.html.php', ['e' => $e]), 500);
} elseif ($errorOutput === 'none') {
// Do nothing (produces an empty feed)
}
}
}
$formatFactory = new FormatFactory();
$format = $formatFactory->create($format);
$format->setItems($items);
$format->setFeed($bridge->getFeed());
$now = time();
$format->setLastModified($now);
$headers = [
'last-modified' => gmdate('D, d M Y H:i:s ', $now) . 'GMT',
'content-type' => $format->getMimeType() . '; charset=UTF-8',
];
$body = $format->render();
// This is supposed to remove non-utf8 byte sequences, but I'm unsure if it works
ini_set('mbstring.substitute_character', 'none');
$body = mb_convert_encoding($body, 'UTF-8', 'UTF-8');
return new Response($body, 200, $headers);
}
private function createFeedItemFromException($e, BridgeAbstract $bridge): array
{
$item = [];
// Create a unique identifier every 24 hours
$uniqueIdentifier = urlencode((int)(time() / 86400));
$title = sprintf('Bridge returned error %s! (%s)', $e->getCode(), $uniqueIdentifier);
$item['title'] = $title;
$item['uri'] = get_current_url();
$item['timestamp'] = time();
// Create an item identifier for feed readers e.g. "staysafetv twitch videos_19389"
$item['uid'] = $bridge->getName() . '_' . $uniqueIdentifier;
$content = render_template(__DIR__ . '/../templates/bridge-error.html.php', [
'error' => render_template(__DIR__ . '/../templates/exception.html.php', ['e' => $e]),
'searchUrl' => self::createGithubSearchUrl($bridge),
'issueUrl' => self::createGithubIssueUrl($bridge, $e),
'maintainer' => $bridge->getMaintainer(),
]);
$item['content'] = $content;
return $item;
}
private function logBridgeError($bridgeName, $code)
{
// todo: it's not really necessary to json encode $report
$cacheKey = 'error_reporting_' . $bridgeName . '_' . $code;
$report = $this->cache->get($cacheKey);
if ($report) {
$report = Json::decode($report);
$report['time'] = time();
$report['count']++;
} else {
$report = [
'error' => $code,
'time' => time(),
'count' => 1,
];
}
$ttl = 86400 * 5;
$this->cache->set($cacheKey, Json::encode($report), $ttl);
return $report['count'];
}
private static function createGithubIssueUrl(BridgeAbstract $bridge, \Throwable $e): string
{
$maintainer = $bridge->getMaintainer();
if (str_contains($maintainer, ',')) {
$maintainers = explode(',', $maintainer);
} else {
$maintainers = [$maintainer];
}
$maintainers = array_map('trim', $maintainers);
$queryString = $_SERVER['QUERY_STRING'] ?? '';
$query = [
'title' => $bridge->getName() . ' failed with: ' . $e->getMessage(),
'body' => sprintf(
"```\n%s\n\n%s\n\nQuery string: %s\nVersion: %s\nOs: %s\nPHP version: %s\n```\nMaintainer: @%s",
create_sane_exception_message($e),
implode("\n", trace_to_call_points(trace_from_exception($e))),
$queryString,
Configuration::getVersion(),
PHP_OS_FAMILY,
phpversion() ?: 'Unknown',
implode(', @', $maintainers),
),
'labels' => 'Bridge-Broken',
'assignee' => $maintainer[0],
];
return 'https://github.com/RSS-Bridge/rss-bridge/issues/new?' . http_build_query($query);
}
private static function createGithubSearchUrl($bridge): string
{
return sprintf(
'https://github.com/RSS-Bridge/rss-bridge/issues?q=%s',
urlencode('is:issue is:open ' . $bridge->getName())
);
}
}

View File

@ -0,0 +1,95 @@
<?php
/**
* This action is used by the frontpage form search.
* It finds a bridge based off of a user input url.
* It uses bridges' detectParameters implementation.
*/
class FindfeedAction implements ActionInterface
{
private BridgeFactory $bridgeFactory;
public function __construct(
BridgeFactory $bridgeFactory
) {
$this->bridgeFactory = $bridgeFactory;
}
public function __invoke(Request $request): Response
{
$url = $request->get('url');
$format = $request->get('format');
if (!$url) {
return new Response('You must specify a url', 400);
}
if (!$format) {
return new Response('You must specify a format', 400);
}
$results = [];
foreach ($this->bridgeFactory->getBridgeClassNames() as $bridgeClassName) {
if (!$this->bridgeFactory->isEnabled($bridgeClassName)) {
continue;
}
$bridge = $this->bridgeFactory->create($bridgeClassName);
$bridgeParams = $bridge->detectParameters($url);
if ($bridgeParams === null) {
continue;
}
// It's allowed to have no 'context' in a bridge (only a default context without any name)
// In this case, the reference to the parameters are found in the first element of the PARAMETERS array
$context = $bridgeParams['context'] ?? 0;
$bridgeData = [];
// Construct the array of parameters
foreach ($bridgeParams as $key => $value) {
// 'context' is a special case : it's a bridge parameters, there is no "name" for this parameter
if ($key == 'context') {
$bridgeData[$key]['name'] = 'Context';
$bridgeData[$key]['value'] = $value;
} else {
$bridgeData[$key]['name'] = $this->getParameterName($bridge, $context, $key);
$bridgeData[$key]['value'] = $value;
}
}
$bridgeParams['bridge'] = $bridgeClassName;
$bridgeParams['format'] = $format;
$content = [
'url' => './?action=display&' . http_build_query($bridgeParams),
'bridgeParams' => $bridgeParams,
'bridgeData' => $bridgeData,
'bridgeMeta' => [
'name' => $bridge::NAME,
'description' => $bridge::DESCRIPTION,
'parameters' => $bridge::PARAMETERS,
'icon' => $bridge->getIcon(),
],
];
$results[] = $content;
}
if ($results === []) {
return new Response(Json::encode(['message' => 'No bridge found for given url']), 404, ['content-type' => 'application/json']);
}
return new Response(Json::encode($results), 200, ['content-type' => 'application/json']);
}
// Get parameter name in the actual context, or in the global parameter
private function getParameterName($bridge, $context, $key)
{
if (isset($bridge::PARAMETERS[$context][$key]['name'])) {
$name = $bridge::PARAMETERS[$context][$key]['name'];
} else if (isset($bridge::PARAMETERS['global'][$key]['name'])) {
$name = $bridge::PARAMETERS['global'][$key]['name'];
} else {
$name = 'Variable "' . $key . '" (No name provided)';
}
return $name;
}
}

View File

@ -0,0 +1,49 @@
<?php
final class FrontpageAction implements ActionInterface
{
private BridgeFactory $bridgeFactory;
public function __construct(
BridgeFactory $bridgeFactory
) {
$this->bridgeFactory = $bridgeFactory;
}
public function __invoke(Request $request): Response
{
$token = $request->getAttribute('token');
$messages = [];
$activeBridges = 0;
$bridgeClassNames = $this->bridgeFactory->getBridgeClassNames();
foreach ($this->bridgeFactory->getMissingEnabledBridges() as $missingEnabledBridge) {
$messages[] = [
'body' => sprintf('Warning : Bridge "%s" not found', $missingEnabledBridge),
'level' => 'warning'
];
}
$body = '';
foreach ($bridgeClassNames as $bridgeClassName) {
if ($this->bridgeFactory->isEnabled($bridgeClassName)) {
$body .= BridgeCard::render($this->bridgeFactory, $bridgeClassName, $token);
$activeBridges++;
}
}
$response = new Response(render(__DIR__ . '/../templates/frontpage.html.php', [
'messages' => $messages,
'admin_email' => Configuration::getConfig('admin', 'email'),
'admin_telegram' => Configuration::getConfig('admin', 'telegram'),
'bridges' => $body,
'active_bridges' => $activeBridges,
'total_bridges' => count($bridgeClassNames),
]));
// TODO: The rendered template could be cached, but beware config changes that changes the html
return $response;
}
}

15
actions/HealthAction.php Normal file
View File

@ -0,0 +1,15 @@
<?php
declare(strict_types=1);
class HealthAction implements ActionInterface
{
public function __invoke(Request $request): Response
{
$response = [
'code' => 200,
'message' => 'all is good',
];
return new Response(Json::encode($response), 200, ['content-type' => 'application/json']);
}
}

36
actions/ListAction.php Normal file
View File

@ -0,0 +1,36 @@
<?php
class ListAction implements ActionInterface
{
private BridgeFactory $bridgeFactory;
public function __construct(
BridgeFactory $bridgeFactory
) {
$this->bridgeFactory = $bridgeFactory;
}
public function __invoke(Request $request): Response
{
$list = new \stdClass();
$list->bridges = [];
$list->total = 0;
foreach ($this->bridgeFactory->getBridgeClassNames() as $bridgeClassName) {
$bridge = $this->bridgeFactory->create($bridgeClassName);
$list->bridges[$bridgeClassName] = [
'status' => $this->bridgeFactory->isEnabled($bridgeClassName) ? 'active' : 'inactive',
'uri' => $bridge->getURI(),
'donationUri' => $bridge->getDonationURI(),
'name' => $bridge->getName(),
'icon' => $bridge->getIcon(),
'parameters' => $bridge->getParameters(),
'maintainer' => $bridge->getMaintainer(),
'description' => $bridge->getDescription()
];
}
$list->total = count($list->bridges);
return new Response(Json::encode($list), 200, ['content-type' => 'application/json']);
}
}

8
app.json Normal file
View File

@ -0,0 +1,8 @@
{
"service": "Heroku",
"name": "rss-bridge-heroku",
"description": "RSS-Bridge is a PHP project capable of generating RSS and Atom feeds for websites which don't have one.",
"repository": "https://github.com/RSS-Bridge/rss-bridge?1651005770",
"keywords": ["php", "rss-bridge", "rss"]
}

16
bin/cache-clear Executable file
View File

@ -0,0 +1,16 @@
#!/usr/bin/env php
<?php
/**
* Remove all items from the cache
*/
require __DIR__ . '/../lib/bootstrap.php';
require __DIR__ . '/../lib/config.php';
$container = require __DIR__ . '/../lib/dependencies.php';
/** @var CacheInterface $cache */
$cache = $container['cache'];
$cache->clear();

24
bin/cache-prune Executable file
View File

@ -0,0 +1,24 @@
#!/usr/bin/env php
<?php
/**
* Remove all expired items from the cache
*/
require __DIR__ . '/../lib/bootstrap.php';
require __DIR__ . '/../lib/config.php';
$container = require __DIR__ . '/../lib/dependencies.php';
if (
Configuration::getConfig('cache', 'type') === 'file'
&& !Configuration::getConfig('FileCache', 'enable_purge')
) {
// Override enable_purge for this particular execution
Configuration::setConfig('FileCache', 'enable_purge', true);
}
/** @var CacheInterface $cache */
$cache = $container['cache'];
$cache->prune();

20
bin/test Executable file
View File

@ -0,0 +1,20 @@
#!/usr/bin/env php
<?php
/**
* Add log records to all three levels (for testing purposes)
*/
require __DIR__ . '/../lib/bootstrap.php';
require __DIR__ . '/../lib/config.php';
$container = require __DIR__ . '/../lib/dependencies.php';
/** @var Logger $logger */
$logger = $container['logger'];
$logger->debug('This is a test debug message');
$logger->info('This is a test info message');
$logger->error('This is a test error message');

49
bridges/ABCNewsBridge.php Normal file
View File

@ -0,0 +1,49 @@
<?php
class ABCNewsBridge extends BridgeAbstract
{
const NAME = 'ABC News Bridge';
const URI = 'https://www.abc.net.au';
const DESCRIPTION = 'Topics of the Australian Broadcasting Corporation';
const MAINTAINER = 'yue-dongchen';
const PARAMETERS = [
[
'topic' => [
'type' => 'list',
'name' => 'Region',
'title' => 'Choose state',
'values' => [
'ACT' => 'act',
'NSW' => 'nsw',
'NT' => 'nt',
'QLD' => 'qld',
'SA' => 'sa',
'TAS' => 'tas',
'VIC' => 'vic',
'WA' => 'wa'
],
]
]
];
public function collectData()
{
$url = sprintf('https://www.abc.net.au/news/%s', $this->getInput('topic'));
$dom = getSimpleHTMLDOM($url);
$dom = $dom->find('div[data-component="PaginationList"]', 0);
if (!$dom) {
throw new \Exception(sprintf('Unable to find css selector on `%s`', $url));
}
$dom = defaultLinkTo($dom, $this->getURI());
foreach ($dom->find('article[data-component="DetailCard"]') as $article) {
$a = $article->find('a', 0);
$this->items[] = [
'title' => $a->plaintext,
'uri' => $a->href,
'content' => $article->find('p', 0)->plaintext,
'timestamp' => strtotime($article->find('time', 0)->datetime),
];
}
}
}

116
bridges/ABolaBridge.php Normal file
View File

@ -0,0 +1,116 @@
<?php
class ABolaBridge extends BridgeAbstract
{
const NAME = 'A Bola';
const URI = 'https://abola.pt/';
const DESCRIPTION = 'Returns news from the Portuguese sports newspaper A BOLA.PT';
const MAINTAINER = 'rmscoelho';
const CACHE_TIMEOUT = 3600;
const PARAMETERS = [
[
'feed' => [
'name' => 'News Feed',
'type' => 'list',
'title' => 'Feeds from the Portuguese sports newspaper A BOLA.PT',
'values' => [
'Últimas' => 'Nnh/Noticias',
'Seleção Nacional' => 'Selecao/Noticias',
'Futebol Nacional' => [
'Notícias' => 'Nacional/Noticias',
'Primeira Liga' => 'Nacional/Liga/Noticias',
'Liga 2' => 'Nacional/Liga2/Noticias',
'Liga 3' => 'Nacional/Liga3/Noticias',
'Liga Revelação' => 'Nacional/Liga-Revelacao/Noticias',
'Campeonato de Portugal' => 'Nacional/Campeonato-Portugal/Noticias',
'Distritais' => 'Nacional/Distritais/Noticias',
'Taça de Portugal' => 'Nacional/TPortugal/Noticias',
'Futebol Feminino' => 'Nacional/FFeminino/Noticias',
'Futsal' => 'Nacional/Futsal/Noticias',
],
'Futebol Internacional' => [
'Notícias' => 'Internacional/Noticias/Noticias',
'Liga dos Campeões' => 'Internacional/Liga-dos-campeoes/Noticias',
'Liga Europa' => 'Internacional/Liga-europa/Noticias',
'Liga Conferência' => 'Internacional/Liga-conferencia/Noticias',
'Liga das Nações' => 'Internacional/Liga-das-nacoes/Noticias',
'UEFA Youth League' => 'Internacional/Uefa-Youth-League/Noticias',
],
'Mercado' => 'Mercado',
'Modalidades' => 'Modalidades/Noticias',
'Motores' => 'Motores/Noticias',
]
]
]
];
public function getIcon()
{
return 'https://abola.pt/img/icons/favicon-96x96.png';
}
public function getName()
{
return !is_null($this->getKey('feed')) ? self::NAME . ' | ' . $this->getKey('feed') : self::NAME;
}
public function getURI()
{
return self::URI . $this->getInput('feed');
}
public function collectData()
{
$url = sprintf('https://abola.pt/%s', $this->getInput('feed'));
$dom = getSimpleHTMLDOM($url);
if ($this->getInput('feed') !== 'Mercado') {
$dom = $dom->find('div#body_Todas1_upNoticiasTodas', 0);
} else {
$dom = $dom->find('div#body_NoticiasMercado_upNoticiasTodas', 0);
}
if (!$dom) {
throw new \Exception(sprintf('Unable to find css selector on `%s`', $url));
}
$dom = defaultLinkTo($dom, $this->getURI());
foreach ($dom->find('div.media') as $key => $article) {
//Get thumbnail
$image = $article->find('.media-img', 0)->style;
$image = preg_replace('/background-image: url\(/i', '', $image);
$image = substr_replace($image, '', -4);
$image = preg_replace('/https:\/\//i', '', $image);
$image = preg_replace('/www\./i', '', $image);
$image = preg_replace('/\/\//', '/', $image);
$image = preg_replace('/\/\/\//', '//', $image);
$image = substr($image, 7);
$image = 'https://' . $image;
$image = preg_replace('/ptimg/', 'pt/img', $image);
$image = preg_replace('/\/\/bola/', 'www.abola', $image);
//Timestamp
$date = date('Y/m/d');
if (!is_null($article->find("span#body_Todas1_rptNoticiasTodas_lblData_$key", 0))) {
$date = $article->find("span#body_Todas1_rptNoticiasTodas_lblData_$key", 0)->plaintext;
$date = preg_replace('/\./', '/', $date);
}
$time = $article->find("span#body_Todas1_rptNoticiasTodas_lblHora_$key", 0)->plaintext;
$date = explode('/', $date);
$time = explode(':', $time);
$year = $date[0];
$month = $date[1];
$day = $date[2];
$hour = $time[0];
$minute = $time[1];
$timestamp = mktime($hour, $minute, 0, $month, $day, $year);
//Content
$image = '<img src="' . $image . '" alt="' . $article->find('h4 span', 0)->plaintext . '" />';
$description = '<p>' . $article->find('.media-texto > span', 0)->plaintext . '</p>';
$content = $image . '</br>' . $description;
$a = $article->find('.media-body > a', 0);
$this->items[] = [
'title' => $a->find('h4 span', 0)->plaintext,
'uri' => $a->href,
'content' => $content,
'timestamp' => $timestamp,
];
}
}
}

238
bridges/AO3Bridge.php Normal file
View File

@ -0,0 +1,238 @@
<?php
class AO3Bridge extends BridgeAbstract
{
const NAME = 'AO3';
const URI = 'https://archiveofourown.org/';
const CACHE_TIMEOUT = 1800;
const DESCRIPTION = 'Returns works or chapters from Archive of Our Own';
const MAINTAINER = 'Obsidienne';
const PARAMETERS = [
'List' => [
'url' => [
'name' => 'url',
'required' => true,
// Example: F/F tag
'exampleValue' => 'https://archiveofourown.org/tags/F*s*F/works',
],
'range' => [
'name' => 'Chapter Content',
'title' => 'Chapter(s) to include in each work\'s feed entry',
'defaultValue' => null,
'type' => 'list',
'values' => [
'None' => null,
'First' => 'first',
'Latest' => 'last',
'Entire work' => 'all',
],
],
'limit' => self::LIMIT,
],
'Bookmarks' => [
'user' => [
'name' => 'user',
'required' => true,
// Example: Nyaaru's bookmarks
'exampleValue' => 'Nyaaru',
],
],
'Work' => [
'id' => [
'name' => 'id',
'required' => true,
// Example: latest chapters from A Better Past by LysSerris
'exampleValue' => '18181853',
],
]
];
private $title;
public function collectData()
{
switch ($this->queriedContext) {
case 'Bookmarks':
$this->collectList($this->getURI());
break;
case 'List':
$this->collectList($this->getURI());
break;
case 'Work':
$this->collectWork($this->getURI());
break;
}
}
/**
* Feed for lists of works (e.g. recent works, search results, filtered tags,
* bookmarks, series, collections).
*/
private function collectList($url)
{
$version = 'v0.0.1';
$headers = [
"useragent: rss-bridge $version (https://github.com/RSS-Bridge/rss-bridge)"
];
$response = getContents($url, $headers);
$html = \str_get_html($response);
$html = defaultLinkTo($html, self::URI);
// Get list title. Will include page range + count in some cases
$heading = ($html->find('#main h2', 0));
if ($heading->find('a.tag')) {
$heading = $heading->find('a.tag', 0);
}
$this->title = $heading->plaintext;
$limit = $this->getInput('limit') ?? 3;
$count = 0;
foreach ($html->find('.index.group > li') as $element) {
$item = [];
$title = $element->find('div h4 a', 0);
if (!isset($title)) {
continue; // discard deleted works
}
$item['title'] = $title->plaintext;
$item['uri'] = $title->href;
$strdate = $element->find('div p.datetime', 0)->plaintext;
$item['timestamp'] = strtotime($strdate);
// detach from rest of page because remove() is buggy
$element = str_get_html($element->outertext());
$tags = $element->find('ul.required-tags', 0);
foreach ($tags->childNodes() as $tag) {
$item['categories'][] = html_entity_decode($tag->plaintext);
}
$tags->remove();
$tags = $element->find('ul.tags', 0);
foreach ($tags->childNodes() as $tag) {
$item['categories'][] = html_entity_decode($tag->plaintext);
}
$tags->remove();
$item['content'] = implode('', $element->childNodes());
$chapters = $element->find('dl dd.chapters', 0);
// bookmarked series and external works do not have a chapters count
$chapters = (isset($chapters) ? $chapters->plaintext : 0);
$item['uid'] = $item['uri'] . "/$strdate/$chapters";
// Fetch workskin of desired chapter(s) in list
if ($this->getInput('range') && ($limit == 0 || $count++ < $limit)) {
$url = $item['uri'];
switch ($this->getInput('range')) {
case ('all'):
$url .= '?view_full_work=true';
break;
case ('first'):
break;
case ('last'):
// only way to get this is using the navigate page unfortunately
$url .= '/navigate';
$response = getContents($url, $headers);
$html = \str_get_html($response);
$html = defaultLinkTo($html, self::URI);
$url = $html->find('ol.index.group > li > a', -1)->href;
break;
}
$response = getContents($url, $headers);
$html = \str_get_html($response);
$html = defaultLinkTo($html, self::URI);
// remove duplicate fic summary
if ($ficsum = $html->find('#workskin > .preface > .summary', 0)) {
$ficsum->remove();
}
$item['content'] .= $html->find('#workskin', 0);
}
// Use predictability of download links to generate enclosures
$wid = explode('/', $item['uri'])[4];
foreach (['azw3', 'epub', 'mobi', 'pdf', 'html'] as $ext) {
$item['enclosures'][] = 'https://archiveofourown.org/downloads/' . $wid . '/work.' . $ext;
}
$this->items[] = $item;
}
}
/**
* Feed for recent chapters of a specific work.
*/
private function collectWork($url)
{
$version = 'v0.0.1';
$headers = [
"useragent: rss-bridge $version (https://github.com/RSS-Bridge/rss-bridge)"
];
$response = getContents($url . '/navigate', $headers);
$html = \str_get_html($response);
$html = defaultLinkTo($html, self::URI);
$response = getContents($url . '?view_full_work=true', $headers);
$workhtml = \str_get_html($response);
$workhtml = defaultLinkTo($workhtml, self::URI);
$this->title = $html->find('h2 a', 0)->plaintext;
$nav = $html->find('ol.index.group > li');
for ($i = 0; $i < count($nav); $i++) {
$item = [];
$element = $nav[$i];
$item['title'] = $element->find('a', 0)->plaintext;
$item['content'] = $workhtml->find('#chapter-' . ($i + 1), 0);
$item['uri'] = $element->find('a', 0)->href;
$strdate = $element->find('span.datetime', 0)->plaintext;
$strdate = str_replace('(', '', $strdate);
$strdate = str_replace(')', '', $strdate);
$item['timestamp'] = strtotime($strdate);
$item['uid'] = $item['uri'] . "/$strdate";
$this->items[] = $item;
}
$this->items = array_reverse($this->items);
}
public function getName()
{
$name = parent::getName() . " $this->queriedContext";
if (isset($this->title)) {
$name .= " - $this->title";
}
return $name;
}
public function getIcon()
{
return self::URI . '/favicon.ico';
}
public function getURI()
{
$url = parent::getURI();
switch ($this->queriedContext) {
case 'Bookmarks':
$user = $this->getInput('user');
$url = self::URI
. '/users/' . $user
. '/bookmarks?bookmark_search[sort_column]=bookmarkable_date';
break;
case 'List':
$url = $this->getInput('url');
break;
case 'Work':
$url = self::URI . '/works/' . $this->getInput('id');
break;
}
return $url;
}
}

View File

@ -0,0 +1,173 @@
<?php
class ARDAudiothekBridge extends BridgeAbstract
{
const NAME = 'ARD-Audiothek Bridge';
const URI = 'https://www.ardaudiothek.de';
const DESCRIPTION = 'Feed of any show in the ARD-Audiothek, specified by its path';
const MAINTAINER = 'Mar-Koeh';
/*
* The URL Prefix of the API
* @const APIENDPOINT https-URL of the used endpoint, ending in `/`
*/
const APIENDPOINT = 'https://api.ardaudiothek.de/';
/*
* The requested width of the preview image
* 448 and 128 have been observed on the wild
* @const IMAGEWIDTH width in px of the preview image
*/
const IMAGEWIDTH = 448;
/*
* Placeholder that will be replace by IMAGEWIDTH in the preview image URL
* @const IMAGEWIDTHPLACEHOLDER
*/
const IMAGEWIDTHPLACEHOLDER = '{width}';
/*
* File extension appended to image link in $this->icon
* @const IMAGEEXTENSION
*/
const IMAGEEXTENSION = '.jpg';
const PARAMETERS = [
[
'path' => [
'name' => 'Show Link or ID',
'required' => true,
'title' => 'Link to the show page or just its numeric suffix',
'defaultValue' => 'https://www.ardaudiothek.de/sendung/kalk-welk/10777871/'
],
'limit' => self::LIMIT,
]
];
/**
* Holds the title of the current show
*
* @var string
*/
private $title;
/**
* Holds the URI of the show
*
* @var string
*/
private $uri;
/**
* Holds the icon of the feed
*
*/
private $icon;
public function collectData()
{
$path = $this->getInput('path');
$limit = $this->getInput('limit');
$oldTz = date_default_timezone_get();
date_default_timezone_set('Europe/Berlin');
$pathComponents = explode('/', $path);
if (empty($pathComponents)) {
returnClientError('Path may not be empty');
}
if (count($pathComponents) < 2) {
$showID = $pathComponents[0];
} else {
$lastKey = count($pathComponents) - 1;
$showID = $pathComponents[$lastKey];
if (strlen($showID) === 0) {
$showID = $pathComponents[$lastKey - 1];
}
}
$url = self::APIENDPOINT . 'programsets/' . $showID . '/';
$json1 = getContents($url);
$data1 = Json::decode($json1, false);
$processedJSON = $data1->data->programSet;
if (!$processedJSON) {
throw new \Exception('Unable to find show id: ' . $showID);
}
$answerLength = 1;
$offset = 0;
$numberOfElements = 1;
while ($answerLength != 0 && $offset < $numberOfElements && (is_null($limit) || $offset < $limit)) {
$json2 = getContents($url . '?offset=' . $offset);
$data2 = Json::decode($json2, false);
$processedJSON = $data2->data->programSet;
$answerLength = count($processedJSON->items->nodes);
$offset = $offset + $answerLength;
$numberOfElements = $processedJSON->numberOfElements;
foreach ($processedJSON->items->nodes as $audio) {
$item = [];
$item['uri'] = $audio->sharingUrl;
$item['title'] = $audio->title;
$imageSquare = str_replace(self::IMAGEWIDTHPLACEHOLDER, self::IMAGEWIDTH, $audio->image->url1X1);
$image = str_replace(self::IMAGEWIDTHPLACEHOLDER, self::IMAGEWIDTH, $audio->image->url);
$item['enclosures'] = [
$audio->audios[0]->url,
$imageSquare
];
// synopsis in list is shortened, full synopsis is available using one request per item
$item['content'] = '<img src="' . $image . '" /><p>' . $audio->synopsis . '</p>';
$item['timestamp'] = $audio->publicationStartDateAndTime;
$item['uid'] = $audio->id;
$item['author'] = $audio->programSet->publicationService->title;
$category = $audio->programSet->editorialCategories->title ?? null;
if ($category) {
$item['categories'] = [$category];
}
$item['itunes'] = [
'duration' => $audio->duration,
];
$this->items[] = $item;
}
}
$this->title = $processedJSON->title;
$this->uri = $processedJSON->sharingUrl;
$this->icon = str_replace(self::IMAGEWIDTHPLACEHOLDER, self::IMAGEWIDTH, $processedJSON->image->url1X1);
// add image file extension to URL so icon is shown in generated RSS feeds, see
// https://github.com/RSS-Bridge/rss-bridge/blob/4aed05c7b678b5673386d61374bba13637d15487/formats/MrssFormat.php#L76
$this->icon = $this->icon . self::IMAGEEXTENSION;
$this->items = array_slice($this->items, 0, $limit);
date_default_timezone_set($oldTz);
}
/** {@inheritdoc} */
public function getURI()
{
if (!empty($this->uri)) {
return $this->uri;
}
return parent::getURI();
}
/** {@inheritdoc} */
public function getName()
{
if (!empty($this->title)) {
return $this->title;
}
return parent::getName();
}
/** {@inheritdoc} */
public function getIcon()
{
if (!empty($this->icon)) {
return $this->icon;
}
return parent::getIcon();
}
}

View File

@ -0,0 +1,114 @@
<?php
class ARDMediathekBridge extends BridgeAbstract
{
const NAME = 'ARD-Mediathek Bridge';
const URI = 'https://www.ardmediathek.de';
const DESCRIPTION = 'Feed of any series in the ARD-Mediathek, specified by its path';
const MAINTAINER = 'yue-dongchen';
/*
* Number of Items to be requested from ARDmediathek API
* 12 has been observed on the wild
* 29 is the highest successfully tested value
* More Items could be fetched via pagination
* The JSON-field pagination holds more information on that
* @const PAGESIZE number of requested items
*/
const PAGESIZE = 29;
/*
* The URL Prefix of the (Webapp-)API
* @const APIENDPOINT https-URL of the used endpoint
*/
const APIENDPOINT = 'https://api.ardmediathek.de/page-gateway/widgets/ard/asset/';
/*
* The URL prefix of the video link
* URLs from the webapp include a slug containing titles of show, episode, and tv station.
* It seems to work without that.
* @const VIDEOLINKPREFIX https-URL prefix of video links
*/
const VIDEOLINKPREFIX = 'https://www.ardmediathek.de/video/';
/*
* The requested width of the preview image
* 432 has been observed on the wild
* The webapp seems to also compute and add the height value
* It seems to works without that.
* @const IMAGEWIDTH width in px of the preview image
*/
const IMAGEWIDTH = 432;
/*
* Placeholder that will be replace by IMAGEWIDTH in the preview image URL
* @const IMAGEWIDTHPLACEHOLDER
*/
const IMAGEWIDTHPLACEHOLDER = '{width}';
/**
* Title of the current show
* @var string
*/
private $title;
const PARAMETERS = [
[
'path' => [
'name' => 'Show Link or ID',
'required' => true,
'title' => 'Link to the show page or just its alphanumeric suffix',
'defaultValue' => 'https://www.ardmediathek.de/sendung/45-min/Y3JpZDovL25kci5kZS8xMzkx/'
]
]
];
public function collectData()
{
$oldTz = date_default_timezone_get();
date_default_timezone_set('Europe/Berlin');
$pathComponents = explode('/', $this->getInput('path'));
if (empty($pathComponents)) {
returnClientError('Path may not be empty');
}
if (count($pathComponents) < 2) {
$showID = $pathComponents[0];
} else {
$lastKey = count($pathComponents) - 1;
$showID = $pathComponents[$lastKey];
if (strlen($showID) === 0) {
$showID = $pathComponents[$lastKey - 1];
}
}
$url = self::APIENDPOINT . $showID . '?pageSize=' . self::PAGESIZE;
$rawJSON = getContents($url);
$processedJSON = json_decode($rawJSON);
foreach ($processedJSON->teasers as $video) {
$item = [];
// there is also ->links->self->id, ->links->self->urlId, ->links->target->id, ->links->target->urlId
$item['uri'] = self::VIDEOLINKPREFIX . $video->id . '/';
// there is also ->mediumTitle and ->shortTitle
$item['title'] = $video->longTitle;
// in the test, aspect16x9 was the only child of images, not sure whether that is always true
$item['enclosures'] = [
str_replace(self::IMAGEWIDTHPLACEHOLDER, self::IMAGEWIDTH, $video->images->aspect16x9->src)
];
$item['content'] = '<img src="' . $item['enclosures'][0] . '" /><p>';
$item['timestamp'] = $video->broadcastedOn;
$item['uid'] = $video->id;
$item['author'] = $video->publicationService->name;
$this->items[] = $item;
}
$this->title = $processedJSON->title;
date_default_timezone_set($oldTz);
}
/** {@inheritdoc} */
public function getName()
{
if (!empty($this->title)) {
return $this->title;
}
return parent::getName();
}
}

View File

@ -0,0 +1,63 @@
<?php
class ASRockNewsBridge extends BridgeAbstract
{
const NAME = 'ASRock News Bridge';
const URI = 'https://www.asrock.com';
const DESCRIPTION = 'Returns latest news articles';
const MAINTAINER = 'VerifiedJoseph';
const PARAMETERS = [];
const CACHE_TIMEOUT = 3600; // 1 hour
public function collectData()
{
$html = getSimpleHTMLDOM(self::URI . '/news/index.asp');
$html = defaultLinkTo($html, self::URI . '/news/');
foreach ($html->find('div.inner > a') as $index => $a) {
$item = [];
$articlePath = $a->href;
$articlePageHtml = getSimpleHTMLDOMCached($articlePath, self::CACHE_TIMEOUT);
$articlePageHtml = defaultLinkTo($articlePageHtml, self::URI);
$contents = $articlePageHtml->find('div.Contents', 0);
$item['uri'] = $articlePath;
$item['title'] = $contents->find('h3', 0)->innertext;
$contents->find('h3', 0)->outertext = '';
$item['content'] = $contents->innertext;
$item['timestamp'] = $this->extractDate($a->plaintext);
$img = $a->find('img', 0);
if ($img) {
$item['enclosures'][] = $img->src;
}
$this->items[] = $item;
if (count($this->items) >= 10) {
break;
}
}
}
private function extractDate($text)
{
$dateRegex = '/^([0-9]{4}\/[0-9]{1,2}\/[0-9]{1,2})/';
$text = trim($text);
if (preg_match($dateRegex, $text, $matches)) {
return $matches[1];
}
return '';
}
}

37
bridges/AcrimedBridge.php Normal file
View File

@ -0,0 +1,37 @@
<?php
class AcrimedBridge extends FeedExpander
{
const MAINTAINER = 'qwertygc';
const NAME = 'Acrimed Bridge';
const URI = 'https://www.acrimed.org/';
const CACHE_TIMEOUT = 4800; //2hours
const DESCRIPTION = 'Returns the newest articles';
const PARAMETERS = [
[
'limit' => [
'name' => 'limit',
'type' => 'number',
'defaultValue' => -1,
]
]
];
public function collectData()
{
$url = 'https://www.acrimed.org/spip.php?page=backend';
$limit = $this->getInput('limit');
$this->collectExpandableDatas($url, $limit);
}
protected function parseItem(array $item)
{
$articlePage = getSimpleHTMLDOM($item['uri']);
$article = sanitize($articlePage->find('article.article1', 0)->innertext);
$article = defaultLinkTo($article, static::URI);
$item['content'] = $article;
return $item;
}
}

View File

@ -0,0 +1,45 @@
<?php
class ActivisionResearchBridge extends BridgeAbstract
{
const NAME = 'Activision Research Blog';
const URI = 'https://research.activision.com';
const DESCRIPTION = 'Posts from the Activision Research blog';
const MAINTAINER = 'thefranke';
const CACHE_TIMEOUT = 86400; // 24h
public function collectData()
{
$dom = getSimpleHTMLDOM(static::URI);
$dom = $dom->find('div[id="home-blog-feed"]', 0);
if (!$dom) {
throw new \Exception(sprintf('Unable to find css selector on `%s`', $url));
}
$dom = defaultLinkTo($dom, $this->getURI());
foreach ($dom->find('div[class="blog-entry"]') as $article) {
$a = $article->find('a', 0);
$blogimg = extractFromDelimiters($article->find('div[class="blog-img"]', 0)->style, 'url(', ')');
$title = htmlspecialchars_decode($article->find('div[class="title"]', 0)->plaintext);
$author = htmlspecialchars_decode($article->find('div[class="author]', 0)->plaintext);
$date = $article->find('div[class="pubdate"]', 0)->plaintext;
$entry = getSimpleHTMLDOMCached($a->href, static::CACHE_TIMEOUT * 7 * 4);
$entry = defaultLinkTo($entry, $this->getURI());
$content = $entry->find('div[class="blog-body"]', 0);
$tagsremove = ['script', 'iframe', 'input', 'form'];
$content = sanitize($content, $tagsremove);
$content = '<img src="' . static::URI . $blogimg . '" alt="">' . $content;
$this->items[] = [
'title' => $title,
'author' => $author,
'uri' => $a->href,
'content' => $content,
'timestamp' => strtotime($date),
];
}
}
}

View File

@ -0,0 +1,56 @@
<?php
class AirBreizhBridge extends BridgeAbstract
{
const MAINTAINER = 'fanch317';
const NAME = 'Air Breizh';
const URI = 'https://www.airbreizh.asso.fr/';
const DESCRIPTION = 'Returns newests publications on Air Breizh';
const PARAMETERS = [
'Publications' => [
'theme' => [
'name' => 'Thematique',
'type' => 'list',
'values' => [
'Tout' => '',
'Rapport d\'activite' => 'rapport-dactivite',
'Etude' => 'etudes',
'Information' => 'information',
'Autres documents' => 'autres-documents',
'Plan Régional de Surveillance de la qualité de lair' => 'prsqa',
'Transport' => 'transport'
]
]
]
];
public function getIcon()
{
return 'https://www.airbreizh.asso.fr/voy_content/uploads/2017/11/favicon.png';
}
public function collectData()
{
$html = '';
$html = getSimpleHTMLDOM(static::URI . 'publications/?fwp_publications_thematiques=' . $this->getInput('theme'));
foreach ($html->find('article') as $article) {
$item = [];
// Title
$item['title'] = $article->find('h2', 0)->plaintext;
// Author
$item['author'] = 'Air Breizh';
// Image
$imagelink = $article->find('.card__image', 0)->find('img', 0)->getAttribute('src');
// Content preview
$item['content'] = '<img src="' . $imagelink . '" />
<br/>'
. $article->find('.card__text', 0)->plaintext;
// URL
$item['uri'] = $article->find('.publi__buttons', 0)->find('a', 0)->getAttribute('href');
// ID
$item['id'] = $article->find('.publi__buttons', 0)->find('a', 0)->getAttribute('href');
$this->items[] = $item;
}
}
}

View File

@ -0,0 +1,76 @@
<?php
class AlbionOnlineBridge extends BridgeAbstract
{
const NAME = 'Albion Online Changelog';
const MAINTAINER = 'otakuf';
const URI = 'https://albiononline.com';
const DESCRIPTION = 'Returns the changes made to the Albion Online';
const CACHE_TIMEOUT = 3600; // 60min
const PARAMETERS = [ [
'postcount' => [
'name' => 'Limit',
'type' => 'number',
'required' => true,
'title' => 'Maximum number of items to return',
'defaultValue' => 5,
],
'language' => [
'name' => 'Language',
'type' => 'list',
'values' => [
'English' => 'en',
'Deutsch' => 'de',
'Polski' => 'pl',
'Français' => 'fr',
'Русский' => 'ru',
'Português' => 'pt',
'Español' => 'es',
],
'title' => 'Language of changelog posts',
'defaultValue' => 'en',
],
'full' => [
'name' => 'Full changelog',
'type' => 'checkbox',
'required' => false,
'title' => 'Enable to receive the full changelog post for each item'
],
]];
public function collectData()
{
$api = 'https://albiononline.com/';
// Example: https://albiononline.com/en/changelog/1/5
$url = $api . $this->getInput('language') . '/changelog/1/' . $this->getInput('postcount');
$html = getSimpleHTMLDOM($url);
foreach ($html->find('li') as $data) {
$item = [];
$item['uri'] = self::URI . $data->find('a', 0)->getAttribute('href');
$item['title'] = trim(explode('|', $data->find('span', 0)->plaintext)[0]);
// Time below work only with en lang. Need to think about solution. May be separate request like getFullChangelog, but to english list for all language
//print_r( date_parse_from_format( 'M j, Y' , 'Sep 9, 2020') );
//$item['timestamp'] = $this->extractDate($a->plaintext);
$item['author'] = 'albiononline.com';
if ($this->getInput('full')) {
$item['content'] = $this->getFullChangelog($item['uri']);
} else {
//$item['content'] = trim(preg_replace('/\s+/', ' ', $data->find('span', 0)->plaintext));
// Just use title, no info at all or use title and date, see above
$item['content'] = $item['title'];
}
$item['uid'] = hash('sha256', $item['title']);
$this->items[] = $item;
}
}
private function getFullChangelog($url)
{
$html = getSimpleHTMLDOMCached($url);
$html = defaultLinkTo($html, self::URI);
return $html->find('div.small-12.columns', 1)->innertext;
}
}

View File

@ -0,0 +1,87 @@
<?php
class AlfaBankByBridge extends BridgeAbstract
{
const MAINTAINER = 'lassana';
const NAME = 'AlfaBank.by Новости';
const URI = 'https://www.alfabank.by';
const DESCRIPTION = 'Уведомления Alfa-Now — новости от Альфа-Банка';
const CACHE_TIMEOUT = 3600; // 1 hour
const PARAMETERS = [
'News' => [
'business' => [
'name' => 'Альфа Бизнес',
'type' => 'list',
'title' => 'В зависимости от выбора, возращает уведомления для" .
" клиентов физ. лиц либо для клиентов-юридических лиц и ИП',
'values' => [
'Новости' => 'news',
'Новости бизнеса' => 'newsBusiness'
],
'defaultValue' => 'news'
],
'fullContent' => [
'name' => 'Включать содержимое',
'type' => 'checkbox',
'title' => 'Если выбрано, содержимое уведомлений вставляется в поток (работает медленно)'
]
]
];
public function collectData()
{
$business = $this->getInput('business') == 'newsBusiness';
$fullContent = $this->getInput('fullContent') == 'on';
$mainPageUrl = self::URI . '/about/articles/uvedomleniya/';
if ($business) {
$mainPageUrl .= '?business=true';
}
$html = getSimpleHTMLDOM($mainPageUrl);
$limit = 0;
foreach ($html->find('a.notifications__item') as $element) {
if ($limit < 10) {
$item = [];
$item['uid'] = 'urn:sha1:' . hash('sha1', $element->getAttribute('data-notification-id'));
$item['title'] = $element->find('div.item-title', 0)->innertext;
$item['timestamp'] = DateTime::createFromFormat(
'd M Y',
$this->ruMonthsToEn($element->find('div.item-date', 0)->innertext)
)->getTimestamp();
$itemUrl = self::URI . $element->href;
if ($business) {
$itemUrl = str_replace('?business=true', '', $itemUrl);
}
$item['uri'] = $itemUrl;
if ($fullContent) {
$itemHtml = getSimpleHTMLDOM($itemUrl);
if ($itemHtml) {
$item['content'] = $itemHtml->find('div.now-p__content-text', 0)->innertext;
}
}
$this->items[] = $item;
$limit++;
}
}
}
public function getIcon()
{
return static::URI . '/local/images/favicon.ico';
}
private function ruMonthsToEn($date)
{
$ruMonths = [
'Января', 'Февраля', 'Марта', 'Апреля', 'Мая', 'Июня',
'Июля', 'Августа', 'Сентября', 'Октября', 'Ноября', 'Декабря' ];
$enMonths = [
'January', 'February', 'March', 'April', 'May', 'June',
'July', 'August', 'September', 'October', 'November', 'December' ];
return str_replace($ruMonths, $enMonths, $date);
}
}

View File

@ -0,0 +1,85 @@
<?php
class AllSidesBridge extends BridgeAbstract
{
const NAME = 'AllSides';
const URI = 'https://www.allsides.com';
const DESCRIPTION = 'Balanced news and media bias ratings.';
const MAINTAINER = 'Oliver Nutter';
const PARAMETERS = [
'global' => [
'limit' => [
'name' => 'Number of posts to return',
'type' => 'number',
'defaultValue' => 10,
'required' => false,
'title' => 'Zero or negative values return all posts (ignored if not fetching full article)',
],
'fetch' => [
'name' => 'Fetch full article content',
'type' => 'checkbox',
'defaultValue' => 'checked',
],
],
'Headline Roundups' => [],
];
private const ROUNDUPS_URI = self::URI . '/headline-roundups';
public function collectData()
{
switch ($this->queriedContext) {
case 'Headline Roundups':
$index = getSimpleHTMLDOM(self::ROUNDUPS_URI);
defaultLinkTo($index, self::ROUNDUPS_URI);
$entries = $index->find('table.views-table > tbody > tr');
$limit = (int) $this->getInput('limit');
$fetch = (bool) $this->getInput('fetch');
if ($limit > 0 && $fetch) {
$entries = array_slice($entries, 0, $limit);
}
foreach ($entries as $entry) {
$item = [
'title' => $entry->find('.views-field-name', 0)->text(),
'uri' => $entry->find('a', 0)->href,
'timestamp' => $entry->find('.date-display-single', 0)->content,
'author' => 'AllSides Staff',
];
if ($fetch) {
$article = getSimpleHTMLDOMCached($item['uri']);
defaultLinkTo($article, $item['uri']);
$item['content'] = $article->find('.story-id-page-description', 0);
foreach ($article->find('.page-tags a') as $tag) {
$item['categories'][] = $tag->text();
}
}
$this->items[] = $item;
}
break;
}
}
public function getName()
{
if ($this->queriedContext) {
return self::NAME . " - {$this->queriedContext}";
}
return self::NAME;
}
public function getURI()
{
switch ($this->queriedContext) {
case 'Headline Roundups':
return self::ROUNDUPS_URI;
}
return self::URI;
}
}

157
bridges/AllegroBridge.php Normal file
View File

@ -0,0 +1,157 @@
<?php
class AllegroBridge extends BridgeAbstract
{
const NAME = 'Allegro';
const URI = 'https://www.allegro.pl';
const DESCRIPTION = 'Returns the search results from the Allegro.pl shopping and bidding portal';
const MAINTAINER = 'wrobelda';
const PARAMETERS = [[
'url' => [
'name' => 'Search URL',
'title' => 'Copy the URL from your browser\'s address bar after searching for your items and paste it here',
'exampleValue' => 'https://allegro.pl/kategoria/swieze-warzywa-cebula-318660',
'required' => true,
],
'cookie' => [
'name' => 'The complete cookie value',
'title' => 'Paste the value of the cookie value from your browser if you want to prevent Allegro imposing rate limits',
'required' => false,
],
'includeSponsoredOffers' => [
'type' => 'checkbox',
'name' => 'Include Sponsored Offers',
'defaultValue' => 'checked'
],
'includePromotedOffers' => [
'type' => 'checkbox',
'name' => 'Include Promoted Offers',
'defaultValue' => 'checked'
]
]];
public function getName()
{
$url = $this->getInput('url');
if (!$url) {
return parent::getName();
}
$parsedUrl = parse_url($url, PHP_URL_QUERY);
if (!$parsedUrl) {
return parent::getName();
}
parse_str($parsedUrl, $fields);
if (array_key_exists('string', $fields)) {
$f = urldecode($fields['string']);
} else {
$f = false;
}
if ($f) {
return $f;
}
return parent::getName();
}
public function getURI()
{
return $this->getInput('url') ?? parent::getURI();
}
public function collectData()
{
# make sure we order by the most recently listed offers
$url = preg_replace('/([?&])order=[^&]+(&|$)/', '$1', $this->getInput('url'));
$url .= (parse_url($url, PHP_URL_QUERY) ? '&' : '?') . 'order=n';
$opts = [];
// If a cookie is provided
if ($cookie = $this->getInput('cookie')) {
$opts[CURLOPT_COOKIE] = $cookie;
}
$html = getSimpleHTMLDOM($url, [], $opts);
# if no results found
if ($html->find('.mzmg_6m.m9qz_yo._6a66d_-fJr5')) {
return;
}
$results = $html->find('article[data-analytics-view-custom-context="REGULAR"]');
if ($this->getInput('includeSponsoredOffers')) {
$results = array_merge($results, $html->find('article[data-analytics-view-custom-context="SPONSORED"]'));
}
if ($this->getInput('includePromotedOffers')) {
$results = array_merge($results, $html->find('article[data-analytics-view-custom-context="PROMOTED"]'));
}
foreach ($results as $post) {
$item = [];
$item['uid'] = $post->{'data-analytics-view-value'};
$item_link = $post->find('a[href*="' . $item['uid'] . '"], a[href*="allegrolokalnie"]', 0);
$item['uri'] = $item_link->href;
$item['title'] = $item_link->find('img', 0)->alt;
$image = $item_link->find('img', 0)->{'data-src'} ?: $item_link->find('img', 0)->src ?? false;
if ($image) {
$item['enclosures'] = [$image . '#.image'];
}
$price = $post->{'data-analytics-view-json-custom-price'};
if ($price) {
$priceDecoded = json_decode(html_entity_decode($price));
$price = $priceDecoded->amount . ' ' . $priceDecoded->currency;
}
$descriptionPatterns = ['/<\s*dt[^>]*>\b/', '/<\/dt>/', '/<\s*dd[^>]*>\b/', '/<\/dd>/'];
$descriptionReplacements = ['<span>', ':</span> ', '<strong>', '&emsp;</strong> '];
$description = $post->find('.m7er_k4.mpof_5r.mpof_z0_s', 0)->innertext;
$descriptionPretty = preg_replace($descriptionPatterns, $descriptionReplacements, $description);
$pricingExtraInfo = array_filter($post->find('.mqu1_g3.mgn2_12'), function ($node) {
return empty($node->find('.mvrt_0'));
});
$pricingExtraInfo = $pricingExtraInfo[0]->plaintext ?? '';
$offerExtraInfo = array_map(function ($node) {
return str_contains($node->plaintext, 'zapłać później') ? '' : $node->outertext;
}, $post->find('div.mpof_ki.mwdn_1.mj7a_4.mgn2_12'));
$isSmart = $post->find('img[alt="Smart!"]', 0) ?? false;
if ($isSmart) {
$pricingExtraInfo .= $isSmart->outertext;
}
$item['categories'] = [];
$parameters = $post->find('dd');
foreach ($parameters as $parameter) {
if (in_array(strtolower($parameter->innertext), ['brak', 'nie'])) {
continue;
}
$item['categories'][] = $parameter->innertext;
}
$item['content'] = $descriptionPretty
. '<div><strong>'
. $price
. '</strong></div><div>'
. implode('</div><div>', $offerExtraInfo)
. '</div><dl>'
. $pricingExtraInfo
. '</dl><hr>';
$this->items[] = $item;
}
}
}

View File

@ -0,0 +1,107 @@
<?php
class AllocineFRBridge extends BridgeAbstract
{
const MAINTAINER = 'superbaillot.net';
const NAME = 'Allo Cine Bridge';
const CACHE_TIMEOUT = 25200; // 7h
const URI = 'https://www.allocine.fr';
const DESCRIPTION = 'Bridge for allocine.fr';
const PARAMETERS = [ [
'category' => [
'name' => 'Emission',
'type' => 'list',
'title' => 'Sélectionner l\'emission',
'values' => [
'Faux Raccord' => 'faux-raccord',
'Fanzone' => 'fanzone',
'Game In Ciné' => 'game-in-cine',
'Pour la faire courte' => 'pour-la-faire-courte',
'Home Cinéma' => 'home-cinema',
'PILS - Par Ici Les Sorties' => 'pils-par-ici-les-sorties',
'AlloCiné : l\'émission, sur LeStream' => 'allocine-lemission-sur-lestream',
'Give Me Five' => 'give-me-five',
'Aviez-vous remarqué ?' => 'aviez-vous-remarque',
'Et paf, il est mort' => 'et-paf-il-est-mort',
'The Big Fan Theory' => 'the-big-fan-theory',
'Clichés' => 'cliches',
'Complètement...' => 'completement',
'#Fun Facts' => 'fun-facts',
'Origin Story' => 'origin-story',
]
]
]];
public function getURI()
{
if (!is_null($this->getInput('category'))) {
$categories = [
'faux-raccord' => '/video/programme-12284/',
'fanzone' => '/video/programme-12298/',
'game-in-cine' => '/video/programme-12288/',
'pour-la-faire-courte' => '/video/programme-20960/',
'home-cinema' => '/video/programme-12287/',
'pils-par-ici-les-sorties' => '/video/programme-25789/',
'allocine-lemission-sur-lestream' => '/video/programme-25123/',
'give-me-five' => '/video/programme-21919/saison-34518/',
'aviez-vous-remarque' => '/video/programme-19518/',
'et-paf-il-est-mort' => '/video/programme-25113/',
'the-big-fan-theory' => '/video/programme-20403/',
'cliches' => '/video/programme-24834/',
'completement' => '/video/programme-23859/',
'fun-facts' => '/video/programme-23040/',
'origin-story' => '/video/programme-25667/'
];
$category = $this->getInput('category');
if (array_key_exists($category, $categories)) {
return static::URI . $this->getLastSeasonURI($categories[$category]);
} else {
returnClientError('Emission inconnue');
}
}
return parent::getURI();
}
private function getLastSeasonURI($category)
{
$html = getSimpleHTMLDOMCached(static::URI . $category, 86400);
$seasonLink = $html->find('section[class=section-wrap section]', 0)->find('div[class=cf]', 0)->find('a', 0);
$URI = $seasonLink->href;
return $URI;
}
public function getName()
{
if (!is_null($this->getInput('category'))) {
return self::NAME . ' : ' . $this->getKey('category');
}
return parent::getName();
}
public function collectData()
{
$html = getSimpleHTMLDOM($this->getURI());
foreach ($html->find('div[class=gd-col-left]', 0)->find('div[class*=video-card]') as $element) {
$item = [];
$title = $element->find('a[class*=meta-title-link]', 0);
$content = trim(defaultLinkTo($element->outertext, static::URI));
// Replace image 'src' with the one in 'data-src'
$content = preg_replace('@src="data:image/gif;base64,[A-Za-z0-9+\/]*"@', '', $content);
$content = preg_replace('@data-src=@', 'src=', $content);
// Remove date in the content to prevent content update while the video is getting older
$content = preg_replace('@<div class="meta-sub light">.*<span>[^<]*</span>[^<]*</div>@', '', $content);
$item['content'] = $content;
$item['title'] = trim($title->innertext);
$item['uri'] = static::URI . '/' . substr($title->href, 1);
$this->items[] = $item;
}
}
}

View File

@ -0,0 +1,66 @@
<?php
class AllocineFRSortiesBridge extends BridgeAbstract
{
const MAINTAINER = 'Simounet';
const NAME = 'AlloCiné Sorties Bridge';
const CACHE_TIMEOUT = 25200; // 7h
const BASE_URI = 'https://www.allocine.fr';
const URI = self::BASE_URI . '/film/sorties-semaine/';
const DESCRIPTION = 'Bridge for AlloCiné - Sorties cinéma cette semaine';
public function getName()
{
return self::NAME;
}
public function collectData()
{
$html = getSimpleHTMLDOM($this->getURI());
foreach ($html->find('section.section.section-wrap', 0)->find('li.mdl') as $element) {
$item = [];
$thumb = $element->find('figure.thumbnail', 0);
$meta = $element->find('div.meta-body', 0);
$synopsis = $element->find('div.synopsis', 0);
$date = $element->find('span.date', 0);
$title = $element->find('a[class*=meta-title-link]', 0);
$content = trim(defaultLinkTo($thumb->outertext . $meta->outertext . $synopsis->outertext, static::URI));
// Replace image 'src' with the one in 'data-src'
$content = preg_replace('@src="data:image/gif;base64,[A-Za-z0-9=+\/]*"@', '', $content);
$content = preg_replace('@data-src=@', 'src=', $content);
$item['content'] = $content;
$item['title'] = trim($title->innertext);
$item['timestamp'] = $this->frenchPubDateToTimestamp($date->plaintext);
$item['uri'] = static::BASE_URI . '/' . substr($title->href, 1);
$this->items[] = $item;
}
}
private function frenchPubDateToTimestamp($date)
{
return strtotime(
strtr(
strtolower($date),
[
'janvier' => 'jan',
'février' => 'feb',
'mars' => 'march',
'avril' => 'apr',
'mai' => 'may',
'juin' => 'jun',
'juillet' => 'jul',
'août' => 'aug',
'septembre' => 'sep',
'octobre' => 'oct',
'novembre' => 'nov',
'décembre' => 'dec'
]
)
);
}
}

105
bridges/AmazonBridge.php Normal file
View File

@ -0,0 +1,105 @@
<?php
class AmazonBridge extends BridgeAbstract
{
const MAINTAINER = 'Alexis CHEMEL';
const NAME = 'Amazon';
const URI = 'https://www.amazon.com/';
const CACHE_TIMEOUT = 3600; // 1h
const DESCRIPTION = 'Returns products from Amazon search';
const PARAMETERS = [[
'q' => [
'name' => 'Keyword',
'required' => true,
'exampleValue' => 'watch',
],
'sort' => [
'name' => 'Sort by',
'type' => 'list',
'values' => [
'Relevance' => 'relevanceblender',
'Price: Low to High' => 'price-asc-rank',
'Price: High to Low' => 'price-desc-rank',
'Average Customer Review' => 'review-rank',
'Newest Arrivals' => 'date-desc-rank',
],
'defaultValue' => 'relevanceblender',
],
'tld' => [
'name' => 'Country',
'type' => 'list',
'values' => [
'Australia' => 'com.au',
'Brazil' => 'com.br',
'Canada' => 'ca',
'China' => 'cn',
'France' => 'fr',
'Germany' => 'de',
'India' => 'in',
'Italy' => 'it',
'Japan' => 'co.jp',
'Mexico' => 'com.mx',
'Netherlands' => 'nl',
'Poland' => 'pl',
'Spain' => 'es',
'Sweden' => 'se',
'Turkey' => 'com.tr',
'United Kingdom' => 'co.uk',
'United States' => 'com',
],
'defaultValue' => 'com',
],
]];
public function collectData()
{
$baseUrl = sprintf('https://www.amazon.%s', $this->getInput('tld'));
$url = sprintf(
'%s/s/?field-keywords=%s&sort=%s',
$baseUrl,
urlencode($this->getInput('q')),
$this->getInput('sort')
);
$dom = getSimpleHTMLDOM($url);
$elements = $dom->find('div.s-result-item');
foreach ($elements as $element) {
$item = [];
$title = $element->find('h2', 0);
if (!$title) {
continue;
}
$item['title'] = $title->innertext;
$itemUrl = $element->find('a', 0)->href;
$item['uri'] = urljoin($baseUrl, $itemUrl);
$image = $element->find('img', 0);
if ($image) {
$item['content'] = '<img src="' . $image->getAttribute('src') . '" /><br />';
}
$price = $element->find('span.a-price > .a-offscreen', 0);
if ($price) {
$item['content'] .= $price->innertext;
}
$this->items[] = $item;
}
}
public function getName()
{
if (!is_null($this->getInput('tld')) && !is_null($this->getInput('q'))) {
return 'Amazon.' . $this->getInput('tld') . ': ' . $this->getInput('q');
}
return parent::getName();
}
}

View File

@ -0,0 +1,253 @@
<?php
class AmazonPriceTrackerBridge extends BridgeAbstract
{
const MAINTAINER = 'captn3m0, sal0max';
const NAME = 'Amazon Price Tracker';
const URI = 'https://www.amazon.com/';
const CACHE_TIMEOUT = 3600; // 1h
const DESCRIPTION = 'Tracks price for a single product on Amazon';
const PARAMETERS = [
[
'asin' => [
'name' => 'ASIN',
'required' => true,
'exampleValue' => 'B071GB1VMQ',
// https://stackoverflow.com/a/12827734
'pattern' => 'B[\dA-Z]{9}|\d{9}(X|\d)',
],
'tld' => [
'name' => 'Country',
'type' => 'list',
'values' => [
'Australia' => 'com.au',
'Brazil' => 'com.br',
'Canada' => 'ca',
'China' => 'cn',
'France' => 'fr',
'Germany' => 'de',
'India' => 'in',
'Italy' => 'it',
'Japan' => 'co.jp',
'Mexico' => 'com.mx',
'Netherlands' => 'nl',
'Poland' => 'pl',
'Spain' => 'es',
'Sweden' => 'se',
'Turkey' => 'com.tr',
'United Kingdom' => 'co.uk',
'United States' => 'com',
],
'defaultValue' => 'com',
],
]];
const PRICE_SELECTORS = [
'#priceblock_ourprice',
'.priceBlockBuyingPriceString',
'#newBuyBoxPrice',
'#tp_price_block_total_price_ww',
'span.offer-price',
'.a-color-price',
];
const WHITESPACE = " \t\n\r\0\x0B\xC2\xA0";
protected $title;
/**
* Generates domain name given a amazon TLD
*/
private function getDomainName()
{
return 'https://www.amazon.' . $this->getInput('tld');
}
/**
* Generates URI for a Amazon product page
*/
public function getURI()
{
if (!is_null($this->getInput('asin'))) {
return $this->getDomainName() . '/dp/' . $this->getInput('asin');
}
return parent::getURI();
}
/**
* Scrapes the product title from the html page
* returns the default title if scraping fails
*/
private function getTitle($html)
{
$titleTag = $html->find('#productTitle', 0);
if (!$titleTag) {
return $this->getDefaultTitle();
} else {
return trim(html_entity_decode($titleTag->innertext, ENT_QUOTES));
}
}
/**
* Title used by the feed if none could be found
*/
private function getDefaultTitle()
{
return 'Amazon.' . $this->getInput('tld') . ': ' . $this->getInput('asin');
}
/**
* Returns name for the feed
* Uses title (already scraped) if it has one
*/
public function getName()
{
if (isset($this->title)) {
return $this->title;
} else {
return parent::getName();
}
}
private function parseDynamicImage($attribute)
{
$json = json_decode(html_entity_decode($attribute), true);
if ($json and count($json) > 0) {
return array_keys($json)[0];
}
}
/**
* Returns a generated image tag for the product
*/
private function getImage($html)
{
$image = 'https://placekitten.com/200/300';
$imageSrc = $html->find('#main-image-container img', 0);
if ($imageSrc) {
$hiresImage = $imageSrc->getAttribute('data-old-hires');
$dynamicImageAttribute = $imageSrc->getAttribute('data-a-dynamic-image');
$image = $hiresImage ?: $this->parseDynamicImage($dynamicImageAttribute);
}
return <<<EOT
<img width="300" style="max-width:300;max-height:300" src="$image" alt="{$this->title}" />
EOT;
}
/**
* Return \simple_html_dom object
* for the entire html of the product page
*/
private function getHtml()
{
$uri = $this->getURI();
return getSimpleHTMLDOM($uri);
}
private function scrapePriceFromMetrics($html)
{
$asinData = $html->find('#cerberus-data-metrics', 0);
// <div id="cerberus-data-metrics" style="display: none;"
// data-asin="B00WTHJ5SU" data-asin-price="14.99" data-asin-shipping="0"
// data-asin-currency-code="USD" data-substitute-count="-1" ... />
if ($asinData) {
return [
'price' => $asinData->getAttribute('data-asin-price'),
'currency' => $asinData->getAttribute('data-asin-currency-code'),
'shipping' => $asinData->getAttribute('data-asin-shipping')
];
}
return false;
}
private function scrapePriceTwister($html)
{
$str = $html->find('.twister-plus-buying-options-price-data', 0);
$data = json_decode($str->innertext, true);
if (count($data) === 1) {
$data = $data[0];
return [
'displayPrice' => $data['displayPrice'],
'currency' => $data['currency'],
'shipping' => '0',
];
}
return false;
}
private function scrapePriceGeneric($html)
{
$default = [
'price' => null,
'displayPrice' => null,
'currency' => null,
'shipping' => null,
];
$priceDiv = null;
foreach (self::PRICE_SELECTORS as $sel) {
$priceDiv = $html->find($sel, 0);
if ($priceDiv) {
break;
}
}
if (!$priceDiv) {
return $default;
}
$priceString = str_replace(str_split(self::WHITESPACE), '', $priceDiv->plaintext);
preg_match('/(\d+\.\d{0,2})/', $priceString, $matches);
$price = $matches[0] ?? null;
$currency = str_replace($price, '', $priceString);
if ($price != null && $currency != null) {
return [
'price' => $price,
'displayPrice' => null,
'currency' => $currency,
'shipping' => '0'
];
}
return $default;
}
public function collectData()
{
$html = $this->getHtml();
$this->title = $this->getTitle($html);
$image = $this->getImage($html);
$data = $this->scrapePriceGeneric($html);
// render
$content = '';
$price = $data['displayPrice'];
if (!$price) {
$price = sprintf('%s %s', $data['price'], $data['currency']);
}
$content .= sprintf('%s<br>Price: %s', $image, $price);
if ($data['shipping'] !== '0') {
$content .= sprintf('<br>Shipping: %s %s</br>', $data['shipping'], $data['currency']);
}
$item = [
'title' => $this->title,
'uri' => $this->getURI(),
'content' => $content,
// This is to ensure that feed readers notice the price change
'uid' => md5($data['price'])
];
$this->items[] = $item;
}
}

278
bridges/AnfrBridge.php Normal file
View File

@ -0,0 +1,278 @@
<?php
class AnfrBridge extends BridgeAbstract
{
const NAME = 'ANFR';
const URI = 'https://data.anfr.fr/';
const DESCRIPTION = 'Fetches data from the French administration "Agence Nationale des Fréquences".';
const CACHE_TIMEOUT = 604800; // 7d
const MAINTAINER = 'quent1';
const PARAMETERS = [
'Données sur les réseaux mobiles' => [
'departement' => [
'name' => 'Département',
'type' => 'list',
'values' => [
'Tous' => null,
'Ain' => '001',
'Aisne' => '002',
'Allier' => '003',
'Alpes-de-Haute-Provence' => '004',
'Hautes-Alpes' => '005',
'Alpes-Maritimes' => '006',
'Ardèche' => '007',
'Ardennes' => '008',
'Ariège' => '009',
'Aube' => '010',
'Aude' => '011',
'Aveyron' => '012',
'Bouches-du-Rhône' => '013',
'Calvados' => '014',
'Cantal' => '015',
'Charente' => '016',
'Charente-Maritime' => '017',
'Cher' => '018',
'Corrèze' => '019',
'Corse-du-Sud' => '02A',
'Haute-Corse' => '02B',
'Côte-d\'Or' => '021',
'Côtes-d\'Armor' => '022',
'Creuse' => '023',
'Dordogne' => '024',
'Doubs' => '025',
'Drôme' => '026',
'Eure' => '027',
'Eure-et-Loir' => '028',
'Finistère' => '029',
'Gard' => '030',
'Haute-Garonne' => '031',
'Gers' => '032',
'Gironde' => '033',
'Hérault' => '034',
'Ille-et-Vilaine' => '035',
'Indre' => '036',
'Indre-et-Loire' => '037',
'Isère' => '038',
'Jura' => '039',
'Landes' => '040',
'Loir-et-Cher' => '041',
'Loire' => '042',
'Haute-Loire' => '043',
'Loire-Atlantique' => '044',
'Loiret' => '045',
'Lot' => '046',
'Lot-et-Garonne' => '047',
'Lozère' => '048',
'Maine-et-Loire' => '049',
'Manche' => '050',
'Marne' => '051',
'Haute-Marne' => '052',
'Mayenne' => '053',
'Meurthe-et-Moselle' => '054',
'Meuse' => '055',
'Morbihan' => '056',
'Moselle' => '057',
'Nièvre' => '058',
'Nord' => '059',
'Oise' => '060',
'Orne' => '061',
'Pas-de-Calais' => '062',
'Puy-de-Dôme' => '063',
'Pyrénées-Atlantiques' => '064',
'Hautes-Pyrénées' => '065',
'Pyrénées-Orientales' => '066',
'Bas-Rhin' => '067',
'Haut-Rhin' => '068',
'Rhône' => '069',
'Haute-Saône' => '070',
'Saône-et-Loire' => '071',
'Sarthe' => '072',
'Savoie' => '073',
'Haute-Savoie' => '074',
'Paris' => '075',
'Seine-Maritime' => '076',
'Seine-et-Marne' => '077',
'Yvelines' => '078',
'Deux-Sèvres' => '079',
'Somme' => '080',
'Tarn' => '081',
'Tarn-et-Garonne' => '082',
'Var' => '083',
'Vaucluse' => '084',
'Vendée' => '085',
'Vienne' => '086',
'Haute-Vienne' => '087',
'Vosges' => '088',
'Yonne' => '089',
'Territoire de Belfort' => '090',
'Essonne' => '091',
'Hauts-de-Seine' => '092',
'Seine-Saint-Denis' => '093',
'Val-de-Marne' => '094',
'Val-d\'Oise' => '095',
'Guadeloupe' => '971',
'Martinique' => '972',
'Guyane' => '973',
'La Réunion' => '974',
'Saint-Pierre-et-Miquelon' => '975',
'Mayotte' => '976',
'Saint-Barthélemy' => '977',
'Saint-Martin' => '978',
'Terres australes et antarctiques françaises' => '984',
'Wallis-et-Futuna' => '986',
'Polynésie française' => '987',
'Nouvelle-Calédonie' => '988',
'Île de Clipperton' => '989'
]
],
'generation' => [
'name' => 'Génération',
'type' => 'list',
'values' => [
'Tous' => null,
'2G' => '2G',
'3G' => '3G',
'4G' => '4G',
'5G' => '5G',
]
],
'operateur' => [
'name' => 'Opérateur',
'type' => 'list',
'values' => [
'Tous' => null,
'Bouygues Télécom' => 'BOUYGUES TELECOM',
'Dauphin Télécom' => 'DAUPHIN TELECOM',
'Digiciel' => 'DIGICEL',
'Free Caraïbes' => 'FREE CARAIBES',
'Free Mobile' => 'FREE MOBILE',
'GLOBALTEL' => 'GLOBALTEL',
'Office des postes et télécommunications de Nouvelle Calédonie' => 'Gouv Nelle Calédonie (OPT)',
'Maore Mobile' => 'MAORE MOBILE',
'ONATi' => 'ONATI',
'Orange' => 'ORANGE',
'Outremer Telecom' => 'OUTREMER TELECOM',
'Vodafone polynésie' => 'PMT/VODAPHONE',
'SFR' => 'SFR',
'SPM Télécom' => 'SPM TELECOM',
'Service des Postes et Télécommunications de Polynésie Française' => 'Gouv Nelle Calédonie (OPT)',
'SRR' => 'SRR',
'Station étrangère' => 'Station étrangère',
'Telco OI' => 'TELCO IO',
'United Telecommunication Services Caraïbes' => 'UTS Caraibes',
'Ora Mobile' => 'VITI SAS',
'Zeop' => 'ZEOP'
]
],
'statut' => [
'name' => 'Statut',
'type' => 'list',
'values' => [
'Tous' => null,
'En service' => 'En service',
'Projet approuvé' => 'Projet approuvé',
'Techniquement opérationnel' => 'Techniquement opérationnel',
]
]
]
];
public function collectData()
{
$urlParts = [
'id' => 'observatoire_2g_3g_4g',
'resource_id' => '88ef0887-6b0f-4d3f-8545-6d64c8f597da',
'fields' => 'id,adm_lb_nom,sta_nm_dpt,emr_lb_systeme,generation,date_maj,sta_nm_anfr,adr_lb_lieu,adr_lb_add1,adr_lb_add2,adr_lb_add3,adr_nm_cp,statut',
'rows' => 10000
];
if (!empty($this->getInput('departement'))) {
$urlParts['refine.sta_nm_dpt'] = urlencode($this->getInput('departement'));
}
if (!empty($this->getInput('generation'))) {
$urlParts['refine.generation'] = $this->getInput('generation');
}
if (!empty($this->getInput('operateur'))) {
// http_build_query() already does urlencoding so this call is redundant
$urlParts['refine.adm_lb_nom'] = urlencode($this->getInput('operateur'));
}
if (!empty($this->getInput('statut'))) {
$urlParts['refine.statut'] = urlencode($this->getInput('statut'));
}
// API seems to not play well with urlencoded data
$url = urljoin(static::URI, '/d4c/api/records/1.0/download/?' . urldecode(http_build_query($urlParts)));
$json = getContents($url);
$data = Json::decode($json, false);
$records = $data->records;
$frequenciesByStation = [];
foreach ($records as $record) {
if (!isset($frequenciesByStation[$record->fields->sta_nm_anfr])) {
$street = sprintf(
'%s %s %s',
$record->fields->adr_lb_add1 ?? '',
$record->fields->adr_lb_add2 ?? '',
$record->fields->adr_lb_add3 ?? ''
);
$frequenciesByStation[$record->fields->sta_nm_anfr] = [
'id' => $record->fields->sta_nm_anfr,
'operator' => $record->fields->adm_lb_nom,
'frequencies' => [],
'lastUpdate' => 0,
'address' => [
'street' => trim($street),
'postCode' => $record->fields->adr_nm_cp,
'city' => $record->fields->adr_lb_lieu
]
];
}
$frequenciesByStation[$record->fields->sta_nm_anfr]['frequencies'][] = [
'generation' => $record->fields->generation,
'frequency' => $record->fields->emr_lb_systeme,
'status' => $record->fields->statut,
'updatedAt' => strtotime($record->fields->date_maj),
];
$frequenciesByStation[$record->fields->sta_nm_anfr]['lastUpdate'] = max(
$frequenciesByStation[$record->fields->sta_nm_anfr]['lastUpdate'],
strtotime($record->fields->date_maj)
);
}
usort($frequenciesByStation, static fn ($a, $b) => $b['lastUpdate'] <=> $a['lastUpdate']);
foreach ($frequenciesByStation as $station) {
$title = sprintf(
'[%s] Mise à jour de la station n°%s à %s (%s)',
$station['operator'],
$station['id'],
$station['address']['city'],
$station['address']['postCode']
);
$array_reduce = array_reduce($station['frequencies'], static function ($carry, $frequency) {
return sprintf('%s<li>%s : %s</li>', $carry, $frequency['frequency'], $frequency['status']);
}, '');
$content = sprintf(
'<h1>Adresse complète</h1><p>%s<br>%s<br>%s</p><h1>Fréquences</h1><p><ul>%s</ul></p>',
$station['address']['street'],
$station['address']['postCode'],
$station['address']['city'],
$array_reduce
);
$this->items[] = [
'uid' => $station['id'],
'timestamp' => $station['lastUpdate'],
'title' => $title,
'content' => $content,
];
}
}
}

218
bridges/AnidexBridge.php Normal file
View File

@ -0,0 +1,218 @@
<?php
class AnidexBridge extends BridgeAbstract
{
const MAINTAINER = 'ORelio';
const NAME = 'Anidex';
const URI = 'http://anidex.info/'; // anidex.info has ddos-guard so we need to use anidex.moe
const ALTERNATE_URI = 'https://anidex.moe/'; // anidex.moe returns 301 unless Host is set to anidex.info
const ALTERNATE_HOST = 'anidex.info'; // Correct host for requesting anidex.moe without 301 redirect
const DESCRIPTION = 'Returns the newest torrents, with optional search criteria.';
const PARAMETERS = [
[
'id' => [
'name' => 'Category',
'type' => 'list',
'values' => [
'All categories' => '0',
'Anime' => '1,2,3',
'Anime - Sub' => '1',
'Anime - Raw' => '2',
'Anime - Dub' => '3',
'Live Action' => '4,5',
'Live Action - Sub' => '4',
'Live Action - Raw' => '5',
'Light Novel' => '6',
'Manga' => '7,8',
'Manga - Translated' => '7',
'Manga - Raw' => '8',
'Music' => '9,10,11',
'Music - Lossy' => '9',
'Music - Lossless' => '10',
'Music - Video' => '11',
'Games' => '12',
'Applications' => '13',
'Pictures' => '14',
'Adult Video' => '15',
'Other' => '16'
]
],
'lang_id' => [
'name' => 'Language',
'type' => 'list',
'values' => [
'All languages' => '0',
'English' => '1',
'Japanese' => '2',
'Polish' => '3',
'Serbo-Croatian' => '4',
'Dutch' => '5',
'Italian' => '6',
'Russian' => '7',
'German' => '8',
'Hungarian' => '9',
'French' => '10',
'Finnish' => '11',
'Vietnamese' => '12',
'Greek' => '13',
'Bulgarian' => '14',
'Spanish (Spain)' => '15',
'Portuguese (Brazil)' => '16',
'Portuguese (Portugal)' => '17',
'Swedish' => '18',
'Arabic' => '19',
'Danish' => '20',
'Chinese (Simplified)' => '21',
'Bengali' => '22',
'Romanian' => '23',
'Czech' => '24',
'Mongolian' => '25',
'Turkish' => '26',
'Indonesian' => '27',
'Korean' => '28',
'Spanish (LATAM)' => '29',
'Persian' => '30',
'Malaysian' => '31'
]
],
'group_id' => [
'name' => 'Group ID',
'type' => 'number'
],
'r' => [
'name' => 'Hide Remakes',
'type' => 'checkbox'
],
'b' => [
'name' => 'Only Batches',
'type' => 'checkbox'
],
'a' => [
'name' => 'Only Authorized',
'type' => 'checkbox'
],
'q' => [
'name' => 'Keyword',
'description' => 'Keyword(s)',
'type' => 'text'
],
'h' => [
'name' => 'Adult content',
'type' => 'list',
'values' => [
'No filter' => '0',
'Hide +18' => '1',
'Only +18' => '2'
]
]
]
];
public function collectData()
{
// Build Search URL from user-provided parameters
$search_url = self::ALTERNATE_URI . '?s=upload_timestamp&o=desc';
foreach (['id', 'lang_id', 'group_id'] as $param_name) {
$param = $this->getInput($param_name);
if (!empty($param) && intval($param) != 0 && ctype_digit(str_replace(',', '', $param))) {
$search_url .= '&' . $param_name . '=' . $param;
}
}
foreach (['r', 'b', 'a'] as $param_name) {
$param = $this->getInput($param_name);
if (!empty($param) && boolval($param)) {
$search_url .= '&' . $param_name . '=1';
}
}
$query = $this->getInput('q');
if (!empty($query)) {
$search_url .= '&q=' . urlencode($query);
}
$opt = [];
$h = $this->getInput('h');
if (!empty($h) && intval($h) != 0 && ctype_digit($h)) {
$opt[CURLOPT_COOKIE] = 'anidex_h_toggle=' . $h;
}
// We need to use a different Host HTTP header to reach the correct page on ALTERNATE_URI
$headers = ['Host: ' . self::ALTERNATE_HOST];
// The HTTPS certificate presented by anidex.moe is for anidex.info. We need to ignore this.
// As a consequence, the bridge is intentionally marked as insecure by setting self::URI to http://
$opt[CURLOPT_SSL_VERIFYHOST] = 0;
$opt[CURLOPT_SSL_VERIFYPEER] = 0;
// Retrieve torrent listing from search results, which does not contain torrent description
$html = getSimpleHTMLDOM($search_url, $headers, $opt);
$links = $html->find('a');
$results = [];
foreach ($links as $link) {
if (strpos($link->href, '/torrent/') === 0 && !in_array($link->href, $results)) {
$results[] = $link->href;
}
}
if (empty($results) && empty($this->getInput('q'))) {
returnServerError('No results from Anidex: ' . $search_url);
}
//Process each item individually
foreach ($results as $element) {
//Limit total amount of requests
if (count($this->items) >= 20) {
break;
}
$torrent_id = str_replace('/torrent/', '', $element);
//Ignore entries without valid torrent ID
if ($torrent_id != 0 && ctype_digit($torrent_id)) {
//Retrieve data for this torrent ID
$item_browse_uri = self::URI . 'torrent/' . $torrent_id;
$item_fetch_uri = self::ALTERNATE_URI . 'torrent/' . $torrent_id;
//Retrieve full description from torrent page (cached for 24 hours: 86400 seconds)
if ($item_html = getSimpleHTMLDOMCached($item_fetch_uri, 86400, $headers, $opt)) {
//Retrieve data from page contents
$item_title = str_replace(' (Torrent) - AniDex ', '', $item_html->find('title', 0)->plaintext);
$item_desc = $item_html->find('div.panel-body', 0);
$item_author = trim($item_html->find('span.fa-user', 0)->parent()->plaintext);
$item_date = strtotime(trim($item_html->find('span.fa-clock', 0)->parent()->plaintext));
$item_image = $this->getURI() . 'images/user_logos/default.png';
//Check for description-less torrent andn optionally extract image
$desc_title_found = false;
foreach ($item_html->find('h3.panel-title') as $h3) {
if (strpos($h3, 'Description') !== false) {
$desc_title_found = true;
break;
}
}
if ($desc_title_found) {
//Retrieve image for thumbnail or generic logo fallback
foreach ($item_desc->find('img') as $img) {
if (strpos($img->src, 'prez') === false) {
$item_image = $img->src;
break;
}
}
$item_desc = trim($item_desc->innertext);
} else {
$item_desc = '<em>No description.</em>';
}
//Build and add final item
$item = [];
$item['uri'] = $item_browse_uri;
$item['title'] = $item_title;
$item['author'] = $item_author;
$item['timestamp'] = $item_date;
$item['enclosures'] = [$item_image];
$item['content'] = $item_desc;
$this->items[] = $item;
}
}
$element = null;
}
$results = null;
}
}

Some files were not shown because too many files have changed in this diff Show More