Compare commits
No commits in common. "gh-pages" and "master" have entirely different histories.
8
.devcontainer/Dockerfile
Normal file
8
.devcontainer/Dockerfile
Normal file
@ -0,0 +1,8 @@
|
||||
FROM rssbridge/rss-bridge:latest
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install --yes --no-install-recommends \
|
||||
git && \
|
||||
pecl install xdebug && \
|
||||
pear install PHP_CodeSniffer && \
|
||||
docker-php-ext-enable xdebug
|
27
.devcontainer/devcontainer.json
Normal file
27
.devcontainer/devcontainer.json
Normal file
@ -0,0 +1,27 @@
|
||||
{
|
||||
"name": "rss-bridge dev",
|
||||
"build": { "dockerfile": "Dockerfile" },
|
||||
"customizations": {
|
||||
// Configure properties specific to VS Code.
|
||||
"vscode": {
|
||||
// Set *default* container specific settings.json values on container create.
|
||||
"settings": {
|
||||
"php.validate.executablePath": "/usr/local/bin/php",
|
||||
"phpSniffer.executablesFolder": "/usr/local/bin/",
|
||||
"phpcs.executablePath": "/usr/local/bin/phpcs",
|
||||
"phpcs.lintOnType": false
|
||||
},
|
||||
|
||||
// Add the IDs of extensions you want installed when the container is created.
|
||||
"extensions": [
|
||||
"xdebug.php-debug",
|
||||
"bmewburn.vscode-intelephense-client",
|
||||
"philfontaine.autolaunch",
|
||||
"eamodio.gitlens",
|
||||
"shevaua.phpcs"
|
||||
]
|
||||
}
|
||||
},
|
||||
"forwardPorts": [3100, 9000, 9003],
|
||||
"postCreateCommand": "cp .devcontainer/nginx.conf /etc/nginx/conf.d/default.conf && cp .devcontainer/xdebug.ini /usr/local/etc/php/conf.d/xdebug.ini && mkdir .vscode && cp .devcontainer/launch.json .vscode && echo '*' > whitelist.txt && chmod a+x \"$(pwd)\" && rm -rf /var/www/html && ln -s \"$(pwd)\" /var/www/html && nginx && php-fpm -D"
|
||||
}
|
49
.devcontainer/launch.json
Normal file
49
.devcontainer/launch.json
Normal file
@ -0,0 +1,49 @@
|
||||
{
|
||||
// Use IntelliSense to learn about possible attributes.
|
||||
// Hover to view descriptions of existing attributes.
|
||||
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"name": "Listen for Xdebug",
|
||||
"type": "php",
|
||||
"request": "launch",
|
||||
"port": 9003,
|
||||
"auto": true
|
||||
},
|
||||
{
|
||||
"name": "Launch currently open script",
|
||||
"type": "php",
|
||||
"request": "launch",
|
||||
"program": "${file}",
|
||||
"cwd": "${fileDirname}",
|
||||
"port": 0,
|
||||
"runtimeArgs": [
|
||||
"-dxdebug.start_with_request=yes"
|
||||
],
|
||||
"env": {
|
||||
"XDEBUG_MODE": "debug,develop",
|
||||
"XDEBUG_CONFIG": "client_port=${port}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Launch Built-in web server",
|
||||
"type": "php",
|
||||
"request": "launch",
|
||||
"runtimeArgs": [
|
||||
"-dxdebug.mode=debug",
|
||||
"-dxdebug.start_with_request=yes",
|
||||
"-S",
|
||||
"localhost:0"
|
||||
],
|
||||
"program": "",
|
||||
"cwd": "${workspaceRoot}",
|
||||
"port": 9003,
|
||||
"serverReadyAction": {
|
||||
"pattern": "Development Server \\(http://localhost:([0-9]+)\\) started",
|
||||
"uriFormat": "http://localhost:%s",
|
||||
"action": "openExternally"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
17
.devcontainer/nginx.conf
Normal file
17
.devcontainer/nginx.conf
Normal file
@ -0,0 +1,17 @@
|
||||
server {
|
||||
listen 3100 default_server;
|
||||
root /workspaces/rss-bridge;
|
||||
access_log /var/log/nginx/rssbridge.access.log;
|
||||
error_log /var/log/nginx/rssbridge.error.log;
|
||||
index index.php;
|
||||
|
||||
location ~ /(\.|vendor|tests) {
|
||||
deny all;
|
||||
return 403; # Forbidden
|
||||
}
|
||||
|
||||
location ~ \.php$ {
|
||||
include snippets/fastcgi-php.conf;
|
||||
fastcgi_pass unix:/var/run/php/php8.2-fpm.sock;
|
||||
}
|
||||
}
|
7
.devcontainer/xdebug.ini
Normal file
7
.devcontainer/xdebug.ini
Normal file
@ -0,0 +1,7 @@
|
||||
[xdebug]
|
||||
xdebug.mode=develop,debug
|
||||
xdebug.client_host=localhost
|
||||
xdebug.client_port=9003
|
||||
xdebug.start_with_request=yes
|
||||
xdebug.discover_client_host=false
|
||||
xdebug.log='/var/www/html/xdebug.log'
|
16
.dockerignore
Normal file
16
.dockerignore
Normal file
@ -0,0 +1,16 @@
|
||||
.git
|
||||
!.git/HEAD
|
||||
!.git/refs/heads/*
|
||||
.gitattributes
|
||||
.github/*
|
||||
.travis.yml
|
||||
cache/*
|
||||
CONTRIBUTING.md
|
||||
DEBUG
|
||||
Dockerfile
|
||||
phpcompatibility.xml
|
||||
phpcs.xml
|
||||
phpcs.xml
|
||||
scalingo.json
|
||||
tests/*
|
||||
whitelist.txt
|
4
.git-blame-ignore-revs
Normal file
4
.git-blame-ignore-revs
Normal file
@ -0,0 +1,4 @@
|
||||
# Reformat code base to PSR12
|
||||
4f75591060d95208a301bc6bf460d875631b29cc
|
||||
# Fix coding style missed by phpbcf
|
||||
951092eef374db048b77bac85e75e3547bfac702
|
68
.gitattributes
vendored
Normal file
68
.gitattributes
vendored
Normal file
@ -0,0 +1,68 @@
|
||||
# Auto detect text files and perform LF normalization
|
||||
* text=auto
|
||||
*.sh text eol=lf
|
||||
|
||||
# Custom for Visual Studio
|
||||
*.cs diff=csharp
|
||||
*.sln merge=union
|
||||
*.csproj merge=union
|
||||
*.vbproj merge=union
|
||||
*.fsproj merge=union
|
||||
*.dbproj merge=union
|
||||
|
||||
# Standard to msysgit
|
||||
*.doc diff=astextplain
|
||||
*.DOC diff=astextplain
|
||||
*.docx diff=astextplain
|
||||
*.DOCX diff=astextplain
|
||||
*.dot diff=astextplain
|
||||
*.DOT diff=astextplain
|
||||
*.pdf diff=astextplain
|
||||
*.PDF diff=astextplain
|
||||
*.rtf diff=astextplain
|
||||
*.RTF diff=astextplain
|
||||
|
||||
# Ignore files in git archive (i.e. GitHub release builds)
|
||||
|
||||
## Docker
|
||||
Dockerfile export-ignore
|
||||
.dockerignore export-ignore
|
||||
|
||||
## Travis
|
||||
.travis.yml export-ignore
|
||||
|
||||
## GitHub
|
||||
.github/ export-ignore
|
||||
|
||||
## Git
|
||||
.gitattributes export-ignore
|
||||
.gitignore export-ignore
|
||||
|
||||
## Scalingo
|
||||
scalingo.json export-ignore
|
||||
|
||||
## RSS-Bridge
|
||||
phpunit.xml export-ignore
|
||||
phpcs.xml export-ignore
|
||||
phpcompatibility.xml export-ignore
|
||||
tests/ export-ignore
|
||||
cache/.gitkeep export-ignore
|
||||
|
||||
## Composer
|
||||
#
|
||||
# Keep the following lines commented out. Heroku does
|
||||
# not function if the composer files are ignored during
|
||||
# export. For more information see
|
||||
# https://github.com/rss-bridge/rss-bridge/issues/1165
|
||||
#
|
||||
# composer.json export-ignore
|
||||
# composer.lock export-ignore
|
||||
|
||||
## Heroku
|
||||
#
|
||||
# Keep the following line commented out. Heroku does
|
||||
# not function if app.json is ignored during export.
|
||||
# For more information see
|
||||
# https://github.com/rss-bridge/rss-bridge/issues/1165
|
||||
#
|
||||
# app.json export-ignore
|
7
.github/.gitignore
vendored
Normal file
7
.github/.gitignore
vendored
Normal file
@ -0,0 +1,7 @@
|
||||
# Visual Studio Code
|
||||
.vscode/*
|
||||
|
||||
# Generated files
|
||||
comment*.md
|
||||
comment*.txt
|
||||
*.html
|
7
.github/CONTRIBUTING.md
vendored
Normal file
7
.github/CONTRIBUTING.md
vendored
Normal file
@ -0,0 +1,7 @@
|
||||
### Pull request policy
|
||||
|
||||
See the [Pull request policy page on the documentation](https://rss-bridge.github.io/rss-bridge/For_Developers/Pull_Request_policy.html) for more information on the pull request policy.
|
||||
|
||||
### Coding style
|
||||
|
||||
See the [Coding style policy page on the documentation](https://rss-bridge.github.io/rss-bridge/For_Developers/Coding_style_policy.html) for more information on the coding style of the project.
|
64
.github/ISSUE_TEMPLATE/bridge-request.md
vendored
Normal file
64
.github/ISSUE_TEMPLATE/bridge-request.md
vendored
Normal file
@ -0,0 +1,64 @@
|
||||
---
|
||||
name: Bridge request
|
||||
about: Use this template for requesting a new bridge
|
||||
title: Bridge request for ...
|
||||
labels: Bridge-Request
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
# Bridge request
|
||||
|
||||
<!--
|
||||
This is a bridge request. Start by adding a descriptive title (i.e. `Bridge request for GitHub`). Use the "Preview" button to see a preview of your request. Make sure your request is complete before submitting!
|
||||
|
||||
Notice: This comment is only visible to you while you work on your request. Please do not remove any of the lines in the template (you may add your own outside the "<!--" and "- ->" lines!)
|
||||
-->
|
||||
|
||||
## General information
|
||||
|
||||
<!--
|
||||
Please describe what you expect from the bridge. Whenever possible provide sample links and screenshots (you can just paste them here) to express your expectations and help others understand your request. If possible, mark relevant areas in your screenshot. Use the following questions for reference:
|
||||
-->
|
||||
|
||||
- _Host URI for the bridge_ (i.e. `https://github.com`):
|
||||
|
||||
- Which information would you like to see?
|
||||
|
||||
|
||||
|
||||
- How should the information be displayed/formatted?
|
||||
|
||||
|
||||
|
||||
- Which of the following parameters do you expect?
|
||||
|
||||
- [X] Title
|
||||
- [X] URI (link to the original article)
|
||||
- [ ] Author
|
||||
- [ ] Timestamp
|
||||
- [X] Content (the content of the article)
|
||||
- [ ] Enclosures (pictures, videos, etc...)
|
||||
- [ ] Categories (categories, tags, etc...)
|
||||
|
||||
## Options
|
||||
|
||||
<!--Select options from the list below. Add your own option if one is missing:-->
|
||||
|
||||
- [ ] Limit number of returned items
|
||||
- _Default limit_: 5
|
||||
- [ ] Load full articles
|
||||
- _Cache articles_ (articles are stored in a local cache on first request): yes
|
||||
- _Cache timeout_ : 24 hours
|
||||
- [X] Balance requests (RSS-Bridge uses cached versions to reduce bandwith usage)
|
||||
- _Timeout_ (default = 5 minutes): 5 minutes
|
||||
|
||||
<!--Be aware that some options might not be available for your specific request due to technical limitations!-->
|
||||
|
||||
<!--
|
||||
## Additional notes
|
||||
|
||||
Keep in mind that opening a request does not guarantee the bridge being implemented! That depends entirely on the interest and time of others to make the bridge for you.
|
||||
|
||||
You can also implement your own bridge (with support of the community if needed). Find more information in the [RSS-Bridge Documentation](https://rss-bridge.github.io/rss-bridge/For_Developers/index.html) developer section.
|
||||
-->
|
38
.github/ISSUE_TEMPLATE/bug_report.md
vendored
Normal file
38
.github/ISSUE_TEMPLATE/bug_report.md
vendored
Normal file
@ -0,0 +1,38 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
title: ''
|
||||
labels: Bug-Report
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Describe the bug**
|
||||
A clear and concise description of what the bug is.
|
||||
|
||||
**To Reproduce**
|
||||
Steps to reproduce the behavior:
|
||||
1. Go to '...'
|
||||
2. Click on '....'
|
||||
3. Scroll down to '....'
|
||||
4. See error
|
||||
|
||||
**Expected behavior**
|
||||
A clear and concise description of what you expected to happen.
|
||||
|
||||
**Screenshots**
|
||||
If applicable, add screenshots to help explain your problem.
|
||||
|
||||
**Desktop (please complete the following information):**
|
||||
- OS: [e.g. iOS]
|
||||
- Browser [e.g. chrome, safari]
|
||||
- Version [e.g. 22]
|
||||
|
||||
**Smartphone (please complete the following information):**
|
||||
- Device: [e.g. iPhone6]
|
||||
- OS: [e.g. iOS8.1]
|
||||
- Browser [e.g. stock browser, safari]
|
||||
- Version [e.g. 22]
|
||||
|
||||
**Additional context**
|
||||
Add any other context about the problem here.
|
20
.github/ISSUE_TEMPLATE/feature_request.md
vendored
Normal file
20
.github/ISSUE_TEMPLATE/feature_request.md
vendored
Normal file
@ -0,0 +1,20 @@
|
||||
---
|
||||
name: Feature request
|
||||
about: Suggest an idea for this project
|
||||
title: ''
|
||||
labels: Feature-Request
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Is your feature request related to a problem? Please describe.**
|
||||
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
|
||||
|
||||
**Describe the solution you'd like**
|
||||
A clear and concise description of what you want to happen.
|
||||
|
||||
**Describe alternatives you've considered**
|
||||
A clear and concise description of any alternative solutions or features you've considered.
|
||||
|
||||
**Additional context**
|
||||
Add any other context or screenshots about the feature request here.
|
2
.github/prtester-requirements.txt
vendored
Normal file
2
.github/prtester-requirements.txt
vendored
Normal file
@ -0,0 +1,2 @@
|
||||
beautifulsoup4>=4.10.0
|
||||
requests>=2.26.0
|
208
.github/prtester.py
vendored
Normal file
208
.github/prtester.py
vendored
Normal file
@ -0,0 +1,208 @@
|
||||
import argparse
|
||||
import requests
|
||||
import re
|
||||
from bs4 import BeautifulSoup
|
||||
from datetime import datetime
|
||||
from typing import Iterable
|
||||
import os
|
||||
import glob
|
||||
import urllib
|
||||
|
||||
# This script is specifically written to be used in automation for https://github.com/RSS-Bridge/rss-bridge
|
||||
#
|
||||
# This will scrape the whitelisted bridges in the current state (port 3000) and the PR state (port 3001) of
|
||||
# RSS-Bridge, generate a feed for each of the bridges and save the output as html files.
|
||||
# It also add a <base> tag with the url of em's public instance, so viewing
|
||||
# the HTML file locally will actually work as designed.
|
||||
|
||||
ARTIFACT_FILE_EXTENSION = '.html'
|
||||
|
||||
class Instance:
|
||||
name = ''
|
||||
url = ''
|
||||
|
||||
def main(instances: Iterable[Instance], with_upload: bool, with_reduced_upload: bool, title: str, output_file: str):
|
||||
start_date = datetime.now()
|
||||
|
||||
prid = os.getenv('PR')
|
||||
artifact_base_url = f'https://rss-bridge.github.io/rss-bridge-tests/prs/{prid}'
|
||||
artifact_directory = os.getcwd()
|
||||
for file in glob.glob(f'*{ARTIFACT_FILE_EXTENSION}', root_dir=artifact_directory):
|
||||
os.remove(file)
|
||||
|
||||
table_rows = []
|
||||
for instance in instances:
|
||||
page = requests.get(instance.url) # Use python requests to grab the rss-bridge main page
|
||||
soup = BeautifulSoup(page.content, "html.parser") # use bs4 to turn the page into soup
|
||||
bridge_cards = soup.select('.bridge-card') # get a soup-formatted list of all bridges on the rss-bridge page
|
||||
table_rows += testBridges(
|
||||
instance=instance,
|
||||
bridge_cards=bridge_cards,
|
||||
with_upload=with_upload,
|
||||
with_reduced_upload=with_reduced_upload,
|
||||
artifact_directory=artifact_directory,
|
||||
artifact_base_url=artifact_base_url) # run the main scraping code with the list of bridges
|
||||
with open(file=output_file, mode='w+', encoding='utf-8') as file:
|
||||
table_rows_value = '\n'.join(sorted(table_rows))
|
||||
file.write(f'''
|
||||
## {title}
|
||||
| Bridge | Context | Status |
|
||||
| - | - | - |
|
||||
{table_rows_value}
|
||||
|
||||
*last change: {start_date.strftime("%A %Y-%m-%d %H:%M:%S")}*
|
||||
'''.strip())
|
||||
|
||||
def testBridges(instance: Instance, bridge_cards: Iterable, with_upload: bool, with_reduced_upload: bool, artifact_directory: str, artifact_base_url: str) -> Iterable:
|
||||
instance_suffix = ''
|
||||
if instance.name:
|
||||
instance_suffix = f' ({instance.name})'
|
||||
table_rows = []
|
||||
for bridge_card in bridge_cards:
|
||||
bridgeid = bridge_card.get('id')
|
||||
bridgeid = bridgeid.split('-')[1] # this extracts a readable bridge name from the bridge metadata
|
||||
print(f'{bridgeid}{instance_suffix}')
|
||||
bridge_name = bridgeid.replace('Bridge', '')
|
||||
context_forms = bridge_card.find_all("form")
|
||||
form_number = 1
|
||||
for context_form in context_forms:
|
||||
# a bridge can have multiple contexts, named 'forms' in html
|
||||
# this code will produce a fully working url that should create a working feed when called
|
||||
# this will create an example feed for every single context, to test them all
|
||||
context_parameters = {}
|
||||
error_messages = []
|
||||
context_name = '*untitled*'
|
||||
context_name_element = context_form.find_previous_sibling('h5')
|
||||
if context_name_element and context_name_element.text.strip() != '':
|
||||
context_name = context_name_element.text
|
||||
parameters = context_form.find_all("input")
|
||||
lists = context_form.find_all("select")
|
||||
# this for/if mess cycles through all available input parameters, checks if it required, then pulls
|
||||
# the default or examplevalue and then combines it all together into the url parameters
|
||||
# if an example or default value is missing for a required attribute, it will throw an error
|
||||
# any non-required fields are not tested!!!
|
||||
for parameter in parameters:
|
||||
parameter_type = parameter.get('type')
|
||||
parameter_name = parameter.get('name')
|
||||
if parameter_type == 'hidden':
|
||||
context_parameters[parameter_name] = parameter.get('value')
|
||||
if parameter_type == 'number' or parameter_type == 'text':
|
||||
if parameter.has_attr('required'):
|
||||
if parameter.get('placeholder') == '':
|
||||
if parameter.get('value') == '':
|
||||
error_messages.append(f'Missing example or default value for parameter "{parameter_name}"')
|
||||
else:
|
||||
context_parameters[parameter_name] = parameter.get('value')
|
||||
else:
|
||||
context_parameters[parameter_name] = parameter.get('placeholder')
|
||||
# same thing, just for checkboxes. If a checkbox is checked per default, it gets added to the url parameters
|
||||
if parameter_type == 'checkbox':
|
||||
if parameter.has_attr('checked'):
|
||||
context_parameters[parameter_name] = 'on'
|
||||
for listing in lists:
|
||||
selectionvalue = ''
|
||||
listname = listing.get('name')
|
||||
cleanlist = []
|
||||
options = listing.find_all('option')
|
||||
for option in options:
|
||||
if 'optgroup' in option.name:
|
||||
cleanlist.extend(option)
|
||||
else:
|
||||
cleanlist.append(option)
|
||||
firstselectionentry = 1
|
||||
for selectionentry in cleanlist:
|
||||
if firstselectionentry:
|
||||
selectionvalue = selectionentry.get('value')
|
||||
firstselectionentry = 0
|
||||
else:
|
||||
if 'selected' in selectionentry.attrs:
|
||||
selectionvalue = selectionentry.get('value')
|
||||
break
|
||||
context_parameters[listname] = selectionvalue
|
||||
artifact_url = 'about:blank'
|
||||
if error_messages:
|
||||
status = '<br>'.join(map(lambda m: f'❌ `{m}`', error_messages))
|
||||
else:
|
||||
# if all example/default values are present, form the full request url, run the request, add a <base> tag with
|
||||
# the url of em's public instance to the response text (so that relative paths work, e.g. to the static css file) and
|
||||
# then save it to a html file.
|
||||
context_parameters.update({
|
||||
'action': 'display',
|
||||
'bridge': bridgeid,
|
||||
'format': 'Html',
|
||||
})
|
||||
request_url = f'{instance.url}/?{urllib.parse.urlencode(context_parameters)}'
|
||||
response = requests.get(request_url)
|
||||
page_text = response.text.replace('<head>','<head><base href="https://rss-bridge.org/bridge01/" target="_blank">')
|
||||
page_text = page_text.encode("utf_8")
|
||||
soup = BeautifulSoup(page_text, "html.parser")
|
||||
status_messages = []
|
||||
if response.status_code != 200:
|
||||
status_messages += [f'❌ `HTTP status {response.status_code} {response.reason}`']
|
||||
else:
|
||||
feed_items = soup.select('.feeditem')
|
||||
feed_items_length = len(feed_items)
|
||||
if feed_items_length <= 0:
|
||||
status_messages += [f'⚠️ `The feed has no items`']
|
||||
elif feed_items_length == 1 and len(soup.select('.error')) > 0:
|
||||
status_messages += [f'❌ `{getFirstLine(feed_items[0].text)}`']
|
||||
status_messages += map(lambda e: f'❌ `{getFirstLine(e.text)}`', soup.select('.error .error-type') + soup.select('.error .error-message'))
|
||||
for item_element in soup.select('.feeditem'): # remove all feed items to not accidentally selected <pre> tags from item content
|
||||
item_element.decompose()
|
||||
status_messages += map(lambda e: f'⚠️ `{getFirstLine(e.text)}`', soup.find_all('pre'))
|
||||
status_messages = list(dict.fromkeys(status_messages)) # remove duplicates
|
||||
status = '<br>'.join(status_messages)
|
||||
status_is_ok = status == '';
|
||||
if status_is_ok:
|
||||
status = '✔️'
|
||||
if with_upload and (not with_reduced_upload or not status_is_ok):
|
||||
filename = f'{bridge_name} {form_number}{instance_suffix}{ARTIFACT_FILE_EXTENSION}'
|
||||
filename = re.sub(r'[^a-z0-9 \_\-\.]', '', filename, flags=re.I).replace(' ', '_')
|
||||
with open(file=f'{artifact_directory}/{filename}', mode='wb') as file:
|
||||
file.write(page_text)
|
||||
artifact_url = f'{artifact_base_url}/{filename}'
|
||||
table_rows.append(f'| {bridge_name} | [{form_number} {context_name}{instance_suffix}]({artifact_url}) | {status} |')
|
||||
form_number += 1
|
||||
return table_rows
|
||||
|
||||
def getFirstLine(value: str) -> str:
|
||||
# trim whitespace and remove text that can break the table or is simply unnecessary
|
||||
clean_value = re.sub(r'^\[[^\]]+\]\s*rssbridge\.|[\|`]', '', value.strip())
|
||||
first_line = next(iter(clean_value.splitlines()), '')
|
||||
max_length = 250
|
||||
if (len(first_line) > max_length):
|
||||
first_line = first_line[:max_length] + '...'
|
||||
return first_line
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--instances', nargs='+')
|
||||
parser.add_argument('--no-upload', action='store_true')
|
||||
parser.add_argument('--reduced-upload', action='store_true')
|
||||
parser.add_argument('--title', default='Pull request artifacts')
|
||||
parser.add_argument('--output-file', default=os.getcwd() + '/comment.txt')
|
||||
args = parser.parse_args()
|
||||
instances = []
|
||||
if args.instances:
|
||||
for instance_arg in args.instances:
|
||||
instance_arg_parts = instance_arg.split('::')
|
||||
instance = Instance()
|
||||
instance.name = instance_arg_parts[1].strip() if len(instance_arg_parts) >= 2 else ''
|
||||
instance.url = instance_arg_parts[0].strip().rstrip("/")
|
||||
instances.append(instance)
|
||||
else:
|
||||
instance = Instance()
|
||||
instance.name = 'current'
|
||||
instance.url = 'http://localhost:3000'
|
||||
instances.append(instance)
|
||||
instance = Instance()
|
||||
instance.name = 'pr'
|
||||
instance.url = 'http://localhost:3001'
|
||||
instances.append(instance)
|
||||
main(
|
||||
instances=instances,
|
||||
with_upload=not args.no_upload,
|
||||
with_reduced_upload=args.reduced_upload and not args.no_upload,
|
||||
title=args.title,
|
||||
output_file=args.output_file
|
||||
);
|
61
.github/workflows/dockerbuild.yml
vendored
Normal file
61
.github/workflows/dockerbuild.yml
vendored
Normal file
@ -0,0 +1,61 @@
|
||||
name: Build Image on Commit and Release
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- 'master'
|
||||
tags:
|
||||
- '20*'
|
||||
|
||||
env:
|
||||
DOCKERHUB_SLUG: rssbridge/rss-bridge
|
||||
GHCR_SLUG: ghcr.io/rss-bridge/rss-bridge
|
||||
|
||||
jobs:
|
||||
bake:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
${{ env.DOCKERHUB_SLUG }}
|
||||
${{ env.GHCR_SLUG }}
|
||||
tags: |
|
||||
type=raw,value=latest
|
||||
type=sha
|
||||
type=ref,event=tag,enable=${{ startsWith(github.ref, 'refs/tags/20') }}
|
||||
type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/20') }}
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
-
|
||||
name: Login to DockerHub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
-
|
||||
name: Build and push
|
||||
uses: docker/bake-action@v5
|
||||
with:
|
||||
files: |
|
||||
./docker-bake.hcl
|
||||
${{ steps.docker_meta.outputs.bake-file }}
|
||||
targets: image-all
|
||||
push: true
|
27
.github/workflows/documentation.yml
vendored
Normal file
27
.github/workflows/documentation.yml
vendored
Normal file
@ -0,0 +1,27 @@
|
||||
name: Documentation
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- 'docs/**'
|
||||
|
||||
jobs:
|
||||
documentation:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Setup PHP
|
||||
uses: shivammathur/setup-php@v2
|
||||
with:
|
||||
php-version: 8.0
|
||||
- name: Install dependencies
|
||||
run: composer global require daux/daux.io
|
||||
- name: Generate documentation
|
||||
run: daux generate
|
||||
- name: Deploy same repository 🚀
|
||||
uses: JamesIves/github-pages-deploy-action@v4
|
||||
with:
|
||||
folder: "static"
|
||||
branch: gh-pages
|
49
.github/workflows/lint.yml
vendored
Normal file
49
.github/workflows/lint.yml
vendored
Normal file
@ -0,0 +1,49 @@
|
||||
name: Lint
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
jobs:
|
||||
phpcs:
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
php-versions: ['7.4']
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: shivammathur/setup-php@v2
|
||||
with:
|
||||
php-version: ${{ matrix.php-versions }}
|
||||
tools: phpcs
|
||||
- run: phpcs . --standard=phpcs.xml --warning-severity=0 --extensions=php -p
|
||||
|
||||
phpcompatibility:
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
php-versions: ['7.4']
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: shivammathur/setup-php@v2
|
||||
with:
|
||||
php-version: ${{ matrix.php-versions }}
|
||||
- run: composer global config --no-plugins allow-plugins.dealerdirect/phpcodesniffer-composer-installer true
|
||||
- run: composer global require dealerdirect/phpcodesniffer-composer-installer
|
||||
- run: composer global require phpcompatibility/php-compatibility
|
||||
- run: ~/.composer/vendor/bin/phpcs . --standard=phpcompatibility.xml --warning-severity=0 --extensions=php -p
|
||||
|
||||
executable_php_files_check:
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: |
|
||||
if find -name "*.php" -executable -type f -print -exec false {} +
|
||||
then
|
||||
echo 'Good, no executable php scripts found'
|
||||
else
|
||||
echo 'Please unmark php scripts above as non-executable'
|
||||
exit 1
|
||||
fi
|
126
.github/workflows/prhtmlgenerator.yml
vendored
Normal file
126
.github/workflows/prhtmlgenerator.yml
vendored
Normal file
@ -0,0 +1,126 @@
|
||||
name: 'PR Testing'
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
branches: [ master ]
|
||||
|
||||
jobs:
|
||||
check-bridges:
|
||||
name: Check if bridges were changed
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
BRIDGES: ${{ steps.check1.outputs.BRIDGES }}
|
||||
steps:
|
||||
- name: Check number of bridges
|
||||
id: check1
|
||||
run: |
|
||||
PR=${{github.event.number}};
|
||||
wget https://patch-diff.githubusercontent.com/raw/$GITHUB_REPOSITORY/pull/$PR.patch;
|
||||
bridgeamount=$(cat $PR.patch | grep "\bbridges/[A-Za-z0-9]*Bridge\.php\b" | sed "s=.*\bbridges/\([A-Za-z0-9]*\)Bridge\.php\b.*=\1=g" | sort | uniq | wc -l);
|
||||
echo "BRIDGES=$bridgeamount" >> "$GITHUB_OUTPUT"
|
||||
test-pr:
|
||||
name: Generate HTML
|
||||
runs-on: ubuntu-latest
|
||||
needs: check-bridges
|
||||
if: needs.check-bridges.outputs.BRIDGES > 0
|
||||
env:
|
||||
PYTHONUNBUFFERED: 1
|
||||
# Needs additional permissions https://github.com/actions/first-interaction/issues/10#issuecomment-1041402989
|
||||
steps:
|
||||
- name: Check out self
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{github.event.pull_request.head.ref}}
|
||||
repository: ${{github.event.pull_request.head.repo.full_name}}
|
||||
- name: Check out rss-bridge
|
||||
run: |
|
||||
PR=${{github.event.number}};
|
||||
wget -O requirements.txt https://raw.githubusercontent.com/$GITHUB_REPOSITORY/${{ github.event.pull_request.base.ref }}/.github/prtester-requirements.txt;
|
||||
wget https://raw.githubusercontent.com/$GITHUB_REPOSITORY/${{ github.event.pull_request.base.ref }}/.github/prtester.py;
|
||||
wget https://patch-diff.githubusercontent.com/raw/$GITHUB_REPOSITORY/pull/$PR.patch;
|
||||
touch DEBUG;
|
||||
cat $PR.patch | grep "\bbridges/[A-Za-z0-9]*Bridge\.php\b" | sed "s=.*\bbridges/\([A-Za-z0-9]*\)Bridge\.php\b.*=\1=g" | sort | uniq > whitelist.txt
|
||||
- name: Start Docker - Current
|
||||
run: |
|
||||
docker run -d -v $GITHUB_WORKSPACE/whitelist.txt:/app/whitelist.txt -v $GITHUB_WORKSPACE/DEBUG:/app/DEBUG -p 3000:80 ghcr.io/rss-bridge/rss-bridge:latest
|
||||
- name: Start Docker - PR
|
||||
run: |
|
||||
docker build -t prbuild .;
|
||||
docker run -d -v $GITHUB_WORKSPACE/whitelist.txt:/app/whitelist.txt -v $GITHUB_WORKSPACE/DEBUG:/app/DEBUG -p 3001:80 prbuild
|
||||
- name: Setup python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.13'
|
||||
cache: 'pip'
|
||||
- name: Install requirements
|
||||
run: |
|
||||
cd $GITHUB_WORKSPACE
|
||||
pip install -r requirements.txt
|
||||
- name: Run bridge tests
|
||||
id: testrun
|
||||
run: |
|
||||
mkdir results;
|
||||
python prtester.py;
|
||||
body="$(cat comment.txt)";
|
||||
body="${body//'%'/'%25'}";
|
||||
body="${body//$'\n'/'%0A'}";
|
||||
body="${body//$'\r'/'%0D'}";
|
||||
echo "bodylength=${#body}" >> $GITHUB_OUTPUT
|
||||
env:
|
||||
PR: ${{ github.event.number }}
|
||||
- name: Upload generated tests
|
||||
uses: actions/upload-artifact@v4
|
||||
id: upload-generated-tests
|
||||
with:
|
||||
name: tests
|
||||
path: '*.html'
|
||||
- name: Find Comment
|
||||
if: ${{ steps.testrun.outputs.bodylength > 130 }}
|
||||
uses: peter-evans/find-comment@v3
|
||||
id: fc
|
||||
with:
|
||||
issue-number: ${{ github.event.pull_request.number }}
|
||||
comment-author: 'github-actions[bot]'
|
||||
body-includes: Pull request artifacts
|
||||
- name: Create or update comment
|
||||
if: ${{ steps.testrun.outputs.bodylength > 130 }}
|
||||
uses: peter-evans/create-or-update-comment@v4
|
||||
with:
|
||||
comment-id: ${{ steps.fc.outputs.comment-id }}
|
||||
issue-number: ${{ github.event.pull_request.number }}
|
||||
body-file: comment.txt
|
||||
edit-mode: replace
|
||||
upload_tests:
|
||||
name: Upload tests
|
||||
runs-on: ubuntu-latest
|
||||
needs: test-pr
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
repository: 'RSS-Bridge/rss-bridge-tests'
|
||||
ref: 'main'
|
||||
token: ${{ secrets.RSSTESTER_ACTION }}
|
||||
|
||||
- name: Setup git config
|
||||
run: |
|
||||
git config --global user.name "GitHub Actions"
|
||||
git config --global user.email "<>"
|
||||
|
||||
- name: Download tests
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: tests
|
||||
|
||||
- name: Move tests
|
||||
run: |
|
||||
cd prs
|
||||
mkdir -p ${{github.event.number}}
|
||||
cd ${{github.event.number}}
|
||||
mv -f $GITHUB_WORKSPACE/*.html .
|
||||
|
||||
- name: Commit and push generated tests
|
||||
run: |
|
||||
export COMMIT_MESSAGE="Added tests for PR ${{github.event.number}}"
|
||||
git add .
|
||||
git commit -m "$COMMIT_MESSAGE"
|
||||
git push
|
23
.github/workflows/tests.yml
vendored
Normal file
23
.github/workflows/tests.yml
vendored
Normal file
@ -0,0 +1,23 @@
|
||||
name: Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
jobs:
|
||||
phpunit8:
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
php-versions: ['7.4', '8.0', '8.1']
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: shivammathur/setup-php@v2
|
||||
with:
|
||||
php-version: ${{ matrix.php-versions }}
|
||||
env:
|
||||
update: true
|
||||
- run: composer install
|
||||
- run: composer test
|
249
.gitignore
vendored
Normal file
249
.gitignore
vendored
Normal file
@ -0,0 +1,249 @@
|
||||
#################
|
||||
## Eclipse
|
||||
#################
|
||||
vendor/*
|
||||
data/
|
||||
*.pydevproject
|
||||
.project
|
||||
.metadata
|
||||
tmp/
|
||||
*.tmp
|
||||
*.bak
|
||||
*.swp
|
||||
*~.nib
|
||||
local.properties
|
||||
.classpath
|
||||
.settings/
|
||||
.loadpath
|
||||
|
||||
# External tool builders
|
||||
.externalToolBuilders/
|
||||
|
||||
# Locally stored "Eclipse launch configurations"
|
||||
*.launch
|
||||
|
||||
# CDT-specific
|
||||
.cproject
|
||||
|
||||
# PDT-specific
|
||||
.buildpath
|
||||
|
||||
|
||||
#################
|
||||
## Visual Studio
|
||||
#################
|
||||
|
||||
## Ignore Visual Studio temporary files, build results, and
|
||||
## files generated by popular Visual Studio add-ons.
|
||||
|
||||
# User-specific files
|
||||
*.suo
|
||||
*.user
|
||||
*.sln.docstates
|
||||
|
||||
# Build results
|
||||
|
||||
[Dd]ebug/
|
||||
[Rr]elease/
|
||||
x64/
|
||||
build/
|
||||
[Bb]in/
|
||||
[Oo]bj/
|
||||
|
||||
# MSTest test Results
|
||||
[Tt]est[Rr]esult*/
|
||||
[Bb]uild[Ll]og.*
|
||||
|
||||
*_i.c
|
||||
*_p.c
|
||||
*.ilk
|
||||
*.meta
|
||||
*.obj
|
||||
*.pch
|
||||
*.pdb
|
||||
*.pgc
|
||||
*.pgd
|
||||
*.rsp
|
||||
*.sbr
|
||||
*.tlb
|
||||
*.tli
|
||||
*.tlh
|
||||
*.tmp
|
||||
*.tmp_proj
|
||||
*.log
|
||||
*.vspscc
|
||||
*.vssscc
|
||||
.builds
|
||||
*.pidb
|
||||
*.log
|
||||
*.scc
|
||||
|
||||
# Visual C++ cache files
|
||||
ipch/
|
||||
*.aps
|
||||
*.ncb
|
||||
*.opensdf
|
||||
*.sdf
|
||||
*.cachefile
|
||||
|
||||
# Visual Studio profiler
|
||||
*.psess
|
||||
*.vsp
|
||||
*.vspx
|
||||
|
||||
# Guidance Automation Toolkit
|
||||
*.gpState
|
||||
|
||||
# ReSharper is a .NET coding add-in
|
||||
_ReSharper*/
|
||||
*.[Rr]e[Ss]harper
|
||||
|
||||
# TeamCity is a build add-in
|
||||
_TeamCity*
|
||||
|
||||
# DotCover is a Code Coverage Tool
|
||||
*.dotCover
|
||||
|
||||
# NCrunch
|
||||
*.ncrunch*
|
||||
.*crunch*.local.xml
|
||||
|
||||
# Installshield output folder
|
||||
[Ee]xpress/
|
||||
|
||||
# DocProject is a documentation generator add-in
|
||||
DocProject/buildhelp/
|
||||
DocProject/Help/*.HxT
|
||||
DocProject/Help/*.HxC
|
||||
DocProject/Help/*.hhc
|
||||
DocProject/Help/*.hhk
|
||||
DocProject/Help/*.hhp
|
||||
DocProject/Help/Html2
|
||||
DocProject/Help/html
|
||||
|
||||
# Click-Once directory
|
||||
publish/
|
||||
|
||||
# Publish Web Output
|
||||
*.Publish.xml
|
||||
*.pubxml
|
||||
|
||||
# NuGet Packages Directory
|
||||
## TODO: If you have NuGet Package Restore enabled, uncomment the next line
|
||||
#packages/
|
||||
|
||||
# Windows Azure Build Output
|
||||
csx
|
||||
*.build.csdef
|
||||
|
||||
# Windows Store app package directory
|
||||
AppPackages/
|
||||
|
||||
# Others
|
||||
sql/
|
||||
*.Cache
|
||||
ClientBin/
|
||||
[Ss]tyle[Cc]op.*
|
||||
~$*
|
||||
*~
|
||||
*.dbmdl
|
||||
*.[Pp]ublish.xml
|
||||
*.pfx
|
||||
*.publishsettings
|
||||
|
||||
# RIA/Silverlight projects
|
||||
Generated_Code/
|
||||
|
||||
# Backup & report files from converting an old project file to a newer
|
||||
# Visual Studio version. Backup files are not needed, because we have git ;-)
|
||||
_UpgradeReport_Files/
|
||||
Backup*/
|
||||
UpgradeLog*.XML
|
||||
UpgradeLog*.htm
|
||||
|
||||
# SQL Server files
|
||||
App_Data/*.mdf
|
||||
App_Data/*.ldf
|
||||
|
||||
#################
|
||||
## Other ide stuff
|
||||
#################
|
||||
.idea/*
|
||||
[#]*[#]
|
||||
|
||||
#############
|
||||
## Windows detritus
|
||||
#############
|
||||
|
||||
# Windows image file caches
|
||||
Thumbs.db
|
||||
ehthumbs.db
|
||||
|
||||
# Folder config file
|
||||
Desktop.ini
|
||||
|
||||
# Recycle Bin used on file shares
|
||||
$RECYCLE.BIN/
|
||||
|
||||
# Mac crap
|
||||
.DS_Store
|
||||
|
||||
|
||||
#############
|
||||
## Python
|
||||
#############
|
||||
|
||||
*.py[co]
|
||||
|
||||
# Packages
|
||||
*.egg
|
||||
*.egg-info
|
||||
dist/
|
||||
build/
|
||||
eggs/
|
||||
parts/
|
||||
var/
|
||||
sdist/
|
||||
develop-eggs/
|
||||
.installed.cfg
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
.coverage
|
||||
.phpunit.result.cache
|
||||
.tox
|
||||
|
||||
#Translations
|
||||
*.mo
|
||||
|
||||
#Mr Developer
|
||||
.mr.developer.cfg
|
||||
|
||||
##############
|
||||
## RSS-Bridge
|
||||
##############
|
||||
/cache
|
||||
/whitelist.txt
|
||||
DEBUG
|
||||
config.ini.php
|
||||
config/*
|
||||
!config/nginx.conf
|
||||
!config/php-fpm.conf
|
||||
!config/php.ini
|
||||
|
||||
######################
|
||||
## VisualStudioCode ##
|
||||
######################
|
||||
.vscode/*
|
||||
|
||||
#Builder
|
||||
.buildconfig
|
||||
|
||||
#Auth
|
||||
.htaccess
|
||||
.htpasswd
|
||||
|
||||
#Crawler
|
||||
robots.txt
|
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
183
CLI/index.html
183
CLI/index.html
File diff suppressed because one or more lines are too long
225
CONTRIBUTORS.md
Normal file
225
CONTRIBUTORS.md
Normal file
@ -0,0 +1,225 @@
|
||||
# Contributors
|
||||
|
||||
* [16mhz](https://github.com/16mhz)
|
||||
* [adamchainz](https://github.com/adamchainz)
|
||||
* [Ahiles3005](https://github.com/Ahiles3005)
|
||||
* [akirk](https://github.com/akirk)
|
||||
* [Albirew](https://github.com/Albirew)
|
||||
* [aledeg](https://github.com/aledeg)
|
||||
* [alex73](https://github.com/alex73)
|
||||
* [alexAubin](https://github.com/alexAubin)
|
||||
* [Alkarex](https://github.com/Alkarex)
|
||||
* [AmauryCarrade](https://github.com/AmauryCarrade)
|
||||
* [arnd-s](https://github.com/arnd-s)
|
||||
* [ArthurHoaro](https://github.com/ArthurHoaro)
|
||||
* [Astalaseven](https://github.com/Astalaseven)
|
||||
* [Astyan-42](https://github.com/Astyan-42)
|
||||
* [austinhuang0131](https://github.com/austinhuang0131)
|
||||
* [axor-mst](https://github.com/axor-mst)
|
||||
* [ayacoo](https://github.com/ayacoo)
|
||||
* [az5he6ch](https://github.com/az5he6ch)
|
||||
* [b1nj](https://github.com/b1nj)
|
||||
* [benasse](https://github.com/benasse)
|
||||
* [Binnette](https://github.com/Binnette)
|
||||
* [BoboTiG](https://github.com/BoboTiG)
|
||||
* [Bockiii](https://github.com/Bockiii)
|
||||
* [brtsos](https://github.com/brtsos)
|
||||
* [captn3m0](https://github.com/captn3m0)
|
||||
* [chemel](https://github.com/chemel)
|
||||
* [Chouchen](https://github.com/Chouchen)
|
||||
* [ckiw](https://github.com/ckiw)
|
||||
* [cn-tools](https://github.com/cn-tools)
|
||||
* [cnlpete](https://github.com/cnlpete)
|
||||
* [corenting](https://github.com/corenting)
|
||||
* [couraudt](https://github.com/couraudt)
|
||||
* [csisoap](https://github.com/csisoap)
|
||||
* [da2x](https://github.com/da2x)
|
||||
* [dabenzel](https://github.com/dabenzel)
|
||||
* [Daiyousei](https://github.com/Daiyousei)
|
||||
* [dawidsowa](https://github.com/dawidsowa)
|
||||
* [DevonHess](https://github.com/DevonHess)
|
||||
* [dhuschde](https://github.com/dhuschde)
|
||||
* [disk0x](https://github.com/disk0x)
|
||||
* [DJCrashdummy](https://github.com/DJCrashdummy)
|
||||
* [Djuuu](https://github.com/Djuuu)
|
||||
* [DnAp](https://github.com/DnAp)
|
||||
* [dominik-th](https://github.com/dominik-th)
|
||||
* [Draeli](https://github.com/Draeli)
|
||||
* [Dreckiger-Dan](https://github.com/Dreckiger-Dan)
|
||||
* [drego85](https://github.com/drego85)
|
||||
* [drklee3](https://github.com/drklee3)
|
||||
* [DRogueRonin](https://github.com/DRogueRonin)
|
||||
* [dvikan](https://github.com/dvikan)
|
||||
* [eggwhalefrog](https://github.com/eggwhalefrog)
|
||||
* [em92](https://github.com/em92)
|
||||
* [eMerzh](https://github.com/eMerzh)
|
||||
* [EtienneM](https://github.com/EtienneM)
|
||||
* [f0086](https://github.com/f0086)
|
||||
* [fanch317](https://github.com/fanch317)
|
||||
* [fatuuse](https://github.com/fatuuse)
|
||||
* [fivefilters](https://github.com/fivefilters)
|
||||
* [floviolleau](https://github.com/floviolleau)
|
||||
* [fluffy-critter](https://github.com/fluffy-critter)
|
||||
* [fmachen](https://github.com/fmachen)
|
||||
* [Frenzie](https://github.com/Frenzie)
|
||||
* [fulmeek](https://github.com/fulmeek)
|
||||
* [ggiessen](https://github.com/ggiessen)
|
||||
* [gileri](https://github.com/gileri)
|
||||
* [Ginko-Aloe](https://github.com/Ginko-Aloe)
|
||||
* [girlpunk](https://github.com/girlpunk)
|
||||
* [Glandos](https://github.com/Glandos)
|
||||
* [gloony](https://github.com/gloony)
|
||||
* [GregThib](https://github.com/GregThib)
|
||||
* [griffaurel](https://github.com/griffaurel)
|
||||
* [Grummfy](https://github.com/Grummfy)
|
||||
* [gsantner](https://github.com/gsantner)
|
||||
* [guigot](https://github.com/guigot)
|
||||
* [hollowleviathan](https://github.com/hollowleviathan)
|
||||
* [hpacleb](https://github.com/hpacleb)
|
||||
* [hunhejj](https://github.com/hunhejj)
|
||||
* [husim0](https://github.com/husim0)
|
||||
* [IceWreck](https://github.com/IceWreck)
|
||||
* [imagoiq](https://github.com/imagoiq)
|
||||
* [j0k3r](https://github.com/j0k3r)
|
||||
* [JackNUMBER](https://github.com/JackNUMBER)
|
||||
* [jacquesh](https://github.com/jacquesh)
|
||||
* [jakubvalenta](https://github.com/jakubvalenta)
|
||||
* [JasonGhent](https://github.com/JasonGhent)
|
||||
* [jcgoette](https://github.com/jcgoette)
|
||||
* [jdesgats](https://github.com/jdesgats)
|
||||
* [jdigilio](https://github.com/jdigilio)
|
||||
* [JeremyRand](https://github.com/JeremyRand)
|
||||
* [JimDog546](https://github.com/JimDog546)
|
||||
* [jNullj](https://github.com/jNullj)
|
||||
* [Jocker666z](https://github.com/Jocker666z)
|
||||
* [johnnygroovy](https://github.com/johnnygroovy)
|
||||
* [johnpc](https://github.com/johnpc)
|
||||
* [joni1993](https://github.com/joni1993)
|
||||
* [jtojnar](https://github.com/jtojnar)
|
||||
* [KamaleiZestri](https://github.com/KamaleiZestri)
|
||||
* [kkoyung](https://github.com/kkoyung)
|
||||
* [klimplant](https://github.com/klimplant)
|
||||
* [KN4CK3R](https://github.com/KN4CK3R)
|
||||
* [kolarcz](https://github.com/kolarcz)
|
||||
* [kranack](https://github.com/kranack)
|
||||
* [kraoc](https://github.com/kraoc)
|
||||
* [krisu5](https://github.com/krisu5)
|
||||
* [l1n](https://github.com/l1n)
|
||||
* [laBecasse](https://github.com/laBecasse)
|
||||
* [lagaisse](https://github.com/lagaisse)
|
||||
* [lalannev](https://github.com/lalannev)
|
||||
* [langfingaz](https://github.com/langfingaz)
|
||||
* [lassana](https://github.com/lassana)
|
||||
* [ldidry](https://github.com/ldidry)
|
||||
* [Leomaradan](https://github.com/Leomaradan)
|
||||
* [leyrer](https://github.com/leyrer)
|
||||
* [liamka](https://github.com/liamka)
|
||||
* [Limero](https://github.com/Limero)
|
||||
* [LogMANOriginal](https://github.com/LogMANOriginal)
|
||||
* [lorenzos](https://github.com/lorenzos)
|
||||
* [lukasklinger](https://github.com/lukasklinger)
|
||||
* [m0zes](https://github.com/m0zes)
|
||||
* [Mar-Koeh](https://github.com/Mar-Koeh)
|
||||
* [marcus-at-localhost](https://github.com/marcus-at-localhost)
|
||||
* [marius8510000-bot](https://github.com/marius8510000-bot)
|
||||
* [matthewseal](https://github.com/matthewseal)
|
||||
* [mcbyte-it](https://github.com/mcbyte-it)
|
||||
* [mdemoss](https://github.com/mdemoss)
|
||||
* [melangue](https://github.com/melangue)
|
||||
* [metaMMA](https://github.com/metaMMA)
|
||||
* [mibe](https://github.com/mibe)
|
||||
* [mickaelBert](https://github.com/mickaelBert)
|
||||
* [mightymt](https://github.com/mightymt)
|
||||
* [mitsukarenai](https://github.com/mitsukarenai)
|
||||
* [Monocularity](https://github.com/Monocularity)
|
||||
* [MonsieurPoutounours](https://github.com/MonsieurPoutounours)
|
||||
* [mr-flibble](https://github.com/mr-flibble)
|
||||
* [mro](https://github.com/mro)
|
||||
* [mschwld](https://github.com/mschwld)
|
||||
* [muekoeff](https://github.com/muekoeff)
|
||||
* [mw80](https://github.com/mw80)
|
||||
* [mxmehl](https://github.com/mxmehl)
|
||||
* [Mynacol](https://github.com/Mynacol)
|
||||
* [nel50n](https://github.com/nel50n)
|
||||
* [niawag](https://github.com/niawag)
|
||||
* [Niehztog](https://github.com/Niehztog)
|
||||
* [NikNikYkt](https://github.com/NikNikYkt)
|
||||
* [Nono-m0le](https://github.com/Nono-m0le)
|
||||
* [NotsoanoNimus](https://github.com/NotsoanoNimus)
|
||||
* [obsiwitch](https://github.com/obsiwitch)
|
||||
* [Ololbu](https://github.com/Ololbu)
|
||||
* [ORelio](https://github.com/ORelio)
|
||||
* [otakuf](https://github.com/otakuf)
|
||||
* [Park0](https://github.com/Park0)
|
||||
* [Paroleen](https://github.com/Paroleen)
|
||||
* [Patricol](https://github.com/Patricol)
|
||||
* [paulchen](https://github.com/paulchen)
|
||||
* [PaulVayssiere](https://github.com/PaulVayssiere)
|
||||
* [pellaeon](https://github.com/pellaeon)
|
||||
* [PeterDaveHello](https://github.com/PeterDaveHello)
|
||||
* [Peterr-K](https://github.com/Peterr-K)
|
||||
* [Piranhaplant](https://github.com/Piranhaplant)
|
||||
* [pirnz](https://github.com/pirnz)
|
||||
* [pit-fgfjiudghdf](https://github.com/pit-fgfjiudghdf)
|
||||
* [pitchoule](https://github.com/pitchoule)
|
||||
* [pmaziere](https://github.com/pmaziere)
|
||||
* [Pofilo](https://github.com/Pofilo)
|
||||
* [prysme01](https://github.com/prysme01)
|
||||
* [pubak42](https://github.com/pubak42)
|
||||
* [Qluxzz](https://github.com/Qluxzz)
|
||||
* [quentinus95](https://github.com/quentinus95)
|
||||
* [quickwick](https://github.com/quickwick)
|
||||
* [rakoo](https://github.com/rakoo)
|
||||
* [RawkBob](https://github.com/RawkBob)
|
||||
* [regisenguehard](https://github.com/regisenguehard)
|
||||
* [Riduidel](https://github.com/Riduidel)
|
||||
* [rogerdc](https://github.com/rogerdc)
|
||||
* [Roliga](https://github.com/Roliga)
|
||||
* [ronansalmon](https://github.com/ronansalmon)
|
||||
* [rremizov](https://github.com/rremizov)
|
||||
* [s0lesurviv0r](https://github.com/s0lesurviv0r)
|
||||
* [sal0max](https://github.com/sal0max)
|
||||
* [sebsauvage](https://github.com/sebsauvage)
|
||||
* [shutosg](https://github.com/shutosg)
|
||||
* [simon816](https://github.com/simon816)
|
||||
* [Simounet](https://github.com/Simounet)
|
||||
* [somini](https://github.com/somini)
|
||||
* [SpangleLabs](https://github.com/SpangleLabs)
|
||||
* [SqrtMinusOne](https://github.com/SqrtMinusOne)
|
||||
* [squeek502](https://github.com/squeek502)
|
||||
* [StelFux](https://github.com/StelFux)
|
||||
* [stjohnjohnson](https://github.com/stjohnjohnson)
|
||||
* [Stopka](https://github.com/Stopka)
|
||||
* [Strubbl](https://github.com/Strubbl)
|
||||
* [sublimz](https://github.com/sublimz)
|
||||
* [sunchaserinfo](https://github.com/sunchaserinfo)
|
||||
* [SuperSandro2000](https://github.com/SuperSandro2000)
|
||||
* [sysadminstory](https://github.com/sysadminstory)
|
||||
* [t0stiman](https://github.com/t0stiman)
|
||||
* [tameroski](https://github.com/tameroski)
|
||||
* [teromene](https://github.com/teromene)
|
||||
* [tgkenney](https://github.com/tgkenney)
|
||||
* [thefranke](https://github.com/thefranke)
|
||||
* [TheRadialActive](https://github.com/TheRadialActive)
|
||||
* [theScrabi](https://github.com/theScrabi)
|
||||
* [thezeroalpha](https://github.com/thezeroalpha)
|
||||
* [thibaultcouraud](https://github.com/thibaultcouraud)
|
||||
* [timendum](https://github.com/timendum)
|
||||
* [TitiTestScalingo](https://github.com/TitiTestScalingo)
|
||||
* [tomaszkane](https://github.com/tomaszkane)
|
||||
* [tomershvueli](https://github.com/tomershvueli)
|
||||
* [TotalCaesar659](https://github.com/TotalCaesar659)
|
||||
* [tpikonen](https://github.com/tpikonen)
|
||||
* [TReKiE](https://github.com/TReKiE)
|
||||
* [triatic](https://github.com/triatic)
|
||||
* [User123698745](https://github.com/User123698745)
|
||||
* [VerifiedJoseph](https://github.com/VerifiedJoseph)
|
||||
* [vitkabele](https://github.com/vitkabele)
|
||||
* [WalterBarrett](https://github.com/WalterBarrett)
|
||||
* [wtuuju](https://github.com/wtuuju)
|
||||
* [xurxof](https://github.com/xurxof)
|
||||
* [yamanq](https://github.com/yamanq)
|
||||
* [yardenac](https://github.com/yardenac)
|
||||
* [ymeister](https://github.com/ymeister)
|
||||
* [yue-dongchen](https://github.com/yue-dongchen)
|
||||
* [ZeNairolf](https://github.com/ZeNairolf)
|
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
72
Dockerfile
Normal file
72
Dockerfile
Normal file
@ -0,0 +1,72 @@
|
||||
FROM debian:12-slim AS rssbridge
|
||||
|
||||
LABEL description="RSS-Bridge is a PHP project capable of generating RSS and Atom feeds for websites that don't have one."
|
||||
LABEL repository="https://github.com/RSS-Bridge/rss-bridge"
|
||||
LABEL website="https://github.com/RSS-Bridge/rss-bridge"
|
||||
|
||||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
RUN set -xe && \
|
||||
apt-get update && \
|
||||
apt-get install --yes --no-install-recommends \
|
||||
ca-certificates \
|
||||
nginx \
|
||||
nss-plugin-pem \
|
||||
php-curl \
|
||||
php-fpm \
|
||||
php-intl \
|
||||
# php-json is enabled by default with PHP 8.2 in Debian 12
|
||||
php-mbstring \
|
||||
php-memcached \
|
||||
# php-opcache is enabled by default with PHP 8.2 in Debian 12
|
||||
# php-openssl is enabled by default with PHP 8.2 in Debian 12
|
||||
php-sqlite3 \
|
||||
php-xml \
|
||||
php-zip \
|
||||
# php-zlib is enabled by default with PHP 8.2 in Debian 12
|
||||
# for downloading libcurl-impersonate
|
||||
curl \
|
||||
&& \
|
||||
# install curl-impersonate library
|
||||
curlimpersonate_version=0.6.0 && \
|
||||
{ \
|
||||
{ \
|
||||
[ $(arch) = 'aarch64' ] && \
|
||||
archive="libcurl-impersonate-v${curlimpersonate_version}.aarch64-linux-gnu.tar.gz" && \
|
||||
sha512sum="d04b1eabe71f3af06aa1ce99b39a49c5e1d33b636acedcd9fad163bc58156af5c3eb3f75aa706f335515791f7b9c7a6c40ffdfa47430796483ecef929abd905d" \
|
||||
; } \
|
||||
|| { \
|
||||
[ $(arch) = 'armv7l' ] && \
|
||||
archive="libcurl-impersonate-v${curlimpersonate_version}.arm-linux-gnueabihf.tar.gz" && \
|
||||
sha512sum="05906b4efa1a6ed8f3b716fd83d476b6eea6bfc68e3dbc5212d65a2962dcaa7bd1f938c9096a7535252b11d1d08fb93adccc633585ff8cb8cec5e58bfe969bc9" \
|
||||
; } \
|
||||
|| { \
|
||||
[ $(arch) = 'x86_64' ] && \
|
||||
archive="libcurl-impersonate-v${curlimpersonate_version}.x86_64-linux-gnu.tar.gz" && \
|
||||
sha512sum="480bbe9452cd9aff2c0daaaf91f1057b3a96385f79011628a9237223757a9b0d090c59cb5982dc54ea0d07191657299ea91ca170a25ced3d7d410fcdff130ace" \
|
||||
; } \
|
||||
} && \
|
||||
curl -LO "https://github.com/lwthiker/curl-impersonate/releases/download/v${curlimpersonate_version}/${archive}" && \
|
||||
echo "$sha512sum $archive" | sha512sum -c - && \
|
||||
mkdir -p /usr/local/lib/curl-impersonate && \
|
||||
tar xaf "$archive" -C /usr/local/lib/curl-impersonate --wildcards 'libcurl-impersonate-ff.so*' && \
|
||||
rm "$archive" && \
|
||||
apt-get purge --assume-yes curl && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENV LD_PRELOAD /usr/local/lib/curl-impersonate/libcurl-impersonate-ff.so
|
||||
ENV CURL_IMPERSONATE ff91esr
|
||||
|
||||
# logs should go to stdout / stderr
|
||||
RUN ln -sfT /dev/stderr /var/log/nginx/error.log; \
|
||||
ln -sfT /dev/stdout /var/log/nginx/access.log; \
|
||||
chown -R --no-dereference www-data:adm /var/log/nginx/
|
||||
|
||||
COPY ./config/nginx.conf /etc/nginx/sites-available/default
|
||||
COPY ./config/php-fpm.conf /etc/php/8.2/fpm/pool.d/rss-bridge.conf
|
||||
COPY ./config/php.ini /etc/php/8.2/fpm/conf.d/90-rss-bridge.ini
|
||||
|
||||
COPY --chown=www-data:www-data ./ /app/
|
||||
|
||||
EXPOSE 80
|
||||
|
||||
ENTRYPOINT ["/app/docker-entrypoint.sh"]
|
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
170
General/FAQ.html
170
General/FAQ.html
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
536
README.md
Normal file
536
README.md
Normal file
@ -0,0 +1,536 @@
|
||||
# RSS-Bridge
|
||||
|
||||

|
||||
|
||||
RSS-Bridge is a PHP web application.
|
||||
|
||||
It generates web feeds for websites that don't have one.
|
||||
|
||||
Officially hosted instance: https://rss-bridge.org/bridge01/
|
||||
|
||||
IRC channel #rssbridge at https://libera.chat/
|
||||
|
||||
[Full documentation](https://rss-bridge.github.io/rss-bridge/index.html)
|
||||
|
||||
Alternatively find another
|
||||
[public instance](https://rss-bridge.github.io/rss-bridge/General/Public_Hosts.html).
|
||||
|
||||
Requires minimum PHP 7.4.
|
||||
|
||||
|
||||
[](UNLICENSE)
|
||||
[](https://github.com/rss-bridge/rss-bridge/releases/latest)
|
||||
[](https://web.libera.chat/#rssbridge)
|
||||
[](https://github.com/RSS-Bridge/rss-bridge/actions)
|
||||
|
||||
|||
|
||||
|:-:|:-:|
|
||||
|||
|
||||
|||
|
||||
|||
|
||||
|
||||
## A subset of bridges (15/447)
|
||||
|
||||
* `CssSelectorBridge`: [Scrape out a feed using CSS selectors](https://rss-bridge.org/bridge01/#bridge-CssSelectorBridge)
|
||||
* `FeedMergeBridge`: [Combine multiple feeds into one](https://rss-bridge.org/bridge01/#bridge-FeedMergeBridge)
|
||||
* `FeedReducerBridge`: [Reduce a noisy feed by some percentage](https://rss-bridge.org/bridge01/#bridge-FeedReducerBridge)
|
||||
* `FilterBridge`: [Filter a feed by excluding/including items by keyword](https://rss-bridge.org/bridge01/#bridge-FilterBridge)
|
||||
* `GettrBridge`: [Fetches the latest posts from a GETTR user](https://rss-bridge.org/bridge01/#bridge-GettrBridge)
|
||||
* `MastodonBridge`: [Fetches statuses from a Mastodon (ActivityPub) instance](https://rss-bridge.org/bridge01/#bridge-MastodonBridge)
|
||||
* `RedditBridge`: [Fetches posts from a user/subredit (with filtering options)](https://rss-bridge.org/bridge01/#bridge-RedditBridge)
|
||||
* `RumbleBridge`: [Fetches channel/user videos](https://rss-bridge.org/bridge01/#bridge-RumbleBridge)
|
||||
* `SoundcloudBridge`: [Fetches music by username](https://rss-bridge.org/bridge01/#bridge-SoundcloudBridge)
|
||||
* `TelegramBridge`: [Fetches posts from a public channel](https://rss-bridge.org/bridge01/#bridge-TelegramBridge)
|
||||
* `ThePirateBayBridge:` [Fetches torrents by search/user/category](https://rss-bridge.org/bridge01/#bridge-ThePirateBayBridge)
|
||||
* `TikTokBridge`: [Fetches posts by username](https://rss-bridge.org/bridge01/#bridge-TikTokBridge)
|
||||
* `TwitchBridge`: [Fetches videos from channel](https://rss-bridge.org/bridge01/#bridge-TwitchBridge)
|
||||
* `XPathBridge`: [Scrape out a feed using XPath expressions](https://rss-bridge.org/bridge01/#bridge-XPathBridge)
|
||||
* `YoutubeBridge`: [Fetches videos by username/channel/playlist/search](https://rss-bridge.org/bridge01/#bridge-YoutubeBridge)
|
||||
* `YouTubeCommunityTabBridge`: [Fetches posts from a channel's community tab](https://rss-bridge.org/bridge01/#bridge-YouTubeCommunityTabBridge)
|
||||
|
||||
## Tutorial
|
||||
|
||||
### How to install on traditional shared web hosting
|
||||
|
||||
RSS-Bridge can basically be unzipped into a web folder. Should be working instantly.
|
||||
|
||||
Latest zip:
|
||||
https://github.com/RSS-Bridge/rss-bridge/archive/refs/heads/master.zip (2MB)
|
||||
|
||||
### How to install on Debian 12 (nginx + php-fpm)
|
||||
|
||||
These instructions have been tested on a fresh Debian 12 VM from Digital Ocean (1vcpu-512mb-10gb, 5 USD/month).
|
||||
|
||||
```shell
|
||||
timedatectl set-timezone Europe/Oslo
|
||||
|
||||
apt install git nginx php8.2-fpm php-mbstring php-simplexml php-curl php-intl
|
||||
|
||||
# Create a user account
|
||||
useradd --shell /bin/bash --create-home rss-bridge
|
||||
|
||||
cd /var/www
|
||||
|
||||
# Create folder and change its ownership to rss-bridge
|
||||
mkdir rss-bridge && chown rss-bridge:rss-bridge rss-bridge/
|
||||
|
||||
# Become rss-bridge
|
||||
su rss-bridge
|
||||
|
||||
# Clone master branch into existing folder
|
||||
git clone https://github.com/RSS-Bridge/rss-bridge.git rss-bridge/
|
||||
cd rss-bridge
|
||||
|
||||
# Copy over the default config (OPTIONAL)
|
||||
cp -v config.default.ini.php config.ini.php
|
||||
|
||||
# Recursively give full permissions to user/owner
|
||||
chmod 700 --recursive ./
|
||||
|
||||
# Give read and execute to others on folder ./static
|
||||
chmod o+rx ./ ./static
|
||||
|
||||
# Recursively give give read to others on folder ./static
|
||||
chmod o+r --recursive ./static
|
||||
```
|
||||
|
||||
Nginx config:
|
||||
|
||||
```nginx
|
||||
# /etc/nginx/sites-enabled/rss-bridge.conf
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
|
||||
# TODO: change to your own server name
|
||||
server_name example.com;
|
||||
|
||||
access_log /var/log/nginx/rss-bridge.access.log;
|
||||
error_log /var/log/nginx/rss-bridge.error.log;
|
||||
log_not_found off;
|
||||
|
||||
# Intentionally not setting a root folder
|
||||
|
||||
# Static content only served here
|
||||
location /static/ {
|
||||
alias /var/www/rss-bridge/static/;
|
||||
}
|
||||
|
||||
# Pass off to php-fpm only when location is EXACTLY == /
|
||||
location = / {
|
||||
root /var/www/rss-bridge/;
|
||||
include snippets/fastcgi-php.conf;
|
||||
fastcgi_read_timeout 45s;
|
||||
fastcgi_pass unix:/run/php/rss-bridge.sock;
|
||||
}
|
||||
|
||||
# Reduce log noise
|
||||
location = /favicon.ico {
|
||||
access_log off;
|
||||
}
|
||||
|
||||
# Reduce log noise
|
||||
location = /robots.txt {
|
||||
access_log off;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
PHP FPM pool config:
|
||||
```ini
|
||||
; /etc/php/8.2/fpm/pool.d/rss-bridge.conf
|
||||
|
||||
[rss-bridge]
|
||||
|
||||
user = rss-bridge
|
||||
group = rss-bridge
|
||||
|
||||
listen = /run/php/rss-bridge.sock
|
||||
|
||||
listen.owner = www-data
|
||||
listen.group = www-data
|
||||
|
||||
; Create 10 workers standing by to serve requests
|
||||
pm = static
|
||||
pm.max_children = 10
|
||||
|
||||
; Respawn worker after 500 requests (workaround for memory leaks etc.)
|
||||
pm.max_requests = 500
|
||||
```
|
||||
|
||||
PHP ini config:
|
||||
```ini
|
||||
; /etc/php/8.2/fpm/conf.d/30-rss-bridge.ini
|
||||
|
||||
max_execution_time = 15
|
||||
memory_limit = 64M
|
||||
```
|
||||
|
||||
Restart fpm and nginx:
|
||||
|
||||
```shell
|
||||
# Lint and restart php-fpm
|
||||
php-fpm8.2 -t && systemctl restart php8.2-fpm
|
||||
|
||||
# Lint and restart nginx
|
||||
nginx -t && systemctl restart nginx
|
||||
```
|
||||
|
||||
### How to install from Composer
|
||||
|
||||
Install the latest release.
|
||||
|
||||
```shell
|
||||
cd /var/www
|
||||
composer create-project -v --no-dev --no-scripts rss-bridge/rss-bridge
|
||||
```
|
||||
|
||||
### How to install with Caddy
|
||||
|
||||
TODO. See https://github.com/RSS-Bridge/rss-bridge/issues/3785
|
||||
|
||||
### Install from Docker Hub:
|
||||
|
||||
Install by downloading the docker image from Docker Hub:
|
||||
|
||||
```bash
|
||||
# Create container
|
||||
docker create --name=rss-bridge --publish 3000:80 --volume $(pwd)/config:/config rssbridge/rss-bridge
|
||||
```
|
||||
|
||||
You can put custom `config.ini.php` and bridges into `./config`.
|
||||
|
||||
**You must restart container for custom changes to take effect.**
|
||||
|
||||
See `docker-entrypoint.sh` for details.
|
||||
|
||||
```bash
|
||||
# Start container
|
||||
docker start rss-bridge
|
||||
```
|
||||
|
||||
Browse http://localhost:3000/
|
||||
|
||||
### Install by locally building from Dockerfile
|
||||
|
||||
```bash
|
||||
# Build image from Dockerfile
|
||||
docker build -t rss-bridge .
|
||||
|
||||
# Create container
|
||||
docker create --name rss-bridge --publish 3000:80 --volume $(pwd)/config:/config rss-bridge
|
||||
```
|
||||
|
||||
You can put custom `config.ini.php` and bridges into `./config`.
|
||||
|
||||
**You must restart container for custom changes to take effect.**
|
||||
|
||||
See `docker-entrypoint.sh` for details.
|
||||
|
||||
```bash
|
||||
# Start container
|
||||
docker start rss-bridge
|
||||
```
|
||||
|
||||
Browse http://localhost:3000/
|
||||
|
||||
### Install with docker-compose (using Docker Hub)
|
||||
|
||||
You can put custom `config.ini.php` and bridges into `./config`.
|
||||
|
||||
**You must restart container for custom changes to take effect.**
|
||||
|
||||
See `docker-entrypoint.sh` for details.
|
||||
|
||||
```bash
|
||||
docker-compose up
|
||||
```
|
||||
|
||||
Browse http://localhost:3000/
|
||||
|
||||
### Other installation methods
|
||||
|
||||
[](https://my.scalingo.com/deploy?source=https://github.com/sebsauvage/rss-bridge)
|
||||
[](https://heroku.com/deploy)
|
||||
[](https://www.cloudron.io/store/com.rssbridgeapp.cloudronapp.html)
|
||||
[](https://www.pikapods.com/pods?run=rssbridge)
|
||||
|
||||
The Heroku quick deploy currently does not work. It might work if you fork this repo and
|
||||
modify the `repository` in `scalingo.json`. See https://github.com/RSS-Bridge/rss-bridge/issues/2688
|
||||
|
||||
Learn more in
|
||||
[Installation](https://rss-bridge.github.io/rss-bridge/For_Hosts/Installation.html).
|
||||
|
||||
## How-to
|
||||
|
||||
### How to fix "Access denied."
|
||||
|
||||
Output is from php-fpm. It is unable to read index.php.
|
||||
|
||||
chown rss-bridge:rss-bridge /var/www/rss-bridge/index.php
|
||||
|
||||
### How to password-protect the instance (token)
|
||||
|
||||
Modify `config.ini.php`:
|
||||
|
||||
[authentication]
|
||||
|
||||
token = "hunter2"
|
||||
|
||||
### How to remove all cache items
|
||||
|
||||
As current user:
|
||||
|
||||
bin/cache-clear
|
||||
|
||||
As user rss-bridge:
|
||||
|
||||
sudo -u rss-bridge bin/cache-clear
|
||||
|
||||
As root:
|
||||
|
||||
sudo bin/cache-clear
|
||||
|
||||
### How to remove all expired cache items
|
||||
|
||||
bin/cache-prune
|
||||
|
||||
### How to fix "PHP Fatal error: Uncaught Exception: The FileCache path is not writable"
|
||||
|
||||
```shell
|
||||
# Give rss-bridge ownership
|
||||
chown rss-bridge:rss-bridge -R /var/www/rss-bridge/cache
|
||||
|
||||
# Or, give www-data ownership
|
||||
chown www-data:www-data -R /var/www/rss-bridge/cache
|
||||
|
||||
# Or, give everyone write permission
|
||||
chmod 777 -R /var/www/rss-bridge/cache
|
||||
|
||||
# Or last ditch effort (CAREFUL)
|
||||
rm -rf /var/www/rss-bridge/cache/ && mkdir /var/www/rss-bridge/cache/
|
||||
```
|
||||
|
||||
### How to fix "attempt to write a readonly database"
|
||||
|
||||
The sqlite files (db, wal and shm) are not writeable.
|
||||
|
||||
chown -v rss-bridge:rss-bridge cache/*
|
||||
|
||||
### How to fix "Unable to prepare statement: 1, no such table: storage"
|
||||
|
||||
rm cache/*
|
||||
|
||||
### How to create a new bridge from scratch
|
||||
|
||||
Create the new bridge in e.g. `bridges/BearBlogBridge.php`:
|
||||
|
||||
```php
|
||||
<?php
|
||||
|
||||
class BearBlogBridge extends BridgeAbstract
|
||||
{
|
||||
const NAME = 'BearBlog (bearblog.dev)';
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$dom = getSimpleHTMLDOM('https://herman.bearblog.dev/blog/');
|
||||
foreach ($dom->find('.blog-posts li') as $li) {
|
||||
$a = $li->find('a', 0);
|
||||
$this->items[] = [
|
||||
'title' => $a->plaintext,
|
||||
'uri' => 'https://herman.bearblog.dev' . $a->href,
|
||||
];
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Learn more in [bridge api](https://rss-bridge.github.io/rss-bridge/Bridge_API/index.html).
|
||||
|
||||
### How to enable all bridges
|
||||
|
||||
enabled_bridges[] = *
|
||||
|
||||
### How to enable some bridges
|
||||
|
||||
```
|
||||
enabled_bridges[] = TwitchBridge
|
||||
enabled_bridges[] = GettrBridge
|
||||
```
|
||||
|
||||
### How to enable debug mode
|
||||
|
||||
The
|
||||
[debug mode](https://rss-bridge.github.io/rss-bridge/For_Developers/Debug_mode.html)
|
||||
disables the majority of caching operations.
|
||||
|
||||
enable_debug_mode = true
|
||||
|
||||
### How to switch to memcached as cache backend
|
||||
|
||||
```
|
||||
[cache]
|
||||
|
||||
; Cache backend: file (default), sqlite, memcached, null
|
||||
type = "memcached"
|
||||
```
|
||||
|
||||
### How to switch to sqlite3 as cache backend
|
||||
|
||||
type = "sqlite"
|
||||
|
||||
### How to disable bridge errors (as feed items)
|
||||
|
||||
When a bridge fails, RSS-Bridge will produce a feed with a single item describing the error.
|
||||
|
||||
This way, feed readers pick it up and you are notified.
|
||||
|
||||
If you don't want this behaviour, switch the error output to `http`:
|
||||
|
||||
[error]
|
||||
|
||||
; Defines how error messages are returned by RSS-Bridge
|
||||
;
|
||||
; "feed" = As part of the feed (default)
|
||||
; "http" = As HTTP error message
|
||||
; "none" = No errors are reported
|
||||
output = "http"
|
||||
|
||||
### How to accumulate errors before finally reporting it
|
||||
|
||||
Modify `report_limit` so that an error must occur 3 times before it is reported.
|
||||
|
||||
; Defines how often an error must occur before it is reported to the user
|
||||
report_limit = 3
|
||||
|
||||
The report count is reset to 0 each day.
|
||||
|
||||
### How to password-protect the instance (HTTP Basic Auth)
|
||||
|
||||
[authentication]
|
||||
|
||||
enable = true
|
||||
username = "alice"
|
||||
password = "cat"
|
||||
|
||||
Will typically require feed readers to be configured with the credentials.
|
||||
|
||||
It may also be possible to manually include the credentials in the URL:
|
||||
|
||||
https://alice:cat@rss-bridge.org/bridge01/?action=display&bridge=FabriceBellardBridge&format=Html
|
||||
|
||||
### How to create a new output format
|
||||
|
||||
See `formats/PlaintextFormat.php` for an example.
|
||||
|
||||
### How to run unit tests and linter
|
||||
|
||||
These commands require that you have installed the dev dependencies in `composer.json`.
|
||||
|
||||
Run all tests:
|
||||
|
||||
./vendor/bin/phpunit
|
||||
|
||||
Run a single test class:
|
||||
|
||||
./vendor/bin/phpunit --filter UrlTest
|
||||
|
||||
Run linter:
|
||||
|
||||
./vendor/bin/phpcs --standard=phpcs.xml --warning-severity=0 --extensions=php -p ./
|
||||
|
||||
https://github.com/squizlabs/PHP_CodeSniffer/wiki
|
||||
|
||||
### How to spawn a minimal development environment
|
||||
|
||||
php -S 127.0.0.1:9001
|
||||
|
||||
http://127.0.0.1:9001/
|
||||
|
||||
## Explanation
|
||||
|
||||
We are RSS-Bridge community, a group of developers continuing the project initiated by sebsauvage,
|
||||
webmaster of
|
||||
[sebsauvage.net](https://sebsauvage.net), author of
|
||||
[Shaarli](https://sebsauvage.net/wiki/doku.php?id=php:shaarli) and
|
||||
[ZeroBin](https://sebsauvage.net/wiki/doku.php?id=php:zerobin).
|
||||
|
||||
See [CONTRIBUTORS.md](CONTRIBUTORS.md)
|
||||
|
||||
RSS-Bridge uses caching to prevent services from banning your server for repeatedly updating feeds.
|
||||
The specific cache duration can be different between bridges.
|
||||
|
||||
RSS-Bridge allows you to take full control over which bridges are displayed to the user.
|
||||
That way you can host your own RSS-Bridge service with your favorite collection of bridges!
|
||||
|
||||
Current maintainers (as of 2024): @dvikan and @Mynacol #2519
|
||||
|
||||
## Reference
|
||||
|
||||
### Feed item structure
|
||||
|
||||
This is the feed item structure that bridges are expected to produce.
|
||||
|
||||
```php
|
||||
$item = [
|
||||
'uri' => 'https://example.com/blog/hello',
|
||||
'title' => 'Hello world',
|
||||
// Publication date in unix timestamp
|
||||
'timestamp' => 1668706254,
|
||||
'author' => 'Alice',
|
||||
'content' => 'Here be item content',
|
||||
'enclosures' => [
|
||||
'https://example.com/foo.png',
|
||||
'https://example.com/bar.png'
|
||||
],
|
||||
'categories' => [
|
||||
'news',
|
||||
'tech',
|
||||
],
|
||||
// Globally unique id
|
||||
'uid' => 'e7147580c8747aad',
|
||||
]
|
||||
```
|
||||
|
||||
### Output formats
|
||||
|
||||
* `Atom`: Atom feed, for use in feed readers
|
||||
* `Html`: Simple HTML page
|
||||
* `Json`: JSON, for consumption by other applications
|
||||
* `Mrss`: MRSS feed, for use in feed readers
|
||||
* `Plaintext`: Raw text, for consumption by other applications
|
||||
* `Sfeed`: Text, TAB separated
|
||||
|
||||
### Cache backends
|
||||
|
||||
* `File`
|
||||
* `SQLite`
|
||||
* `Memcached`
|
||||
* `Array`
|
||||
* `Null`
|
||||
|
||||
### Licenses
|
||||
|
||||
The source code for RSS-Bridge is [Public Domain](UNLICENSE).
|
||||
|
||||
RSS-Bridge uses third party libraries with their own license:
|
||||
|
||||
* [`Parsedown`](https://github.com/erusev/parsedown) licensed under the [MIT License](https://opensource.org/licenses/MIT)
|
||||
* [`PHP Simple HTML DOM Parser`](https://simplehtmldom.sourceforge.io/docs/1.9/index.html) licensed under the [MIT License](https://opensource.org/licenses/MIT)
|
||||
* [`php-urljoin`](https://github.com/fluffy-critter/php-urljoin) licensed under the [MIT License](https://opensource.org/licenses/MIT)
|
||||
* [`Laravel framework`](https://github.com/laravel/framework/) licensed under the [MIT License](https://opensource.org/licenses/MIT)
|
||||
|
||||
## Rant
|
||||
|
||||
*Dear so-called "social" websites.*
|
||||
|
||||
Your catchword is "share", but you don't want us to share. You want to keep us within your walled gardens. That's why you've been removing RSS links from webpages, hiding them deep on your website, or removed feeds entirely, replacing it with crippled or demented proprietary API. **FUCK YOU.**
|
||||
|
||||
You're not social when you hamper sharing by removing feeds. You're happy to have customers creating content for your ecosystem, but you don't want this content out - a content you do not even own. Google Takeout is just a gimmick. We want our data to flow, we want RSS or Atom feeds.
|
||||
|
||||
We want to share with friends, using open protocols: RSS, Atom, XMPP, whatever. Because no one wants to have *your* service with *your* applications using *your* API force-feeding them. Friends must be free to choose whatever software and service they want.
|
||||
|
||||
We are rebuilding bridges you have willfully destroyed.
|
||||
|
||||
Get your shit together: Put RSS/Atom back in.
|
File diff suppressed because one or more lines are too long
25
UNLICENSE
Normal file
25
UNLICENSE
Normal file
@ -0,0 +1,25 @@
|
||||
This is free and unencumbered software released into the public domain.
|
||||
|
||||
Anyone is free to copy, modify, publish, use, compile, sell, or
|
||||
distribute this software, either in source code form or as a compiled
|
||||
binary, for any purpose, commercial or non-commercial, and by any
|
||||
means.
|
||||
|
||||
In jurisdictions that recognize copyright laws, the author or authors
|
||||
of this software dedicate any and all copyright interest in the
|
||||
software to the public domain. We make this dedication for the benefit
|
||||
of the public at large and to the detriment of our heirs and
|
||||
successors. We intend this dedication to be an overt act of
|
||||
relinquishment in perpetuity of all present and future rights to this
|
||||
software under copyright law.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
For more information, please refer to <http://unlicense.org>
|
||||
|
67
actions/ConnectivityAction.php
Normal file
67
actions/ConnectivityAction.php
Normal file
@ -0,0 +1,67 @@
|
||||
<?php
|
||||
|
||||
/**
|
||||
* Checks if the website for a given bridge is reachable.
|
||||
*
|
||||
* **Remarks**
|
||||
* - This action is only available in debug mode.
|
||||
* - Returns the bridge status as Json-formatted string.
|
||||
* - Returns an error if the bridge is not whitelisted.
|
||||
* - Returns a responsive web page that automatically checks all whitelisted
|
||||
* bridges (using JavaScript) if no bridge is specified.
|
||||
*/
|
||||
class ConnectivityAction implements ActionInterface
|
||||
{
|
||||
private BridgeFactory $bridgeFactory;
|
||||
|
||||
public function __construct(
|
||||
BridgeFactory $bridgeFactory
|
||||
) {
|
||||
$this->bridgeFactory = $bridgeFactory;
|
||||
}
|
||||
|
||||
public function __invoke(Request $request): Response
|
||||
{
|
||||
if (!Debug::isEnabled()) {
|
||||
return new Response('This action is only available in debug mode!', 403);
|
||||
}
|
||||
|
||||
$bridgeName = $request->get('bridge');
|
||||
if (!$bridgeName) {
|
||||
return new Response(render_template('connectivity.html.php'));
|
||||
}
|
||||
$bridgeClassName = $this->bridgeFactory->createBridgeClassName($bridgeName);
|
||||
if (!$bridgeClassName) {
|
||||
return new Response('Bridge not found', 404);
|
||||
}
|
||||
return $this->reportBridgeConnectivity($bridgeClassName);
|
||||
}
|
||||
|
||||
private function reportBridgeConnectivity($bridgeClassName)
|
||||
{
|
||||
if (!$this->bridgeFactory->isEnabled($bridgeClassName)) {
|
||||
throw new \Exception('Bridge is not whitelisted!');
|
||||
}
|
||||
|
||||
$bridge = $this->bridgeFactory->create($bridgeClassName);
|
||||
$curl_opts = [
|
||||
CURLOPT_CONNECTTIMEOUT => 5,
|
||||
CURLOPT_FOLLOWLOCATION => true,
|
||||
];
|
||||
$result = [
|
||||
'bridge' => $bridgeClassName,
|
||||
'successful' => false,
|
||||
'http_code' => null,
|
||||
];
|
||||
try {
|
||||
$response = getContents($bridge::URI, [], $curl_opts, true);
|
||||
$result['http_code'] = $response->getCode();
|
||||
if (in_array($result['http_code'], [200])) {
|
||||
$result['successful'] = true;
|
||||
}
|
||||
} catch (\Exception $e) {
|
||||
}
|
||||
|
||||
return new Response(Json::encode($result), 200, ['content-type' => 'text/json']);
|
||||
}
|
||||
}
|
51
actions/DetectAction.php
Normal file
51
actions/DetectAction.php
Normal file
@ -0,0 +1,51 @@
|
||||
<?php
|
||||
|
||||
class DetectAction implements ActionInterface
|
||||
{
|
||||
private BridgeFactory $bridgeFactory;
|
||||
|
||||
public function __construct(
|
||||
BridgeFactory $bridgeFactory
|
||||
) {
|
||||
$this->bridgeFactory = $bridgeFactory;
|
||||
}
|
||||
|
||||
public function __invoke(Request $request): Response
|
||||
{
|
||||
$url = $request->get('url');
|
||||
$format = $request->get('format');
|
||||
|
||||
if (!$url) {
|
||||
return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'You must specify a url']));
|
||||
}
|
||||
if (!$format) {
|
||||
return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'You must specify a format']));
|
||||
}
|
||||
|
||||
foreach ($this->bridgeFactory->getBridgeClassNames() as $bridgeClassName) {
|
||||
if (!$this->bridgeFactory->isEnabled($bridgeClassName)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$bridge = $this->bridgeFactory->create($bridgeClassName);
|
||||
|
||||
$bridgeParams = $bridge->detectParameters($url);
|
||||
|
||||
if (!$bridgeParams) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$query = [
|
||||
'action' => 'display',
|
||||
'bridge' => $bridgeClassName,
|
||||
'format' => $format,
|
||||
];
|
||||
$query = array_merge($query, $bridgeParams);
|
||||
return new Response('', 301, ['location' => '?' . http_build_query($query)]);
|
||||
}
|
||||
|
||||
return new Response(render(__DIR__ . '/../templates/error.html.php', [
|
||||
'message' => 'No bridge found for given URL: ' . $url,
|
||||
]));
|
||||
}
|
||||
}
|
231
actions/DisplayAction.php
Normal file
231
actions/DisplayAction.php
Normal file
@ -0,0 +1,231 @@
|
||||
<?php
|
||||
|
||||
class DisplayAction implements ActionInterface
|
||||
{
|
||||
private CacheInterface $cache;
|
||||
private Logger $logger;
|
||||
private BridgeFactory $bridgeFactory;
|
||||
|
||||
public function __construct(
|
||||
CacheInterface $cache,
|
||||
Logger $logger,
|
||||
BridgeFactory $bridgeFactory
|
||||
) {
|
||||
$this->cache = $cache;
|
||||
$this->logger = $logger;
|
||||
$this->bridgeFactory = $bridgeFactory;
|
||||
}
|
||||
|
||||
public function __invoke(Request $request): Response
|
||||
{
|
||||
$bridgeName = $request->get('bridge');
|
||||
$format = $request->get('format');
|
||||
$noproxy = $request->get('_noproxy');
|
||||
|
||||
if (!$bridgeName) {
|
||||
return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'Missing bridge name parameter']), 400);
|
||||
}
|
||||
$bridgeClassName = $this->bridgeFactory->createBridgeClassName($bridgeName);
|
||||
if (!$bridgeClassName) {
|
||||
return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'Bridge not found']), 404);
|
||||
}
|
||||
|
||||
if (!$format) {
|
||||
return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'You must specify a format']), 400);
|
||||
}
|
||||
if (!$this->bridgeFactory->isEnabled($bridgeClassName)) {
|
||||
return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'This bridge is not whitelisted']), 400);
|
||||
}
|
||||
|
||||
// Disable proxy (if enabled and per user's request)
|
||||
if (
|
||||
Configuration::getConfig('proxy', 'url')
|
||||
&& Configuration::getConfig('proxy', 'by_bridge')
|
||||
&& $noproxy
|
||||
) {
|
||||
// This const is only used once in getContents()
|
||||
define('NOPROXY', true);
|
||||
}
|
||||
|
||||
$cacheKey = 'http_' . json_encode($request->toArray());
|
||||
|
||||
$bridge = $this->bridgeFactory->create($bridgeClassName);
|
||||
|
||||
$response = $this->createResponse($request, $bridge, $format);
|
||||
|
||||
if ($response->getCode() === 200) {
|
||||
$ttl = $request->get('_cache_timeout');
|
||||
if (Configuration::getConfig('cache', 'custom_timeout') && $ttl) {
|
||||
$ttl = (int) $ttl;
|
||||
} else {
|
||||
$ttl = $bridge->getCacheTimeout();
|
||||
}
|
||||
$this->cache->set($cacheKey, $response, $ttl);
|
||||
}
|
||||
|
||||
return $response;
|
||||
}
|
||||
|
||||
private function createResponse(Request $request, BridgeAbstract $bridge, string $format)
|
||||
{
|
||||
$items = [];
|
||||
|
||||
try {
|
||||
$bridge->loadConfiguration();
|
||||
// Remove parameters that don't concern bridges
|
||||
$remove = [
|
||||
'token',
|
||||
'action',
|
||||
'bridge',
|
||||
'format',
|
||||
'_noproxy',
|
||||
'_cache_timeout',
|
||||
'_error_time',
|
||||
'_', // Some RSS readers add a cache-busting parameter (_=<timestamp>) to feed URLs, detect and ignore them.
|
||||
];
|
||||
$requestArray = $request->toArray();
|
||||
$input = array_diff_key($requestArray, array_fill_keys($remove, ''));
|
||||
$bridge->setInput($input);
|
||||
$bridge->collectData();
|
||||
$items = $bridge->getItems();
|
||||
} catch (\Throwable $e) {
|
||||
if ($e instanceof RateLimitException) {
|
||||
// These are internally generated by bridges
|
||||
$this->logger->info(sprintf('RateLimitException in DisplayAction(%s): %s', $bridge->getShortName(), create_sane_exception_message($e)));
|
||||
return new Response(render(__DIR__ . '/../templates/exception.html.php', ['e' => $e]), 429);
|
||||
}
|
||||
if ($e instanceof HttpException) {
|
||||
if (in_array($e->getCode(), [429, 503])) {
|
||||
// Log with debug, immediately reproduce and return
|
||||
$this->logger->debug(sprintf('Exception in DisplayAction(%s): %s', $bridge->getShortName(), create_sane_exception_message($e)));
|
||||
return new Response(render(__DIR__ . '/../templates/exception.html.php', ['e' => $e]), $e->getCode());
|
||||
}
|
||||
// Some other status code which we let fail normally (but don't log it)
|
||||
} else {
|
||||
// Log error if it's not an HttpException
|
||||
$this->logger->error(sprintf('Exception in DisplayAction(%s)', $bridge->getShortName()), ['e' => $e]);
|
||||
}
|
||||
$errorOutput = Configuration::getConfig('error', 'output');
|
||||
$reportLimit = Configuration::getConfig('error', 'report_limit');
|
||||
$errorCount = 1;
|
||||
if ($reportLimit > 1) {
|
||||
$errorCount = $this->logBridgeError($bridge->getName(), $e->getCode());
|
||||
}
|
||||
// Let clients know about the error if we are passed the report limit
|
||||
if ($errorCount >= $reportLimit) {
|
||||
if ($errorOutput === 'feed') {
|
||||
// Render the exception as a feed item
|
||||
$items = [$this->createFeedItemFromException($e, $bridge)];
|
||||
} elseif ($errorOutput === 'http') {
|
||||
return new Response(render(__DIR__ . '/../templates/exception.html.php', ['e' => $e]), 500);
|
||||
} elseif ($errorOutput === 'none') {
|
||||
// Do nothing (produces an empty feed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$formatFactory = new FormatFactory();
|
||||
$format = $formatFactory->create($format);
|
||||
|
||||
$format->setItems($items);
|
||||
$format->setFeed($bridge->getFeed());
|
||||
$now = time();
|
||||
$format->setLastModified($now);
|
||||
$headers = [
|
||||
'last-modified' => gmdate('D, d M Y H:i:s ', $now) . 'GMT',
|
||||
'content-type' => $format->getMimeType() . '; charset=UTF-8',
|
||||
];
|
||||
$body = $format->render();
|
||||
|
||||
// This is supposed to remove non-utf8 byte sequences, but I'm unsure if it works
|
||||
ini_set('mbstring.substitute_character', 'none');
|
||||
$body = mb_convert_encoding($body, 'UTF-8', 'UTF-8');
|
||||
|
||||
return new Response($body, 200, $headers);
|
||||
}
|
||||
|
||||
private function createFeedItemFromException($e, BridgeAbstract $bridge): array
|
||||
{
|
||||
$item = [];
|
||||
|
||||
// Create a unique identifier every 24 hours
|
||||
$uniqueIdentifier = urlencode((int)(time() / 86400));
|
||||
$title = sprintf('Bridge returned error %s! (%s)', $e->getCode(), $uniqueIdentifier);
|
||||
|
||||
$item['title'] = $title;
|
||||
$item['uri'] = get_current_url();
|
||||
$item['timestamp'] = time();
|
||||
|
||||
// Create an item identifier for feed readers e.g. "staysafetv twitch videos_19389"
|
||||
$item['uid'] = $bridge->getName() . '_' . $uniqueIdentifier;
|
||||
|
||||
$content = render_template(__DIR__ . '/../templates/bridge-error.html.php', [
|
||||
'error' => render_template(__DIR__ . '/../templates/exception.html.php', ['e' => $e]),
|
||||
'searchUrl' => self::createGithubSearchUrl($bridge),
|
||||
'issueUrl' => self::createGithubIssueUrl($bridge, $e),
|
||||
'maintainer' => $bridge->getMaintainer(),
|
||||
]);
|
||||
$item['content'] = $content;
|
||||
|
||||
return $item;
|
||||
}
|
||||
|
||||
private function logBridgeError($bridgeName, $code)
|
||||
{
|
||||
// todo: it's not really necessary to json encode $report
|
||||
$cacheKey = 'error_reporting_' . $bridgeName . '_' . $code;
|
||||
$report = $this->cache->get($cacheKey);
|
||||
if ($report) {
|
||||
$report = Json::decode($report);
|
||||
$report['time'] = time();
|
||||
$report['count']++;
|
||||
} else {
|
||||
$report = [
|
||||
'error' => $code,
|
||||
'time' => time(),
|
||||
'count' => 1,
|
||||
];
|
||||
}
|
||||
$ttl = 86400 * 5;
|
||||
$this->cache->set($cacheKey, Json::encode($report), $ttl);
|
||||
return $report['count'];
|
||||
}
|
||||
|
||||
private static function createGithubIssueUrl(BridgeAbstract $bridge, \Throwable $e): string
|
||||
{
|
||||
$maintainer = $bridge->getMaintainer();
|
||||
if (str_contains($maintainer, ',')) {
|
||||
$maintainers = explode(',', $maintainer);
|
||||
} else {
|
||||
$maintainers = [$maintainer];
|
||||
}
|
||||
$maintainers = array_map('trim', $maintainers);
|
||||
|
||||
$queryString = $_SERVER['QUERY_STRING'] ?? '';
|
||||
$query = [
|
||||
'title' => $bridge->getName() . ' failed with: ' . $e->getMessage(),
|
||||
'body' => sprintf(
|
||||
"```\n%s\n\n%s\n\nQuery string: %s\nVersion: %s\nOs: %s\nPHP version: %s\n```\nMaintainer: @%s",
|
||||
create_sane_exception_message($e),
|
||||
implode("\n", trace_to_call_points(trace_from_exception($e))),
|
||||
$queryString,
|
||||
Configuration::getVersion(),
|
||||
PHP_OS_FAMILY,
|
||||
phpversion() ?: 'Unknown',
|
||||
implode(', @', $maintainers),
|
||||
),
|
||||
'labels' => 'Bridge-Broken',
|
||||
'assignee' => $maintainer[0],
|
||||
];
|
||||
|
||||
return 'https://github.com/RSS-Bridge/rss-bridge/issues/new?' . http_build_query($query);
|
||||
}
|
||||
|
||||
private static function createGithubSearchUrl($bridge): string
|
||||
{
|
||||
return sprintf(
|
||||
'https://github.com/RSS-Bridge/rss-bridge/issues?q=%s',
|
||||
urlencode('is:issue is:open ' . $bridge->getName())
|
||||
);
|
||||
}
|
||||
}
|
95
actions/FindfeedAction.php
Normal file
95
actions/FindfeedAction.php
Normal file
@ -0,0 +1,95 @@
|
||||
<?php
|
||||
|
||||
/**
|
||||
* This action is used by the frontpage form search.
|
||||
* It finds a bridge based off of a user input url.
|
||||
* It uses bridges' detectParameters implementation.
|
||||
*/
|
||||
class FindfeedAction implements ActionInterface
|
||||
{
|
||||
private BridgeFactory $bridgeFactory;
|
||||
|
||||
public function __construct(
|
||||
BridgeFactory $bridgeFactory
|
||||
) {
|
||||
$this->bridgeFactory = $bridgeFactory;
|
||||
}
|
||||
|
||||
public function __invoke(Request $request): Response
|
||||
{
|
||||
$url = $request->get('url');
|
||||
$format = $request->get('format');
|
||||
|
||||
if (!$url) {
|
||||
return new Response('You must specify a url', 400);
|
||||
}
|
||||
if (!$format) {
|
||||
return new Response('You must specify a format', 400);
|
||||
}
|
||||
|
||||
$results = [];
|
||||
foreach ($this->bridgeFactory->getBridgeClassNames() as $bridgeClassName) {
|
||||
if (!$this->bridgeFactory->isEnabled($bridgeClassName)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$bridge = $this->bridgeFactory->create($bridgeClassName);
|
||||
|
||||
$bridgeParams = $bridge->detectParameters($url);
|
||||
|
||||
if ($bridgeParams === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// It's allowed to have no 'context' in a bridge (only a default context without any name)
|
||||
// In this case, the reference to the parameters are found in the first element of the PARAMETERS array
|
||||
|
||||
$context = $bridgeParams['context'] ?? 0;
|
||||
|
||||
$bridgeData = [];
|
||||
// Construct the array of parameters
|
||||
foreach ($bridgeParams as $key => $value) {
|
||||
// 'context' is a special case : it's a bridge parameters, there is no "name" for this parameter
|
||||
if ($key == 'context') {
|
||||
$bridgeData[$key]['name'] = 'Context';
|
||||
$bridgeData[$key]['value'] = $value;
|
||||
} else {
|
||||
$bridgeData[$key]['name'] = $this->getParameterName($bridge, $context, $key);
|
||||
$bridgeData[$key]['value'] = $value;
|
||||
}
|
||||
}
|
||||
|
||||
$bridgeParams['bridge'] = $bridgeClassName;
|
||||
$bridgeParams['format'] = $format;
|
||||
$content = [
|
||||
'url' => './?action=display&' . http_build_query($bridgeParams),
|
||||
'bridgeParams' => $bridgeParams,
|
||||
'bridgeData' => $bridgeData,
|
||||
'bridgeMeta' => [
|
||||
'name' => $bridge::NAME,
|
||||
'description' => $bridge::DESCRIPTION,
|
||||
'parameters' => $bridge::PARAMETERS,
|
||||
'icon' => $bridge->getIcon(),
|
||||
],
|
||||
];
|
||||
$results[] = $content;
|
||||
}
|
||||
if ($results === []) {
|
||||
return new Response(Json::encode(['message' => 'No bridge found for given url']), 404, ['content-type' => 'application/json']);
|
||||
}
|
||||
return new Response(Json::encode($results), 200, ['content-type' => 'application/json']);
|
||||
}
|
||||
|
||||
// Get parameter name in the actual context, or in the global parameter
|
||||
private function getParameterName($bridge, $context, $key)
|
||||
{
|
||||
if (isset($bridge::PARAMETERS[$context][$key]['name'])) {
|
||||
$name = $bridge::PARAMETERS[$context][$key]['name'];
|
||||
} else if (isset($bridge::PARAMETERS['global'][$key]['name'])) {
|
||||
$name = $bridge::PARAMETERS['global'][$key]['name'];
|
||||
} else {
|
||||
$name = 'Variable "' . $key . '" (No name provided)';
|
||||
}
|
||||
return $name;
|
||||
}
|
||||
}
|
49
actions/FrontpageAction.php
Normal file
49
actions/FrontpageAction.php
Normal file
@ -0,0 +1,49 @@
|
||||
<?php
|
||||
|
||||
final class FrontpageAction implements ActionInterface
|
||||
{
|
||||
private BridgeFactory $bridgeFactory;
|
||||
|
||||
public function __construct(
|
||||
BridgeFactory $bridgeFactory
|
||||
) {
|
||||
$this->bridgeFactory = $bridgeFactory;
|
||||
}
|
||||
|
||||
public function __invoke(Request $request): Response
|
||||
{
|
||||
$token = $request->getAttribute('token');
|
||||
|
||||
$messages = [];
|
||||
$activeBridges = 0;
|
||||
|
||||
$bridgeClassNames = $this->bridgeFactory->getBridgeClassNames();
|
||||
|
||||
foreach ($this->bridgeFactory->getMissingEnabledBridges() as $missingEnabledBridge) {
|
||||
$messages[] = [
|
||||
'body' => sprintf('Warning : Bridge "%s" not found', $missingEnabledBridge),
|
||||
'level' => 'warning'
|
||||
];
|
||||
}
|
||||
|
||||
$body = '';
|
||||
foreach ($bridgeClassNames as $bridgeClassName) {
|
||||
if ($this->bridgeFactory->isEnabled($bridgeClassName)) {
|
||||
$body .= BridgeCard::render($this->bridgeFactory, $bridgeClassName, $token);
|
||||
$activeBridges++;
|
||||
}
|
||||
}
|
||||
|
||||
$response = new Response(render(__DIR__ . '/../templates/frontpage.html.php', [
|
||||
'messages' => $messages,
|
||||
'admin_email' => Configuration::getConfig('admin', 'email'),
|
||||
'admin_telegram' => Configuration::getConfig('admin', 'telegram'),
|
||||
'bridges' => $body,
|
||||
'active_bridges' => $activeBridges,
|
||||
'total_bridges' => count($bridgeClassNames),
|
||||
]));
|
||||
|
||||
// TODO: The rendered template could be cached, but beware config changes that changes the html
|
||||
return $response;
|
||||
}
|
||||
}
|
15
actions/HealthAction.php
Normal file
15
actions/HealthAction.php
Normal file
@ -0,0 +1,15 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
class HealthAction implements ActionInterface
|
||||
{
|
||||
public function __invoke(Request $request): Response
|
||||
{
|
||||
$response = [
|
||||
'code' => 200,
|
||||
'message' => 'all is good',
|
||||
];
|
||||
return new Response(Json::encode($response), 200, ['content-type' => 'application/json']);
|
||||
}
|
||||
}
|
36
actions/ListAction.php
Normal file
36
actions/ListAction.php
Normal file
@ -0,0 +1,36 @@
|
||||
<?php
|
||||
|
||||
class ListAction implements ActionInterface
|
||||
{
|
||||
private BridgeFactory $bridgeFactory;
|
||||
|
||||
public function __construct(
|
||||
BridgeFactory $bridgeFactory
|
||||
) {
|
||||
$this->bridgeFactory = $bridgeFactory;
|
||||
}
|
||||
|
||||
public function __invoke(Request $request): Response
|
||||
{
|
||||
$list = new \stdClass();
|
||||
$list->bridges = [];
|
||||
$list->total = 0;
|
||||
|
||||
foreach ($this->bridgeFactory->getBridgeClassNames() as $bridgeClassName) {
|
||||
$bridge = $this->bridgeFactory->create($bridgeClassName);
|
||||
|
||||
$list->bridges[$bridgeClassName] = [
|
||||
'status' => $this->bridgeFactory->isEnabled($bridgeClassName) ? 'active' : 'inactive',
|
||||
'uri' => $bridge->getURI(),
|
||||
'donationUri' => $bridge->getDonationURI(),
|
||||
'name' => $bridge->getName(),
|
||||
'icon' => $bridge->getIcon(),
|
||||
'parameters' => $bridge->getParameters(),
|
||||
'maintainer' => $bridge->getMaintainer(),
|
||||
'description' => $bridge->getDescription()
|
||||
];
|
||||
}
|
||||
$list->total = count($list->bridges);
|
||||
return new Response(Json::encode($list), 200, ['content-type' => 'application/json']);
|
||||
}
|
||||
}
|
8
app.json
Normal file
8
app.json
Normal file
@ -0,0 +1,8 @@
|
||||
{
|
||||
"service": "Heroku",
|
||||
"name": "rss-bridge-heroku",
|
||||
"description": "RSS-Bridge is a PHP project capable of generating RSS and Atom feeds for websites which don't have one.",
|
||||
"repository": "https://github.com/RSS-Bridge/rss-bridge?1651005770",
|
||||
"keywords": ["php", "rss-bridge", "rss"]
|
||||
}
|
||||
|
16
bin/cache-clear
Executable file
16
bin/cache-clear
Executable file
@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env php
|
||||
<?php
|
||||
|
||||
/**
|
||||
* Remove all items from the cache
|
||||
*/
|
||||
|
||||
require __DIR__ . '/../lib/bootstrap.php';
|
||||
require __DIR__ . '/../lib/config.php';
|
||||
|
||||
$container = require __DIR__ . '/../lib/dependencies.php';
|
||||
|
||||
/** @var CacheInterface $cache */
|
||||
$cache = $container['cache'];
|
||||
|
||||
$cache->clear();
|
24
bin/cache-prune
Executable file
24
bin/cache-prune
Executable file
@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env php
|
||||
<?php
|
||||
|
||||
/**
|
||||
* Remove all expired items from the cache
|
||||
*/
|
||||
|
||||
require __DIR__ . '/../lib/bootstrap.php';
|
||||
require __DIR__ . '/../lib/config.php';
|
||||
|
||||
$container = require __DIR__ . '/../lib/dependencies.php';
|
||||
|
||||
if (
|
||||
Configuration::getConfig('cache', 'type') === 'file'
|
||||
&& !Configuration::getConfig('FileCache', 'enable_purge')
|
||||
) {
|
||||
// Override enable_purge for this particular execution
|
||||
Configuration::setConfig('FileCache', 'enable_purge', true);
|
||||
}
|
||||
|
||||
/** @var CacheInterface $cache */
|
||||
$cache = $container['cache'];
|
||||
|
||||
$cache->prune();
|
20
bin/test
Executable file
20
bin/test
Executable file
@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env php
|
||||
<?php
|
||||
|
||||
/**
|
||||
* Add log records to all three levels (for testing purposes)
|
||||
*/
|
||||
|
||||
require __DIR__ . '/../lib/bootstrap.php';
|
||||
require __DIR__ . '/../lib/config.php';
|
||||
|
||||
$container = require __DIR__ . '/../lib/dependencies.php';
|
||||
|
||||
/** @var Logger $logger */
|
||||
$logger = $container['logger'];
|
||||
|
||||
$logger->debug('This is a test debug message');
|
||||
|
||||
$logger->info('This is a test info message');
|
||||
|
||||
$logger->error('This is a test error message');
|
49
bridges/ABCNewsBridge.php
Normal file
49
bridges/ABCNewsBridge.php
Normal file
@ -0,0 +1,49 @@
|
||||
<?php
|
||||
|
||||
class ABCNewsBridge extends BridgeAbstract
|
||||
{
|
||||
const NAME = 'ABC News Bridge';
|
||||
const URI = 'https://www.abc.net.au';
|
||||
const DESCRIPTION = 'Topics of the Australian Broadcasting Corporation';
|
||||
const MAINTAINER = 'yue-dongchen';
|
||||
|
||||
const PARAMETERS = [
|
||||
[
|
||||
'topic' => [
|
||||
'type' => 'list',
|
||||
'name' => 'Region',
|
||||
'title' => 'Choose state',
|
||||
'values' => [
|
||||
'ACT' => 'act',
|
||||
'NSW' => 'nsw',
|
||||
'NT' => 'nt',
|
||||
'QLD' => 'qld',
|
||||
'SA' => 'sa',
|
||||
'TAS' => 'tas',
|
||||
'VIC' => 'vic',
|
||||
'WA' => 'wa'
|
||||
],
|
||||
]
|
||||
]
|
||||
];
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$url = sprintf('https://www.abc.net.au/news/%s', $this->getInput('topic'));
|
||||
$dom = getSimpleHTMLDOM($url);
|
||||
$dom = $dom->find('div[data-component="PaginationList"]', 0);
|
||||
if (!$dom) {
|
||||
throw new \Exception(sprintf('Unable to find css selector on `%s`', $url));
|
||||
}
|
||||
$dom = defaultLinkTo($dom, $this->getURI());
|
||||
foreach ($dom->find('article[data-component="DetailCard"]') as $article) {
|
||||
$a = $article->find('a', 0);
|
||||
$this->items[] = [
|
||||
'title' => $a->plaintext,
|
||||
'uri' => $a->href,
|
||||
'content' => $article->find('p', 0)->plaintext,
|
||||
'timestamp' => strtotime($article->find('time', 0)->datetime),
|
||||
];
|
||||
}
|
||||
}
|
||||
}
|
116
bridges/ABolaBridge.php
Normal file
116
bridges/ABolaBridge.php
Normal file
@ -0,0 +1,116 @@
|
||||
<?php
|
||||
|
||||
class ABolaBridge extends BridgeAbstract
|
||||
{
|
||||
const NAME = 'A Bola';
|
||||
const URI = 'https://abola.pt/';
|
||||
const DESCRIPTION = 'Returns news from the Portuguese sports newspaper A BOLA.PT';
|
||||
const MAINTAINER = 'rmscoelho';
|
||||
const CACHE_TIMEOUT = 3600;
|
||||
const PARAMETERS = [
|
||||
[
|
||||
'feed' => [
|
||||
'name' => 'News Feed',
|
||||
'type' => 'list',
|
||||
'title' => 'Feeds from the Portuguese sports newspaper A BOLA.PT',
|
||||
'values' => [
|
||||
'Últimas' => 'Nnh/Noticias',
|
||||
'Seleção Nacional' => 'Selecao/Noticias',
|
||||
'Futebol Nacional' => [
|
||||
'Notícias' => 'Nacional/Noticias',
|
||||
'Primeira Liga' => 'Nacional/Liga/Noticias',
|
||||
'Liga 2' => 'Nacional/Liga2/Noticias',
|
||||
'Liga 3' => 'Nacional/Liga3/Noticias',
|
||||
'Liga Revelação' => 'Nacional/Liga-Revelacao/Noticias',
|
||||
'Campeonato de Portugal' => 'Nacional/Campeonato-Portugal/Noticias',
|
||||
'Distritais' => 'Nacional/Distritais/Noticias',
|
||||
'Taça de Portugal' => 'Nacional/TPortugal/Noticias',
|
||||
'Futebol Feminino' => 'Nacional/FFeminino/Noticias',
|
||||
'Futsal' => 'Nacional/Futsal/Noticias',
|
||||
],
|
||||
'Futebol Internacional' => [
|
||||
'Notícias' => 'Internacional/Noticias/Noticias',
|
||||
'Liga dos Campeões' => 'Internacional/Liga-dos-campeoes/Noticias',
|
||||
'Liga Europa' => 'Internacional/Liga-europa/Noticias',
|
||||
'Liga Conferência' => 'Internacional/Liga-conferencia/Noticias',
|
||||
'Liga das Nações' => 'Internacional/Liga-das-nacoes/Noticias',
|
||||
'UEFA Youth League' => 'Internacional/Uefa-Youth-League/Noticias',
|
||||
],
|
||||
'Mercado' => 'Mercado',
|
||||
'Modalidades' => 'Modalidades/Noticias',
|
||||
'Motores' => 'Motores/Noticias',
|
||||
]
|
||||
]
|
||||
]
|
||||
];
|
||||
|
||||
public function getIcon()
|
||||
{
|
||||
return 'https://abola.pt/img/icons/favicon-96x96.png';
|
||||
}
|
||||
|
||||
public function getName()
|
||||
{
|
||||
return !is_null($this->getKey('feed')) ? self::NAME . ' | ' . $this->getKey('feed') : self::NAME;
|
||||
}
|
||||
|
||||
public function getURI()
|
||||
{
|
||||
return self::URI . $this->getInput('feed');
|
||||
}
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$url = sprintf('https://abola.pt/%s', $this->getInput('feed'));
|
||||
$dom = getSimpleHTMLDOM($url);
|
||||
if ($this->getInput('feed') !== 'Mercado') {
|
||||
$dom = $dom->find('div#body_Todas1_upNoticiasTodas', 0);
|
||||
} else {
|
||||
$dom = $dom->find('div#body_NoticiasMercado_upNoticiasTodas', 0);
|
||||
}
|
||||
if (!$dom) {
|
||||
throw new \Exception(sprintf('Unable to find css selector on `%s`', $url));
|
||||
}
|
||||
$dom = defaultLinkTo($dom, $this->getURI());
|
||||
foreach ($dom->find('div.media') as $key => $article) {
|
||||
//Get thumbnail
|
||||
$image = $article->find('.media-img', 0)->style;
|
||||
$image = preg_replace('/background-image: url\(/i', '', $image);
|
||||
$image = substr_replace($image, '', -4);
|
||||
$image = preg_replace('/https:\/\//i', '', $image);
|
||||
$image = preg_replace('/www\./i', '', $image);
|
||||
$image = preg_replace('/\/\//', '/', $image);
|
||||
$image = preg_replace('/\/\/\//', '//', $image);
|
||||
$image = substr($image, 7);
|
||||
$image = 'https://' . $image;
|
||||
$image = preg_replace('/ptimg/', 'pt/img', $image);
|
||||
$image = preg_replace('/\/\/bola/', 'www.abola', $image);
|
||||
//Timestamp
|
||||
$date = date('Y/m/d');
|
||||
if (!is_null($article->find("span#body_Todas1_rptNoticiasTodas_lblData_$key", 0))) {
|
||||
$date = $article->find("span#body_Todas1_rptNoticiasTodas_lblData_$key", 0)->plaintext;
|
||||
$date = preg_replace('/\./', '/', $date);
|
||||
}
|
||||
$time = $article->find("span#body_Todas1_rptNoticiasTodas_lblHora_$key", 0)->plaintext;
|
||||
$date = explode('/', $date);
|
||||
$time = explode(':', $time);
|
||||
$year = $date[0];
|
||||
$month = $date[1];
|
||||
$day = $date[2];
|
||||
$hour = $time[0];
|
||||
$minute = $time[1];
|
||||
$timestamp = mktime($hour, $minute, 0, $month, $day, $year);
|
||||
//Content
|
||||
$image = '<img src="' . $image . '" alt="' . $article->find('h4 span', 0)->plaintext . '" />';
|
||||
$description = '<p>' . $article->find('.media-texto > span', 0)->plaintext . '</p>';
|
||||
$content = $image . '</br>' . $description;
|
||||
$a = $article->find('.media-body > a', 0);
|
||||
$this->items[] = [
|
||||
'title' => $a->find('h4 span', 0)->plaintext,
|
||||
'uri' => $a->href,
|
||||
'content' => $content,
|
||||
'timestamp' => $timestamp,
|
||||
];
|
||||
}
|
||||
}
|
||||
}
|
238
bridges/AO3Bridge.php
Normal file
238
bridges/AO3Bridge.php
Normal file
@ -0,0 +1,238 @@
|
||||
<?php
|
||||
|
||||
class AO3Bridge extends BridgeAbstract
|
||||
{
|
||||
const NAME = 'AO3';
|
||||
const URI = 'https://archiveofourown.org/';
|
||||
const CACHE_TIMEOUT = 1800;
|
||||
const DESCRIPTION = 'Returns works or chapters from Archive of Our Own';
|
||||
const MAINTAINER = 'Obsidienne';
|
||||
const PARAMETERS = [
|
||||
'List' => [
|
||||
'url' => [
|
||||
'name' => 'url',
|
||||
'required' => true,
|
||||
// Example: F/F tag
|
||||
'exampleValue' => 'https://archiveofourown.org/tags/F*s*F/works',
|
||||
],
|
||||
'range' => [
|
||||
'name' => 'Chapter Content',
|
||||
'title' => 'Chapter(s) to include in each work\'s feed entry',
|
||||
'defaultValue' => null,
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'None' => null,
|
||||
'First' => 'first',
|
||||
'Latest' => 'last',
|
||||
'Entire work' => 'all',
|
||||
],
|
||||
],
|
||||
'limit' => self::LIMIT,
|
||||
],
|
||||
'Bookmarks' => [
|
||||
'user' => [
|
||||
'name' => 'user',
|
||||
'required' => true,
|
||||
// Example: Nyaaru's bookmarks
|
||||
'exampleValue' => 'Nyaaru',
|
||||
],
|
||||
],
|
||||
'Work' => [
|
||||
'id' => [
|
||||
'name' => 'id',
|
||||
'required' => true,
|
||||
// Example: latest chapters from A Better Past by LysSerris
|
||||
'exampleValue' => '18181853',
|
||||
],
|
||||
]
|
||||
];
|
||||
private $title;
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
switch ($this->queriedContext) {
|
||||
case 'Bookmarks':
|
||||
$this->collectList($this->getURI());
|
||||
break;
|
||||
case 'List':
|
||||
$this->collectList($this->getURI());
|
||||
break;
|
||||
case 'Work':
|
||||
$this->collectWork($this->getURI());
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Feed for lists of works (e.g. recent works, search results, filtered tags,
|
||||
* bookmarks, series, collections).
|
||||
*/
|
||||
private function collectList($url)
|
||||
{
|
||||
$version = 'v0.0.1';
|
||||
$headers = [
|
||||
"useragent: rss-bridge $version (https://github.com/RSS-Bridge/rss-bridge)"
|
||||
];
|
||||
$response = getContents($url, $headers);
|
||||
|
||||
$html = \str_get_html($response);
|
||||
$html = defaultLinkTo($html, self::URI);
|
||||
|
||||
// Get list title. Will include page range + count in some cases
|
||||
$heading = ($html->find('#main h2', 0));
|
||||
if ($heading->find('a.tag')) {
|
||||
$heading = $heading->find('a.tag', 0);
|
||||
}
|
||||
$this->title = $heading->plaintext;
|
||||
|
||||
$limit = $this->getInput('limit') ?? 3;
|
||||
$count = 0;
|
||||
foreach ($html->find('.index.group > li') as $element) {
|
||||
$item = [];
|
||||
|
||||
$title = $element->find('div h4 a', 0);
|
||||
if (!isset($title)) {
|
||||
continue; // discard deleted works
|
||||
}
|
||||
$item['title'] = $title->plaintext;
|
||||
$item['uri'] = $title->href;
|
||||
|
||||
$strdate = $element->find('div p.datetime', 0)->plaintext;
|
||||
$item['timestamp'] = strtotime($strdate);
|
||||
|
||||
// detach from rest of page because remove() is buggy
|
||||
$element = str_get_html($element->outertext());
|
||||
$tags = $element->find('ul.required-tags', 0);
|
||||
foreach ($tags->childNodes() as $tag) {
|
||||
$item['categories'][] = html_entity_decode($tag->plaintext);
|
||||
}
|
||||
$tags->remove();
|
||||
$tags = $element->find('ul.tags', 0);
|
||||
foreach ($tags->childNodes() as $tag) {
|
||||
$item['categories'][] = html_entity_decode($tag->plaintext);
|
||||
}
|
||||
$tags->remove();
|
||||
|
||||
$item['content'] = implode('', $element->childNodes());
|
||||
|
||||
$chapters = $element->find('dl dd.chapters', 0);
|
||||
// bookmarked series and external works do not have a chapters count
|
||||
$chapters = (isset($chapters) ? $chapters->plaintext : 0);
|
||||
$item['uid'] = $item['uri'] . "/$strdate/$chapters";
|
||||
|
||||
// Fetch workskin of desired chapter(s) in list
|
||||
if ($this->getInput('range') && ($limit == 0 || $count++ < $limit)) {
|
||||
$url = $item['uri'];
|
||||
switch ($this->getInput('range')) {
|
||||
case ('all'):
|
||||
$url .= '?view_full_work=true';
|
||||
break;
|
||||
case ('first'):
|
||||
break;
|
||||
case ('last'):
|
||||
// only way to get this is using the navigate page unfortunately
|
||||
$url .= '/navigate';
|
||||
$response = getContents($url, $headers);
|
||||
$html = \str_get_html($response);
|
||||
$html = defaultLinkTo($html, self::URI);
|
||||
$url = $html->find('ol.index.group > li > a', -1)->href;
|
||||
break;
|
||||
}
|
||||
$response = getContents($url, $headers);
|
||||
|
||||
$html = \str_get_html($response);
|
||||
$html = defaultLinkTo($html, self::URI);
|
||||
// remove duplicate fic summary
|
||||
if ($ficsum = $html->find('#workskin > .preface > .summary', 0)) {
|
||||
$ficsum->remove();
|
||||
}
|
||||
$item['content'] .= $html->find('#workskin', 0);
|
||||
}
|
||||
|
||||
// Use predictability of download links to generate enclosures
|
||||
$wid = explode('/', $item['uri'])[4];
|
||||
foreach (['azw3', 'epub', 'mobi', 'pdf', 'html'] as $ext) {
|
||||
$item['enclosures'][] = 'https://archiveofourown.org/downloads/' . $wid . '/work.' . $ext;
|
||||
}
|
||||
|
||||
$this->items[] = $item;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Feed for recent chapters of a specific work.
|
||||
*/
|
||||
private function collectWork($url)
|
||||
{
|
||||
$version = 'v0.0.1';
|
||||
$headers = [
|
||||
"useragent: rss-bridge $version (https://github.com/RSS-Bridge/rss-bridge)"
|
||||
];
|
||||
$response = getContents($url . '/navigate', $headers);
|
||||
|
||||
$html = \str_get_html($response);
|
||||
$html = defaultLinkTo($html, self::URI);
|
||||
|
||||
$response = getContents($url . '?view_full_work=true', $headers);
|
||||
|
||||
$workhtml = \str_get_html($response);
|
||||
$workhtml = defaultLinkTo($workhtml, self::URI);
|
||||
|
||||
$this->title = $html->find('h2 a', 0)->plaintext;
|
||||
|
||||
$nav = $html->find('ol.index.group > li');
|
||||
for ($i = 0; $i < count($nav); $i++) {
|
||||
$item = [];
|
||||
|
||||
$element = $nav[$i];
|
||||
$item['title'] = $element->find('a', 0)->plaintext;
|
||||
$item['content'] = $workhtml->find('#chapter-' . ($i + 1), 0);
|
||||
$item['uri'] = $element->find('a', 0)->href;
|
||||
|
||||
$strdate = $element->find('span.datetime', 0)->plaintext;
|
||||
$strdate = str_replace('(', '', $strdate);
|
||||
$strdate = str_replace(')', '', $strdate);
|
||||
$item['timestamp'] = strtotime($strdate);
|
||||
|
||||
$item['uid'] = $item['uri'] . "/$strdate";
|
||||
|
||||
$this->items[] = $item;
|
||||
}
|
||||
|
||||
$this->items = array_reverse($this->items);
|
||||
}
|
||||
|
||||
public function getName()
|
||||
{
|
||||
$name = parent::getName() . " $this->queriedContext";
|
||||
if (isset($this->title)) {
|
||||
$name .= " - $this->title";
|
||||
}
|
||||
return $name;
|
||||
}
|
||||
|
||||
public function getIcon()
|
||||
{
|
||||
return self::URI . '/favicon.ico';
|
||||
}
|
||||
|
||||
public function getURI()
|
||||
{
|
||||
$url = parent::getURI();
|
||||
switch ($this->queriedContext) {
|
||||
case 'Bookmarks':
|
||||
$user = $this->getInput('user');
|
||||
$url = self::URI
|
||||
. '/users/' . $user
|
||||
. '/bookmarks?bookmark_search[sort_column]=bookmarkable_date';
|
||||
break;
|
||||
case 'List':
|
||||
$url = $this->getInput('url');
|
||||
break;
|
||||
case 'Work':
|
||||
$url = self::URI . '/works/' . $this->getInput('id');
|
||||
break;
|
||||
}
|
||||
return $url;
|
||||
}
|
||||
}
|
173
bridges/ARDAudiothekBridge.php
Normal file
173
bridges/ARDAudiothekBridge.php
Normal file
@ -0,0 +1,173 @@
|
||||
<?php
|
||||
|
||||
class ARDAudiothekBridge extends BridgeAbstract
|
||||
{
|
||||
const NAME = 'ARD-Audiothek Bridge';
|
||||
const URI = 'https://www.ardaudiothek.de';
|
||||
const DESCRIPTION = 'Feed of any show in the ARD-Audiothek, specified by its path';
|
||||
const MAINTAINER = 'Mar-Koeh';
|
||||
/*
|
||||
* The URL Prefix of the API
|
||||
* @const APIENDPOINT https-URL of the used endpoint, ending in `/`
|
||||
*/
|
||||
const APIENDPOINT = 'https://api.ardaudiothek.de/';
|
||||
/*
|
||||
* The requested width of the preview image
|
||||
* 448 and 128 have been observed on the wild
|
||||
* @const IMAGEWIDTH width in px of the preview image
|
||||
*/
|
||||
const IMAGEWIDTH = 448;
|
||||
/*
|
||||
* Placeholder that will be replace by IMAGEWIDTH in the preview image URL
|
||||
* @const IMAGEWIDTHPLACEHOLDER
|
||||
*/
|
||||
const IMAGEWIDTHPLACEHOLDER = '{width}';
|
||||
/*
|
||||
* File extension appended to image link in $this->icon
|
||||
* @const IMAGEEXTENSION
|
||||
*/
|
||||
const IMAGEEXTENSION = '.jpg';
|
||||
|
||||
const PARAMETERS = [
|
||||
[
|
||||
'path' => [
|
||||
'name' => 'Show Link or ID',
|
||||
'required' => true,
|
||||
'title' => 'Link to the show page or just its numeric suffix',
|
||||
'defaultValue' => 'https://www.ardaudiothek.de/sendung/kalk-welk/10777871/'
|
||||
],
|
||||
'limit' => self::LIMIT,
|
||||
]
|
||||
];
|
||||
|
||||
|
||||
/**
|
||||
* Holds the title of the current show
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
private $title;
|
||||
|
||||
/**
|
||||
* Holds the URI of the show
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
private $uri;
|
||||
|
||||
/**
|
||||
* Holds the icon of the feed
|
||||
*
|
||||
*/
|
||||
private $icon;
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$path = $this->getInput('path');
|
||||
$limit = $this->getInput('limit');
|
||||
|
||||
$oldTz = date_default_timezone_get();
|
||||
date_default_timezone_set('Europe/Berlin');
|
||||
|
||||
$pathComponents = explode('/', $path);
|
||||
if (empty($pathComponents)) {
|
||||
returnClientError('Path may not be empty');
|
||||
}
|
||||
if (count($pathComponents) < 2) {
|
||||
$showID = $pathComponents[0];
|
||||
} else {
|
||||
$lastKey = count($pathComponents) - 1;
|
||||
$showID = $pathComponents[$lastKey];
|
||||
if (strlen($showID) === 0) {
|
||||
$showID = $pathComponents[$lastKey - 1];
|
||||
}
|
||||
}
|
||||
|
||||
$url = self::APIENDPOINT . 'programsets/' . $showID . '/';
|
||||
$json1 = getContents($url);
|
||||
$data1 = Json::decode($json1, false);
|
||||
$processedJSON = $data1->data->programSet;
|
||||
if (!$processedJSON) {
|
||||
throw new \Exception('Unable to find show id: ' . $showID);
|
||||
}
|
||||
|
||||
$answerLength = 1;
|
||||
$offset = 0;
|
||||
$numberOfElements = 1;
|
||||
|
||||
while ($answerLength != 0 && $offset < $numberOfElements && (is_null($limit) || $offset < $limit)) {
|
||||
$json2 = getContents($url . '?offset=' . $offset);
|
||||
$data2 = Json::decode($json2, false);
|
||||
$processedJSON = $data2->data->programSet;
|
||||
|
||||
$answerLength = count($processedJSON->items->nodes);
|
||||
$offset = $offset + $answerLength;
|
||||
$numberOfElements = $processedJSON->numberOfElements;
|
||||
|
||||
foreach ($processedJSON->items->nodes as $audio) {
|
||||
$item = [];
|
||||
$item['uri'] = $audio->sharingUrl;
|
||||
$item['title'] = $audio->title;
|
||||
$imageSquare = str_replace(self::IMAGEWIDTHPLACEHOLDER, self::IMAGEWIDTH, $audio->image->url1X1);
|
||||
$image = str_replace(self::IMAGEWIDTHPLACEHOLDER, self::IMAGEWIDTH, $audio->image->url);
|
||||
$item['enclosures'] = [
|
||||
$audio->audios[0]->url,
|
||||
$imageSquare
|
||||
];
|
||||
// synopsis in list is shortened, full synopsis is available using one request per item
|
||||
$item['content'] = '<img src="' . $image . '" /><p>' . $audio->synopsis . '</p>';
|
||||
$item['timestamp'] = $audio->publicationStartDateAndTime;
|
||||
$item['uid'] = $audio->id;
|
||||
$item['author'] = $audio->programSet->publicationService->title;
|
||||
|
||||
$category = $audio->programSet->editorialCategories->title ?? null;
|
||||
if ($category) {
|
||||
$item['categories'] = [$category];
|
||||
}
|
||||
|
||||
$item['itunes'] = [
|
||||
'duration' => $audio->duration,
|
||||
];
|
||||
|
||||
$this->items[] = $item;
|
||||
}
|
||||
}
|
||||
$this->title = $processedJSON->title;
|
||||
$this->uri = $processedJSON->sharingUrl;
|
||||
$this->icon = str_replace(self::IMAGEWIDTHPLACEHOLDER, self::IMAGEWIDTH, $processedJSON->image->url1X1);
|
||||
// add image file extension to URL so icon is shown in generated RSS feeds, see
|
||||
// https://github.com/RSS-Bridge/rss-bridge/blob/4aed05c7b678b5673386d61374bba13637d15487/formats/MrssFormat.php#L76
|
||||
$this->icon = $this->icon . self::IMAGEEXTENSION;
|
||||
|
||||
$this->items = array_slice($this->items, 0, $limit);
|
||||
|
||||
date_default_timezone_set($oldTz);
|
||||
}
|
||||
|
||||
/** {@inheritdoc} */
|
||||
public function getURI()
|
||||
{
|
||||
if (!empty($this->uri)) {
|
||||
return $this->uri;
|
||||
}
|
||||
return parent::getURI();
|
||||
}
|
||||
|
||||
/** {@inheritdoc} */
|
||||
public function getName()
|
||||
{
|
||||
if (!empty($this->title)) {
|
||||
return $this->title;
|
||||
}
|
||||
return parent::getName();
|
||||
}
|
||||
|
||||
/** {@inheritdoc} */
|
||||
public function getIcon()
|
||||
{
|
||||
if (!empty($this->icon)) {
|
||||
return $this->icon;
|
||||
}
|
||||
return parent::getIcon();
|
||||
}
|
||||
}
|
114
bridges/ARDMediathekBridge.php
Normal file
114
bridges/ARDMediathekBridge.php
Normal file
@ -0,0 +1,114 @@
|
||||
<?php
|
||||
|
||||
class ARDMediathekBridge extends BridgeAbstract
|
||||
{
|
||||
const NAME = 'ARD-Mediathek Bridge';
|
||||
const URI = 'https://www.ardmediathek.de';
|
||||
const DESCRIPTION = 'Feed of any series in the ARD-Mediathek, specified by its path';
|
||||
const MAINTAINER = 'yue-dongchen';
|
||||
/*
|
||||
* Number of Items to be requested from ARDmediathek API
|
||||
* 12 has been observed on the wild
|
||||
* 29 is the highest successfully tested value
|
||||
* More Items could be fetched via pagination
|
||||
* The JSON-field pagination holds more information on that
|
||||
* @const PAGESIZE number of requested items
|
||||
*/
|
||||
const PAGESIZE = 29;
|
||||
/*
|
||||
* The URL Prefix of the (Webapp-)API
|
||||
* @const APIENDPOINT https-URL of the used endpoint
|
||||
*/
|
||||
const APIENDPOINT = 'https://api.ardmediathek.de/page-gateway/widgets/ard/asset/';
|
||||
/*
|
||||
* The URL prefix of the video link
|
||||
* URLs from the webapp include a slug containing titles of show, episode, and tv station.
|
||||
* It seems to work without that.
|
||||
* @const VIDEOLINKPREFIX https-URL prefix of video links
|
||||
*/
|
||||
const VIDEOLINKPREFIX = 'https://www.ardmediathek.de/video/';
|
||||
/*
|
||||
* The requested width of the preview image
|
||||
* 432 has been observed on the wild
|
||||
* The webapp seems to also compute and add the height value
|
||||
* It seems to works without that.
|
||||
* @const IMAGEWIDTH width in px of the preview image
|
||||
*/
|
||||
const IMAGEWIDTH = 432;
|
||||
/*
|
||||
* Placeholder that will be replace by IMAGEWIDTH in the preview image URL
|
||||
* @const IMAGEWIDTHPLACEHOLDER
|
||||
*/
|
||||
const IMAGEWIDTHPLACEHOLDER = '{width}';
|
||||
/**
|
||||
* Title of the current show
|
||||
* @var string
|
||||
*/
|
||||
private $title;
|
||||
|
||||
const PARAMETERS = [
|
||||
[
|
||||
'path' => [
|
||||
'name' => 'Show Link or ID',
|
||||
'required' => true,
|
||||
'title' => 'Link to the show page or just its alphanumeric suffix',
|
||||
'defaultValue' => 'https://www.ardmediathek.de/sendung/45-min/Y3JpZDovL25kci5kZS8xMzkx/'
|
||||
]
|
||||
]
|
||||
];
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$oldTz = date_default_timezone_get();
|
||||
|
||||
date_default_timezone_set('Europe/Berlin');
|
||||
|
||||
$pathComponents = explode('/', $this->getInput('path'));
|
||||
if (empty($pathComponents)) {
|
||||
returnClientError('Path may not be empty');
|
||||
}
|
||||
if (count($pathComponents) < 2) {
|
||||
$showID = $pathComponents[0];
|
||||
} else {
|
||||
$lastKey = count($pathComponents) - 1;
|
||||
$showID = $pathComponents[$lastKey];
|
||||
if (strlen($showID) === 0) {
|
||||
$showID = $pathComponents[$lastKey - 1];
|
||||
}
|
||||
}
|
||||
|
||||
$url = self::APIENDPOINT . $showID . '?pageSize=' . self::PAGESIZE;
|
||||
$rawJSON = getContents($url);
|
||||
$processedJSON = json_decode($rawJSON);
|
||||
|
||||
foreach ($processedJSON->teasers as $video) {
|
||||
$item = [];
|
||||
// there is also ->links->self->id, ->links->self->urlId, ->links->target->id, ->links->target->urlId
|
||||
$item['uri'] = self::VIDEOLINKPREFIX . $video->id . '/';
|
||||
// there is also ->mediumTitle and ->shortTitle
|
||||
$item['title'] = $video->longTitle;
|
||||
// in the test, aspect16x9 was the only child of images, not sure whether that is always true
|
||||
$item['enclosures'] = [
|
||||
str_replace(self::IMAGEWIDTHPLACEHOLDER, self::IMAGEWIDTH, $video->images->aspect16x9->src)
|
||||
];
|
||||
$item['content'] = '<img src="' . $item['enclosures'][0] . '" /><p>';
|
||||
$item['timestamp'] = $video->broadcastedOn;
|
||||
$item['uid'] = $video->id;
|
||||
$item['author'] = $video->publicationService->name;
|
||||
$this->items[] = $item;
|
||||
}
|
||||
|
||||
$this->title = $processedJSON->title;
|
||||
|
||||
date_default_timezone_set($oldTz);
|
||||
}
|
||||
|
||||
/** {@inheritdoc} */
|
||||
public function getName()
|
||||
{
|
||||
if (!empty($this->title)) {
|
||||
return $this->title;
|
||||
}
|
||||
return parent::getName();
|
||||
}
|
||||
}
|
63
bridges/ASRockNewsBridge.php
Normal file
63
bridges/ASRockNewsBridge.php
Normal file
@ -0,0 +1,63 @@
|
||||
<?php
|
||||
|
||||
class ASRockNewsBridge extends BridgeAbstract
|
||||
{
|
||||
const NAME = 'ASRock News Bridge';
|
||||
const URI = 'https://www.asrock.com';
|
||||
const DESCRIPTION = 'Returns latest news articles';
|
||||
const MAINTAINER = 'VerifiedJoseph';
|
||||
const PARAMETERS = [];
|
||||
|
||||
const CACHE_TIMEOUT = 3600; // 1 hour
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$html = getSimpleHTMLDOM(self::URI . '/news/index.asp');
|
||||
|
||||
$html = defaultLinkTo($html, self::URI . '/news/');
|
||||
|
||||
foreach ($html->find('div.inner > a') as $index => $a) {
|
||||
$item = [];
|
||||
|
||||
$articlePath = $a->href;
|
||||
|
||||
$articlePageHtml = getSimpleHTMLDOMCached($articlePath, self::CACHE_TIMEOUT);
|
||||
|
||||
$articlePageHtml = defaultLinkTo($articlePageHtml, self::URI);
|
||||
|
||||
$contents = $articlePageHtml->find('div.Contents', 0);
|
||||
|
||||
$item['uri'] = $articlePath;
|
||||
$item['title'] = $contents->find('h3', 0)->innertext;
|
||||
|
||||
$contents->find('h3', 0)->outertext = '';
|
||||
|
||||
$item['content'] = $contents->innertext;
|
||||
$item['timestamp'] = $this->extractDate($a->plaintext);
|
||||
|
||||
$img = $a->find('img', 0);
|
||||
if ($img) {
|
||||
$item['enclosures'][] = $img->src;
|
||||
}
|
||||
|
||||
$this->items[] = $item;
|
||||
|
||||
if (count($this->items) >= 10) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private function extractDate($text)
|
||||
{
|
||||
$dateRegex = '/^([0-9]{4}\/[0-9]{1,2}\/[0-9]{1,2})/';
|
||||
|
||||
$text = trim($text);
|
||||
|
||||
if (preg_match($dateRegex, $text, $matches)) {
|
||||
return $matches[1];
|
||||
}
|
||||
|
||||
return '';
|
||||
}
|
||||
}
|
37
bridges/AcrimedBridge.php
Normal file
37
bridges/AcrimedBridge.php
Normal file
@ -0,0 +1,37 @@
|
||||
<?php
|
||||
|
||||
class AcrimedBridge extends FeedExpander
|
||||
{
|
||||
const MAINTAINER = 'qwertygc';
|
||||
const NAME = 'Acrimed Bridge';
|
||||
const URI = 'https://www.acrimed.org/';
|
||||
const CACHE_TIMEOUT = 4800; //2hours
|
||||
const DESCRIPTION = 'Returns the newest articles';
|
||||
|
||||
const PARAMETERS = [
|
||||
[
|
||||
'limit' => [
|
||||
'name' => 'limit',
|
||||
'type' => 'number',
|
||||
'defaultValue' => -1,
|
||||
]
|
||||
]
|
||||
];
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$url = 'https://www.acrimed.org/spip.php?page=backend';
|
||||
$limit = $this->getInput('limit');
|
||||
$this->collectExpandableDatas($url, $limit);
|
||||
}
|
||||
|
||||
protected function parseItem(array $item)
|
||||
{
|
||||
$articlePage = getSimpleHTMLDOM($item['uri']);
|
||||
$article = sanitize($articlePage->find('article.article1', 0)->innertext);
|
||||
$article = defaultLinkTo($article, static::URI);
|
||||
$item['content'] = $article;
|
||||
|
||||
return $item;
|
||||
}
|
||||
}
|
45
bridges/ActivisionResearchBridge.php
Normal file
45
bridges/ActivisionResearchBridge.php
Normal file
@ -0,0 +1,45 @@
|
||||
<?php
|
||||
|
||||
class ActivisionResearchBridge extends BridgeAbstract
|
||||
{
|
||||
const NAME = 'Activision Research Blog';
|
||||
const URI = 'https://research.activision.com';
|
||||
const DESCRIPTION = 'Posts from the Activision Research blog';
|
||||
const MAINTAINER = 'thefranke';
|
||||
const CACHE_TIMEOUT = 86400; // 24h
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$dom = getSimpleHTMLDOM(static::URI);
|
||||
$dom = $dom->find('div[id="home-blog-feed"]', 0);
|
||||
if (!$dom) {
|
||||
throw new \Exception(sprintf('Unable to find css selector on `%s`', $url));
|
||||
}
|
||||
$dom = defaultLinkTo($dom, $this->getURI());
|
||||
foreach ($dom->find('div[class="blog-entry"]') as $article) {
|
||||
$a = $article->find('a', 0);
|
||||
|
||||
$blogimg = extractFromDelimiters($article->find('div[class="blog-img"]', 0)->style, 'url(', ')');
|
||||
|
||||
$title = htmlspecialchars_decode($article->find('div[class="title"]', 0)->plaintext);
|
||||
$author = htmlspecialchars_decode($article->find('div[class="author]', 0)->plaintext);
|
||||
$date = $article->find('div[class="pubdate"]', 0)->plaintext;
|
||||
|
||||
$entry = getSimpleHTMLDOMCached($a->href, static::CACHE_TIMEOUT * 7 * 4);
|
||||
$entry = defaultLinkTo($entry, $this->getURI());
|
||||
|
||||
$content = $entry->find('div[class="blog-body"]', 0);
|
||||
$tagsremove = ['script', 'iframe', 'input', 'form'];
|
||||
$content = sanitize($content, $tagsremove);
|
||||
$content = '<img src="' . static::URI . $blogimg . '" alt="">' . $content;
|
||||
|
||||
$this->items[] = [
|
||||
'title' => $title,
|
||||
'author' => $author,
|
||||
'uri' => $a->href,
|
||||
'content' => $content,
|
||||
'timestamp' => strtotime($date),
|
||||
];
|
||||
}
|
||||
}
|
||||
}
|
56
bridges/AirBreizhBridge.php
Normal file
56
bridges/AirBreizhBridge.php
Normal file
@ -0,0 +1,56 @@
|
||||
<?php
|
||||
|
||||
class AirBreizhBridge extends BridgeAbstract
|
||||
{
|
||||
const MAINTAINER = 'fanch317';
|
||||
const NAME = 'Air Breizh';
|
||||
const URI = 'https://www.airbreizh.asso.fr/';
|
||||
const DESCRIPTION = 'Returns newests publications on Air Breizh';
|
||||
const PARAMETERS = [
|
||||
'Publications' => [
|
||||
'theme' => [
|
||||
'name' => 'Thematique',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'Tout' => '',
|
||||
'Rapport d\'activite' => 'rapport-dactivite',
|
||||
'Etude' => 'etudes',
|
||||
'Information' => 'information',
|
||||
'Autres documents' => 'autres-documents',
|
||||
'Plan Régional de Surveillance de la qualité de l’air' => 'prsqa',
|
||||
'Transport' => 'transport'
|
||||
]
|
||||
]
|
||||
]
|
||||
];
|
||||
|
||||
public function getIcon()
|
||||
{
|
||||
return 'https://www.airbreizh.asso.fr/voy_content/uploads/2017/11/favicon.png';
|
||||
}
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$html = '';
|
||||
$html = getSimpleHTMLDOM(static::URI . 'publications/?fwp_publications_thematiques=' . $this->getInput('theme'));
|
||||
|
||||
foreach ($html->find('article') as $article) {
|
||||
$item = [];
|
||||
// Title
|
||||
$item['title'] = $article->find('h2', 0)->plaintext;
|
||||
// Author
|
||||
$item['author'] = 'Air Breizh';
|
||||
// Image
|
||||
$imagelink = $article->find('.card__image', 0)->find('img', 0)->getAttribute('src');
|
||||
// Content preview
|
||||
$item['content'] = '<img src="' . $imagelink . '" />
|
||||
<br/>'
|
||||
. $article->find('.card__text', 0)->plaintext;
|
||||
// URL
|
||||
$item['uri'] = $article->find('.publi__buttons', 0)->find('a', 0)->getAttribute('href');
|
||||
// ID
|
||||
$item['id'] = $article->find('.publi__buttons', 0)->find('a', 0)->getAttribute('href');
|
||||
$this->items[] = $item;
|
||||
}
|
||||
}
|
||||
}
|
76
bridges/AlbionOnlineBridge.php
Normal file
76
bridges/AlbionOnlineBridge.php
Normal file
@ -0,0 +1,76 @@
|
||||
<?php
|
||||
|
||||
class AlbionOnlineBridge extends BridgeAbstract
|
||||
{
|
||||
const NAME = 'Albion Online Changelog';
|
||||
const MAINTAINER = 'otakuf';
|
||||
const URI = 'https://albiononline.com';
|
||||
const DESCRIPTION = 'Returns the changes made to the Albion Online';
|
||||
const CACHE_TIMEOUT = 3600; // 60min
|
||||
|
||||
const PARAMETERS = [ [
|
||||
'postcount' => [
|
||||
'name' => 'Limit',
|
||||
'type' => 'number',
|
||||
'required' => true,
|
||||
'title' => 'Maximum number of items to return',
|
||||
'defaultValue' => 5,
|
||||
],
|
||||
'language' => [
|
||||
'name' => 'Language',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'English' => 'en',
|
||||
'Deutsch' => 'de',
|
||||
'Polski' => 'pl',
|
||||
'Français' => 'fr',
|
||||
'Русский' => 'ru',
|
||||
'Português' => 'pt',
|
||||
'Español' => 'es',
|
||||
],
|
||||
'title' => 'Language of changelog posts',
|
||||
'defaultValue' => 'en',
|
||||
],
|
||||
'full' => [
|
||||
'name' => 'Full changelog',
|
||||
'type' => 'checkbox',
|
||||
'required' => false,
|
||||
'title' => 'Enable to receive the full changelog post for each item'
|
||||
],
|
||||
]];
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$api = 'https://albiononline.com/';
|
||||
// Example: https://albiononline.com/en/changelog/1/5
|
||||
$url = $api . $this->getInput('language') . '/changelog/1/' . $this->getInput('postcount');
|
||||
|
||||
$html = getSimpleHTMLDOM($url);
|
||||
|
||||
foreach ($html->find('li') as $data) {
|
||||
$item = [];
|
||||
$item['uri'] = self::URI . $data->find('a', 0)->getAttribute('href');
|
||||
$item['title'] = trim(explode('|', $data->find('span', 0)->plaintext)[0]);
|
||||
// Time below work only with en lang. Need to think about solution. May be separate request like getFullChangelog, but to english list for all language
|
||||
//print_r( date_parse_from_format( 'M j, Y' , 'Sep 9, 2020') );
|
||||
//$item['timestamp'] = $this->extractDate($a->plaintext);
|
||||
$item['author'] = 'albiononline.com';
|
||||
if ($this->getInput('full')) {
|
||||
$item['content'] = $this->getFullChangelog($item['uri']);
|
||||
} else {
|
||||
//$item['content'] = trim(preg_replace('/\s+/', ' ', $data->find('span', 0)->plaintext));
|
||||
// Just use title, no info at all or use title and date, see above
|
||||
$item['content'] = $item['title'];
|
||||
}
|
||||
$item['uid'] = hash('sha256', $item['title']);
|
||||
$this->items[] = $item;
|
||||
}
|
||||
}
|
||||
|
||||
private function getFullChangelog($url)
|
||||
{
|
||||
$html = getSimpleHTMLDOMCached($url);
|
||||
$html = defaultLinkTo($html, self::URI);
|
||||
return $html->find('div.small-12.columns', 1)->innertext;
|
||||
}
|
||||
}
|
87
bridges/AlfaBankByBridge.php
Normal file
87
bridges/AlfaBankByBridge.php
Normal file
@ -0,0 +1,87 @@
|
||||
<?php
|
||||
|
||||
class AlfaBankByBridge extends BridgeAbstract
|
||||
{
|
||||
const MAINTAINER = 'lassana';
|
||||
const NAME = 'AlfaBank.by Новости';
|
||||
const URI = 'https://www.alfabank.by';
|
||||
const DESCRIPTION = 'Уведомления Alfa-Now — новости от Альфа-Банка';
|
||||
const CACHE_TIMEOUT = 3600; // 1 hour
|
||||
const PARAMETERS = [
|
||||
'News' => [
|
||||
'business' => [
|
||||
'name' => 'Альфа Бизнес',
|
||||
'type' => 'list',
|
||||
'title' => 'В зависимости от выбора, возращает уведомления для" .
|
||||
" клиентов физ. лиц либо для клиентов-юридических лиц и ИП',
|
||||
'values' => [
|
||||
'Новости' => 'news',
|
||||
'Новости бизнеса' => 'newsBusiness'
|
||||
],
|
||||
'defaultValue' => 'news'
|
||||
],
|
||||
'fullContent' => [
|
||||
'name' => 'Включать содержимое',
|
||||
'type' => 'checkbox',
|
||||
'title' => 'Если выбрано, содержимое уведомлений вставляется в поток (работает медленно)'
|
||||
]
|
||||
]
|
||||
];
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$business = $this->getInput('business') == 'newsBusiness';
|
||||
$fullContent = $this->getInput('fullContent') == 'on';
|
||||
|
||||
$mainPageUrl = self::URI . '/about/articles/uvedomleniya/';
|
||||
if ($business) {
|
||||
$mainPageUrl .= '?business=true';
|
||||
}
|
||||
$html = getSimpleHTMLDOM($mainPageUrl);
|
||||
$limit = 0;
|
||||
|
||||
foreach ($html->find('a.notifications__item') as $element) {
|
||||
if ($limit < 10) {
|
||||
$item = [];
|
||||
$item['uid'] = 'urn:sha1:' . hash('sha1', $element->getAttribute('data-notification-id'));
|
||||
$item['title'] = $element->find('div.item-title', 0)->innertext;
|
||||
$item['timestamp'] = DateTime::createFromFormat(
|
||||
'd M Y',
|
||||
$this->ruMonthsToEn($element->find('div.item-date', 0)->innertext)
|
||||
)->getTimestamp();
|
||||
|
||||
$itemUrl = self::URI . $element->href;
|
||||
if ($business) {
|
||||
$itemUrl = str_replace('?business=true', '', $itemUrl);
|
||||
}
|
||||
$item['uri'] = $itemUrl;
|
||||
|
||||
if ($fullContent) {
|
||||
$itemHtml = getSimpleHTMLDOM($itemUrl);
|
||||
if ($itemHtml) {
|
||||
$item['content'] = $itemHtml->find('div.now-p__content-text', 0)->innertext;
|
||||
}
|
||||
}
|
||||
|
||||
$this->items[] = $item;
|
||||
$limit++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function getIcon()
|
||||
{
|
||||
return static::URI . '/local/images/favicon.ico';
|
||||
}
|
||||
|
||||
private function ruMonthsToEn($date)
|
||||
{
|
||||
$ruMonths = [
|
||||
'Января', 'Февраля', 'Марта', 'Апреля', 'Мая', 'Июня',
|
||||
'Июля', 'Августа', 'Сентября', 'Октября', 'Ноября', 'Декабря' ];
|
||||
$enMonths = [
|
||||
'January', 'February', 'March', 'April', 'May', 'June',
|
||||
'July', 'August', 'September', 'October', 'November', 'December' ];
|
||||
return str_replace($ruMonths, $enMonths, $date);
|
||||
}
|
||||
}
|
85
bridges/AllSidesBridge.php
Normal file
85
bridges/AllSidesBridge.php
Normal file
@ -0,0 +1,85 @@
|
||||
<?php
|
||||
|
||||
class AllSidesBridge extends BridgeAbstract
|
||||
{
|
||||
const NAME = 'AllSides';
|
||||
const URI = 'https://www.allsides.com';
|
||||
const DESCRIPTION = 'Balanced news and media bias ratings.';
|
||||
const MAINTAINER = 'Oliver Nutter';
|
||||
const PARAMETERS = [
|
||||
'global' => [
|
||||
'limit' => [
|
||||
'name' => 'Number of posts to return',
|
||||
'type' => 'number',
|
||||
'defaultValue' => 10,
|
||||
'required' => false,
|
||||
'title' => 'Zero or negative values return all posts (ignored if not fetching full article)',
|
||||
],
|
||||
'fetch' => [
|
||||
'name' => 'Fetch full article content',
|
||||
'type' => 'checkbox',
|
||||
'defaultValue' => 'checked',
|
||||
],
|
||||
],
|
||||
'Headline Roundups' => [],
|
||||
];
|
||||
|
||||
private const ROUNDUPS_URI = self::URI . '/headline-roundups';
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
switch ($this->queriedContext) {
|
||||
case 'Headline Roundups':
|
||||
$index = getSimpleHTMLDOM(self::ROUNDUPS_URI);
|
||||
defaultLinkTo($index, self::ROUNDUPS_URI);
|
||||
$entries = $index->find('table.views-table > tbody > tr');
|
||||
|
||||
$limit = (int) $this->getInput('limit');
|
||||
$fetch = (bool) $this->getInput('fetch');
|
||||
|
||||
if ($limit > 0 && $fetch) {
|
||||
$entries = array_slice($entries, 0, $limit);
|
||||
}
|
||||
|
||||
foreach ($entries as $entry) {
|
||||
$item = [
|
||||
'title' => $entry->find('.views-field-name', 0)->text(),
|
||||
'uri' => $entry->find('a', 0)->href,
|
||||
'timestamp' => $entry->find('.date-display-single', 0)->content,
|
||||
'author' => 'AllSides Staff',
|
||||
];
|
||||
|
||||
if ($fetch) {
|
||||
$article = getSimpleHTMLDOMCached($item['uri']);
|
||||
defaultLinkTo($article, $item['uri']);
|
||||
|
||||
$item['content'] = $article->find('.story-id-page-description', 0);
|
||||
|
||||
foreach ($article->find('.page-tags a') as $tag) {
|
||||
$item['categories'][] = $tag->text();
|
||||
}
|
||||
}
|
||||
|
||||
$this->items[] = $item;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
public function getName()
|
||||
{
|
||||
if ($this->queriedContext) {
|
||||
return self::NAME . " - {$this->queriedContext}";
|
||||
}
|
||||
return self::NAME;
|
||||
}
|
||||
|
||||
public function getURI()
|
||||
{
|
||||
switch ($this->queriedContext) {
|
||||
case 'Headline Roundups':
|
||||
return self::ROUNDUPS_URI;
|
||||
}
|
||||
return self::URI;
|
||||
}
|
||||
}
|
157
bridges/AllegroBridge.php
Normal file
157
bridges/AllegroBridge.php
Normal file
@ -0,0 +1,157 @@
|
||||
<?php
|
||||
|
||||
class AllegroBridge extends BridgeAbstract
|
||||
{
|
||||
const NAME = 'Allegro';
|
||||
const URI = 'https://www.allegro.pl';
|
||||
const DESCRIPTION = 'Returns the search results from the Allegro.pl shopping and bidding portal';
|
||||
const MAINTAINER = 'wrobelda';
|
||||
const PARAMETERS = [[
|
||||
'url' => [
|
||||
'name' => 'Search URL',
|
||||
'title' => 'Copy the URL from your browser\'s address bar after searching for your items and paste it here',
|
||||
'exampleValue' => 'https://allegro.pl/kategoria/swieze-warzywa-cebula-318660',
|
||||
'required' => true,
|
||||
],
|
||||
'cookie' => [
|
||||
'name' => 'The complete cookie value',
|
||||
'title' => 'Paste the value of the cookie value from your browser if you want to prevent Allegro imposing rate limits',
|
||||
'required' => false,
|
||||
],
|
||||
'includeSponsoredOffers' => [
|
||||
'type' => 'checkbox',
|
||||
'name' => 'Include Sponsored Offers',
|
||||
'defaultValue' => 'checked'
|
||||
],
|
||||
'includePromotedOffers' => [
|
||||
'type' => 'checkbox',
|
||||
'name' => 'Include Promoted Offers',
|
||||
'defaultValue' => 'checked'
|
||||
]
|
||||
]];
|
||||
|
||||
public function getName()
|
||||
{
|
||||
$url = $this->getInput('url');
|
||||
if (!$url) {
|
||||
return parent::getName();
|
||||
}
|
||||
$parsedUrl = parse_url($url, PHP_URL_QUERY);
|
||||
if (!$parsedUrl) {
|
||||
return parent::getName();
|
||||
}
|
||||
parse_str($parsedUrl, $fields);
|
||||
|
||||
if (array_key_exists('string', $fields)) {
|
||||
$f = urldecode($fields['string']);
|
||||
} else {
|
||||
$f = false;
|
||||
}
|
||||
if ($f) {
|
||||
return $f;
|
||||
}
|
||||
|
||||
return parent::getName();
|
||||
}
|
||||
|
||||
public function getURI()
|
||||
{
|
||||
return $this->getInput('url') ?? parent::getURI();
|
||||
}
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
# make sure we order by the most recently listed offers
|
||||
$url = preg_replace('/([?&])order=[^&]+(&|$)/', '$1', $this->getInput('url'));
|
||||
$url .= (parse_url($url, PHP_URL_QUERY) ? '&' : '?') . 'order=n';
|
||||
|
||||
$opts = [];
|
||||
|
||||
// If a cookie is provided
|
||||
if ($cookie = $this->getInput('cookie')) {
|
||||
$opts[CURLOPT_COOKIE] = $cookie;
|
||||
}
|
||||
|
||||
$html = getSimpleHTMLDOM($url, [], $opts);
|
||||
|
||||
# if no results found
|
||||
if ($html->find('.mzmg_6m.m9qz_yo._6a66d_-fJr5')) {
|
||||
return;
|
||||
}
|
||||
|
||||
$results = $html->find('article[data-analytics-view-custom-context="REGULAR"]');
|
||||
|
||||
if ($this->getInput('includeSponsoredOffers')) {
|
||||
$results = array_merge($results, $html->find('article[data-analytics-view-custom-context="SPONSORED"]'));
|
||||
}
|
||||
|
||||
if ($this->getInput('includePromotedOffers')) {
|
||||
$results = array_merge($results, $html->find('article[data-analytics-view-custom-context="PROMOTED"]'));
|
||||
}
|
||||
|
||||
foreach ($results as $post) {
|
||||
$item = [];
|
||||
|
||||
$item['uid'] = $post->{'data-analytics-view-value'};
|
||||
|
||||
$item_link = $post->find('a[href*="' . $item['uid'] . '"], a[href*="allegrolokalnie"]', 0);
|
||||
|
||||
$item['uri'] = $item_link->href;
|
||||
|
||||
$item['title'] = $item_link->find('img', 0)->alt;
|
||||
|
||||
$image = $item_link->find('img', 0)->{'data-src'} ?: $item_link->find('img', 0)->src ?? false;
|
||||
|
||||
if ($image) {
|
||||
$item['enclosures'] = [$image . '#.image'];
|
||||
}
|
||||
|
||||
$price = $post->{'data-analytics-view-json-custom-price'};
|
||||
if ($price) {
|
||||
$priceDecoded = json_decode(html_entity_decode($price));
|
||||
$price = $priceDecoded->amount . ' ' . $priceDecoded->currency;
|
||||
}
|
||||
|
||||
$descriptionPatterns = ['/<\s*dt[^>]*>\b/', '/<\/dt>/', '/<\s*dd[^>]*>\b/', '/<\/dd>/'];
|
||||
$descriptionReplacements = ['<span>', ':</span> ', '<strong>', ' </strong> '];
|
||||
$description = $post->find('.m7er_k4.mpof_5r.mpof_z0_s', 0)->innertext;
|
||||
$descriptionPretty = preg_replace($descriptionPatterns, $descriptionReplacements, $description);
|
||||
|
||||
$pricingExtraInfo = array_filter($post->find('.mqu1_g3.mgn2_12'), function ($node) {
|
||||
return empty($node->find('.mvrt_0'));
|
||||
});
|
||||
|
||||
$pricingExtraInfo = $pricingExtraInfo[0]->plaintext ?? '';
|
||||
|
||||
$offerExtraInfo = array_map(function ($node) {
|
||||
return str_contains($node->plaintext, 'zapłać później') ? '' : $node->outertext;
|
||||
}, $post->find('div.mpof_ki.mwdn_1.mj7a_4.mgn2_12'));
|
||||
|
||||
$isSmart = $post->find('img[alt="Smart!"]', 0) ?? false;
|
||||
if ($isSmart) {
|
||||
$pricingExtraInfo .= $isSmart->outertext;
|
||||
}
|
||||
|
||||
$item['categories'] = [];
|
||||
$parameters = $post->find('dd');
|
||||
foreach ($parameters as $parameter) {
|
||||
if (in_array(strtolower($parameter->innertext), ['brak', 'nie'])) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$item['categories'][] = $parameter->innertext;
|
||||
}
|
||||
|
||||
$item['content'] = $descriptionPretty
|
||||
. '<div><strong>'
|
||||
. $price
|
||||
. '</strong></div><div>'
|
||||
. implode('</div><div>', $offerExtraInfo)
|
||||
. '</div><dl>'
|
||||
. $pricingExtraInfo
|
||||
. '</dl><hr>';
|
||||
|
||||
$this->items[] = $item;
|
||||
}
|
||||
}
|
||||
}
|
107
bridges/AllocineFRBridge.php
Normal file
107
bridges/AllocineFRBridge.php
Normal file
@ -0,0 +1,107 @@
|
||||
<?php
|
||||
|
||||
class AllocineFRBridge extends BridgeAbstract
|
||||
{
|
||||
const MAINTAINER = 'superbaillot.net';
|
||||
const NAME = 'Allo Cine Bridge';
|
||||
const CACHE_TIMEOUT = 25200; // 7h
|
||||
const URI = 'https://www.allocine.fr';
|
||||
const DESCRIPTION = 'Bridge for allocine.fr';
|
||||
const PARAMETERS = [ [
|
||||
'category' => [
|
||||
'name' => 'Emission',
|
||||
'type' => 'list',
|
||||
'title' => 'Sélectionner l\'emission',
|
||||
'values' => [
|
||||
'Faux Raccord' => 'faux-raccord',
|
||||
'Fanzone' => 'fanzone',
|
||||
'Game In Ciné' => 'game-in-cine',
|
||||
'Pour la faire courte' => 'pour-la-faire-courte',
|
||||
'Home Cinéma' => 'home-cinema',
|
||||
'PILS - Par Ici Les Sorties' => 'pils-par-ici-les-sorties',
|
||||
'AlloCiné : l\'émission, sur LeStream' => 'allocine-lemission-sur-lestream',
|
||||
'Give Me Five' => 'give-me-five',
|
||||
'Aviez-vous remarqué ?' => 'aviez-vous-remarque',
|
||||
'Et paf, il est mort' => 'et-paf-il-est-mort',
|
||||
'The Big Fan Theory' => 'the-big-fan-theory',
|
||||
'Clichés' => 'cliches',
|
||||
'Complètement...' => 'completement',
|
||||
'#Fun Facts' => 'fun-facts',
|
||||
'Origin Story' => 'origin-story',
|
||||
]
|
||||
]
|
||||
]];
|
||||
|
||||
public function getURI()
|
||||
{
|
||||
if (!is_null($this->getInput('category'))) {
|
||||
$categories = [
|
||||
'faux-raccord' => '/video/programme-12284/',
|
||||
'fanzone' => '/video/programme-12298/',
|
||||
'game-in-cine' => '/video/programme-12288/',
|
||||
'pour-la-faire-courte' => '/video/programme-20960/',
|
||||
'home-cinema' => '/video/programme-12287/',
|
||||
'pils-par-ici-les-sorties' => '/video/programme-25789/',
|
||||
'allocine-lemission-sur-lestream' => '/video/programme-25123/',
|
||||
'give-me-five' => '/video/programme-21919/saison-34518/',
|
||||
'aviez-vous-remarque' => '/video/programme-19518/',
|
||||
'et-paf-il-est-mort' => '/video/programme-25113/',
|
||||
'the-big-fan-theory' => '/video/programme-20403/',
|
||||
'cliches' => '/video/programme-24834/',
|
||||
'completement' => '/video/programme-23859/',
|
||||
'fun-facts' => '/video/programme-23040/',
|
||||
'origin-story' => '/video/programme-25667/'
|
||||
];
|
||||
|
||||
$category = $this->getInput('category');
|
||||
if (array_key_exists($category, $categories)) {
|
||||
return static::URI . $this->getLastSeasonURI($categories[$category]);
|
||||
} else {
|
||||
returnClientError('Emission inconnue');
|
||||
}
|
||||
}
|
||||
|
||||
return parent::getURI();
|
||||
}
|
||||
|
||||
private function getLastSeasonURI($category)
|
||||
{
|
||||
$html = getSimpleHTMLDOMCached(static::URI . $category, 86400);
|
||||
$seasonLink = $html->find('section[class=section-wrap section]', 0)->find('div[class=cf]', 0)->find('a', 0);
|
||||
$URI = $seasonLink->href;
|
||||
return $URI;
|
||||
}
|
||||
|
||||
public function getName()
|
||||
{
|
||||
if (!is_null($this->getInput('category'))) {
|
||||
return self::NAME . ' : ' . $this->getKey('category');
|
||||
}
|
||||
|
||||
return parent::getName();
|
||||
}
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$html = getSimpleHTMLDOM($this->getURI());
|
||||
|
||||
foreach ($html->find('div[class=gd-col-left]', 0)->find('div[class*=video-card]') as $element) {
|
||||
$item = [];
|
||||
|
||||
$title = $element->find('a[class*=meta-title-link]', 0);
|
||||
$content = trim(defaultLinkTo($element->outertext, static::URI));
|
||||
|
||||
// Replace image 'src' with the one in 'data-src'
|
||||
$content = preg_replace('@src="data:image/gif;base64,[A-Za-z0-9+\/]*"@', '', $content);
|
||||
$content = preg_replace('@data-src=@', 'src=', $content);
|
||||
|
||||
// Remove date in the content to prevent content update while the video is getting older
|
||||
$content = preg_replace('@<div class="meta-sub light">.*<span>[^<]*</span>[^<]*</div>@', '', $content);
|
||||
|
||||
$item['content'] = $content;
|
||||
$item['title'] = trim($title->innertext);
|
||||
$item['uri'] = static::URI . '/' . substr($title->href, 1);
|
||||
$this->items[] = $item;
|
||||
}
|
||||
}
|
||||
}
|
66
bridges/AllocineFRSortiesBridge.php
Normal file
66
bridges/AllocineFRSortiesBridge.php
Normal file
@ -0,0 +1,66 @@
|
||||
<?php
|
||||
|
||||
class AllocineFRSortiesBridge extends BridgeAbstract
|
||||
{
|
||||
const MAINTAINER = 'Simounet';
|
||||
const NAME = 'AlloCiné Sorties Bridge';
|
||||
const CACHE_TIMEOUT = 25200; // 7h
|
||||
const BASE_URI = 'https://www.allocine.fr';
|
||||
const URI = self::BASE_URI . '/film/sorties-semaine/';
|
||||
const DESCRIPTION = 'Bridge for AlloCiné - Sorties cinéma cette semaine';
|
||||
|
||||
public function getName()
|
||||
{
|
||||
return self::NAME;
|
||||
}
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$html = getSimpleHTMLDOM($this->getURI());
|
||||
|
||||
foreach ($html->find('section.section.section-wrap', 0)->find('li.mdl') as $element) {
|
||||
$item = [];
|
||||
|
||||
$thumb = $element->find('figure.thumbnail', 0);
|
||||
$meta = $element->find('div.meta-body', 0);
|
||||
$synopsis = $element->find('div.synopsis', 0);
|
||||
$date = $element->find('span.date', 0);
|
||||
|
||||
$title = $element->find('a[class*=meta-title-link]', 0);
|
||||
$content = trim(defaultLinkTo($thumb->outertext . $meta->outertext . $synopsis->outertext, static::URI));
|
||||
|
||||
// Replace image 'src' with the one in 'data-src'
|
||||
$content = preg_replace('@src="data:image/gif;base64,[A-Za-z0-9=+\/]*"@', '', $content);
|
||||
$content = preg_replace('@data-src=@', 'src=', $content);
|
||||
|
||||
$item['content'] = $content;
|
||||
$item['title'] = trim($title->innertext);
|
||||
$item['timestamp'] = $this->frenchPubDateToTimestamp($date->plaintext);
|
||||
$item['uri'] = static::BASE_URI . '/' . substr($title->href, 1);
|
||||
$this->items[] = $item;
|
||||
}
|
||||
}
|
||||
|
||||
private function frenchPubDateToTimestamp($date)
|
||||
{
|
||||
return strtotime(
|
||||
strtr(
|
||||
strtolower($date),
|
||||
[
|
||||
'janvier' => 'jan',
|
||||
'février' => 'feb',
|
||||
'mars' => 'march',
|
||||
'avril' => 'apr',
|
||||
'mai' => 'may',
|
||||
'juin' => 'jun',
|
||||
'juillet' => 'jul',
|
||||
'août' => 'aug',
|
||||
'septembre' => 'sep',
|
||||
'octobre' => 'oct',
|
||||
'novembre' => 'nov',
|
||||
'décembre' => 'dec'
|
||||
]
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
105
bridges/AmazonBridge.php
Normal file
105
bridges/AmazonBridge.php
Normal file
@ -0,0 +1,105 @@
|
||||
<?php
|
||||
|
||||
class AmazonBridge extends BridgeAbstract
|
||||
{
|
||||
const MAINTAINER = 'Alexis CHEMEL';
|
||||
const NAME = 'Amazon';
|
||||
const URI = 'https://www.amazon.com/';
|
||||
const CACHE_TIMEOUT = 3600; // 1h
|
||||
const DESCRIPTION = 'Returns products from Amazon search';
|
||||
|
||||
const PARAMETERS = [[
|
||||
'q' => [
|
||||
'name' => 'Keyword',
|
||||
'required' => true,
|
||||
'exampleValue' => 'watch',
|
||||
],
|
||||
'sort' => [
|
||||
'name' => 'Sort by',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'Relevance' => 'relevanceblender',
|
||||
'Price: Low to High' => 'price-asc-rank',
|
||||
'Price: High to Low' => 'price-desc-rank',
|
||||
'Average Customer Review' => 'review-rank',
|
||||
'Newest Arrivals' => 'date-desc-rank',
|
||||
],
|
||||
'defaultValue' => 'relevanceblender',
|
||||
],
|
||||
'tld' => [
|
||||
'name' => 'Country',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'Australia' => 'com.au',
|
||||
'Brazil' => 'com.br',
|
||||
'Canada' => 'ca',
|
||||
'China' => 'cn',
|
||||
'France' => 'fr',
|
||||
'Germany' => 'de',
|
||||
'India' => 'in',
|
||||
'Italy' => 'it',
|
||||
'Japan' => 'co.jp',
|
||||
'Mexico' => 'com.mx',
|
||||
'Netherlands' => 'nl',
|
||||
'Poland' => 'pl',
|
||||
'Spain' => 'es',
|
||||
'Sweden' => 'se',
|
||||
'Turkey' => 'com.tr',
|
||||
'United Kingdom' => 'co.uk',
|
||||
'United States' => 'com',
|
||||
],
|
||||
'defaultValue' => 'com',
|
||||
],
|
||||
]];
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$baseUrl = sprintf('https://www.amazon.%s', $this->getInput('tld'));
|
||||
|
||||
$url = sprintf(
|
||||
'%s/s/?field-keywords=%s&sort=%s',
|
||||
$baseUrl,
|
||||
urlencode($this->getInput('q')),
|
||||
$this->getInput('sort')
|
||||
);
|
||||
|
||||
$dom = getSimpleHTMLDOM($url);
|
||||
|
||||
$elements = $dom->find('div.s-result-item');
|
||||
|
||||
foreach ($elements as $element) {
|
||||
$item = [];
|
||||
|
||||
$title = $element->find('h2', 0);
|
||||
if (!$title) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$item['title'] = $title->innertext;
|
||||
|
||||
$itemUrl = $element->find('a', 0)->href;
|
||||
$item['uri'] = urljoin($baseUrl, $itemUrl);
|
||||
|
||||
$image = $element->find('img', 0);
|
||||
if ($image) {
|
||||
$item['content'] = '<img src="' . $image->getAttribute('src') . '" /><br />';
|
||||
}
|
||||
|
||||
$price = $element->find('span.a-price > .a-offscreen', 0);
|
||||
if ($price) {
|
||||
$item['content'] .= $price->innertext;
|
||||
}
|
||||
|
||||
$this->items[] = $item;
|
||||
}
|
||||
}
|
||||
|
||||
public function getName()
|
||||
{
|
||||
if (!is_null($this->getInput('tld')) && !is_null($this->getInput('q'))) {
|
||||
return 'Amazon.' . $this->getInput('tld') . ': ' . $this->getInput('q');
|
||||
}
|
||||
|
||||
return parent::getName();
|
||||
}
|
||||
}
|
253
bridges/AmazonPriceTrackerBridge.php
Normal file
253
bridges/AmazonPriceTrackerBridge.php
Normal file
@ -0,0 +1,253 @@
|
||||
<?php
|
||||
|
||||
class AmazonPriceTrackerBridge extends BridgeAbstract
|
||||
{
|
||||
const MAINTAINER = 'captn3m0, sal0max';
|
||||
const NAME = 'Amazon Price Tracker';
|
||||
const URI = 'https://www.amazon.com/';
|
||||
const CACHE_TIMEOUT = 3600; // 1h
|
||||
const DESCRIPTION = 'Tracks price for a single product on Amazon';
|
||||
|
||||
const PARAMETERS = [
|
||||
[
|
||||
'asin' => [
|
||||
'name' => 'ASIN',
|
||||
'required' => true,
|
||||
'exampleValue' => 'B071GB1VMQ',
|
||||
// https://stackoverflow.com/a/12827734
|
||||
'pattern' => 'B[\dA-Z]{9}|\d{9}(X|\d)',
|
||||
],
|
||||
'tld' => [
|
||||
'name' => 'Country',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'Australia' => 'com.au',
|
||||
'Brazil' => 'com.br',
|
||||
'Canada' => 'ca',
|
||||
'China' => 'cn',
|
||||
'France' => 'fr',
|
||||
'Germany' => 'de',
|
||||
'India' => 'in',
|
||||
'Italy' => 'it',
|
||||
'Japan' => 'co.jp',
|
||||
'Mexico' => 'com.mx',
|
||||
'Netherlands' => 'nl',
|
||||
'Poland' => 'pl',
|
||||
'Spain' => 'es',
|
||||
'Sweden' => 'se',
|
||||
'Turkey' => 'com.tr',
|
||||
'United Kingdom' => 'co.uk',
|
||||
'United States' => 'com',
|
||||
],
|
||||
'defaultValue' => 'com',
|
||||
],
|
||||
]];
|
||||
|
||||
const PRICE_SELECTORS = [
|
||||
'#priceblock_ourprice',
|
||||
'.priceBlockBuyingPriceString',
|
||||
'#newBuyBoxPrice',
|
||||
'#tp_price_block_total_price_ww',
|
||||
'span.offer-price',
|
||||
'.a-color-price',
|
||||
];
|
||||
|
||||
const WHITESPACE = " \t\n\r\0\x0B\xC2\xA0";
|
||||
|
||||
protected $title;
|
||||
|
||||
/**
|
||||
* Generates domain name given a amazon TLD
|
||||
*/
|
||||
private function getDomainName()
|
||||
{
|
||||
return 'https://www.amazon.' . $this->getInput('tld');
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates URI for a Amazon product page
|
||||
*/
|
||||
public function getURI()
|
||||
{
|
||||
if (!is_null($this->getInput('asin'))) {
|
||||
return $this->getDomainName() . '/dp/' . $this->getInput('asin');
|
||||
}
|
||||
return parent::getURI();
|
||||
}
|
||||
|
||||
/**
|
||||
* Scrapes the product title from the html page
|
||||
* returns the default title if scraping fails
|
||||
*/
|
||||
private function getTitle($html)
|
||||
{
|
||||
$titleTag = $html->find('#productTitle', 0);
|
||||
|
||||
if (!$titleTag) {
|
||||
return $this->getDefaultTitle();
|
||||
} else {
|
||||
return trim(html_entity_decode($titleTag->innertext, ENT_QUOTES));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Title used by the feed if none could be found
|
||||
*/
|
||||
private function getDefaultTitle()
|
||||
{
|
||||
return 'Amazon.' . $this->getInput('tld') . ': ' . $this->getInput('asin');
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns name for the feed
|
||||
* Uses title (already scraped) if it has one
|
||||
*/
|
||||
public function getName()
|
||||
{
|
||||
if (isset($this->title)) {
|
||||
return $this->title;
|
||||
} else {
|
||||
return parent::getName();
|
||||
}
|
||||
}
|
||||
|
||||
private function parseDynamicImage($attribute)
|
||||
{
|
||||
$json = json_decode(html_entity_decode($attribute), true);
|
||||
|
||||
if ($json and count($json) > 0) {
|
||||
return array_keys($json)[0];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a generated image tag for the product
|
||||
*/
|
||||
private function getImage($html)
|
||||
{
|
||||
$image = 'https://placekitten.com/200/300';
|
||||
$imageSrc = $html->find('#main-image-container img', 0);
|
||||
if ($imageSrc) {
|
||||
$hiresImage = $imageSrc->getAttribute('data-old-hires');
|
||||
$dynamicImageAttribute = $imageSrc->getAttribute('data-a-dynamic-image');
|
||||
$image = $hiresImage ?: $this->parseDynamicImage($dynamicImageAttribute);
|
||||
}
|
||||
|
||||
return <<<EOT
|
||||
<img width="300" style="max-width:300;max-height:300" src="$image" alt="{$this->title}" />
|
||||
EOT;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return \simple_html_dom object
|
||||
* for the entire html of the product page
|
||||
*/
|
||||
private function getHtml()
|
||||
{
|
||||
$uri = $this->getURI();
|
||||
|
||||
return getSimpleHTMLDOM($uri);
|
||||
}
|
||||
|
||||
private function scrapePriceFromMetrics($html)
|
||||
{
|
||||
$asinData = $html->find('#cerberus-data-metrics', 0);
|
||||
|
||||
// <div id="cerberus-data-metrics" style="display: none;"
|
||||
// data-asin="B00WTHJ5SU" data-asin-price="14.99" data-asin-shipping="0"
|
||||
// data-asin-currency-code="USD" data-substitute-count="-1" ... />
|
||||
if ($asinData) {
|
||||
return [
|
||||
'price' => $asinData->getAttribute('data-asin-price'),
|
||||
'currency' => $asinData->getAttribute('data-asin-currency-code'),
|
||||
'shipping' => $asinData->getAttribute('data-asin-shipping')
|
||||
];
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private function scrapePriceTwister($html)
|
||||
{
|
||||
$str = $html->find('.twister-plus-buying-options-price-data', 0);
|
||||
|
||||
$data = json_decode($str->innertext, true);
|
||||
if (count($data) === 1) {
|
||||
$data = $data[0];
|
||||
return [
|
||||
'displayPrice' => $data['displayPrice'],
|
||||
'currency' => $data['currency'],
|
||||
'shipping' => '0',
|
||||
];
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private function scrapePriceGeneric($html)
|
||||
{
|
||||
$default = [
|
||||
'price' => null,
|
||||
'displayPrice' => null,
|
||||
'currency' => null,
|
||||
'shipping' => null,
|
||||
];
|
||||
$priceDiv = null;
|
||||
|
||||
foreach (self::PRICE_SELECTORS as $sel) {
|
||||
$priceDiv = $html->find($sel, 0);
|
||||
if ($priceDiv) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!$priceDiv) {
|
||||
return $default;
|
||||
}
|
||||
|
||||
$priceString = str_replace(str_split(self::WHITESPACE), '', $priceDiv->plaintext);
|
||||
preg_match('/(\d+\.\d{0,2})/', $priceString, $matches);
|
||||
|
||||
$price = $matches[0] ?? null;
|
||||
$currency = str_replace($price, '', $priceString);
|
||||
|
||||
if ($price != null && $currency != null) {
|
||||
return [
|
||||
'price' => $price,
|
||||
'displayPrice' => null,
|
||||
'currency' => $currency,
|
||||
'shipping' => '0'
|
||||
];
|
||||
}
|
||||
return $default;
|
||||
}
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$html = $this->getHtml();
|
||||
$this->title = $this->getTitle($html);
|
||||
$image = $this->getImage($html);
|
||||
$data = $this->scrapePriceGeneric($html);
|
||||
|
||||
// render
|
||||
$content = '';
|
||||
$price = $data['displayPrice'];
|
||||
if (!$price) {
|
||||
$price = sprintf('%s %s', $data['price'], $data['currency']);
|
||||
}
|
||||
$content .= sprintf('%s<br>Price: %s', $image, $price);
|
||||
if ($data['shipping'] !== '0') {
|
||||
$content .= sprintf('<br>Shipping: %s %s</br>', $data['shipping'], $data['currency']);
|
||||
}
|
||||
|
||||
$item = [
|
||||
'title' => $this->title,
|
||||
'uri' => $this->getURI(),
|
||||
'content' => $content,
|
||||
// This is to ensure that feed readers notice the price change
|
||||
'uid' => md5($data['price'])
|
||||
];
|
||||
|
||||
$this->items[] = $item;
|
||||
}
|
||||
}
|
278
bridges/AnfrBridge.php
Normal file
278
bridges/AnfrBridge.php
Normal file
@ -0,0 +1,278 @@
|
||||
<?php
|
||||
|
||||
class AnfrBridge extends BridgeAbstract
|
||||
{
|
||||
const NAME = 'ANFR';
|
||||
const URI = 'https://data.anfr.fr/';
|
||||
const DESCRIPTION = 'Fetches data from the French administration "Agence Nationale des Fréquences".';
|
||||
const CACHE_TIMEOUT = 604800; // 7d
|
||||
const MAINTAINER = 'quent1';
|
||||
const PARAMETERS = [
|
||||
'Données sur les réseaux mobiles' => [
|
||||
'departement' => [
|
||||
'name' => 'Département',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'Tous' => null,
|
||||
'Ain' => '001',
|
||||
'Aisne' => '002',
|
||||
'Allier' => '003',
|
||||
'Alpes-de-Haute-Provence' => '004',
|
||||
'Hautes-Alpes' => '005',
|
||||
'Alpes-Maritimes' => '006',
|
||||
'Ardèche' => '007',
|
||||
'Ardennes' => '008',
|
||||
'Ariège' => '009',
|
||||
'Aube' => '010',
|
||||
'Aude' => '011',
|
||||
'Aveyron' => '012',
|
||||
'Bouches-du-Rhône' => '013',
|
||||
'Calvados' => '014',
|
||||
'Cantal' => '015',
|
||||
'Charente' => '016',
|
||||
'Charente-Maritime' => '017',
|
||||
'Cher' => '018',
|
||||
'Corrèze' => '019',
|
||||
'Corse-du-Sud' => '02A',
|
||||
'Haute-Corse' => '02B',
|
||||
'Côte-d\'Or' => '021',
|
||||
'Côtes-d\'Armor' => '022',
|
||||
'Creuse' => '023',
|
||||
'Dordogne' => '024',
|
||||
'Doubs' => '025',
|
||||
'Drôme' => '026',
|
||||
'Eure' => '027',
|
||||
'Eure-et-Loir' => '028',
|
||||
'Finistère' => '029',
|
||||
'Gard' => '030',
|
||||
'Haute-Garonne' => '031',
|
||||
'Gers' => '032',
|
||||
'Gironde' => '033',
|
||||
'Hérault' => '034',
|
||||
'Ille-et-Vilaine' => '035',
|
||||
'Indre' => '036',
|
||||
'Indre-et-Loire' => '037',
|
||||
'Isère' => '038',
|
||||
'Jura' => '039',
|
||||
'Landes' => '040',
|
||||
'Loir-et-Cher' => '041',
|
||||
'Loire' => '042',
|
||||
'Haute-Loire' => '043',
|
||||
'Loire-Atlantique' => '044',
|
||||
'Loiret' => '045',
|
||||
'Lot' => '046',
|
||||
'Lot-et-Garonne' => '047',
|
||||
'Lozère' => '048',
|
||||
'Maine-et-Loire' => '049',
|
||||
'Manche' => '050',
|
||||
'Marne' => '051',
|
||||
'Haute-Marne' => '052',
|
||||
'Mayenne' => '053',
|
||||
'Meurthe-et-Moselle' => '054',
|
||||
'Meuse' => '055',
|
||||
'Morbihan' => '056',
|
||||
'Moselle' => '057',
|
||||
'Nièvre' => '058',
|
||||
'Nord' => '059',
|
||||
'Oise' => '060',
|
||||
'Orne' => '061',
|
||||
'Pas-de-Calais' => '062',
|
||||
'Puy-de-Dôme' => '063',
|
||||
'Pyrénées-Atlantiques' => '064',
|
||||
'Hautes-Pyrénées' => '065',
|
||||
'Pyrénées-Orientales' => '066',
|
||||
'Bas-Rhin' => '067',
|
||||
'Haut-Rhin' => '068',
|
||||
'Rhône' => '069',
|
||||
'Haute-Saône' => '070',
|
||||
'Saône-et-Loire' => '071',
|
||||
'Sarthe' => '072',
|
||||
'Savoie' => '073',
|
||||
'Haute-Savoie' => '074',
|
||||
'Paris' => '075',
|
||||
'Seine-Maritime' => '076',
|
||||
'Seine-et-Marne' => '077',
|
||||
'Yvelines' => '078',
|
||||
'Deux-Sèvres' => '079',
|
||||
'Somme' => '080',
|
||||
'Tarn' => '081',
|
||||
'Tarn-et-Garonne' => '082',
|
||||
'Var' => '083',
|
||||
'Vaucluse' => '084',
|
||||
'Vendée' => '085',
|
||||
'Vienne' => '086',
|
||||
'Haute-Vienne' => '087',
|
||||
'Vosges' => '088',
|
||||
'Yonne' => '089',
|
||||
'Territoire de Belfort' => '090',
|
||||
'Essonne' => '091',
|
||||
'Hauts-de-Seine' => '092',
|
||||
'Seine-Saint-Denis' => '093',
|
||||
'Val-de-Marne' => '094',
|
||||
'Val-d\'Oise' => '095',
|
||||
'Guadeloupe' => '971',
|
||||
'Martinique' => '972',
|
||||
'Guyane' => '973',
|
||||
'La Réunion' => '974',
|
||||
'Saint-Pierre-et-Miquelon' => '975',
|
||||
'Mayotte' => '976',
|
||||
'Saint-Barthélemy' => '977',
|
||||
'Saint-Martin' => '978',
|
||||
'Terres australes et antarctiques françaises' => '984',
|
||||
'Wallis-et-Futuna' => '986',
|
||||
'Polynésie française' => '987',
|
||||
'Nouvelle-Calédonie' => '988',
|
||||
'Île de Clipperton' => '989'
|
||||
]
|
||||
],
|
||||
'generation' => [
|
||||
'name' => 'Génération',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'Tous' => null,
|
||||
'2G' => '2G',
|
||||
'3G' => '3G',
|
||||
'4G' => '4G',
|
||||
'5G' => '5G',
|
||||
]
|
||||
],
|
||||
'operateur' => [
|
||||
'name' => 'Opérateur',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'Tous' => null,
|
||||
'Bouygues Télécom' => 'BOUYGUES TELECOM',
|
||||
'Dauphin Télécom' => 'DAUPHIN TELECOM',
|
||||
'Digiciel' => 'DIGICEL',
|
||||
'Free Caraïbes' => 'FREE CARAIBES',
|
||||
'Free Mobile' => 'FREE MOBILE',
|
||||
'GLOBALTEL' => 'GLOBALTEL',
|
||||
'Office des postes et télécommunications de Nouvelle Calédonie' => 'Gouv Nelle Calédonie (OPT)',
|
||||
'Maore Mobile' => 'MAORE MOBILE',
|
||||
'ONATi' => 'ONATI',
|
||||
'Orange' => 'ORANGE',
|
||||
'Outremer Telecom' => 'OUTREMER TELECOM',
|
||||
'Vodafone polynésie' => 'PMT/VODAPHONE',
|
||||
'SFR' => 'SFR',
|
||||
'SPM Télécom' => 'SPM TELECOM',
|
||||
'Service des Postes et Télécommunications de Polynésie Française' => 'Gouv Nelle Calédonie (OPT)',
|
||||
'SRR' => 'SRR',
|
||||
'Station étrangère' => 'Station étrangère',
|
||||
'Telco OI' => 'TELCO IO',
|
||||
'United Telecommunication Services Caraïbes' => 'UTS Caraibes',
|
||||
'Ora Mobile' => 'VITI SAS',
|
||||
'Zeop' => 'ZEOP'
|
||||
]
|
||||
],
|
||||
'statut' => [
|
||||
'name' => 'Statut',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'Tous' => null,
|
||||
'En service' => 'En service',
|
||||
'Projet approuvé' => 'Projet approuvé',
|
||||
'Techniquement opérationnel' => 'Techniquement opérationnel',
|
||||
]
|
||||
]
|
||||
]
|
||||
];
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
$urlParts = [
|
||||
'id' => 'observatoire_2g_3g_4g',
|
||||
'resource_id' => '88ef0887-6b0f-4d3f-8545-6d64c8f597da',
|
||||
'fields' => 'id,adm_lb_nom,sta_nm_dpt,emr_lb_systeme,generation,date_maj,sta_nm_anfr,adr_lb_lieu,adr_lb_add1,adr_lb_add2,adr_lb_add3,adr_nm_cp,statut',
|
||||
'rows' => 10000
|
||||
];
|
||||
|
||||
if (!empty($this->getInput('departement'))) {
|
||||
$urlParts['refine.sta_nm_dpt'] = urlencode($this->getInput('departement'));
|
||||
}
|
||||
|
||||
if (!empty($this->getInput('generation'))) {
|
||||
$urlParts['refine.generation'] = $this->getInput('generation');
|
||||
}
|
||||
|
||||
if (!empty($this->getInput('operateur'))) {
|
||||
// http_build_query() already does urlencoding so this call is redundant
|
||||
$urlParts['refine.adm_lb_nom'] = urlencode($this->getInput('operateur'));
|
||||
}
|
||||
|
||||
if (!empty($this->getInput('statut'))) {
|
||||
$urlParts['refine.statut'] = urlencode($this->getInput('statut'));
|
||||
}
|
||||
|
||||
// API seems to not play well with urlencoded data
|
||||
$url = urljoin(static::URI, '/d4c/api/records/1.0/download/?' . urldecode(http_build_query($urlParts)));
|
||||
|
||||
$json = getContents($url);
|
||||
$data = Json::decode($json, false);
|
||||
$records = $data->records;
|
||||
$frequenciesByStation = [];
|
||||
foreach ($records as $record) {
|
||||
if (!isset($frequenciesByStation[$record->fields->sta_nm_anfr])) {
|
||||
$street = sprintf(
|
||||
'%s %s %s',
|
||||
$record->fields->adr_lb_add1 ?? '',
|
||||
$record->fields->adr_lb_add2 ?? '',
|
||||
$record->fields->adr_lb_add3 ?? ''
|
||||
);
|
||||
$frequenciesByStation[$record->fields->sta_nm_anfr] = [
|
||||
'id' => $record->fields->sta_nm_anfr,
|
||||
'operator' => $record->fields->adm_lb_nom,
|
||||
'frequencies' => [],
|
||||
'lastUpdate' => 0,
|
||||
'address' => [
|
||||
'street' => trim($street),
|
||||
'postCode' => $record->fields->adr_nm_cp,
|
||||
'city' => $record->fields->adr_lb_lieu
|
||||
]
|
||||
];
|
||||
}
|
||||
|
||||
$frequenciesByStation[$record->fields->sta_nm_anfr]['frequencies'][] = [
|
||||
'generation' => $record->fields->generation,
|
||||
'frequency' => $record->fields->emr_lb_systeme,
|
||||
'status' => $record->fields->statut,
|
||||
'updatedAt' => strtotime($record->fields->date_maj),
|
||||
];
|
||||
|
||||
$frequenciesByStation[$record->fields->sta_nm_anfr]['lastUpdate'] = max(
|
||||
$frequenciesByStation[$record->fields->sta_nm_anfr]['lastUpdate'],
|
||||
strtotime($record->fields->date_maj)
|
||||
);
|
||||
}
|
||||
|
||||
usort($frequenciesByStation, static fn ($a, $b) => $b['lastUpdate'] <=> $a['lastUpdate']);
|
||||
|
||||
foreach ($frequenciesByStation as $station) {
|
||||
$title = sprintf(
|
||||
'[%s] Mise à jour de la station n°%s à %s (%s)',
|
||||
$station['operator'],
|
||||
$station['id'],
|
||||
$station['address']['city'],
|
||||
$station['address']['postCode']
|
||||
);
|
||||
|
||||
$array_reduce = array_reduce($station['frequencies'], static function ($carry, $frequency) {
|
||||
return sprintf('%s<li>%s : %s</li>', $carry, $frequency['frequency'], $frequency['status']);
|
||||
}, '');
|
||||
|
||||
$content = sprintf(
|
||||
'<h1>Adresse complète</h1><p>%s<br>%s<br>%s</p><h1>Fréquences</h1><p><ul>%s</ul></p>',
|
||||
$station['address']['street'],
|
||||
$station['address']['postCode'],
|
||||
$station['address']['city'],
|
||||
$array_reduce
|
||||
);
|
||||
|
||||
$this->items[] = [
|
||||
'uid' => $station['id'],
|
||||
'timestamp' => $station['lastUpdate'],
|
||||
'title' => $title,
|
||||
'content' => $content,
|
||||
];
|
||||
}
|
||||
}
|
||||
}
|
218
bridges/AnidexBridge.php
Normal file
218
bridges/AnidexBridge.php
Normal file
@ -0,0 +1,218 @@
|
||||
<?php
|
||||
|
||||
class AnidexBridge extends BridgeAbstract
|
||||
{
|
||||
const MAINTAINER = 'ORelio';
|
||||
const NAME = 'Anidex';
|
||||
const URI = 'http://anidex.info/'; // anidex.info has ddos-guard so we need to use anidex.moe
|
||||
const ALTERNATE_URI = 'https://anidex.moe/'; // anidex.moe returns 301 unless Host is set to anidex.info
|
||||
const ALTERNATE_HOST = 'anidex.info'; // Correct host for requesting anidex.moe without 301 redirect
|
||||
const DESCRIPTION = 'Returns the newest torrents, with optional search criteria.';
|
||||
const PARAMETERS = [
|
||||
[
|
||||
'id' => [
|
||||
'name' => 'Category',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'All categories' => '0',
|
||||
'Anime' => '1,2,3',
|
||||
'Anime - Sub' => '1',
|
||||
'Anime - Raw' => '2',
|
||||
'Anime - Dub' => '3',
|
||||
'Live Action' => '4,5',
|
||||
'Live Action - Sub' => '4',
|
||||
'Live Action - Raw' => '5',
|
||||
'Light Novel' => '6',
|
||||
'Manga' => '7,8',
|
||||
'Manga - Translated' => '7',
|
||||
'Manga - Raw' => '8',
|
||||
'Music' => '9,10,11',
|
||||
'Music - Lossy' => '9',
|
||||
'Music - Lossless' => '10',
|
||||
'Music - Video' => '11',
|
||||
'Games' => '12',
|
||||
'Applications' => '13',
|
||||
'Pictures' => '14',
|
||||
'Adult Video' => '15',
|
||||
'Other' => '16'
|
||||
]
|
||||
],
|
||||
'lang_id' => [
|
||||
'name' => 'Language',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'All languages' => '0',
|
||||
'English' => '1',
|
||||
'Japanese' => '2',
|
||||
'Polish' => '3',
|
||||
'Serbo-Croatian' => '4',
|
||||
'Dutch' => '5',
|
||||
'Italian' => '6',
|
||||
'Russian' => '7',
|
||||
'German' => '8',
|
||||
'Hungarian' => '9',
|
||||
'French' => '10',
|
||||
'Finnish' => '11',
|
||||
'Vietnamese' => '12',
|
||||
'Greek' => '13',
|
||||
'Bulgarian' => '14',
|
||||
'Spanish (Spain)' => '15',
|
||||
'Portuguese (Brazil)' => '16',
|
||||
'Portuguese (Portugal)' => '17',
|
||||
'Swedish' => '18',
|
||||
'Arabic' => '19',
|
||||
'Danish' => '20',
|
||||
'Chinese (Simplified)' => '21',
|
||||
'Bengali' => '22',
|
||||
'Romanian' => '23',
|
||||
'Czech' => '24',
|
||||
'Mongolian' => '25',
|
||||
'Turkish' => '26',
|
||||
'Indonesian' => '27',
|
||||
'Korean' => '28',
|
||||
'Spanish (LATAM)' => '29',
|
||||
'Persian' => '30',
|
||||
'Malaysian' => '31'
|
||||
]
|
||||
],
|
||||
'group_id' => [
|
||||
'name' => 'Group ID',
|
||||
'type' => 'number'
|
||||
],
|
||||
'r' => [
|
||||
'name' => 'Hide Remakes',
|
||||
'type' => 'checkbox'
|
||||
],
|
||||
'b' => [
|
||||
'name' => 'Only Batches',
|
||||
'type' => 'checkbox'
|
||||
],
|
||||
'a' => [
|
||||
'name' => 'Only Authorized',
|
||||
'type' => 'checkbox'
|
||||
],
|
||||
'q' => [
|
||||
'name' => 'Keyword',
|
||||
'description' => 'Keyword(s)',
|
||||
'type' => 'text'
|
||||
],
|
||||
'h' => [
|
||||
'name' => 'Adult content',
|
||||
'type' => 'list',
|
||||
'values' => [
|
||||
'No filter' => '0',
|
||||
'Hide +18' => '1',
|
||||
'Only +18' => '2'
|
||||
]
|
||||
]
|
||||
]
|
||||
];
|
||||
|
||||
public function collectData()
|
||||
{
|
||||
// Build Search URL from user-provided parameters
|
||||
$search_url = self::ALTERNATE_URI . '?s=upload_timestamp&o=desc';
|
||||
foreach (['id', 'lang_id', 'group_id'] as $param_name) {
|
||||
$param = $this->getInput($param_name);
|
||||
if (!empty($param) && intval($param) != 0 && ctype_digit(str_replace(',', '', $param))) {
|
||||
$search_url .= '&' . $param_name . '=' . $param;
|
||||
}
|
||||
}
|
||||
foreach (['r', 'b', 'a'] as $param_name) {
|
||||
$param = $this->getInput($param_name);
|
||||
if (!empty($param) && boolval($param)) {
|
||||
$search_url .= '&' . $param_name . '=1';
|
||||
}
|
||||
}
|
||||
$query = $this->getInput('q');
|
||||
if (!empty($query)) {
|
||||
$search_url .= '&q=' . urlencode($query);
|
||||
}
|
||||
$opt = [];
|
||||
$h = $this->getInput('h');
|
||||
if (!empty($h) && intval($h) != 0 && ctype_digit($h)) {
|
||||
$opt[CURLOPT_COOKIE] = 'anidex_h_toggle=' . $h;
|
||||
}
|
||||
|
||||
// We need to use a different Host HTTP header to reach the correct page on ALTERNATE_URI
|
||||
$headers = ['Host: ' . self::ALTERNATE_HOST];
|
||||
|
||||
// The HTTPS certificate presented by anidex.moe is for anidex.info. We need to ignore this.
|
||||
// As a consequence, the bridge is intentionally marked as insecure by setting self::URI to http://
|
||||
$opt[CURLOPT_SSL_VERIFYHOST] = 0;
|
||||
$opt[CURLOPT_SSL_VERIFYPEER] = 0;
|
||||
|
||||
// Retrieve torrent listing from search results, which does not contain torrent description
|
||||
$html = getSimpleHTMLDOM($search_url, $headers, $opt);
|
||||
$links = $html->find('a');
|
||||
$results = [];
|
||||
foreach ($links as $link) {
|
||||
if (strpos($link->href, '/torrent/') === 0 && !in_array($link->href, $results)) {
|
||||
$results[] = $link->href;
|
||||
}
|
||||
}
|
||||
if (empty($results) && empty($this->getInput('q'))) {
|
||||
returnServerError('No results from Anidex: ' . $search_url);
|
||||
}
|
||||
|
||||
//Process each item individually
|
||||
foreach ($results as $element) {
|
||||
//Limit total amount of requests
|
||||
if (count($this->items) >= 20) {
|
||||
break;
|
||||
}
|
||||
|
||||
$torrent_id = str_replace('/torrent/', '', $element);
|
||||
|
||||
//Ignore entries without valid torrent ID
|
||||
if ($torrent_id != 0 && ctype_digit($torrent_id)) {
|
||||
//Retrieve data for this torrent ID
|
||||
$item_browse_uri = self::URI . 'torrent/' . $torrent_id;
|
||||
$item_fetch_uri = self::ALTERNATE_URI . 'torrent/' . $torrent_id;
|
||||
|
||||
//Retrieve full description from torrent page (cached for 24 hours: 86400 seconds)
|
||||
if ($item_html = getSimpleHTMLDOMCached($item_fetch_uri, 86400, $headers, $opt)) {
|
||||
//Retrieve data from page contents
|
||||
$item_title = str_replace(' (Torrent) - AniDex ', '', $item_html->find('title', 0)->plaintext);
|
||||
$item_desc = $item_html->find('div.panel-body', 0);
|
||||
$item_author = trim($item_html->find('span.fa-user', 0)->parent()->plaintext);
|
||||
$item_date = strtotime(trim($item_html->find('span.fa-clock', 0)->parent()->plaintext));
|
||||
$item_image = $this->getURI() . 'images/user_logos/default.png';
|
||||
|
||||
//Check for description-less torrent andn optionally extract image
|
||||
$desc_title_found = false;
|
||||
foreach ($item_html->find('h3.panel-title') as $h3) {
|
||||
if (strpos($h3, 'Description') !== false) {
|
||||
$desc_title_found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if ($desc_title_found) {
|
||||
//Retrieve image for thumbnail or generic logo fallback
|
||||
foreach ($item_desc->find('img') as $img) {
|
||||
if (strpos($img->src, 'prez') === false) {
|
||||
$item_image = $img->src;
|
||||
break;
|
||||
}
|
||||
}
|
||||
$item_desc = trim($item_desc->innertext);
|
||||
} else {
|
||||
$item_desc = '<em>No description.</em>';
|
||||
}
|
||||
|
||||
//Build and add final item
|
||||
$item = [];
|
||||
$item['uri'] = $item_browse_uri;
|
||||
$item['title'] = $item_title;
|
||||
$item['author'] = $item_author;
|
||||
$item['timestamp'] = $item_date;
|
||||
$item['enclosures'] = [$item_image];
|
||||
$item['content'] = $item_desc;
|
||||
$this->items[] = $item;
|
||||
}
|
||||
}
|
||||
$element = null;
|
||||
}
|
||||
$results = null;
|
||||
}
|
||||
}
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user