Added webpage
This commit is contained in:
parent
74dde9116e
commit
032351e8f1
3
.gitignore
vendored
3
.gitignore
vendored
@ -168,3 +168,6 @@ temp/
|
||||
Network Trash Folder
|
||||
Temporary Items
|
||||
.apdisk
|
||||
|
||||
*.fiber.gz
|
||||
.vscode
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
FROM golang:alpine AS builder
|
||||
FROM golang:1.16-alpine AS builder
|
||||
RUN apk update && apk add --no-cache git && apk add -U --no-cache ca-certificates
|
||||
WORKDIR /app/
|
||||
ADD go.mod go.sum ./
|
||||
|
||||
3
go.mod
3
go.mod
@ -1,8 +1,9 @@
|
||||
module feed-fetcher
|
||||
|
||||
go 1.15
|
||||
go 1.16
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.5.1
|
||||
github.com/gofiber/fiber/v2 v2.6.0
|
||||
github.com/mmcdole/gofeed v1.1.0
|
||||
)
|
||||
|
||||
124
index.html
Normal file
124
index.html
Normal file
@ -0,0 +1,124 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>feed-fetcher</title>
|
||||
|
||||
<link rel="icon" type="image/svg+xml" href="data:image/svg+xml, %3Csvg xmlns='http://www.w3.org/2000/svg' xmlns:xlink='http://www.w3.org/1999/xlink' viewBox='0 0 512 512' style='height:50px;'%3E%3Ccircle cx='256' cy='256' r='247.9' fill='%2371cad1'/%3E%3Cg%3E%3Cpath d='M159 397a44 44 0 110-88 44 44 0 010 88z' fill='%23ef71a8'/%3E%3Cpath d='M250 384v-10c0-62-50-112-112-112h-10l-2-2v-56l2-2h10c95 0 172 77 172 172v10l-2 2h-56l-2-2z' fill='%23ef71a8'/%3E%3Cpath d='M358 384v-10a220 220 0 00-220-220h-10l-2-2V96l2-2h12c74 0 144 29 197 81a276 276 0 0181 197v12l-2 2h-56l-2-2z' fill='%23ef71a8'/%3E%3C/g%3E%3Cpath d='M437 75a254 254 0 00-362 0 255 255 0 000 362 255 255 0 00362 0 255 255 0 000-362zM256 496a240 240 0 111-480 240 240 0 01-1 480z' fill='%2327162a'/%3E%3C/svg%3E">
|
||||
|
||||
|
||||
<meta property="og:title" content="feed-fetcher">
|
||||
<meta property="og:site_name" content="feed-fetcher">
|
||||
<meta property="og:url" content="https://feed-fetcher.cluster.fun">
|
||||
<meta property="og:description" content="Returns the RSS feed associated with the given URL">
|
||||
<meta property="og:type" content="website">
|
||||
<meta property="og:image" content="data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB2aWV3Qm94PSItNTAwIC0xMDAgMTYxMiA3MTIiIHdpZHRoPSIxMDAwIiBoZWlnaHQ9IjEwMDAiPjxjaXJjbGUgY3g9IjI1NiIgY3k9IjI1NiIgcj0iMjQ3LjkiIGZpbGw9IiM3MWNhZDEiLz48Zz48cGF0aCBkPSJNMTU5IDM5N2E0NCA0NCAwIDExMC04OCA0NCA0NCAwIDAxMCA4OHoiIGZpbGw9IiNlZjcxYTgiLz48cGF0aCBkPSJNMjUwIDM4NHYtMTBjMC02Mi01MC0xMTItMTEyLTExMmgtMTBsLTItMnYtNTZsMi0yaDEwYzk1IDAgMTcyIDc3IDE3MiAxNzJ2MTBsLTIgMmgtNTZsLTItMnoiIGZpbGw9IiNlZjcxYTgiLz48cGF0aCBkPSJNMzU4IDM4NHYtMTBhMjIwIDIyMCAwIDAwLTIyMC0yMjBoLTEwbC0yLTJWOTZsMi0yaDEyYzc0IDAgMTQ0IDI5IDE5NyA4MWEyNzYgMjc2IDAgMDE4MSAxOTd2MTJsLTIgMmgtNTZsLTItMnoiIGZpbGw9IiNlZjcxYTgiLz48L2c+PHBhdGggZD0iTTQzNyA3NWEyNTQgMjU0IDAgMDAtMzYyIDAgMjU1IDI1NSAwIDAwMCAzNjIgMjU1IDI1NSAwIDAwMzYyIDAgMjU1IDI1NSAwIDAwMC0zNjJ6TTI1NiA0OTZhMjQwIDI0MCAwIDExMS00ODAgMjQwIDI0MCAwIDAxLTEgNDgweiIgZmlsbD0iIzI3MTYyYSIvPjwvc3ZnPgo=">
|
||||
<meta name="twitter:card" content="summary" />
|
||||
<meta name="twitter:creator" content="@Marcus_Noble_" />
|
||||
|
||||
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300italic,700,700italic">
|
||||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/normalize/5.0.0/normalize.css">
|
||||
<link rel="stylesheet" href="https://cdn.githubraw.com/AverageMarcus/milligram/fb7f0444/dist/milligram.min.css">
|
||||
<style>
|
||||
body {
|
||||
min-height: 100vh;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
justify-content: space-between;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<h1 class="heading-fancy">
|
||||
feed-fetcher
|
||||
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 512 512" style="height:50px;"><circle cx="256" cy="256" r="247.9" fill="#71cad1"/><g><path d="M159 397a44 44 0 110-88 44 44 0 010 88z" fill="#ef71a8"/><path d="M250 384v-10c0-62-50-112-112-112h-10l-2-2v-56l2-2h10c95 0 172 77 172 172v10l-2 2h-56l-2-2z" fill="#ef71a8"/><path d="M358 384v-10a220 220 0 00-220-220h-10l-2-2V96l2-2h12c74 0 144 29 197 81a276 276 0 0181 197v12l-2 2h-56l-2-2z" fill="#ef71a8"/></g><path d="M437 75a254 254 0 00-362 0 255 255 0 000 362 255 255 0 00362 0 255 255 0 000-362zM256 496a240 240 0 111-480 240 240 0 01-1 480z" fill="#27162a"/></svg>
|
||||
</h1>
|
||||
<blockquote>Returns the RSS feed associated with the given URL</blockquote>
|
||||
|
||||
<p>
|
||||
Enter an URL and press "Fetch Feed" to see all feeds (if any) found associated with that page.
|
||||
</p>
|
||||
|
||||
<div class="row">
|
||||
<form id="fetchForm" class="column column-80 column-offset-10" method="GET" action="/">
|
||||
<fieldset>
|
||||
<label for="url">URL of webpage</label>
|
||||
<input type="text" placeholder="e.g. https://marcusnoble.co.uk" name="url" id="url">
|
||||
<input id="fetchFeed" class="button-primary button-outline float-right" type="submit" value="Fetch Feed">
|
||||
</fieldset>
|
||||
</form>
|
||||
</div>
|
||||
<div class="row">
|
||||
<div id="results" class="column column-80 column-offset-10 text-center"></div>
|
||||
</div>
|
||||
|
||||
<hr>
|
||||
|
||||
<p>
|
||||
<em>Alternatively</em>, you can navigate to <code>https://feed-fetcher.cluster.fun/?url=YOUR_URL_HERE</code> and your browser will redirect to the associated feed URL if found.
|
||||
</p>
|
||||
<p>
|
||||
<h4>Calling as an API:</h4>
|
||||
If you set the <code>Content-Type</code> request header to <code>application/json</code> the response will return as a JSON array of all found feed URLs. If no feeds are found an empty array will be returned and the response status code will be <strong>404</strong>. If multiple feeds are found all will be returned in the array with a response status code of <strong>300</strong>.
|
||||
<pre><code>
|
||||
✨ curl -H "Content-Type: application/json" https://feed-fetcher.cluster.fun/\?url\=https://marcusnoble.co.uk
|
||||
HTTP/1.1 200 OK
|
||||
Date: Sun, 21 Mar 2021 07:24:37 GMT
|
||||
Content-Type: application/json
|
||||
Content-Length: 38
|
||||
|
||||
[
|
||||
"https://marcusnoble.co.uk/feed.xml"
|
||||
]
|
||||
</code></pre>
|
||||
</p>
|
||||
|
||||
<div>
|
||||
Source code available on <a href="https://github.com/AverageMarcus/feed-fetcher" target="_blank" rel="noopener noreferrer">GitHub</a>, <a href="https://gitlab.com/AverageMarcus/feed-fetcher" target="_blank" rel="noopener noreferrer">GitLab</a>, <a href="https://bitbucket.org/AverageMarcus/feed-fetcher/" target="_blank" rel="noopener noreferrer">Bitbucket</a> & <a href="https://git.cluster.fun/AverageMarcus/feed-fetcher" target="_blank" rel="noopener noreferrer">my own Gitea server</a>.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="container">
|
||||
<div class="row">
|
||||
<div class="column column-60 column-offset-20">
|
||||
<footer>
|
||||
Made with
|
||||
<svg height="20" class="fill-primary" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 449.3 449.3" xmlns:xlink="http://www.w3.org/1999/xlink"><title>love</title><g><path d="M0 162.7c1.5-7.7 2.7-15.4 4.5-23A125.5 125.5 0 0132 88a136.3 136.3 0 0162.7-40.6c8.3-2.9 17.7-3.2 26.6-3.7a134 134 0 0155.6 6.6c14.9 5.7 30 11 41 23.6 17-20 36.4-36.3 60-46.4 12-5.2 25.7-6.9 38.7-9.4a79.4 79.4 0 0140.3 3.2 96.4 96.4 0 0143.2 26 209.8 209.8 0 0137.8 55.4 133.2 133.2 0 0111 65.7c-3.2 42.2-21 79-41.5 114.8a431.2 431.2 0 01-47.6 64.3c-19.6 23-39.7 45.7-59.6 68.5-3.7 4.3-7.2 9-11.7 12.4-7.3 5.4-15.9 4.9-23.8 1.5-21.9-9.2-43.8-18.5-65.3-28.5a520.1 520.1 0 01-98-58.7c-28.2-21.5-55.5-44.3-74-75.3a183.8 183.8 0 01-25-61.4c-1-6.2-1.6-12.6-2.4-18.8v-24.5zM138 281l2.5.1c0 2.1.3 4.3 0 6.3l-9 55.5c-.3 1.6 0 4.2 1 5 5.8 4.1 11.8 7.8 17.9 11.8l11-59.7 2.5.3c3.7 21.6-6.3 42-7.6 63.5l19.4 9.6c.5-18 2.2-35 6.9-51.6 1.3 4 1.5 8 1.2 12-.7 11.2-1 22.5-2.6 33.7-1 7.3 1.9 10.6 8 13.3 27 11.7 54.1 23.5 81 35.7 5.5 2.5 8.8 1.5 12.6-2.9 12.5-14.4 25.7-28.1 38-42.7 17.2-20.2 34.5-40.5 50.5-61.7 29.5-39.2 52.7-81.7 61-131 3.6-21.7 2-43-5.6-63.5a176 176 0 00-33.9-54.5 84.8 84.8 0 00-38-26 91.2 91.2 0 00-44.2-2.5c-13 2-25.6 5.1-37.2 12.3a208.6 208.6 0 00-45.9 37c-5.7 6.3-7.8 6.4-15 1.4-5.7-4-11-8.8-17.2-11.9-15.1-7.5-30.7-14.4-48-14.9-13.7-.4-28-2-41.2 1-36 8.4-62.7 30-80.3 62.8a111 111 0 00-11.7 56.6c.3 10 1.4 20 2.2 29.9 7-18.9 11-38.3 19.7-56.3.6 3 .6 6 0 8.7l-14.5 53.2c-.7 2.6-2 5.7-1.2 8 2.7 8.4 6.2 16.6 9.4 24.8 0-20 13.7-63.9 21.8-66.8 0 .6.3 1.1.2 1.6a11936 11936 0 01-17 64c-.3 1.4-1.6 2.6-2.9 4.5l9.7 17a573 573 0 0120-67.3l2.9.7c-1 5-1.5 10-2.8 14.9-4.7 17.2-9.6 34.3-14.2 51.5-.7 2.5-1.5 6-.3 7.8 3.6 5.6 8 10.7 12.4 16.1 1.8-8.6 3.1-16.6 5.2-24.4 3.1-11.5 6.6-22.9 10.2-34.3.8-2.6 2.7-5 4-7.4l2 .8c-.1 1.6-.1 3.4-.5 5l-12.2 47.4c-4.6 17.9-4.3 18.9 7.8 29.4 2.2 1.8 4.5 3.5 7 5.3 3.7-31 12.5-64.7 18.4-68-.3 3.5-.3 6.6-.9 9.6l-12.7 58.8c-.3 1.2-.7 3-.1 3.5 4.7 4.4 9.6 8.6 14.5 12.9 3.5-21.4 7.6-41.7 13.6-61.6 3.4 8.7.6 17-.8 25.2-1.9 11.6-4.5 23.1-6.5 34.7-.4 2.3-.4 6 1 7 4.8 4.2 10.3 7.4 16 11.3 3.9-21.5 5.5-42.6 12.6-62.5z"/><path d="M323.2 180.5a24 24 0 01-24.5-19.7c-2-9.7.7-20.2 15-27 16.6-7.8 38.3 2.3 41.6 19.5 1.2 6.6-1.6 12.1-6 16.3a35.1 35.1 0 01-26 11z"/><path d="M138.9 167.1a31 31 0 0125 12.7 22 22 0 01-13.6 34.9 29.9 29.9 0 01-31-9.9c-6-7-7.6-15.3-4.1-24.1 3.7-9.5 12-12 21-13.5.8-.2 1.8 0 2.7 0z"/><path d="M233.2 202c-18.5-.4-33.7-13.6-34-29.7 0-4.2.4-8 5.4-8.7 4.5-.7 6.6 2.3 8 6 2.3 6.6 5.5 12.4 12.4 15.4 7.5 3.2 14 1.5 20.4-3.3 6.2-4.7 6.4-11 4.7-17.5-1.4-5.2.4-8.4 4.7-10.2 4.3-1.7 9 1.3 10.9 6.2 7.2 19.8-5.7 34.6-22.8 40.2-3 1-6.5 1-9.7 1.6z"/><path d="M201.2 384.7c-.9-2-2.7-4.2-2.5-6.2 1.2-14 2.8-28.1 4.4-42.2 0-.9.7-1.7 1-2.6l2.6 1.1-3.1 49.1-2.4.8z"/><path d="M222 387.4c-4.8-5.7-5-15-1-37.8 3.1 3 4 30.5 1 37.8z"/><path d="M240.9 361.7l-1.4 20.2c-4.8-3.3-4.6-11.9-.2-20.4l1.6.2z"/><path d="M257.4 380.4c-4.1-5.2-3.7-9.7 1-14.7l-1 14.7z"/></g></svg>
|
||||
by <a href="https://marcusnoble.co.uk" class="fancy-link">Marcus Noble</a>
|
||||
</footer>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
document.getElementById('fetchForm').addEventListener('submit', function(e) {
|
||||
let submitUrl = e.target.action + '?url=' + document.getElementById('url').value;
|
||||
|
||||
let resultsText = document.getElementById('results');
|
||||
fetch(submitUrl, { headers: { "Content-Type": "application/json" } })
|
||||
.then(response => {
|
||||
if (response.status == 404) {
|
||||
resultsText.innerHTML = "No feeds found";
|
||||
return;
|
||||
} else if (response.status > 400) {
|
||||
resultsText.innerHTML = "Unable to fetch feed";
|
||||
return;
|
||||
}
|
||||
|
||||
return response.json()
|
||||
})
|
||||
.then(json => {
|
||||
if (json) {
|
||||
resultsText.innerHTML = `<h3 class="heading-fancy">Results</h3><br><ul>` + json.reduce((acc, url) => `${acc}<li><a href="${url}" class="fancy-link">${url}</a></li>`, '') + '</ul>';
|
||||
}
|
||||
})
|
||||
.catch(ex => {
|
||||
console.log(ex);
|
||||
resultsText.innerHTML = "Unable to fetch feed";
|
||||
});
|
||||
|
||||
e.preventDefault();
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
118
main.go
118
main.go
@ -7,13 +7,18 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"embed"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/mmcdole/gofeed"
|
||||
)
|
||||
|
||||
//go:embed index.html
|
||||
|
||||
var content embed.FS
|
||||
|
||||
func main() {
|
||||
fp := gofeed.NewParser()
|
||||
port, ok := os.LookupEnv("PORT")
|
||||
if !ok {
|
||||
port = "8080"
|
||||
@ -24,56 +29,32 @@ func main() {
|
||||
app.Get("/", func(c *fiber.Ctx) error {
|
||||
feedUrl := c.Query("url")
|
||||
if feedUrl == "" {
|
||||
fmt.Println("No URL provided")
|
||||
return c.SendStatus(fiber.StatusBadRequest)
|
||||
c.Type("html", "UTF8")
|
||||
body, _ := content.ReadFile("index.html")
|
||||
return c.Send(body)
|
||||
}
|
||||
|
||||
_, err := fp.ParseURL(feedUrl)
|
||||
if err != nil && err == gofeed.ErrFeedTypeNotDetected {
|
||||
res, err := http.Get(feedUrl)
|
||||
if err != nil {
|
||||
fmt.Println("Failed to fetch URL")
|
||||
return c.SendStatus(fiber.StatusInternalServerError)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 400 {
|
||||
fmt.Println("Provided URL returned an error status code")
|
||||
return c.SendStatus(res.StatusCode)
|
||||
}
|
||||
feeds, statusCode := getFeeds(feedUrl)
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(res.Body)
|
||||
if err != nil {
|
||||
fmt.Println("Failed to parse response body")
|
||||
return c.SendStatus(fiber.StatusInternalServerError)
|
||||
if c.Is("json") {
|
||||
if statusCode >= 400 || statusCode == 300 {
|
||||
c.Status(statusCode)
|
||||
}
|
||||
|
||||
matches := doc.Find(`[rel="alternate"][type="application/rss+xml"]`)
|
||||
if matches.Length() == 0 {
|
||||
fmt.Println("No RSS feeds found on page")
|
||||
return c.SendStatus(fiber.StatusNotFound)
|
||||
}
|
||||
|
||||
foundUrl, ok := matches.First().Attr("href")
|
||||
if !ok {
|
||||
fmt.Println("href attribute missing from tag")
|
||||
return c.SendStatus(fiber.StatusNotFound)
|
||||
}
|
||||
c.Set("Location", absoluteUrl(feedUrl, foundUrl))
|
||||
if matches.Length() > 1 {
|
||||
fmt.Println("Multiple feeds found on page")
|
||||
return c.SendStatus(fiber.StatusMultipleChoices)
|
||||
return c.JSON(feeds)
|
||||
} else {
|
||||
fmt.Println("Feed found on page")
|
||||
return c.SendStatus(fiber.StatusTemporaryRedirect)
|
||||
c.Status(statusCode)
|
||||
c.Location(feeds[0])
|
||||
|
||||
if len(feeds) > 1 {
|
||||
responseBody := "Multiple Choices\n\n"
|
||||
for _, feed := range feeds {
|
||||
responseBody += feed + "\n"
|
||||
}
|
||||
} else if err != nil {
|
||||
fmt.Println("Failed while attempting to parse feed")
|
||||
return c.SendStatus(fiber.StatusInternalServerError)
|
||||
return c.SendString(responseBody)
|
||||
}
|
||||
|
||||
fmt.Println("URL provided is already a feed")
|
||||
c.Set("Location", feedUrl)
|
||||
return c.SendStatus(fiber.StatusMovedPermanently)
|
||||
return c.Send(nil)
|
||||
}
|
||||
})
|
||||
|
||||
fmt.Println(app.Listen(fmt.Sprintf(":%s", port)))
|
||||
@ -87,3 +68,54 @@ func absoluteUrl(requestUrl, foundUrl string) string {
|
||||
|
||||
return foundUrl
|
||||
}
|
||||
|
||||
func getFeeds(requestURL string) ([]string, int) {
|
||||
feeds := []string{}
|
||||
|
||||
fp := gofeed.NewParser()
|
||||
_, err := fp.ParseURL(requestURL)
|
||||
if err == nil {
|
||||
feeds = []string{requestURL}
|
||||
} else if err != nil && err == gofeed.ErrFeedTypeNotDetected {
|
||||
res, err := http.Get(requestURL)
|
||||
if err != nil {
|
||||
fmt.Println("Failed to fetch URL")
|
||||
return feeds, fiber.StatusInternalServerError
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode >= 400 {
|
||||
fmt.Println("Provided URL returned an error status code")
|
||||
return feeds, res.StatusCode
|
||||
}
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(res.Body)
|
||||
if err != nil {
|
||||
fmt.Println("Failed to parse response body")
|
||||
return feeds, fiber.StatusInternalServerError
|
||||
}
|
||||
|
||||
matches := doc.Find(`[rel="alternate"][type="application/rss+xml"]`)
|
||||
if matches.Length() == 0 {
|
||||
fmt.Println("No RSS feeds found on page")
|
||||
return feeds, fiber.StatusNotFound
|
||||
}
|
||||
|
||||
matches.Each(func(i int, s *goquery.Selection) {
|
||||
feeds = append(feeds, absoluteUrl(requestURL, s.AttrOr("href", "")))
|
||||
})
|
||||
|
||||
if matches.Length() > 1 {
|
||||
fmt.Println("Multiple feeds found on page")
|
||||
return feeds, fiber.StatusMultipleChoices
|
||||
} else {
|
||||
fmt.Println("Feed found on page")
|
||||
return feeds, fiber.StatusTemporaryRedirect
|
||||
}
|
||||
} else if err != nil {
|
||||
fmt.Println("Failed while attempting to parse feed")
|
||||
return feeds, fiber.StatusInternalServerError
|
||||
}
|
||||
|
||||
return feeds, 200
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user