Initial release

This commit is contained in:
Marcus Noble 2021-03-17 09:51:00 +00:00
parent 13830565f7
commit 30ad5889bd
6 changed files with 216 additions and 35 deletions

View File

@ -0,0 +1,17 @@
FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.14-alpine as builder
ARG TARGETPLATFORM
ARG BUILDPLATFORM
ARG TARGETOS
ARG TARGETARCH
WORKDIR /app/
RUN apk update && apk add --no-cache --update gcc musl-dev git && adduser -D -g '' gopher && apk add -U --no-cache ca-certificates
ADD go.mod go.sum ./
RUN go mod download
ADD . .
RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o feed-fetcher .
COPY ./views /app/
ENTRYPOINT ["/app/feed-fetcher"]

View File

@ -1,49 +1,33 @@
.DEFAULT_GOAL := default .DEFAULT_GOAL := default
IMAGE ?= docker.cluster.fun/private/feed-fetcher:latest IMAGE ?= docker.cluster.fun/averagemarcus/feed-fetcher:latest
.PHONY: test # Run all tests, linting and format checks .PHONY: test # Run all tests, linting and format checks
test: lint check-format run-tests test: lint check-format run-tests
.PHONY: lint # Perform lint checks against code .PHONY: lint # Perform lint checks against code
lint: lint:
@echo "⚠️ 'lint' unimplemented" @go vet && golint -set_exit_status ./...
# GO Projects
# @go vet && golint -set_exit_status ./...
.PHONY: check-format # Checks code formatting and returns a non-zero exit code if formatting errors found .PHONY: check-format # Checks code formatting and returns a non-zero exit code if formatting errors found
check-format: check-format:
@echo "⚠️ 'check-format' unimplemented" @gofmt -e -l .
# GO Projects
# @gofmt -e -l .
.PHONY: format # Performs automatic format fixes on all code .PHONY: format # Performs automatic format fixes on all code
format: format:
@echo "⚠️ 'format' unimplemented" @gofmt -s -w .
# GO Projects
# @gofmt -s -w .
.PHONY: run-tests # Runs all tests .PHONY: run-tests # Runs all tests
run-tests: run-tests:
@echo "⚠️ 'run-tests' unimplemented" @go test
# GO Projects
# @go test
# Node Projects
# @npm test
.PHONY: fetch-deps # Fetch all project dependencies .PHONY: fetch-deps # Fetch all project dependencies
fetch-deps: fetch-deps:s
@echo "⚠️ 'fetch-deps' unimplemented" @go mod tidy
# GO Projects
# @go mod tidy
# Node Projects
# @npm install
.PHONY: build # Build the project .PHONY: build # Build the project
build: lint check-format fetch-deps build: lint check-format fetch-deps
@echo "⚠️ 'build' unimplemented" @go build -o PROJECT_NAME main.go
# GO Projects
# @go build -o PROJECT_NAME main.go
.PHONY: docker-build # Build the docker image .PHONY: docker-build # Build the docker image
docker-build: docker-build:
@ -55,11 +39,7 @@ docker-publish:
.PHONY: run # Run the application .PHONY: run # Run the application
run: run:
@echo "⚠️ 'run' unimplemented" @go run main.go
# GO Projects
# @go run main.go
# Node Projects
# @npm start
.PHONY: ci # Perform CI specific tasks to perform on a pull request .PHONY: ci # Perform CI specific tasks to perform on a pull request
ci: ci:

View File

@ -2,14 +2,43 @@
Returns the RSS feed associated with the given URL Returns the RSS feed associated with the given URL
## Features ## Usage
## Install
```sh ```sh
GET https://feed-fetcher.cluster.fun/?url=${URL_TO_CHECK}
``` ```
Example:
```sh
curl -v http://localhost:8000/\?url\=https://marcusnoble.co.uk/
* Trying 127.0.0.1...
* TCP_NODELAY set
* Connected to localhost (127.0.0.1) port 8000 (#0)
> GET /?url=https://marcusnoble.co.uk/ HTTP/1.1
> Host: localhost:8000
> User-Agent: curl/7.64.1
> Accept: */*
>
< HTTP/1.1 307 Temporary Redirect
< Date: Wed, 17 Mar 2021 09:44:32 GMT
< Content-Type: text/plain; charset=utf-8
< Content-Length: 18
< Location: https://marcusnoble.co.uk/feed.xml
<
* Connection #0 to host localhost left intact
Temporary Redirect* Closing connection 0
```
### Possible status code responses
* **300** - Multiple possible feeds found on page (the first is returned on the `Location` header)
* **301** - URL provided was already a valid feed URL
* **307** - Feed URL found on provided page
* **400** - No URL provided
* **404** - No feed URL found on provided webpage
* **500** - Server error while trying to fetch feed
## Building from source ## Building from source
With Docker: With Docker:
@ -24,8 +53,6 @@ Standalone:
make build make build
``` ```
## Resources
## Contributing ## Contributing
If you find a bug or have an idea for a new feature please [raise an issue](issues/new) to discuss it. If you find a bug or have an idea for a new feature please [raise an issue](issues/new) to discuss it.

8
go.mod Normal file
View File

@ -0,0 +1,8 @@
module feed-fetcher
go 1.15
require (
github.com/gofiber/fiber/v2 v2.6.0
github.com/mmcdole/gofeed v1.1.0
)

60
go.sum Normal file
View File

@ -0,0 +1,60 @@
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4=
github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gofiber/fiber/v2 v2.6.0 h1:OywSUL6QPY/+/b89Ulnb8reovwm5QGjZQfk74v0R7Uc=
github.com/gofiber/fiber/v2 v2.6.0/go.mod h1:f8BRRIMjMdRyt2qmJ/0Sea3j3rwwfufPrh9WNBRiVZ0=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/klauspost/compress v1.10.7 h1:7rix8v8GpI3ZBb0nSozFRgbtXKv+hOe+qfEpZqybrAg=
github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/mmcdole/gofeed v1.1.0 h1:T2WrGLVJRV04PY2qwhEJLHCt9JiCtBhb6SmC8ZvJH08=
github.com/mmcdole/gofeed v1.1.0/go.mod h1:PPiVwgDXLlz2N83KB4TrIim2lyYM5Zn7ZWH9Pi4oHUk=
github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI=
github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/urfave/cli v1.22.3/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.18.0 h1:IV0DdMlatq9QO1Cr6wGJPVW1sV1Q8HvZXAIcjorylyM=
github.com/valyala/fasthttp v1.18.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A=
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc=
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0 h1:5kGOVHlq0euqwzgTC9Vu15p6fV1Wi0ArVi8da2urnVg=
golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201210223839-7e3030f88018 h1:XKi8B/gRBuTZN1vU9gFsLMm6zVz5FSCDzm8JYACnjy8=
golang.org/x/sys v0.0.0-20201210223839-7e3030f88018/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

89
main.go Normal file
View File

@ -0,0 +1,89 @@
package main
import (
"fmt"
"net/http"
"net/url"
"os"
"strings"
"github.com/PuerkitoBio/goquery"
"github.com/gofiber/fiber/v2"
"github.com/mmcdole/gofeed"
)
func main() {
fp := gofeed.NewParser()
port, ok := os.LookupEnv("PORT")
if !ok {
port = "8080"
}
app := fiber.New(fiber.Config{})
app.Get("/", func(c *fiber.Ctx) error {
feedUrl := c.Query("url")
if feedUrl == "" {
fmt.Println("No URL provided")
return c.SendStatus(fiber.StatusBadRequest)
}
_, err := fp.ParseURL(feedUrl)
if err != nil && err == gofeed.ErrFeedTypeNotDetected {
res, err := http.Get(feedUrl)
if err != nil {
fmt.Println("Failed to fetch URL")
return c.SendStatus(fiber.StatusInternalServerError)
}
defer res.Body.Close()
if res.StatusCode >= 400 {
fmt.Println("Provided URL returned an error status code")
return c.SendStatus(res.StatusCode)
}
doc, err := goquery.NewDocumentFromReader(res.Body)
if err != nil {
fmt.Println("Failed to parse response body")
return c.SendStatus(fiber.StatusInternalServerError)
}
matches := doc.Find(`[rel="alternate"][type="application/rss+xml"]`)
if matches.Length() == 0 {
fmt.Println("No RSS feeds found on page")
return c.SendStatus(fiber.StatusNotFound)
}
foundUrl, ok := matches.First().Attr("href")
if !ok {
fmt.Println("href attribute missing from tag")
return c.SendStatus(fiber.StatusNotFound)
}
c.Set("Location", absoluteUrl(feedUrl, foundUrl))
if matches.Length() > 1 {
fmt.Println("Multiple feeds found on page")
return c.SendStatus(fiber.StatusMultipleChoices)
} else {
fmt.Println("Feed found on page")
return c.SendStatus(fiber.StatusTemporaryRedirect)
}
} else if err != nil {
fmt.Println("Failed while attempting to parse feed")
return c.SendStatus(fiber.StatusInternalServerError)
}
fmt.Println("URL provided is already a feed")
c.Set("Location", feedUrl)
return c.SendStatus(fiber.StatusMovedPermanently)
})
fmt.Println(app.Listen(fmt.Sprintf(":%s", port)))
}
func absoluteUrl(requestUrl, foundUrl string) string {
if !strings.HasPrefix(foundUrl, "http") {
parsedUrl, _ := url.Parse(requestUrl)
foundUrl = fmt.Sprintf("%s://%s%s", parsedUrl.Scheme, parsedUrl.Host, foundUrl)
}
return foundUrl
}