Initial release
This commit is contained in:
parent
13830565f7
commit
30ad5889bd
17
Dockerfile
17
Dockerfile
@ -0,0 +1,17 @@
|
|||||||
|
FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.14-alpine as builder
|
||||||
|
|
||||||
|
ARG TARGETPLATFORM
|
||||||
|
ARG BUILDPLATFORM
|
||||||
|
ARG TARGETOS
|
||||||
|
ARG TARGETARCH
|
||||||
|
|
||||||
|
WORKDIR /app/
|
||||||
|
RUN apk update && apk add --no-cache --update gcc musl-dev git && adduser -D -g '' gopher && apk add -U --no-cache ca-certificates
|
||||||
|
ADD go.mod go.sum ./
|
||||||
|
RUN go mod download
|
||||||
|
ADD . .
|
||||||
|
RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o feed-fetcher .
|
||||||
|
|
||||||
|
COPY ./views /app/
|
||||||
|
|
||||||
|
ENTRYPOINT ["/app/feed-fetcher"]
|
||||||
38
Makefile
38
Makefile
@ -1,49 +1,33 @@
|
|||||||
.DEFAULT_GOAL := default
|
.DEFAULT_GOAL := default
|
||||||
|
|
||||||
IMAGE ?= docker.cluster.fun/private/feed-fetcher:latest
|
IMAGE ?= docker.cluster.fun/averagemarcus/feed-fetcher:latest
|
||||||
|
|
||||||
.PHONY: test # Run all tests, linting and format checks
|
.PHONY: test # Run all tests, linting and format checks
|
||||||
test: lint check-format run-tests
|
test: lint check-format run-tests
|
||||||
|
|
||||||
.PHONY: lint # Perform lint checks against code
|
.PHONY: lint # Perform lint checks against code
|
||||||
lint:
|
lint:
|
||||||
@echo "⚠️ 'lint' unimplemented"
|
@go vet && golint -set_exit_status ./...
|
||||||
# GO Projects
|
|
||||||
# @go vet && golint -set_exit_status ./...
|
|
||||||
|
|
||||||
.PHONY: check-format # Checks code formatting and returns a non-zero exit code if formatting errors found
|
.PHONY: check-format # Checks code formatting and returns a non-zero exit code if formatting errors found
|
||||||
check-format:
|
check-format:
|
||||||
@echo "⚠️ 'check-format' unimplemented"
|
@gofmt -e -l .
|
||||||
# GO Projects
|
|
||||||
# @gofmt -e -l .
|
|
||||||
|
|
||||||
.PHONY: format # Performs automatic format fixes on all code
|
.PHONY: format # Performs automatic format fixes on all code
|
||||||
format:
|
format:
|
||||||
@echo "⚠️ 'format' unimplemented"
|
@gofmt -s -w .
|
||||||
# GO Projects
|
|
||||||
# @gofmt -s -w .
|
|
||||||
|
|
||||||
.PHONY: run-tests # Runs all tests
|
.PHONY: run-tests # Runs all tests
|
||||||
run-tests:
|
run-tests:
|
||||||
@echo "⚠️ 'run-tests' unimplemented"
|
@go test
|
||||||
# GO Projects
|
|
||||||
# @go test
|
|
||||||
# Node Projects
|
|
||||||
# @npm test
|
|
||||||
|
|
||||||
.PHONY: fetch-deps # Fetch all project dependencies
|
.PHONY: fetch-deps # Fetch all project dependencies
|
||||||
fetch-deps:
|
fetch-deps:s
|
||||||
@echo "⚠️ 'fetch-deps' unimplemented"
|
@go mod tidy
|
||||||
# GO Projects
|
|
||||||
# @go mod tidy
|
|
||||||
# Node Projects
|
|
||||||
# @npm install
|
|
||||||
|
|
||||||
.PHONY: build # Build the project
|
.PHONY: build # Build the project
|
||||||
build: lint check-format fetch-deps
|
build: lint check-format fetch-deps
|
||||||
@echo "⚠️ 'build' unimplemented"
|
@go build -o PROJECT_NAME main.go
|
||||||
# GO Projects
|
|
||||||
# @go build -o PROJECT_NAME main.go
|
|
||||||
|
|
||||||
.PHONY: docker-build # Build the docker image
|
.PHONY: docker-build # Build the docker image
|
||||||
docker-build:
|
docker-build:
|
||||||
@ -55,11 +39,7 @@ docker-publish:
|
|||||||
|
|
||||||
.PHONY: run # Run the application
|
.PHONY: run # Run the application
|
||||||
run:
|
run:
|
||||||
@echo "⚠️ 'run' unimplemented"
|
@go run main.go
|
||||||
# GO Projects
|
|
||||||
# @go run main.go
|
|
||||||
# Node Projects
|
|
||||||
# @npm start
|
|
||||||
|
|
||||||
.PHONY: ci # Perform CI specific tasks to perform on a pull request
|
.PHONY: ci # Perform CI specific tasks to perform on a pull request
|
||||||
ci:
|
ci:
|
||||||
|
|||||||
39
README.md
39
README.md
@ -2,14 +2,43 @@
|
|||||||
|
|
||||||
Returns the RSS feed associated with the given URL
|
Returns the RSS feed associated with the given URL
|
||||||
|
|
||||||
## Features
|
## Usage
|
||||||
|
|
||||||
## Install
|
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
|
GET https://feed-fetcher.cluster.fun/?url=${URL_TO_CHECK}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
curl -v http://localhost:8000/\?url\=https://marcusnoble.co.uk/
|
||||||
|
* Trying 127.0.0.1...
|
||||||
|
* TCP_NODELAY set
|
||||||
|
* Connected to localhost (127.0.0.1) port 8000 (#0)
|
||||||
|
> GET /?url=https://marcusnoble.co.uk/ HTTP/1.1
|
||||||
|
> Host: localhost:8000
|
||||||
|
> User-Agent: curl/7.64.1
|
||||||
|
> Accept: */*
|
||||||
|
>
|
||||||
|
< HTTP/1.1 307 Temporary Redirect
|
||||||
|
< Date: Wed, 17 Mar 2021 09:44:32 GMT
|
||||||
|
< Content-Type: text/plain; charset=utf-8
|
||||||
|
< Content-Length: 18
|
||||||
|
< Location: https://marcusnoble.co.uk/feed.xml
|
||||||
|
<
|
||||||
|
* Connection #0 to host localhost left intact
|
||||||
|
Temporary Redirect* Closing connection 0
|
||||||
|
```
|
||||||
|
|
||||||
|
### Possible status code responses
|
||||||
|
|
||||||
|
* **300** - Multiple possible feeds found on page (the first is returned on the `Location` header)
|
||||||
|
* **301** - URL provided was already a valid feed URL
|
||||||
|
* **307** - Feed URL found on provided page
|
||||||
|
* **400** - No URL provided
|
||||||
|
* **404** - No feed URL found on provided webpage
|
||||||
|
* **500** - Server error while trying to fetch feed
|
||||||
|
|
||||||
## Building from source
|
## Building from source
|
||||||
|
|
||||||
With Docker:
|
With Docker:
|
||||||
@ -24,8 +53,6 @@ Standalone:
|
|||||||
make build
|
make build
|
||||||
```
|
```
|
||||||
|
|
||||||
## Resources
|
|
||||||
|
|
||||||
## Contributing
|
## Contributing
|
||||||
|
|
||||||
If you find a bug or have an idea for a new feature please [raise an issue](issues/new) to discuss it.
|
If you find a bug or have an idea for a new feature please [raise an issue](issues/new) to discuss it.
|
||||||
|
|||||||
8
go.mod
Normal file
8
go.mod
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
module feed-fetcher
|
||||||
|
|
||||||
|
go 1.15
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/gofiber/fiber/v2 v2.6.0
|
||||||
|
github.com/mmcdole/gofeed v1.1.0
|
||||||
|
)
|
||||||
60
go.sum
Normal file
60
go.sum
Normal file
@ -0,0 +1,60 @@
|
|||||||
|
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||||
|
github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
|
||||||
|
github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
|
||||||
|
github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4=
|
||||||
|
github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
|
||||||
|
github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
|
||||||
|
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
|
||||||
|
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||||
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/gofiber/fiber/v2 v2.6.0 h1:OywSUL6QPY/+/b89Ulnb8reovwm5QGjZQfk74v0R7Uc=
|
||||||
|
github.com/gofiber/fiber/v2 v2.6.0/go.mod h1:f8BRRIMjMdRyt2qmJ/0Sea3j3rwwfufPrh9WNBRiVZ0=
|
||||||
|
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||||
|
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
|
||||||
|
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||||
|
github.com/klauspost/compress v1.10.7 h1:7rix8v8GpI3ZBb0nSozFRgbtXKv+hOe+qfEpZqybrAg=
|
||||||
|
github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
|
||||||
|
github.com/mmcdole/gofeed v1.1.0 h1:T2WrGLVJRV04PY2qwhEJLHCt9JiCtBhb6SmC8ZvJH08=
|
||||||
|
github.com/mmcdole/gofeed v1.1.0/go.mod h1:PPiVwgDXLlz2N83KB4TrIim2lyYM5Zn7ZWH9Pi4oHUk=
|
||||||
|
github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI=
|
||||||
|
github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8=
|
||||||
|
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
|
||||||
|
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||||
|
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
|
||||||
|
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||||
|
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||||
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
|
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
||||||
|
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||||
|
github.com/urfave/cli v1.22.3/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
|
||||||
|
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||||
|
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||||
|
github.com/valyala/fasthttp v1.18.0 h1:IV0DdMlatq9QO1Cr6wGJPVW1sV1Q8HvZXAIcjorylyM=
|
||||||
|
github.com/valyala/fasthttp v1.18.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A=
|
||||||
|
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc=
|
||||||
|
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
|
||||||
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
|
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||||
|
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
|
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||||
|
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0 h1:5kGOVHlq0euqwzgTC9Vu15p6fV1Wi0ArVi8da2urnVg=
|
||||||
|
golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||||
|
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
|
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20201210223839-7e3030f88018 h1:XKi8B/gRBuTZN1vU9gFsLMm6zVz5FSCDzm8JYACnjy8=
|
||||||
|
golang.org/x/sys v0.0.0-20201210223839-7e3030f88018/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
|
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||||
|
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
|
||||||
|
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
|
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
89
main.go
Normal file
89
main.go
Normal file
@ -0,0 +1,89 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/PuerkitoBio/goquery"
|
||||||
|
"github.com/gofiber/fiber/v2"
|
||||||
|
"github.com/mmcdole/gofeed"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
fp := gofeed.NewParser()
|
||||||
|
port, ok := os.LookupEnv("PORT")
|
||||||
|
if !ok {
|
||||||
|
port = "8080"
|
||||||
|
}
|
||||||
|
|
||||||
|
app := fiber.New(fiber.Config{})
|
||||||
|
|
||||||
|
app.Get("/", func(c *fiber.Ctx) error {
|
||||||
|
feedUrl := c.Query("url")
|
||||||
|
if feedUrl == "" {
|
||||||
|
fmt.Println("No URL provided")
|
||||||
|
return c.SendStatus(fiber.StatusBadRequest)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := fp.ParseURL(feedUrl)
|
||||||
|
if err != nil && err == gofeed.ErrFeedTypeNotDetected {
|
||||||
|
res, err := http.Get(feedUrl)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Println("Failed to fetch URL")
|
||||||
|
return c.SendStatus(fiber.StatusInternalServerError)
|
||||||
|
}
|
||||||
|
defer res.Body.Close()
|
||||||
|
if res.StatusCode >= 400 {
|
||||||
|
fmt.Println("Provided URL returned an error status code")
|
||||||
|
return c.SendStatus(res.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
doc, err := goquery.NewDocumentFromReader(res.Body)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Println("Failed to parse response body")
|
||||||
|
return c.SendStatus(fiber.StatusInternalServerError)
|
||||||
|
}
|
||||||
|
|
||||||
|
matches := doc.Find(`[rel="alternate"][type="application/rss+xml"]`)
|
||||||
|
if matches.Length() == 0 {
|
||||||
|
fmt.Println("No RSS feeds found on page")
|
||||||
|
return c.SendStatus(fiber.StatusNotFound)
|
||||||
|
}
|
||||||
|
|
||||||
|
foundUrl, ok := matches.First().Attr("href")
|
||||||
|
if !ok {
|
||||||
|
fmt.Println("href attribute missing from tag")
|
||||||
|
return c.SendStatus(fiber.StatusNotFound)
|
||||||
|
}
|
||||||
|
c.Set("Location", absoluteUrl(feedUrl, foundUrl))
|
||||||
|
if matches.Length() > 1 {
|
||||||
|
fmt.Println("Multiple feeds found on page")
|
||||||
|
return c.SendStatus(fiber.StatusMultipleChoices)
|
||||||
|
} else {
|
||||||
|
fmt.Println("Feed found on page")
|
||||||
|
return c.SendStatus(fiber.StatusTemporaryRedirect)
|
||||||
|
}
|
||||||
|
} else if err != nil {
|
||||||
|
fmt.Println("Failed while attempting to parse feed")
|
||||||
|
return c.SendStatus(fiber.StatusInternalServerError)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println("URL provided is already a feed")
|
||||||
|
c.Set("Location", feedUrl)
|
||||||
|
return c.SendStatus(fiber.StatusMovedPermanently)
|
||||||
|
})
|
||||||
|
|
||||||
|
fmt.Println(app.Listen(fmt.Sprintf(":%s", port)))
|
||||||
|
}
|
||||||
|
|
||||||
|
func absoluteUrl(requestUrl, foundUrl string) string {
|
||||||
|
if !strings.HasPrefix(foundUrl, "http") {
|
||||||
|
parsedUrl, _ := url.Parse(requestUrl)
|
||||||
|
foundUrl = fmt.Sprintf("%s://%s%s", parsedUrl.Scheme, parsedUrl.Host, foundUrl)
|
||||||
|
}
|
||||||
|
|
||||||
|
return foundUrl
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user