diff --git a/Dockerfile b/Dockerfile index e69de29..abae878 100644 --- a/Dockerfile +++ b/Dockerfile @@ -0,0 +1,17 @@ +FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.14-alpine as builder + +ARG TARGETPLATFORM +ARG BUILDPLATFORM +ARG TARGETOS +ARG TARGETARCH + +WORKDIR /app/ +RUN apk update && apk add --no-cache --update gcc musl-dev git && adduser -D -g '' gopher && apk add -U --no-cache ca-certificates +ADD go.mod go.sum ./ +RUN go mod download +ADD . . +RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o feed-fetcher . + +COPY ./views /app/ + +ENTRYPOINT ["/app/feed-fetcher"] diff --git a/Makefile b/Makefile index 34526dc..689b88a 100644 --- a/Makefile +++ b/Makefile @@ -1,49 +1,33 @@ .DEFAULT_GOAL := default -IMAGE ?= docker.cluster.fun/private/feed-fetcher:latest +IMAGE ?= docker.cluster.fun/averagemarcus/feed-fetcher:latest .PHONY: test # Run all tests, linting and format checks test: lint check-format run-tests .PHONY: lint # Perform lint checks against code lint: - @echo "⚠️ 'lint' unimplemented" - # GO Projects - # @go vet && golint -set_exit_status ./... + @go vet && golint -set_exit_status ./... .PHONY: check-format # Checks code formatting and returns a non-zero exit code if formatting errors found check-format: - @echo "⚠️ 'check-format' unimplemented" - # GO Projects - # @gofmt -e -l . + @gofmt -e -l . .PHONY: format # Performs automatic format fixes on all code format: - @echo "⚠️ 'format' unimplemented" - # GO Projects - # @gofmt -s -w . + @gofmt -s -w . .PHONY: run-tests # Runs all tests run-tests: - @echo "⚠️ 'run-tests' unimplemented" - # GO Projects - # @go test - # Node Projects - # @npm test + @go test .PHONY: fetch-deps # Fetch all project dependencies -fetch-deps: - @echo "⚠️ 'fetch-deps' unimplemented" - # GO Projects - # @go mod tidy - # Node Projects - # @npm install +fetch-deps:s + @go mod tidy .PHONY: build # Build the project build: lint check-format fetch-deps - @echo "⚠️ 'build' unimplemented" - # GO Projects - # @go build -o PROJECT_NAME main.go + @go build -o PROJECT_NAME main.go .PHONY: docker-build # Build the docker image docker-build: @@ -55,11 +39,7 @@ docker-publish: .PHONY: run # Run the application run: - @echo "⚠️ 'run' unimplemented" - # GO Projects - # @go run main.go - # Node Projects - # @npm start + @go run main.go .PHONY: ci # Perform CI specific tasks to perform on a pull request ci: diff --git a/README.md b/README.md index 59d8f0f..16f457f 100644 --- a/README.md +++ b/README.md @@ -2,14 +2,43 @@ Returns the RSS feed associated with the given URL -## Features - -## Install +## Usage ```sh - +GET https://feed-fetcher.cluster.fun/?url=${URL_TO_CHECK} ``` +Example: + +```sh +curl -v http://localhost:8000/\?url\=https://marcusnoble.co.uk/ +* Trying 127.0.0.1... +* TCP_NODELAY set +* Connected to localhost (127.0.0.1) port 8000 (#0) +> GET /?url=https://marcusnoble.co.uk/ HTTP/1.1 +> Host: localhost:8000 +> User-Agent: curl/7.64.1 +> Accept: */* +> +< HTTP/1.1 307 Temporary Redirect +< Date: Wed, 17 Mar 2021 09:44:32 GMT +< Content-Type: text/plain; charset=utf-8 +< Content-Length: 18 +< Location: https://marcusnoble.co.uk/feed.xml +< +* Connection #0 to host localhost left intact +Temporary Redirect* Closing connection 0 +``` + +### Possible status code responses + +* **300** - Multiple possible feeds found on page (the first is returned on the `Location` header) +* **301** - URL provided was already a valid feed URL +* **307** - Feed URL found on provided page +* **400** - No URL provided +* **404** - No feed URL found on provided webpage +* **500** - Server error while trying to fetch feed + ## Building from source With Docker: @@ -24,8 +53,6 @@ Standalone: make build ``` -## Resources - ## Contributing If you find a bug or have an idea for a new feature please [raise an issue](issues/new) to discuss it. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..5b3de66 --- /dev/null +++ b/go.mod @@ -0,0 +1,8 @@ +module feed-fetcher + +go 1.15 + +require ( + github.com/gofiber/fiber/v2 v2.6.0 + github.com/mmcdole/gofeed v1.1.0 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..d3b89e8 --- /dev/null +++ b/go.sum @@ -0,0 +1,60 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE= +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= +github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gofiber/fiber/v2 v2.6.0 h1:OywSUL6QPY/+/b89Ulnb8reovwm5QGjZQfk74v0R7Uc= +github.com/gofiber/fiber/v2 v2.6.0/go.mod h1:f8BRRIMjMdRyt2qmJ/0Sea3j3rwwfufPrh9WNBRiVZ0= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/klauspost/compress v1.10.7 h1:7rix8v8GpI3ZBb0nSozFRgbtXKv+hOe+qfEpZqybrAg= +github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/mmcdole/gofeed v1.1.0 h1:T2WrGLVJRV04PY2qwhEJLHCt9JiCtBhb6SmC8ZvJH08= +github.com/mmcdole/gofeed v1.1.0/go.mod h1:PPiVwgDXLlz2N83KB4TrIim2lyYM5Zn7ZWH9Pi4oHUk= +github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI= +github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/urfave/cli v1.22.3/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.18.0 h1:IV0DdMlatq9QO1Cr6wGJPVW1sV1Q8HvZXAIcjorylyM= +github.com/valyala/fasthttp v1.18.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0 h1:5kGOVHlq0euqwzgTC9Vu15p6fV1Wi0ArVi8da2urnVg= +golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201210223839-7e3030f88018 h1:XKi8B/gRBuTZN1vU9gFsLMm6zVz5FSCDzm8JYACnjy8= +golang.org/x/sys v0.0.0-20201210223839-7e3030f88018/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/main.go b/main.go new file mode 100644 index 0000000..9bb5c01 --- /dev/null +++ b/main.go @@ -0,0 +1,89 @@ +package main + +import ( + "fmt" + "net/http" + "net/url" + "os" + "strings" + + "github.com/PuerkitoBio/goquery" + "github.com/gofiber/fiber/v2" + "github.com/mmcdole/gofeed" +) + +func main() { + fp := gofeed.NewParser() + port, ok := os.LookupEnv("PORT") + if !ok { + port = "8080" + } + + app := fiber.New(fiber.Config{}) + + app.Get("/", func(c *fiber.Ctx) error { + feedUrl := c.Query("url") + if feedUrl == "" { + fmt.Println("No URL provided") + return c.SendStatus(fiber.StatusBadRequest) + } + + _, err := fp.ParseURL(feedUrl) + if err != nil && err == gofeed.ErrFeedTypeNotDetected { + res, err := http.Get(feedUrl) + if err != nil { + fmt.Println("Failed to fetch URL") + return c.SendStatus(fiber.StatusInternalServerError) + } + defer res.Body.Close() + if res.StatusCode >= 400 { + fmt.Println("Provided URL returned an error status code") + return c.SendStatus(res.StatusCode) + } + + doc, err := goquery.NewDocumentFromReader(res.Body) + if err != nil { + fmt.Println("Failed to parse response body") + return c.SendStatus(fiber.StatusInternalServerError) + } + + matches := doc.Find(`[rel="alternate"][type="application/rss+xml"]`) + if matches.Length() == 0 { + fmt.Println("No RSS feeds found on page") + return c.SendStatus(fiber.StatusNotFound) + } + + foundUrl, ok := matches.First().Attr("href") + if !ok { + fmt.Println("href attribute missing from tag") + return c.SendStatus(fiber.StatusNotFound) + } + c.Set("Location", absoluteUrl(feedUrl, foundUrl)) + if matches.Length() > 1 { + fmt.Println("Multiple feeds found on page") + return c.SendStatus(fiber.StatusMultipleChoices) + } else { + fmt.Println("Feed found on page") + return c.SendStatus(fiber.StatusTemporaryRedirect) + } + } else if err != nil { + fmt.Println("Failed while attempting to parse feed") + return c.SendStatus(fiber.StatusInternalServerError) + } + + fmt.Println("URL provided is already a feed") + c.Set("Location", feedUrl) + return c.SendStatus(fiber.StatusMovedPermanently) + }) + + fmt.Println(app.Listen(fmt.Sprintf(":%s", port))) +} + +func absoluteUrl(requestUrl, foundUrl string) string { + if !strings.HasPrefix(foundUrl, "http") { + parsedUrl, _ := url.Parse(requestUrl) + foundUrl = fmt.Sprintf("%s://%s%s", parsedUrl.Scheme, parsedUrl.Host, foundUrl) + } + + return foundUrl +}