@@ -16,3 +16,4 @@ lerna-debug.log*
|
|||||||
.vscode
|
.vscode
|
||||||
*.code-workspace
|
*.code-workspace
|
||||||
.history/
|
.history/
|
||||||
|
README.md
|
||||||
|
|||||||
@@ -168,3 +168,7 @@ temp/
|
|||||||
Network Trash Folder
|
Network Trash Folder
|
||||||
Temporary Items
|
Temporary Items
|
||||||
.apdisk
|
.apdisk
|
||||||
|
|
||||||
|
|
||||||
|
# App specific
|
||||||
|
config.yaml
|
||||||
|
|||||||
+11
@@ -0,0 +1,11 @@
|
|||||||
|
FROM golang:1.27-alpine AS builder
|
||||||
|
WORKDIR /app/
|
||||||
|
ADD go.mod go.sum ./
|
||||||
|
RUN go mod download
|
||||||
|
ADD . .
|
||||||
|
RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags="-w -s" -o backup-to-nextcloud main.go
|
||||||
|
|
||||||
|
FROM golang:1.27-alpine
|
||||||
|
WORKDIR /app/
|
||||||
|
COPY --from=builder /app/backup-to-nextcloud /app/backup-to-nextcloud
|
||||||
|
ENTRYPOINT ["/app/backup-to-nextcloud"]
|
||||||
|
|||||||
@@ -1,49 +1,33 @@
|
|||||||
.DEFAULT_GOAL := default
|
.DEFAULT_GOAL := default
|
||||||
|
|
||||||
IMAGE ?= rg.fr-par.scw.cloud/averagemarcus-private/backup-to-nextcloud:latest
|
IMAGE ?= rg.fr-par.scw.cloud/averagemarcus/backup-to-nextcloud:latest
|
||||||
|
|
||||||
.PHONY: test # Run all tests, linting and format checks
|
.PHONY: test # Run all tests, linting and format checks
|
||||||
test: lint check-format run-tests
|
test: lint check-format run-tests
|
||||||
|
|
||||||
.PHONY: lint # Perform lint checks against code
|
.PHONY: lint # Perform lint checks against code
|
||||||
lint:
|
lint:
|
||||||
@echo "⚠️ 'lint' unimplemented"
|
@go vet && golint -set_exit_status ./...
|
||||||
# GO Projects
|
|
||||||
# @go vet && golint -set_exit_status ./...
|
|
||||||
|
|
||||||
.PHONY: check-format # Checks code formatting and returns a non-zero exit code if formatting errors found
|
.PHONY: check-format # Checks code formatting and returns a non-zero exit code if formatting errors found
|
||||||
check-format:
|
check-format:
|
||||||
@echo "⚠️ 'check-format' unimplemented"
|
@gofmt -e -l .
|
||||||
# GO Projects
|
|
||||||
# @gofmt -e -l .
|
|
||||||
|
|
||||||
.PHONY: format # Performs automatic format fixes on all code
|
.PHONY: format # Performs automatic format fixes on all code
|
||||||
format:
|
format:
|
||||||
@echo "⚠️ 'format' unimplemented"
|
@gofmt -s -w .
|
||||||
# GO Projects
|
|
||||||
# @gofmt -s -w .
|
|
||||||
|
|
||||||
.PHONY: run-tests # Runs all tests
|
.PHONY: run-tests # Runs all tests
|
||||||
run-tests:
|
run-tests:
|
||||||
@echo "⚠️ 'run-tests' unimplemented"
|
@go test
|
||||||
# GO Projects
|
|
||||||
# @go test
|
|
||||||
# Node Projects
|
|
||||||
# @npm test
|
|
||||||
|
|
||||||
.PHONY: fetch-deps # Fetch all project dependencies
|
.PHONY: fetch-deps # Fetch all project dependencies
|
||||||
fetch-deps:
|
fetch-deps:
|
||||||
@echo "⚠️ 'fetch-deps' unimplemented"
|
@go mod tidy
|
||||||
# GO Projects
|
|
||||||
# @go mod tidy
|
|
||||||
# Node Projects
|
|
||||||
# @npm install
|
|
||||||
|
|
||||||
.PHONY: build # Build the project
|
.PHONY: build # Build the project
|
||||||
build: lint check-format fetch-deps
|
build: lint check-format fetch-deps
|
||||||
@echo "⚠️ 'build' unimplemented"
|
@go build -o backup-to-nextcloud main.go
|
||||||
# GO Projects
|
|
||||||
# @go build -o PROJECT_NAME main.go
|
|
||||||
|
|
||||||
.PHONY: docker-build # Build the docker image
|
.PHONY: docker-build # Build the docker image
|
||||||
docker-build:
|
docker-build:
|
||||||
@@ -55,11 +39,7 @@ docker-publish:
|
|||||||
|
|
||||||
.PHONY: run # Run the application
|
.PHONY: run # Run the application
|
||||||
run:
|
run:
|
||||||
@echo "⚠️ 'run' unimplemented"
|
@go run main.go
|
||||||
# GO Projects
|
|
||||||
# @go run main.go
|
|
||||||
# Node Projects
|
|
||||||
# @npm start
|
|
||||||
|
|
||||||
.PHONY: ci # Perform CI specific tasks to perform on a pull request
|
.PHONY: ci # Perform CI specific tasks to perform on a pull request
|
||||||
ci:
|
ci:
|
||||||
|
|||||||
@@ -1,15 +1,37 @@
|
|||||||
# backup-to-nextcloud
|
# backup-to-nextcloud
|
||||||
|
|
||||||
A simple application that will backup files to Nextcloud
|
A simple application that will backup files to Nextcloud.
|
||||||
|
|
||||||
|
Designed to be run as a CronJob with a focus on directories containing backup archives (e.g. daily zip files).
|
||||||
|
|
||||||
## Features
|
## Features
|
||||||
|
|
||||||
## Install
|
* Define jobs via yaml config
|
||||||
|
* Optionally clean up files on Nextcloud older than a given age
|
||||||
|
* Optionally only store the latest X number of files
|
||||||
|
|
||||||
```sh
|
## Example Config
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
nextcloudURL: https://nextcloud.example.com
|
||||||
|
jobs:
|
||||||
|
- sourceDirectory: backups/example-daily
|
||||||
|
destinationDirectory: backups/example
|
||||||
|
# Optional
|
||||||
|
maxAge: 168h # 7 days
|
||||||
|
# Optional
|
||||||
|
maxItems: 7
|
||||||
```
|
```
|
||||||
|
|
||||||
|
By default, this loads from `./config.yaml` but the file path can be specified by setting the `NEXTCLOUD_CONFIG_PATH` environment variable.
|
||||||
|
|
||||||
|
## Credentials
|
||||||
|
|
||||||
|
Generate an app password for your user in Nextcloud then populate the following environment variables:
|
||||||
|
|
||||||
|
* `NEXTCLOUD_USER`
|
||||||
|
* `NEXTCLOUD_PASSWORD`
|
||||||
|
|
||||||
## Building from source
|
## Building from source
|
||||||
|
|
||||||
With Docker:
|
With Docker:
|
||||||
@@ -26,6 +48,8 @@ make build
|
|||||||
|
|
||||||
## Resources
|
## Resources
|
||||||
|
|
||||||
|
* [Nextcloud WebDav docs](https://docs.nextcloud.com/server/stable/developer_manual/client_apis/WebDAV/basic.html)
|
||||||
|
|
||||||
## Contributing
|
## Contributing
|
||||||
|
|
||||||
If you find a bug or have an idea for a new feature please [raise an issue](issues/new) to discuss it.
|
If you find a bug or have an idea for a new feature please [raise an issue](issues/new) to discuss it.
|
||||||
|
|||||||
@@ -0,0 +1,180 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/xml"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Client struct {
|
||||||
|
nextcloudURL string
|
||||||
|
username string
|
||||||
|
password string
|
||||||
|
}
|
||||||
|
|
||||||
|
type DirectoryItem struct {
|
||||||
|
Directory string
|
||||||
|
FileName string
|
||||||
|
LastModified *time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewClient(nextcloudURL, username, password string) *Client {
|
||||||
|
return &Client{
|
||||||
|
nextcloudURL: nextcloudURL,
|
||||||
|
username: username,
|
||||||
|
password: password,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) CreateDestDirectory(ctx context.Context, destinationDirectory string) error {
|
||||||
|
path := fmt.Sprintf("%s/remote.php/dav/files/%s/%s", c.nextcloudURL, c.username, destinationDirectory)
|
||||||
|
method := "MKCOL"
|
||||||
|
|
||||||
|
response, err := c.doRequest(ctx, path, method, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
if strings.Contains(response, "The resource you tried to create already exists") {
|
||||||
|
// Directory already exists so safe to skip
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) UploadFile(ctx context.Context, destinationDirectory string, file *os.File) error {
|
||||||
|
path := fmt.Sprintf("%s/remote.php/dav/files/%s/%s/%s", c.nextcloudURL, c.username, destinationDirectory, filepath.Base(file.Name()))
|
||||||
|
method := "PUT"
|
||||||
|
|
||||||
|
headers := map[string]string{}
|
||||||
|
info, err := os.Stat(file.Name())
|
||||||
|
if err == nil {
|
||||||
|
headers["X-OC-Mtime"] = fmt.Sprintf("%d", info.ModTime().Unix())
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = c.doRequest(ctx, path, method, file, headers)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) ListDirectoryContents(ctx context.Context, destinationDirectory string) ([]DirectoryItem, error) {
|
||||||
|
directoryItems := []DirectoryItem{}
|
||||||
|
|
||||||
|
path := fmt.Sprintf("%s/remote.php/dav/files/%s/%s", c.nextcloudURL, c.username, destinationDirectory)
|
||||||
|
method := "PROPFIND"
|
||||||
|
|
||||||
|
response, err := c.doRequest(ctx, path, method, nil, nil)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return directoryItems, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var directoryContents Multistatus
|
||||||
|
if err := xml.Unmarshal([]byte(response), &directoryContents); err != nil {
|
||||||
|
logger.ErrorContext(ctx, "Error unmarshalling XML response",
|
||||||
|
slog.String("destinationDirectory", destinationDirectory),
|
||||||
|
slog.Any("error", err),
|
||||||
|
)
|
||||||
|
return directoryItems, err
|
||||||
|
}
|
||||||
|
|
||||||
|
hrefPrefix := fmt.Sprintf("/remote.php/dav/files/%s/%s/", c.username, destinationDirectory)
|
||||||
|
|
||||||
|
for _, item := range directoryContents.Responses {
|
||||||
|
if !item.IsCollection() {
|
||||||
|
var lastModified *time.Time
|
||||||
|
|
||||||
|
if item.GetLastModified() != "" {
|
||||||
|
parsedDate, err := time.Parse(time.RFC1123, item.GetLastModified())
|
||||||
|
if err == nil {
|
||||||
|
lastModified = &parsedDate
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
file := DirectoryItem{
|
||||||
|
Directory: destinationDirectory,
|
||||||
|
FileName: strings.TrimPrefix(item.Href, hrefPrefix),
|
||||||
|
LastModified: lastModified,
|
||||||
|
}
|
||||||
|
directoryItems = append(directoryItems, file)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return directoryItems, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) DeleteFile(ctx context.Context, destinationDirectory, fileName string) error {
|
||||||
|
path := fmt.Sprintf("%s/remote.php/dav/files/%s/%s/%s", c.nextcloudURL, c.username, destinationDirectory, fileName)
|
||||||
|
method := "DELETE"
|
||||||
|
|
||||||
|
_, err := c.doRequest(ctx, path, method, nil, nil)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) doRequest(ctx context.Context, path, method string, payload *os.File, additionalHeaders map[string]string) (string, error) {
|
||||||
|
client := &http.Client{}
|
||||||
|
|
||||||
|
var err error
|
||||||
|
var req *http.Request
|
||||||
|
|
||||||
|
if payload != nil {
|
||||||
|
req, err = http.NewRequestWithContext(ctx, method, path, payload)
|
||||||
|
} else {
|
||||||
|
req, err = http.NewRequestWithContext(ctx, method, path, nil)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
logger.ErrorContext(ctx, "Failed to create request",
|
||||||
|
slog.String("path", path),
|
||||||
|
slog.String("method", method),
|
||||||
|
slog.Any("error", err),
|
||||||
|
)
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
req.SetBasicAuth(c.username, c.password)
|
||||||
|
|
||||||
|
if payload != nil {
|
||||||
|
req.Header.Set("Content-Type", "application/octet-stream")
|
||||||
|
}
|
||||||
|
|
||||||
|
for key, val := range additionalHeaders {
|
||||||
|
req.Header.Set(key, val)
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
logger.ErrorContext(ctx, "Failed to perform request",
|
||||||
|
slog.String("path", path),
|
||||||
|
slog.String("method", method),
|
||||||
|
slog.Any("error", err),
|
||||||
|
)
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
bodyBytes, _ := io.ReadAll(resp.Body)
|
||||||
|
|
||||||
|
if resp.StatusCode >= 400 {
|
||||||
|
logger.ErrorContext(ctx, "Unexpected error response returned",
|
||||||
|
slog.String("path", path),
|
||||||
|
slog.String("method", method),
|
||||||
|
slog.Int("statusCode", resp.StatusCode),
|
||||||
|
slog.String("statusMessage", resp.Status),
|
||||||
|
slog.String("response", string(bodyBytes)),
|
||||||
|
)
|
||||||
|
return string(bodyBytes), fmt.Errorf("Unexpected error response returned")
|
||||||
|
} else {
|
||||||
|
logger.DebugContext(ctx, "Request successful",
|
||||||
|
slog.String("path", path),
|
||||||
|
slog.String("method", method),
|
||||||
|
slog.Int("statusCode", resp.StatusCode),
|
||||||
|
slog.String("statusMessage", resp.Status),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return string(bodyBytes), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
module backup-to-nextcloud
|
||||||
|
|
||||||
|
go 1.27.1
|
||||||
|
|
||||||
|
require (
|
||||||
|
go.opentelemetry.io/otel/trace v1.46.0
|
||||||
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
|
)
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||||
|
go.opentelemetry.io/otel v1.46.0 // indirect
|
||||||
|
)
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||||
|
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||||
|
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||||
|
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||||
|
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
|
||||||
|
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
|
||||||
|
go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc=
|
||||||
|
go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE=
|
||||||
|
go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c=
|
||||||
|
go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI=
|
||||||
|
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||||
|
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||||
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||||
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
@@ -0,0 +1,259 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
CONFIG_PATH_VAR = "NEXTCLOUD_CONFIG_PATH"
|
||||||
|
NEXTCLOUD_USER_VAR = "NEXTCLOUD_USER"
|
||||||
|
NEXTCLOUD_PASSWORD_VAR = "NEXTCLOUD_PASSWORD"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
config Config
|
||||||
|
logger *slog.Logger
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
jsonHandler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
|
||||||
|
Level: slog.LevelDebug,
|
||||||
|
AddSource: true,
|
||||||
|
})
|
||||||
|
logger = slog.New(jsonHandler)
|
||||||
|
slog.SetDefault(logger)
|
||||||
|
|
||||||
|
configPath, exists := os.LookupEnv(CONFIG_PATH_VAR)
|
||||||
|
if !exists {
|
||||||
|
configPath = "./config.yaml"
|
||||||
|
}
|
||||||
|
logger.InfoContext(ctx,
|
||||||
|
"Using config path",
|
||||||
|
slog.String("configPath", configPath),
|
||||||
|
)
|
||||||
|
|
||||||
|
file, err := os.ReadFile(configPath)
|
||||||
|
if err != nil {
|
||||||
|
logger.ErrorContext(ctx, "Error reading config file", slog.Any("error", err))
|
||||||
|
os.Exit(1)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
err = yaml.Unmarshal(file, &config)
|
||||||
|
if err != nil {
|
||||||
|
logger.ErrorContext(ctx, "Error parsing config file", slog.Any("error", err))
|
||||||
|
os.Exit(1)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case os.Getenv(NEXTCLOUD_USER_VAR) == "":
|
||||||
|
logger.ErrorContext(ctx, "NEXTCLOUD_USER environment variable is not set")
|
||||||
|
os.Exit(1)
|
||||||
|
return
|
||||||
|
case os.Getenv(NEXTCLOUD_PASSWORD_VAR) == "":
|
||||||
|
logger.ErrorContext(ctx, "NEXTCLOUD_PASSWORD environment variable is not set")
|
||||||
|
os.Exit(1)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
if err := config.Validate(); err != nil {
|
||||||
|
logger.ErrorContext(ctx, "Invalid config", slog.Any("error", err))
|
||||||
|
os.Exit(1)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
client := NewClient(config.NextcloudURL, os.Getenv(NEXTCLOUD_USER_VAR), os.Getenv(NEXTCLOUD_PASSWORD_VAR))
|
||||||
|
|
||||||
|
for _, job := range config.Jobs {
|
||||||
|
err := processJob(ctx, client, job)
|
||||||
|
if err != nil {
|
||||||
|
logger.ErrorContext(ctx, "Failed to process job",
|
||||||
|
slog.String("sourceDirectory", job.SourceDirectory),
|
||||||
|
slog.String("destinationDirectory", job.DestinationDirectory),
|
||||||
|
slog.Any("error", err),
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
} else {
|
||||||
|
logger.DebugContext(ctx, "Finished processing job",
|
||||||
|
slog.String("sourceDirectory", job.SourceDirectory),
|
||||||
|
slog.String("destinationDirectory", job.DestinationDirectory),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func processJob(ctx context.Context, client *Client, job Job) error {
|
||||||
|
// 1. Check that we actually have files to upload...
|
||||||
|
files, err := os.ReadDir(job.SourceDirectory)
|
||||||
|
if err != nil {
|
||||||
|
logger.ErrorContext(ctx, "Failed to read source directory",
|
||||||
|
slog.String("sourceDirectory", job.SourceDirectory),
|
||||||
|
slog.Any("error", err),
|
||||||
|
)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(files) == 0 {
|
||||||
|
logger.InfoContext(ctx, "No files found in source directory, skipping job",
|
||||||
|
slog.String("sourceDirectory", job.SourceDirectory),
|
||||||
|
)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Ensure the destination exists in Nextcloud
|
||||||
|
if err := client.CreateDestDirectory(ctx, job.DestinationDirectory); err != nil {
|
||||||
|
logger.ErrorContext(ctx, "Failed to create destination directory in Nextcloud",
|
||||||
|
slog.String("destinationDirectory", job.DestinationDirectory),
|
||||||
|
slog.Any("error", err),
|
||||||
|
)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Upload files
|
||||||
|
for _, file := range files {
|
||||||
|
if file.IsDir() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
fileName := filepath.Base(file.Name())
|
||||||
|
filePath := fmt.Sprintf("%s/%s", job.SourceDirectory, fileName)
|
||||||
|
|
||||||
|
file, err := os.Open(filePath)
|
||||||
|
if err != nil {
|
||||||
|
logger.ErrorContext(ctx, "Error opening file",
|
||||||
|
slog.String("sourceDirectory", job.SourceDirectory),
|
||||||
|
slog.String("fileName", fileName),
|
||||||
|
slog.String("filePath", filePath),
|
||||||
|
slog.Any("error", err),
|
||||||
|
)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
logger.DebugContext(ctx, "Uploading file",
|
||||||
|
slog.String("sourceDirectory", job.SourceDirectory),
|
||||||
|
slog.String("destinationDirectory", job.DestinationDirectory),
|
||||||
|
slog.String("fileName", fileName),
|
||||||
|
slog.String("filePath", filePath),
|
||||||
|
)
|
||||||
|
if err := client.UploadFile(ctx, job.DestinationDirectory, file); err != nil {
|
||||||
|
logger.ErrorContext(ctx, "Failed to upload file",
|
||||||
|
slog.String("sourceDirectory", job.SourceDirectory),
|
||||||
|
slog.String("destinationDirectory", job.DestinationDirectory),
|
||||||
|
slog.String("fileName", fileName),
|
||||||
|
slog.String("filePath", filePath),
|
||||||
|
slog.Any("error", err),
|
||||||
|
)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. Remove any files older than the max age specified
|
||||||
|
if job.MaxAge != nil {
|
||||||
|
duration, _ := time.ParseDuration(*job.MaxAge)
|
||||||
|
threshold := time.Now().Add(-duration)
|
||||||
|
logger.InfoContext(ctx, "Removing files older than max age",
|
||||||
|
slog.String("destinationDirectory", job.DestinationDirectory),
|
||||||
|
slog.String("maxAge", *job.MaxAge),
|
||||||
|
slog.Time("dateThreshold", threshold),
|
||||||
|
)
|
||||||
|
|
||||||
|
contents, err := client.ListDirectoryContents(ctx, job.DestinationDirectory)
|
||||||
|
if err != nil {
|
||||||
|
logger.ErrorContext(ctx, "Failed to list directory contents",
|
||||||
|
slog.String("sourceDirectory", job.SourceDirectory),
|
||||||
|
slog.String("destinationDirectory", job.DestinationDirectory),
|
||||||
|
slog.Any("error", err),
|
||||||
|
)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, file := range contents {
|
||||||
|
if file.LastModified != nil {
|
||||||
|
if file.LastModified.Before(threshold) {
|
||||||
|
logger.InfoContext(ctx, "Removing old file",
|
||||||
|
slog.String("destinationDirectory", job.DestinationDirectory),
|
||||||
|
slog.String("fileName", file.FileName),
|
||||||
|
slog.Time("lastModified", *file.LastModified),
|
||||||
|
)
|
||||||
|
if err := client.DeleteFile(ctx, job.DestinationDirectory, file.FileName); err != nil {
|
||||||
|
logger.ErrorContext(ctx, "Failed to delete file",
|
||||||
|
slog.String("destinationDirectory", job.DestinationDirectory),
|
||||||
|
slog.String("fileName", file.FileName),
|
||||||
|
slog.Any("error", err),
|
||||||
|
)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 5. Ensure only newest X files are kept
|
||||||
|
if job.MaxItems != nil {
|
||||||
|
logger.InfoContext(ctx, "Removing excess files",
|
||||||
|
slog.String("destinationDirectory", job.DestinationDirectory),
|
||||||
|
slog.Int("maxItem", *job.MaxItems),
|
||||||
|
)
|
||||||
|
|
||||||
|
contents, err := client.ListDirectoryContents(ctx, job.DestinationDirectory)
|
||||||
|
if err != nil {
|
||||||
|
logger.ErrorContext(ctx, "Failed to list directory contents",
|
||||||
|
slog.String("sourceDirectory", job.SourceDirectory),
|
||||||
|
slog.String("destinationDirectory", job.DestinationDirectory),
|
||||||
|
slog.Any("error", err),
|
||||||
|
)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.SliceStable(contents, func(i, j int) bool {
|
||||||
|
if contents[i].LastModified == nil && contents[j].LastModified == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if contents[i].LastModified == nil {
|
||||||
|
return false // nil goes to the end
|
||||||
|
}
|
||||||
|
if contents[j].LastModified == nil {
|
||||||
|
return true // nil goes to the end
|
||||||
|
}
|
||||||
|
return contents[i].LastModified.After(*contents[j].LastModified) // flipped: After instead of Before
|
||||||
|
})
|
||||||
|
|
||||||
|
if len(contents) > *job.MaxItems {
|
||||||
|
for _, file := range contents[*job.MaxItems:] {
|
||||||
|
logger.InfoContext(ctx, "Removing old files until max files met",
|
||||||
|
slog.String("destinationDirectory", job.DestinationDirectory),
|
||||||
|
slog.String("fileName", file.FileName),
|
||||||
|
slog.Time("lastModified", *file.LastModified),
|
||||||
|
slog.String("maxItems", *job.MaxAge),
|
||||||
|
)
|
||||||
|
if err := client.DeleteFile(ctx, job.DestinationDirectory, file.FileName); err != nil {
|
||||||
|
logger.ErrorContext(ctx, "Failed to delete file",
|
||||||
|
slog.String("destinationDirectory", job.DestinationDirectory),
|
||||||
|
slog.String("fileName", file.FileName),
|
||||||
|
slog.String("maxItems", *job.MaxAge),
|
||||||
|
slog.Any("error", err),
|
||||||
|
)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
+268
@@ -0,0 +1,268 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/xml"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Namespace prefixes used in the DAV XML
|
||||||
|
const (
|
||||||
|
NSDAV = "DAV:"
|
||||||
|
NSSabre = "http://sabredav.org/ns"
|
||||||
|
NSOwn = "http://owncloud.org/ns"
|
||||||
|
NSNextcl = "http://nextcloud.org/ns"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Multistatus represents the root <d:multistatus> element
|
||||||
|
type Multistatus struct {
|
||||||
|
Responses []Response `xml:"DAV:response"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Response represents a <d:response> element containing path and status info
|
||||||
|
type Response struct {
|
||||||
|
Href string `xml:"DAV:href"`
|
||||||
|
Propstats []Propstat `xml:"DAV:propstat"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Propstat contains a set of properties and their HTTP status
|
||||||
|
type Propstat struct {
|
||||||
|
Prop Prop `xml:"DAV:prop"`
|
||||||
|
Status string `xml:"DAV:status"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Prop holds the various DAV properties (getlastmodified, getetag, etc.)
|
||||||
|
type Prop struct {
|
||||||
|
GetLastModified string `xml:"DAV:getlastmodified"`
|
||||||
|
GetContentLength *int64 `xml:"DAV:getcontentlength"`
|
||||||
|
GetContentType string `xml:"DAV:getcontenttype"`
|
||||||
|
GetETag string `xml:"DAV:getetag"`
|
||||||
|
QuotaUsedBytes *int64 `xml:"DAV:quota-used-bytes"`
|
||||||
|
QuotaAvailableBytes *int64 `xml:"DAV:quota-available-bytes"`
|
||||||
|
// ResourceType is special - it's a single element that may contain
|
||||||
|
// a <d:collection/> child (indicating a directory) or be empty (file)
|
||||||
|
ResourceType ResourceType `xml:"DAV:resourcetype"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResourceType determines if a resource is a collection (directory) or a file
|
||||||
|
type ResourceType struct {
|
||||||
|
Collection bool `xml:"DAV:collection"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// UnmarshalXML implements custom XML unmarshaling for Prop
|
||||||
|
// to handle properties that may not always be present
|
||||||
|
func (p *Prop) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||||
|
for {
|
||||||
|
tok, err := d.Token()
|
||||||
|
if err != nil {
|
||||||
|
if err == io.EOF {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
switch elem := tok.(type) {
|
||||||
|
case xml.StartElement:
|
||||||
|
switch elem.Name.Local {
|
||||||
|
case "getlastmodified":
|
||||||
|
var content string
|
||||||
|
if err := d.DecodeElement(&content, &elem); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
p.GetLastModified = content
|
||||||
|
case "getcontentlength":
|
||||||
|
var content string
|
||||||
|
if err := d.DecodeElement(&content, &elem); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if val, err := parseInt64(content); err == nil {
|
||||||
|
p.GetContentLength = &val
|
||||||
|
}
|
||||||
|
case "getcontenttype":
|
||||||
|
var content string
|
||||||
|
if err := d.DecodeElement(&content, &elem); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
p.GetContentType = content
|
||||||
|
case "getetag":
|
||||||
|
var content string
|
||||||
|
if err := d.DecodeElement(&content, &elem); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
p.GetETag = content
|
||||||
|
case "quota-used-bytes":
|
||||||
|
var content string
|
||||||
|
if err := d.DecodeElement(&content, &elem); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if val, err := parseInt64(content); err == nil {
|
||||||
|
p.QuotaUsedBytes = &val
|
||||||
|
}
|
||||||
|
case "quota-available-bytes":
|
||||||
|
var content string
|
||||||
|
if err := d.DecodeElement(&content, &elem); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if val, err := parseInt64(content); err == nil {
|
||||||
|
p.QuotaAvailableBytes = &val
|
||||||
|
}
|
||||||
|
case "resourcetype":
|
||||||
|
// For resourcetype, we need to check if it contains <collection/>
|
||||||
|
// rather than just counting children
|
||||||
|
for {
|
||||||
|
innerTok, err := d.Token()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if innerTok, ok := innerTok.(xml.EndElement); ok && innerTok.Name.Local == "resourcetype" {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if innerTok, ok := innerTok.(xml.StartElement); ok && innerTok.Name.Local == "collection" {
|
||||||
|
p.ResourceType.Collection = true
|
||||||
|
// Skip past the collection element
|
||||||
|
for {
|
||||||
|
innerTok, err := d.Token()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if end, ok := innerTok.(xml.EndElement); ok && end.Name.Local == "collection" {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case xml.EndElement:
|
||||||
|
if elem.Name.Local == start.Name.Local {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// UnmarshalXML handles multistatus response parsing
|
||||||
|
func (m *Multistatus) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||||
|
m.Responses = []Response{}
|
||||||
|
|
||||||
|
for {
|
||||||
|
tok, err := d.Token()
|
||||||
|
if err != nil {
|
||||||
|
if err == io.EOF {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if start, ok := tok.(xml.StartElement); ok && start.Name.Local == "response" {
|
||||||
|
var resp Response
|
||||||
|
if err := d.DecodeElement(&resp, &start); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
m.Responses = append(m.Responses, resp)
|
||||||
|
}
|
||||||
|
|
||||||
|
if end, ok := tok.(xml.EndElement); ok && end.Name.Local == "multistatus" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// UnmarshalXML handles response parsing
|
||||||
|
func (r *Response) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||||
|
for {
|
||||||
|
tok, err := d.Token()
|
||||||
|
if err != nil {
|
||||||
|
if err == io.EOF {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
switch elem := tok.(type) {
|
||||||
|
case xml.StartElement:
|
||||||
|
switch elem.Name.Local {
|
||||||
|
case "href":
|
||||||
|
if err := d.DecodeElement(&r.Href, &elem); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
case "propstat":
|
||||||
|
var ps Propstat
|
||||||
|
if err := d.DecodeElement(&ps, &elem); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
r.Propstats = append(r.Propstats, ps)
|
||||||
|
}
|
||||||
|
case xml.EndElement:
|
||||||
|
if elem.Name.Local == start.Name.Local {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// UnmarshalXML handles propstat parsing
|
||||||
|
func (p *Propstat) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||||
|
for {
|
||||||
|
tok, err := d.Token()
|
||||||
|
if err != nil {
|
||||||
|
if err == io.EOF {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
switch elem := tok.(type) {
|
||||||
|
case xml.StartElement:
|
||||||
|
switch elem.Name.Local {
|
||||||
|
case "prop":
|
||||||
|
if err := d.DecodeElement(&p.Prop, &elem); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
case "status":
|
||||||
|
var content string
|
||||||
|
if err := d.DecodeElement(&content, &elem); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
p.Status = content
|
||||||
|
}
|
||||||
|
case xml.EndElement:
|
||||||
|
if elem.Name.Local == start.Name.Local {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// RemoveQuotes removes surrounding double quotes from a string (for etags like `"abc123"`)
|
||||||
|
func RemoveQuotes(s string) string {
|
||||||
|
if len(s) >= 2 && s[0] == '"' && s[len(s)-1] == '"' {
|
||||||
|
return s[1 : len(s)-1]
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseInt64 converts a string to int64, returning 0 and error on failure
|
||||||
|
func parseInt64(s string) (int64, error) {
|
||||||
|
var val int64
|
||||||
|
_, err := fmt.Sscanf(s, "%d", &val)
|
||||||
|
return val, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsCollection returns true if the resource is a directory
|
||||||
|
func (r *Response) IsCollection() bool {
|
||||||
|
for _, ps := range r.Propstats {
|
||||||
|
if ps.Prop.ResourceType.Collection {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Response) GetLastModified() string {
|
||||||
|
for _, ps := range r.Propstats {
|
||||||
|
if ps.Prop.GetLastModified != "" {
|
||||||
|
return ps.Prop.GetLastModified
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Config struct {
|
||||||
|
NextcloudURL string `yaml:"nextcloudURL"`
|
||||||
|
Jobs []Job `yaml:"jobs"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Job struct {
|
||||||
|
SourceDirectory string `yaml:"sourceDirectory"`
|
||||||
|
DestinationDirectory string `yaml:"destinationDirectory"`
|
||||||
|
MaxItems *int `yaml:"maxItems"`
|
||||||
|
MaxAge *string `yaml:"maxAge"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c Config) Validate() error {
|
||||||
|
if c.NextcloudURL == "" {
|
||||||
|
return fmt.Errorf("nextcloudURL is required")
|
||||||
|
}
|
||||||
|
for _, job := range c.Jobs {
|
||||||
|
if job.SourceDirectory == "" {
|
||||||
|
return fmt.Errorf("sourceDirectory is required for job")
|
||||||
|
}
|
||||||
|
if job.DestinationDirectory == "" {
|
||||||
|
return fmt.Errorf("destinationDirectory is required for job")
|
||||||
|
}
|
||||||
|
if job.MaxItems != nil && *job.MaxItems <= 0 {
|
||||||
|
return fmt.Errorf("maxItems must be greater than 0 for job")
|
||||||
|
}
|
||||||
|
if job.MaxAge != nil {
|
||||||
|
if _, err := time.ParseDuration(*job.MaxAge); err != nil {
|
||||||
|
return fmt.Errorf("maxAge must be a valid duration for job - %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user