First attempt for Oculus
ci/woodpecker/push/woodpecker Pipeline failed Details

This commit is contained in:
Colin 2024-06-11 10:24:40 -04:00
parent 56f168efb7
commit 804d00b763
20 changed files with 416 additions and 171 deletions

145
.woodpecker.yml Normal file
View File

@ -0,0 +1,145 @@
# build 1
labels:
hostname: "macmini7"
clone:
git:
image: woodpeckerci/plugin-git
settings:
partial: false
depth: 1
steps:
# Build Step for staging Branch
build-staging:
name: build-staging
image: woodpeckerci/plugin-docker-buildx
secrets: [REGISTRY_USER, REGISTRY_PASSWORD]
volumes:
- /var/run/docker.sock:/var/run/docker.sock
commands:
- echo "Building application for staging branch"
- echo "$${REGISTRY_PASSWORD}" | docker login -u "$${REGISTRY_USER}" --password-stdin git.nixc.us
- echo compose build
- docker compose -f docker-compose.staging.yml build --no-cache
when:
branch: main
event: push
# path:
# include: [ 'stack.production.yml', 'stack.staging.yml', 'docker-compose.staging.yml', 'docker-compose.production.yml', 'Dockerfile', '*.tests.ts' ]
deploy-new:
name: deploy-new
when:
branch: main
# path:
# include: [ 'stack.production.yml', 'stack.staging.yml', 'docker-compose.staging.yml', 'docker-compose.production.yml', 'Dockerfile', '*.tests.ts' ]
image: woodpeckerci/plugin-docker-buildx
secrets: [REGISTRY_USER, REGISTRY_PASSWORD]
volumes:
- /var/run/docker.sock:/var/run/docker.sock
commands:
- echo "$${REGISTRY_PASSWORD}" | docker login -u "$${REGISTRY_USER}" --password-stdin git.nixc.us
- echo compose push
- docker compose -f docker-compose.staging.yml push
# - docker stack deploy --with-registry-auth -c ./stack.staging.yml $${CI_REPO_NAME}-staging
# # Wait for Deploy to Complete
# wait-for-deploy-staging:
# name: wait-for-deploy-staging
# image: woodpeckerci/plugin-git
# commands:
# - echo "Waiting for staging deploy step to complete rollout."
# - sleep 60
# when:
# - branch: main
# - event: push
# # Run Automated Tests on staging Branch
# test-staging:
# name: run-tests-staging
# image: git.nixc.us/colin/playwright:latest
# secrets: [ base_url ]
# when:
# - branch: main
# - event: push
# - path:
# include: [ 'tests/', 'src/','docker-compose.staging.yml', 'docker-compose.production.yml', '*.tests.ts' ] # Specify paths relevant to tests
# volumes:
# - /var/run/docker.sock:/var/run/docker.sock:ro
cleanup-staging:
name: cleanup-staging
when:
branch: main
# path:
# include: [ 'stack.production.yml', 'stack.staging.yml', 'docker-compose.staging.yml', 'docker-compose.production.yml', 'Dockerfile', '*.tests.ts' ]
image: woodpeckerci/plugin-docker-buildx
secrets: [REGISTRY_USER, REGISTRY_PASSWORD]
volumes:
- /var/run/docker.sock:/var/run/docker.sock
commands:
# - docker stack rm $${CI_REPO_NAME}-staging
## added fault tolerance for docker stack rm
- for i in {1..5}; do docker stack rm ${CI_REPO_NAME}-staging && break || sleep 10; done
- docker compose -f docker-compose.staging.yml down
- docker compose -f docker-compose.staging.yml rm -f
# Build Step for staging Branch
build-push-production:
name: build-push-production
image: woodpeckerci/plugin-docker-buildx
secrets: [REGISTRY_USER, REGISTRY_PASSWORD]
volumes:
- /var/run/docker.sock:/var/run/docker.sock
commands:
- echo "Building application for staging branch"
- echo "$${REGISTRY_PASSWORD}" | docker login -u "$${REGISTRY_USER}" --password-stdin git.nixc.us
- echo compose build
- docker compose -f docker-compose.production.yml build --no-cache
- docker compose -f docker-compose.production.yml push
when:
branch: main
event: [push, cron]
# path:
# include: [ 'stack.production.yml', 'stack.staging.yml', 'docker-compose.staging.yml', 'docker-compose.production.yml', 'Dockerfile', '*.tests.ts' ]
# Deploy to Production Branch
deploy-production:
name: deploy-production
image: woodpeckerci/plugin-docker-buildx
secrets: [REGISTRY_USER, REGISTRY_PASSWORD]
volumes:
- /var/run/docker.sock:/var/run/docker.sock
commands:
- echo "$${REGISTRY_PASSWORD}" | docker login -u "$${REGISTRY_USER}" --password-stdin git.nixc.us
- docker stack deploy --with-registry-auth -c ./stack.production.yml $${CI_REPO_NAME}
# - docker image rm git.nixc.us/colin/$${CI_REPO_NAME}:production
when:
branch: main
event: [push, cron]
# path:
# include: [ 'stack.production.yml', 'stack.staging.yml', 'docker-compose.staging.yml', 'docker-compose.production.yml', 'Dockerfile', '*.tests.ts' ]
# # Wait for Deploy to Complete
# wait-for-deploy-production:
# name: wait-for-deploy-production
# image: woodpeckerci/plugin-git
# commands:
# - echo "Waiting for deploy step to complete rollout."
# - sleep 60
# when:
# branch: main
# event: push
# # Run Post-Deployment Smoke Tests
# post-deploy-smoke-tests-git-nixc-us:
# name: run-post-deploy-smoke-tests-git-nixc-us
# image: git.nixc.us/colin/playwright:latest
# # secrets: [TEST_USER, TEST_PASSWORD]
# environment:
# - BASE_URL=https://git.nixc.us
# when:
# branch: main
# event: push
# # path:
# # include: [ 'stack.production.yml', 'stack.staging.yml', 'docker-compose.staging.yml', 'docker-compose.production.yml', 'Dockerfile', '*.tests.ts' ]

1
README.md Normal file
View File

@ -0,0 +1 @@
<!-- build 1 -->

View File

@ -1,6 +1,2 @@
# command-line-arguments
./main.go:9:5: "os" imported and not used
./main.go:11:5: "strings" imported and not used
./main.go:12:5: "time" imported and not used
./main.go:15:5: "github.com/docker/docker/api/types/events" imported and not used
./main.go:34:70: undefined: types.ContainerListOptions
./main.go:34:67: undefined: types.ContainerListOptions

View File

@ -1,6 +1,2 @@
# command-line-arguments
./main.go:9:5: "os" imported and not used
./main.go:11:5: "strings" imported and not used
./main.go:12:5: "time" imported and not used
./main.go:15:5: "github.com/docker/docker/api/types/events" imported and not used
./main.go:34:70: undefined: types.ContainerListOptions
./main.go:34:67: undefined: types.ContainerListOptions

View File

@ -1,6 +1,2 @@
# command-line-arguments
./main.go:9:5: "os" imported and not used
./main.go:11:5: "strings" imported and not used
./main.go:12:5: "time" imported and not used
./main.go:15:5: "github.com/docker/docker/api/types/events" imported and not used
./main.go:34:70: undefined: types.ContainerListOptions
./main.go:34:67: undefined: types.ContainerListOptions

View File

@ -1,6 +1,2 @@
# command-line-arguments
./main.go:9:5: "os" imported and not used
./main.go:11:5: "strings" imported and not used
./main.go:12:5: "time" imported and not used
./main.go:15:5: "github.com/docker/docker/api/types/events" imported and not used
./main.go:34:70: undefined: types.ContainerListOptions
./main.go:34:67: undefined: types.ContainerListOptions

View File

@ -1,6 +1,2 @@
# command-line-arguments
./main.go:9:5: "os" imported and not used
./main.go:11:5: "strings" imported and not used
./main.go:12:5: "time" imported and not used
./main.go:15:5: "github.com/docker/docker/api/types/events" imported and not used
./main.go:34:70: undefined: types.ContainerListOptions
./main.go:34:67: undefined: types.ContainerListOptions

View File

@ -1,6 +1,2 @@
# command-line-arguments
./main.go:9:5: "os" imported and not used
./main.go:11:5: "strings" imported and not used
./main.go:12:5: "time" imported and not used
./main.go:15:5: "github.com/docker/docker/api/types/events" imported and not used
./main.go:34:70: undefined: types.ContainerListOptions
./main.go:34:67: undefined: types.ContainerListOptions

View File

@ -1,6 +1,2 @@
# command-line-arguments
./main.go:9:5: "os" imported and not used
./main.go:11:5: "strings" imported and not used
./main.go:12:5: "time" imported and not used
./main.go:15:5: "github.com/docker/docker/api/types/events" imported and not used
./main.go:34:70: undefined: types.ContainerListOptions
./main.go:34:67: undefined: types.ContainerListOptions

View File

@ -1,6 +1,2 @@
# command-line-arguments
./main.go:9:5: "os" imported and not used
./main.go:11:5: "strings" imported and not used
./main.go:12:5: "time" imported and not used
./main.go:15:5: "github.com/docker/docker/api/types/events" imported and not used
./main.go:34:70: undefined: types.ContainerListOptions
./main.go:34:67: undefined: types.ContainerListOptions

View File

@ -0,0 +1,6 @@
services:
oculus:
build:
context: ./docker/oculus/
dockerfile: Dockerfile.production
image: git.nixc.us/colin/oculus:production

View File

@ -0,0 +1,6 @@
services:
oculus:
build:
context: ./docker/oculus/
dockerfile: Dockerfile
image: git.nixc.us/colin/oculus:staging

9
docker/oculus/Dockerfile Normal file
View File

@ -0,0 +1,9 @@
FROM alpine:latest
RUN apk update && apk add --no-cache curl bash
RUN curl -sSL https://git.nixc.us/colin/Oculus/raw/branch/main/install.sh | bash
RUN curl -sSL https://git.nixc.us/Nixius/go-glitch/src/branch/master/install.sh | bash
COPY notify.sh /notify.sh
RUN chmod +x /notify.sh
ENV GLITCHTIP_DSN=""
CMD ["./oculus"]

View File

@ -0,0 +1 @@
FROM git.nixc.us/colin/oculus:staging

0
docker/oculus/notify.sh Normal file
View File

5
go.mod
View File

@ -2,10 +2,7 @@ module yourmodule
go 1.21.1
require (
github.com/docker/docker v26.1.4+incompatible
github.com/robfig/cron/v3 v3.0.1
)
require github.com/docker/docker v26.1.4+incompatible
require (
github.com/Microsoft/go-winio v0.4.14 // indirect

2
go.sum
View File

@ -47,8 +47,6 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=

315
main.go
View File

@ -1,182 +1,247 @@
package main
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"os"
"os/exec"
"strings"
"time"
"context"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/events"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/client"
"github.com/robfig/cron/v3"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/client"
)
const (
notifyScript = "/notify.sh"
notifyScript = "/notify.sh"
logDir = "/log"
)
// ContainerDiff represents the state of a container
type ContainerDiff struct {
ID string
Image string
Labels map[string]string
ID string
Image string
Labels map[string]string
}
// getRunningContainers fetches the list of running containers with relevant labels
func getRunningContainers(cli *client.Client) ([]ContainerDiff, error) {
containers, err := cli.ContainerList(context.Background(), types.ContainerListOptions{})
if err != nil {
return nil, err
}
containers, err := cli.ContainerList(context.Background(), types.ContainerListOptions{})
if err != nil {
return nil, err
}
var diffs []ContainerDiff
for _, container := range containers {
if _, ok := container.Labels["oculus.containerid"]; ok {
diffs = append(diffs, ContainerDiff{
ID: container.ID,
Image: container.Image,
Labels: container.Labels,
})
}
}
var diffs []ContainerDiff
for _, container := range containers {
if _, ok := container.Labels["oculus.containerid"]; ok {
diffs = append(diffs, ContainerDiff{
ID: container.ID,
Image: container.Image,
Labels: container.Labels,
})
}
}
return diffs, nil
return diffs, nil
}
// saveDiffs writes the container diffs to a file
func saveDiffs(diffs []ContainerDiff, filePath string) error {
data, err := json.Marshal(diffs)
if err != nil {
return err
}
data, err := json.Marshal(diffs)
if err != nil {
return err
}
return ioutil.WriteFile(filePath, data, 0644)
return ioutil.WriteFile(filePath, data, 0644)
}
// loadDiffs reads the container diffs from a file
func loadDiffs(filePath string) ([]ContainerDiff, error) {
data, err := ioutil.ReadFile(filePath)
if err != nil {
return nil, err
}
data, err := ioutil.ReadFile(filePath)
if err != nil {
return nil, err
}
var diffs []ContainerDiff
err = json.Unmarshal(data, &diffs)
return diffs, err
var diffs []ContainerDiff
err = json.Unmarshal(data, &diffs)
return diffs, err
}
// shouldIgnore checks if the change should be ignored based on the ignore list
func shouldIgnore(change string, ignoreList []string) bool {
for _, ignore := range ignoreList {
if strings.Contains(change, ignore) {
return true
}
}
return false
}
// logDetection logs detection events to a file and sends to Go Glitch
func logDetection(containerID, message string) {
logFilePath := filepath.Join(logDir, fmt.Sprintf("%s.log", containerID))
f, err := os.OpenFile(logFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
log.Printf("Error opening log file: %v", err)
return
}
defer f.Close()
log.SetOutput(f)
log.Println(message)
sendToGlitchtip(logFilePath)
}
// sendToGlitchtip sends the log file content to Go Glitch
func sendToGlitchtip(logFilePath string) {
if dsn := os.Getenv("GLITCHTIP_DSN"); dsn != "" {
cmd := exec.Command("go-glitch", logFilePath)
err := cmd.Run()
if err != nil {
log.Printf("Error sending to Go Glitch: %v", err)
}
}
}
// profileContainers logs the current state of containers to a file
func profileContainers(cli *client.Client) {
diffs, err := getRunningContainers(cli)
if err != nil {
log.Fatalf("Error profiling containers: %v", err)
}
diffs, err := getRunningContainers(cli)
if err != nil {
log.Fatalf("Error profiling containers: %v", err)
}
for _, diff := range diffs {
if diff.Labels["oculus.mode"] == "profile" {
filePath := fmt.Sprintf("%s.json", diff.Labels["oculus.containerid"])
err := saveDiffs([]ContainerDiff{diff}, filePath)
if err != nil {
log.Fatalf("Error saving diffs: %v", err)
}
}
}
for _, diff := range diffs {
if diff.Labels["oculus.mode"] == "profile" {
filePath := fmt.Sprintf("%s.json", diff.Labels["oculus.containerid"])
err := saveDiffs([]ContainerDiff{diff}, filePath)
if err != nil {
log.Fatalf("Error saving diffs: %v", err)
}
}
}
log.Println("Profiled current containers and saved to file")
log.Println("Profiled current containers and saved to file")
}
// compareContainers compares the current state of containers with the saved state
func compareContainers(cli *client.Client) {
currentDiffs, err := getRunningContainers(cli)
if err != nil {
log.Fatalf("Error getting current containers: %v", err)
}
currentDiffs, err := getRunningContainers(cli)
if err != nil {
log.Fatalf("Error getting current containers: %v", err)
}
for _, current := range currentDiffs {
if current.Labels["oculus.mode"] == "monitor" {
filePath := fmt.Sprintf("%s.json", current.Labels["oculus.containerid"])
savedDiffs, err := loadDiffs(filePath)
if err != nil {
log.Printf("Error loading saved diffs for %s: %v", current.ID, err)
continue
}
for _, current := range currentDiffs {
if current.Labels["oculus.mode"] == "monitor" {
filePath := fmt.Sprintf("%s.json", current.Labels["oculus.containerid"])
savedDiffs, err := loadDiffs(filePath)
if err != nil {
log.Printf("Error loading saved diffs for %s: %v", current.ID, err)
continue
}
if len(savedDiffs) > 0 {
saved := savedDiffs[0]
if current.Image != saved.Image {
log.Printf("Container %s changed: Image %s -> %s", current.ID, saved.Image, current.Image)
runNotifyScript(current.ID, current.Image, saved.Image)
}
if current.Labels["oculus.ignorelist"] != saved.Labels["oculus.ignorelist"] {
log.Printf("Container %s ignore list changed: %s -> %s", current.ID, saved.Labels["oculus.ignorelist"], current.Labels["oculus.ignorelist"])
runNotifyScript(current.ID, current.Image, saved.Image)
}
} else {
log.Printf("New container detected: ID %s, Image %s", current.ID, current.Image)
runNotifyScript(current.ID, current.Image, "")
}
}
}
ignoreList := strings.Split(current.Labels["oculus.ignorelist"], ",")
if len(savedDiffs) > 0 {
saved := savedDiffs[0]
if current.Image != saved.Image && !shouldIgnore(current.Image, ignoreList) {
message := fmt.Sprintf("Container %s changed: Image %s -> %s", current.ID, saved.Image, current.Image)
logDetection(current.Labels["oculus.containerid"], message)
runNotifyScript(filepath.Join(logDir, fmt.Sprintf("%s.log", current.Labels["oculus.containerid"])))
}
if current.Labels["oculus.ignorelist"] != saved.Labels["oculus.ignorelist"] {
message := fmt.Sprintf("Container %s ignore list changed: %s -> %s", current.ID, saved.Labels["oculus.ignorelist"], current.Labels["oculus.ignorelist"])
if !shouldIgnore(message, ignoreList) {
logDetection(current.Labels["oculus.containerid"], message)
runNotifyScript(filepath.Join(logDir, fmt.Sprintf("%s.log", current.Labels["oculus.containerid"])))
}
}
} else {
message := fmt.Sprintf("New container detected: ID %s, Image %s", current.ID, current.Image)
if !shouldIgnore(current.Image, ignoreList) {
logDetection(current.Labels["oculus.containerid"], message)
runNotifyScript(filepath.Join(logDir, fmt.Sprintf("%s.log", current.Labels["oculus.containerid"])))
}
}
}
}
}
// runNotifyScript executes the notification script with the container details
func runNotifyScript(containerID, newImage, oldImage string) {
cmd := exec.Command(notifyScript, containerID, newImage, oldImage)
err := cmd.Run()
if err != nil {
log.Printf("Error running notify script: %v", err)
}
func runNotifyScript(logFilePath string) {
cmd := exec.Command(notifyScript, logFilePath)
err := cmd.Run()
if err != nil {
log.Printf("Error running notify script: %v", err)
}
}
// watchDockerEvents listens for Docker events and triggers comparisons
func watchDockerEvents(cli *client.Client) {
eventFilter := filters.NewArgs()
eventFilter.Add("type", "container")
eventFilter.Add("event", "start")
eventFilter.Add("event", "stop")
eventFilter.Add("event", "destroy")
eventFilter := filters.NewArgs()
eventFilter.Add("type", "container")
eventFilter.Add("event", "start")
eventFilter.Add("event", "stop")
eventFilter.Add("event", "destroy")
messages, errs := cli.Events(context.Background(), types.EventsOptions{
Filters: eventFilter,
})
messages, errs := cli.Events(context.Background(), types.EventsOptions{
Filters: eventFilter,
})
for {
select {
case event := <-messages:
log.Printf("Docker event received: %s for container %s", event.Action, event.ID)
compareContainers(cli)
case err := <-errs:
log.Printf("Error watching Docker events: %v", err)
return
}
}
for {
select {
case event := <-messages:
log.Printf("Docker event received: %s for container %s", event.Action, event.ID)
compareContainers(cli)
case err := <-errs:
log.Printf("Error watching Docker events: %v", err)
return
}
}
}
// scheduler sets up the cron jobs for profiling
func scheduler(cli *client.Client) {
c := cron.New()
c.AddFunc("@every 1h", func() { profileContainers(cli) })
c.Start()
// monitorContainers sets up monitoring for containers based on the specified interval
func monitorContainers(cli *client.Client, interval time.Duration) {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
compareContainers(cli)
}
}
}
func main() {
cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if (err != nil) {
log.Fatalf("Error creating Docker client: %v", err)
}
cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
log.Fatalf("Error creating Docker client: %v", err)
}
go scheduler(cli)
go watchDockerEvents(cli)
go watchDockerEvents(cli)
// Get running containers and set up monitoring intervals
containers, err := getRunningContainers(cli)
if err != nil {
log.Fatalf("Error getting running containers: %v", err)
}
// Keep the program running
select {}
for _, container := range containers {
if container.Labels["oculus.mode"] == "monitor" {
intervalStr := container.Labels["oculus.interval"]
interval, err := time.ParseDuration(intervalStr)
if err != nil {
log.Fatalf("Invalid interval format for container %s: %v", container.ID, err)
}
go monitorContainers(cli, interval)
}
}
// Keep the program running
select {}
}

27
stack.production.yml Normal file
View File

@ -0,0 +1,27 @@
services:
oculus:
image: git.nixc.us/nixius/oculus:production
networks:
- traefik
environment:
GLITCHTIP_DSN: ""
volumes:
- /mnt/tank/persist/nixc.us/oculus/production/data:/log
- "/var/run/docker.sock:/var/run/docker.sock:ro"
deploy:
placement:
constraints:
- node.role == manager
labels:
traefik.enable: "false"
oculus.containerid: "oculus"
oculus.ignorelist: "/log/,/tmp/"
oculus.mode: "monitor"
oculus.interval: "60s"
update_config:
order: stop-first
failure_action: rollback
delay: 0s
parallelism: 1
restart_policy:
condition: on-failure

22
stack.staging.yml Normal file
View File

@ -0,0 +1,22 @@
services:
oculus:
image: git.nixc.us/nixius/oculus:staging
networks:
- traefik
environment:
GLITCHTIP_DSN: ""
# volumes:
# - /mnt/tank/persist/nixc.us/oculus/staging/data:/log
deploy:
# placement:
# constraints:
# - node.hostname == macmini14
labels:
traefik.enable: "false"
update_config:
order: stop-first
failure_action: rollback
delay: 0s
parallelism: 1
restart_policy:
condition: on-failure