.woodpecker.yml

@@ -114,7 +114,7 @@ steps:
   deploy-production:
     name: deploy-production
     image: woodpeckerci/plugin-docker-buildx
-    secrets: [REGISTRY_USER, REGISTRY_PASSWORD, SENTRY_DSN]
+    secrets: [REGISTRY_USER, REGISTRY_PASSWORD]
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
     commands:

docker-compose.production.yml

@@ -2,6 +2,6 @@ version: "3.9"
 services:
   fluentd:
     build:
-      context: docker/fluentd-ingest
+      context: docker/fluentd
       dockerfile: Dockerfile.production
-    image: git.nixc.us/nixius/fluentd-ingest:production
+    image: git.nixc.us/nixius/fluentd:production

docker-compose.staging.yml

@@ -2,5 +2,5 @@ version: "3.9"
 services:
   fluentd:
     build:
-      context: docker/fluentd-ingest
+      context: docker/fluentd
-    image: git.nixc.us/nixius/fluentd-ingest:staging
+    image: git.nixc.us/nixius/fluentd:staging

docker/fluentd-ingest/Dockerfile

@@ -1,11 +0,0 @@
-FROM fluentd/fluentd:alpine
-
-# Install dependencies
-RUN pip install jinja2 pyyaml
-
-# Copy files into the container 
-COPY fluentd.conf.template /templates/
-COPY render_config.py /render_config.py
-
-# Render configuration at container start
-ENTRYPOINT ["python", "/render_config.py"]  

docker/fluentd-ingest/README.md

@@ -1,88 +0,0 @@
-Here's a breakdown of how you would use Jinja to dynamically inject environment variables into a config file and then execute a program:
-
-**1. Install Jinja2**
-
-Make sure you have Jinja2 installed in your Python environment:
-
-```bash
-pip install jinja2
-```
-
-**2. Create a Jinja2 Template**
-
-Your template file (e.g., `config.conf.j2`) will contain the structure of your config with placeholders for variables from the environment:
-
-```
-server {
-    listen {{ SERVER_PORT }};
-    root {{ DOCUMENT_ROOT }};
-}
-
-logging {
-    error_log {{ ERROR_LOG_PATH }};
-}
-```
-
-**3. Python Script**
-
-Here's a Python script to render the template and execute a program:
-
-```python
-import os
-from jinja2 import Environment, FileSystemLoader
-
-# Load the template
-file_loader = FileSystemLoader('templates') # Assuming a 'templates' directory
-env = Environment(loader=file_loader)
-template = env.get_template('config.conf.j2')
-
-# Pass environment variables as context
-output = template.render(
-    SERVER_PORT=os.environ.get('SERVER_PORT'),
-    DOCUMENT_ROOT=os.environ.get('DOCUMENT_ROOT'),
-    ERROR_LOG_PATH=os.environ.get('ERROR_LOG_PATH')
-)
-
-# Write the rendered config
-with open('config.conf', 'w') as f:
-    f.write(output)
-
-# Execute your program
-os.system('your_program')  # Replace 'your_program' with actual command
-```
-
-**Explanation**
-
-1. **Environment Variables:** The script fetches environment variables (`SERVER_PORT`, `DOCUMENT_ROOT`, etc.) using `os.environ.get()`.
-
-2. **Rendering:** Jinja renders the template, substituting the environment variable values into the placeholders.
-
-3. **Writing Config:** The rendered output is written to a file (`config.conf`).
-
-4. **Program Execution:** The `os.system()` function executes your program.
-
-**Remember:**
-
-* **Set Environment Variables:** Ensure the necessary environment variables are set before running the script.
-
-* **Security:** Be cautious when injecting values directly from environment variables. Validate them to prevent unexpected inputs that could break your configuration or introduce vulnerabilities.
-
-**Let's Customize It**
-
-* **Share parts of your config template and the corresponding environment variables.**
-
-* **Let me know if there are specific constraints or error-handling you'd like to incorporate.**
-
-I'm here to help you tailor the Jinja solution to your exact requirements!
-
-
-```yaml
-    environment:
-      - FLUENTD_FILTER_REGEX='/ERROR|CRITICAL/'  # Example regex
-      - FLUENTD_MATCH_SECTIONS: > 
-          - match docker.**
-            @type stdout
-          - match error.log
-            @type file
-            path /var/log/error.log
-```

docker/fluentd-ingest/entrypoint.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-# Generate fluent.conf from the template
-ERB_VERSION=$(gem list erubis -i -v | cut -d' ' -f3)
-erubis --version "$ERB_VERSION" /fluentd/fluentd.conf.erb > /fluentd/fluentd.conf
-
-# Replace default destination with ingress host from environment variable
-sed -i "s/ingress.nixc.us/$(echo $FLUENTD_FORWARD_HOST)/" /fluentd/fluentd.conf
-
-# Start Fluentd
-exec /usr/sbin/td-agent -c /fluentd/fluentd.conf

docker/fluentd-ingest/fluent.conf.template

@@ -1,21 +0,0 @@
-<source>
-  @type forward
-  port 24224
-  bind 0.0.0.0
-</source>
-
-<filter docker.**>
-  @type grep
-  <regexp>
-    key log
-    pattern {{ FLUENTD_FILTER_REGEX }}
-  </regexp>
-</filter>
-
-<match **> 
-  @type forward
-  <server>
-    host ingress.nixc.us
-    port 24224
-  </server>
-</match>

docker/fluentd-ingest/render_config.py

@@ -1,13 +0,0 @@
-import os
-from jinja2 import Environment, FileSystemLoader
-
-file_loader = FileSystemLoader('templates')
-env = Environment(loader=file_loader)
-template = env.get_template('fluentd.conf.template')
-
-filter_regex = os.environ.get('FLUENTD_FILTER_REGEX')
-
-output = template.render(FLUENTD_FILTER_REGEX=filter_regex)
-
-with open('/fluentd/etc/fluent.conf', 'w') as f:
-    f.write(output)

docker/fluentd/Dockerfile

@@ -0,0 +1,8 @@
+FROM fluent/fluentd:v1.12-debian-1
+ENV SENTRY_DSN="https://1be9f9ba4adf4278a664455a98d5d744@glitch.nixc.us/6"
+USER root
+COPY go-glitch /
+RUN chmod 777 /go-glitch 
+COPY fluent.conf /fluentd/etc/ 
+RUN chown -R fluent:fluent /fluentd && chmod -R 700 /fluentd/etc
+USER fluent

docker/fluentd/fluent.conf

@@ -0,0 +1,27 @@
+<source>
+  @type forward
+  port 24224
+  bind 0.0.0.0
+</source>
+
+<filter docker.**>
+  @type parser
+  key_name log
+  reserve_data true
+  <parse>
+    @type regexp
+    expression /^(?<message>.*)\b(SIGTERM|SEVERE ERROR|FATAL|CRITICAL)\b/i
+  </parse>
+</filter>
+
+<match docker.**>
+  @type exec
+  command "/go-glitch"
+  <buffer>
+    flush_interval 10s
+  </buffer>
+  <format>
+    @type json
+  </format>
+  run_interval 10s
+</match>

docker/fluentd/fluent.conf.greylog

@@ -0,0 +1,39 @@
+<source>
+  @type forward
+  port 24224
+  bind 0.0.0.0
+</source>
+
+<filter docker.**>
+  @type parser
+  key_name log
+  reserve_data true
+  <parse>
+    @type regexp
+    expression /^(?<message>.*)\b(SIGTERM|SEVERE ERROR|FATAL|CRITICAL)\b/i
+  </parse>
+</filter>
+
+<match docker.**>
+  @type copy  # Introduce a 'copy' stage
+
+  # Original output
+  <store>
+    @type exec
+    command "/go-glitch"
+    <buffer>
+      flush_interval 10s
+    </buffer>
+    <format>
+      @type json
+    </format>
+    run_interval 10s
+  </store>
+
+  # Graylog output
+  <store>
+    @type gelf
+    host graylog_server_hostname  # Replace with your Graylog server's hostname
+    port 12201                    # Standard GELF port
+  </store>
+</match>

stack.production.yml

@@ -3,11 +3,11 @@ networks:
   logging:
     external: true
 services:
-  fluentd-ingest:
+  fluentd:
-    image: git.nixc.us/nixius/fluentd-ingest:production
+    image: git.nixc.us/nixius/fluentd:production
     ports:
       - target: 24224
-        published: 24224
+        published: 9880
         protocol: tcp
         mode: host
     networks:

stack.staging.yml

@@ -3,12 +3,12 @@ networks:
   logging:
     external: true
 services:
-  fluentd-ingest:
+  fluentd:
-    image: git.nixc.us/nixius/fluentd-ingest:production
+    image: git.nixc.us/nixius/fluentd:production
     networks:
       - logging
     deploy:
       replicas: 1
-      # placement:
+      placement:
-      #   constraints:
+        constraints:
-      #     - node.hostname == ingress.nixc.us
+          - node.hostname == ingress.nixc.us