first commit

docker-compose.production.yml

@@ -2,6 +2,6 @@ version: "3.9"
 services:
   fluentd:
     build:
-      context: docker/fluentd
+      context: docker/fluentd-ingest
       dockerfile: Dockerfile.production
-    image: git.nixc.us/nixius/fluentd:production
+    image: git.nixc.us/nixius/fluentd-ingest:production

docker-compose.staging.yml

@@ -2,5 +2,5 @@ version: "3.9"
 services:
   fluentd:
     build:
-      context: docker/fluentd
-    image: git.nixc.us/nixius/fluentd:staging
+      context: docker/fluentd-ingest
+    image: git.nixc.us/nixius/fluentd-ingest:staging

docker/fluentd-ingest/Dockerfile

@@ -0,0 +1,11 @@
+FROM fluentd/fluentd:alpine
+
+# Install dependencies
+RUN pip install jinja2 pyyaml
+
+# Copy files into the container 
+COPY fluentd.conf.template /templates/
+COPY render_config.py /render_config.py
+
+# Render configuration at container start
+ENTRYPOINT ["python", "/render_config.py"]  

docker/fluentd-ingest/README.md

@@ -0,0 +1,88 @@
+Here's a breakdown of how you would use Jinja to dynamically inject environment variables into a config file and then execute a program:
+
+**1. Install Jinja2**
+
+Make sure you have Jinja2 installed in your Python environment:
+
+```bash
+pip install jinja2
+```
+
+**2. Create a Jinja2 Template**
+
+Your template file (e.g., `config.conf.j2`) will contain the structure of your config with placeholders for variables from the environment:
+
+```
+server {
+    listen {{ SERVER_PORT }};
+    root {{ DOCUMENT_ROOT }};
+}
+
+logging {
+    error_log {{ ERROR_LOG_PATH }};
+}
+```
+
+**3. Python Script**
+
+Here's a Python script to render the template and execute a program:
+
+```python
+import os
+from jinja2 import Environment, FileSystemLoader
+
+# Load the template
+file_loader = FileSystemLoader('templates') # Assuming a 'templates' directory
+env = Environment(loader=file_loader)
+template = env.get_template('config.conf.j2')
+
+# Pass environment variables as context
+output = template.render(
+    SERVER_PORT=os.environ.get('SERVER_PORT'),
+    DOCUMENT_ROOT=os.environ.get('DOCUMENT_ROOT'),
+    ERROR_LOG_PATH=os.environ.get('ERROR_LOG_PATH')
+)
+
+# Write the rendered config
+with open('config.conf', 'w') as f:
+    f.write(output)
+
+# Execute your program
+os.system('your_program')  # Replace 'your_program' with actual command
+```
+
+**Explanation**
+
+1. **Environment Variables:** The script fetches environment variables (`SERVER_PORT`, `DOCUMENT_ROOT`, etc.) using `os.environ.get()`.
+
+2. **Rendering:** Jinja renders the template, substituting the environment variable values into the placeholders.
+
+3. **Writing Config:** The rendered output is written to a file (`config.conf`).
+
+4. **Program Execution:** The `os.system()` function executes your program.
+
+**Remember:**
+
+* **Set Environment Variables:** Ensure the necessary environment variables are set before running the script.
+
+* **Security:** Be cautious when injecting values directly from environment variables. Validate them to prevent unexpected inputs that could break your configuration or introduce vulnerabilities.
+
+**Let's Customize It**
+
+* **Share parts of your config template and the corresponding environment variables.**
+
+* **Let me know if there are specific constraints or error-handling you'd like to incorporate.**
+
+I'm here to help you tailor the Jinja solution to your exact requirements!
+
+
+```yaml
+    environment:
+      - FLUENTD_FILTER_REGEX='/ERROR|CRITICAL/'  # Example regex
+      - FLUENTD_MATCH_SECTIONS: > 
+          - match docker.**
+            @type stdout
+          - match error.log
+            @type file
+            path /var/log/error.log
+```

docker/fluentd-ingest/entrypoint.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Generate fluent.conf from the template
+ERB_VERSION=$(gem list erubis -i -v | cut -d' ' -f3)
+erubis --version "$ERB_VERSION" /fluentd/fluentd.conf.erb > /fluentd/fluentd.conf
+
+# Replace default destination with ingress host from environment variable
+sed -i "s/ingress.nixc.us/$(echo $FLUENTD_FORWARD_HOST)/" /fluentd/fluentd.conf
+
+# Start Fluentd
+exec /usr/sbin/td-agent -c /fluentd/fluentd.conf

docker/fluentd-ingest/fluent.conf.template

@@ -0,0 +1,21 @@
+<source>
+  @type forward
+  port 24224
+  bind 0.0.0.0
+</source>
+
+<filter docker.**>
+  @type grep
+  <regexp>
+    key log
+    pattern {{ FLUENTD_FILTER_REGEX }}
+  </regexp>
+</filter>
+
+<match **> 
+  @type forward
+  <server>
+    host ingress.nixc.us
+    port 24224
+  </server>
+</match>

docker/fluentd-ingest/render_config.py

@@ -0,0 +1,13 @@
+import os
+from jinja2 import Environment, FileSystemLoader
+
+file_loader = FileSystemLoader('templates')
+env = Environment(loader=file_loader)
+template = env.get_template('fluentd.conf.template')
+
+filter_regex = os.environ.get('FLUENTD_FILTER_REGEX')
+
+output = template.render(FLUENTD_FILTER_REGEX=filter_regex)
+
+with open('/fluentd/etc/fluent.conf', 'w') as f:
+    f.write(output)

docker/fluentd/Dockerfile

@@ -1,9 +0,0 @@
-FROM fluent/fluentd:v1.12-debian-1
-ENV SENTRY_DSN="https://1be9f9ba4adf4278a664455a98d5d744@glitch.nixc.us/6"
-USER root
-COPY go-glitch /
-RUN chmod 777 /go-glitch 
-COPY fluent.conf /fluentd/etc/ 
-RUN chown -R fluent:fluent /fluentd && chmod -R 700 /fluentd/etc
-    # gem install fluent-plugin-gelf
-USER fluent

docker/fluentd/fluent.conf

@@ -1,32 +0,0 @@
-<source>
-  @type forward
-  port 24224
-  bind 0.0.0.0
-</source>
-
-<filter docker.**>
-  @type grep
-  <regexp>
-    key log
-    pattern /SIGTERM|SEVERE ERROR|FATAL|CRITICAL/i
-  </regexp>
-</filter>
-
-<match docker.**>
-    @type exec     
-    command "/go-glitch"     
-    <buffer>     
-        flush_interval 10s     
-    </buffer>     
-    <format>     
-        @type json     
-    </format>      
-    run_interval 10s    
-</match>
-
-<match **>  
-  @type copy  
-  <store>
-    @type stdout 
-  </store>
-</match> 

docker/fluentd/fluent.conf.greylog

@@ -1,39 +0,0 @@
-<source>
-  @type forward
-  port 24224
-  bind 0.0.0.0
-</source>
-
-<filter docker.**>
-  @type parser
-  key_name log
-  reserve_data true
-  <parse>
-    @type regexp
-    expression /^(?<message>.*)\b(SIGTERM|SEVERE ERROR|FATAL|CRITICAL)\b/i
-  </parse>
-</filter>
-
-<match docker.**>
-  @type copy  # Introduce a 'copy' stage
-
-  # Original output
-  <store>
-    @type exec
-    command "/go-glitch"
-    <buffer>
-      flush_interval 10s
-    </buffer>
-    <format>
-      @type json
-    </format>
-    run_interval 10s
-  </store>
-
-  # Graylog output
-  <store>
-    @type gelf
-    host graylog_server_hostname  # Replace with your Graylog server's hostname
-    port 12201                    # Standard GELF port
-  </store>
-</match>

stack.production.yml

@@ -3,8 +3,8 @@ networks:
   logging:
     external: true
 services:
-  fluentd:
-    image: git.nixc.us/nixius/fluentd:production
+  fluentd-ingest:
+    image: git.nixc.us/nixius/fluentd-ingest:production
     ports:
       - target: 24224
         published: 24224

stack.staging.yml

@@ -3,12 +3,12 @@ networks:
   logging:
     external: true
 services:
-  fluentd:
-    image: git.nixc.us/nixius/fluentd:production
+  fluentd-ingest:
+    image: git.nixc.us/nixius/fluentd-ingest:production
     networks:
       - logging
     deploy:
       replicas: 1
-      placement:
-        constraints:
-          - node.hostname == ingress.nixc.us
+      # placement:
+      #   constraints:
+      #     - node.hostname == ingress.nixc.us