diff --git a/docker-compose.production.yml b/docker-compose.production.yml
index 12a7cda..2db9130 100644
--- a/docker-compose.production.yml
+++ b/docker-compose.production.yml
@@ -2,6 +2,6 @@ version: "3.9"
services:
fluentd:
build:
- context: docker/fluentd
+ context: docker/fluentd-ingest
dockerfile: Dockerfile.production
- image: git.nixc.us/nixius/fluentd:production
+ image: git.nixc.us/nixius/fluentd-ingest:production
diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml
index 0dcada3..b33f599 100644
--- a/docker-compose.staging.yml
+++ b/docker-compose.staging.yml
@@ -2,5 +2,5 @@ version: "3.9"
services:
fluentd:
build:
- context: docker/fluentd
- image: git.nixc.us/nixius/fluentd:staging
+ context: docker/fluentd-ingest
+ image: git.nixc.us/nixius/fluentd-ingest:staging
diff --git a/docker/fluentd-ingest/Dockerfile b/docker/fluentd-ingest/Dockerfile
new file mode 100644
index 0000000..8cfbac5
--- /dev/null
+++ b/docker/fluentd-ingest/Dockerfile
@@ -0,0 +1,11 @@
+FROM fluentd/fluentd:alpine
+
+# Install dependencies
+RUN pip install jinja2 pyyaml
+
+# Copy files into the container
+COPY fluentd.conf.template /templates/
+COPY render_config.py /render_config.py
+
+# Render configuration at container start
+ENTRYPOINT ["python", "/render_config.py"]
diff --git a/docker/fluentd/Dockerfile.production b/docker/fluentd-ingest/Dockerfile.production
similarity index 100%
rename from docker/fluentd/Dockerfile.production
rename to docker/fluentd-ingest/Dockerfile.production
diff --git a/docker/fluentd-ingest/README.md b/docker/fluentd-ingest/README.md
new file mode 100644
index 0000000..b8fae6b
--- /dev/null
+++ b/docker/fluentd-ingest/README.md
@@ -0,0 +1,88 @@
+Here's a breakdown of how you would use Jinja to dynamically inject environment variables into a config file and then execute a program:
+
+**1. Install Jinja2**
+
+Make sure you have Jinja2 installed in your Python environment:
+
+```bash
+pip install jinja2
+```
+
+**2. Create a Jinja2 Template**
+
+Your template file (e.g., `config.conf.j2`) will contain the structure of your config with placeholders for variables from the environment:
+
+```
+server {
+ listen {{ SERVER_PORT }};
+ root {{ DOCUMENT_ROOT }};
+}
+
+logging {
+ error_log {{ ERROR_LOG_PATH }};
+}
+```
+
+**3. Python Script**
+
+Here's a Python script to render the template and execute a program:
+
+```python
+import os
+from jinja2 import Environment, FileSystemLoader
+
+# Load the template
+file_loader = FileSystemLoader('templates') # Assuming a 'templates' directory
+env = Environment(loader=file_loader)
+template = env.get_template('config.conf.j2')
+
+# Pass environment variables as context
+output = template.render(
+ SERVER_PORT=os.environ.get('SERVER_PORT'),
+ DOCUMENT_ROOT=os.environ.get('DOCUMENT_ROOT'),
+ ERROR_LOG_PATH=os.environ.get('ERROR_LOG_PATH')
+)
+
+# Write the rendered config
+with open('config.conf', 'w') as f:
+ f.write(output)
+
+# Execute your program
+os.system('your_program') # Replace 'your_program' with actual command
+```
+
+**Explanation**
+
+1. **Environment Variables:** The script fetches environment variables (`SERVER_PORT`, `DOCUMENT_ROOT`, etc.) using `os.environ.get()`.
+
+2. **Rendering:** Jinja renders the template, substituting the environment variable values into the placeholders.
+
+3. **Writing Config:** The rendered output is written to a file (`config.conf`).
+
+4. **Program Execution:** The `os.system()` function executes your program.
+
+**Remember:**
+
+* **Set Environment Variables:** Ensure the necessary environment variables are set before running the script.
+
+* **Security:** Be cautious when injecting values directly from environment variables. Validate them to prevent unexpected inputs that could break your configuration or introduce vulnerabilities.
+
+**Let's Customize It**
+
+* **Share parts of your config template and the corresponding environment variables.**
+
+* **Let me know if there are specific constraints or error-handling you'd like to incorporate.**
+
+I'm here to help you tailor the Jinja solution to your exact requirements!
+
+
+```yaml
+ environment:
+ - FLUENTD_FILTER_REGEX='/ERROR|CRITICAL/' # Example regex
+ - FLUENTD_MATCH_SECTIONS: >
+ - match docker.**
+ @type stdout
+ - match error.log
+ @type file
+ path /var/log/error.log
+```
\ No newline at end of file
diff --git a/docker/fluentd-ingest/entrypoint.sh b/docker/fluentd-ingest/entrypoint.sh
new file mode 100755
index 0000000..39656ad
--- /dev/null
+++ b/docker/fluentd-ingest/entrypoint.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Generate fluent.conf from the template
+ERB_VERSION=$(gem list erubis -i -v | cut -d' ' -f3)
+erubis --version "$ERB_VERSION" /fluentd/fluentd.conf.erb > /fluentd/fluentd.conf
+
+# Replace default destination with ingress host from environment variable
+sed -i "s/ingress.nixc.us/$(echo $FLUENTD_FORWARD_HOST)/" /fluentd/fluentd.conf
+
+# Start Fluentd
+exec /usr/sbin/td-agent -c /fluentd/fluentd.conf
diff --git a/docker/fluentd-ingest/fluent.conf.template b/docker/fluentd-ingest/fluent.conf.template
new file mode 100644
index 0000000..b73512d
--- /dev/null
+++ b/docker/fluentd-ingest/fluent.conf.template
@@ -0,0 +1,21 @@
+
+ @type forward
+ port 24224
+ bind 0.0.0.0
+
+
+
+ @type grep
+
+ key log
+ pattern {{ FLUENTD_FILTER_REGEX }}
+
+
+
+
+ @type forward
+
+ host ingress.nixc.us
+ port 24224
+
+
diff --git a/docker/fluentd-ingest/render_config.py b/docker/fluentd-ingest/render_config.py
new file mode 100644
index 0000000..96f9b48
--- /dev/null
+++ b/docker/fluentd-ingest/render_config.py
@@ -0,0 +1,13 @@
+import os
+from jinja2 import Environment, FileSystemLoader
+
+file_loader = FileSystemLoader('templates')
+env = Environment(loader=file_loader)
+template = env.get_template('fluentd.conf.template')
+
+filter_regex = os.environ.get('FLUENTD_FILTER_REGEX')
+
+output = template.render(FLUENTD_FILTER_REGEX=filter_regex)
+
+with open('/fluentd/etc/fluent.conf', 'w') as f:
+ f.write(output)
diff --git a/docker/fluentd/Dockerfile b/docker/fluentd/Dockerfile
deleted file mode 100644
index 79368e8..0000000
--- a/docker/fluentd/Dockerfile
+++ /dev/null
@@ -1,9 +0,0 @@
-FROM fluent/fluentd:v1.12-debian-1
-ENV SENTRY_DSN="https://1be9f9ba4adf4278a664455a98d5d744@glitch.nixc.us/6"
-USER root
-COPY go-glitch /
-RUN chmod 777 /go-glitch
-COPY fluent.conf /fluentd/etc/
-RUN chown -R fluent:fluent /fluentd && chmod -R 700 /fluentd/etc
- # gem install fluent-plugin-gelf
-USER fluent
\ No newline at end of file
diff --git a/docker/fluentd/fluent.conf b/docker/fluentd/fluent.conf
deleted file mode 100644
index 6dd4c31..0000000
--- a/docker/fluentd/fluent.conf
+++ /dev/null
@@ -1,32 +0,0 @@
-
- @type forward
- port 24224
- bind 0.0.0.0
-
-
-
- @type grep
-
- key log
- pattern /SIGTERM|SEVERE ERROR|FATAL|CRITICAL/i
-
-
-
-
- @type exec
- command "/go-glitch"
-
- flush_interval 10s
-
-
- @type json
-
- run_interval 10s
-
-
-
- @type copy
-
- @type stdout
-
-
\ No newline at end of file
diff --git a/docker/fluentd/fluent.conf.greylog b/docker/fluentd/fluent.conf.greylog
deleted file mode 100644
index f99ee54..0000000
--- a/docker/fluentd/fluent.conf.greylog
+++ /dev/null
@@ -1,39 +0,0 @@
-
- @type forward
- port 24224
- bind 0.0.0.0
-
-
-
- @type parser
- key_name log
- reserve_data true
-
- @type regexp
- expression /^(?.*)\b(SIGTERM|SEVERE ERROR|FATAL|CRITICAL)\b/i
-
-
-
-
- @type copy # Introduce a 'copy' stage
-
- # Original output
-
- @type exec
- command "/go-glitch"
-
- flush_interval 10s
-
-
- @type json
-
- run_interval 10s
-
-
- # Graylog output
-
- @type gelf
- host graylog_server_hostname # Replace with your Graylog server's hostname
- port 12201 # Standard GELF port
-
-
\ No newline at end of file
diff --git a/docker/fluentd/go-glitch b/docker/fluentd/go-glitch
deleted file mode 100644
index eb5be21..0000000
Binary files a/docker/fluentd/go-glitch and /dev/null differ
diff --git a/stack.production.yml b/stack.production.yml
index 90f6729..2d727eb 100644
--- a/stack.production.yml
+++ b/stack.production.yml
@@ -3,8 +3,8 @@ networks:
logging:
external: true
services:
- fluentd:
- image: git.nixc.us/nixius/fluentd:production
+ fluentd-ingest:
+ image: git.nixc.us/nixius/fluentd-ingest:production
ports:
- target: 24224
published: 24224
diff --git a/stack.staging.yml b/stack.staging.yml
index 9b825fb..e24e6e2 100644
--- a/stack.staging.yml
+++ b/stack.staging.yml
@@ -3,12 +3,12 @@ networks:
logging:
external: true
services:
- fluentd:
- image: git.nixc.us/nixius/fluentd:production
+ fluentd-ingest:
+ image: git.nixc.us/nixius/fluentd-ingest:production
networks:
- logging
deploy:
replicas: 1
- placement:
- constraints:
- - node.hostname == ingress.nixc.us
+ # placement:
+ # constraints:
+ # - node.hostname == ingress.nixc.us