diff --git a/docker-compose.production.yml b/docker-compose.production.yml index 12a7cda..2db9130 100644 --- a/docker-compose.production.yml +++ b/docker-compose.production.yml @@ -2,6 +2,6 @@ version: "3.9" services: fluentd: build: - context: docker/fluentd + context: docker/fluentd-ingest dockerfile: Dockerfile.production - image: git.nixc.us/nixius/fluentd:production + image: git.nixc.us/nixius/fluentd-ingest:production diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml index 0dcada3..b33f599 100644 --- a/docker-compose.staging.yml +++ b/docker-compose.staging.yml @@ -2,5 +2,5 @@ version: "3.9" services: fluentd: build: - context: docker/fluentd - image: git.nixc.us/nixius/fluentd:staging + context: docker/fluentd-ingest + image: git.nixc.us/nixius/fluentd-ingest:staging diff --git a/docker/fluentd-ingest/Dockerfile b/docker/fluentd-ingest/Dockerfile new file mode 100644 index 0000000..8cfbac5 --- /dev/null +++ b/docker/fluentd-ingest/Dockerfile @@ -0,0 +1,11 @@ +FROM fluentd/fluentd:alpine + +# Install dependencies +RUN pip install jinja2 pyyaml + +# Copy files into the container +COPY fluentd.conf.template /templates/ +COPY render_config.py /render_config.py + +# Render configuration at container start +ENTRYPOINT ["python", "/render_config.py"] diff --git a/docker/fluentd/Dockerfile.production b/docker/fluentd-ingest/Dockerfile.production similarity index 100% rename from docker/fluentd/Dockerfile.production rename to docker/fluentd-ingest/Dockerfile.production diff --git a/docker/fluentd-ingest/README.md b/docker/fluentd-ingest/README.md new file mode 100644 index 0000000..b8fae6b --- /dev/null +++ b/docker/fluentd-ingest/README.md @@ -0,0 +1,88 @@ +Here's a breakdown of how you would use Jinja to dynamically inject environment variables into a config file and then execute a program: + +**1. Install Jinja2** + +Make sure you have Jinja2 installed in your Python environment: + +```bash +pip install jinja2 +``` + +**2. Create a Jinja2 Template** + +Your template file (e.g., `config.conf.j2`) will contain the structure of your config with placeholders for variables from the environment: + +``` +server { + listen {{ SERVER_PORT }}; + root {{ DOCUMENT_ROOT }}; +} + +logging { + error_log {{ ERROR_LOG_PATH }}; +} +``` + +**3. Python Script** + +Here's a Python script to render the template and execute a program: + +```python +import os +from jinja2 import Environment, FileSystemLoader + +# Load the template +file_loader = FileSystemLoader('templates') # Assuming a 'templates' directory +env = Environment(loader=file_loader) +template = env.get_template('config.conf.j2') + +# Pass environment variables as context +output = template.render( + SERVER_PORT=os.environ.get('SERVER_PORT'), + DOCUMENT_ROOT=os.environ.get('DOCUMENT_ROOT'), + ERROR_LOG_PATH=os.environ.get('ERROR_LOG_PATH') +) + +# Write the rendered config +with open('config.conf', 'w') as f: + f.write(output) + +# Execute your program +os.system('your_program') # Replace 'your_program' with actual command +``` + +**Explanation** + +1. **Environment Variables:** The script fetches environment variables (`SERVER_PORT`, `DOCUMENT_ROOT`, etc.) using `os.environ.get()`. + +2. **Rendering:** Jinja renders the template, substituting the environment variable values into the placeholders. + +3. **Writing Config:** The rendered output is written to a file (`config.conf`). + +4. **Program Execution:** The `os.system()` function executes your program. + +**Remember:** + +* **Set Environment Variables:** Ensure the necessary environment variables are set before running the script. + +* **Security:** Be cautious when injecting values directly from environment variables. Validate them to prevent unexpected inputs that could break your configuration or introduce vulnerabilities. + +**Let's Customize It** + +* **Share parts of your config template and the corresponding environment variables.** + +* **Let me know if there are specific constraints or error-handling you'd like to incorporate.** + +I'm here to help you tailor the Jinja solution to your exact requirements! + + +```yaml + environment: + - FLUENTD_FILTER_REGEX='/ERROR|CRITICAL/' # Example regex + - FLUENTD_MATCH_SECTIONS: > + - match docker.** + @type stdout + - match error.log + @type file + path /var/log/error.log +``` \ No newline at end of file diff --git a/docker/fluentd-ingest/entrypoint.sh b/docker/fluentd-ingest/entrypoint.sh new file mode 100755 index 0000000..39656ad --- /dev/null +++ b/docker/fluentd-ingest/entrypoint.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# Generate fluent.conf from the template +ERB_VERSION=$(gem list erubis -i -v | cut -d' ' -f3) +erubis --version "$ERB_VERSION" /fluentd/fluentd.conf.erb > /fluentd/fluentd.conf + +# Replace default destination with ingress host from environment variable +sed -i "s/ingress.nixc.us/$(echo $FLUENTD_FORWARD_HOST)/" /fluentd/fluentd.conf + +# Start Fluentd +exec /usr/sbin/td-agent -c /fluentd/fluentd.conf diff --git a/docker/fluentd-ingest/fluent.conf.template b/docker/fluentd-ingest/fluent.conf.template new file mode 100644 index 0000000..b73512d --- /dev/null +++ b/docker/fluentd-ingest/fluent.conf.template @@ -0,0 +1,21 @@ + + @type forward + port 24224 + bind 0.0.0.0 + + + + @type grep + + key log + pattern {{ FLUENTD_FILTER_REGEX }} + + + + + @type forward + + host ingress.nixc.us + port 24224 + + diff --git a/docker/fluentd-ingest/render_config.py b/docker/fluentd-ingest/render_config.py new file mode 100644 index 0000000..96f9b48 --- /dev/null +++ b/docker/fluentd-ingest/render_config.py @@ -0,0 +1,13 @@ +import os +from jinja2 import Environment, FileSystemLoader + +file_loader = FileSystemLoader('templates') +env = Environment(loader=file_loader) +template = env.get_template('fluentd.conf.template') + +filter_regex = os.environ.get('FLUENTD_FILTER_REGEX') + +output = template.render(FLUENTD_FILTER_REGEX=filter_regex) + +with open('/fluentd/etc/fluent.conf', 'w') as f: + f.write(output) diff --git a/docker/fluentd/Dockerfile b/docker/fluentd/Dockerfile deleted file mode 100644 index 79368e8..0000000 --- a/docker/fluentd/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM fluent/fluentd:v1.12-debian-1 -ENV SENTRY_DSN="https://1be9f9ba4adf4278a664455a98d5d744@glitch.nixc.us/6" -USER root -COPY go-glitch / -RUN chmod 777 /go-glitch -COPY fluent.conf /fluentd/etc/ -RUN chown -R fluent:fluent /fluentd && chmod -R 700 /fluentd/etc - # gem install fluent-plugin-gelf -USER fluent \ No newline at end of file diff --git a/docker/fluentd/fluent.conf b/docker/fluentd/fluent.conf deleted file mode 100644 index 6dd4c31..0000000 --- a/docker/fluentd/fluent.conf +++ /dev/null @@ -1,32 +0,0 @@ - - @type forward - port 24224 - bind 0.0.0.0 - - - - @type grep - - key log - pattern /SIGTERM|SEVERE ERROR|FATAL|CRITICAL/i - - - - - @type exec - command "/go-glitch" - - flush_interval 10s - - - @type json - - run_interval 10s - - - - @type copy - - @type stdout - - \ No newline at end of file diff --git a/docker/fluentd/fluent.conf.greylog b/docker/fluentd/fluent.conf.greylog deleted file mode 100644 index f99ee54..0000000 --- a/docker/fluentd/fluent.conf.greylog +++ /dev/null @@ -1,39 +0,0 @@ - - @type forward - port 24224 - bind 0.0.0.0 - - - - @type parser - key_name log - reserve_data true - - @type regexp - expression /^(?.*)\b(SIGTERM|SEVERE ERROR|FATAL|CRITICAL)\b/i - - - - - @type copy # Introduce a 'copy' stage - - # Original output - - @type exec - command "/go-glitch" - - flush_interval 10s - - - @type json - - run_interval 10s - - - # Graylog output - - @type gelf - host graylog_server_hostname # Replace with your Graylog server's hostname - port 12201 # Standard GELF port - - \ No newline at end of file diff --git a/docker/fluentd/go-glitch b/docker/fluentd/go-glitch deleted file mode 100644 index eb5be21..0000000 Binary files a/docker/fluentd/go-glitch and /dev/null differ diff --git a/stack.production.yml b/stack.production.yml index 90f6729..2d727eb 100644 --- a/stack.production.yml +++ b/stack.production.yml @@ -3,8 +3,8 @@ networks: logging: external: true services: - fluentd: - image: git.nixc.us/nixius/fluentd:production + fluentd-ingest: + image: git.nixc.us/nixius/fluentd-ingest:production ports: - target: 24224 published: 24224 diff --git a/stack.staging.yml b/stack.staging.yml index 9b825fb..e24e6e2 100644 --- a/stack.staging.yml +++ b/stack.staging.yml @@ -3,12 +3,12 @@ networks: logging: external: true services: - fluentd: - image: git.nixc.us/nixius/fluentd:production + fluentd-ingest: + image: git.nixc.us/nixius/fluentd-ingest:production networks: - logging deploy: replicas: 1 - placement: - constraints: - - node.hostname == ingress.nixc.us + # placement: + # constraints: + # - node.hostname == ingress.nixc.us