diff --git a/.woodpecker.yml b/.woodpecker.yml index 4559e3a..043f6f6 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,3 +1,14 @@ +pipeline: + build_app: + image: docker:24 + commands: + - docker build -t ploughshares:ci -f docker/ploughshares/Dockerfile docker/ploughshares + + build_crawler_google_alerts: + image: docker:24 + commands: + - docker build -t ploughshares-crawler-google-alerts:ci docker/crawler-google-alerts + # build:0 labels: location: manager diff --git a/README.md b/README.md index 63dc218..7d07a9f 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,14 @@ docker-compose up The application will be available at http://localhost:5001. +### Crawler - Google Alerts + +To run the Google Alerts crawler locally (requires GOOGLE_API_KEY): + +```bash +GOOGLE_API_KEY=your_key docker-compose -f docker-compose.dev.yml run --rm crawler_google_alerts +``` + ## Features - Transaction management (create, view, edit) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 79e0f92..8add1bc 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -46,6 +46,19 @@ services: timeout: 5s retries: 5 + crawler_google_alerts: + build: + context: ./docker/crawler-google-alerts + image: ploughshares-crawler-google-alerts:dev + environment: + - GOOGLE_API_KEY=${GOOGLE_API_KEY} + depends_on: + db: + condition: service_started + command: python main.py + volumes: + - ./docker/crawler-google-alerts:/app + volumes: postgres_dev_data: diff --git a/docker-compose.yml b/docker-compose.yml index f88cbdb..703abb7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,6 +38,17 @@ services: timeout: 5s retries: 5 + crawler_google_alerts: + build: + context: ./docker/crawler-google-alerts + image: ploughshares-crawler-google-alerts:latest + environment: + - GOOGLE_API_KEY=${GOOGLE_API_KEY} + depends_on: + db: + condition: service_started + command: python main.py + volumes: postgres_data: diff --git a/docker/crawler-google-alerts/Dockerfile b/docker/crawler-google-alerts/Dockerfile new file mode 100644 index 0000000..7c272fc --- /dev/null +++ b/docker/crawler-google-alerts/Dockerfile @@ -0,0 +1,21 @@ +FROM python:3.11-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app + +# System deps (kept minimal; playwright browsers not installed unless needed at runtime) +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY . /app + +# GOOGLE_API_KEY must be provided at runtime +CMD ["python", "main.py"] + +