# Add this block inside the Nginx http section.
log_format reachllm_json escape=json '{'
  '"domain":"$host",'
  '"path":"$request_uri",'
  '"user_agent":"$http_user_agent",'
  '"timestamp":"$time_iso8601",'
  '"method":"$request_method",'
  '"status_code":$status,'
  '"referer":"$http_referer",'
  '"request_id":"$request_id"'
'}';
access_log /var/log/nginx/reachllm-access.json reachllm_json;

# Put this Fluent Bit input and filter in fluent-bit.conf.
[INPUT]
    Name              tail
    Path              /var/log/nginx/reachllm-access.json
    Parser            json
    Tag               reachllm.crawler
    DB                /var/lib/fluent-bit/reachllm-crawler.db
    Read_from_Head    false

[FILTER]
    Name              grep
    Match             reachllm.crawler
    Regex             user_agent (GPTBot|OAI-SearchBot|ChatGPT-User|OAI-AdsBot|ClaudeBot|Claude-SearchBot|Claude-User|anthropic-ai|Claude-Web|PerplexityBot|Perplexity-User|Google-CloudVertexBot|Google-Agent|Google-NotebookLM|Applebot|meta-externalagent|meta-externalfetcher|Amazonbot|Amzn-SearchBot|Amzn-User|MistralAI-Index|MistralAI-User|YouBot|DuckAssistBot|CCBot|AI2Bot|DeepSeekBot|Grok-DeepSearch|xAI-Web-Crawler|xAI-SearchBot|Bytespider|cohere-ai)

[OUTPUT]
    Name              http
    Match             reachllm.crawler
    Host              app.reachllm.com
    Port              443
    URI               /api/ai-crawler-traffic/ingest
    Format            json_lines
    Header            X-ReachLLM-Site-Token ${REACHLLM_SITE_TOKEN}
    tls               On
    tls.verify        On