refactor: rethink monitoring structure | add Loki && Promtail | new Grafana conf

templates/monitoring/promtail/promtail.yml

@@ -0,0 +1,50 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  # Caddy access logs
+  - job_name: caddy
+    static_configs:
+      - targets:
+          - localhost
+        labels:
+          job: caddy
+          __path__: /var/log/caddy/access.log
+    pipeline_stages:
+      - json:
+          expressions:
+            timestamp: ts
+            level: level
+            message: msg
+            method: request.method
+            uri: request.uri
+            status: resp_headers.status
+            duration: duration
+      - labels:
+          method:
+          status:
+      - timestamp:
+          source: timestamp
+          format: Unix
+
+  # System logs
+  - job_name: syslog
+    static_configs:
+      - targets:
+          - localhost
+        labels:
+          job: syslog
+          __path__: /var/log/syslog
+    pipeline_stages:
+      - regex:
+          expression: '^(?P<timestamp>\w+\s+\d+\s+\d+:\d+:\d+)\s+(?P<hostname>\w+)\s+(?P<service>\w+).*'
+      - labels:
+          hostname:
+          service: