base/grafana-alloy: add base config.alloy

base/grafana-alloy.nix

@@ -7,10 +7,84 @@
   ...
 }:
 
+let
+  alloy_prometheus_endpoint = "http://log-01.tail755c5.ts.net:9090/api/v1/write";
+  alloy_loki_endpoint = "http://log-01.tail755c5.ts.net:3100/loki/api/v1/push";
+in
 {
   environment.systemPackages = with pkgs; [ grafana-alloy ];
   services.alloy.enable = true;
 
   # Fixes issue with node_exporter
   services.dbus.implementation = "broker";
+
+  environment.etc."alloy/config.alloy".text = ''
+    // Do not modify this or any of the below lines
+    // We want to silence most output except for any errors
+    logging {
+    	level  = "error"
+    	format = "logfmt"
+    }
+
+    // Default write target to log server
+    prometheus.remote_write "default" {
+    	endpoint {
+    		url = "${alloy_prometheus_endpoint}"
+    	}
+    	external_labels = {host = ${config.networking.hostName}}
+    }
+
+    // ========== Default Prometheus configuration ==========
+    prometheus.exporter.unix "local_system" {
+    	enable_collectors = ["systemd"]
+    }
+
+    prometheus.scrape "local_system" {
+    	targets         = prometheus.exporter.unix.local_system.targets
+    	forward_to      = [prometheus.remote_write.default.receiver]
+    	scrape_interval = "10s"
+    }
+
+    // ========== Default Loki configuration ==========
+    loki.write "default" {
+    	endpoint {
+    		url = "${alloy_loki_endpoint}"
+    	}
+    	external_labels = {host = "${config.networking.hostName}}"
+    }
+
+    local.file_match "local_logfiles" {
+    	path_targets = [{"__path__" = "/var/log/*.log"}]
+    	sync_period  = "5s"
+    }
+
+    discovery.relabel "systemd_journal" {
+    	targets = []
+
+    	rule {
+    		source_labels = ["__journal__systemd_unit"]
+    		target_label  = "systemd_unit"
+    	}
+
+    	rule {
+    		source_labels = ["__journal__hostname"]
+    		target_label  = "host"
+    	}
+
+    	rule {
+    		source_labels = ["__journal_syslog_identifier"]
+    		target_label  = "syslog_identifier"
+    	}
+    }
+
+    loki.source.journal "systemd_journal" {
+    	path          = "/var/log/journal"
+    	relabel_rules = discovery.relabel.systemd_journal.rules
+    	forward_to    = [loki.write.default.receiver]
+    	labels        = {
+    		cluster = "ops-tools1",
+    		job     = "default/systemd-journal",
+    	}
+    }
+  '';
 }