nixos/modules/server/monitor.nix

  # MONITORING: services run on loopback interface
  #             nginx reverse proxy exposes services to network

{ config, lib, pkgs, ... }:

let 
	grafana_port = 7000;
in {
	# Logging metrics
	# Disable loki
	services.loki = {
		enable = false;
		configuration = {
			server.http_listen_port = 3100;
			auth_enabled = false;

			ingester = {
				lifecycler = {
					address = "127.0.0.1";
					ring = {
						kvstore = {
							store = "inmemory";
						};
						replication_factor = 1;
					};
				};
				chunk_idle_period = "1h";
				max_chunk_age = "1h";
				chunk_target_size = 999999;
				chunk_retain_period = "30s";
			};

			schema_config = {
				configs = [{
					store = "tsdb";
					object_store = "filesystem";
					schema = "v13";
					index = {
						prefix = "index_";
						period = "24h";
					};
				}];
			};

			storage_config = {
				tsdb_shipper = {
					active_index_directory = "/var/lib/loki/tsdb-shipper-active";
					cache_location = "/var/lib/loki/tsdb-shipper-cache";
					cache_ttl = "24h";
				};

				filesystem = {
					directory = "/var/lib/loki/chunks";
				};
			};

			limits_config = {
				reject_old_samples = true;
				reject_old_samples_max_age = "168h";
			};


			table_manager = {
				retention_deletes_enabled = false;
				retention_period = "0s";
			};

			compactor = {
				working_directory = "/var/lib/loki";
				compactor_ring = {
					kvstore = {
						store = "inmemory";
					};
				};
			};
		};
	};

	# Disable
	services.promtail = {
		enable = false;
		configuration = {
			server = {
				http_listen_port = 3031;
				grpc_listen_port = 0;
			};
			positions = {
				filename = "/tmp/positions.yaml";
			};
			clients = [{
				url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
			}];
			scrape_configs = [{
				job_name = "journal";
				journal = {
					max_age = "12h";
					labels = {
						job = "systemd-journal";
						host = "misox";
					};
				};
				relabel_configs = [{
					source_labels = [ "__journal__systemd_unit" ];
					target_label = "unit";
				}];
			}];
		};
	};

	# System metrics
	services.prometheus = {
		enable = true;
		exporters = {
			node = {
				enable = true;
				enabledCollectors = [ "systemd" "tcpstat" "processes" "interrupts" ];
				port = 9100;
			};
		};
		#globalConfig.scrape_interval = "1m";
		scrapeConfigs = [{
			job_name = "node";
			static_configs = [{
				targets = [
					"localhost:${toString config.services.prometheus.exporters.node.port}"
				];
			}];
		}];
	};

	# Dashboard
	services.grafana = {
		enable = true;
		settings = {
			server = {
				http_addr = "127.0.0.1";
				http_port = 3000;
				domain = "grafana.misox";
				#root_url = "http://misox:${toString grafana_port}/";
			};
		};
	};
}