grafana prometheus loki promtail foo

8 files changed, 335 insertions, 208 deletions

diff --git a/nix/hosts/corrino/configuration.nix b/nix/hosts/corrino/configuration.nix
index b49688e..f4157b4 100644
--- a/nix/hosts/corrino/configuration.nix
+++ b/nix/hosts/corrino/configuration.nix
@@ -29,6 +29,9 @@ in
     ./www/hydra.emile.space.nix
     ./www/netbox.emile.space.nix
     ./www/grafana.emile.space.nix
+    ./www/prometheus.emile.space.nix
+    ./www/loki.emile.space.nix
+    ./www/promtail.emile.space.nix
 
     # ./www/photo.emile.space.nix
     # ./www/photo
diff --git a/nix/hosts/corrino/ports.nix b/nix/hosts/corrino/ports.nix
index 0c2bd2a..13ef9a4 100644
--- a/nix/hosts/corrino/ports.nix
+++ b/nix/hosts/corrino/ports.nix
@@ -30,11 +30,18 @@
       s3 = 9000;
       web = 9001;
     };
+    promtail = 9033;
+    loki = 9034;
     authelia = 9091;
-    prometheus = 9003;
     gitDaemon = 9418;
-    prometheus_node_exporter = 9002;
-    prometheus_systemd_exporter = 9558;
-    prometheus_smartctl_exporter = 9633;
+    prometheus = {
+      web = 9003;
+      exporter = {
+        node = 9002;
+        nginx = 9913;
+        systemd = 9558;
+        smartctl = 9633;
+      };
+    };
   };
 }
diff --git a/nix/hosts/corrino/secrets/grafana_env_vars.age b/nix/hosts/corrino/secrets/grafana_env_vars.age
new file mode 100644
index 0000000..0365676
--- /dev/null
+++ b/nix/hosts/corrino/secrets/grafana_env_vars.age
@@ -0,0 +1,8 @@
+age-encryption.org/v1
+-> ssh-ed25519 gvwQ2Q VS/mHcaCvW6k56oTsRXMMppF+dliSaibw/34cARQpjs
+6h5gpw1nIB0/+Zon8EKVbxmizBVxhk1/QFeYJzrJs0Y
+-> ssh-ed25519 m8VklA FCypRodQTcpMk46i1kcG71VK5IYP09/WjqEEt9EnzHE
+pGO0EQA1pYa0KbvmgwKgms09Le0olfBzosP5fb8Smvo
+--- IJ2Guf9UWjppq+ZJYbSh37a88jHuqE8Zxob9fxtQ74I
+�b�A���=�fu��$�O!O]v;�~A�_ľ�g�8w��Q�(��{�NգY��Էts��ܿ`��
�
|�{�)'�-Н�}|�G���X���n��}���.!�.�µ提N+(���9�7ew~A�B�,�r`?��
+G
\ No newline at end of file
diff --git a/nix/hosts/corrino/www/grafana.emile.space.nix b/nix/hosts/corrino/www/grafana.emile.space.nix
index 8e7d529..b521c05 100644
--- a/nix/hosts/corrino/www/grafana.emile.space.nix
+++ b/nix/hosts/corrino/www/grafana.emile.space.nix
@@ -1,6 +1,8 @@
 { config, ... }:
 
 {
+  systemd.services.grafana.serviceConfig.EnvironmentFile = config.age.secrets.grafana_env_vars.path;
+
   services = {
     nginx.virtualHosts = {
       "grafana.emile.space" = {
@@ -11,16 +13,48 @@
           proxyWebsockets = true;
         };
       };
-      "prometheus.emile.space" = {
-        addSSL = true;
-        enableACME = true;
-        locations."/" = {
-          proxyPass = "http://${config.services.prometheus.listenAddress}:${toString config.services.prometheus.port}/";
-          proxyWebsockets = true;
-        };
-      };
     };
 
+    authelia.instances.main.settings.identity_providers.oidc.clients = [
+      {
+        id = "Grafana";
+
+        # ; nix run nixpkgs#authelia -- crypto hash generate pbkdf2 --variant sha512 --random --random.length 72 --random.charset rfc3986
+        secret = "$pbkdf2-sha512$310000$S.RE0jcmr7Sn/tjJDNxV/A$1tsYhQ/YEcVfE4JyzszHemrcUqy.84Fb6xVSmz87if5C9N46Mz2lRWB5l8s4EIrLsiumPnt4HQMkYZ4MoovJzA";
+        public = false;
+        authorization_policy = "two_factor";
+        redirect_uris = [ "https://grafana.emile.space/login/generic_oauth" ];
+        scopes = [
+          "openid"
+          "email"
+          "profile"
+        ];
+        grant_types = [
+          "refresh_token"
+          "authorization_code"
+        ];
+        response_types = [ "code" ];
+        response_modes = [
+          "form_post"
+          "query"
+          "fragment"
+        ];
+      }
+    ];
+
+    # example from md.emile.space
+    # CMD_OAUTH2_PROVIDERNAME=Authelia
+    # CMD_OAUTH2_CLIENT_ID=HedgeDoc
+    # CMD_OAUTH2_CLIENT_SECRET=
+    # CMD_OAUTH2_SCOPE=openid email profile
+    # CMD_OAUTH2_USER_PROFILE_USERNAME_ATTR=sub
+    # CMD_OAUTH2_USER_PROFILE_DISPLAY_NAME_ATTR=name
+    # CMD_OAUTH2_USER_PROFILE_EMAIL_ATTR=email
+    # CMD_OAUTH2_USER_PROFILE_URL=https://sso.emile.space/api/oidc/userinfo
+    # CMD_OAUTH2_TOKEN_URL=https://sso.emile.space/api/oidc/token
+    # CMD_OAUTH2_AUTHORIZATION_URL=https://sso.emile.space/api/oidc/authorize
+    # CMD_DOCUMENT_MAX_LENGTH=1000000
+
     grafana = {
       enable = true;
       settings = {
@@ -30,6 +64,37 @@
           domain = "grafana.emile.space";
           root_url = "https://grafana.emile.space/";
         };
+
+        "auth.generic_oauth" = let
+          sso = "https://sso.emile.space/api/oidc";
+        in {
+          enabled = true;
+          client_id = "Grafana";
+
+          # [auth.generic_oauth]
+          # client_secret = ... 
+          #   set in env var as 
+          #   GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET 
+          client_secret = "set in env var this is just a placeholder";
+
+          token_url = "${sso}/token";
+          auth_url = "${sso}/authorization";
+          api_url = "${sso}/userinfo";
+          scopes = [
+            "openid"
+            "email"
+            "profile"
+          ];
+          use_refresh_token = true;
+
+          # scopes = "openid email profile offline_access roles";
+          # email_attribute_path = "email";
+          # login_attribute_path = "username";
+          # name_attribute_path = "full_name";
+          # role_attribute_path = "contains(roles[*], 'admin') && 'Admin' || contains(roles[*], 'editor') && 'Editor' || 'Viewer'";
+
+          role_attribute_path = "contains(groups[*], 'admin') && 'Admin' || contains(groups[*], 'editor') && 'Editor' || 'Viewer'";
+        };
       };
 
       provision = {
@@ -43,205 +108,15 @@
                 editable = false;
                 access = "proxy"; # server = "proxy", browser = "direct"
               }
-              # {
-              #   name = "loki";
-              #   url = "http://localhost:${toString config.services.loki.configuration.server.http_listen_port}";
-              #   type = "loki";
-              # }
+              {
+                name = "loki";
+                url = "http://localhost:${toString config.services.loki.configuration.server.http_listen_port}";
+                type = "loki";
+              }
             ];
           };
         };
       };
     };
-
-    prometheus = {
-      enable = true;
-      retentionTime = "356d";
-
-      listenAddress = "[::1]";
-      port = config.emile.ports.prometheus;
-
-      exporters = {
-        node = {
-          enable = true;
-          enabledCollectors = [ "systemd" ];
-          port = config.emile.ports.prometheus_node_exporter;
-        };
-        systemd = {
-          enable = true;
-          port = config.emile.ports.prometheus_systemd_exporter;
-        };
-        smartctl = {
-          enable = true;
-          port = config.emile.ports.prometheus_smartctl_exporter;
-        };
-      };
-      scrapeConfigs = [
-        {
-          job_name = "corrino";
-          static_configs = [
-            { targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.node.port}" ]; }
-            { targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.systemd.port}" ]; }
-            { targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}" ]; }
-          ];
-        }
-        {
-          job_name = "lampadas";
-          static_configs = [
-            { targets = [ "lampadas:9100" ]; }
-            { targets = [ "lampadas:9558" ]; }
-            { targets = [ "lampadas:9633" ]; }
-          ];
-        }
-      ];
-    };
-
-    # loki = {
-    #   enable = true;
-    #   configuration = {
-    #     auth_enabled = false;
-    #     server = {
-    #       http_listen_port = ports.loki;
-    #     };
-
-    #     limits_config = {
-    #       reject_old_samples = true;
-    #       reject_old_samples_max_age = "7d";
-    #       max_global_streams_per_user = 100000;
-    #     };
-
-    #     common = {
-    #       instance_addr = "127.0.0.1";
-    #       ring = {
-    #         instance_addr = "127.0.0.1";
-    #         kvstore.store = "inmemory";
-    #       };
-    #       replication_factor = 1;
-    #       path_prefix = "/tmp/loki";
-    #     };
-
-    #     schema_config.configs = [{
-    #       from = "2023-05-09";
-    #       store = "boltdb-shipper";
-    #       object_store = "filesystem";
-    #       schema = "v11";
-    #       index = {
-    #         prefix = "index_";
-    #         period = "24h";
-    #       };
-    #     }];
-    #   };
-    # };
   };
-
-  # # allow the promtail user to read the nginx access files
-  # users.users.promtail.extraGroups = [ "nginx" ];
-
-  # services = {
-  #   promtail = {
-  #     enable = true;
-  #     configuration = {
-  #       server = {
-  #         http_listen_port = ports.promtail;
-  #         grpc_listen_port = 0;
-  #       };
-  #       positions.filename = "/tmp/positions.yml";
-  #       clients = [{
-  #         url = "http://localhost:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
-  #       }];
-  #       scrape_configs = [
-
-  #         # systemd
-  #         {
-  #           job_name = "journal";
-  #           journal = {
-  #             max_age = "12h";
-  #             labels = {
-  #               job = "systemd-journal";
-  #               host = config.networking.hostName;
-  #             };
-  #           };
-  #           relabel_configs = [
-  #             {
-  #               source_labels = [ "__journal__systemd_unit" ];
-  #               target_label = "unit";
-  #             }
-  #           ];
-  #         }
-
-  #         # nginx error log
-  #         {
-  #           job_name = "nginx-error-logs";
-  #           static_configs = [{
-  #             targets = [ "localhost" ];
-  #             labels = {
-  #               job = "nginx-error-logs";
-  #               host = "corrino";
-  #               __path__ = "/var/log/nginx/*error.log";
-  #             };
-  #           }];
-  #         }
-
-  #         # nginx
-  #         {
-  #           job_name = "nginx";
-  #           static_configs = [
-  #             {
-  #               targets = [ "localhost" ];
-  #               labels = {
-  #                 job = "nginx";
-  #                 host = "corrino";
-  #                 __path__ = "/var/log/nginx/*access.log";
-  #               };   
-  #             }
-  #           ];
-  #           pipeline_stages = [
-  #             # {
-  #             #   regex = {
-  #             #     expression = "(?:[0-9]{1,3}\.){3}([0-9]{1,3})";
-  #             #     replace = "***";
-  #             #   };
-  #             # }
-  #             {
-  #               regex = {
-  #                 expression = ''(?P<remote_addr>.+) - - \[(?P<time_local>.+)\] "(?P<method>.+) (?P<url>.+) (HTTP\/(?P<version>\d.\d))" (?P<status>\d{3}) (?P<body_bytes_sent>\d+) (["](?P<http_referer>(\-)|(.+))["]) (["](?P<http_user_agent>.+)["])'';
-  #               };
-  #             }
-  #             {
-  #               labels = {
-  #                 remote_addr = null;
-  #                 time_local = null;
-  #                 method = null;
-  #                 url = null;
-  #                 status = null;
-  #                 body_bytes_sent = null;
-  #                 http_referer = null;
-  #                 http_user_agent = null;
-  #               };
-  #             }
-  #             # {
-  #             #   timestamp = {
-  #             #     source = "time_local";
-  #             #     format = "02/Jan/2006:15:04:05 -0700";
-  #             #   };
-  #             # }
-  #             {
-  #               drop = {
-  #                 source = "url";
-  #                 expression = ''/(_matrix|.well-known|notifications|api|identity).*'';
-  #               };
-  #             }
-  #             {
-  #               drop = {
-  #                 source = "url";
-  #                 expression = ''grafana.*'';
-  #               };
-  #             }
-  #           ];
-  #         }
-
-  #       ];
-  #     };
-  #   };
-  # };
 }
diff --git a/nix/hosts/corrino/www/loki.emile.space.nix b/nix/hosts/corrino/www/loki.emile.space.nix
new file mode 100644
index 0000000..1c1786f
--- /dev/null
+++ b/nix/hosts/corrino/www/loki.emile.space.nix
@@ -0,0 +1,57 @@
+{ config, ... }:
+
+{
+  services = {
+    loki = {
+      enable = false;
+      configuration = {
+        auth_enabled = false;
+        server = {
+          http_listen_port = config.emile.ports.loki;
+        };
+
+        limits_config = {
+          reject_old_samples = true;
+          reject_old_samples_max_age = "7d";
+          max_global_streams_per_user = 100000;
+          max_streams_per_user = 100000;
+        };
+
+        common = {
+          instance_addr = "127.0.0.1";
+          ring = {
+            instance_addr = "127.0.0.1";
+            kvstore.store = "inmemory";
+          };
+          replication_factor = 1;
+          path_prefix = "/tmp/loki";
+        };
+
+        # limits_config.allow_structured_metadata = false;
+
+        schema_config.configs = [
+          # {
+          #   from = "2023-05-09";
+          #   store = "boltdb-shipper";
+          #   object_store = "filesystem";
+          #   schema = "v13";
+          #   index = {
+          #     prefix = "index_";
+          #     period = "24h";
+          #   };
+          # }
+          {
+            from = "2024-10-18";
+            store = "tsdb";
+            object_store = "filesystem";
+            schema = "v13";
+            index = {
+              prefix = "index_";
+              period = "24h";
+            };
+          }
+        ];
+      };
+    };
+  };
+}
diff --git a/nix/hosts/corrino/www/prometheus.emile.space.nix b/nix/hosts/corrino/www/prometheus.emile.space.nix
new file mode 100644
index 0000000..3fcbd32
--- /dev/null
+++ b/nix/hosts/corrino/www/prometheus.emile.space.nix
@@ -0,0 +1,63 @@
+{ config, ... }:
+
+{
+  services = {
+    nginx.virtualHosts = {
+      "prometheus.emile.space" = {
+        addSSL = true;
+        enableACME = true;
+        locations."/" = {
+          proxyPass = "http://${config.services.prometheus.listenAddress}:${toString config.services.prometheus.port}/";
+          proxyWebsockets = true;
+        };
+      };
+    };
+
+    prometheus = {
+      enable = true;
+      retentionTime = "356d";
+
+      listenAddress = "[::1]";
+      port = config.emile.ports.prometheus.web;
+
+      exporters = {
+        node = {
+          enable = true;
+          enabledCollectors = [ "systemd" ];
+          port = config.emile.ports.prometheus.exporter.node;
+        };
+        systemd = {
+          enable = true;
+          port = config.emile.ports.prometheus.exporter.systemd;
+        };
+        smartctl = {
+          enable = true;
+          port = config.emile.ports.prometheus.exporter.smartctl;
+        };
+        nginx = {
+          enable = true;
+          port = config.emile.ports.prometheus.exporter.nginx;
+        };
+      };
+      scrapeConfigs = [
+        {
+          job_name = "corrino";
+          static_configs = [
+            { targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.node.port}" ]; }
+            { targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.systemd.port}" ]; }
+            { targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}" ]; }
+            { targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.nginx.port}" ]; }
+          ];
+        }
+        {
+          job_name = "lampadas";
+          static_configs = [
+            { targets = [ "lampadas:9100" ]; }
+            { targets = [ "lampadas:9558" ]; }
+            { targets = [ "lampadas:9633" ]; }
+          ];
+        }
+      ];
+    };
+  };
+}
diff --git a/nix/hosts/corrino/www/promtail.emile.space.nix b/nix/hosts/corrino/www/promtail.emile.space.nix
new file mode 100644
index 0000000..7205bf9
--- /dev/null
+++ b/nix/hosts/corrino/www/promtail.emile.space.nix
@@ -0,0 +1,114 @@
+{ config, ... }:
+
+{
+  # allow the promtail user to read the nginx access files
+  users.users.promtail.extraGroups = [ "nginx" ];
+
+  services = {
+    promtail = {
+      enable = true;
+      configuration = {
+        server = {
+          http_listen_port = config.emile.ports.promtail;
+          grpc_listen_port = 0;
+        };
+        positions.filename = "/tmp/positions.yml";
+        clients = [{
+          url = "http://localhost:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
+        }];
+        scrape_configs = [
+
+          # systemd
+          {
+            job_name = "journal";
+            journal = {
+              max_age = "12h";
+              labels = {
+                job = "systemd-journal";
+                host = config.networking.hostName;
+              };
+            };
+            relabel_configs = [
+              {
+                source_labels = [ "__journal__systemd_unit" ];
+                target_label = "unit";
+              }
+            ];
+          }
+
+          # nginx error log
+          {
+            job_name = "nginx-error-logs";
+            static_configs = [{
+              targets = [ "localhost" ];
+              labels = {
+                job = "nginx-error-logs";
+                host = "corrino";
+                __path__ = "/var/log/nginx/*error.log";
+              };
+            }];
+          }
+
+          # nginx
+          {
+            job_name = "nginx";
+            static_configs = [
+              {
+                targets = [ "localhost" ];
+                labels = {
+                  job = "nginx";
+                  host = "corrino";
+                  __path__ = "/var/log/nginx/*access.log";
+                };   
+              }
+            ];
+            pipeline_stages = [
+              # {
+              #   regex = {
+              #     expression = "(?:[0-9]{1,3}\.){3}([0-9]{1,3})";
+              #     replace = "***";
+              #   };
+              # }
+              {
+                regex = {
+                  expression = ''(?P<remote_addr>.+) - - \[(?P<time_local>.+)\] "(?P<method>.+) (?P<url>.+) (HTTP\/(?P<version>\d.\d))" (?P<status>\d{3}) (?P<body_bytes_sent>\d+) (["](?P<http_referer>(\-)|(.+))["]) (["](?P<http_user_agent>.+)["])'';
+                };
+              }
+              {
+                labels = {
+                  remote_addr = null;
+                  time_local = null;
+                  method = null;
+                  url = null;
+                  status = null;
+                  body_bytes_sent = null;
+                  http_referer = null;
+                  http_user_agent = null;
+                };
+              }
+              # {
+              #   timestamp = {
+              #     source = "time_local";
+              #     format = "02/Jan/2006:15:04:05 -0700";
+              #   };
+              # }
+              {
+                drop = {
+                  source = "url";
+                  expression = ''/(_matrix|.well-known|notifications|api|identity).*'';
+                };
+              }
+              {
+                drop = {
+                  source = "url";
+                  expression = ''grafana.*'';
+                };
+              }
+            ];
+          }
+
+        ];
+      };
+    };
+  };
+}
diff --git a/nix/hosts/corrino/www/sso.emile.space.nix b/nix/hosts/corrino/www/sso.emile.space.nix
index e51db9a..3a742c5 100644
--- a/nix/hosts/corrino/www/sso.emile.space.nix
+++ b/nix/hosts/corrino/www/sso.emile.space.nix
@@ -110,7 +110,7 @@ in
 
         # we're using a file to store the user information
         authentication_backend = {
-          refresh_interval = "20s";
+          refresh_interval = "60s";
           file = {
             path = "/var/lib/authelia-main/user.yml";
             watch = true;