__meta_kubernetes_node_name: The name of the node object.
__meta_kubernetes_node_label_<labelname>: Each label from the node object.
__meta_kubernetes_node_labelpresent_<labelname>: true for each label from the node object.
__meta_kubernetes_node_annotation_<annotationname>: Each annotation from the node object.
__meta_kubernetes_node_annotationpresent_<annotationname>: true for each annotation from the node object.
__meta_kubernetes_node_address_<address_type>: The first address for each node address type, if it exists.

__meta_kubernetes_namespace: The namespace of the service object.
__meta_kubernetes_service_annotation_<annotationname>: Each annotation from the service object.
__meta_kubernetes_service_annotationpresent_<annotationname>: "true" for each annotation of the service object.
__meta_kubernetes_service_cluster_ip: The cluster IP address of the service. (Does not apply to services of type ExternalName)
__meta_kubernetes_service_external_name: The DNS name of the service. (Applies to services of type ExternalName)
__meta_kubernetes_service_label_<labelname>: Each label from the service object.
__meta_kubernetes_service_labelpresent_<labelname>: true for each label of the service object.
__meta_kubernetes_service_name: The name of the service object.
__meta_kubernetes_service_port_name: Name of the service port for the target.
__meta_kubernetes_service_port_protocol: Protocol of the service port for the target.

__meta_kubernetes_namespace: The namespace of the pod object.
__meta_kubernetes_pod_name: The name of the pod object.
__meta_kubernetes_pod_ip: The pod IP of the pod object.
__meta_kubernetes_pod_label_<labelname>: Each label from the pod object.
__meta_kubernetes_pod_labelpresent_<labelname>: truefor each label from the pod object.
__meta_kubernetes_pod_annotation_<annotationname>: Each annotation from the pod object.
__meta_kubernetes_pod_annotationpresent_<annotationname>: true for each annotation from the pod object.
__meta_kubernetes_pod_container_init: true if the container is an InitContainer
__meta_kubernetes_pod_container_name: Name of the container the target address points to.
__meta_kubernetes_pod_container_port_name: Name of the container port.
__meta_kubernetes_pod_container_port_number: Number of the container port.
__meta_kubernetes_pod_container_port_protocol: Protocol of the container port.
__meta_kubernetes_pod_ready: Set to true or false for the pod's ready state.
__meta_kubernetes_pod_phase: Set to Pending, Running, Succeeded, Failed or Unknown in the lifecycle.
__meta_kubernetes_pod_node_name: The name of the node the pod is scheduled onto.
__meta_kubernetes_pod_host_ip: The current host IP of the pod object.
__meta_kubernetes_pod_uid: The UID of the pod object.
__meta_kubernetes_pod_controller_kind: Object kind of the pod controller.
__meta_kubernetes_pod_controller_name: Name of the pod controller.

__meta_kubernetes_namespace: The namespace of the endpoints object.
__meta_kubernetes_endpoints_name: The names of the endpoints object.
For all targets discovered directly from the endpoints list (those not additionally inferred from underlying pods), the following labels are attached:
__meta_kubernetes_endpoint_hostname: Hostname of the endpoint.
__meta_kubernetes_endpoint_node_name: Name of the node hosting the endpoint.
__meta_kubernetes_endpoint_ready: Set to true or false for the endpoint's ready state.
__meta_kubernetes_endpoint_port_name: Name of the endpoint port.
__meta_kubernetes_endpoint_port_protocol: Protocol of the endpoint port.
__meta_kubernetes_endpoint_address_target_kind: Kind of the endpoint address target.
__meta_kubernetes_endpoint_address_target_name: Name of the endpoint address target.
If the endpoints belong to a service, all labels of the role: service discovery are attached.
For all targets backed by a pod, all labels of the role: pod discovery are attached.

__meta_kubernetes_namespace: The namespace of the ingress object.
__meta_kubernetes_ingress_name: The name of the ingress object.
__meta_kubernetes_ingress_label_<labelname>: Each label from the ingress object.
__meta_kubernetes_ingress_labelpresent_<labelname>: true for each label from the ingress object.
__meta_kubernetes_ingress_annotation_<annotationname>: Each annotation from the ingress object.
__meta_kubernetes_ingress_annotationpresent_<annotationname>: true for each annotation from the ingress object.
__meta_kubernetes_ingress_scheme: Protocol scheme of ingress, https if TLS config is set. Defaults to http.
__meta_kubernetes_ingress_path: Path from ingress spec. Defaults to /.

apiVersion: apps/v1kind: StatefulSet
metadata:labels:run: jx3recipename: jx3recipeannotations:prometheus.io/scrape: "true"
spec:selector:matchLabels:app: jx3recipeserviceName: jx3recipe-servicereplicas: 2template:metadata:labels:app: jx3recipeappCluster: jx3recipe-clusterspec:terminationGracePeriodSeconds: 20containers:- image: hub.kce.ooo.com/jx3pvp/jx3recipe:qa-latestimagePullPolicy: AlwayssecurityContext:runAsUser: 1000name: jx3recipelifecycle:preStop:exec:command: ["kill","-s","SIGINT","1"]volumeMounts:- name: config-volumemountPath: /data/conf.ymlsubPath: conf.ymlresources:requests:cpu: "100m"memory: "500Mi"env:- name: JX3PVP_ENVvalue: "qa"- name: JX3PVP_RUN_MODEvalue: "k8s"- name: JX3PVP_SERVICE_IDvalueFrom:fieldRef:fieldPath: metadata.name- name: JX3PVP_LOCAL_IPvalueFrom:fieldRef:fieldPath: status.podIP- name: JX3PVP_CONSUL_IPvalue: $(CONSUL_AGENT_SERVICE_HOST)ports:- name: bizcontainerPort: 8000protocol: "TCP"- name: admincontainerPort: 7000protocol: "TCP"volumes:- name: config-volumeconfigMap:name: app-configure-file-jx3recipeitems:- key: jx3recipe.ymlpath: conf.yml

- job_name: 'kubernetes-service-pod'kubernetes_sd_configs:- role: podrelabel_configs:- source_labels: [__meta_kubernetes_pod_container_name]action: replacetarget_label: jx3recipe- action: labelmapregex: __meta_kubernetes_pod_label_(.+)- source_labels:  ["__meta_kubernetes_pod_label_appCluster"]regex: "jx3recipe-cluster"action: keep- source_labels: [__address__]action: keepregex: '(.*):7000'

kubectl create secret generic additional-configs --from-file=prometheus-additional.yaml -n monitoring

apiVersion: v1
data:prometheus-additional.yaml: LSBqb2JfbmFtZTogJ2t1YmVybmV0ZXMtc2VydmljZS1wb2QnCiAga3ViZXJuZXRlc19zZF9jb25maWdzOgogIC0gcm9sZTogcG9kCiAgcmVsYWJlbF9jb25maWdzOgogIC0gc291cmNlX2xhYmVsczogW19fbWV0YV9rdWJlcm5ldGVzX3BvZF9jb250YWluZXJfbmFtZV0KICAgIGFjdGlvbjogcmVwbGFjZQogICAgdGFyZ2V0X2xhYmVsOiBqeDNyZWNpcGUKICAtIGFjdGlvbjogbGFiZWxtYXAKICAgIHJlZ2V4OiBfX21ldGFfa3ViZXJuZXRlc19wb2RfbGFiZWxfKC4rKQogIC0gc291cmNlX2xhYmVsczogIFsiX19tZXRhX2t1YmVybmV0ZXNfcG9kX2xhYmVsX2FwcENsdXN0ZXIiXQogICAgcmVnZXg6ICJqeDNyZWNpcGUtY2x1c3RlciIKICAgIGFjdGlvbjoga2VlcAogIC0gc291cmNlX2xhYmVsczogW19fYWRkcmVzc19fXQogICAgYWN0aW9uOiBrZWVwCiAgICByZWdleDogJyguKik6NzAwMCcK
kind: Secret
metadata:creationTimestamp: "2019-09-10T09:32:22Z"name: additional-configsnamespace: monitoringresourceVersion: "1004681"selfLink: /api/v1/namespaces/monitoring/secrets/additional-configsuid: e455d657-d3ad-11e9-95b4-fa163e3c10ff
type: Opaque

apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:labels:prometheus: k8sname: k8snamespace: monitoring
spec:alerting:alertmanagers:- name: alertmanager-mainnamespace: monitoringport: webbaseImage: quay.io/prometheus/prometheusnodeSelector:beta.kubernetes.io/os: linuxreplicas: 2secrets:- etcd-certsresources:requests:memory: 400MiruleSelector:matchLabels:prometheus: k8srole: alert-rulessecurityContext:fsGroup: 2000runAsNonRoot: truerunAsUser: 1000additionalScrapeConfigs:name: additional-configskey: prometheus-additional.yamlserviceAccountName: prometheus-k8sserviceMonitorNamespaceSelector: {}serviceMonitorSelector: {}version: v2.5.0

additionalScrapeConfigs:name: additional-configskey: prometheus-additional.yaml

kubectl apply -f prometheus-prometheus.yaml

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:name: prometheus-k8s
rules:
- apiGroups:- ""resources:- nodes- services- endpoints- pods- nodes/proxyverbs:- get- list- watch
- apiGroups:- ""resources:- configmaps- nodes/metricsverbs:- get
- nonResourceURLs:- /metricsverbs:- get