Skip to content

Commit 497ecfe

Browse files
author
Benjamin Huo
authored
Merge pull request #39 from wanjunlei/master
add loki outpt support
2 parents 296884d + d01b990 commit 497ecfe

15 files changed

+3701
-3012
lines changed

‎Makefile

+1
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ deploy: manifests
4343
# Generate manifests e.g. CRD, RBAC etc.
4444
manifests: controller-gen
4545
$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
46+
kustomize build config/crd | sed -e '/creationTimestamp/d' > manifests/setup/fluentbit-operator-crd.yaml
4647

4748
# Run go fmt against code
4849
fmt:

‎api/v1alpha2/output_types.go

+2
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ type OutputSpec struct {
5151
Stdout *output.Stdout `json:"stdout,omitempty"`
5252
// TCP defines TCP Output configuration.
5353
TCP *output.TCP `json:"tcp,omitempty"`
54+
// Loki defines Loki Output configuration.
55+
Loki *output.Loki `json:"loki,omitempty"`
5456
}
5557

5658
// +kubebuilder:object:root=true

‎api/v1alpha2/plugins/output/loki_types.go

+99
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
package output
2+
3+
import (
4+
"fmt"
5+
"kubesphere.io/fluentbit-operator/api/v1alpha2/plugins"
6+
"kubesphere.io/fluentbit-operator/pkg/utils"
7+
)
8+
9+
// +kubebuilder:object:generate:=true
10+
11+
// The loki output plugin, allows to ingest your records into a Loki service.
12+
type Loki struct {
13+
// Loki hostname or IP address.
14+
Host string `json:"host"`
15+
// Loki TCP port
16+
// +kubebuilder:validation:Minimum:=1
17+
// +kubebuilder:validation:Maximum:=65535
18+
Port *int32 `json:"port,omitempty"`
19+
// Set HTTP basic authentication user name.
20+
HTTPUser *plugins.Secret `json:"httpUser,omitempty"`
21+
// Password for user defined in HTTP_User
22+
// Set HTTP basic authentication password
23+
HTTPPasswd *plugins.Secret `json:"httpPassword,omitempty"`
24+
// Tenant ID used by default to push logs to Loki.
25+
// If omitted or empty it assumes Loki is running in single-tenant mode and no X-Scope-OrgID header is sent.
26+
TenantID *plugins.Secret `json:"tenantID,omitempty"`
27+
// Stream labels for API request. It can be multiple comma separated of strings specifying key=value pairs.
28+
// In addition to fixed parameters, it also allows to add custom record keys (similar to label_keys property).
29+
Labels []string `json:"labels,omitempty"`
30+
// Optional list of record keys that will be placed as stream labels.
31+
// This configuration property is for records key only.
32+
LabelKeys []string `json:"labelKeys,omitempty"`
33+
// Format to use when flattening the record to a log line. Valid values are json or key_value.
34+
// If set to json, the log line sent to Loki will be the Fluent Bit record dumped as JSON.
35+
// If set to key_value, the log line will be each item in the record concatenated together (separated by a single space) in the format.
36+
// +kubebuilder:validation:Enum:=json;key_value
37+
LineFormat string `json:"lineFormat,omitempty"`
38+
// If set to true, it will add all Kubernetes labels to the Stream labels.
39+
// +kubebuilder:validation:Enum:=on;off
40+
AutoKubernetesLabels string `json:"autoKubernetesLabels,omitempty"`
41+
*plugins.TLS `json:"tls,omitempty"`
42+
}
43+
44+
// implement Section() method
45+
func (_ *Loki) Name() string {
46+
return "loki"
47+
}
48+
49+
// implement Section() method
50+
func (l *Loki) Params(sl plugins.SecretLoader) (*plugins.KVs, error) {
51+
kvs := plugins.NewKVs()
52+
if l.Host != "" {
53+
kvs.Insert("host", l.Host)
54+
}
55+
if l.Port != nil {
56+
kvs.Insert("port", fmt.Sprint(*l.Port))
57+
}
58+
if l.HTTPUser != nil {
59+
u, err := sl.LoadSecret(*l.HTTPUser)
60+
if err != nil {
61+
return nil, err
62+
}
63+
kvs.Insert("http_user", u)
64+
}
65+
if l.HTTPPasswd != nil {
66+
pwd, err := sl.LoadSecret(*l.HTTPPasswd)
67+
if err != nil {
68+
return nil, err
69+
}
70+
kvs.Insert("http_passwd", pwd)
71+
}
72+
if l.TenantID != nil {
73+
id, err := sl.LoadSecret(*l.TenantID)
74+
if err != nil {
75+
return nil, err
76+
}
77+
kvs.Insert("tenant_id", id)
78+
}
79+
if l.Labels != nil && len(l.Labels) > 0 {
80+
kvs.Insert("labels", utils.ConcatString(l.Labels, ","))
81+
}
82+
if l.LabelKeys != nil && len(l.LabelKeys) > 0 {
83+
kvs.Insert("label_keys", utils.ConcatString(l.LabelKeys, ","))
84+
}
85+
if l.LineFormat != "" {
86+
kvs.Insert("line_format", l.LineFormat)
87+
}
88+
if l.AutoKubernetesLabels != "" {
89+
kvs.Insert("auto_kubernetes_labels", l.AutoKubernetesLabels)
90+
}
91+
if l.TLS != nil {
92+
tls, err := l.TLS.Params(sl)
93+
if err != nil {
94+
return nil, err
95+
}
96+
kvs.Merge(tls)
97+
}
98+
return kvs, nil
99+
}

‎api/v1alpha2/plugins/output/zz_generated.deepcopy.go

+50
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/v1alpha2/zz_generated.deepcopy.go

+5
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎config/crd/bases/logging.kubesphere.io_outputs.yaml

+196
Original file line numberDiff line numberDiff line change
@@ -677,6 +677,202 @@ spec:
677677
will be used.
678678
type: string
679679
type: object
680+
loki:
681+
description: Loki defines Loki Output configuration.
682+
properties:
683+
autoKubernetesLabels:
684+
description: If set to true, it will add all Kubernetes labels to
685+
the Stream labels.
686+
enum:
687+
- "on"
688+
- "off"
689+
type: string
690+
host:
691+
description: Loki hostname or IP address.
692+
type: string
693+
httpPassword:
694+
description: Password for user defined in HTTP_User Set HTTP basic
695+
authentication password
696+
properties:
697+
valueFrom:
698+
description: ValueSource represents a source for the value of
699+
a secret.
700+
properties:
701+
secretKeyRef:
702+
description: Selects a key of a secret in the pod's namespace
703+
properties:
704+
key:
705+
description: The key of the secret to select from. Must
706+
be a valid secret key.
707+
type: string
708+
name:
709+
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
710+
TODO: Add other useful fields. apiVersion, kind, uid?'
711+
type: string
712+
optional:
713+
description: Specify whether the Secret or its key must
714+
be defined
715+
type: boolean
716+
required:
717+
- key
718+
type: object
719+
type: object
720+
type: object
721+
httpUser:
722+
description: Set HTTP basic authentication user name.
723+
properties:
724+
valueFrom:
725+
description: ValueSource represents a source for the value of
726+
a secret.
727+
properties:
728+
secretKeyRef:
729+
description: Selects a key of a secret in the pod's namespace
730+
properties:
731+
key:
732+
description: The key of the secret to select from. Must
733+
be a valid secret key.
734+
type: string
735+
name:
736+
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
737+
TODO: Add other useful fields. apiVersion, kind, uid?'
738+
type: string
739+
optional:
740+
description: Specify whether the Secret or its key must
741+
be defined
742+
type: boolean
743+
required:
744+
- key
745+
type: object
746+
type: object
747+
type: object
748+
labelKeys:
749+
description: Optional list of record keys that will be placed as
750+
stream labels. This configuration property is for records key
751+
only.
752+
items:
753+
type: string
754+
type: array
755+
labels:
756+
description: Stream labels for API request. It can be multiple comma
757+
separated of strings specifying key=value pairs. In addition
758+
to fixed parameters, it also allows to add custom record keys
759+
(similar to label_keys property).
760+
items:
761+
type: string
762+
type: array
763+
lineFormat:
764+
description: Format to use when flattening the record to a log line.
765+
Valid values are json or key_value. If set to json, the log line
766+
sent to Loki will be the Fluent Bit record dumped as JSON. If
767+
set to key_value, the log line will be each item in the record
768+
concatenated together (separated by a single space) in the format.
769+
enum:
770+
- json
771+
- key_value
772+
type: string
773+
port:
774+
description: Loki TCP port
775+
format: int32
776+
maximum: 65535
777+
minimum: 1
778+
type: integer
779+
tenantID:
780+
description: Tenant ID used by default to push logs to Loki. If
781+
omitted or empty it assumes Loki is running in single-tenant mode
782+
and no X-Scope-OrgID header is sent.
783+
properties:
784+
valueFrom:
785+
description: ValueSource represents a source for the value of
786+
a secret.
787+
properties:
788+
secretKeyRef:
789+
description: Selects a key of a secret in the pod's namespace
790+
properties:
791+
key:
792+
description: The key of the secret to select from. Must
793+
be a valid secret key.
794+
type: string
795+
name:
796+
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
797+
TODO: Add other useful fields. apiVersion, kind, uid?'
798+
type: string
799+
optional:
800+
description: Specify whether the Secret or its key must
801+
be defined
802+
type: boolean
803+
required:
804+
- key
805+
type: object
806+
type: object
807+
type: object
808+
tls:
809+
description: Fluent Bit provides integrated support for Transport
810+
Layer Security (TLS) and it predecessor Secure Sockets Layer (SSL)
811+
respectively.
812+
properties:
813+
caFile:
814+
description: Absolute path to CA certificate file
815+
type: string
816+
caPath:
817+
description: Absolute path to scan for certificate files
818+
type: string
819+
crtFile:
820+
description: Absolute path to Certificate file
821+
type: string
822+
debug:
823+
description: 'Set TLS debug verbosity level. It accept the following
824+
values: 0 (No debug), 1 (Error), 2 (State change), 3 (Informational)
825+
and 4 Verbose'
826+
enum:
827+
- 0
828+
- 1
829+
- 2
830+
- 3
831+
- 4
832+
format: int32
833+
type: integer
834+
keyFile:
835+
description: Absolute path to private Key file
836+
type: string
837+
keyPassword:
838+
description: Optional password for tls.key_file file
839+
properties:
840+
valueFrom:
841+
description: ValueSource represents a source for the value
842+
of a secret.
843+
properties:
844+
secretKeyRef:
845+
description: Selects a key of a secret in the pod's
846+
namespace
847+
properties:
848+
key:
849+
description: The key of the secret to select from. Must
850+
be a valid secret key.
851+
type: string
852+
name:
853+
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
854+
TODO: Add other useful fields. apiVersion, kind,
855+
uid?'
856+
type: string
857+
optional:
858+
description: Specify whether the Secret or its key
859+
must be defined
860+
type: boolean
861+
required:
862+
- key
863+
type: object
864+
type: object
865+
type: object
866+
verify:
867+
description: Force certificate validation
868+
type: boolean
869+
vhost:
870+
description: Hostname to be used for TLS SNI extension
871+
type: string
872+
type: object
873+
required:
874+
- host
875+
type: object
680876
match:
681877
description: A pattern to match against the tags of incoming records.
682878
It's case sensitive and support the star (*) character as a wildcard.

‎docs/crd.md

+1
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,7 @@ OutputSpec defines the desired state of Output
233233
| null | Null defines Null Output configuration. | *[output.Null](plugins/output/null.md) |
234234
| stdout | Stdout defines Stdout Output configuration. | *[output.Stdout](plugins/output/stdout.md) |
235235
| tcp | TCP defines TCP Output configuration. | *[output.TCP](plugins/output/tcp.md) |
236+
| loki | Loki defines Loki Output configuration. | *[output.Loki](plugins/output/loki.md) |
236237

237238
[Back to TOC](#table-of-contents)
238239
## Parser

0 commit comments

Comments
 (0)