Skip to content

Commit 2f5bce8

Browse files
anapsixanapsix
committedJun 1, 2020
customize ilm policy
·
v0.6.0v0.5.0-rc2
1 parent 567632c commit 2f5bce8

13 files changed

+381
-115
lines changed
 

‎README.md

+14-2
Original file line numberDiff line numberDiff line change
@@ -35,13 +35,24 @@ After launching local environment, access Kibana via http://localhost:5601/app/k
3535
#### With Docker Compose
3636
3737
```sh
38+
# (re)build
39+
docker-compose build
40+
3841
# launch Elasticsearch, Kibana, and get-logs container instances
3942
docker-compose up -d
4043
4144
# keep an eye on the logs
4245
docker-compose logs -f get-logs
4346
```
4447
48+
After launching local environment, access Kibana via http://localhost:5601/app/kibana#/discover.
49+
50+
> NOTE: since log collection is running on schedule, data will not appear in ES
51+
> immediately. Keen an eye on the logs, and `./logs` directory. ES index will
52+
> receive data shortly after you see new file appearing, and / or
53+
> `Harvester started for file:...` log message in the logs. Create an index
54+
> pattern while you are waiting.
55+
4556
#### Launch manually
4657

4758
```sh
@@ -67,7 +78,7 @@ docker run -it --rm \
6778
-e SAMPLE_RATE="0.01" \
6879
-e ES_HOST="http://elasticsearch:9200" \
6980
-e ES_INDEX="cloudflare-test" \
70-
-e ES_INDEX_SHARD=6 \
81+
-e ES_INDEX_SHARD=5 \
7182
-e ES_INDEX_REPLICAS=0 \
7283
-e ES_INDEX_REFRESH=10s \
7384
--link es:elasticsearch \
@@ -82,7 +93,8 @@ Open-sourced software licensed under the MIT license.
8293

8394
## Acknowledgments
8495

85-
This repo includes and relies on [go-tasks][go-tasks], created by [Martin Fabrizzio Vilche][mvilche]
96+
This repo includes and relies on [go-tasks][go-tasks], created by
97+
[Martin Fabrizzio Vilche][mvilche]. Thank you 🙏, Martin.
8698

8799
[link reference]::
88100
[logpull]: https://developers.cloudflare.com/logs/logpull-api/

‎config/filebeat.yml

+10-11
Original file line numberDiff line numberDiff line change
@@ -50,28 +50,27 @@ setup.dashboard:
5050
enabled: false
5151

5252
setup.ilm:
53+
enabled: false # ES_ILM_ENABLED in startup script
5354
check_exists: true
54-
enabled: false
55-
overwrite: false
55+
overwrite: false # ES_ILM_OVERWRITE in startup script
5656
pattern: '{now/M{YYYY.MM}}-000001'
5757
policy_name: '${ES_INDEX:cloudflare-test}'
58-
# policy_file: ${ES_ILM_POLICY_FILE}
58+
# policy_file: ${ES_ILM_POLICY_FILE} in startup script
5959
rollover_alias: '${ES_INDEX:cloudflare-test}'
6060

6161
setup.kibana:
6262
enabled: false
6363

6464
setup.template:
65-
enabled: false
65+
enabled: false # ES_TEMPLATE_ENABLED in startup script
6666
name: '${ES_INDEX:cloudflare-test}'
67-
overwrite: false
67+
overwrite: false # ES_TEMPLATE_OVERWRITE in startup script
6868
pattern: '${ES_INDEX:cloudflare-test}-*'
6969
# fields: "fields.yml"
70-
json.enabled: true
71-
json.path: '/opt/filebeat/index-template.json'
70+
json.enabled: false # ES_TEMPLATE_JSON_ENABLED in startup script
7271
json.name: '${ES_INDEX:cloudflare-test}'
72+
json.path: '${ES_TEMPLATE_JSON_FILE:/opt/filebeat/index-template.json}'
7373
settings:
74-
# _source.enabled: false
75-
index.number_of_shards: ${ES_INDEX_SHARDS:6}
76-
index.number_of_replicas: ${ES_INDEX_REPLICAS:0}
77-
index.refresh_interval: ${ES_INDEX_REFRESH:5s}
74+
index.number_of_shards: ${ES_TEMPLATE_INDEX_SHARDS:5}
75+
index.number_of_replicas: ${ES_TEMPLATE_INDEX_REPLICAS:0}
76+
index.refresh_interval: ${ES_TEMPLATE_INDEX_REFRESH:10s}

‎config/templates/ilm-default-policy.json

+5-5
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
2-
"policy" : {
3-
"phases" : {
2+
"policy": {
3+
"phases": {
44
"hot": {
55
"min_age": "0ms",
66
"actions": {
@@ -10,15 +10,15 @@
1010
}
1111
}
1212
},
13-
"warm" : {
13+
"warm": {
1414
"min_age": "0ms",
1515
"actions": {
1616
"forcemerge": {
17-
"max_num_segments" : 1
17+
"max_num_segments": 1
1818
}
1919
}
2020
},
21-
"delete" : {
21+
"delete": {
2222
"min_age": "7d",
2323
"actions": {
2424
"delete": { }

‎docker-compose.yaml

+6-4
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,13 @@ services:
1515
- SAMPLE_RATE=0.01
1616
- ES_HOST=http://elasticsearch:9200
1717
- ES_INDEX=cloudflare-test
18-
- ES_INDEX_SHARDS=1
19-
- ES_INDEX_REPLICAS=0
20-
- ES_INDEX_REFRESH=10s
18+
- ES_TEMPLATE_ENABLED=true
19+
- ES_TEMPLATE_INDEX_SHARDS=1
20+
- ES_TEMPLATE_INDEX_REPLICAS=0
21+
- ES_TEMPLATE_INDEX_REFRESH=10s
22+
- ES_ILM_ENABLED=true
2123
- ES_ILM_DEFAULT_POLICY_ENABLED=true
22-
- ES_INDEX_DEFAULT_PIPELINE_ENABLED=true
24+
- ES_PIPELINE_DEFAULT_ENABLED=true
2325
healthcheck:
2426
test: 'pgrep filebeat || exit 1'
2527
interval: 30s

‎helm/Chart.yaml

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
apiVersion: v2
22
name: get-cloudflare-logs
3-
description: Helm chart installing
3+
description: Helm chart to retrieve Cloudflare Logs via LogPull API
44

55
# A chart can be either an 'application' or a 'library' chart.
66
#
@@ -14,8 +14,8 @@ type: application
1414

1515
# This is the chart version. This version number should be incremented each time you make changes
1616
# to the chart and its templates, including the app version.
17-
version: 0.3.0
17+
version: 0.4.0-rc1
1818

1919
# This is the version number of the application being deployed. This version number should be
2020
# incremented each time you make changes to the application.
21-
appVersion: 0.4.0 # assessed as .Chart.AppVersion
21+
appVersion: 0.5.0-rc1 # assessed as .Chart.AppVersion

‎helm/README.md

+50-4
Original file line numberDiff line numberDiff line change
@@ -55,8 +55,54 @@ $ helm delete my-release
5555
`config.elasticsearch.username` | Elasticsearch connection username | `nil`
5656
`config.elasticsearch.password` | Elasticsearch connection password | `nil`
5757
`config.elasticsearch.index.name` | Elasticsearch dst index | `cloudflare-access`
58-
`config.elasticsearch.index.shards` | Elasticsearch dst index shards | `5`
59-
`config.elasticsearch.index.replicas` | Elasticsearch dst index replicas | `0`
60-
`config.elasticsearch.index.refreshInterval` | Elasticsearch dst index refresh interval | `5s`
61-
`config.elasticsearch.ilm.default` | Enables use of default ILM policy | `true`
58+
`config.elasticsearch.index.template.shards` | Elasticsearch dst index shards | `5`
59+
`config.elasticsearch.index.template.replicas` | Elasticsearch dst index replicas | `0`
60+
`config.elasticsearch.index.template.refreshInterval` | Elasticsearch dst index refresh interval | `10s`
61+
`config.elasticsearch.ilm.enabled` | Enables ILM use | `true`
62+
`config.elasticsearch.ilm.policyYAML` | Specifies policy via YAML | see [`values.yaml`][values]
63+
`config.elasticsearch.ilm.policyJSON` | Specifies policy via literal JSON | see [`values.yaml`][values]
64+
`config.elasticsearch.ilm.policyFile` | Specifies a file on local filesystem to use as ILM policy | `files/ilm-default-policy.json`
65+
`config.elasticsearch.pipeline.enabled` | Enables Ingest Pipeline | `true` (at the moment, has no effect)
6266
`config.elasticsearch.pipeline.default` | Enables use of default Ingest Pipeline | `true`
67+
68+
69+
## Advanced Configuration
70+
71+
### ILM Policy
72+
73+
By default, ILM policy setup is enabled, and policy defined with
74+
`config.elasticsearch.ilm.policyYAML` (same as one included in Docker image)
75+
will be used.
76+
To customize ILM policy, change `config.elasticsearch.ilm.policyYAML`.
77+
78+
> ILM policy is stored as ConfigMap, and passed to the pods via read-only mount.
79+
80+
ILM policy can be defined using any of the following attributes, listed in the
81+
order of precedence.
82+
- `config.elasticsearch.ilm.policyYAML` - inline, convenient for tweaking individual phases
83+
- `config.elasticsearch.ilm.policyJSON` - inline, convenient when copying from ES
84+
- `config.elasticsearch.ilm.policyFile` - from local file, convenient when copying from ES
85+
86+
If `policyYAML` is unset, or evaluates to `false`, `policyJSON` will be used.
87+
If both `policyYAML`, and `policyJSON` are unset, or evaluate to `false`,
88+
`policyFile` will be used
89+
90+
Read more about ILM and ILM policy in [Elasticsearch docs][ilm-docs].
91+
92+
### Ingest Pipeline
93+
94+
By default, Ingest Pipeline included in Docker image is created, and enabled as
95+
["default pipeline"][index-docs]. Read more about Pipelines and pipeline
96+
processors in [Elasticsearch docs][pipeline-docs].
97+
98+
Default pipeline is configured with following processors:
99+
- `user_agent` - to process `cloudflare.ClientRequestUserAgent`
100+
- `geoip` - to process `cloudflare.ClientIP`
101+
102+
103+
104+
[ link reference ]::
105+
[ilm-docs]: https://www.elastic.co/guide/en/elasticsearch/reference/current/index-lifecycle-management.html
106+
[pipeline-docs]: https://www.elastic.co/guide/en/elasticsearch/reference/current/pipeline.html
107+
[index-docs]: https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules.html#dynamic-index-settings
108+
[values]: ./values.yaml

‎helm/files/ilm-default-policy.json

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
{
2+
"policy": {
3+
"phases": {
4+
"hot": {
5+
"min_age": "0ms",
6+
"actions": {
7+
"rollover": {
8+
"max_size": "60gb",
9+
"max_age": "1d"
10+
}
11+
}
12+
},
13+
"warm": {
14+
"min_age": "0ms",
15+
"actions": {
16+
"forcemerge": {
17+
"max_num_segments": 1
18+
}
19+
}
20+
},
21+
"delete": {
22+
"min_age": "7d",
23+
"actions": {
24+
"delete": { }
25+
}
26+
}
27+
}
28+
}
29+
}

‎helm/templates/configmap-ilm-policy.yaml

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{{- if .Values.config.elasticsearch.ilm.enabled -}}
2+
apiVersion: v1
3+
kind: ConfigMap
4+
metadata:
5+
name: {{ .Release.Name }}-ilm-policy
6+
namespace: {{ .Release.Namespace }}
7+
labels:
8+
{{- include "get-cloudflare-logs.labels" . | nindent 4 }}
9+
data:
10+
ilm-policy.json: |-
11+
{{- if .Values.config.elasticsearch.ilm.policyYAML }}
12+
{
13+
"policy": {{- toJson .Values.config.elasticsearch.ilm.policyYAML | nindent 6 }}
14+
}
15+
{{ else if .Values.config.elasticsearch.ilm.policyJSON }}
16+
{{ .Values.config.elasticsearch.ilm.policyJSON | indent 4 }}
17+
{{ else }}
18+
{{ .Files.Get .Values.config.elasticsearch.ilm.policyFile | indent 4 }}
19+
{{ end -}}
20+
{{- end -}}

‎helm/templates/deployment.yaml

+64-41
Original file line numberDiff line numberDiff line change
@@ -31,47 +31,64 @@ spec:
3131
image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}"
3232
imagePullPolicy: {{ .Values.image.pullPolicy }}
3333
env:
34-
- name: CF_ZONE_ID
35-
valueFrom:
36-
secretKeyRef:
37-
name: {{ include "get-cloudflare-logs.fullname" . }}
38-
key: CF_ZONE_ID
39-
- name: CF_AUTH_EMAIL
40-
valueFrom:
41-
secretKeyRef:
42-
name: {{ include "get-cloudflare-logs.fullname" . }}
43-
key: CF_AUTH_EMAIL
44-
- name: CF_AUTH_KEY
45-
valueFrom:
46-
secretKeyRef:
47-
name: {{ include "get-cloudflare-logs.fullname" . }}
48-
key: CF_AUTH_KEY
49-
- name: SAMPLE_RATE
50-
value: {{ .Values.config.cloudflare.sampleRate | quote }}
51-
- name: ES_HOST
52-
value: {{ .Values.config.elasticsearch.host | quote }}
53-
- name: ES_USERNAME
54-
valueFrom:
55-
secretKeyRef:
56-
name: {{ include "get-cloudflare-logs.fullname" . }}
57-
key: ES_USERNAME
58-
- name: ES_PASSWORD
59-
valueFrom:
60-
secretKeyRef:
61-
name: {{ include "get-cloudflare-logs.fullname" . }}
62-
key: ES_PASSWORD
63-
- name: ES_INDEX
64-
value: {{ .Values.config.elasticsearch.index.name | quote }}
65-
- name: ES_INDEX_SHARDS
66-
value: {{ .Values.config.elasticsearch.index.shards | quote }}
67-
- name: ES_INDEX_REPLICAS
68-
value: {{ .Values.config.elasticsearch.index.replicas | quote }}
69-
- name: ES_INDEX_REFRESH
70-
value: {{ .Values.config.elasticsearch.index.refreshInterval | quote }}
71-
- name: ES_INDEX_DEFAULT_PIPELINE_ENABLED
72-
value: {{ default "false" .Values.config.elasticsearch.pipeline.default | quote }}
73-
- name: ES_ILM_DEFAULT_POLICY_ENABLED
74-
value: {{ default "false" .Values.config.elasticsearch.ilm.default | quote }}
34+
- name: CF_ZONE_ID
35+
valueFrom:
36+
secretKeyRef:
37+
name: {{ include "get-cloudflare-logs.fullname" . }}
38+
key: CF_ZONE_ID
39+
- name: CF_AUTH_EMAIL
40+
valueFrom:
41+
secretKeyRef:
42+
name: {{ include "get-cloudflare-logs.fullname" . }}
43+
key: CF_AUTH_EMAIL
44+
- name: CF_AUTH_KEY
45+
valueFrom:
46+
secretKeyRef:
47+
name: {{ include "get-cloudflare-logs.fullname" . }}
48+
key: CF_AUTH_KEY
49+
- name: SAMPLE_RATE
50+
value: {{ .Values.config.cloudflare.sampleRate | quote }}
51+
- name: ES_HOST
52+
value: {{ .Values.config.elasticsearch.host | quote }}
53+
- name: ES_USERNAME
54+
valueFrom:
55+
secretKeyRef:
56+
name: {{ include "get-cloudflare-logs.fullname" . }}
57+
key: ES_USERNAME
58+
- name: ES_PASSWORD
59+
valueFrom:
60+
secretKeyRef:
61+
name: {{ include "get-cloudflare-logs.fullname" . }}
62+
key: ES_PASSWORD
63+
- name: ES_INDEX
64+
value: {{ .Values.config.elasticsearch.index.name | quote }}
65+
- name: ES_TEMPLATE_ENABLED
66+
value: {{ default "false" .Values.config.elasticsearch.index.template.enabled | quote }}
67+
- name: ES_TEMPLATE_INDEX_SHARDS
68+
value: {{ .Values.config.elasticsearch.index.template.shards | quote }}
69+
- name: ES_TEMPLATE_INDEX_REPLICAS
70+
value: {{ .Values.config.elasticsearch.index.template.replicas | quote }}
71+
- name: ES_TEMPLATE_INDEX_REFRESH
72+
value: {{ .Values.config.elasticsearch.index.template.refreshInterval | quote }}
73+
- name: ES_PIPELINE_ENABLED
74+
value: {{ default "false" .Values.config.elasticsearch.pipeline.enabled | quote }}
75+
- name: ES_PIPELINE_DEFAULT_ENABLED
76+
value: {{ default "false" .Values.config.elasticsearch.pipeline.default | quote }}
77+
- name: ES_ILM_ENABLED
78+
value: {{ .Values.config.elasticsearch.ilm.enabled | quote }}
79+
{{- if .Values.config.elasticsearch.ilm.enabled }}
80+
- name: ES_ILM_DEFAULT_POLICY_ENABLED
81+
value: "false"
82+
- name: ES_ILM_POLICY_FILE
83+
value: "/opt/filebeat/ilm-policy.json"
84+
{{- end }}
85+
{{- if .Values.config.elasticsearch.ilm.enabled }}
86+
volumeMounts:
87+
- name: ilm-policy
88+
mountPath: /opt/filebeat/ilm-policy.json
89+
subPath: ilm-policy.json
90+
readOnly: true
91+
{{- end }}
7592
livenessProbe:
7693
exec:
7794
command:
@@ -88,6 +105,12 @@ spec:
88105
periodSeconds: 5
89106
resources:
90107
{{- toYaml .Values.resources | nindent 12 }}
108+
{{- if .Values.config.elasticsearch.ilm.enabled }}
109+
volumes:
110+
- name: ilm-policy
111+
configMap:
112+
name: {{ .Release.Name }}-ilm-policy
113+
{{- end }}
91114
{{- with .Values.nodeSelector }}
92115
nodeSelector:
93116
{{- toYaml . | nindent 8 }}

‎helm/templates/secrets.yaml

+7-2
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,10 @@ data:
1010
CF_ZONE_ID: {{ .Values.config.cloudflare.zoneId | b64enc }}
1111
CF_AUTH_EMAIL: {{ .Values.config.cloudflare.authEmail | b64enc }}
1212
CF_AUTH_KEY: {{ .Values.config.cloudflare.authKey | b64enc }}
13-
ES_USERNAME: {{ default "" .Values.config.elasticsearch.username | b64enc }}
14-
ES_PASSWORD: {{ default "" .Values.config.elasticsearch.password | b64enc }}
13+
{{- if .Values.config.elasticsearch.username }}
14+
ES_USERNAME: {{ .Values.config.elasticsearch.username | b64enc }}
15+
{{- end -}}
16+
{{ if .Values.config.elasticsearch.password }}
17+
ES_PASSWORD: {{ .Values.config.elasticsearch.password | b64enc }}
18+
{{ end -}}
19+

‎helm/values.yaml

+92-6
Original file line numberDiff line numberDiff line change
@@ -4,24 +4,110 @@
44

55
# application config
66
config:
7+
78
cloudflare:
89
zoneId: "51e241f08e014feb95d1b2760228d12a"
910
authEmail: "admin@example.com"
1011
authKey: "51e241f08e014feb95d1b2760228d12a2df50"
11-
sampleRate: "0.01"
12+
sampleRate: "0.01" # sample percent of logs, 0.01 = 1%
13+
14+
1215
elasticsearch:
1316
host: "http://elasticsearch:9200" # make sure to always include port!
1417
username: ~ # for password protected Elasticsearch instances
1518
password: ~ # set username and password
19+
1620
index:
1721
name: "cloudflare-access"
18-
shards: "5"
19-
replicas: "0"
20-
refreshInterval: "5s"
22+
template:
23+
enabled: true # enables template setup
24+
shards: "5" # sets number of index shards
25+
replicas: "0" # sets number of index replicas
26+
refreshInterval: "10s" # sets index refresh interval
27+
28+
# ingest pipeline is using processors to extract helpful info, and more
2129
pipeline:
22-
default: true # enabled use of default pipeline
30+
enabled: true # enables pipeline (unless true, UserAgent, and Lat/Lon
31+
# will not be available)
32+
default: true # enables use of default pipeline, customization is not
33+
# supported at the moment, and it's unlikely you'll want to
34+
# change it anyways
35+
# default pipeline is configured with following processors:
36+
# user_agent - to process cloudflare.ClientRequestUserAgent
37+
# geoip - to process cloudflare.ClientIP
38+
2339
ilm:
24-
default: true # enables use of default ilm policy
40+
# When ILM is enabled, and ilm.default is set to "true"
41+
# default policy from the image will be used
42+
# Custom ILM policy can be used by setting ilm.default to "false"
43+
# and specifying policyFile location as local filepath of JSON file.
44+
# See example ILM policy in ./files/ilm-default-policy.json
45+
# To learn more about Elasticsearch ILM see Elasticsearch docs
46+
# https://www.elastic.co/guide/en/elasticsearch/reference/current/index-lifecycle-management.html
47+
enabled: true # enables setup of ilm on start
48+
49+
# ILM policy can be defined via either of these,
50+
# listed in the order of priority
51+
# - policyYAML
52+
# - policyJSON
53+
# - policyFile
54+
#
55+
# all default policies are identical
56+
57+
# takes precedence
58+
policyYAML:
59+
phases:
60+
hot:
61+
min_age: 0ms
62+
actions:
63+
rollover:
64+
max_size: 60gb
65+
max_age: 1d
66+
warm:
67+
min_age: 0ms
68+
actions:
69+
forcemerge:
70+
max_num_segments: 1
71+
delete:
72+
min_age: 7d
73+
actions:
74+
delete: {}
75+
76+
# used if policyYAML is unset, empty or nil
77+
policyJSON: |
78+
{
79+
"policy": {
80+
"phases": {
81+
"hot": {
82+
"min_age": "0ms",
83+
"actions": {
84+
"rollover": {
85+
"max_size": "60gb",
86+
"max_age": "1d"
87+
}
88+
}
89+
},
90+
"warm": {
91+
"min_age": "0ms",
92+
"actions": {
93+
"forcemerge": {
94+
"max_num_segments" : 1
95+
}
96+
}
97+
},
98+
"delete": {
99+
"min_age": "7d",
100+
"actions": {
101+
"delete": { }
102+
}
103+
}
104+
}
105+
}
106+
}
107+
108+
# used if both policyYAML and policyJSON are unset, empty or nil
109+
policyFile: "files/ilm-default-policy.json"
110+
25111

26112
# k8s objects config
27113
terminationGracePeriodSeconds: 15

‎scripts/docker-entrypoint.sh

+76-35
Original file line numberDiff line numberDiff line change
@@ -19,21 +19,30 @@ set -o pipefail
1919
: ${ES_PASSWORD:='_unset_'}
2020

2121
: ${ES_INDEX:='cloudflare-test'}
22-
: ${ES_INDEX_SHARDS:=6}
23-
: ${ES_INDEX_REPLICAS:=0}
24-
: ${ES_INDEX_REFRESH:='5s'}
25-
: ${ES_INDEX_JSON_ENABLED:='true'}
2622

23+
: ${ES_TEMPLATE_ENABLED:='true'}
24+
: ${ES_TEMPLATE_OVERWRITE:='true'}
25+
: ${ES_TEMPLATE_INDEX_SHARDS:=6}
26+
: ${ES_TEMPLATE_INDEX_REPLICAS:=0}
27+
: ${ES_TEMPLATE_INDEX_REFRESH:='10s'}
28+
29+
: ${ES_TEMPLATE_JSON_ENABLED:='true'}
30+
: ${ES_TEMPLATE_JSON_FILE:='/opt/filebeat/index-template.json'}
31+
32+
: ${ES_ILM_ENABLED:='true'}
33+
: ${ES_ILM_OVERWRITE:='true'}
34+
: ${ES_ILM_POLICY_FILE:='_unset_'}
2735
: ${ES_ILM_DEFAULT_POLICY_FILE:='/opt/filebeat/ilm-default-policy.json'}
2836
: ${ES_ILM_DEFAULT_POLICY_ENABLED:='true'}
2937

30-
: ${ES_INDEX_DEFAULT_PIPELINE:='cloudflare'}
31-
: ${ES_INDEX_DEFAULT_PIPELINE_FILE:='/opt/filebeat/ingest-default-pipeline.json'}
32-
: ${ES_INDEX_DEFAULT_PIPELINE_ENABLED:='true'}
38+
: ${ES_PIPELINE_ENABLED:='true'}
39+
: ${ES_PIPELINE_DEFAULT:='cloudflare'}
40+
: ${ES_PIPELINE_DEFAULT_FILE:='/opt/filebeat/ingest-default-pipeline.json'}
41+
: ${ES_PIPELINE_DEFAULT_ENABLED:='true'}
3342

3443
export TZ='UTC'
3544
export CF_AUTH_EMAIL CF_AUTH_KEY CF_ZONE_ID CF_LOGS_DIRECTORY
36-
export FILEBEAT_CONFIG ES_INDEX_JSON_ENABLED
45+
export FILEBEAT_CONFIG
3746
export ES_INDEX ES_INDEX_SHARDS ES_INDEX_REPLICAS ES_INDEX_REFRESH
3847

3948
rawurlencode() {
@@ -55,31 +64,31 @@ rawurlencode() {
5564

5665
install_pipeline() {
5766
local ES_CREDENTIALS
58-
if [[ ! -r "${ES_INDEX_DEFAULT_PIPELINE_FILE}" ]]; then
59-
echo >&2 "ERROR: default index pipeline cannot be read at \"${ES_INDEX_DEFAULT_PIPELINE_FILE}\""
67+
if [[ ! -r "${ES_PIPELINE_DEFAULT_FILE}" ]]; then
68+
echo >&2 "ERROR: default index pipeline cannot be read at \"${ES_PIPELINE_DEFAULT_FILE}\""
6069
exit 1
6170
fi
6271
if [[ "${ES_USERNAME}" != "_unset_" ]] && [[ "${ES_PASSWORD}" != "_unset_" ]]; then
6372
ES_CREDENTIALS="$(rawurlencode "${ES_USERNAME}"):$(rawurlencode "${ES_PASSWORD}")@"
6473
fi
65-
local ES_URL="${ES_HOST%%/*}//${ES_CREDENTIALS:-}${ES_HOST##*/}/_ingest/pipeline/${ES_INDEX_DEFAULT_PIPELINE}"
74+
local ES_URL="${ES_HOST%%/*}//${ES_CREDENTIALS:-}${ES_HOST##*/}/_ingest/pipeline/${ES_PIPELINE_DEFAULT}"
6675
curl \
6776
-sS \
6877
-X PUT \
6978
-H "Content-Type: application/json" \
70-
-d @"${ES_INDEX_DEFAULT_PIPELINE_FILE}" \
79+
-d @"${ES_PIPELINE_DEFAULT_FILE}" \
7180
"${ES_URL}"
7281
}
7382

7483
generate_index_template() {
7584
jq \
7685
--arg idx "${ES_INDEX}" \
7786
--arg ip "${ES_INDEX}-*" \
78-
--arg shards "${ES_INDEX_SHARDS}" \
79-
--arg replicas "${ES_INDEX_REPLICAS}" \
80-
--arg refresh_interval "${ES_INDEX_REFRESH}" \
81-
--arg default_pipeline "${ES_INDEX_DEFAULT_PIPELINE}" \
82-
--arg default_pipeline_enabled "${ES_INDEX_DEFAULT_PIPELINE_ENABLED}" \
87+
--arg shards "${ES_TEMPLATE_INDEX_SHARDS}" \
88+
--arg replicas "${ES_TEMPLATE_INDEX_REPLICAS}" \
89+
--arg refresh_interval "${ES_TEMPLATE_INDEX_REFRESH}" \
90+
--arg default_pipeline "${ES_PIPELINE_DEFAULT}" \
91+
--arg default_pipeline_enabled "${ES_PIPELINE_DEFAULT_ENABLED}" \
8392
'
8493
.index_patterns = $ip |
8594
.settings.index.lifecycle.name = $idx |
@@ -93,7 +102,7 @@ generate_index_template() {
93102
.
94103
end' \
95104
"${INDEX_TEMPLATE_FILE}" \
96-
> /opt/filebeat/index-template.json
105+
> "${ES_TEMPLATE_JSON_FILE}"
97106
}
98107

99108
init_message() {
@@ -138,9 +147,9 @@ cat <<EOM
138147
"EdgeResponseStatus": 200,
139148
"EdgeServerIP": "127.0.0.1",
140149
"EdgeStartTimestamp": 0000000010000000000,
141-
"FirewallMatchesActions": [],
142-
"FirewallMatchesRuleIDs": [],
143-
"FirewallMatchesSources": [],
150+
"FirewallMatchesActions": [ "simulate", "challenge" ],
151+
"FirewallMatchesRuleIDs": [ "47b718f2f84149e4a2973d6271c4aa6a", "1cb257e2891d4c108c0a9b527ab2a76d" ],
152+
"FirewallMatchesSources": [ "firewallRules", "firewallRules" ],
144153
"OriginIP": "127.0.0.1",
145154
"OriginResponseBytes": 0,
146155
"OriginResponseHTTPExpires": "Thu, 01 Jan 1970 01:00:00 GMT",
@@ -215,34 +224,66 @@ else
215224
echo >&2
216225
setup_cron
217226

218-
if [[ "${ES_ILM_DEFAULT_POLICY_ENABLED}" == "true" ]]; then
219-
echo >&2 '### using default ilm policy'
227+
if [[ "${ES_ILM_ENABLED}" == "true" ]]; then
228+
echo >&2 '## ilm setup enabled'
220229
echo >&2
221-
ilm_policy_file_arg="-E setup.ilm.policy_file='${ES_ILM_DEFAULT_POLICY_FILE}'"
230+
if [[ "${ES_ILM_DEFAULT_POLICY_ENABLED}" == "true" ]]; then
231+
echo >&2 '### using default ilm policy'
232+
echo >&2
233+
ilm_policy_file_arg="-E setup.ilm.policy_file='${ES_ILM_DEFAULT_POLICY_FILE}'"
234+
elif [[ "${ES_ILM_POLICY_FILE}" != "_unset_" ]]; then
235+
echo >&2 "### using custom ilm policy from \"${ES_ILM_POLICY_FILE}\""
236+
echo >&2
237+
if [[ ! -r "{ES_ILM_POLICY_FILE}" ]]; then
238+
echo >&2 "ERROR: unable to read policy file \"${ES_ILM_POLICY_FILE}\""
239+
exit 1
240+
fi
241+
ilm_policy_file_arg="-E setup.ilm.policy_file='${ES_ILM_POLICY_FILE}'"
242+
else
243+
echo >&2 '### ilm policy is not specified'
244+
echo >&2
245+
fi
246+
else
247+
echo >&2 '## ilm setup disabled'
222248
fi
223249

224-
if [[ "${ES_INDEX_DEFAULT_PIPELINE_ENABLED}" == "true" ]]; then
225-
echo >&2 '## installing default pipeline'
250+
if [[ "${ES_PIPELINE_ENABLED}" ]]; then
251+
echo >&2 '## ingest pipeline enabled'
226252
echo >&2
227-
install_pipeline
253+
if [[ "${ES_PIPELINE_DEFAULT_ENABLED}" == "true" ]]; then
254+
echo >&2 '### installing default pipeline'
255+
echo >&2
256+
install_pipeline
257+
else
258+
echo >&2 '### custom ingest pipeline is not supported at the moment'
259+
echo >&2 '### though you could overwrite $ES_PIPELINE_DEFAULT_FILE value'
260+
exit 1
261+
fi
228262
fi
229263

230-
if [[ "${ES_INDEX_JSON_ENABLED}" == "true" ]]; then
231-
echo >&2 '## generating index template'
264+
if [[ "${ES_TEMPLATE_ENABLED}" == "true" ]]; then
265+
echo >&2 '## template setup enabled'
266+
echo >&2
267+
if [[ "${ES_TEMPLATE_JSON_ENABLED}" == "true" ]]; then
268+
echo >&2 '### generating index template'
269+
echo >&2
270+
generate_index_template
271+
fi
272+
else
273+
echo >&2 '## template setup disabled'
232274
echo >&2
233-
generate_index_template
234275
fi
235276

236277
echo >&2 '## running Filebeat setup'
237278
echo >&2
238279
filebeat \
239280
-c "${FILEBEAT_CONFIG}" \
240-
-E setup.ilm.enabled=true \
241-
-E setup.ilm.overwrite=true \
281+
-E setup.ilm.enabled=${ES_ILM_ENABLED} \
282+
-E setup.ilm.overwrite=${ES_ILM_OVERWRITE} \
242283
${ilm_policy_file_arg:-} \
243-
-E setup.template.enabled=true \
244-
-E setup.template.overwrite=true \
245-
-E setup.template.json.enabled=${ES_INDEX_JSON_ENABLED} \
284+
-E setup.template.enabled=${ES_TEMPLATE_ENABLED} \
285+
-E setup.template.overwrite=${ES_TEMPLATE_OVERWRITE} \
286+
-E setup.template.json.enabled=${ES_TEMPLATE_JSON_ENABLED} \
246287
setup --index-management
247288

248289
echo >&2

‎scripts/get_asn/get_asn.py

+5-2
Original file line numberDiff line numberDiff line change
@@ -14,5 +14,8 @@
1414
for line in data_file:
1515
ip = line.strip()
1616
obj = IPWhois(ip)
17-
results = obj.lookup_rdap(depth=0,inc_raw=False,inc_nir=False,asn_methods=['dns'])
18-
print('%s %s' % (ip, results['asn']))
17+
try:
18+
results = obj.lookup_rdap(depth=0,inc_raw=False,inc_nir=False,asn_methods=['dns'])
19+
print('%s %s' % (ip, results['asn']))
20+
except:
21+
print('%s %s' % (ip, 'unknown'))

0 commit comments

Comments
 (0)
Please sign in to comment.