Skip to content

Commit 90f5130

Browse files
adam-irisadam-iris
committed
Basic outline of some components strung together:
- kafka backbone - avro schemas - nginx front-end - django (python) app framework
1 parent 7a69ab7 commit 90f5130

23 files changed

+756
-0
lines changed

‎.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
.env
2+
django-static/

‎avro_schemas/example-key.avsc

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"type": "record",
3+
"name": "example",
4+
"namespace": "example",
5+
"fields": [
6+
{
7+
"name": "key",
8+
"type": "string",
9+
"doc": "Identifying key"
10+
}
11+
]
12+
}

‎avro_schemas/example-value.avsc

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"namespace": "example",
3+
"name": "example",
4+
"type": "record",
5+
"fields": [
6+
{
7+
"name": "timestamp",
8+
"type": ["string","null"],
9+
"doc":"YYYY-MM-DD HH:MM:SS.fff"
10+
},
11+
{
12+
"name": "value",
13+
"type": ["string","null"],
14+
"doc":"Arbitrary value"
15+
}
16+
]
17+
}

‎avro_schemas/gtsm_etl-key.avsc

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"type": "record",
3+
"name": "gtsm_etl",
4+
"namespace": "gtsm_etl",
5+
"fields": [
6+
{
7+
"name": "site",
8+
"type": "string",
9+
"doc": "Four Character ID of Site"
10+
}
11+
]
12+
}

‎avro_schemas/gtsm_etl-value.avsc

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
{
2+
"namespace": "gtsm_etl",
3+
"name": "gtsm_etl",
4+
"type": "record",
5+
"fields": [
6+
{"name": "site", "type": ["string","null"], "doc":"four character id of site"},
7+
{"name": "time", "type": ["long","null"], "logicaltype": "timestamp-millis", "doc":"unix epoch milliseconds timestamp"},
8+
{"name": "timestamp", "type": ["string","null"], "doc":"yyyy-mm-dd hh:mm:ss.fff"},
9+
{"name": "rs1_t0", "type": ["int","null"], "doc":"10 min strain counts ch0"},
10+
{"name": "rs2_t0", "type": ["int","null"], "doc":"10 min strain counts ch1"},
11+
{"name": "rs3_t0", "type": ["int","null"], "doc":"10 min strain counts ch2"},
12+
{"name": "rs4_t0", "type": ["int","null"], "doc":"10 min strain counts ch3"},
13+
{"name": "ls1_t0", "type": ["int","null"], "doc":"1 hz strain counts ch0"},
14+
{"name": "ls2_t0", "type": ["int","null"], "doc":"1 hz strain counts ch1"},
15+
{"name": "ls3_t0", "type": ["int","null"], "doc":"1 hz strain counts ch2"},
16+
{"name": "ls4_t0", "type": ["int","null"], "doc":"1 hz strain counts ch3"},
17+
{"name": "bs1_t0", "type": ["int","null"], "doc":"20 hz strain counts ch0"},
18+
{"name": "bs2_t0", "type": ["int","null"], "doc":"20 hz strain counts ch1"},
19+
{"name": "bs3_t0", "type": ["int","null"], "doc":"20 hz strain counts ch2"},
20+
{"name": "bs4_t0", "type": ["int","null"], "doc":"20 hz strain counts ch3"},
21+
{"name": "re1_t0", "type": ["int","null"], "doc":"batteryvolts"},
22+
{"name": "rkd_t0", "type": ["int","null"], "doc":"downholedegc"},
23+
{"name": "rk1_t0", "type": ["int","null"], "doc":"loggerdegc"},
24+
{"name": "rk2_t0", "type": ["int","null"], "doc":"powerboxdegc"},
25+
{"name": "rdo_ts", "type": ["int","null"], "doc":"pressurekpa"},
26+
{"name": "rro_ts", "type": ["int","null"], "doc":"rainfallmm"},
27+
{"name": "reo_t0", "type": ["int","null"], "doc":"solaramps"},
28+
{"name": "re2_t0", "type": ["int","null"], "doc":"systemamps"},
29+
{"name": "rca_t0", "type": ["int","null"], "doc":"rtsettingch0"},
30+
{"name": "rcb_t0", "type": ["int","null"], "doc":"rtsettingch1"},
31+
{"name": "rcc_t0", "type": ["int","null"], "doc":"rtsettingch2"},
32+
{"name": "rcd_t0", "type": ["int","null"], "doc":"rtsettingch3"},
33+
{"name": "rca_t1", "type": ["int","null"], "doc":"caloffsetch0g1"},
34+
{"name": "rca_t2", "type": ["int","null"], "doc":"caloffsetch0g2"},
35+
{"name": "rca_t3", "type": ["int","null"], "doc":"caloffsetch0g3"},
36+
{"name": "rcb_t1", "type": ["int","null"], "doc":"caloffsetch1g1"},
37+
{"name": "rcb_t2", "type": ["int","null"], "doc":"caloffsetch1g2"},
38+
{"name": "rcb_t3", "type": ["int","null"], "doc":"caloffsetch1g3"},
39+
{"name": "rcc_t1", "type": ["int","null"], "doc":"caloffsetch2g1"},
40+
{"name": "rcc_t2", "type": ["int","null"], "doc":"caloffsetch2g2"},
41+
{"name": "rcc_t3", "type": ["int","null"], "doc":"caloffsetch2g3"},
42+
{"name": "rcd_t1", "type": ["int","null"], "doc":"caloffsetch3g1"},
43+
{"name": "rcd_t2", "type": ["int","null"], "doc":"caloffsetch3g2"},
44+
{"name": "rcd_t3", "type": ["int","null"], "doc":"caloffsetch3g3"},
45+
{"name": "rca_t4", "type": ["int","null"], "doc":"calstepch0g1"},
46+
{"name": "rca_t5", "type": ["int","null"], "doc":"calstepch0g2"},
47+
{"name": "rca_t6", "type": ["int","null"], "doc":"calstepch0g3"},
48+
{"name": "rcb_t4", "type": ["int","null"], "doc":"calstepch1g1"},
49+
{"name": "rcb_t5", "type": ["int","null"], "doc":"calstepch1g2"},
50+
{"name": "rcb_t6", "type": ["int","null"], "doc":"calstepch1g3"},
51+
{"name": "rcc_t4", "type": ["int","null"], "doc":"calstepch2g1"},
52+
{"name": "rcc_t5", "type": ["int","null"], "doc":"calstepch2g2"},
53+
{"name": "rcc_t6", "type": ["int","null"], "doc":"calstepch2g3"},
54+
{"name": "rcd_t4", "type": ["int","null"], "doc":"calstepch3g1"},
55+
{"name": "rcd_t5", "type": ["int","null"], "doc":"calstepch3g2"},
56+
{"name": "rcd_t6", "type": ["int","null"], "doc":"calstepch3g3"},
57+
{"name": "rca_t7", "type": ["int","null"], "doc":"caloffsetch0g0"},
58+
{"name": "rcb_t7", "type": ["int","null"], "doc":"caloffsetch1g0"},
59+
{"name": "rcc_t7", "type": ["int","null"], "doc":"caloffsetch2g0"},
60+
{"name": "rcd_t7", "type": ["int","null"], "doc":"caloffsetch3g0"},
61+
{"name": "rca_t8", "type": ["int","null"], "doc":"calstepch0g0"},
62+
{"name": "rcb_t8", "type": ["int","null"], "doc":"calstepch1g0"},
63+
{"name": "rcc_t8", "type": ["int","null"], "doc":"calstepch2g0"},
64+
{"name": "rcd_t8", "type": ["int","null"], "doc":"calstepch3g0"}
65+
]
66+
}

‎avro_schemas/strain-key.avsc

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"type": "record",
3+
"name": "strain",
4+
"namespace": "strain",
5+
"fields": [
6+
{
7+
"name": "site",
8+
"type": "string",
9+
"doc": "Four Character ID of Site"
10+
}
11+
]
12+
}

‎avro_schemas/strain-value.avsc

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"namespace": "strain",
3+
"name": "strain",
4+
"type": "record",
5+
"fields": [
6+
{"name": "site", "type": ["string","null"], "doc":"Four Character ID of Site"},
7+
{"name": "time", "type": ["long","null"], "logicalType": "timestamp-millis", "doc":"Unix epoch milliseconds timestamp"},
8+
{"name": "timestamp", "type": ["string","null"], "doc":"YYYY-MM-DD HH:MM:SS.fff"},
9+
{"name": "ch0", "type": ["int","null"], "doc":"raw counts from CH0"},
10+
{"name": "ch1", "type": ["int","null"], "doc":"raw counts from CH1"},
11+
{"name": "ch2", "type": ["int","null"], "doc":"raw counts from CH2"},
12+
{"name": "ch3", "type": ["int","null"], "doc":"raw counts from CH3"}
13+
]
14+
}

‎django/.env-local

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# When developing locally, install dotenv and copy this to .env
2+
BOOTSTRAP_SERVERS="localhost:9092"
3+
SCHEMA_REGISTRY_URL="http://localhost:8081"
4+
AVRO_SCHEMAS_ROOT="${CWD}/avro_schemas"

‎django/Dockerfile

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
FROM python:3
2+
ENV PYTHONUNBUFFERED=1
3+
WORKDIR /django
4+
COPY requirements.txt .
5+
RUN pip install -r requirements.txt
6+
COPY . ./
7+
RUN python manage.py migrate

‎django/gunicorn.conf.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
bind = "0.0.0.0:9090"
2+
workers = 4
3+
worker_class = 'gevent'
4+
threads = 4
5+
loglevel = "debug"
6+
# chdir = "/django"
7+
wsgi_app = "www.wsgi"
8+
reload = True

0 commit comments

Comments
 (0)