Skip to content

Commit d32c854

Browse files
AdheipSinghAdheipSingh
authored
Merge pull request #29 from datainfrahq/aws
add aws examples
2 parents 496a6b3 + 59e043d commit d32c854

File tree

4 files changed

+749
-0
lines changed

4 files changed

+749
-0
lines changed

‎docs/getting_started_aws.md‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
#### Export your StorageClassName
2+
```
3+
export STORAGE_CLASS_NAME=gp2
4+
```
5+
6+
#### Install Pinot Control Plane
7+
```
8+
make helm-install-pinot-control-plane
9+
```
10+
11+
#### Install Zookeeper Opoerator and CR
12+
```
13+
make helm-install-zk-operator
14+
```
15+
#### Install Pinot Cluster
16+
```
17+
envsubst < examples/05-pinot-aws/pinot-aws.yaml | kubectl apply -f - -n pinot
18+
```
19+
#### Deploy Kafka Cluster and Create Topics
20+
```
21+
# Add Kafka
22+
helm repo add kafka https://charts.bitnami.com/bitnami
23+
# Deploy kafka
24+
helm install -n pinot kafka kafka/kafka --set replicas=1,zookeeper.image.tag=latest
25+
# Create topics
26+
kubectl -n pinot exec kafka-0 -- kafka-topics.sh --bootstrap-server kafka-0:9092 --topic flights-realtime --create --partitions 1 --replication-factor 1
27+
kubectl -n pinot exec kafka-0 -- kafka-topics.sh --bootstrap-server kafka-0:9092 --topic flights-realtime-avro --create --partitions 1 --replication-factor 1
28+
```
29+
30+
#### Access Pinot Console
31+
32+
```
33+
kubectl port-forward svc/pinot-controller-controller-svc -n pinot 9000
34+
```
35+
36+
### Once Pinot Cluster is up and running
37+
38+
#### Create Schema
39+
```
40+
kubectl apply -f examples/05-pinot-aws/pinotschema-aws.yaml -n pinot
41+
```
42+
43+
#### Create Table
44+
```
45+
kubectl apply -f examples/05-pinot-aws/pinottable-aws.yaml -n pinot
46+
```
47+
48+
#### Check All Custom Resources created by the control plane
49+
```
50+
kubectl get pinot -A
51+
kubectl get pinotschema -A
52+
kubectl get pinottable -A
53+
```
54+
55+
#### Load Data Into Kafka
56+
```
57+
kubectl apply -f examples/03-pinot-minio/pinot-realtime-kafka.yaml
58+
```
59+
60+
#### Port-forward and query on console
61+
```
62+
kubectl port-forward pinot-controller-controller-0 -n pinot 9000
63+
```

‎examples/05-pinot-aws/pinotbasic-aws.yaml‎

Lines changed: 286 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,286 @@
1+
apiVersion: datainfra.io/v1beta1
2+
kind: Pinot
3+
metadata:
4+
name: pinot-aws
5+
spec:
6+
7+
plugins:
8+
- pinot-s3
9+
10+
external:
11+
12+
zookeeper:
13+
spec:
14+
zkAddress: zk-pinot-zookeeper-headless.pinot:2181
15+
16+
deepStorage:
17+
spec:
18+
- nodeType: controller
19+
data: |-
20+
controller.data.dir=s3://pinot-datainfra/pinot-realtime
21+
pinot.controller.storage.factory.class.s3=org.apache.pinot.plugin.filesystem.S3PinotFS
22+
pinot.controller.storage.factory.s3.region=us-east-1
23+
pinot.controller.storage.factory.s3.accessKey=
24+
pinot.controller.storage.factory.s3.secretKey=
25+
pinot.controller.segment.fetcher.protocols=file,http,s3
26+
pinot.controller.segment.fetcher.s3.class=org.apache.pinot.common.utils.fetcher.PinotFSSegmentFetcher
27+
- nodeType: server
28+
data: |-
29+
pinot.server.instance.enable.split.commit=true
30+
pinot.server.storage.factory.class.s3=org.apache.pinot.plugin.filesystem.S3PinotFS
31+
pinot.server.storage.factory.s3.region=us-west-2
32+
pinot.server.storage.factory.s3.httpclient.maxConnections=50
33+
pinot.server.storage.factory.s3.httpclient.socketTimeout=30s
34+
pinot.server.storage.factory.s3.httpclient.connectionTimeout=2s
35+
pinot.server.storage.factory.s3.httpclient.connectionTimeToLive=0s
36+
pinot.server.storage.factory.s3.httpclient.connectionAcquisitionTimeout=10s
37+
pinot.server.segment.fetcher.protocols=file,http,s3
38+
pinot.server.segment.fetcher.s3.class=org.apache.pinot.common.utils.fetcher.PinotFSSegmentFetcher
39+
- nodeType: minion
40+
data: |-
41+
pinot.minion.storage.factory.class.s3=org.apache.pinot.plugin.filesystem.S3PinotFS
42+
pinot.minion.storage.factory.s3.region=us-west-2
43+
pinot.minion.segment.fetcher.protocols=file,http,s3
44+
pinot.minion.segment.fetcher.s3.class=org.apache.pinot.common.utils.fetcher.PinotFSSegmentFetcher
45+
46+
nodes:
47+
48+
- name: pinot-controller
49+
kind: Statefulset
50+
replicas: 1
51+
nodeType: controller
52+
k8sConfig: controller
53+
pinotNodeConfig: controller
54+
55+
- name: pinot-broker
56+
kind: Statefulset
57+
replicas: 1
58+
nodeType: broker
59+
k8sConfig: broker
60+
pinotNodeConfig: broker
61+
62+
- name: pinot-server
63+
kind: Statefulset
64+
replicas: 1
65+
nodeType: server
66+
k8sConfig: server
67+
pinotNodeConfig: server
68+
69+
- name: pinot-minion
70+
kind: Statefulset
71+
replicas: 1
72+
nodeType: minion
73+
k8sConfig: minion
74+
pinotNodeConfig: minion
75+
76+
deploymentOrder:
77+
- controller
78+
- broker
79+
- server
80+
- minion
81+
82+
k8sConfig:
83+
84+
- name: controller
85+
serviceAccountName: "default"
86+
port:
87+
- name: controller
88+
containerPort: 9000
89+
protocol: TCP
90+
livenessProbe:
91+
initialDelaySeconds: 60
92+
periodSeconds: 10
93+
httpGet:
94+
path: "/health"
95+
port: 9000
96+
readinessProbe:
97+
initialDelaySeconds: 60
98+
periodSeconds: 10
99+
httpGet:
100+
path: "/health"
101+
port: 9000
102+
service:
103+
type: ClusterIP
104+
ports:
105+
- protocol: TCP
106+
port: 9000
107+
targetPort: 9000
108+
env:
109+
- name: LOG4J_CONSOLE_LEVEL
110+
value: debug
111+
image: apachepinot/pinot:latest-11-amazoncorretto-linux-amd64
112+
storageConfig:
113+
- name: pinotcontroller
114+
mountPath: "/var/pinot/controller/data"
115+
spec:
116+
accessModes:
117+
- ReadWriteOnce
118+
storageClassName: ${STORAGE_CLASS_NAME}
119+
resources:
120+
requests:
121+
storage: 10Gi
122+
123+
- name: broker
124+
serviceAccountName: "default"
125+
port:
126+
- name: broker
127+
containerPort: 8099
128+
protocol: TCP
129+
livenessProbe:
130+
initialDelaySeconds: 60
131+
periodSeconds: 10
132+
httpGet:
133+
path: "/health"
134+
port: 8099
135+
readinessProbe:
136+
initialDelaySeconds: 60
137+
periodSeconds: 10
138+
httpGet:
139+
path: "/health"
140+
port: 8099
141+
service:
142+
type: ClusterIP
143+
ports:
144+
- name: broker
145+
port: 8099
146+
protocol: TCP
147+
targetPort: 8099
148+
env:
149+
- name: LOG4J_CONSOLE_LEVEL
150+
value: info
151+
image: apachepinot/pinot:latest-11-amazoncorretto-linux-amd64
152+
storageConfig:
153+
- name: broker
154+
mountPath: "/var/pinot/broker/data"
155+
spec:
156+
accessModes:
157+
- ReadWriteOnce
158+
storageClassName: ${STORAGE_CLASS_NAME}
159+
resources:
160+
requests:
161+
storage: 10Gi
162+
163+
- name: server
164+
serviceAccountName: "default"
165+
port:
166+
- name: server
167+
containerPort: 8097
168+
protocol: TCP
169+
livenessProbe:
170+
initialDelaySeconds: 60
171+
periodSeconds: 10
172+
httpGet:
173+
path: "/health"
174+
port: 8097
175+
readinessProbe:
176+
initialDelaySeconds: 60
177+
periodSeconds: 10
178+
httpGet:
179+
path: "/health"
180+
port: 8097
181+
service:
182+
type: ClusterIP
183+
ports:
184+
- name: netty
185+
port: 8098
186+
protocol: TCP
187+
targetPort: 8098
188+
- protocol: TCP
189+
port: 80
190+
targetPort: 8097
191+
name: admin
192+
image: apachepinot/pinot:latest-11-amazoncorretto-linux-amd64
193+
env:
194+
- name: LOG4J_CONSOLE_LEVEL
195+
value: info
196+
storageConfig:
197+
- name: server
198+
mountPath: "/var/pinot/server/data"
199+
spec:
200+
accessModes:
201+
- ReadWriteOnce
202+
storageClassName: ${STORAGE_CLASS_NAME}
203+
resources:
204+
requests:
205+
storage: 10Gi
206+
207+
- name: minion
208+
serviceAccountName: "default"
209+
image: apachepinot/pinot:latest-11-amazoncorretto-linux-amd64
210+
livenessProbe:
211+
initialDelaySeconds: 60
212+
periodSeconds: 10
213+
httpGet:
214+
path: "/health"
215+
port: 9514
216+
readinessProbe:
217+
initialDelaySeconds: 60
218+
periodSeconds: 10
219+
httpGet:
220+
path: "/health"
221+
port: 9514
222+
service:
223+
type: ClusterIP
224+
ports:
225+
- name: minion
226+
port: 9514
227+
protocol: TCP
228+
targetPort: 9514
229+
port:
230+
- name: minion
231+
containerPort: 9514
232+
protocol: TCP
233+
env:
234+
- name: LOG4J_CONSOLE_LEVEL
235+
value: info
236+
storageConfig:
237+
- name: minion
238+
mountPath: "/var/pinot/minion/data"
239+
spec:
240+
accessModes:
241+
- ReadWriteOnce
242+
storageClassName: ${STORAGE_CLASS_NAME}
243+
resources:
244+
requests:
245+
storage: 10Gi
246+
247+
pinotNodeConfig:
248+
249+
- name: controller
250+
java_opts: "-XX:ActiveProcessorCount=2 -Xms256M -Xmx1G -XX:+UseG1GC -XX:MaxGCPauseMillis=200
251+
-Xlog:gc*:file=/opt/pinot/gc-pinot-controller.log -Dlog4j2.configurationFile=/opt/pinot/conf/log4j2.xml
252+
-Dplugins.dir=/opt/pinot/plugins"
253+
data: |-
254+
controller.port=9000
255+
controller.local.temp.dir=/var/pinot/controller/data
256+
pinot.set.instance.id.to.hostname=true
257+
controller.task.scheduler.enabled=true
258+
259+
- name: broker
260+
java_opts: "-XX:ActiveProcessorCount=2 -Xms256M -Xmx1G -XX:+UseG1GC -XX:MaxGCPauseMillis=200
261+
-Xlog:gc*:file=/opt/pinot/gc-pinot-broker.log -Dlog4j2.configurationFile=/opt/pinot/conf/log4j2.xml
262+
-Dplugins.dir=/opt/pinot/plugins"
263+
data: |-
264+
pinot.broker.client.queryPort=8099
265+
pinot.broker.routing.table.builder.class=random
266+
pinot.set.instance.id.to.hostname=true
267+
268+
- name: server
269+
java_opts: "-Xms512M -Xmx1G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xlog:gc*:file=/opt/pinot/gc-pinot-server.log
270+
-Dlog4j2.configurationFile=/opt/pinot/conf/log4j2.xml -Dplugins.dir=/opt/pinot/plugins"
271+
data: |-
272+
pinot.server.netty.port=8098
273+
pinot.server.adminapi.port=8097
274+
pinot.server.instance.dataDir=/var/pinot/server/data/index
275+
pinot.server.instance.segmentTarDir=/var/pinot/server/data/segment
276+
pinot.set.instance.id.to.hostname=true
277+
pinot.server.instance.realtime.alloc.offheap=true
278+
279+
- name: minion
280+
java_opts: "-XX:ActiveProcessorCount=2 -Xms256M -Xmx1G -XX:+UseG1GC -XX:MaxGCPauseMillis=200
281+
-Xlog:gc*:file=/opt/pinot/gc-pinot-minion.log -Dlog4j2.configurationFile=/opt/pinot/conf/log4j2.xml
282+
-Dplugins.dir=/opt/pinot/plugins"
283+
data: |-
284+
pinot.minion.port=9514
285+
dataDir=/var/pinot/minion/data
286+
pinot.set.instance.id.to.hostname=true

0 commit comments

Comments
 (0)