From 039b4370ce3b3a208656e2293acc31a4d28fd025 Mon Sep 17 00:00:00 2001 From: Monica Pardeshi Date: Thu, 13 Jul 2017 19:42:03 -0500 Subject: [PATCH 01/19] Added dockerfile and updated README --- CHANGELOG.md | 1 + fili-generic-example/Dockerfile | 37 +++++++++++++++++++++++++ fili-generic-example/README.md | 48 +++++++++++++++++++++++---------- 3 files changed, 72 insertions(+), 14 deletions(-) create mode 100644 fili-generic-example/Dockerfile diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cc3907e04..9b1199c4db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ Current ------- ### Added: +- [Added dockerfile and updated README](https://github.com/yahoo/fili/pull/429) - [Add Uptime Status Metric](https://github.com/yahoo/fili/pull/518) * Add a metric to show how long Fili has been running diff --git a/fili-generic-example/Dockerfile b/fili-generic-example/Dockerfile new file mode 100644 index 0000000000..094bb9af70 --- /dev/null +++ b/fili-generic-example/Dockerfile @@ -0,0 +1,37 @@ +FROM druidio/example-cluster:latest +LABEL maintainer="https://groups.google.com/forum/#!forum/fili-developers" + +# fili github information +ENV GITHUB_OWNER=yahoo BRANCH=master + +RUN apt-get update && apt-get install -y --no-install-recommends git + +RUN wget -q -O - http://archive.apache.org/dist/maven/maven-3/3.2.5/binaries/apache-maven-3.2.5-bin.tar.gz | tar -xzf - -C /usr/local \ + && ln -s /usr/local/apache-maven-3.2.5 /usr/local/apache-maven \ + && ln -s -f /usr/local/apache-maven/bin/mvn /usr/local/bin/mvn + +# fili code goes here +RUN mkdir -p /usr/local/fili + +# update in case of new commits +ADD https://api.github.com/repos/$GITHUB_OWNER/fili/git/refs/heads/$BRANCH \ +fili-branch.json +RUN git clone -q --branch "$BRANCH" --depth 1 \ +https://github.com/$GITHUB_OWNER/fili.git /usr/local/fili + +WORKDIR /usr/local/fili + +# install Fili in the container +RUN mvn -U -B install -DskipTests=true -Dmaven.javadoc.skip=true -Dcheckstyle.skip + +EXPOSE 9998 + +RUN apt-get purge --auto-remove -y git + +ENTRYPOINT export HOSTIP="$(resolveip -s $HOSTNAME)" \ +&& exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf \ + & mvn -pl fili-generic-example exec:java -Dbard__fili_port=9998 \ + -Dbard__druid_coord=http://localhost:8081/druid/coordinator/v1 \ + -Dbard__non_ui_druid_broker=http://localhost:8082/druid/v2 \ + -Dbard__ui_druid_broker=http://localhost:8082/druid/v2 + diff --git a/fili-generic-example/README.md b/fili-generic-example/README.md index c414199792..fbf87104af 100644 --- a/fili-generic-example/README.md +++ b/fili-generic-example/README.md @@ -1,7 +1,7 @@ Fili Generic Loader Application ================================== -This application will automatically configure fili to work with **any** instance +This application will automatically configure Fili to work with **any** instance of Druid and show the basic metrics and dimensions. This lets you test what it's like using Fili without putting any effort into setting it up. @@ -13,6 +13,15 @@ Note that this was last tested using [version 0.9.1](https://github.com/yahoo/fi ## Setup and Launching +### Docker +There is a Fili Dockerfile which is available on Dockerhub. To start playing around with Fili, you can download and open Docker. Then +run the command +```bash +docker run --name fili-generic-example -i --rm -p 3001:8081 -p 3000:8082 fili:1.0 +``` +Once you run this command, please wait a few minutes for Druid and Fili to spin up. + +### Manual 1. Have a [Druid](http://druid.io/docs/latest/tutorials/quickstart.html) cluster running on your Unix based machine. 2. Clone this repository to your computer. @@ -58,26 +67,37 @@ Here are some sample queries that you can run to verify your server: ### Specific to Wikipedia data -- If everything is working, the [query below](http://localhost:9998/v1/data/wikiticker/day/?metrics=deleted&dateTime=2015-09-12/PT24H) +- If everything is working, the [query below](http://localhost:9998/v1/data/wikipedia/day/?metrics=deleted&dateTime=2015-09-12/PT24H) ```bash - curl "http://localhost:9998/v1/data/wikiticker/day/?metrics=deleted&dateTime=2015-09-12/PT24H" -H "Content-Type: application/json" | python -m json.tool + curl "http://localhost:9998/v1/data/wikipedia/day/?metrics=deleted&dateTime=2013-08-01/PT24H" -H "Content-Type: application/json" | python -m json.tool ``` should show something like: ``` { "rows": [{ - "dateTime": "2015-09-12 00:00:00.000", - "deleted": 394298.0 + "dateTime": "2013-08-01 00:00:00.000", + "deleted": -39917308 }] } ``` -- Count of edits by hour for the last 72 hours: - - GET http://localhost:9998/v1/data/wikiticker/day/?metrics=count&dateTime=PT72H/current - - Note: this will should be something like the response below since the - wikiticker table doesn't have data for the past 72 hours from now. +If you see this message while running Docker it means that Druid isn't ready yet: + + ```json + { + "rows": [], + "meta": { + "missingIntervals": ["2013-08-01 00:00:00.000/2013-08-02 00:00:00.000"] + } + } + ``` + +- Count of edits by hour for the last 72 hours: + + GET http://localhost:9998/v1/data/wikipedia/day/?metrics=count&dateTime=PT72H/current + + Note: this will should be something like the response below since the + wikipedia table doesn't have data for the past 72 hours from now. ```json { "rows": [], @@ -87,14 +107,14 @@ Here are some sample queries that you can run to verify your server: } ``` -- Show [debug info](http://localhost:9998/v1/data/wikiticker/day/?format=debug&metrics=count&dateTime=PT72H/current), +- Show [debug info](http://localhost:9998/v1/data/wikipedia/day/?format=debug&metrics=count&dateTime=PT72H/current), including the query sent to Druid: - GET http://localhost:9998/v1/data/wikiticker/day/?format=debug&metrics=count&dateTime=PT72H/current + GET http://localhost:9998/v1/data/wikipedia/day/?format=debug&metrics=count&dateTime=PT72H/current ## Notable Restrictions -- Using this is great for testing out fili and druid, but it can't do interesting things with metrics. +- Using this is great for testing out Fili and druid, but it can't do interesting things with metrics. - This can only use 1 timegrain even though a datasource in druid *could* have more. ## Importing and Running in IntelliJ From 530e78476fce6db37b2557210a46013016c30945 Mon Sep 17 00:00:00 2001 From: Monica Pardeshi Date: Thu, 13 Jul 2017 19:42:03 -0500 Subject: [PATCH 02/19] Added dockerfile and updated README --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b1199c4db..cc71ba715b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,8 @@ Current ------- ### Added: -- [Added dockerfile and updated README](https://github.com/yahoo/fili/pull/429) +- [Add dockerfile and update README](https://github.com/yahoo/fili/pull/429) + * Make a Dockerfile for the Wikipedia example - [Add Uptime Status Metric](https://github.com/yahoo/fili/pull/518) * Add a metric to show how long Fili has been running From 277f8d02114f7f460d58836e82b7db265738658a Mon Sep 17 00:00:00 2001 From: Monica Pardeshi Date: Mon, 17 Jul 2017 18:06:06 -0500 Subject: [PATCH 03/19] changed BRANCH --- fili-generic-example/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fili-generic-example/Dockerfile b/fili-generic-example/Dockerfile index 094bb9af70..81b1eace56 100644 --- a/fili-generic-example/Dockerfile +++ b/fili-generic-example/Dockerfile @@ -16,7 +16,7 @@ RUN mkdir -p /usr/local/fili # update in case of new commits ADD https://api.github.com/repos/$GITHUB_OWNER/fili/git/refs/heads/$BRANCH \ fili-branch.json -RUN git clone -q --branch "$BRANCH" --depth 1 \ +RUN git clone -q --branch ${BRANCH} --depth 1 \ https://github.com/$GITHUB_OWNER/fili.git /usr/local/fili WORKDIR /usr/local/fili From c36a249f59fc436c26d18e5b40aa524e23ee54bc Mon Sep 17 00:00:00 2001 From: Monica Pardeshi Date: Mon, 17 Jul 2017 18:24:12 -0500 Subject: [PATCH 04/19] moved Dockerfile and removed Docker instruction from README --- fili-generic-example/README.md | 34 ++++--------------- .../Dockerfile | 0 2 files changed, 7 insertions(+), 27 deletions(-) rename {fili-generic-example => fili-wikipedia-example}/Dockerfile (100%) diff --git a/fili-generic-example/README.md b/fili-generic-example/README.md index fbf87104af..5f76ce3508 100644 --- a/fili-generic-example/README.md +++ b/fili-generic-example/README.md @@ -13,15 +13,6 @@ Note that this was last tested using [version 0.9.1](https://github.com/yahoo/fi ## Setup and Launching -### Docker -There is a Fili Dockerfile which is available on Dockerhub. To start playing around with Fili, you can download and open Docker. Then -run the command -```bash -docker run --name fili-generic-example -i --rm -p 3001:8081 -p 3000:8082 fili:1.0 -``` -Once you run this command, please wait a few minutes for Druid and Fili to spin up. - -### Manual 1. Have a [Druid](http://druid.io/docs/latest/tutorials/quickstart.html) cluster running on your Unix based machine. 2. Clone this repository to your computer. @@ -67,34 +58,23 @@ Here are some sample queries that you can run to verify your server: ### Specific to Wikipedia data -- If everything is working, the [query below](http://localhost:9998/v1/data/wikipedia/day/?metrics=deleted&dateTime=2015-09-12/PT24H) +- If everything is working, the [query below](http://localhost:9998/v1/data/wikiticker/day/?metrics=deleted&dateTime=2015-09-12/PT24H) ```bash - curl "http://localhost:9998/v1/data/wikipedia/day/?metrics=deleted&dateTime=2013-08-01/PT24H" -H "Content-Type: application/json" | python -m json.tool + curl "http://localhost:9998/v1/data/wikiticker/day/?metrics=deleted&dateTime=2015-09-12/PT24H" -H "Content-Type: application/json" | python -m json.tool ``` should show something like: ``` { "rows": [{ - "dateTime": "2013-08-01 00:00:00.000", - "deleted": -39917308 + "dateTime": "2015-09-12 00:00:00.000", + "deleted": 394298.0 }] } ``` -If you see this message while running Docker it means that Druid isn't ready yet: - - ```json - { - "rows": [], - "meta": { - "missingIntervals": ["2013-08-01 00:00:00.000/2013-08-02 00:00:00.000"] - } - } - ``` - - Count of edits by hour for the last 72 hours: - GET http://localhost:9998/v1/data/wikipedia/day/?metrics=count&dateTime=PT72H/current + GET http://localhost:9998/v1/data/wikiticker/day/?metrics=count&dateTime=PT72H/current Note: this will should be something like the response below since the wikipedia table doesn't have data for the past 72 hours from now. @@ -107,10 +87,10 @@ If you see this message while running Docker it means that Druid isn't ready yet } ``` -- Show [debug info](http://localhost:9998/v1/data/wikipedia/day/?format=debug&metrics=count&dateTime=PT72H/current), +- Show [debug info](http://localhost:9998/v1/data/wikiticker/day/?format=debug&metrics=count&dateTime=PT72H/current), including the query sent to Druid: - GET http://localhost:9998/v1/data/wikipedia/day/?format=debug&metrics=count&dateTime=PT72H/current + GET http://localhost:9998/v1/data/wikiticker/day/?format=debug&metrics=count&dateTime=PT72H/current ## Notable Restrictions diff --git a/fili-generic-example/Dockerfile b/fili-wikipedia-example/Dockerfile similarity index 100% rename from fili-generic-example/Dockerfile rename to fili-wikipedia-example/Dockerfile From 23dc17115203c730963599f5fab72e1cc7a0057b Mon Sep 17 00:00:00 2001 From: Monica Pardeshi Date: Mon, 17 Jul 2017 21:40:41 -0500 Subject: [PATCH 05/19] Added Docker instructions --- fili-wikipedia-example/README.md | 50 ++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/fili-wikipedia-example/README.md b/fili-wikipedia-example/README.md index 22c25e3681..0a3124311d 100644 --- a/fili-wikipedia-example/README.md +++ b/fili-wikipedia-example/README.md @@ -65,6 +65,56 @@ Here are some sample queries that you can run to verify your server: GET http://localhost:9998/v1/data/wikipedia/hour/?format=debug&metrics=count&dateTime=PT72H/current +## Running Fili with Docker + +There is a [Docker image](https://hub.docker.com/r/mpardesh/fili/) for Fili which can be found on +Dockerhub. If you would like to experiment with Fili without having to download its dependencies, you can +[download](https://www.docker.com/) and start Docker. Then run these commands: + ``` + docker pull mpardesh/fili + ``` + + ``` + docker run --name fili-generic-example -i --rm -p 3001:8081 -p 3000:8082 fili:1.0 + ``` + +This will start a container. Please wait a few minutes for Druid to ingest the data. + +Once Druid is ready, you can start querying! Here is a +[sample query](http://localhost:9998/v1/data/wikipedia/day/?metrics=deleted&dateTime=2013-08-01/PT24H) +to get started: + ``` + http://localhost:9998/v1/data/wikipedia/day/?metrics=deleted&dateTime=2013-08-01/PT24H + ``` + +If Druid isn't ready yet, you will see this message: + ``` + { + "rows": [], + "meta": { + "missingIntervals": ["2013-08-01 00:00:00.000/2013-08-02 00:00:00.000"] + } + } + ``` + +If the query is successful, you should see this: + ``` + { + "rows": [{ + "dateTime": "2013-08-01 00:00:00.000", + "deleted": -39917308 + }] + } + ``` + +To stop the container, run + ``` + docker stop + ``` +in a different terminal tab. + +Note: the data used with Docker is from a different day than the data used with Druid quickstart. + ## Importing and Running in IntelliJ 1. In IntelliJ, go to `File -> Open` From 4b608fd63b80e1bbb99c44392bb8d0dc106172a1 Mon Sep 17 00:00:00 2001 From: Monica Pardeshi Date: Tue, 18 Jul 2017 12:33:56 -0500 Subject: [PATCH 06/19] updated commands and links --- fili-wikipedia-example/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fili-wikipedia-example/README.md b/fili-wikipedia-example/README.md index 0a3124311d..c5a3e0acfa 100644 --- a/fili-wikipedia-example/README.md +++ b/fili-wikipedia-example/README.md @@ -69,16 +69,16 @@ Here are some sample queries that you can run to verify your server: There is a [Docker image](https://hub.docker.com/r/mpardesh/fili/) for Fili which can be found on Dockerhub. If you would like to experiment with Fili without having to download its dependencies, you can -[download](https://www.docker.com/) and start Docker. Then run these commands: +[install](https://www.docker.com/community-edition) and start Docker. Then run these commands: ``` docker pull mpardesh/fili ``` ``` - docker run --name fili-generic-example -i --rm -p 3001:8081 -p 3000:8082 fili:1.0 + docker run --name fili-generic-example -i --rm -p 3001:8081 -p 3000:8082 mpardesh/fili:1.0 ``` -This will start a container. Please wait a few minutes for Druid to ingest the data. +This will start a container. Please wait a few minutes for Druid to get ready. Once Druid is ready, you can start querying! Here is a [sample query](http://localhost:9998/v1/data/wikipedia/day/?metrics=deleted&dateTime=2013-08-01/PT24H) From aff7b9ba5dae8c9e6653d160b632ef34cc3f89e2 Mon Sep 17 00:00:00 2001 From: Monica Pardeshi Date: Tue, 18 Jul 2017 13:11:48 -0500 Subject: [PATCH 07/19] kept git in Dockerfile --- fili-wikipedia-example/Dockerfile | 4 +--- fili-wikipedia-example/README.md | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/fili-wikipedia-example/Dockerfile b/fili-wikipedia-example/Dockerfile index 81b1eace56..84b5fe7805 100644 --- a/fili-wikipedia-example/Dockerfile +++ b/fili-wikipedia-example/Dockerfile @@ -26,11 +26,9 @@ RUN mvn -U -B install -DskipTests=true -Dmaven.javadoc.skip=true -Dcheckstyle.sk EXPOSE 9998 -RUN apt-get purge --auto-remove -y git - ENTRYPOINT export HOSTIP="$(resolveip -s $HOSTNAME)" \ && exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf \ - & mvn -pl fili-generic-example exec:java -Dbard__fili_port=9998 \ + & mvn -pl fili-wikipedia-example exec:java -Dbard__fili_port=9998 \ -Dbard__druid_coord=http://localhost:8081/druid/coordinator/v1 \ -Dbard__non_ui_druid_broker=http://localhost:8082/druid/v2 \ -Dbard__ui_druid_broker=http://localhost:8082/druid/v2 diff --git a/fili-wikipedia-example/README.md b/fili-wikipedia-example/README.md index c5a3e0acfa..312a2322d3 100644 --- a/fili-wikipedia-example/README.md +++ b/fili-wikipedia-example/README.md @@ -75,7 +75,7 @@ Dockerhub. If you would like to experiment with Fili without having to download ``` ``` - docker run --name fili-generic-example -i --rm -p 3001:8081 -p 3000:8082 mpardesh/fili:1.0 + docker run --name fili-wikipedia-example -i --rm -p 3001:8081 -p 3000:8082 mpardesh/fili:1.0 ``` This will start a container. Please wait a few minutes for Druid to get ready. From 1e355102f4ca36b345e9f5ee9288f982b3f8caa7 Mon Sep 17 00:00:00 2001 From: Monica Pardeshi Date: Tue, 18 Jul 2017 14:00:32 -0500 Subject: [PATCH 08/19] kept git --- fili-wikipedia-example/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fili-wikipedia-example/Dockerfile b/fili-wikipedia-example/Dockerfile index 84b5fe7805..a22f83a8ca 100644 --- a/fili-wikipedia-example/Dockerfile +++ b/fili-wikipedia-example/Dockerfile @@ -28,7 +28,7 @@ EXPOSE 9998 ENTRYPOINT export HOSTIP="$(resolveip -s $HOSTNAME)" \ && exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf \ - & mvn -pl fili-wikipedia-example exec:java -Dbard__fili_port=9998 \ + & mvn -pl fili-generic-example exec:java -Dbard__fili_port=9998 \ -Dbard__druid_coord=http://localhost:8081/druid/coordinator/v1 \ -Dbard__non_ui_druid_broker=http://localhost:8082/druid/v2 \ -Dbard__ui_druid_broker=http://localhost:8082/druid/v2 From f883950229134fe62b592c8d2d11d34827b10eee Mon Sep 17 00:00:00 2001 From: Monica Pardeshi Date: Tue, 18 Jul 2017 15:10:26 -0500 Subject: [PATCH 09/19] fixed formatting --- fili-wikipedia-example/README.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/fili-wikipedia-example/README.md b/fili-wikipedia-example/README.md index 312a2322d3..e73c6ef67a 100644 --- a/fili-wikipedia-example/README.md +++ b/fili-wikipedia-example/README.md @@ -70,11 +70,9 @@ Here are some sample queries that you can run to verify your server: There is a [Docker image](https://hub.docker.com/r/mpardesh/fili/) for Fili which can be found on Dockerhub. If you would like to experiment with Fili without having to download its dependencies, you can [install](https://www.docker.com/community-edition) and start Docker. Then run these commands: - ``` - docker pull mpardesh/fili - ``` - ``` + ``` bash + docker pull mpardesh/fili docker run --name fili-wikipedia-example -i --rm -p 3001:8081 -p 3000:8082 mpardesh/fili:1.0 ``` @@ -83,11 +81,13 @@ This will start a container. Please wait a few minutes for Druid to get ready. Once Druid is ready, you can start querying! Here is a [sample query](http://localhost:9998/v1/data/wikipedia/day/?metrics=deleted&dateTime=2013-08-01/PT24H) to get started: + ``` http://localhost:9998/v1/data/wikipedia/day/?metrics=deleted&dateTime=2013-08-01/PT24H ``` If Druid isn't ready yet, you will see this message: + ``` { "rows": [], @@ -98,6 +98,7 @@ If Druid isn't ready yet, you will see this message: ``` If the query is successful, you should see this: + ``` { "rows": [{ @@ -108,9 +109,11 @@ If the query is successful, you should see this: ``` To stop the container, run - ``` + + ``` bash docker stop ``` + in a different terminal tab. Note: the data used with Docker is from a different day than the data used with Druid quickstart. From c6c4a1fb9766438fbbfe26ef4a53a9cf6f0147a0 Mon Sep 17 00:00:00 2001 From: Monica Pardeshi Date: Tue, 18 Jul 2017 15:51:15 -0500 Subject: [PATCH 10/19] more formatting changes --- fili-wikipedia-example/README.md | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/fili-wikipedia-example/README.md b/fili-wikipedia-example/README.md index e73c6ef67a..d5870936f6 100644 --- a/fili-wikipedia-example/README.md +++ b/fili-wikipedia-example/README.md @@ -71,10 +71,10 @@ There is a [Docker image](https://hub.docker.com/r/mpardesh/fili/) for Fili whic Dockerhub. If you would like to experiment with Fili without having to download its dependencies, you can [install](https://www.docker.com/community-edition) and start Docker. Then run these commands: - ``` bash + docker pull mpardesh/fili docker run --name fili-wikipedia-example -i --rm -p 3001:8081 -p 3000:8082 mpardesh/fili:1.0 - ``` + This will start a container. Please wait a few minutes for Druid to get ready. @@ -82,38 +82,30 @@ Once Druid is ready, you can start querying! Here is a [sample query](http://localhost:9998/v1/data/wikipedia/day/?metrics=deleted&dateTime=2013-08-01/PT24H) to get started: - ``` http://localhost:9998/v1/data/wikipedia/day/?metrics=deleted&dateTime=2013-08-01/PT24H - ``` - + If Druid isn't ready yet, you will see this message: - - ``` + { "rows": [], "meta": { "missingIntervals": ["2013-08-01 00:00:00.000/2013-08-02 00:00:00.000"] } } - ``` - + If the query is successful, you should see this: - ``` { "rows": [{ "dateTime": "2013-08-01 00:00:00.000", "deleted": -39917308 }] } - ``` - + To stop the container, run - ``` bash docker stop - ``` - + in a different terminal tab. Note: the data used with Docker is from a different day than the data used with Druid quickstart. From 17e5000cb77f37c841533ba96364d6bb28f98e51 Mon Sep 17 00:00:00 2001 From: Monica Pardeshi Date: Tue, 18 Jul 2017 21:25:23 -0500 Subject: [PATCH 11/19] updated docker run and stop commands --- fili-wikipedia-example/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fili-wikipedia-example/README.md b/fili-wikipedia-example/README.md index d5870936f6..8db20bc055 100644 --- a/fili-wikipedia-example/README.md +++ b/fili-wikipedia-example/README.md @@ -72,8 +72,8 @@ Dockerhub. If you would like to experiment with Fili without having to download [install](https://www.docker.com/community-edition) and start Docker. Then run these commands: - docker pull mpardesh/fili - docker run --name fili-wikipedia-example -i --rm -p 3001:8081 -p 3000:8082 mpardesh/fili:1.0 + docker pull mpardesh/fili:1.0 + docker run --name fili-wikipedia-example -i --rm -p 3001:8081 -p 3000:8082 -p 9998:9998 mpardesh/fili:1.0 This will start a container. Please wait a few minutes for Druid to get ready. @@ -104,7 +104,7 @@ If the query is successful, you should see this: To stop the container, run - docker stop + docker stop fili-wikipedia-example in a different terminal tab. From b8412f858ea0a40c7e038342bed6a8dfcd31bfb1 Mon Sep 17 00:00:00 2001 From: Monica Pardeshi Date: Tue, 18 Jul 2017 22:40:49 -0500 Subject: [PATCH 12/19] edited entrypoint in Dockerfile and moved those commands to entrypoint.sh --- fili-wikipedia-example/Dockerfile | 9 +++------ fili-wikipedia-example/entrypoint.sh | 13 +++++++++++++ 2 files changed, 16 insertions(+), 6 deletions(-) create mode 100644 fili-wikipedia-example/entrypoint.sh diff --git a/fili-wikipedia-example/Dockerfile b/fili-wikipedia-example/Dockerfile index a22f83a8ca..85ea042f3f 100644 --- a/fili-wikipedia-example/Dockerfile +++ b/fili-wikipedia-example/Dockerfile @@ -26,10 +26,7 @@ RUN mvn -U -B install -DskipTests=true -Dmaven.javadoc.skip=true -Dcheckstyle.sk EXPOSE 9998 -ENTRYPOINT export HOSTIP="$(resolveip -s $HOSTNAME)" \ -&& exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf \ - & mvn -pl fili-generic-example exec:java -Dbard__fili_port=9998 \ - -Dbard__druid_coord=http://localhost:8081/druid/coordinator/v1 \ - -Dbard__non_ui_druid_broker=http://localhost:8082/druid/v2 \ - -Dbard__ui_druid_broker=http://localhost:8082/druid/v2 +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] diff --git a/fili-wikipedia-example/entrypoint.sh b/fili-wikipedia-example/entrypoint.sh new file mode 100644 index 0000000000..6847f94591 --- /dev/null +++ b/fili-wikipedia-example/entrypoint.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +export HOSTIP="$(resolveip -s $HOSTNAME)" +/usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf +echo "Waiting for Druid to finish setting up" +while ! curl http://localhost:3001/druid/coordinator/v1/datasources | grep -q "wikipedia"; do + sleep 5 +done +echo "Druid finished setting up. Starting Fili" +mvn -pl fili-generic-example exec:java -Dbard__fili_port=9998 + -Dbard__druid_coord=http://localhost:8081/druid/coordinator/v1 + -Dbard__non_ui_druid_broker=http://localhost:8082/druid/v2 + -Dbard__ui_druid_broker=http://localhost:8082/druid/v2 From 816284968d93c029cb8a24c4d90eea2e9a97ca07 Mon Sep 17 00:00:00 2001 From: Monica Pardeshi Date: Wed, 19 Jul 2017 10:08:13 -0500 Subject: [PATCH 13/19] updated path in entrypoint --- fili-wikipedia-example/Dockerfile | 5 ++--- fili-wikipedia-example/entrypoint.sh | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/fili-wikipedia-example/Dockerfile b/fili-wikipedia-example/Dockerfile index 85ea042f3f..7a0e6b5286 100644 --- a/fili-wikipedia-example/Dockerfile +++ b/fili-wikipedia-example/Dockerfile @@ -26,7 +26,6 @@ RUN mvn -U -B install -DskipTests=true -Dmaven.javadoc.skip=true -Dcheckstyle.sk EXPOSE 9998 -COPY entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh -ENTRYPOINT ["/entrypoint.sh"] +RUN chmod +x /usr/local/fili/fili-wikipedia-example/entrypoint.sh +ENTRYPOINT ["/usr/local/fili/fili-wikipedia-example/entrypoint.sh"] diff --git a/fili-wikipedia-example/entrypoint.sh b/fili-wikipedia-example/entrypoint.sh index 6847f94591..64704c2404 100644 --- a/fili-wikipedia-example/entrypoint.sh +++ b/fili-wikipedia-example/entrypoint.sh @@ -3,7 +3,7 @@ export HOSTIP="$(resolveip -s $HOSTNAME)" /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf echo "Waiting for Druid to finish setting up" -while ! curl http://localhost:3001/druid/coordinator/v1/datasources | grep -q "wikipedia"; do +while ! curl http://localhost:8081/druid/coordinator/v1/datasources | grep -q "wikipedia"; do sleep 5 done echo "Druid finished setting up. Starting Fili" From c99fe36cdd94cad636e427151032f804fff53356 Mon Sep 17 00:00:00 2001 From: Monica Pardeshi Date: Wed, 19 Jul 2017 13:46:11 -0500 Subject: [PATCH 14/19] added & after starting Druid --- fili-wikipedia-example/Dockerfile | 3 +-- fili-wikipedia-example/entrypoint.sh | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/fili-wikipedia-example/Dockerfile b/fili-wikipedia-example/Dockerfile index 7a0e6b5286..6d2c251953 100644 --- a/fili-wikipedia-example/Dockerfile +++ b/fili-wikipedia-example/Dockerfile @@ -2,7 +2,7 @@ FROM druidio/example-cluster:latest LABEL maintainer="https://groups.google.com/forum/#!forum/fili-developers" # fili github information -ENV GITHUB_OWNER=yahoo BRANCH=master +ENV GITHUB_OWNER=yahoo BRANCH=fili RUN apt-get update && apt-get install -y --no-install-recommends git @@ -28,4 +28,3 @@ EXPOSE 9998 RUN chmod +x /usr/local/fili/fili-wikipedia-example/entrypoint.sh ENTRYPOINT ["/usr/local/fili/fili-wikipedia-example/entrypoint.sh"] - diff --git a/fili-wikipedia-example/entrypoint.sh b/fili-wikipedia-example/entrypoint.sh index 64704c2404..75ef1f31f2 100644 --- a/fili-wikipedia-example/entrypoint.sh +++ b/fili-wikipedia-example/entrypoint.sh @@ -1,7 +1,7 @@ #!/bin/sh export HOSTIP="$(resolveip -s $HOSTNAME)" -/usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf +/usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf & echo "Waiting for Druid to finish setting up" while ! curl http://localhost:8081/druid/coordinator/v1/datasources | grep -q "wikipedia"; do sleep 5 From 14add72e19bae0ab20db753b1ffad571e690edd1 Mon Sep 17 00:00:00 2001 From: Monica Pardeshi Date: Wed, 19 Jul 2017 14:01:30 -0500 Subject: [PATCH 15/19] changed branch to master --- fili-wikipedia-example/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fili-wikipedia-example/Dockerfile b/fili-wikipedia-example/Dockerfile index 6d2c251953..d2e1bbb785 100644 --- a/fili-wikipedia-example/Dockerfile +++ b/fili-wikipedia-example/Dockerfile @@ -2,7 +2,7 @@ FROM druidio/example-cluster:latest LABEL maintainer="https://groups.google.com/forum/#!forum/fili-developers" # fili github information -ENV GITHUB_OWNER=yahoo BRANCH=fili +ENV GITHUB_OWNER=yahoo BRANCH=master RUN apt-get update && apt-get install -y --no-install-recommends git From aec1c1c11e1f0eb77d89a558ea51ed328f731578 Mon Sep 17 00:00:00 2001 From: Monica Pardeshi Date: Wed, 19 Jul 2017 15:04:30 -0500 Subject: [PATCH 16/19] added command to install curl --- fili-wikipedia-example/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fili-wikipedia-example/Dockerfile b/fili-wikipedia-example/Dockerfile index d2e1bbb785..12e6c99eec 100644 --- a/fili-wikipedia-example/Dockerfile +++ b/fili-wikipedia-example/Dockerfile @@ -4,7 +4,7 @@ LABEL maintainer="https://groups.google.com/forum/#!forum/fili-developers" # fili github information ENV GITHUB_OWNER=yahoo BRANCH=master -RUN apt-get update && apt-get install -y --no-install-recommends git +RUN apt-get update && apt-get install -y --no-install-recommends git curl RUN wget -q -O - http://archive.apache.org/dist/maven/maven-3/3.2.5/binaries/apache-maven-3.2.5-bin.tar.gz | tar -xzf - -C /usr/local \ && ln -s /usr/local/apache-maven-3.2.5 /usr/local/apache-maven \ From ec2b1ba49c38fb1706eefd418f3f6c7db3b4035a Mon Sep 17 00:00:00 2001 From: Monica Pardeshi Date: Fri, 21 Jul 2017 11:43:45 -0500 Subject: [PATCH 17/19] made sure that Docker doesn't run forever if Druid fails to start --- fili-wikipedia-example/entrypoint.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/fili-wikipedia-example/entrypoint.sh b/fili-wikipedia-example/entrypoint.sh index 75ef1f31f2..59f82fc9db 100644 --- a/fili-wikipedia-example/entrypoint.sh +++ b/fili-wikipedia-example/entrypoint.sh @@ -3,9 +3,19 @@ export HOSTIP="$(resolveip -s $HOSTNAME)" /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf & echo "Waiting for Druid to finish setting up" + +time_left=300 + while ! curl http://localhost:8081/druid/coordinator/v1/datasources | grep -q "wikipedia"; do - sleep 5 + if [ "$time_left" -le 0 ] + then + echo "Druid is having trouble setting up" + exit + fi + sleep 5 + time_left=$(( time_left - 5 )) done + echo "Druid finished setting up. Starting Fili" mvn -pl fili-generic-example exec:java -Dbard__fili_port=9998 -Dbard__druid_coord=http://localhost:8081/druid/coordinator/v1 From d42f55df2c749892e0a6e3fb03fbfc4371da3445 Mon Sep 17 00:00:00 2001 From: Monica Pardeshi Date: Tue, 25 Jul 2017 14:46:44 -0500 Subject: [PATCH 18/19] fixed formatting --- CHANGELOG.md | 1 + fili-generic-example/README.md | 2 +- fili-wikipedia-example/Dockerfile | 6 ++---- fili-wikipedia-example/README.md | 2 -- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc71ba715b..798e003e7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ Current ------- ### Added: + - [Add dockerfile and update README](https://github.com/yahoo/fili/pull/429) * Make a Dockerfile for the Wikipedia example diff --git a/fili-generic-example/README.md b/fili-generic-example/README.md index 5f76ce3508..8b77e0b813 100644 --- a/fili-generic-example/README.md +++ b/fili-generic-example/README.md @@ -77,7 +77,7 @@ Here are some sample queries that you can run to verify your server: GET http://localhost:9998/v1/data/wikiticker/day/?metrics=count&dateTime=PT72H/current Note: this will should be something like the response below since the - wikipedia table doesn't have data for the past 72 hours from now. + wikiticker table doesn't have data for the past 72 hours from now. ```json { "rows": [], diff --git a/fili-wikipedia-example/Dockerfile b/fili-wikipedia-example/Dockerfile index 12e6c99eec..b9c4b09d32 100644 --- a/fili-wikipedia-example/Dockerfile +++ b/fili-wikipedia-example/Dockerfile @@ -14,10 +14,8 @@ RUN wget -q -O - http://archive.apache.org/dist/maven/maven-3/3.2.5/binaries/apa RUN mkdir -p /usr/local/fili # update in case of new commits -ADD https://api.github.com/repos/$GITHUB_OWNER/fili/git/refs/heads/$BRANCH \ -fili-branch.json -RUN git clone -q --branch ${BRANCH} --depth 1 \ -https://github.com/$GITHUB_OWNER/fili.git /usr/local/fili +ADD https://api.github.com/repos/$GITHUB_OWNER/fili/git/refs/heads/$BRANCH fili-branch.json +RUN git clone -q --branch ${BRANCH} --depth 1 https://github.com/$GITHUB_OWNER/fili.git /usr/local/fili WORKDIR /usr/local/fili diff --git a/fili-wikipedia-example/README.md b/fili-wikipedia-example/README.md index 8db20bc055..621c3d75cf 100644 --- a/fili-wikipedia-example/README.md +++ b/fili-wikipedia-example/README.md @@ -70,12 +70,10 @@ Here are some sample queries that you can run to verify your server: There is a [Docker image](https://hub.docker.com/r/mpardesh/fili/) for Fili which can be found on Dockerhub. If you would like to experiment with Fili without having to download its dependencies, you can [install](https://www.docker.com/community-edition) and start Docker. Then run these commands: - docker pull mpardesh/fili:1.0 docker run --name fili-wikipedia-example -i --rm -p 3001:8081 -p 3000:8082 -p 9998:9998 mpardesh/fili:1.0 - This will start a container. Please wait a few minutes for Druid to get ready. Once Druid is ready, you can start querying! Here is a From b352a5b36a95392d48a671966176977298d78c53 Mon Sep 17 00:00:00 2001 From: Monica Pardeshi Date: Sun, 20 Aug 2017 10:23:53 -0500 Subject: [PATCH 19/19] changed style --- fili-wikipedia-example/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fili-wikipedia-example/Dockerfile b/fili-wikipedia-example/Dockerfile index b9c4b09d32..14c730dc3b 100644 --- a/fili-wikipedia-example/Dockerfile +++ b/fili-wikipedia-example/Dockerfile @@ -2,7 +2,8 @@ FROM druidio/example-cluster:latest LABEL maintainer="https://groups.google.com/forum/#!forum/fili-developers" # fili github information -ENV GITHUB_OWNER=yahoo BRANCH=master +ENV GITHUB_OWNER yahoo +ENV BRANCH master RUN apt-get update && apt-get install -y --no-install-recommends git curl