From dd59cf877f26a259020de0faa6c7ef9490030bbb Mon Sep 17 00:00:00 2001 From: Keeley Erhardt Date: Mon, 2 Apr 2018 17:43:19 +0100 Subject: [PATCH 1/2] Dockerize Ingestion service --- .dockerignore | 8 +++++ .travis.yml | 48 +++++++++++++++---------- Dockerfile | 34 ++++++++++++++++++ cmd/{omniscience_server => api}/main.go | 16 ++++++--- docker-compose.yml | 38 ++++++++++++++++++++ ui/Dockerfile | 21 ++++++----- ui/package.json | 3 ++ ui/tsconfig.json | 3 +- 8 files changed, 139 insertions(+), 32 deletions(-) create mode 100644 .dockerignore rename cmd/{omniscience_server => api}/main.go (64%) create mode 100644 docker-compose.yml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..84bc22c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.dockerignore +.gitignore +README.md + +node_modules/ +build/ +public/ +npm-debug.log diff --git a/.travis.yml b/.travis.yml index 14b9433..2db9c4f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,34 +1,46 @@ -# .travis.yml +sudo: required + +services: + - docker + env: global: - secure: "i9TsESPUaytcKK4NYOtKgQuiKwvwyRKRqnCWJ9GUBmN7nnk6LeOeD63qFDhF3Kyap9Vkml2Ky6Dzg1enWYOsvCs+8Mb+0ZGbUGGn86cGSYQauOscH3BzMZBLKYon7LzPY183ucoT83E4gHxlnzpf43B+UvwLsnWUf6h8FRhknuk+Ff6AgEDct1bDGNwJITWCnS75Q4q6Wl3GtTusZtFe2oHhgcYpsWa6u7/b8PfAoCsxu5FeYbaRUWmUYAFqSuTJfUvVnGRGraNGaWosIV6yeYpJMco4kkdZH46ReAU1aGRTobrYEpiA2kzmhVDuWHiK+75sItHeQMHENGztfxaPGPT61L6t2r51MXJBKzCq9Xe2Er5/fK7vgL6fzBCLrY2A3ICIxwmMJgRUEl7k+0ED9ezdUMgWfGU/LQZ1XWgpYB+DKC1ZJ+19NoswBJAWe3SUVvkSV+I+NYoMVGPwR73qIDglUXwyR7Ol4uRlQrIBGH+xjK4RdSQbZIPGM+fK9unJgD1k3SWD1QQSc0/BXY52KN/VkJpsUL0uV2YoYv/uvxh5OHZP+9PVfz6+7jxUUJw4iha0rMNb+R4kqpR9yjSOYvEP58ctSGtaDWkIiAI7wpS2oib2eTxhQe1Vtfdq/aLCfhThZYzflWWkueFxFNnRQ27dMCPLz9NfByQUfcxz1Ec=" + - secure: "i9TsESPUaytcKK4NYOtKgQuiKwvwyRKRqnCWJ9GUBmN7nnk6LeOeD63qFDhF3Kyap9Vkml2Ky6Dzg1enWYOsvCs+8Mb+0ZGbUGGn86cGSYQauOscH3BzMZBLKYon7LzPY183ucoT83E4gHxlnzpf43B+UvwLsnWUf6h8FRhknuk+Ff6AgEDct1bDGNwJITWCnS75Q4q6Wl3GtTusZtFe2oHhgcYpsWa6u7/b8PfAoCsxu5FeYbaRUWmUYAFqSuTJfUvVnGRGraNGaWosIV6yeYpJMco4kkdZH46ReAU1aGRTobrYEpiA2kzmhVDuWHiK+75sItHeQMHENGztfxaPGPT61L6t2r51MXJBKzCq9Xe2Er5/fK7vgL6fzBCLrY2A3ICIxwmMJgRUEl7k+0ED9ezdUMgWfGU/LQZ1XWgpYB+DKC1ZJ+19NoswBJAWe3SUVvkSV+I+NYoMVGPwR73qIDglUXwyR7Ol4uRlQrIBGH+xjK4RdSQbZIPGM+fK9unJgD1k3SWD1QQSc0/BXY52KN/VkJpsUL0uV2YoYv/uvxh5OHZP+9PVfz6+7jxUUJw4iha0rMNb+R4kqpR9yjSOYvEP58ctSGtaDWkIiAI7wpS2oib2eTxhQe1Vtfdq/aLCfhThZYzflWWkueFxFNnRQ27dMCPLz9NfByQUfcxz1Ec=" + - secure: "lhgrjdfq/PbYx6xguYXTE1u8fx/UGi56sR4oE5lJPTeViBNsaiQFpwoUlMQS5IlRHWX6ilH1l4XIh6Glp23iycSX73W1AJqzJa9UKEFm4D6tW2R3UWP6fOprekQ7Hl+Sy/uNoKGEA5d7u+U+/cxJ0rhlai2FiBEulr6PaoWcnbivW53bwlRjewIVmITjYzDhAapn4Gl7Ki3rScslJ/aDyPJHd4/N4xAWJ99Y4kDSSFSmdo+EWoeXOb1v6lStB6ULxl/xcGGNsbqP08E3k3tER25lYFnyx7qFA3az338sFrlVU0uGnwU8sgyeLkg1WZM7tqYutGc+5hPqG0Ko4yrrY08rpKOdcLfQVC9L94YdgTwaSwaA2TGN7O78RbCQfX8QWhAnKn2xpVa8zuuh4262oJh43HZFWFoR+afW/9NNaFksHDtfVWw1K+IhzgAnGHPXxbJzGYalcUCT2BsriUSHmP5qsgbMfoOAUfVvQAt6d6Dal9Mz0LlOOe/Twa8FZapcpsq77vIrLhlFUNEZUzd/LjFNy9VlvMjw6lWQfNAg3Zfuty0bd5vmEgj6lVnv83VGB6R+fPeLfTrBbMIWMfM/IUZVsAJ1AMGy8oQynjCAsuGO5P0jSycN3Fyy69PmGL0C7wph4xkEAPlrAgAxfy1KFTVxzcZ1RmVxMtZ1iDXVbwY=" # DOCKER_USER + - secure: "JevDYrp+pskYhXBoXGwbrMQd398DTswwEsxj+4GEPt9yE9ppi1ey8USjr3/VdmrhkzaEuyuH7wLsqD8TYog+F8n6ALbkQUXiOQZcZL7J7f+fRS1Ox79xyfH97ABOsUQXvKhwGSYBaSv5UIVqrwrkOIvUu4nsUKhw0cP0SBFuNAka5iSk5wOHw8uQXM/WGqbt0BULijH4cEtVt0kyTgzj4xaGvrom+vmvy6UuKo0LR+vRXWHV+ApE9Oib5a9aduAWJuGF/glEUZF7x5atT8jjO7C/B7EhlqV4/+lo9x0JAINPWJ01JSJmeBqn4xg0HayPcLhJ2jiipEgMNeB+2nNsjYaSMZlBQy8fZ5q5MHXTeGwLY24J6kkYT54y+zFfw+R6fGFd1AJn30s1cT/pCnyDoog9IAAAticnJgQIrbk2JXbM4RD234/zeOPKGvGto86781sFcJDxI9zZ1m+6/b96lgAgXnNEW/MzgQ3vv4zr2HZAnSuBGVESJhocdmfMs+3n1Aq3b8KCI/3yc/mysht1QULN9CLB+atzKMnWnlzfWWrZedBusL2VTbPruPnSLTQ55YIaff/4VcIu+oirUGfZS+S+Eu5qvMzI7CrXcS1BXYcwWB906zFyK2O6/q9163agHwPUPZhTpBPdMwxSmSQfnq0FWH5B09nxGeDr06khwms=" # DOCKER_PASS + - DOCKER_COMPOSE_VERSION=1.20.1 + - COMMIT=${TRAVIS_COMMIT::8} + - REPO=keelerh/omniscience language: go go: - "1.10" -before_install: +install: + # Coveralls - go get github.com/mattn/goveralls + + # dep - curl -L -s https://github.com/golang/dep/releases/download/v0.4.1/dep-linux-amd64 -o $GOPATH/bin/dep - chmod +x $GOPATH/bin/dep -install: - - dep ensure - script: - $HOME/gopath/bin/goveralls -v -service=travis-ci - -services: - - docker + - dep ensure after_success: - - test -n "$TRAVIS_TAG" && curl -sL https://git.io/goreleaser | bash -# - mkdir -p dist -# - CGO_ENABLED="0" GOARCH="amd64" GOOS="linux" go build -a -installsuffix cgo -o ./dist/gomotics -# - docker login -u $DOCKER_USER -p $DOCKER_PASSWORD -# - export REPO=$DOCKER_USER/gomotics -# - export TAG=`if [ "$TRAVIS_BRANCH" == "master" ]; then echo "latest"; else echo $TRAVIS_TAG ; fi` -# - echo $REPO:$TAG -# - docker build -f Dockerfile -t $REPO:$TAG . -# - docker push $REPO \ No newline at end of file + - echo "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin + - export TAG=`if [ "$TRAVIS_BRANCH" == "master" ]; then echo "latest"; else echo $TRAVIS_BRANCH ; fi` + + # Build Ingestion service image + - docker build -t $REPO:api/$COMMIT . + - docker tag $REPO:$COMMIT $REPO:api/$TAG + - docker tag $REPO:$COMMIT $REPO:api/travis-$TRAVIS_BUILD_NUMBER + + # Build React frontend image + - docker build -t $REPO:ui/$COMMIT ./ui + - docker tag $REPO:$COMMIT $REPO:ui/$TAG + - docker tag $REPO:$COMMIT $REPO:ui/travis-$TRAVIS_BUILD_NUMBER + + - docker push $REPO diff --git a/Dockerfile b/Dockerfile index e69de29..f2e166e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -0,0 +1,34 @@ +# 1) BUILD INGESTION API +FROM golang:alpine AS build-go +RUN apk --no-cache add git + +WORKDIR /go/src/github.com/keelerh/omniscience + +RUN go get -u github.com/golang/dep/... +COPY Gopkg.toml Gopkg.lock ./ +RUN dep ensure -v --vendor-only + +COPY ./pkg ./pkg +COPY ./protos ./protos +COPY ./cmd/api ./cmd/api + +RUN cd cmd/api && go build -o api && cp api /tmp/ + + +# 2) INSTALL DOCKERIZE +FROM alpine AS install-dockerize +RUN apk update && apk add wget +RUN wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz +RUN tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + + +# 3) BUILD FINAL IMAGE +FROM alpine +RUN apk --no-cache add ca-certificates + +WORKDIR /app/server/ + +COPY --from=build-go /tmp/api /app/server/ +COPY --from=install-dockerize /usr/local/bin/dockerize /usr/local/bin/dockerize + +EXPOSE 50051 diff --git a/cmd/omniscience_server/main.go b/cmd/api/main.go similarity index 64% rename from cmd/omniscience_server/main.go rename to cmd/api/main.go index 0f3e3a2..e24a61d 100644 --- a/cmd/omniscience_server/main.go +++ b/cmd/api/main.go @@ -3,6 +3,7 @@ package main import ( + "flag" "log" "net" @@ -12,10 +13,16 @@ import ( "google.golang.org/grpc" ) -const ( - port = ":50051" +var ( + fElasticsearchUrl = flag.String( + "elasticsearch_url", + "http://elasticsearch:9200", + "Defines the URL endpoint of the Elasticsearch node") ) +// TODO: Port should be set as an ENV variable. +const port = ":50051" + func main() { lis, err := net.Listen("tcp", port) if err != nil { @@ -23,8 +30,9 @@ func main() { } s := grpc.NewServer() - // Obtain a client and connect to the default Elasticsearch installation on 127.0.0.1:9200. - esClient, err := elastic.NewClient() + // Obtain a client and connect to the Elasticsearch installation at the URL endpoint. + opts := elastic.SetURL(*fElasticsearchUrl) + esClient, err := elastic.NewClient(opts) if err != nil { log.Fatalf("failed to initialise Elasticsearch client: %v", err) } diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..57ee456 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,38 @@ +version: '3' + +services: + api: # Go Ingestion service + container_name: om-api + build: . + ports: + - "50051:50051" # Expose API port + environment: # Set ENV vars + - ES_HOST=elasticsearch + - PORT=50051 + depends_on: + - elasticsearch + command: dockerize -wait http://elasticsearch:9200 -timeout 30s ./api + + ui: # React frontend + # TODO: Enable hot reloading. + container_name: om-ui + build: ./ui + ports: + - "3000:3000" # Expose Node dev port + environment: # Set ENV vars + - NODE_ENV=local + depends_on: + - elasticsearch + + elasticsearch: + container_name: om-es + image: docker.elastic.co/elasticsearch/elasticsearch:6.2.3 + ports: + - "9200:9200" + environment: + - http.cors.enabled=true + - http.cors.allow-origin=* + - http.cors.allow-methods=OPTIONS, HEAD, GET, POST, PUT, DELETE + - http.cors.allow-headers=X-Requested-With, X-Auth-Token, Content-Type, Content-Length + volumes: + - "/tmp/esdata:/usr/share/elasticsearch/data" diff --git a/ui/Dockerfile b/ui/Dockerfile index 89ab81b..900b3a7 100644 --- a/ui/Dockerfile +++ b/ui/Dockerfile @@ -1,9 +1,12 @@ -FROM node:alpine AS build-react -ENV D=/go/src/github.com/keelerh/omniscience -# WORKDIR $D/ui -RUN mkdir -p /src/ui -ADD ./ui/package.json /src/ui/ -ADD ./ui /src/ui -RUN cd /src/ui && npm install -COPY . . -CMD [ "npm", "start" ] +# 1) BUILD REACT FRONTEND +FROM node:9.10.0-alpine +RUN apk --no-cache add ca-certificates + +COPY package.json package.json +RUN npm install + +COPY . /app/ui +WORKDIR /app/ui + +EXPOSE 3000 +ENTRYPOINT ["npm", "start"] diff --git a/ui/package.json b/ui/package.json index 34422dc..dc2a7fd 100644 --- a/ui/package.json +++ b/ui/package.json @@ -15,6 +15,9 @@ "test": "react-scripts-ts test --env=jsdom", "eject": "react-scripts-ts eject" }, + "optionalDependencies": { + "fsevents": "*" + }, "devDependencies": { "@types/jest": "^22.1.4", "@types/node": "^9.4.6", diff --git a/ui/tsconfig.json b/ui/tsconfig.json index 8597354..089c6fe 100644 --- a/ui/tsconfig.json +++ b/ui/tsconfig.json @@ -15,7 +15,8 @@ "noImplicitAny": true, "strictNullChecks": true, "suppressImplicitAnyIndexErrors": true, - "noUnusedLocals": true + "noUnusedLocals": true, + "baseUrl": "." }, "exclude": [ "node_modules", From c12108566943c39d003d91c58f8232a19b0e8fe0 Mon Sep 17 00:00:00 2001 From: Keeley Erhardt Date: Sun, 8 Apr 2018 21:05:42 +0100 Subject: [PATCH 2/2] Seperate services into different docker repos --- .travis.yml | 19 ++++++++++--------- Dockerfile | 1 + 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2db9c4f..5360296 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,8 @@ env: - secure: "JevDYrp+pskYhXBoXGwbrMQd398DTswwEsxj+4GEPt9yE9ppi1ey8USjr3/VdmrhkzaEuyuH7wLsqD8TYog+F8n6ALbkQUXiOQZcZL7J7f+fRS1Ox79xyfH97ABOsUQXvKhwGSYBaSv5UIVqrwrkOIvUu4nsUKhw0cP0SBFuNAka5iSk5wOHw8uQXM/WGqbt0BULijH4cEtVt0kyTgzj4xaGvrom+vmvy6UuKo0LR+vRXWHV+ApE9Oib5a9aduAWJuGF/glEUZF7x5atT8jjO7C/B7EhlqV4/+lo9x0JAINPWJ01JSJmeBqn4xg0HayPcLhJ2jiipEgMNeB+2nNsjYaSMZlBQy8fZ5q5MHXTeGwLY24J6kkYT54y+zFfw+R6fGFd1AJn30s1cT/pCnyDoog9IAAAticnJgQIrbk2JXbM4RD234/zeOPKGvGto86781sFcJDxI9zZ1m+6/b96lgAgXnNEW/MzgQ3vv4zr2HZAnSuBGVESJhocdmfMs+3n1Aq3b8KCI/3yc/mysht1QULN9CLB+atzKMnWnlzfWWrZedBusL2VTbPruPnSLTQ55YIaff/4VcIu+oirUGfZS+S+Eu5qvMzI7CrXcS1BXYcwWB906zFyK2O6/q9163agHwPUPZhTpBPdMwxSmSQfnq0FWH5B09nxGeDr06khwms=" # DOCKER_PASS - DOCKER_COMPOSE_VERSION=1.20.1 - COMMIT=${TRAVIS_COMMIT::8} - - REPO=keelerh/omniscience + - API_REPO=keelerh/omniscience-api + - UI_REPO=keelerh/omniscience-ui language: go @@ -34,13 +35,13 @@ after_success: - export TAG=`if [ "$TRAVIS_BRANCH" == "master" ]; then echo "latest"; else echo $TRAVIS_BRANCH ; fi` # Build Ingestion service image - - docker build -t $REPO:api/$COMMIT . - - docker tag $REPO:$COMMIT $REPO:api/$TAG - - docker tag $REPO:$COMMIT $REPO:api/travis-$TRAVIS_BUILD_NUMBER + - docker build -t $API_REPO:$COMMIT . + - docker tag $API_REPO:$COMMIT $API_REPO:$TAG + - docker tag $API_REPO:$COMMIT $API_REPO:travis-$TRAVIS_BUILD_NUMBER + - docker push $API_REPO # Build React frontend image - - docker build -t $REPO:ui/$COMMIT ./ui - - docker tag $REPO:$COMMIT $REPO:ui/$TAG - - docker tag $REPO:$COMMIT $REPO:ui/travis-$TRAVIS_BUILD_NUMBER - - - docker push $REPO + - docker build -t $UI_REPO:$COMMIT ./ui + - docker tag $UI_REPO:$COMMIT $UI_REPO:$TAG + - docker tag $UI_REPO:$COMMIT $UI_REPO:travis-$TRAVIS_BUILD_NUMBER + - docker push $UI_REPO diff --git a/Dockerfile b/Dockerfile index f2e166e..e10d399 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,3 +32,4 @@ COPY --from=build-go /tmp/api /app/server/ COPY --from=install-dockerize /usr/local/bin/dockerize /usr/local/bin/dockerize EXPOSE 50051 +ENTRYPOINT ["dockerize", "-wait", "http://elasticsearch:9200", "-timeout", "30s", "./api"]