diff --git a/.babelrc b/.babelrc new file mode 100644 index 0000000..d018deb --- /dev/null +++ b/.babelrc @@ -0,0 +1,4 @@ +{ + "presets": ["es2015"] +} + diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e9c4b10 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,17 @@ +# Wild +*.log +*.tmp +*.bak +*.bck + +# Files +Dockerfile + +# Folders +.git/ +data/ +docker/*.* +docks/ +doc/ +node_modules/ +tmp/ diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..347e7e6 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,26 @@ +; indicate this is the root of the project +root = true + +[*] +indent_style = space +end_of_line = lf +indent_size = 2 +charset = utf-8 +trim_trailing_whitespace = true + +[*.c] +indent_size = 4 + +[*.cpp] +indent_size = 4 + +[*.h] +indent_size = 4 + +[*.md] +max_line_length = 0 +trim_trailing_whitespace = false + +[Makefile] +indent_style = tab +indent_size = 8 \ No newline at end of file diff --git a/.eslintrc b/.eslintrc new file mode 100644 index 0000000..6b75e74 --- /dev/null +++ b/.eslintrc @@ -0,0 +1,43 @@ +{ + "extends": "airbnb", + "parser": "babel-eslint", + "parserOptions": { + "ecmaVersion": 7, + "sourceType": "module", + "ecmaFeatures": { + "experimentalObjectRestSpread": true, + "jsx": true + } + }, + "rules": { + "arrow-body-style": 0, + "arrow-parens": 0, + "camelcase": 0, + "comma-dangle": 0, + "global-require": 0, + "import/no-dynamic-require": 0, + "import/prefer-default-export": 0, + "no-constant-condition": 0, + "max-len": 0, + "no-console": 0, + "object-curly-spacing": 0, + "react/forbid-prop-types": 0, + "react/jsx-curly-spacing": 0, + "react/jsx-filename-extension": 0, + "react/jsx-space-before-closing": 0, + "react/prefer-stateless-function": 0, + "semi": 2 + }, + "globals": { + "__DEVELOPMENT__": true, + "__CLIENT__": true, + "__SERVER__": true, + "__DISABLE_SSR__": true, + "__DEVTOOLS__": true, + "beforeEach": true, + "describe": true, + "document": true, + "it": true, + "window": true, + } +} diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..9197c71 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,29 @@ +sudo: required + +services: + - docker + +language: node_js +node_js: + - "6" + - "6.1" + - "5.11" + +env: + - CXX=g++-4.8 +addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-4.8 + +before_install: + - "docker-compose up -d postgres" + - "sleep 15" + - "docker-compose up -d" + - "sleep 15" + - "curl http://localhost:4000" + +before_script: + - npm run gulp diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..bdc0d40 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,39 @@ +FROM mhart/alpine-node:6.7.0 + +# Install required dependencies (Alpine Linux packages) +RUN apk update && \ + apk add --no-cache \ + cmake \ + g++ \ + gcc \ + gd-dev \ + git \ + libev-dev \ + libevent-dev \ + libuv-dev \ + make \ + openssl-dev \ + perl \ + python + +RUN git clone git://github.com/couchbase/libcouchbase.git && \ + cd libcouchbase && \ + git checkout 2.5.8 && \ + mkdir build && \ + cd build && \ + ../cmake/configure && \ + make && \ + make install && \ + cd .. + +# Copy required stuff +ADD . . + +# If you need npm, don't use a base tag +RUN npm install + +# RUN npm install -g babel-cli gulp + +RUN npm run gulp + +ENTRYPOINT ["./bin/microcrawler-worker.js"] diff --git a/README.md b/README.md index d47e1bb..d69b195 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,237 @@ -# microcrawler-twitter -Simple PoC of Twitter Stream Crawler +# Microcrawler Worker + +## Status + +[![Build Status](https://travis-ci.org/ApolloCrawler/microcrawler-worker.svg?branch=master)](https://travis-ci.org/ApolloCrawler/microcrawler-worker) +[![License](https://img.shields.io/github/license/ApolloCrawler/microcrawler-worker.svg)]() +[![Dependency Status](https://gemnasium.com/badges/github.com/ApolloCrawler/microcrawler-worker.svg)](https://gemnasium.com/github.com/ApolloCrawler/microcrawler-worker) + +## Prerequisities + +- [microcrawler-webapp](https://github.com/ApolloCrawler/microcrawler-webapp) + + +## Getting started + +***Clone Sources*** + +``` +git clone https://github.com/AplloCrawler/microcrawler-worker.git +``` + +***Enter Directory with Sources*** + +``` +cd microcrawler-worker +``` + +***Install Dependencies*** + +``` +npm install +``` + +***Run Microcrawler Worker*** + +``` +bin/microcrawler-worker.js +``` + +## Usage + +***Show Help*** + +``` +$ ./bin/microcrawler-worker.js -h + + Usage: microcrawler-worker [options] + + Options: + + -h, --help output usage information + -V, --version output the version number + -c, --channel Channel to connect to, default: worker:lobby + --heartbeat-interval Heartbeat interval in milliseconds, default: 10000 + -i, --interactive Run interactive mode + -u, --url URL to connect to, default: ws://localhost:4000/socket + -t, --token Token used for authorization, default: null +``` + +***Print Version*** + +``` +$ ./bin/microcrawler-worker.js --version +0.0.1 +``` + +***Connect to Default URL*** + +``` +$ ./bin/microcrawler-worker.js +Connecting to "ws://localhost:4000/socket" +``` + +***Connect to Custom URL*** + +``` +$ ./bin/microcrawler-worker.js -u ws://example.com/socket +Connecting to "ws://example.com/socket" +``` + +***Sample Output*** + +``` +$ ./bin/microcrawler-worker.js --heartbeat-interval 10000 +Connecting to "ws://localhost:4000/socket" +Received ok +{ + "msg": "Welcome!" +} +Received event - pong +{ + "version": "0.0.1", + "os": { + "uptime": 139871, + "platform": "darwin", + "mem": { + "total": 8589934592, + "free": 134340608 + }, + "load": [ + 2.14794921875, + 2.11328125, + 2.318359375 + ], + "hostname": "kx-mac.local", + "endian": "LE", + "cpus": [ + { + "times": { + "user": 17079150, + "sys": 8949760, + "nice": 0, + "irq": 0, + "idle": 65412150 + }, + "speed": 1700, + "model": "Intel(R) Core(TM) i7-4650U CPU @ 1.70GHz" + }, + { + "times": { + "user": 7933380, + "sys": 3558830, + "nice": 0, + "irq": 0, + "idle": 79947730 + }, + "speed": 1700, + "model": "Intel(R) Core(TM) i7-4650U CPU @ 1.70GHz" + }, + { + "times": { + "user": 17069960, + "sys": 7510340, + "nice": 0, + "irq": 0, + "idle": 66859640 + }, + "speed": 1700, + "model": "Intel(R) Core(TM) i7-4650U CPU @ 1.70GHz" + }, + { + "times": { + "user": 8092410, + "sys": 3646170, + "nice": 0, + "irq": 0, + "idle": 79701350 + }, + "speed": 1700, + "model": "Intel(R) Core(TM) i7-4650U CPU @ 1.70GHz" + } + ] + }, + "name": "microcrawler-worker", + "msg": "I am still alive!", + "id": 0, + "ts": 1475360236777 +} +``` + +***Interactive Mode*** + +*Server* + +``` +$ iex -S mix phoenix.server +Erlang/OTP 19 [erts-8.1] [source] [64-bit] [smp:4:4] [async-threads:10] [hipe] [kernel-poll:false] [dtrace] + +[info] Running MicrocrawlerWebapp.Endpoint with Cowboy using http://localhost:4000 +Interactive Elixir (1.3.3) - press Ctrl+C to exit (type h() ENTER for help) +iex(1)> 05 Oct 20:54:35 - info: compiled 6 files into 2 files, copied 6 in 1.6 sec +[info] JOIN worker:lobby to MicrocrawlerWebapp.WorkerChannel + Transport: Phoenix.Transports.WebSocket + Parameters: %{"token" => nil} +Received join - worker:lobby +{ + "token": null +} +#PID<0.366.0> +[info] Replied worker:lobby :ok +{:basic_consume_ok, %{consumer_tag: "amq.ctag-Rb-fiGVc4xTncGoOKjxyKA"}} +#PID<0.366.0> +Received event - msg +{ + "msg": "Hello World!" +} +Received event - msg +"Hi, how are you?" +Received event - msg +"Received my message?" +Received event - msg +42 +Received event - msg +{ + "abc": "123" +} +Received event - msg +{ + "pole": [ + 1, + 2, + 3, + 4 + ], + "msg": "to pole je fakt urodny" +} +{:shutdown, :closed} +#PID<0.366.0> +%Phoenix.Socket{assigns: %{rabb_chan: %AMQP.Channel{conn: %AMQP.Connection{pid: #PID<0.369.0>}, + pid: #PID<0.379.0>}, rabb_conn: %AMQP.Connection{pid: #PID<0.369.0>}}, + channel: MicrocrawlerWebapp.WorkerChannel, channel_pid: #PID<0.366.0>, + endpoint: MicrocrawlerWebapp.Endpoint, handler: MicrocrawlerWebapp.UserSocket, + id: nil, joined: true, pubsub_server: MicrocrawlerWebapp.PubSub, ref: nil, + serializer: Phoenix.Transports.WebSocketSerializer, topic: "worker:lobby", + transport: Phoenix.Transports.WebSocket, transport_name: :websocket, + transport_pid: #PID<0.364.0>} +``` + +*Client* + +``` +$ ./bin/microcrawler-worker.js -i --heartbeat-interval 3600000 +Connecting to "ws://localhost:4000/socket" +Running in interactive mode. +Type "quit" or press ctrl+c twice to exit. +msg> Received ok +{ + "msg": "Welcome!" +} +msg> Hi, how are you? +msg> Received my message? +msg> 42 +msg> {"abc": "123"} +msg> {"pole": [1, 2, 3, 4], "msg": "to pole je fakt urodny"} +msg> quit +$ +``` diff --git a/bin/microcrawler-twitter.js b/bin/microcrawler-twitter.js new file mode 100755 index 0000000..95f355e --- /dev/null +++ b/bin/microcrawler-twitter.js @@ -0,0 +1,5 @@ +#! /usr/bin/env node + +var App = require('../lib/app').default; +var app = new App(); +app.main(); diff --git a/config/cmds/quit.json b/config/cmds/quit.json new file mode 100644 index 0000000..c503772 --- /dev/null +++ b/config/cmds/quit.json @@ -0,0 +1,16 @@ +[ + "exit", + "exit()", + "x", + "x()", + "\\x", + "/x", + "/exit", + "quit", + "quit()", + "q", + "q()", + "\\q", + "/q", + "/quit" +] \ No newline at end of file diff --git a/gulpfile.js b/gulpfile.js new file mode 100644 index 0000000..2742c87 --- /dev/null +++ b/gulpfile.js @@ -0,0 +1,51 @@ +require('babel-core/register'); + +const gulp = require('gulp'); +const babel = require('gulp-babel'); +const eslint = require('gulp-eslint'); +const mocha = require('gulp-mocha'); + +const files = { + sources: [ + 'src/**/*.js' + ], + tests: [ + 'test/**/*.js' + ] +}; + +gulp.task('build', ['lint'], () => { + return gulp.src(files.sources) + .pipe(babel()) + .pipe(gulp.dest('lib')); +}); + +gulp.task('default', ['build']); + +gulp.task('lint', () => { + // ESLint ignores files with "node_modules" paths. + // So, it's best to have gulp ignore the directory as well. + // Also, Be sure to return the stream from the task; + // Otherwise, the task may end before the stream has finished. + return gulp.src([...files.sources, ...files.tests, '!node_modules/**']) + // eslint() attaches the lint output to the "eslint" property + // of the file object so it can be used by other modules. + .pipe(eslint()) + // eslint.format() outputs the lint results to the console. + // Alternatively use eslint.formatEach() (see Docs). + .pipe(eslint.format()) + // To have the process exit with an error code (1) on + // lint error, return the stream and pipe to failAfterError last. + .pipe(eslint.failAfterError()); +}); + +gulp.task('test', ['build'], () => + gulp.src(...files.tests, {read: false}) + // gulp-mocha needs filepaths so you can't have any plugins before it + .pipe(mocha()) +); + + +gulp.task('watch', ['test'], () => { + gulp.watch([...files.sources, ...files.tests], ['test']); +}); \ No newline at end of file diff --git a/lib/app/app.js b/lib/app/app.js new file mode 100644 index 0000000..a801e1b --- /dev/null +++ b/lib/app/app.js @@ -0,0 +1,75 @@ +'use strict'; + +Object.defineProperty(exports, "__esModule", { + value: true +}); + +var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); + +var _commander = require('commander'); + +var _commander2 = _interopRequireDefault(_commander); + +var _twit = require('twit'); + +var _twit2 = _interopRequireDefault(_twit); + +var _package = require('../../package.json'); + +var _package2 = _interopRequireDefault(_package); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } + +var DEFAULT_CONSUMER_KEY = 'wkcWm5RTq8JsWtlY7tIXD5e83'; +var DEFAULT_CONSUMER_SECRET = 'qS6YBe7AQXO0MxFyUvelqE1El6UoTPlpQ31OBU7HjDJFolvo4y'; +var DEFAULT_ACCESS_TOKEN = '203465100-qsUXAoYYpVFZ6tWTr17yVHHv0VEaPR5q1vlDQXd7'; +var DEFAULT_ACCESS_TOKEN_SECRET = 'G3cxHx6FqrQlqhZDIwnGmCBdWqoJIUrB3BUvldFKXJTxT'; + +var DEFAULT_TIMEOUT_MS = 60 * 1000; + +var DEFAULT_TOPIC = '#trump'; + +var App = function () { + function App() { + _classCallCheck(this, App); + + this.twitter = null; + } + + _createClass(App, [{ + key: 'main', + value: function main() { + var args = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : process.argv; + + _commander2.default.version(_package2.default.version).option('-t, --topic ', 'Topic to Crawl', DEFAULT_TOPIC).parse(args); + + this.twitter = new _twit2.default({ + consumer_key: DEFAULT_CONSUMER_KEY, + consumer_secret: DEFAULT_CONSUMER_SECRET, + access_token: DEFAULT_ACCESS_TOKEN, + access_token_secret: DEFAULT_ACCESS_TOKEN_SECRET, + timeout_ms: DEFAULT_TIMEOUT_MS + }); + + // See https://github.com/ttezel/twit + var stream = this.twitter.stream('statuses/filter', { track: _commander2.default.topic }); + stream.on('tweet', function (tweet) { + console.log(tweet.text); + }); + + stream.on('error', function (tweet) { + console.log(tweet); + }); + + stream.on('limitation', function (tweet) { + console.log(tweet); + }); + } + }]); + + return App; +}(); + +exports.default = App; \ No newline at end of file diff --git a/lib/app/index.js b/lib/app/index.js new file mode 100644 index 0000000..202cd9b --- /dev/null +++ b/lib/app/index.js @@ -0,0 +1,13 @@ +'use strict'; + +Object.defineProperty(exports, "__esModule", { + value: true +}); + +var _app = require('./app'); + +var _app2 = _interopRequireDefault(_app); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +exports.default = _app2.default; \ No newline at end of file diff --git a/lib/app/prompt/prompt.js b/lib/app/prompt/prompt.js new file mode 100644 index 0000000..9a390c3 --- /dev/null +++ b/lib/app/prompt/prompt.js @@ -0,0 +1 @@ +"use strict"; \ No newline at end of file diff --git a/lib/index.js b/lib/index.js new file mode 100644 index 0000000..202cd9b --- /dev/null +++ b/lib/index.js @@ -0,0 +1,13 @@ +'use strict'; + +Object.defineProperty(exports, "__esModule", { + value: true +}); + +var _app = require('./app'); + +var _app2 = _interopRequireDefault(_app); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +exports.default = _app2.default; \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..4c5a1ad --- /dev/null +++ b/package.json @@ -0,0 +1,59 @@ +{ + "name": "microcrawler-twitter", + "version": "0.0.1", + "description": "Microcrawler Twitter", + "main": "index.js", + "scripts": { + "test": "./node_modules/gulp/bin/gulp.js test", + "gulp": "./node_modules/gulp/bin/gulp.js" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/ApolloCrawler/microcrawler-twitter.git" + }, + "keywords": [ + "microcrawler", + "crawler", + "scrapper" + ], + "author": "Tomas Korcak ", + "license": "MIT", + "bugs": { + "url": "https://github.com/ApolloCrawler/microcrawler-twitter/issues" + }, + "homepage": "https://github.com/ApolloCrawler/microcrawler-twitter#readme", + "dependencies": { + "babel-eslint": "^7.0.0", + "bufferutil": "^1.2.1", + "chai": "^3.5.0", + "cheerio": "^0.22.0", + "colors": "^1.1.2", + "commander": "^2.9.0", + "glob": "^7.1.1", + "homedir": "^0.6.0", + "microcrawler-crawler-all": "0.0.7", + "mocha": "^3.1.2", + "node-uuid": "^1.4.7", + "phoenix-socket": "git://github.com/bossek/phoenix_socket.git#v1.2.patched", + "ramda": "^0.22.1", + "superagent": "^2.3.0", + "twit": "^2.2.5", + "utf-8-validate": "^1.2.1", + "websocket": "^1.0.23", + "xhr2": "^0.1.3" + }, + "devDependencies": { + "babel-core": "^6.16.0", + "babel-preset-es2015": "^6.16.0", + "eslint": "^3.7.0", + "eslint-config-airbnb": "^12.0.0", + "eslint-config-es2015": "^1.1.0", + "eslint-plugin-import": "^1.16.0", + "eslint-plugin-jsx-a11y": "^2.2.2", + "eslint-plugin-react": "^6.3.0", + "gulp": "^3.9.1", + "gulp-babel": "^6.1.2", + "gulp-eslint": "^3.0.1", + "gulp-mocha": "^3.0.1" + } +} diff --git a/src/app/app.js b/src/app/app.js new file mode 100644 index 0000000..cfa9fba --- /dev/null +++ b/src/app/app.js @@ -0,0 +1,48 @@ +import program from 'commander'; +import Twit from 'twit'; + +import pkg from '../../package.json'; + +const DEFAULT_CONSUMER_KEY = 'wkcWm5RTq8JsWtlY7tIXD5e83'; +const DEFAULT_CONSUMER_SECRET = 'qS6YBe7AQXO0MxFyUvelqE1El6UoTPlpQ31OBU7HjDJFolvo4y'; +const DEFAULT_ACCESS_TOKEN = '203465100-qsUXAoYYpVFZ6tWTr17yVHHv0VEaPR5q1vlDQXd7'; +const DEFAULT_ACCESS_TOKEN_SECRET = 'G3cxHx6FqrQlqhZDIwnGmCBdWqoJIUrB3BUvldFKXJTxT'; + +const DEFAULT_TIMEOUT_MS = 60 * 1000; + +const DEFAULT_TOPIC = '#trump'; + +export default class App { + constructor() { + this.twitter = null; + } + + main(args = process.argv) { + program + .version(pkg.version) + .option('-t, --topic ', 'Topic to Crawl', DEFAULT_TOPIC) + .parse(args); + + this.twitter = new Twit({ + consumer_key: DEFAULT_CONSUMER_KEY, + consumer_secret: DEFAULT_CONSUMER_SECRET, + access_token: DEFAULT_ACCESS_TOKEN, + access_token_secret: DEFAULT_ACCESS_TOKEN_SECRET, + timeout_ms: DEFAULT_TIMEOUT_MS + }); + + // See https://github.com/ttezel/twit + const stream = this.twitter.stream('statuses/filter', { track: program.topic }); + stream.on('tweet', (tweet) => { + console.log(tweet.text); + }); + + stream.on('error', (tweet) => { + console.log(tweet); + }); + + stream.on('limitation', (tweet) => { + console.log(tweet); + }); + } +} diff --git a/src/app/index.js b/src/app/index.js new file mode 100644 index 0000000..ee338bc --- /dev/null +++ b/src/app/index.js @@ -0,0 +1,3 @@ +import App from './app'; + +export default App; diff --git a/src/index.js b/src/index.js new file mode 100644 index 0000000..ee338bc --- /dev/null +++ b/src/index.js @@ -0,0 +1,3 @@ +import App from './app'; + +export default App; diff --git a/test/microcrawler-twitter.spec.babel.js b/test/microcrawler-twitter.spec.babel.js new file mode 100644 index 0000000..ef211bd --- /dev/null +++ b/test/microcrawler-twitter.spec.babel.js @@ -0,0 +1,17 @@ +import {expect} from 'chai'; + +import App from '../lib'; + + +describe('microcrawler-twitter', () => { + it('should work', () => { + expect(1 + 1).to.equal(2); + }); +}); + +describe('Command Line Interface', () => { + it('accepts', () => { + const app = new App(); + app.main(); + }); +});