diff --git a/README.md b/README.md index 658cb5e..725753c 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,7 @@ 可能是正方教务系统最优雅的一只爬虫。 -#项目地址： - -http://lcrawl.lzjtuhand.com +# 项目地址： http://www.luoning.me/lcrawl.html diff --git a/composer.json b/composer.json index 1cefed7..8515a19 100644 --- a/composer.json +++ b/composer.json @@ -17,7 +17,8 @@ "guzzlehttp/guzzle": "~6.0", "symfony/dom-crawler": "^3.0", "symfony/css-selector": "^3.0", - "doctrine/cache": "^1.6" + "doctrine/cache": "^1.6", + "monolog/monolog": "1.21.0" }, "autoload": { "psr-4": { diff --git a/composer.lock b/composer.lock index 4b7f642..3d411e5 100644 --- a/composer.lock +++ b/composer.lock @@ -4,8 +4,8 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", "This file is @generated automatically" ], - "hash": "7acf45218ba37a646c45c238171cde33", - "content-hash": "09531e0eba7631895b89bffb924555fd", + "hash": "111f3b83508d473d9ae6f6a57d26277f", + "content-hash": "65d5b1e680e018b5be13a9a7f67b4701", "packages": [ { "name": "doctrine/cache", @@ -17,7 +17,7 @@ }, "dist": { "type": "zip", - "url": "https://packagist.phpcomposer.com/files/doctrine/cache/f8af318d14bdb0eff0336795b428b547bd39ccb6.zip", + "url": "https://api.github.com/repos/doctrine/cache/zipball/f8af318d14bdb0eff0336795b428b547bd39ccb6", "reference": "f8af318d14bdb0eff0336795b428b547bd39ccb6", "shasum": "" }, @@ -87,7 +87,7 @@ }, "dist": { "type": "zip", - "url": "https://packagist.phpcomposer.com/files/guzzle/guzzle/d094e337976dff9d8e2424e8485872194e768662.zip", + "url": "https://api.github.com/repos/guzzle/guzzle/zipball/d094e337976dff9d8e2424e8485872194e768662", "reference": "d094e337976dff9d8e2424e8485872194e768662", "shasum": "" }, @@ -149,7 +149,7 @@ }, "dist": { "type": "zip", - "url": "https://packagist.phpcomposer.com/files/guzzle/promises/bb9024c526b22f3fe6ae55a561fd70653d470aa8.zip", + "url": "https://api.github.com/repos/guzzle/promises/zipball/bb9024c526b22f3fe6ae55a561fd70653d470aa8", "reference": "bb9024c526b22f3fe6ae55a561fd70653d470aa8", "shasum": "" }, @@ -200,7 +200,7 @@ }, "dist": { "type": "zip", - "url": "https://packagist.phpcomposer.com/files/guzzle/psr7/31382fef2889136415751badebbd1cb022a4ed72.zip", + "url": "https://api.github.com/repos/guzzle/psr7/zipball/31382fef2889136415751badebbd1cb022a4ed72", "reference": "31382fef2889136415751badebbd1cb022a4ed72", "shasum": "" }, @@ -258,7 +258,7 @@ }, "dist": { "type": "zip", - "url": "https://packagist.phpcomposer.com/files/Imangazaliev/DiDOM/8e29148bbb5c9f1d4ca8e1c4b882e0326a389201.zip", + "url": "https://api.github.com/repos/Imangazaliev/DiDOM/zipball/8e29148bbb5c9f1d4ca8e1c4b882e0326a389201", "reference": "8e29148bbb5c9f1d4ca8e1c4b882e0326a389201", "shasum": "" }, @@ -294,6 +294,84 @@ ], "time": "2016-04-22 18:01:52" }, + { + "name": "monolog/monolog", + "version": "1.21.0", + "source": { + "type": "git", + "url": "https://github.com/Seldaek/monolog.git", + "reference": "f42fbdfd53e306bda545845e4dbfd3e72edb4952" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/Seldaek/monolog/zipball/f42fbdfd53e306bda545845e4dbfd3e72edb4952", + "reference": "f42fbdfd53e306bda545845e4dbfd3e72edb4952", + "shasum": "" + }, + "require": { + "php": ">=5.3.0", + "psr/log": "~1.0" + }, + "provide": { + "psr/log-implementation": "1.0.0" + }, + "require-dev": { + "aws/aws-sdk-php": "^2.4.9", + "doctrine/couchdb": "~1.0@dev", + "graylog2/gelf-php": "~1.0", + "jakub-onderka/php-parallel-lint": "0.9", + "php-amqplib/php-amqplib": "~2.4", + "php-console/php-console": "^3.1.3", + "phpunit/phpunit": "~4.5", + "phpunit/phpunit-mock-objects": "2.3.0", + "ruflin/elastica": ">=0.90 <3.0", + "sentry/sentry": "^0.13", + "swiftmailer/swiftmailer": "~5.3" + }, + "suggest": { + "aws/aws-sdk-php": "Allow sending log messages to AWS services like DynamoDB", + "doctrine/couchdb": "Allow sending log messages to a CouchDB server", + "ext-amqp": "Allow sending log messages to an AMQP server (1.0+ required)", + "ext-mongo": "Allow sending log messages to a MongoDB server", + "graylog2/gelf-php": "Allow sending log messages to a GrayLog2 server", + "mongodb/mongodb": "Allow sending log messages to a MongoDB server via PHP Driver", + "php-amqplib/php-amqplib": "Allow sending log messages to an AMQP server using php-amqplib", + "php-console/php-console": "Allow sending log messages to Google Chrome", + "rollbar/rollbar": "Allow sending log messages to Rollbar", + "ruflin/elastica": "Allow sending log messages to an Elastic Search server", + "sentry/sentry": "Allow sending log messages to a Sentry server" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Monolog\\": "src/Monolog" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Jordi Boggiano", + "email": "j.boggiano@seld.be", + "homepage": "http://seld.be" + } + ], + "description": "Sends your logs to files, sockets, inboxes, databases and various web services", + "homepage": "http://github.com/Seldaek/monolog", + "keywords": [ + "log", + "logging", + "psr-3" + ], + "time": "2016-07-29 03:23:52" + }, { "name": "psr/http-message", "version": "1.0", @@ -304,7 +382,7 @@ }, "dist": { "type": "zip", - "url": "https://packagist.phpcomposer.com/files/php-fig/http-message/85d63699f0dbedb190bbd4b0d2b9dc707ea4c298.zip", + "url": "https://api.github.com/repos/php-fig/http-message/zipball/85d63699f0dbedb190bbd4b0d2b9dc707ea4c298", "reference": "85d63699f0dbedb190bbd4b0d2b9dc707ea4c298", "shasum": "" }, @@ -343,6 +421,44 @@ ], "time": "2015-05-04 20:22:00" }, + { + "name": "psr/log", + "version": "1.0.0", + "source": { + "type": "git", + "url": "https://github.com/php-fig/log.git", + "reference": "fe0936ee26643249e916849d48e3a51d5f5e278b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/log/zipball/fe0936ee26643249e916849d48e3a51d5f5e278b", + "reference": "fe0936ee26643249e916849d48e3a51d5f5e278b", + "shasum": "" + }, + "type": "library", + "autoload": { + "psr-0": { + "Psr\\Log\\": "" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "http://www.php-fig.org/" + } + ], + "description": "Common interface for logging libraries", + "keywords": [ + "log", + "psr", + "psr-3" + ], + "time": "2012-12-21 11:40:51" + }, { "name": "symfony/css-selector", "version": "v3.0.5", @@ -353,7 +469,7 @@ }, "dist": { "type": "zip", - "url": "https://packagist.phpcomposer.com/files/symfony/css-selector/65e764f404685f2dc20c057e889b3ad04b2e2db0.zip", + "url": "https://api.github.com/repos/symfony/css-selector/zipball/65e764f404685f2dc20c057e889b3ad04b2e2db0", "reference": "65e764f404685f2dc20c057e889b3ad04b2e2db0", "shasum": "" }, @@ -406,7 +522,7 @@ }, "dist": { "type": "zip", - "url": "https://packagist.phpcomposer.com/files/symfony/dom-crawler/49b588841225b205700e5122fa01911cabada857.zip", + "url": "https://api.github.com/repos/symfony/dom-crawler/zipball/49b588841225b205700e5122fa01911cabada857", "reference": "49b588841225b205700e5122fa01911cabada857", "shasum": "" }, @@ -462,7 +578,7 @@ }, "dist": { "type": "zip", - "url": "https://packagist.phpcomposer.com/files/symfony/polyfill-mbstring/1289d16209491b584839022f29257ad859b8532d.zip", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/1289d16209491b584839022f29257ad859b8532d", "reference": "1289d16209491b584839022f29257ad859b8532d", "shasum": "" }, @@ -518,6 +634,8 @@ "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, - "platform": [], + "platform": { + "php": ">=5.4.0" + }, "platform-dev": [] } diff --git a/index.php b/index.php index 0dccf61..28271df 100644 --- a/index.php +++ b/index.php @@ -21,7 +21,6 @@ $client = new Lcrawl('http://xuanke.lzjtu.edu.cn/', $user, false); -$client->login(); $all = $client->setUa('Lcrawl Spider V2.0.2')->getGrade(); //setTimeOut() diff --git a/src/Lcrawl.php b/src/Lcrawl.php index 8ed095d..68bec11 100644 --- a/src/Lcrawl.php +++ b/src/Lcrawl.php @@ -20,6 +20,11 @@ use Lndj\Supports\Log; use Lndj\Traits\Parser; use Lndj\Traits\BuildRequest; +use Lndj\Support\Log; +use Monolog\Handler\NullHandler; +use Monolog\Handler\StreamHandler; +use Monolog\Logger; +use Symfony\Component\CssSelector\Exception\ExpressionErrorException; class Lcrawl { @@ -37,14 +42,35 @@ class Lcrawl //课表查询uri const ZF_SCHEDULE_URI = 'xskbcx.aspx'; + /** + * The client of guzzle. + * @var Client + */ private $client; - private $base_uri; //The base_uri of your Academic Network Systems. Like 'http://xuanke.lzjtu.edu.cn/' + /** + * The base url. + * Like 'http://xuanke.lzjtu.edu.cn/' + * @var + */ + private $base_url; + /** + * Login uri. + * @var string + */ private $login_uri = 'default_ysdx.aspx'; + /** + * The main page of ZF after login. + * @var string + */ private $main_page_uri = 'xs_main.aspx'; + /** + * The Header when request. + * @var array + */ private $headers = [ 'timeout' => 3.0, 'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.94 Safari/537.36', @@ -52,23 +78,64 @@ class Lcrawl 'Content-Type' => 'application/x-www-form-urlencoded' ]; + /** + * The student id. + * @var + */ private $stu_id; + /** + * Password of ZF. + * @var + */ private $password; - private $cacheCookie = false; // Is cookie cached + /** + * Is cookie cached + * @var bool + */ + private $cacheCookie = false; - private $cache; //Doctrine\Common\Cache\Cache + /** + * //Doctrine\Common\Cache\Cache + * @var + */ + private $cache; + /** + * Cache prefix. + * @var string + */ private $cachePrefix = 'Lcrawl'; //The login post param private $loginParam = []; - function __construct($base_uri, $user, $isCacheCookie = false, $loginParam = []) + /** + * The monolog file. + * @var string + */ + private $logFile = './Lcrawl.log'; + + /** + * Monolog level. + * @var int + */ + private $logLevel = Logger::ERROR; + + /** + * Init the Lcrawl. + * Lcrawl constructor. + * @param $base_url + * @param $user + * @param bool $isCacheCookie + * @param array $loginParam + * @param array $logOption + */ + function __construct($base_url, $user, $isCacheCookie = false, $loginParam = [], $logOption = []) { - //Set the base_uri. - $this->base_uri = $base_uri; + //Set the base_url. + $this->base_url = $base_url; //Set the stu_id and password if (is_array($user) && $user['stu_id'] && $user['stu_pwd']) { @@ -83,7 +150,7 @@ function __construct($base_uri, $user, $isCacheCookie = false, $loginParam = []) } $client_param = [ // Base URI is used with relative requests - 'base_uri' => $this->base_uri, + 'base_url' => $this->base_url, ]; //If this value is true, Lcrawl will cache the cookie jar when logining. @@ -99,6 +166,13 @@ function __construct($base_uri, $user, $isCacheCookie = false, $loginParam = []) $this->loginParam = $loginParam; } + //Set the log option. + if (!empty($logOption)) { + $this->initializeLogger($logOption['file'], $logOption['level']); + } else { + $this->initializeLogger($this->logFile, $this->logLevel); + } + $this->client = new Client($client_param); } @@ -290,8 +364,15 @@ public function login() { //Get the hidden value from login page. $response = $this->client->get($this->login_uri); - $viewstate = $this->parserHiddenValue($response->getBody()); - + try { + $viewstate = $this->parserHiddenValue($response->getBody()); + } catch (ExpressionErrorException $e) { + Log::debug('login process: parser the hidden value from login page failed!', [ + 'Body' => $response->getBody(), + 'Message' => $e->getMessage(), + 'TraceString' => $e->getTraceAsString(), + ]); + } //The default login post param $loginParam = [ @@ -433,24 +514,25 @@ public function getExam() $response = $this->buildGetRequest(self::ZF_EXAM_URI); return $this->parserCommonTable($response->getBody()); } -} -/** - * Just a debug function - * - * @param Obeject /Array/string $arr - * @param String $hint debug hint - * @return void - */ -function dd($arr, $hint = '') -{ - if (is_object($arr) || is_array($arr)) { - echo "
";
-        print_r($arr);
-        echo PHP_EOL . $hint;
-        echo "
"; - } else { - var_dump($arr); - echo PHP_EOL . $hint; + /** + * Initialize the logger instanes. + * @param $logFile + * @param int $level + */ + private function initializeLogger($logFile, $level = Logger::WARNING) + { + if (Log::hasLogger()) { + return; + } + + $logger = new Logger('Lcrawl'); + if (defined('PHPUNIT_RUNNING')) { + $logger->pushHandler(new NullHandler()); + } else { + $logger->pushHandler(new StreamHandler($logFile, $level)); + } + + Log::setLogger($logger); } -} +} \ No newline at end of file diff --git a/src/Support/Log.php b/src/Support/Log.php new file mode 100644 index 0000000..33f85a9 --- /dev/null +++ b/src/Support/Log.php @@ -0,0 +1,87 @@ +pushHandler(new NullHandler()); + } else { + $log->pushHandler(new ErrorLogHandler()); + } + + return $log; + } +}