app.js

// Generated by CoffeeScript 1.6.3
(function() {
  var BufferHelper, City, allCity, async, cheerio, city, code, e, fs, http, iconv, k, keys, loadCity, running, tasks, _i, _len;

  fs = require('fs');

  http = require('http');

  async = require('async');

  cheerio = require('cheerio');

  iconv = require('iconv-lite');

  BufferHelper = require('bufferhelper');

  keys = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'.split('');

  allCity = [];

  tasks = [];

  running = {};

  loadCity = function(k, cb) {
    return http.get({
      host: 'm.soufun.com',
      path: '/template/city/hotcity.jsp?key=' + k,
      headers: {
        'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36'
      }
    }, function(res) {
      var bufferHelper;
      bufferHelper = new BufferHelper();
      return bufferHelper.load(res, function(err, buffer) {
        var $;
        $ = cheerio.load(iconv.decode(buffer, 'GBK'));
        $('.ablack').each(function(i, el) {
          var code;
          code = el.attribs.href.replace('/', '').replace('.html', '');
          return allCity.push({
            code: code,
            name: $(el).text()
          });
        });
        cb();
        return console.log('loaded', k);
      });
    });
  };

  City = function(opts) {
    this.name = opts.name;
    this.code = opts.code;
    this.page = opts.page || 1;
    this.opts = {
      host: 'm.soufun.com',
      path: '/xf.d?m=xflist&city=' + this.code + '&datatype=json',
      headers: {
        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36',
        'Content-Type': 'application/x-www-form-urlencoded'
      }
    };
    this.bufferHelper = new BufferHelper();
    if (this.page === 1) {
      return fs.writeFileSync('data/' + this.name + '.xls', '<table>');
    }
  };

  City.prototype.log = function(str) {
    return fs.appendFileSync('data/' + this.name + '.xls', str + '\r\n');
  };

  City.prototype.load = function() {
    var param, req, self;
    self = this;
    param = 'p=' + this.page;
    this.opts.headers['Content-Length'] = param.length;
    this.log('<tr><td>page-' + this.page + '</td></tr>');
    req = http.request(this.opts, function(res) {
      return self.bufferHelper.load(res, function(err, buffer) {
        var $, city, hasData;
        if (err) {
          console.log(err);
          return self.load();
        }
        hasData = false;
        $ = cheerio.load(iconv.decode(buffer, 'GBK'));
        $('.houselist-txt').each(function(i, el) {
          var item, items, tmp, _i, _j, _len, _len1;
          tmp = '<tr>';
          items = $(el).text().split('\n').filter(function(str) {
            return str.trim();
          });
          for (_i = 0, _len = items.length; _i < _len; _i++) {
            item = items[_i];
            item = item.trim();
          }
          for (_j = 0, _len1 = items.length; _j < _len1; _j++) {
            item = items[_j];
            tmp += "<td>" + (item.trim()) + "</td>";
          }
          tmp += '</tr>';
          self.log(tmp);
          return hasData = true;
        });
        if (hasData) {
          console.log('loaded', self.name, self.page);
          running[self.code] = {
            code: self.code,
            name: self.name,
            page: self.page
          };
          self.page++;
          return self.load();
        } else {
          self.log('</table>');
          city = allCity.pop();
          delete running[self.code];
          running[city.code] = city;
          if (city) {
            new City(city).load();
          }
          return console.log('end', self.name);
        }
      });
    });
    req.on('error', function(err) {
      console.log(err);
      return self.load();
    });
    req.write(param);
    return req.end();
  };

  try {
    allCity = JSON.parse(fs.readFileSync('allCity').toString());
    running = JSON.parse(fs.readFileSync('running').toString());
    for (code in running) {
      city = running[code];
      new City(city).load();
    }
  } catch (_error) {
    e = _error;
    console.log(e);
    for (_i = 0, _len = keys.length; _i < _len; _i++) {
      k = keys[_i];
      tasks.push((function(arg) {
        return function(cb) {
          return loadCity(arg, cb);
        };
      })(k));
    }
    async.auto(tasks, function(err) {
      var i, _j, _results;
      if (err) {
        return console.log(err);
      }
      fs.writeFileSync('allCity', JSON.stringify(allCity));
      _results = [];
      for (i = _j = 1; _j <= 10; i = ++_j) {
        city = allCity.pop();
        running[city.code] = city;
        _results.push(new City(city).load());
      }
      return _results;
    });
  }

  setInterval(function() {
    return fs.writeFileSync('running', JSON.stringify(running));
  }, 1000);

}).call(this);